diff --git a/.github/helper/semgrep_rules/frappe_correctness.yml b/.github/helper/semgrep_rules/frappe_correctness.yml index faab3344a6..d9603e89aa 100644 --- a/.github/helper/semgrep_rules/frappe_correctness.yml +++ b/.github/helper/semgrep_rules/frappe_correctness.yml @@ -98,8 +98,6 @@ rules: languages: [python] severity: WARNING paths: - exclude: - - test_*.py include: - "*/**/doctype/*" diff --git a/.github/helper/semgrep_rules/security.yml b/.github/helper/semgrep_rules/security.yml index b2cc4b16fc..5a5098bf50 100644 --- a/.github/helper/semgrep_rules/security.yml +++ b/.github/helper/semgrep_rules/security.yml @@ -8,10 +8,6 @@ rules: dynamic content. Avoid it or use safe_eval(). languages: [python] severity: ERROR - paths: - exclude: - - frappe/__init__.py - - frappe/commands/utils.py - id: frappe-sqli-format-strings patterns: diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 389524e968..e27b406df0 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -1,34 +1,18 @@ name: Semgrep on: - pull_request: - branches: - - develop - - version-13-hotfix - - version-13-pre-release + pull_request: { } + jobs: semgrep: name: Frappe Linter runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Setup python3 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Setup semgrep - run: | - python -m pip install -q semgrep - git fetch origin $GITHUB_BASE_REF:$GITHUB_BASE_REF -q - - - name: Semgrep errors - run: | - files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF) - [[ -d .github/helper/semgrep_rules ]] && semgrep --severity ERROR --config=.github/helper/semgrep_rules --quiet --error $files - semgrep --config="r/python.lang.correctness" --quiet --error $files - - - name: Semgrep warnings - run: | - files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF) - [[ -d .github/helper/semgrep_rules ]] && semgrep --severity WARNING --severity INFO --config=.github/helper/semgrep_rules --quiet $files + - uses: actions/checkout@v2 + - uses: returntocorp/semgrep-action@v1 + env: + SEMGREP_TIMEOUT: 120 + with: + config: >- + r/python.lang.correctness + .github/helper/semgrep_rules diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index ca58e78870..212642e01b 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import json import os import subprocess @@ -14,6 +12,13 @@ from frappe.exceptions import SiteNotSpecifiedError from frappe.utils import get_bench_path, update_progress_bar, cint +DATA_IMPORT_DEPRECATION = click.style( + "[DEPRECATED] The `import-csv` command used 'Data Import Legacy' which has been deprecated.\n" + "Use `data-import` command instead to import data via 'Data Import'.", + fg="yellow" +) + + @click.command('build') @click.option('--app', help='Build assets for app') @click.option('--apps', help='Build assets for specific apps') @@ -350,7 +355,8 @@ def import_doc(context, path, force=False): if not context.sites: raise SiteNotSpecifiedError -@click.command('import-csv') + +@click.command('import-csv', help=DATA_IMPORT_DEPRECATION) @click.argument('path') @click.option('--only-insert', default=False, is_flag=True, help='Do not overwrite existing records') @click.option('--submit-after-import', default=False, is_flag=True, help='Submit document after importing it') @@ -358,32 +364,8 @@ def import_doc(context, path, force=False): @click.option('--no-email', default=True, is_flag=True, help='Send email if applicable') @pass_context def import_csv(context, path, only_insert=False, submit_after_import=False, ignore_encoding_errors=False, no_email=True): - "Import CSV using data import" - from frappe.core.doctype.data_import_legacy import importer - from frappe.utils.csvutils import read_csv_content - site = get_site(context) - - if not os.path.exists(path): - path = os.path.join('..', path) - if not os.path.exists(path): - print('Invalid path {0}'.format(path)) - sys.exit(1) - - with open(path, 'r') as csvfile: - content = read_csv_content(csvfile.read()) - - frappe.init(site=site) - frappe.connect() - - try: - importer.upload(content, submit_after_import=submit_after_import, no_email=no_email, - ignore_encoding_errors=ignore_encoding_errors, overwrite=not only_insert, - via_console=True) - frappe.db.commit() - except Exception: - print(frappe.get_traceback()) - - frappe.destroy() + click.secho(DATA_IMPORT_DEPRECATION) + sys.exit(1) @click.command('data-import') @@ -767,26 +749,49 @@ def set_config(context, key, value, global_=False, parse=False, as_dict=False): frappe.destroy() -@click.command('version') -def get_version(): - "Show the versions of all the installed apps" +@click.command("version") +@click.option("-f", "--format", "output", + type=click.Choice(["plain", "table", "json", "legacy"]), help="Output format", default="legacy") +def get_version(output): + """Show the versions of all the installed apps.""" from git import Repo + from frappe.utils.commands import render_table from frappe.utils.change_log import get_app_branch - frappe.init('') + + frappe.init("") + data = [] for app in sorted(frappe.get_all_apps()): - branch_name = get_app_branch(app) module = frappe.get_module(app) app_hooks = frappe.get_module(app + ".hooks") repo = Repo(frappe.get_app_path(app, "..")) - branch = repo.head.ref.name - commit = repo.head.ref.commit.hexsha[:7] - - if hasattr(app_hooks, '{0}_version'.format(branch_name)): - click.echo("{0} {1} {2} ({3})".format(app, getattr(app_hooks, '{0}_version'.format(branch_name)), branch, commit)) - elif hasattr(module, "__version__"): - click.echo("{0} {1} {2} ({3})".format(app, module.__version__, branch, commit)) + app_info = frappe._dict() + app_info.app = app + app_info.branch = get_app_branch(app) + app_info.commit = repo.head.object.hexsha[:7] + app_info.version = getattr(app_hooks, f"{app_info.branch}_version", None) or module.__version__ + + data.append(app_info) + + { + "legacy": lambda: [ + click.echo(f"{app_info.app} {app_info.version}") + for app_info in data + ], + "plain": lambda: [ + click.echo(f"{app_info.app} {app_info.version} {app_info.branch} ({app_info.commit})") + for app_info in data + ], + "table": lambda: render_table( + [["App", "Version", "Branch", "Commit"]] + + [ + [app_info.app, app_info.version, app_info.branch, app_info.commit] + for app_info in data + ] + ), + "json": lambda: click.echo(json.dumps(data, indent=4)), + }[output]() @click.command('rebuild-global-search') diff --git a/frappe/core/doctype/access_log/access_log.py b/frappe/core/doctype/access_log/access_log.py index d2fbee108b..2ea014f981 100644 --- a/frappe/core/doctype/access_log/access_log.py +++ b/frappe/core/doctype/access_log/access_log.py @@ -1,9 +1,5 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt - -# imports - standard imports -# imports - module imports import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/communication/mixins.py b/frappe/core/doctype/communication/mixins.py index 82a47d24d9..09a8a0ac22 100644 --- a/frappe/core/doctype/communication/mixins.py +++ b/frappe/core/doctype/communication/mixins.py @@ -89,7 +89,7 @@ class CommunicationEmailMixin: return self._final_cc def get_mail_cc_with_displayname(self, is_inbound_mail_communcation=False, include_sender = False): - cc_list = self.mail_cc(is_inbound_mail_communcation=False, include_sender = False) + cc_list = self.mail_cc(is_inbound_mail_communcation=is_inbound_mail_communcation, include_sender = include_sender) return [self.get_email_with_displayname(email) for email in cc_list] def mail_bcc(self, is_inbound_mail_communcation=False): @@ -176,8 +176,8 @@ class CommunicationEmailMixin: def mail_attachments(self, print_format=None, print_html=None): final_attachments = [] - if print_format and print_html: - d = {'print_format': print_format, 'print_html': print_html, 'print_format_attachment': 1, + if print_format or print_html: + d = {'print_format': print_format, 'html': print_html, 'print_format_attachment': 1, 'doctype': self.reference_doctype, 'name': self.reference_name} final_attachments.append(d) diff --git a/frappe/core/doctype/data_export/exporter.py b/frappe/core/doctype/data_export/exporter.py index 389948449e..ffd828bfdb 100644 --- a/frappe/core/doctype/data_export/exporter.py +++ b/frappe/core/doctype/data_export/exporter.py @@ -7,7 +7,6 @@ import frappe.permissions import re, csv, os from frappe.utils.csvutils import UnicodeWriter from frappe.utils import cstr, formatdate, format_datetime, parse_json, cint, format_duration -from frappe.core.doctype.data_import_legacy.importer import get_data_keys from frappe.core.doctype.access_log.access_log import make_access_log reflags = { @@ -20,6 +19,15 @@ reflags = { "D": re.DEBUG } +def get_data_keys(): + return frappe._dict({ + "data_separator": _('Start entering data below this line'), + "main_table": _("Table") + ":", + "parent_table": _("Parent Table") + ":", + "columns": _("Column Name") + ":", + "doctype": _("DocType") + ":" + }) + @frappe.whitelist() def export_data(doctype=None, parent_doctype=None, all_doctypes=True, with_data=False, select_columns=None, file_type='CSV', template=False, filters=None): diff --git a/frappe/core/doctype/data_import/data_import.py b/frappe/core/doctype/data_import/data_import.py index 7e8374a0a2..50469eeb4d 100644 --- a/frappe/core/doctype/data_import/data_import.py +++ b/frappe/core/doctype/data_import/data_import.py @@ -171,9 +171,6 @@ def import_file( i.import_data() -############## - - def import_doc(path, pre_process=None): if os.path.isdir(path): files = [os.path.join(path, f) for f in os.listdir(path)] @@ -192,19 +189,8 @@ def import_doc(path, pre_process=None): ) frappe.flags.mute_emails = False frappe.db.commit() - elif f.endswith(".csv"): - validate_csv_import_file(f) - frappe.db.commit() - - -def validate_csv_import_file(path): - if path.endswith(".csv"): - print() - print("This method is deprecated.") - print('Import CSV files using the command "bench --site sitename data-import"') - print("Or use the method frappe.core.doctype.data_import.data_import.import_file") - print() - raise Exception("Method deprecated") + else: + raise NotImplementedError("Only .json files can be imported") def export_json( diff --git a/frappe/core/doctype/data_import_legacy/__init__.py b/frappe/core/doctype/data_import_legacy/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/core/doctype/data_import_legacy/data_import_legacy.js b/frappe/core/doctype/data_import_legacy/data_import_legacy.js deleted file mode 100644 index 8e4f397171..0000000000 --- a/frappe/core/doctype/data_import_legacy/data_import_legacy.js +++ /dev/null @@ -1,324 +0,0 @@ -// Copyright (c) 2017, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Data Import Legacy', { - onload: function(frm) { - if (frm.doc.__islocal) { - frm.set_value("action", ""); - } - - frappe.call({ - method: "frappe.core.doctype.data_import_legacy.data_import_legacy.get_importable_doctypes", - callback: function (r) { - let importable_doctypes = r.message; - frm.set_query("reference_doctype", function () { - return { - "filters": { - "issingle": 0, - "istable": 0, - "name": ['in', importable_doctypes] - } - }; - }); - } - }), - - // should never check public - frm.fields_dict["import_file"].df.is_private = 1; - - frappe.realtime.on("data_import_progress", function(data) { - if (data.data_import === frm.doc.name) { - if (data.reload && data.reload === true) { - frm.reload_doc(); - } - if (data.progress) { - let progress_bar = $(frm.dashboard.progress_area.body).find(".progress-bar"); - if (progress_bar) { - $(progress_bar).removeClass("progress-bar-danger").addClass("progress-bar-success progress-bar-striped"); - $(progress_bar).css("width", data.progress + "%"); - } - } - } - }); - }, - - reference_doctype: function(frm){ - if (frm.doc.reference_doctype) { - frappe.model.with_doctype(frm.doc.reference_doctype); - } - }, - - refresh: function(frm) { - frm.disable_save(); - frm.dashboard.clear_headline(); - if (frm.doc.reference_doctype && !frm.doc.import_file) { - frm.page.set_indicator(__('Attach file'), 'orange'); - } else { - if (frm.doc.import_status) { - const listview_settings = frappe.listview_settings['Data Import Legacy']; - const indicator = listview_settings.get_indicator(frm.doc); - - frm.page.set_indicator(indicator[0], indicator[1]); - - if (frm.doc.import_status === "In Progress") { - frm.dashboard.add_progress("Data Import Progress", "0"); - frm.set_read_only(); - frm.refresh_fields(); - } - } - } - - if (frm.doc.reference_doctype) { - frappe.model.with_doctype(frm.doc.reference_doctype); - } - - if(frm.doc.action == "Insert new records" || frm.doc.action == "Update records") { - frm.set_df_property("action", "read_only", 1); - } - - frm.add_custom_button(__("Help"), function() { - frappe.help.show_video("6wiriRKPhmg"); - }); - - if (frm.doc.reference_doctype && frm.doc.docstatus === 0) { - frm.add_custom_button(__("Download template"), function() { - frappe.data_import.download_dialog(frm).show(); - }); - } - - if (frm.doc.reference_doctype && frm.doc.import_file && frm.doc.total_rows && - frm.doc.docstatus === 0 && (!frm.doc.import_status || frm.doc.import_status == "Failed")) { - frm.page.set_primary_action(__("Start Import"), function() { - frappe.call({ - btn: frm.page.btn_primary, - method: "frappe.core.doctype.data_import_legacy.data_import_legacy.import_data", - args: { - data_import: frm.doc.name - } - }); - }).addClass('btn btn-primary'); - } - - if (frm.doc.log_details) { - frm.events.create_log_table(frm); - } else { - $(frm.fields_dict.import_log.wrapper).empty(); - } - }, - - action: function(frm) { - if(!frm.doc.action) return; - if(!frm.doc.reference_doctype) { - frappe.msgprint(__("Please select document type first.")); - frm.set_value("action", ""); - return; - } - - if(frm.doc.action == "Insert new records") { - frm.doc.insert_new = 1; - } else if (frm.doc.action == "Update records"){ - frm.doc.overwrite = 1; - } - frm.save(); - }, - - only_update: function(frm) { - frm.save(); - }, - - submit_after_import: function(frm) { - frm.save(); - }, - - skip_errors: function(frm) { - frm.save(); - }, - - ignore_encoding_errors: function(frm) { - frm.save(); - }, - - no_email: function(frm) { - frm.save(); - }, - - show_only_errors: function(frm) { - frm.events.create_log_table(frm); - }, - - create_log_table: function(frm) { - let msg = JSON.parse(frm.doc.log_details); - var $log_wrapper = $(frm.fields_dict.import_log.wrapper).empty(); - $(frappe.render_template("log_details", { - data: msg.messages, - import_status: frm.doc.import_status, - show_only_errors: frm.doc.show_only_errors, - })).appendTo($log_wrapper); - } -}); - -frappe.provide('frappe.data_import'); -frappe.data_import.download_dialog = function(frm) { - var dialog; - const filter_fields = df => frappe.model.is_value_type(df) && !df.hidden; - const get_fields = dt => frappe.meta.get_docfields(dt).filter(filter_fields); - - const get_doctype_checkbox_fields = () => { - return dialog.fields.filter(df => df.fieldname.endsWith('_fields')) - .map(df => dialog.fields_dict[df.fieldname]); - }; - - const doctype_fields = get_fields(frm.doc.reference_doctype) - .map(df => { - let reqd = (df.reqd || df.fieldname == 'naming_series') ? 1 : 0; - return { - label: df.label, - reqd: reqd, - danger: reqd, - value: df.fieldname, - checked: 1 - }; - }); - - let fields = [ - { - "label": __("Select Columns"), - "fieldname": "select_columns", - "fieldtype": "Select", - "options": "All\nMandatory\nManually", - "reqd": 1, - "onchange": function() { - const fields = get_doctype_checkbox_fields(); - fields.map(f => f.toggle(true)); - if(this.value == 'Mandatory' || this.value == 'Manually') { - checkbox_toggle(true); - fields.map(multicheck_field => { - multicheck_field.options.map(option => { - if(!option.reqd) return; - $(multicheck_field.$wrapper).find(`:checkbox[data-unit="${option.value}"]`) - .prop('checked', false) - .trigger('click'); - }); - }); - } else if(this.value == 'All'){ - $(dialog.body).find(`[data-fieldtype="MultiCheck"] :checkbox`) - .prop('disabled', true); - } - } - }, - { - "label": __("File Type"), - "fieldname": "file_type", - "fieldtype": "Select", - "options": "Excel\nCSV", - "default": "Excel" - }, - { - "label": __("Download with Data"), - "fieldname": "with_data", - "fieldtype": "Check", - "hidden": !frm.doc.overwrite, - "default": 1 - }, - { - "label": __("Select All"), - "fieldname": "select_all", - "fieldtype": "Button", - "depends_on": "eval:doc.select_columns=='Manually'", - click: function() { - checkbox_toggle(); - } - }, - { - "label": __("Unselect All"), - "fieldname": "unselect_all", - "fieldtype": "Button", - "depends_on": "eval:doc.select_columns=='Manually'", - click: function() { - checkbox_toggle(true); - } - }, - { - "label": frm.doc.reference_doctype, - "fieldname": "doctype_fields", - "fieldtype": "MultiCheck", - "options": doctype_fields, - "columns": 2, - "hidden": 1 - } - ]; - - const child_table_fields = frappe.meta.get_table_fields(frm.doc.reference_doctype) - .map(df => { - return { - "label": df.options, - "fieldname": df.fieldname + '_fields', - "fieldtype": "MultiCheck", - "options": frappe.meta.get_docfields(df.options) - .filter(filter_fields) - .map(df => ({ - label: df.label, - reqd: df.reqd ? 1 : 0, - value: df.fieldname, - checked: 1, - danger: df.reqd - })), - "columns": 2, - "hidden": 1 - }; - }); - - fields = fields.concat(child_table_fields); - - dialog = new frappe.ui.Dialog({ - title: __('Download Template'), - fields: fields, - primary_action: function(values) { - var data = values; - if (frm.doc.reference_doctype) { - var export_params = () => { - let columns = {}; - if(values.select_columns) { - columns = get_doctype_checkbox_fields().reduce((columns, field) => { - const options = field.get_checked_options(); - columns[field.df.label] = options; - return columns; - }, {}); - } - - return { - doctype: frm.doc.reference_doctype, - parent_doctype: frm.doc.reference_doctype, - select_columns: JSON.stringify(columns), - with_data: frm.doc.overwrite && data.with_data, - all_doctypes: true, - file_type: data.file_type, - template: true - }; - }; - let get_template_url = '/api/method/frappe.core.doctype.data_export.exporter.export_data'; - open_url_post(get_template_url, export_params()); - } else { - frappe.msgprint(__("Please select the Document Type.")); - } - dialog.hide(); - }, - primary_action_label: __('Download') - }); - - $(dialog.body).find('div[data-fieldname="select_all"], div[data-fieldname="unselect_all"]') - .wrapAll('
'); - const button_container = $(dialog.body).find('.inline-buttons'); - button_container.addClass('flex'); - $(button_container).find('.frappe-control').map((index, button) => { - $(button).css({"margin-right": "1em"}); - }); - - function checkbox_toggle(checked=false) { - $(dialog.body).find('[data-fieldtype="MultiCheck"]').map((index, element) => { - $(element).find(`:checkbox`).prop("checked", checked).trigger('click'); - }); - } - - return dialog; -}; diff --git a/frappe/core/doctype/data_import_legacy/data_import_legacy.json b/frappe/core/doctype/data_import_legacy/data_import_legacy.json deleted file mode 100644 index 852ccba156..0000000000 --- a/frappe/core/doctype/data_import_legacy/data_import_legacy.json +++ /dev/null @@ -1,218 +0,0 @@ -{ - "actions": [], - "allow_copy": 1, - "creation": "2020-06-11 16:13:23.813709", - "doctype": "DocType", - "document_type": "Document", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "reference_doctype", - "action", - "insert_new", - "overwrite", - "only_update", - "section_break_4", - "import_file", - "column_break_4", - "error_file", - "section_break_6", - "skip_errors", - "submit_after_import", - "ignore_encoding_errors", - "no_email", - "import_detail", - "import_status", - "show_only_errors", - "import_log", - "log_details", - "amended_from", - "total_rows", - "amended_from" - ], - "fields": [ - { - "fieldname": "reference_doctype", - "fieldtype": "Link", - "ignore_user_permissions": 1, - "in_list_view": 1, - "label": "Document Type", - "options": "DocType", - "reqd": 1 - }, - { - "fieldname": "action", - "fieldtype": "Select", - "label": "Action", - "options": "Insert new records\nUpdate records", - "reqd": 1 - }, - { - "default": "0", - "depends_on": "eval:!doc.overwrite", - "description": "New data will be inserted.", - "fieldname": "insert_new", - "fieldtype": "Check", - "hidden": 1, - "label": "Insert new records", - "set_only_once": 1 - }, - { - "default": "0", - "depends_on": "eval:!doc.insert_new", - "description": "If you are updating/overwriting already created records.", - "fieldname": "overwrite", - "fieldtype": "Check", - "hidden": 1, - "label": "Update records", - "set_only_once": 1 - }, - { - "default": "0", - "depends_on": "overwrite", - "description": "If you don't want to create any new records while updating the older records.", - "fieldname": "only_update", - "fieldtype": "Check", - "label": "Don't create new records" - }, - { - "depends_on": "eval:(!doc.__islocal)", - "fieldname": "section_break_4", - "fieldtype": "Section Break" - }, - { - "fieldname": "import_file", - "fieldtype": "Attach", - "label": "Attach file for Import" - }, - { - "fieldname": "column_break_4", - "fieldtype": "Column Break" - }, - { - "depends_on": "eval: doc.import_status == \"Partially Successful\"", - "description": "This is the template file generated with only the rows having some error. You should use this file for correction and import.", - "fieldname": "error_file", - "fieldtype": "Attach", - "label": "Generated File" - }, - { - "depends_on": "eval:(!doc.__islocal)", - "fieldname": "section_break_6", - "fieldtype": "Section Break" - }, - { - "default": "0", - "description": "If this is checked, rows with valid data will be imported and invalid rows will be dumped into a new file for you to import later.", - "fieldname": "skip_errors", - "fieldtype": "Check", - "label": "Skip rows with errors" - }, - { - "default": "0", - "fieldname": "submit_after_import", - "fieldtype": "Check", - "label": "Submit after importing" - }, - { - "default": "0", - "fieldname": "ignore_encoding_errors", - "fieldtype": "Check", - "label": "Ignore encoding errors" - }, - { - "default": "1", - "fieldname": "no_email", - "fieldtype": "Check", - "label": "Do not send Emails" - }, - { - "collapsible": 1, - "collapsible_depends_on": "eval: doc.import_status == \"Failed\"", - "depends_on": "import_status", - "fieldname": "import_detail", - "fieldtype": "Section Break", - "label": "Import Log" - }, - { - "fieldname": "import_status", - "fieldtype": "Select", - "label": "Import Status", - "options": "\nSuccessful\nFailed\nIn Progress\nPartially Successful", - "read_only": 1 - }, - { - "allow_on_submit": 1, - "default": "1", - "fieldname": "show_only_errors", - "fieldtype": "Check", - "label": "Show only errors", - "no_copy": 1, - "print_hide": 1 - }, - { - "allow_on_submit": 1, - "depends_on": "import_status", - "fieldname": "import_log", - "fieldtype": "HTML", - "label": "Import Log" - }, - { - "allow_on_submit": 1, - "fieldname": "log_details", - "fieldtype": "Code", - "hidden": 1, - "label": "Log Details", - "read_only": 1 - }, - { - "fieldname": "amended_from", - "fieldtype": "Link", - "label": "Amended From", - "no_copy": 1, - "options": "Data Import", - "print_hide": 1, - "read_only": 1 - }, - { - "fieldname": "total_rows", - "fieldtype": "Int", - "hidden": 1, - "label": "Total Rows", - "read_only": 1 - }, - { - "fieldname": "amended_from", - "fieldtype": "Link", - "label": "Amended From", - "no_copy": 1, - "options": "Data Import Legacy", - "print_hide": 1, - "read_only": 1 - } - ], - "is_submittable": 1, - "links": [], - "max_attachments": 1, - "modified": "2020-06-11 16:13:23.813709", - "modified_by": "Administrator", - "module": "Core", - "name": "Data Import Legacy", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "read": 1, - "role": "System Manager", - "share": 1, - "submit": 1, - "write": 1 - } - ], - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 1 -} \ No newline at end of file diff --git a/frappe/core/doctype/data_import_legacy/data_import_legacy.py b/frappe/core/doctype/data_import_legacy/data_import_legacy.py deleted file mode 100644 index 63f806d75b..0000000000 --- a/frappe/core/doctype/data_import_legacy/data_import_legacy.py +++ /dev/null @@ -1,126 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# For license information, please see license.txt - -import os - -import frappe -import frappe.modules.import_file -from frappe import _ -from frappe.core.doctype.data_import_legacy.importer import upload -from frappe.model.document import Document -from frappe.modules.import_file import import_file_by_path as _import_file_by_path -from frappe.utils.background_jobs import enqueue -from frappe.utils.data import format_datetime - - -class DataImportLegacy(Document): - def autoname(self): - if not self.name: - self.name = "Import on " + format_datetime(self.creation) - - def validate(self): - if not self.import_file: - self.db_set("total_rows", 0) - if self.import_status == "In Progress": - frappe.throw(_("Can't save the form as data import is in progress.")) - - # validate the template just after the upload - # if there is total_rows in the doc, it means that the template is already validated and error free - if self.import_file and not self.total_rows: - upload(data_import_doc=self, from_data_import="Yes", validate_template=True) - - -@frappe.whitelist() -def get_importable_doctypes(): - return frappe.cache().hget("can_import", frappe.session.user) - - -@frappe.whitelist() -def import_data(data_import): - frappe.db.set_value("Data Import Legacy", data_import, "import_status", "In Progress", update_modified=False) - frappe.publish_realtime("data_import_progress", {"progress": "0", - "data_import": data_import, "reload": True}, user=frappe.session.user) - - from frappe.core.page.background_jobs.background_jobs import get_info - enqueued_jobs = [d.get("job_name") for d in get_info()] - - if data_import not in enqueued_jobs: - enqueue(upload, queue='default', timeout=6000, event='data_import', job_name=data_import, - data_import_doc=data_import, from_data_import="Yes", user=frappe.session.user) - - -def import_doc(path, overwrite=False, ignore_links=False, ignore_insert=False, - insert=False, submit=False, pre_process=None): - if os.path.isdir(path): - files = [os.path.join(path, f) for f in os.listdir(path)] - else: - files = [path] - - for f in files: - if f.endswith(".json"): - frappe.flags.mute_emails = True - _import_file_by_path(f, data_import=True, force=True, pre_process=pre_process, reset_permissions=True) - frappe.flags.mute_emails = False - frappe.db.commit() - elif f.endswith(".csv"): - import_file_by_path(f, ignore_links=ignore_links, overwrite=overwrite, submit=submit, pre_process=pre_process) - frappe.db.commit() - - -def import_file_by_path(path, ignore_links=False, overwrite=False, submit=False, pre_process=None, no_email=True): - from frappe.utils.csvutils import read_csv_content - print("Importing " + path) - with open(path, "r") as infile: - upload(rows=read_csv_content(infile.read()), ignore_links=ignore_links, no_email=no_email, overwrite=overwrite, - submit_after_import=submit, pre_process=pre_process) - - -def export_json(doctype, path, filters=None, or_filters=None, name=None, order_by="creation asc"): - def post_process(out): - del_keys = ('modified_by', 'creation', 'owner', 'idx') - for doc in out: - for key in del_keys: - if key in doc: - del doc[key] - for k, v in doc.items(): - if isinstance(v, list): - for child in v: - for key in del_keys + ('docstatus', 'doctype', 'modified', 'name'): - if key in child: - del child[key] - - out = [] - if name: - out.append(frappe.get_doc(doctype, name).as_dict()) - elif frappe.db.get_value("DocType", doctype, "issingle"): - out.append(frappe.get_doc(doctype).as_dict()) - else: - for doc in frappe.get_all(doctype, fields=["name"], filters=filters, or_filters=or_filters, limit_page_length=0, order_by=order_by): - out.append(frappe.get_doc(doctype, doc.name).as_dict()) - post_process(out) - - dirname = os.path.dirname(path) - if not os.path.exists(dirname): - path = os.path.join('..', path) - - with open(path, "w") as outfile: - outfile.write(frappe.as_json(out)) - - -def export_csv(doctype, path): - from frappe.core.doctype.data_export.exporter import export_data - with open(path, "wb") as csvfile: - export_data(doctype=doctype, all_doctypes=True, template=True, with_data=True) - csvfile.write(frappe.response.result.encode("utf-8")) - - -@frappe.whitelist() -def export_fixture(doctype, app): - if frappe.session.user != "Administrator": - raise frappe.PermissionError - - if not os.path.exists(frappe.get_app_path(app, "fixtures")): - os.mkdir(frappe.get_app_path(app, "fixtures")) - - export_json(doctype, frappe.get_app_path(app, "fixtures", frappe.scrub(doctype) + ".json"), order_by="name asc") diff --git a/frappe/core/doctype/data_import_legacy/data_import_legacy_list.js b/frappe/core/doctype/data_import_legacy/data_import_legacy_list.js deleted file mode 100644 index fcf2391313..0000000000 --- a/frappe/core/doctype/data_import_legacy/data_import_legacy_list.js +++ /dev/null @@ -1,24 +0,0 @@ -frappe.listview_settings['Data Import Legacy'] = { - add_fields: ["import_status"], - has_indicator_for_draft: 1, - get_indicator: function(doc) { - - let status = { - 'Successful': [__("Success"), "green", "import_status,=,Successful"], - 'Partially Successful': [__("Partial Success"), "blue", "import_status,=,Partially Successful"], - 'In Progress': [__("In Progress"), "orange", "import_status,=,In Progress"], - 'Failed': [__("Failed"), "red", "import_status,=,Failed"], - 'Pending': [__("Pending"), "orange", "import_status,=,"] - } - - if (doc.import_status) { - return status[doc.import_status]; - } - - if (doc.docstatus == 0) { - return status['Pending']; - } - - return status['Pending']; - } -}; diff --git a/frappe/core/doctype/data_import_legacy/importer.py b/frappe/core/doctype/data_import_legacy/importer.py deleted file mode 100644 index ceefff4410..0000000000 --- a/frappe/core/doctype/data_import_legacy/importer.py +++ /dev/null @@ -1,538 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -import requests -import frappe, json -import frappe.permissions - -from frappe import _ - -from frappe.utils.csvutils import getlink -from frappe.utils.dateutils import parse_date - -from frappe.utils import cint, cstr, flt, getdate, get_datetime, get_url, get_absolute_url, duration_to_seconds - - -@frappe.whitelist() -def get_data_keys(): - return frappe._dict({ - "data_separator": _('Start entering data below this line'), - "main_table": _("Table") + ":", - "parent_table": _("Parent Table") + ":", - "columns": _("Column Name") + ":", - "doctype": _("DocType") + ":" - }) - - - -@frappe.whitelist() -def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None, - update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No", - skip_errors = True, data_import_doc=None, validate_template=False, user=None): - """upload data""" - - # for translations - if user: - frappe.cache().hdel("lang", user) - frappe.set_user_lang(user) - - if data_import_doc and isinstance(data_import_doc, str): - data_import_doc = frappe.get_doc("Data Import Legacy", data_import_doc) - if data_import_doc and from_data_import == "Yes": - no_email = data_import_doc.no_email - ignore_encoding_errors = data_import_doc.ignore_encoding_errors - update_only = data_import_doc.only_update - submit_after_import = data_import_doc.submit_after_import - overwrite = data_import_doc.overwrite - skip_errors = data_import_doc.skip_errors - else: - # extra input params - params = json.loads(frappe.form_dict.get("params") or '{}') - if params.get("submit_after_import"): - submit_after_import = True - if params.get("ignore_encoding_errors"): - ignore_encoding_errors = True - if not params.get("no_email"): - no_email = False - if params.get('update_only'): - update_only = True - if params.get('from_data_import'): - from_data_import = params.get('from_data_import') - if not params.get('skip_errors'): - skip_errors = params.get('skip_errors') - - frappe.flags.in_import = True - frappe.flags.mute_emails = no_email - - def get_data_keys_definition(): - return get_data_keys() - - def bad_template(): - frappe.throw(_("Please do not change the rows above {0}").format(get_data_keys_definition().data_separator)) - - def check_data_length(): - if not data: - frappe.throw(_("No data found in the file. Please reattach the new file with data.")) - - def get_start_row(): - for i, row in enumerate(rows): - if row and row[0]==get_data_keys_definition().data_separator: - return i+1 - bad_template() - - def get_header_row(key): - return get_header_row_and_idx(key)[0] - - def get_header_row_and_idx(key): - for i, row in enumerate(header): - if row and row[0]==key: - return row, i - return [], -1 - - def filter_empty_columns(columns): - empty_cols = list(filter(lambda x: x in ("", None), columns)) - - if empty_cols: - if columns[-1*len(empty_cols):] == empty_cols: - # filter empty columns if they exist at the end - columns = columns[:-1*len(empty_cols)] - else: - frappe.msgprint(_("Please make sure that there are no empty columns in the file."), - raise_exception=1) - - return columns - - def make_column_map(): - doctype_row, row_idx = get_header_row_and_idx(get_data_keys_definition().doctype) - if row_idx == -1: # old style - return - - dt = None - for i, d in enumerate(doctype_row[1:]): - if d not in ("~", "-"): - if d and doctype_row[i] in (None, '' ,'~', '-', _("DocType") + ":"): - dt, parentfield = d, None - # xls format truncates the row, so it may not have more columns - if len(doctype_row) > i+2: - parentfield = doctype_row[i+2] - doctypes.append((dt, parentfield)) - column_idx_to_fieldname[(dt, parentfield)] = {} - column_idx_to_fieldtype[(dt, parentfield)] = {} - if dt: - column_idx_to_fieldname[(dt, parentfield)][i+1] = rows[row_idx + 2][i+1] - column_idx_to_fieldtype[(dt, parentfield)][i+1] = rows[row_idx + 4][i+1] - - def get_doc(start_idx): - if doctypes: - doc = {} - attachments = [] - last_error_row_idx = None - for idx in range(start_idx, len(rows)): - last_error_row_idx = idx # pylint: disable=W0612 - if (not doc) or main_doc_empty(rows[idx]): - for dt, parentfield in doctypes: - d = {} - for column_idx in column_idx_to_fieldname[(dt, parentfield)]: - try: - fieldname = column_idx_to_fieldname[(dt, parentfield)][column_idx] - fieldtype = column_idx_to_fieldtype[(dt, parentfield)][column_idx] - - if not fieldname or not rows[idx][column_idx]: - continue - - d[fieldname] = rows[idx][column_idx] - if fieldtype in ("Int", "Check"): - d[fieldname] = cint(d[fieldname]) - elif fieldtype in ("Float", "Currency", "Percent"): - d[fieldname] = flt(d[fieldname]) - elif fieldtype == "Date": - if d[fieldname] and isinstance(d[fieldname], str): - d[fieldname] = getdate(parse_date(d[fieldname])) - elif fieldtype == "Datetime": - if d[fieldname]: - if " " in d[fieldname]: - _date, _time = d[fieldname].split() - else: - _date, _time = d[fieldname], '00:00:00' - _date = parse_date(d[fieldname]) - d[fieldname] = get_datetime(_date + " " + _time) - else: - d[fieldname] = None - elif fieldtype == "Duration": - d[fieldname] = duration_to_seconds(cstr(d[fieldname])) - elif fieldtype in ("Image", "Attach Image", "Attach"): - # added file to attachments list - attachments.append(d[fieldname]) - - elif fieldtype in ("Link", "Dynamic Link", "Data") and d[fieldname]: - # as fields can be saved in the number format(long type) in data import template - d[fieldname] = cstr(d[fieldname]) - - except IndexError: - pass - - # scrub quotes from name and modified - if d.get("name") and d["name"].startswith('"'): - d["name"] = d["name"][1:-1] - - if sum(0 if not val else 1 for val in d.values()): - d['doctype'] = dt - if dt == doctype: - doc.update(d) - else: - if not overwrite and doc.get("name"): - d['parent'] = doc["name"] - d['parenttype'] = doctype - d['parentfield'] = parentfield - doc.setdefault(d['parentfield'], []).append(d) - else: - break - - return doc, attachments, last_error_row_idx - else: - doc = frappe._dict(zip(columns, rows[start_idx][1:])) - doc['doctype'] = doctype - return doc, [], None - - # used in testing whether a row is empty or parent row or child row - # checked only 3 first columns since first two columns can be blank for example the case of - # importing the item variant where item code and item name will be blank. - def main_doc_empty(row): - if row: - for i in range(3,0,-1): - if len(row) > i and row[i]: - return False - return True - - def validate_naming(doc): - autoname = frappe.get_meta(doctype).autoname - if autoname: - if autoname[0:5] == 'field': - autoname = autoname[6:] - elif autoname == 'naming_series:': - autoname = 'naming_series' - else: - return True - - if (autoname not in doc) or (not doc[autoname]): - from frappe.model.base_document import get_controller - if not hasattr(get_controller(doctype), "autoname"): - frappe.throw(_("{0} is a mandatory field").format(autoname)) - return True - - users = frappe.db.sql_list("select name from tabUser") - def prepare_for_insert(doc): - # don't block data import if user is not set - # migrating from another system - if not doc.owner in users: - doc.owner = frappe.session.user - if not doc.modified_by in users: - doc.modified_by = frappe.session.user - - def is_valid_url(url): - is_valid = False - if url.startswith("/files") or url.startswith("/private/files"): - url = get_url(url) - - try: - r = requests.get(url) - is_valid = True if r.status_code == 200 else False - except Exception: - pass - - return is_valid - - def attach_file_to_doc(doctype, docname, file_url): - # check if attachment is already available - # check if the attachement link is relative or not - if not file_url: - return - if not is_valid_url(file_url): - return - - files = frappe.db.sql("""Select name from `tabFile` where attached_to_doctype='{doctype}' and - attached_to_name='{docname}' and (file_url='{file_url}' or thumbnail_url='{file_url}')""".format( - doctype=doctype, - docname=docname, - file_url=file_url - )) - - if files: - # file is already attached - return - - _file = frappe.get_doc({ - "doctype": "File", - "file_url": file_url, - "attached_to_name": docname, - "attached_to_doctype": doctype, - "attached_to_field": 0, - "folder": "Home/Attachments"}) - _file.save() - - - # header - filename, file_extension = ['',''] - if not rows: - _file = frappe.get_doc("File", {"file_url": data_import_doc.import_file}) - fcontent = _file.get_content() - filename, file_extension = _file.get_extension() - - if file_extension == '.xlsx' and from_data_import == 'Yes': - from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file - rows = read_xlsx_file_from_attached_file(file_url=data_import_doc.import_file) - - elif file_extension == '.csv': - from frappe.utils.csvutils import read_csv_content - rows = read_csv_content(fcontent, ignore_encoding_errors) - - else: - frappe.throw(_("Unsupported File Format")) - - start_row = get_start_row() - header = rows[:start_row] - data = rows[start_row:] - try: - doctype = get_header_row(get_data_keys_definition().main_table)[1] - columns = filter_empty_columns(get_header_row(get_data_keys_definition().columns)[1:]) - except: - frappe.throw(_("Cannot change header content")) - doctypes = [] - column_idx_to_fieldname = {} - column_idx_to_fieldtype = {} - - if skip_errors: - data_rows_with_error = header - - if submit_after_import and not cint(frappe.db.get_value("DocType", - doctype, "is_submittable")): - submit_after_import = False - - parenttype = get_header_row(get_data_keys_definition().parent_table) - - if len(parenttype) > 1: - parenttype = parenttype[1] - - # check permissions - if not frappe.permissions.can_import(parenttype or doctype): - frappe.flags.mute_emails = False - return {"messages": [_("Not allowed to Import") + ": " + _(doctype)], "error": True} - - # Throw expception in case of the empty data file - check_data_length() - make_column_map() - total = len(data) - - if validate_template: - if total: - data_import_doc.total_rows = total - return True - - if overwrite==None: - overwrite = params.get('overwrite') - - # delete child rows (if parenttype) - parentfield = None - if parenttype: - parentfield = get_parent_field(doctype, parenttype) - - if overwrite: - delete_child_rows(data, doctype) - - import_log = [] - def log(**kwargs): - if via_console: - print((kwargs.get("title") + kwargs.get("message")).encode('utf-8')) - else: - import_log.append(kwargs) - - def as_link(doctype, name): - if via_console: - return "{0}: {1}".format(doctype, name) - else: - return getlink(doctype, name) - - # publish realtime task update - def publish_progress(achieved, reload=False): - if data_import_doc: - frappe.publish_realtime("data_import_progress", {"progress": str(int(100.0*achieved/total)), - "data_import": data_import_doc.name, "reload": reload}, user=frappe.session.user) - - - error_flag = rollback_flag = False - - batch_size = frappe.conf.data_import_batch_size or 1000 - - for batch_start in range(0, total, batch_size): - batch = data[batch_start:batch_start + batch_size] - - for i, row in enumerate(batch): - # bypass empty rows - if main_doc_empty(row): - continue - - row_idx = i + start_row - doc = None - - publish_progress(i) - - try: - doc, attachments, last_error_row_idx = get_doc(row_idx) - validate_naming(doc) - if pre_process: - pre_process(doc) - - original = None - if parentfield: - parent = frappe.get_doc(parenttype, doc["parent"]) - doc = parent.append(parentfield, doc) - parent.save() - else: - if overwrite and doc.get("name") and frappe.db.exists(doctype, doc["name"]): - original = frappe.get_doc(doctype, doc["name"]) - original_name = original.name - original.update(doc) - # preserve original name for case sensitivity - original.name = original_name - original.flags.ignore_links = ignore_links - original.save() - doc = original - else: - if not update_only: - doc = frappe.get_doc(doc) - prepare_for_insert(doc) - doc.flags.ignore_links = ignore_links - doc.insert() - if attachments: - # check file url and create a File document - for file_url in attachments: - attach_file_to_doc(doc.doctype, doc.name, file_url) - if submit_after_import: - doc.submit() - - # log errors - if parentfield: - log(**{"row": doc.idx, "title": 'Inserted row for "%s"' % (as_link(parenttype, doc.parent)), - "link": get_absolute_url(parenttype, doc.parent), "message": 'Document successfully saved', "indicator": "green"}) - elif submit_after_import: - log(**{"row": row_idx + 1, "title":'Submitted row for "%s"' % (as_link(doc.doctype, doc.name)), - "message": "Document successfully submitted", "link": get_absolute_url(doc.doctype, doc.name), "indicator": "blue"}) - elif original: - log(**{"row": row_idx + 1,"title":'Updated row for "%s"' % (as_link(doc.doctype, doc.name)), - "message": "Document successfully updated", "link": get_absolute_url(doc.doctype, doc.name), "indicator": "green"}) - elif not update_only: - log(**{"row": row_idx + 1, "title":'Inserted row for "%s"' % (as_link(doc.doctype, doc.name)), - "message": "Document successfully saved", "link": get_absolute_url(doc.doctype, doc.name), "indicator": "green"}) - else: - log(**{"row": row_idx + 1, "title":'Ignored row for %s' % (row[1]), "link": None, - "message": "Document updation ignored", "indicator": "orange"}) - - except Exception as e: - error_flag = True - - # build error message - if frappe.local.message_log: - err_msg = "\n".join(['

{}

'.format(json.loads(msg).get('message')) for msg in frappe.local.message_log]) - else: - err_msg = '

{}

'.format(cstr(e)) - - error_trace = frappe.get_traceback() - if error_trace: - error_log_doc = frappe.log_error(error_trace) - error_link = get_absolute_url("Error Log", error_log_doc.name) - else: - error_link = None - - log(**{ - "row": row_idx + 1, - "title": 'Error for row %s' % (len(row)>1 and frappe.safe_decode(row[1]) or ""), - "message": err_msg, - "indicator": "red", - "link":error_link - }) - - # data with error to create a new file - # include the errored data in the last row as last_error_row_idx will not be updated for the last row - if skip_errors: - if last_error_row_idx == len(rows)-1: - last_error_row_idx = len(rows) - data_rows_with_error += rows[row_idx:last_error_row_idx] - else: - rollback_flag = True - finally: - frappe.local.message_log = [] - - start_row += batch_size - if rollback_flag: - frappe.db.rollback() - else: - frappe.db.commit() - - frappe.flags.mute_emails = False - frappe.flags.in_import = False - - log_message = {"messages": import_log, "error": error_flag} - if data_import_doc: - data_import_doc.log_details = json.dumps(log_message) - - import_status = None - if error_flag and data_import_doc.skip_errors and len(data) != len(data_rows_with_error): - import_status = "Partially Successful" - # write the file with the faulty row - file_name = 'error_' + filename + file_extension - if file_extension == '.xlsx': - from frappe.utils.xlsxutils import make_xlsx - xlsx_file = make_xlsx(data_rows_with_error, "Data Import Template") - file_data = xlsx_file.getvalue() - else: - from frappe.utils.csvutils import to_csv - file_data = to_csv(data_rows_with_error) - _file = frappe.get_doc({ - "doctype": "File", - "file_name": file_name, - "attached_to_doctype": "Data Import Legacy", - "attached_to_name": data_import_doc.name, - "folder": "Home/Attachments", - "content": file_data}) - _file.save() - data_import_doc.error_file = _file.file_url - - elif error_flag: - import_status = "Failed" - else: - import_status = "Successful" - - data_import_doc.import_status = import_status - data_import_doc.save() - if data_import_doc.import_status in ["Successful", "Partially Successful"]: - data_import_doc.submit() - publish_progress(100, True) - else: - publish_progress(0, True) - frappe.db.commit() - else: - return log_message - -def get_parent_field(doctype, parenttype): - parentfield = None - - # get parentfield - if parenttype: - for d in frappe.get_meta(parenttype).get_table_fields(): - if d.options==doctype: - parentfield = d.fieldname - break - - if not parentfield: - frappe.msgprint(_("Did not find {0} for {0} ({1})").format("parentfield", parenttype, doctype)) - raise Exception - - return parentfield - -def delete_child_rows(rows, doctype): - """delete child rows for all parents""" - for p in list(set(r[1] for r in rows)): - if p: - frappe.db.sql("""delete from `tab{0}` where parent=%s""".format(doctype), p) diff --git a/frappe/core/doctype/data_import_legacy/log_details.html b/frappe/core/doctype/data_import_legacy/log_details.html deleted file mode 100644 index aa160a742b..0000000000 --- a/frappe/core/doctype/data_import_legacy/log_details.html +++ /dev/null @@ -1,38 +0,0 @@ -
-
- - - - - - - - {% for row in data %} - {% if (!show_only_errors) || (show_only_errors && row.indicator == "red") %} - - - - - - {% endif %} - {% endfor %} -
{{ __("Row No") }} {{ __("Row Status") }} {{ __("Message") }}
- {{ row.row }} - - {{ row.title }} - - {% if (import_status != "Failed" || (row.indicator == "red")) { %} -
{{ row.message }}
- {% if row.link %} - - - - - - {% endif %} - {% } else { %} - {{ __("Document can't saved.") }} - {% } %} -
-
-
\ No newline at end of file diff --git a/frappe/core/doctype/data_import_legacy/test_data_import_legacy.py b/frappe/core/doctype/data_import_legacy/test_data_import_legacy.py deleted file mode 100644 index 6f9964e8f5..0000000000 --- a/frappe/core/doctype/data_import_legacy/test_data_import_legacy.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2020, Frappe Technologies and Contributors -# See license.txt -# import frappe -import unittest - -class TestDataImportLegacy(unittest.TestCase): - pass diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 5b605504e8..53d1c9ffe5 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -53,6 +53,7 @@ class User(Document): def after_insert(self): create_notification_settings(self.name) frappe.cache().delete_key('users_for_mentions') + frappe.cache().delete_key('enabled_users') def validate(self): self.check_demo() @@ -129,6 +130,9 @@ class User(Document): if self.has_value_changed('allow_in_mentions') or self.has_value_changed('user_type'): frappe.cache().delete_key('users_for_mentions') + if self.has_value_changed('enabled'): + frappe.cache().delete_key('enabled_users') + def has_website_permission(self, ptype, user, verbose=False): """Returns true if current user is the session user""" return self.name == frappe.session.user @@ -392,6 +396,8 @@ class User(Document): if self.get('allow_in_mentions'): frappe.cache().delete_key('users_for_mentions') + frappe.cache().delete_key('enabled_users') + def before_rename(self, old_name, new_name, merge=False): self.check_demo() @@ -1230,3 +1236,10 @@ def generate_keys(user): def switch_theme(theme): if theme in ["Dark", "Light"]: frappe.db.set_value("User", frappe.session.user, "desk_theme", theme) + +def get_enabled_users(): + def _get_enabled_users(): + enabled_users = frappe.get_all("User", filters={"enabled": "1"}, pluck="name") + return enabled_users + + return frappe.cache().get_value("enabled_users", _get_enabled_users) \ No newline at end of file diff --git a/frappe/custom/doctype/custom_field/custom_field.json b/frappe/custom/doctype/custom_field/custom_field.json index 2f0819ab68..19462e79de 100644 --- a/frappe/custom/doctype/custom_field/custom_field.json +++ b/frappe/custom/doctype/custom_field/custom_field.json @@ -120,7 +120,7 @@ "label": "Field Type", "oldfieldname": "fieldtype", "oldfieldtype": "Select", - "options": "Attach\nAttach Image\nBarcode\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDuration\nDynamic Link\nFloat\nGeolocation\nHTML\nImage\nInt\nLink\nLong Text\nMarkdown Editor\nPassword\nPercent\nRating\nRead Only\nSection Break\nSelect\nSmall Text\nTable\nTable MultiSelect\nText\nText Editor\nTime\nSignature", + "options": "Attach\nAttach Image\nBarcode\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDuration\nDynamic Link\nFloat\nFold\nGeolocation\nHeading\nHTML\nHTML Editor\nImage\nInt\nLink\nLong Text\nMarkdown Editor\nPassword\nPercent\nRead Only\nRating\nSection Break\nSelect\nSmall Text\nTable\nTable MultiSelect\nText\nText Editor\nTime\nSignature", "reqd": 1 }, { @@ -417,7 +417,7 @@ "idx": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2020-10-29 06:14:43.073329", + "modified": "2021-07-12 04:54:12.042319", "modified_by": "Administrator", "module": "Custom", "name": "Custom Field", diff --git a/frappe/database/database.py b/frappe/database/database.py index 81e24cc7ad..6012e47445 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -15,7 +15,6 @@ from frappe import _ from time import time from frappe.utils import now, getdate, cast_fieldtype, get_datetime from frappe.model.utils.link_count import flush_local_link_count -from frappe.utils import cint class Database(object): @@ -556,8 +555,7 @@ class Database(object): if not df: frappe.throw(_('Invalid field name: {0}').format(frappe.bold(fieldname)), self.InvalidColumnName) - if df.fieldtype in frappe.model.numeric_fieldtypes: - val = cint(val) + val = cast_fieldtype(df.fieldtype, val) self.value_cache[doctype][fieldname] = val diff --git a/frappe/desk/desktop.py b/frappe/desk/desktop.py index cdab7d6d1b..ca53e6cba4 100644 --- a/frappe/desk/desktop.py +++ b/frappe/desk/desktop.py @@ -452,6 +452,7 @@ def get_custom_report_list(module): "type": "Link", "link_type": "report", "doctype": r.ref_doctype, + "dependencies": r.ref_doctype, "is_query_report": 1 if r.report_type in ("Query Report", "Script Report", "Custom Report") else 0, "label": _(r.name), "link_to": r.name, diff --git a/frappe/desk/doctype/notification_log/notification_log.py b/frappe/desk/doctype/notification_log/notification_log.py index 414f272f59..d7d7f68b74 100644 --- a/frappe/desk/doctype/notification_log/notification_log.py +++ b/frappe/desk/doctype/notification_log/notification_log.py @@ -12,7 +12,10 @@ class NotificationLog(Document): frappe.publish_realtime('notification', after_commit=True, user=self.for_user) set_notifications_as_unseen(self.for_user) if is_email_notifications_enabled_for_type(self.for_user, self.type): - send_notification_email(self) + try: + send_notification_email(self) + except frappe.OutgoingEmailError: + frappe.log_error(message=frappe.get_traceback(), title=_("Failed to send notification email")) def get_permission_query_conditions(for_user): diff --git a/frappe/desk/doctype/tag/test_tag.py b/frappe/desk/doctype/tag/test_tag.py index 442a891fd8..6eb7219c26 100644 --- a/frappe/desk/doctype/tag/test_tag.py +++ b/frappe/desk/doctype/tag/test_tag.py @@ -1,8 +1,26 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2019, Frappe Technologies and Contributors -# See license.txt -# import frappe import unittest +import frappe + +from frappe.desk.reportview import get_stats +from frappe.desk.doctype.tag.tag import add_tag class TestTag(unittest.TestCase): - pass + def setUp(self) -> None: + frappe.db.sql("DELETE from `tabTag`") + frappe.db.sql("UPDATE `tabDocType` set _user_tags=''") + + def test_tag_count_query(self): + self.assertDictEqual(get_stats('["_user_tags"]', 'DocType'), + {'_user_tags': [['No Tags', frappe.db.count('DocType')]]}) + add_tag('Standard', 'DocType', 'User') + add_tag('Standard', 'DocType', 'ToDo') + + # count with no filter + self.assertDictEqual(get_stats('["_user_tags"]', 'DocType'), + {'_user_tags': [['Standard', 2], ['No Tags', frappe.db.count('DocType') - 2]]}) + + # count with child table field filter + self.assertDictEqual(get_stats('["_user_tags"]', + 'DocType', + filters='[["DocField", "fieldname", "like", "%last_name%"], ["DocType", "name", "like", "%use%"]]'), + {'_user_tags': [['Standard', 1], ['No Tags', 0]]}) \ No newline at end of file diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index 55515856f1..1dbc52eb5b 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -445,24 +445,36 @@ def get_stats(stats, doctype, filters=[]): for tag in tags: if not tag in columns: continue try: - tagcount = frappe.get_list(doctype, fields=[tag, "count(*)"], - #filters=["ifnull(`%s`,'')!=''" % tag], group_by=tag, as_list=True) - filters = filters + ["ifnull(`%s`,'')!=''" % tag], group_by = tag, as_list = True) - - if tag=='_user_tags': - stats[tag] = scrub_user_tags(tagcount) - stats[tag].append([_("No Tags"), frappe.get_list(doctype, + tag_count = frappe.get_list(doctype, + fields=[tag, "count(*)"], + filters=filters + [[tag, '!=', '']], + group_by=tag, + as_list=True, + distinct=1, + ) + + if tag == '_user_tags': + stats[tag] = scrub_user_tags(tag_count) + no_tag_count = frappe.get_list(doctype, fields=[tag, "count(*)"], - filters=filters +["({0} = ',' or {0} = '' or {0} is null)".format(tag)], as_list=True)[0][1]]) + filters=filters + [[tag, "in", ('', ',')]], + as_list=True, + group_by=tag, + order_by=tag, + ) + + no_tag_count = no_tag_count[0][1] if no_tag_count else 0 + + stats[tag].append([_("No Tags"), no_tag_count]) else: - stats[tag] = tagcount + stats[tag] = tag_count except frappe.db.SQLError: - # does not work for child tables pass - except frappe.db.InternalError: + except frappe.db.InternalError as e: # raised when _user_tags column is added on the fly pass + return stats @frappe.whitelist() diff --git a/frappe/desk/search.py b/frappe/desk/search.py index 040a8c2118..f9b65fc98e 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -168,7 +168,18 @@ def search_widget(doctype, txt, query=None, searchfield=None, start=0, strict=False) if doctype in UNTRANSLATED_DOCTYPES: - values = tuple([v for v in list(values) if re.search(re.escape(txt)+".*", (_(v.name) if as_dict else _(v[0])), re.IGNORECASE)]) + # Filtering the values array so that query is included in very element + values = ( + v for v in values + if re.search( + f"{re.escape(txt)}.*", _(v.name if as_dict else v[0]), re.IGNORECASE + ) + ) + + # Sorting the values array so that relevant results always come first + # This will first bring elements on top in which query is a prefix of element + # Then it will bring the rest of the elements and sort them in lexicographical order + values = sorted(values, key=lambda x: relevance_sorter(x, txt, as_dict)) # remove _relevance from results if as_dict: @@ -208,6 +219,13 @@ def scrub_custom_query(query, key, txt): query = query.replace('%s', ((txt or '') + '%')) return query +def relevance_sorter(key, query, as_dict): + value = _(key.name if as_dict else key[0]) + return ( + value.lower().startswith(query.lower()) is not True, + value + ) + @wrapt.decorator def validate_and_sanitize_search_inputs(fn, instance, args, kwargs): kwargs.update(dict(zip(fn.__code__.co_varnames, args))) diff --git a/frappe/email/test_email_body.py b/frappe/email/test_email_body.py index 8e637273ed..2c7d119fce 100644 --- a/frappe/email/test_email_body.py +++ b/frappe/email/test_email_body.py @@ -127,7 +127,7 @@ w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> ''' transformed_html = '''

Hi John

-

This is a test email

+

This is a test email

''' self.assertTrue(transformed_html in inline_style_in_html(html)) diff --git a/frappe/hooks.py b/frappe/hooks.py index ac42a03461..f3d25d6bf4 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -171,6 +171,9 @@ doc_events = { "frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions", "frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers" ], + "on_update_after_submit": [ + "frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions" + ], "on_change": [ "frappe.social.doctype.energy_point_rule.energy_point_rule.process_energy_points", "frappe.automation.doctype.milestone_tracker.milestone_tracker.evaluate_milestone" diff --git a/frappe/public/js/frappe/form/controls/signature.js b/frappe/public/js/frappe/form/controls/signature.js index bcbdfa0d27..445f5b7b3b 100644 --- a/frappe/public/js/frappe/form/controls/signature.js +++ b/frappe/public/js/frappe/form/controls/signature.js @@ -53,13 +53,15 @@ frappe.ui.form.ControlSignature = class ControlSignature extends frappe.ui.form. this.img = $("") .appendTo(this.img_wrapper).toggle(false); } - refresh_input(e) { + refresh_input() { + // signature dom is not ready + if (!this.body) return; // prevent to load the second time this.make_pad(); this.$wrapper.find(".control-input").toggle(false); this.set_editable(this.get_status()=="Write"); this.load_pad(); - if(this.get_status()=="Read") { + if (this.get_status() == "Read") { $(this.disp_area).toggle(false); } } diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 8064f90a98..faaa3dfbd9 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -1265,7 +1265,9 @@ frappe.ui.form.Form = class FrappeForm { if (df && df[property] != value) { df[property] = value; if (table_field && table_row_name) { - this.fields_dict[fieldname].grid.grid_rows_by_docname[table_row_name].refresh_field(fieldname); + if (this.fields_dict[fieldname].grid.grid_rows_by_docname[table_row_name]) { + this.fields_dict[fieldname].grid.grid_rows_by_docname[table_row_name].refresh_field(fieldname); + } } else { this.refresh_field(fieldname); } diff --git a/frappe/public/js/frappe/form/layout.js b/frappe/public/js/frappe/form/layout.js index 045e5dc1b3..528f485354 100644 --- a/frappe/public/js/frappe/form/layout.js +++ b/frappe/public/js/frappe/form/layout.js @@ -263,9 +263,6 @@ frappe.ui.form.Layout = class Layout { section.addClass("empty-section"); } }); - - this.frm && this.frm.dashboard.refresh(); - } refresh_fields (fields) { diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js index 8d3d0edd53..65635ec1dd 100644 --- a/frappe/public/js/frappe/utils/utils.js +++ b/frappe/public/js/frappe/utils/utils.js @@ -1202,6 +1202,8 @@ Object.assign(frappe.utils, { } else if (type === "report") { if (item.is_query_report) { route = "query-report/" + item.name; + } else if (!item.doctype) { + route = "/report/" + item.name; } else { route = frappe.router.slug(item.doctype) + "/view/report/" + item.name; } diff --git a/frappe/public/js/frappe/widgets/links_widget.js b/frappe/public/js/frappe/widgets/links_widget.js index 2cc824899f..9d0cadc1d0 100644 --- a/frappe/public/js/frappe/widgets/links_widget.js +++ b/frappe/public/js/frappe/widgets/links_widget.js @@ -68,7 +68,7 @@ export default class LinksWidget extends Widget { is_query_report: item.is_query_report }; - if (item.link_type == "Report" && !item.is_query_report) { + if (item.link_type.toLowerCase() == "report" && !item.is_query_report) { opts.doctype = item.dependencies; } diff --git a/frappe/public/scss/common/controls.scss b/frappe/public/scss/common/controls.scss index c939c6de39..a7a8d49510 100644 --- a/frappe/public/scss/common/controls.scss +++ b/frappe/public/scss/common/controls.scss @@ -166,6 +166,9 @@ select.form-control { .ace_print-margin { background-color: var(--dark-border-color); } + .ace_scrollbar { + z-index: 3; + } } .frappe-control[data-fieldtype="Attach"], diff --git a/frappe/public/scss/common/css_variables.scss b/frappe/public/scss/common/css_variables.scss index 48a8a48f5f..112238bfe5 100644 --- a/frappe/public/scss/common/css_variables.scss +++ b/frappe/public/scss/common/css_variables.scss @@ -179,9 +179,20 @@ --text-on-pink: var(--pink-500); --text-on-cyan: var(--cyan-600); + // Layout Colors + --bg-color: var(--gray-50); + --fg-color: white; + --navbar-bg: white; + --fg-hover-color: var(--gray-100); + --card-bg: var(--fg-color); + --disabled-text-color: var(--gray-700); --disabled-control-bg: var(--gray-50); --control-bg: var(--gray-100); --control-bg-on-gray: var(--gray-200); + --awesomebar-focus-bg: var(--fg-color); + --modal-bg: white; + --toast-bg: var(--modal-bg); + --popover-bg: white; --awesomplete-hover-bg: var(--control-bg); diff --git a/frappe/public/scss/desk/css_variables.scss b/frappe/public/scss/desk/css_variables.scss index c099a8bdeb..255d0a91e4 100644 --- a/frappe/public/scss/desk/css_variables.scss +++ b/frappe/public/scss/desk/css_variables.scss @@ -25,20 +25,7 @@ $input-height: 28px !default; --navbar-height: 60px; - // Layout Colors - --bg-color: var(--gray-50); - --fg-color: white; - --navbar-bg: white; - --fg-hover-color: var(--gray-100); - --card-bg: var(--fg-color); - --disabled-text-color: var(--gray-700); - --disabled-control-bg: var(--gray-50); - --control-bg: var(--gray-100); - --control-bg-on-gray: var(--gray-200); - --awesomebar-focus-bg: var(--fg-color); - --modal-bg: white; - --toast-bg: var(--modal-bg); - --popover-bg: white; + --appreciation-color: var(--dark-green-600); --appreciation-bg: var(--dark-green-100); diff --git a/frappe/public/scss/email.bundle.scss b/frappe/public/scss/email.bundle.scss index b1a8d6c485..9b58c130a2 100644 --- a/frappe/public/scss/email.bundle.scss +++ b/frappe/public/scss/email.bundle.scss @@ -36,7 +36,13 @@ a { } p { - margin: 5px 0 !important; + margin: 1em 0 !important; +} + +.with-container { + p { + margin: 5px 0 !important; + } } .ql-editor { diff --git a/frappe/public/scss/website.bundle.scss b/frappe/public/scss/website.bundle.scss index bcbb6f3c6a..06ec73c386 100644 --- a/frappe/public/scss/website.bundle.scss +++ b/frappe/public/scss/website.bundle.scss @@ -1 +1 @@ -@import './website/index'; \ No newline at end of file +@import './website/index'; diff --git a/frappe/public/scss/website/page_builder.scss b/frappe/public/scss/website/page_builder.scss index 6eb6dae5d2..ff9f4ae1e6 100644 --- a/frappe/public/scss/website/page_builder.scss +++ b/frappe/public/scss/website/page_builder.scss @@ -145,20 +145,21 @@ .section-with-cards .card { @include transition(); + border: none; + + .card-body { + padding: 0 1.5rem 2rem 0; + } &:hover { border-color: $gray-500; } .card-title { - line-height: 1; + line-height: 1.3; } &.card-sm { - .card-body { - padding: 1.5rem; - } - .card-title { font-size: $font-size-base; font-weight: 600; @@ -169,10 +170,6 @@ } } &.card-md { - .card-body { - padding: 1.75rem; - } - .card-title { font-size: $font-size-lg; font-weight: 600; @@ -186,10 +183,6 @@ } } &.card-lg { - .card-body { - padding: 2rem; - } - .card-title { font-size: $font-size-xl; font-weight: bold; diff --git a/frappe/social/doctype/energy_point_log/test_energy_point_log.py b/frappe/social/doctype/energy_point_log/test_energy_point_log.py index 8c4dba5d6b..4a6e86463e 100644 --- a/frappe/social/doctype/energy_point_log/test_energy_point_log.py +++ b/frappe/social/doctype/energy_point_log/test_energy_point_log.py @@ -273,6 +273,31 @@ class TestEnergyPointLog(unittest.TestCase): self.assertEqual(points_after_reverting_todo, points_after_closing_todo - rule_points) self.assertEqual(points_after_saving_todo_again, points_after_reverting_todo + rule_points) + def test_energy_points_disabled_user(self): + frappe.set_user('test@example.com') + user = frappe.get_doc('User', 'test@example.com') + user.enabled = 0 + user.save() + todo_point_rule = create_energy_point_rule_for_todo() + energy_point_of_user = get_points('test@example.com') + + created_todo = create_a_todo() + + created_todo.status = 'Closed' + created_todo.save() + points_after_closing_todo = get_points('test@example.com') + + # do not update energy points for disabled user + self.assertEqual(points_after_closing_todo, energy_point_of_user) + + user.enabled = 1 + user.save() + + created_todo.save() + points_after_re_saving_todo = get_points('test@example.com') + self.assertEqual(points_after_re_saving_todo, energy_point_of_user + todo_point_rule.points) + + def create_energy_point_rule_for_todo(multiplier_field=None, for_doc_event='Custom', max_points=None, for_assigned_users=0, field_to_check=None, apply_once=False, user_field='owner'): name = 'ToDo Closed' diff --git a/frappe/social/doctype/energy_point_rule/energy_point_rule.py b/frappe/social/doctype/energy_point_rule/energy_point_rule.py index 1c736528de..b673e8e199 100644 --- a/frappe/social/doctype/energy_point_rule/energy_point_rule.py +++ b/frappe/social/doctype/energy_point_rule/energy_point_rule.py @@ -5,6 +5,7 @@ import frappe from frappe import _ import frappe.cache_manager +from frappe.core.doctype.user.user import get_enabled_users from frappe.model import log_types from frappe.model.document import Document from frappe.social.doctype.energy_point_settings.energy_point_settings import is_energy_point_enabled @@ -44,7 +45,7 @@ class EnergyPointRule(Document): try: for user in users: - if not user or user == 'Administrator': continue + if not is_eligible_user(user): continue create_energy_points_log(reference_doctype, reference_name, { 'points': points, 'user': user, @@ -119,3 +120,8 @@ def get_energy_point_doctypes(): d.reference_doctype for d in frappe.get_all('Energy Point Rule', ['reference_doctype'], {'enabled': 1}) ] + +def is_eligible_user(user): + '''Checks if user is eligible to get energy points''' + enabled_users = get_enabled_users() + return user and user in enabled_users and user != 'Administrator' diff --git a/frappe/tests/test_boilerplate.py b/frappe/tests/test_boilerplate.py index 4ae78c94de..259d5a9194 100644 --- a/frappe/tests/test_boilerplate.py +++ b/frappe/tests/test_boilerplate.py @@ -20,7 +20,7 @@ class TestBoilerPlate(unittest.TestCase): def test_create_app(self): title = "Test App" - description = "Test app for unit testing" + description = "This app's description contains 'single quotes' and \"double quotes\"." publisher = "Test Publisher" email = "example@example.org" icon = "" # empty -> default diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index 07bdf8791e..54103f0151 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -426,3 +426,13 @@ class TestCommands(BaseTestCommands): self.assertEqual(self.returncode, 0) self.assertIn("pong", self.stdout) + def test_version(self): + self.execute("bench version") + self.assertEqual(self.returncode, 0) + + for output in ["legacy", "plain", "table", "json"]: + self.execute(f"bench version -f {output}") + self.assertEqual(self.returncode, 0) + + self.execute("bench version -f invalid") + self.assertEqual(self.returncode, 2) diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index a31a898d73..04c9a525b1 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -5,6 +5,7 @@ import unittest from random import choice +import datetime import frappe from frappe.custom.doctype.custom_field.custom_field import create_custom_field @@ -45,11 +46,35 @@ class TestDB(unittest.TestCase): frappe.db.escape("香港濟生堂製藥有限公司 - IT".encode("utf-8")) def test_get_single_value(self): - frappe.db.set_value('System Settings', 'System Settings', 'backup_limit', 5) - frappe.db.commit() + #setup + values_dict = { + "Float": 1.5, + "Int": 1, + "Percent": 55.5, + "Currency": 12.5, + "Data": "Test", + "Date": datetime.datetime.now().date(), + "Datetime": datetime.datetime.now(), + "Time": datetime.timedelta(hours=9, minutes=45, seconds=10) + } + test_inputs = [{ + "fieldtype": fieldtype, + "value": value} for fieldtype, value in values_dict.items()] + for fieldtype in values_dict.keys(): + create_custom_field("Print Settings", { + "fieldname": f"test_{fieldtype.lower()}", + "label": f"Test {fieldtype}", + "fieldtype": fieldtype, + }) + + #test + for inp in test_inputs: + fieldname = f"test_{inp['fieldtype'].lower()}" + frappe.db.set_value("Print Settings", "Print Settings", fieldname, inp["value"]) + self.assertEqual(frappe.db.get_single_value("Print Settings", fieldname), inp["value"]) - limit = frappe.db.get_single_value('System Settings', 'backup_limit') - self.assertEqual(limit, 5) + #teardown + clear_custom_fields("Print Settings") def test_log_touched_tables(self): frappe.flags.in_migrate = True @@ -132,29 +157,29 @@ class TestDB(unittest.TestCase): # Testing read self.assertEqual(list(frappe.get_all("ToDo", fields=[random_field], limit=1)[0])[0], random_field) - self.assertEqual(list(frappe.get_all("ToDo", fields=["`{0}` as total".format(random_field)], limit=1)[0])[0], "total") + self.assertEqual(list(frappe.get_all("ToDo", fields=[f"`{random_field}` as total"], limit=1)[0])[0], "total") # Testing read for distinct and sql functions self.assertEqual(list( frappe.get_all("ToDo", - fields=["`{0}` as total".format(random_field)], + fields=[f"`{random_field}` as total"], distinct=True, limit=1, )[0] )[0], "total") self.assertEqual(list( frappe.get_all("ToDo", - fields=["`{0}`".format(random_field)], + fields=[f"`{random_field}`"], distinct=True, limit=1, )[0] )[0], random_field) self.assertEqual(list( frappe.get_all("ToDo", - fields=["count(`{0}`)".format(random_field)], + fields=[f"count(`{random_field}`)"], limit=1 )[0] - )[0], "count" if frappe.conf.db_type == "postgres" else "count(`{0}`)".format(random_field)) + )[0], "count" if frappe.conf.db_type == "postgres" else f"count(`{random_field}`)") # Testing update frappe.db.set_value(test_doctype, random_doc, random_field, random_value) diff --git a/frappe/tests/test_search.py b/frappe/tests/test_search.py index 9ad02f49a6..afb584ea15 100644 --- a/frappe/tests/test_search.py +++ b/frappe/tests/test_search.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt import unittest @@ -6,7 +6,16 @@ import frappe from frappe.desk.search import search_link from frappe.desk.search import search_widget + class TestSearch(unittest.TestCase): + def setUp(self): + if self._testMethodName == "test_link_field_order": + setup_test_link_field_order(self) + + def tearDown(self): + if self._testMethodName == "test_link_field_order": + teardown_test_link_field_order(self) + def test_search_field_sanitizer(self): # pass search_link('DocType', 'User', query=None, filters=None, page_length=20, searchfield='name') @@ -38,6 +47,18 @@ class TestSearch(unittest.TestCase): search_link, 'DocType', 'Customer', query=None, filters=None, page_length=20, searchfield=';') + def test_link_field_order(self): + # Making a request to the search_link with the tree doctype + search_link(doctype=self.tree_doctype_name, txt='all', query=None, + filters=None, page_length=20, searchfield=None) + result = frappe.response['results'] + + # Check whether the result is sorted or not + self.assertEquals(self.parent_doctype_name, result[0]['value']) + + # Check whether searching for parent also list out children + self.assertEquals(len(result), len(self.child_doctypes_names) + 1) + #Search for the word "pay", part of the word "pays" (country) in french. def test_link_search_in_foreign_language(self): try: @@ -80,4 +101,58 @@ class TestSearch(unittest.TestCase): @frappe.validate_and_sanitize_search_inputs def get_data(doctype, txt, searchfield, start, page_len, filters): - return [doctype, txt, searchfield, start, page_len, filters] \ No newline at end of file + return [doctype, txt, searchfield, start, page_len, filters] + +def setup_test_link_field_order(TestCase): + TestCase.tree_doctype_name = 'Test Tree Order' + TestCase.child_doctype_list = [] + TestCase.child_doctypes_names = ['USA', 'India', 'Russia', 'China'] + TestCase.parent_doctype_name = 'All Territories' + + # Create Tree doctype + TestCase.tree_doc = frappe.get_doc({ + 'doctype': 'DocType', + 'name': TestCase.tree_doctype_name, + 'module': 'Custom', + 'custom': 1, + 'is_tree': 1, + 'autoname': 'field:random', + 'fields': [{ + 'fieldname': 'random', + 'label': 'Random', + 'fieldtype': 'Data' + }] + }).insert() + TestCase.tree_doc.search_fields = 'parent_test_tree_order' + TestCase.tree_doc.save() + + # Create root for the tree doctype + frappe.get_doc({ + "doctype": TestCase.tree_doctype_name, + "random": TestCase.parent_doctype_name, + "is_group": 1 + }).insert() + + # Create children for the root + for child_name in TestCase.child_doctypes_names: + temp = frappe.get_doc({ + "doctype": TestCase.tree_doctype_name, + "random": child_name, + "parent_test_tree_order": TestCase.parent_doctype_name + }).insert() + TestCase.child_doctype_list.append(temp) + +def teardown_test_link_field_order(TestCase): + # Deleting all the created doctype + for child_doctype in TestCase.child_doctype_list: + child_doctype.delete() + + frappe.delete_doc( + TestCase.tree_doctype_name, + TestCase.parent_doctype_name, + ignore_permissions=True, + force=True, + for_reload=True, + ) + + TestCase.tree_doc.delete() diff --git a/frappe/utils/boilerplate.py b/frappe/utils/boilerplate.py index 8dab9b748f..d88eaa5745 100755 --- a/frappe/utils/boilerplate.py +++ b/frappe/utils/boilerplate.py @@ -66,9 +66,6 @@ def make_boilerplate(dest, app_name): with open(os.path.join(dest, hooks.app_name, ".gitignore"), "w") as f: f.write(frappe.as_unicode(gitignore_template.format(app_name = hooks.app_name))) - with open(os.path.join(dest, hooks.app_name, "setup.py"), "w") as f: - f.write(frappe.as_unicode(setup_template.format(**hooks))) - with open(os.path.join(dest, hooks.app_name, "requirements.txt"), "w") as f: f.write("# frappe -- https://github.com/frappe/frappe is installed via 'bench init'") @@ -82,6 +79,14 @@ def make_boilerplate(dest, app_name): with open(os.path.join(dest, hooks.app_name, hooks.app_name, "modules.txt"), "w") as f: f.write(frappe.as_unicode(hooks.app_title)) + # These values could contain quotes and can break string declarations + # So escaping them before setting variables in setup.py and hooks.py + for key in ("app_publisher", "app_description", "app_license"): + hooks[key] = hooks[key].replace("\\", "\\\\").replace("'", "\\'").replace("\"", "\\\"") + + with open(os.path.join(dest, hooks.app_name, "setup.py"), "w") as f: + f.write(frappe.as_unicode(setup_template.format(**hooks))) + with open(os.path.join(dest, hooks.app_name, hooks.app_name, "hooks.py"), "w") as f: f.write(frappe.as_unicode(hooks_template.format(**hooks))) @@ -328,18 +333,18 @@ def get_data(): setup_template = """from setuptools import setup, find_packages -with open('requirements.txt') as f: - install_requires = f.read().strip().split('\\n') +with open("requirements.txt") as f: + install_requires = f.read().strip().split("\\n") # get version from __version__ variable in {app_name}/__init__.py from {app_name} import __version__ as version setup( - name='{app_name}', + name="{app_name}", version=version, - description='{app_description}', - author='{app_publisher}', - author_email='{app_email}', + description="{app_description}", + author="{app_publisher}", + author_email="{app_email}", packages=find_packages(), zip_safe=False, include_package_data=True, diff --git a/frappe/utils/fixtures.py b/frappe/utils/fixtures.py index 1f33c36b13..895a3c8373 100644 --- a/frappe/utils/fixtures.py +++ b/frappe/utils/fixtures.py @@ -20,7 +20,7 @@ def sync_fixtures(app=None): if os.path.exists(frappe.get_app_path(app, "fixtures")): fixture_files = sorted(os.listdir(frappe.get_app_path(app, "fixtures"))) for fname in fixture_files: - if fname.endswith(".json") or fname.endswith(".csv"): + if fname.endswith(".json"): import_doc(frappe.get_app_path(app, "fixtures", fname)) import_custom_scripts(app) diff --git a/frappe/utils/global_search.py b/frappe/utils/global_search.py index efe92232d9..8fa2ea474f 100644 --- a/frappe/utils/global_search.py +++ b/frappe/utils/global_search.py @@ -231,9 +231,6 @@ def update_global_search(doc): if frappe.local.conf.get('disable_global_search'): return - if frappe.local.conf.get('disable_global_search'): - return - if doc.docstatus > 1 or (doc.meta.has_field("enabled") and not doc.get("enabled")) \ or doc.get("disabled"): return diff --git a/frappe/website/page_renderers/not_found_page.py b/frappe/website/page_renderers/not_found_page.py index af510fecfc..de631f5dfe 100644 --- a/frappe/website/page_renderers/not_found_page.py +++ b/frappe/website/page_renderers/not_found_page.py @@ -8,11 +8,11 @@ from frappe.website.utils import can_cache HOMEPAGE_PATHS = ('/', '/index', 'index') class NotFoundPage(TemplatePage): - def __init__(self, path, http_status_code): + def __init__(self, path, http_status_code=None): self.request_path = path self.request_url = frappe.local.request.url if hasattr(frappe.local, 'request') else '' path = '404' - http_status_code = 404 + http_status_code = http_status_code or 404 super().__init__(path=path, http_status_code=http_status_code) def can_render(self): diff --git a/yarn.lock b/yarn.lock index bd1b5ba3ef..f21f493aef 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4163,16 +4163,11 @@ minimatch@^3.0.4, minimatch@~3.0.2: dependencies: brace-expansion "^1.1.7" -minimist@^1.1.3, minimist@^1.2.5: +minimist@^1.1.3, minimist@^1.2.0, minimist@^1.2.5: version "1.2.5" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== -minimist@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" - integrity sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ= - minipass@^3.0.0: version "3.1.3" resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.1.3.tgz#7d42ff1f39635482e15f9cdb53184deebd5815fd"