diff --git a/.github/helper/install.sh b/.github/helper/install.sh index 19a7c68e19..246bdbe096 100644 --- a/.github/helper/install.sh +++ b/.github/helper/install.sh @@ -50,7 +50,9 @@ if [ "$TYPE" == "server" ]; then sed -i 's/^socketio:/# socketio:/g' Procfile; f if [ "$TYPE" == "server" ]; then sed -i 's/^redis_socketio:/# redis_socketio:/g' Procfile; fi if [ "$TYPE" == "ui" ]; then bench setup requirements --node; fi -if [ "$TYPE" == "server" ]; then bench setup requirements --dev; fi +bench setup requirements --dev + +if [ "$TYPE" == "ui" ]; then sed -i 's/^web: bench serve/web: bench serve --with-coverage/g' Procfile; fi # install node-sass which is required for website theme test cd ./apps/frappe || exit @@ -60,4 +62,4 @@ cd ../.. bench start & bench --site test_site reinstall --yes if [ "$TYPE" == "server" ]; then bench --site test_site_producer reinstall --yes; fi -CI=Yes bench build --app frappe +if [ "$TYPE" == "server" ]; then CI=Yes bench build --app frappe; fi diff --git a/.github/helper/roulette.py b/.github/helper/roulette.py index 9831df7f30..90f4608a22 100644 --- a/.github/helper/roulette.py +++ b/.github/helper/roulette.py @@ -41,6 +41,7 @@ if __name__ == "__main__": # this is a push build, run all builds if not pr_number: os.system('echo "::set-output name=build::strawberry"') + os.system('echo "::set-output name=build-server::strawberry"') sys.exit(0) files_list = files_list or get_files_list(pr_number=pr_number, repo=repo) @@ -52,7 +53,8 @@ if __name__ == "__main__": ci_files_changed = any(f for f in files_list if is_ci(f)) only_docs_changed = len(list(filter(is_docs, files_list))) == len(files_list) only_frontend_code_changed = len(list(filter(is_frontend_code, files_list))) == len(files_list) - only_py_changed = len(list(filter(is_py, files_list))) == len(files_list) + updated_py_file_count = len(list(filter(is_py, files_list))) + only_py_changed = updated_py_file_count == len(files_list) if ci_files_changed: print("CI related files were updated, running all build processes.") @@ -65,8 +67,12 @@ if __name__ == "__main__": print("Only Frontend code was updated; Stopping Python build process.") sys.exit(0) - elif only_py_changed and build_type == "ui": - print("Only Python code was updated, stopping Cypress build process.") - sys.exit(0) + elif build_type == "ui": + if only_py_changed: + print("Only Python code was updated, stopping Cypress build process.") + sys.exit(0) + elif updated_py_file_count > 0: + # both frontend and backend code were updated + os.system('echo "::set-output name=build-server::strawberry"') os.system('echo "::set-output name=build::strawberry"') diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 3eefd1ce82..fc8093444e 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -141,6 +141,12 @@ jobs: env: CYPRESS_RECORD_KEY: 4a48f41c-11b3-425b-aa88-c58048fa69eb + - name: Stop server + if: ${{ steps.check-build.outputs.build-server == 'strawberry' }} + run: | + ps -ef | grep "frappe serve" | awk '{print $2}' | xargs kill -s SIGINT 2> /dev/null || true + sleep 5 + - name: Check If Coverage Report Exists id: check_coverage uses: andstor/file-existence-action@v1 @@ -156,3 +162,13 @@ jobs: directory: /home/runner/frappe-bench/apps/frappe/.cypress-coverage/ verbose: true flags: ui-tests + + - name: Upload Server Coverage Data + if: ${{ steps.check-build.outputs.build-server == 'strawberry' }} + uses: codecov/codecov-action@v2 + with: + name: MariaDB + fail_ci_if_error: true + files: /home/runner/frappe-bench/sites/coverage.xml + verbose: true + flags: server diff --git a/cypress/integration/depends_on.js b/cypress/integration/depends_on.js index 9aa6b5d89d..12f54f2b6e 100644 --- a/cypress/integration/depends_on.js +++ b/cypress/integration/depends_on.js @@ -55,10 +55,31 @@ context('Depends On', () => { 'read_only_depends_on': "eval:doc.test_field=='Some Other Value'", 'options': "Child Test Depends On" }, + { + "label": "Dependent Tab", + "fieldname": "dependent_tab", + "fieldtype": "Tab Break", + "depends_on": "eval:doc.test_field=='Show Tab'" + }, + { + "fieldname": "tab_section", + "fieldtype": "Section Break", + }, + { + "label": "Field in Tab", + "fieldname": "field_in_tab", + "fieldtype": "Data", + } ] }); }); }); + it('should show the tab on other setting field value', () => { + cy.new_form('Test Depends On'); + cy.fill_field('test_field', 'Show Tab'); + cy.get('body').click(); + cy.findByRole("tab", {name: "Dependent Tab"}).should('be.visible'); + }); it('should set the field as mandatory depending on other fields value', () => { cy.new_form('Test Depends On'); cy.fill_field('test_field', 'Some Value'); diff --git a/frappe/build.py b/frappe/build.py index 6b93b8b93a..7a06ee3a22 100644 --- a/frappe/build.py +++ b/frappe/build.py @@ -1,25 +1,21 @@ -# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import os -import re -import json import shutil +import re import subprocess +from distutils.spawn import find_executable from subprocess import getoutput -from io import StringIO from tempfile import mkdtemp, mktemp -from distutils.spawn import find_executable - -import frappe -from frappe.utils.minify import JavascriptMinify +from urllib.parse import urlparse import click import psutil -from urllib.parse import urlparse -from semantic_version import Version from requests import head from requests.exceptions import HTTPError +from semantic_version import Version +import frappe timestamps = {} app_paths = None @@ -32,6 +28,7 @@ class AssetsNotDownloadedError(Exception): class AssetsDontExistError(HTTPError): pass + def download_file(url, prefix): from requests import get @@ -277,12 +274,14 @@ def check_node_executable(): click.echo(f"{warn} Please install yarn using below command and try again.\nnpm install -g yarn") click.echo() + def get_node_env(): node_env = { "NODE_OPTIONS": f"--max_old_space_size={get_safe_max_old_space_size()}" } return node_env + def get_safe_max_old_space_size(): safe_max_old_space_size = 0 try: @@ -296,6 +295,7 @@ def get_safe_max_old_space_size(): return safe_max_old_space_size + def generate_assets_map(): symlinks = {} @@ -344,7 +344,6 @@ def clear_broken_symlinks(): os.remove(path) - def unstrip(message: str) -> str: """Pads input string on the right side until the last available column in the terminal """ @@ -397,94 +396,6 @@ def link_assets_dir(source, target, hard_link=False): symlink(source, target, overwrite=True) -def build(no_compress=False, verbose=False): - for target, sources in get_build_maps().items(): - pack(os.path.join(assets_path, target), sources, no_compress, verbose) - - -def get_build_maps(): - """get all build.jsons with absolute paths""" - # framework js and css files - - build_maps = {} - for app_path in app_paths: - path = os.path.join(app_path, "public", "build.json") - if os.path.exists(path): - with open(path) as f: - try: - for target, sources in (json.loads(f.read() or "{}")).items(): - # update app path - source_paths = [] - for source in sources: - if isinstance(source, list): - s = frappe.get_pymodule_path(source[0], *source[1].split("/")) - else: - s = os.path.join(app_path, source) - source_paths.append(s) - - build_maps[target] = source_paths - except ValueError as e: - print(path) - print("JSON syntax error {0}".format(str(e))) - return build_maps - - -def pack(target, sources, no_compress, verbose): - outtype, outtxt = target.split(".")[-1], "" - jsm = JavascriptMinify() - - for f in sources: - suffix = None - if ":" in f: - f, suffix = f.split(":") - if not os.path.exists(f) or os.path.isdir(f): - print("did not find " + f) - continue - timestamps[f] = os.path.getmtime(f) - try: - with open(f, "r") as sourcefile: - data = str(sourcefile.read(), "utf-8", errors="ignore") - - extn = f.rsplit(".", 1)[1] - - if ( - outtype == "js" - and extn == "js" - and (not no_compress) - and suffix != "concat" - and (".min." not in f) - ): - tmpin, tmpout = StringIO(data.encode("utf-8")), StringIO() - jsm.minify(tmpin, tmpout) - minified = tmpout.getvalue() - if minified: - outtxt += str(minified or "", "utf-8").strip("\n") + ";" - - if verbose: - print("{0}: {1}k".format(f, int(len(minified) / 1024))) - elif outtype == "js" and extn == "html": - # add to frappe.templates - outtxt += html_to_js_template(f, data) - else: - outtxt += "\n/*\n *\t%s\n */" % f - outtxt += "\n" + data + "\n" - - except Exception: - print("--Error in:" + f + "--") - print(frappe.get_traceback()) - - with open(target, "w") as f: - f.write(outtxt.encode("utf-8")) - - print("Wrote %s - %sk" % (target, str(int(os.path.getsize(target) / 1024)))) - - -def html_to_js_template(path, content): - """returns HTML template content as Javascript code, adding it to `frappe.templates`""" - return """frappe.templates["{key}"] = '{content}';\n""".format( - key=path.rsplit("/", 1)[-1][:-5], content=scrub_html_template(content)) - - def scrub_html_template(content): """Returns HTML content with removed whitespace and comments""" # remove whitespace to a single space @@ -496,37 +407,7 @@ def scrub_html_template(content): return content.replace("'", "\'") -def files_dirty(): - for target, sources in get_build_maps().items(): - for f in sources: - if ":" in f: - f, suffix = f.split(":") - if not os.path.exists(f) or os.path.isdir(f): - continue - if os.path.getmtime(f) != timestamps.get(f): - print(f + " dirty") - return True - else: - return False - - -def compile_less(): - if not find_executable("lessc"): - return - - for path in app_paths: - less_path = os.path.join(path, "public", "less") - if os.path.exists(less_path): - for fname in os.listdir(less_path): - if fname.endswith(".less") and fname != "variables.less": - fpath = os.path.join(less_path, fname) - mtime = os.path.getmtime(fpath) - if fpath in timestamps and mtime == timestamps[fpath]: - continue - - timestamps[fpath] = mtime - - print("compiling {0}".format(fpath)) - - css_path = os.path.join(path, "public", "css", fname.rsplit(".", 1)[0] + ".css") - os.system("lessc {0} > {1}".format(fpath, css_path)) +def html_to_js_template(path, content): + """returns HTML template content as Javascript code, adding it to `frappe.templates`""" + return """frappe.templates["{key}"] = '{content}';\n""".format( + key=path.rsplit("/", 1)[-1][:-5], content=scrub_html_template(content)) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 62488525b0..1684f26d49 100755 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -447,21 +447,17 @@ def disable_user(context, email): @pass_context def migrate(context, skip_failing=False, skip_search_index=False): "Run patches, sync schema and rebuild files/translations" - from frappe.migrate import migrate + from frappe.migrate import SiteMigration for site in context.sites: click.secho(f"Migrating {site}", fg="green") - frappe.init(site=site) - frappe.connect() try: - migrate( - context.verbose, + SiteMigration( skip_failing=skip_failing, - skip_search_index=skip_search_index - ) + skip_search_index=skip_search_index, + ).run(site=site) finally: print() - frappe.destroy() if not context.sites: raise SiteNotSpecifiedError diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index e3379a43aa..7246df8aa7 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -742,8 +742,9 @@ def run_ui_tests(context, app, headless=False, parallel=True, with_coverage=Fals @click.option('--profile', is_flag=True, default=False) @click.option('--noreload', "no_reload", is_flag=True, default=False) @click.option('--nothreading', "no_threading", is_flag=True, default=False) +@click.option('--with-coverage', is_flag=True, default=False) @pass_context -def serve(context, port=None, profile=False, no_reload=False, no_threading=False, sites_path='.', site=None): +def serve(context, port=None, profile=False, no_reload=False, no_threading=False, sites_path='.', site=None, with_coverage=False): "Start development web server" import frappe.app @@ -751,8 +752,12 @@ def serve(context, port=None, profile=False, no_reload=False, no_threading=False site = None else: site = context.sites[0] - - frappe.app.serve(port=port, profile=profile, no_reload=no_reload, no_threading=no_threading, site=site, sites_path='.') + with CodeCoverage(with_coverage, 'frappe'): + if with_coverage: + # unable to track coverage with threading enabled + no_threading = True + no_reload = True + frappe.app.serve(port=port, profile=profile, no_reload=no_reload, no_threading=no_threading, site=site, sites_path='.') @click.command('request') diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index 5a22304f32..bf63afa5c5 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -3,7 +3,7 @@ import frappe, json, os import unittest -from frappe.desk.query_report import run, save_report +from frappe.desk.query_report import run, save_report, add_total_row from frappe.desk.reportview import delete_report, save_report as _save_report from frappe.custom.doctype.customize_form.customize_form import reset_customization from frappe.core.doctype.user_permission.test_user_permission import create_user @@ -282,3 +282,55 @@ result = [ # Set user back to administrator frappe.set_user('Administrator') + + def test_add_total_row_for_tree_reports(self): + report_settings = { + 'tree': True, + 'parent_field': 'parent_value' + } + + columns = [ + { + "fieldname": "parent_column", + "label": "Parent Column", + "fieldtype": "Data", + "width": 10 + }, + { + "fieldname": "column_1", + "label": "Column 1", + "fieldtype": "Float", + "width": 10 + }, + { + "fieldname": "column_2", + "label": "Column 2", + "fieldtype": "Float", + "width": 10 + } + ] + + result = [ + { + "parent_column": "Parent 1", + "column_1": 200, + "column_2": 150.50 + }, + { + "parent_column": "Child 1", + "column_1": 100, + "column_2": 75.25, + "parent_value": "Parent 1" + }, + { + "parent_column": "Child 2", + "column_1": 100, + "column_2": 75.25, + "parent_value": "Parent 1" + } + ] + + result = add_total_row(result, columns, meta=None, report_settings=report_settings) + self.assertEqual(result[-1][0], "Total") + self.assertEqual(result[-1][1], 200) + self.assertEqual(result[-1][2], 150.50) diff --git a/frappe/coverage.py b/frappe/coverage.py index 1969cae141..5f89800deb 100644 --- a/frappe/coverage.py +++ b/frappe/coverage.py @@ -29,6 +29,7 @@ FRAPPE_EXCLUSIONS = [ "*/commands/*", "*/frappe/change_log/*", "*/frappe/exceptions*", + "*/frappe/coverage.py", "*frappe/setup.py", "*/doctype/*/*_dashboard.py", "*/patches/*", diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index 97bceeb725..9ed956e986 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -73,7 +73,7 @@ def get_report_result(report, filters): return res @frappe.read_only() -def generate_report_result(report, filters=None, user=None, custom_columns=None): +def generate_report_result(report, filters=None, user=None, custom_columns=None, report_settings=None): user = user or frappe.session.user filters = filters or [] @@ -108,7 +108,7 @@ def generate_report_result(report, filters=None, user=None, custom_columns=None) result = get_filtered_data(report.ref_doctype, columns, result, user) if cint(report.add_total_row) and result and not skip_total_row: - result = add_total_row(result, columns) + result = add_total_row(result, columns, report_settings=report_settings) return { "result": result, @@ -210,7 +210,7 @@ def get_script(report_name): @frappe.whitelist() @frappe.read_only() -def run(report_name, filters=None, user=None, ignore_prepared_report=False, custom_columns=None): +def run(report_name, filters=None, user=None, ignore_prepared_report=False, custom_columns=None, report_settings=None): report = get_report_doc(report_name) if not user: user = frappe.session.user @@ -238,7 +238,7 @@ def run(report_name, filters=None, user=None, ignore_prepared_report=False, cust dn = "" result = get_prepared_report_result(report, filters, dn, user) else: - result = generate_report_result(report, filters, user, custom_columns) + result = generate_report_result(report, filters, user, custom_columns, report_settings) result["add_total_row"] = report.add_total_row and not result.get( "skip_total_row", False @@ -435,9 +435,19 @@ def build_xlsx_data(columns, data, visible_idx, include_indentation, ignore_visi return result, column_widths -def add_total_row(result, columns, meta=None): +def add_total_row(result, columns, meta=None, report_settings=None): total_row = [""] * len(columns) has_percent = [] + is_tree = False + parent_field = '' + + if report_settings: + if isinstance(report_settings, (str,)): + report_settings = json.loads(report_settings) + + is_tree = report_settings.get('tree') + parent_field = report_settings.get('parent_field') + for i, col in enumerate(columns): fieldtype, options, fieldname = None, None, None if isinstance(col, str): @@ -464,12 +474,12 @@ def add_total_row(result, columns, meta=None): for row in result: if i >= len(row): continue - cell = row.get(fieldname) if isinstance(row, dict) else row[i] if fieldtype in ["Currency", "Int", "Float", "Percent", "Duration"] and flt( cell ): - total_row[i] = flt(total_row[i]) + flt(cell) + if not (is_tree and row.get(parent_field)): + total_row[i] = flt(total_row[i]) + flt(cell) if fieldtype == "Percent" and i not in has_percent: has_percent.append(i) diff --git a/frappe/installer.py b/frappe/installer.py index d892ff4ddc..0948620ab0 100644 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -529,10 +529,9 @@ def extract_sql_gzip(sql_gz_path): import subprocess try: - # dvf - decompress, verbose, force original_file = sql_gz_path decompressed_file = original_file.rstrip(".gz") - cmd = 'gzip -dvf < {0} > {1}'.format(original_file, decompressed_file) + cmd = 'gzip --decompress --force < {0} > {1}'.format(original_file, decompressed_file) subprocess.check_call(cmd, shell=True) except Exception: raise diff --git a/frappe/migrate.py b/frappe/migrate.py index d13fe858f7..eabd0ff3e0 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -1,30 +1,54 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import json import os -import sys +from textwrap import dedent + import frappe -import frappe.translate -import frappe.modules.patch_handler import frappe.model.sync -from frappe.utils.fixtures import sync_fixtures -from frappe.utils.connections import check_connection -from frappe.utils.dashboard import sync_dashboards +import frappe.modules.patch_handler +import frappe.translate from frappe.cache_manager import clear_global_cache -from frappe.desk.notifications import clear_notifications -from frappe.website.utils import clear_website_cache from frappe.core.doctype.language.language import sync_languages -from frappe.modules.utils import sync_customizations from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs -from frappe.search.website_search import build_index_for_all_routes from frappe.database.schema import add_column +from frappe.desk.notifications import clear_notifications from frappe.modules.patch_handler import PatchType +from frappe.modules.utils import sync_customizations +from frappe.search.website_search import build_index_for_all_routes +from frappe.utils.connections import check_connection +from frappe.utils.dashboard import sync_dashboards +from frappe.utils.fixtures import sync_fixtures +from frappe.website.utils import clear_website_cache + +BENCH_START_MESSAGE = dedent( + """ + Cannot run bench migrate without the services running. + If you are running bench in development mode, make sure that bench is running: + + $ bench start + Otherwise, check the server logs and ensure that all the required services are running. + """ +) -def migrate(verbose=True, skip_failing=False, skip_search_index=False): - '''Migrate all apps to the current version, will: +def atomic(method): + def wrapper(*args, **kwargs): + try: + ret = method(*args, **kwargs) + frappe.db.commit() + return ret + except Exception: + frappe.db.rollback() + raise + + return wrapper + + +class SiteMigration: + """Migrate all apps to the current version, will: - run before migrate hooks - run patches - sync doctypes (schema) @@ -35,70 +59,117 @@ def migrate(verbose=True, skip_failing=False, skip_search_index=False): - sync languages - sync web pages (from /www) - run after migrate hooks - ''' - - service_status = check_connection(redis_services=["redis_cache"]) - if False in service_status.values(): - for service in service_status: - if not service_status.get(service, True): - print("{} service is not running.".format(service)) - print("""Cannot run bench migrate without the services running. -If you are running bench in development mode, make sure that bench is running: + """ -$ bench start + def __init__(self, skip_failing: bool = False, skip_search_index: bool = False) -> None: + self.skip_failing = skip_failing + self.skip_search_index = skip_search_index -Otherwise, check the server logs and ensure that all the required services are running.""") - sys.exit(1) + def setUp(self): + """Complete setup required for site migration + """ + frappe.flags.touched_tables = set() + self.touched_tables_file = frappe.get_site_path("touched_tables.json") + add_column(doctype="DocType", column_name="migration_hash", fieldtype="Data") + clear_global_cache() - touched_tables_file = frappe.get_site_path('touched_tables.json') - if os.path.exists(touched_tables_file): - os.remove(touched_tables_file) + if os.path.exists(self.touched_tables_file): + os.remove(self.touched_tables_file) - try: - add_column(doctype="DocType", column_name="migration_hash", fieldtype="Data") - frappe.flags.touched_tables = set() frappe.flags.in_migrate = True - clear_global_cache() + def tearDown(self): + """Run operations that should be run post schema updation processes + This should be executed irrespective of outcome + """ + frappe.translate.clear_cache() + clear_website_cache() + clear_notifications() + + with open(self.touched_tables_file, "w") as f: + json.dump(list(frappe.flags.touched_tables), f, sort_keys=True, indent=4) + + if not self.skip_search_index: + print(f"Building search index for {frappe.local.site}") + build_index_for_all_routes() + + frappe.publish_realtime("version-update") + frappe.flags.touched_tables.clear() + frappe.flags.in_migrate = False + @atomic + def pre_schema_updates(self): + """Executes `before_migrate` hooks + """ for app in frappe.get_installed_apps(): - for fn in frappe.get_hooks('before_migrate', app_name=app): + for fn in frappe.get_hooks("before_migrate", app_name=app): frappe.get_attr(fn)() - frappe.modules.patch_handler.run_all(skip_failing=skip_failing, patch_type=PatchType.pre_model_sync) + @atomic + def run_schema_updates(self): + """Run patches as defined in patches.txt, sync schema changes as defined in the {doctype}.json files + """ + frappe.modules.patch_handler.run_all(skip_failing=self.skip_failing, patch_type=PatchType.pre_model_sync) frappe.model.sync.sync_all() - frappe.modules.patch_handler.run_all(skip_failing=skip_failing, patch_type=PatchType.post_model_sync) - frappe.translate.clear_cache() + frappe.modules.patch_handler.run_all(skip_failing=self.skip_failing, patch_type=PatchType.post_model_sync) + + @atomic + def post_schema_updates(self): + """Execute pending migration tasks post patches execution & schema sync + This includes: + * Sync `Scheduled Job Type` and scheduler events defined in hooks + * Sync fixtures & custom scripts + * Sync in-Desk Module Dashboards + * Sync customizations: Custom Fields, Property Setters, Custom Permissions + * Sync Frappe's internal language master + * Sync Portal Menu Items + * Sync Installed Applications Version History + * Execute `after_migrate` hooks + """ sync_jobs() sync_fixtures() sync_dashboards() sync_customizations() sync_languages() - frappe.get_doc('Portal Settings', 'Portal Settings').sync_menu() - - # syncs static files - clear_website_cache() - - # updating installed applications data - frappe.get_single('Installed Applications').update_versions() + frappe.get_single("Portal Settings").sync_menu() + frappe.get_single("Installed Applications").update_versions() for app in frappe.get_installed_apps(): - for fn in frappe.get_hooks('after_migrate', app_name=app): + for fn in frappe.get_hooks("after_migrate", app_name=app): frappe.get_attr(fn)() - if not skip_search_index: - # Run this last as it updates the current session - print('Building search index for {}'.format(frappe.local.site)) - build_index_for_all_routes() - - frappe.db.commit() - - clear_notifications() - - frappe.publish_realtime("version-update") - frappe.flags.in_migrate = False - finally: - with open(touched_tables_file, 'w') as f: - json.dump(list(frappe.flags.touched_tables), f, sort_keys=True, indent=4) - frappe.flags.touched_tables.clear() + def required_services_running(self) -> bool: + """Returns True if all required services are running. Returns False and prints + instructions to stdout when required services are not available. + """ + service_status = check_connection(redis_services=["redis_cache"]) + are_services_running = all(service_status.values()) + + if not are_services_running: + for service in service_status: + if not service_status.get(service, True): + print(f"Service {service} is not running.") + print(BENCH_START_MESSAGE) + + return are_services_running + + def run(self, site: str): + """Run Migrate operation on site specified. This method initializes + and destroys connections to the site database. + """ + if not self.required_services_running(): + raise SystemExit(1) + + if site: + frappe.init(site=site) + frappe.connect() + + self.setUp() + try: + self.pre_schema_updates() + self.run_schema_updates() + finally: + self.post_schema_updates() + self.tearDown() + frappe.destroy() diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index a40f428969..0ee5a180e0 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -5,11 +5,7 @@ export default class GridRow { this.on_grid_fields_dict = {}; this.on_grid_fields = []; $.extend(this, opts); - if (this.doc && this.parent_df.options) { - frappe.meta.make_docfield_copy_for(this.parent_df.options, this.doc.name, this.docfields); - const docfields = frappe.meta.get_docfields(this.parent_df.options, this.doc.name); - this.docfields = docfields.length ? docfields : opts.docfields; - } + this.set_docfields(); this.columns = {}; this.columns_list = []; this.row_check_html = ''; @@ -41,6 +37,22 @@ export default class GridRow { this.set_data(); } } + + set_docfields(update=false) { + if (this.doc && this.parent_df.options) { + frappe.meta.make_docfield_copy_for(this.parent_df.options, this.doc.name, this.docfields); + const docfields = frappe.meta.get_docfields(this.parent_df.options, this.doc.name); + if (update) { + // to maintain references + this.docfields.forEach(df => { + Object.assign(df, docfields.find(d => d.fieldname === df.fieldname)); + }); + } else { + this.docfields = docfields; + } + } + } + set_data() { this.wrapper.data({ "doc": this.doc @@ -148,6 +160,11 @@ export default class GridRow { }, __('Move To'), 'Update'); } refresh() { + // update docfields for new record + if (this.frm && this.doc && this.doc.__islocal) { + this.set_docfields(true); + } + if(this.frm && this.doc) { this.doc = locals[this.doc.doctype][this.doc.name]; } diff --git a/frappe/public/js/frappe/form/layout.js b/frappe/public/js/frappe/form/layout.js index 1378718495..8c0f8953c5 100644 --- a/frappe/public/js/frappe/form/layout.js +++ b/frappe/public/js/frappe/form/layout.js @@ -554,19 +554,21 @@ frappe.ui.form.Layout = class Layout { let has_dep = false; - for (let fkey in this.fields_list) { - let f = this.fields_list[fkey]; - f.dependencies_clear = true; + const fields = this.fields_list.concat(this.tabs); + + for (let fkey in fields) { + let f = fields[fkey]; if (f.df.depends_on || f.df.mandatory_depends_on || f.df.read_only_depends_on) { has_dep = true; + break; } } if (!has_dep) return; // show / hide based on values - for (let i = this.fields_list.length - 1; i >= 0; i--) { - let f = this.fields_list[i]; + for (let i = fields.length - 1; i >= 0; i--) { + let f = fields[i]; f.guardian_has_value = true; if (f.df.depends_on) { // evaluate guardian diff --git a/frappe/public/js/frappe/form/tab.js b/frappe/public/js/frappe/form/tab.js index c8ca016398..0e740ce49c 100644 --- a/frappe/public/js/frappe/form/tab.js +++ b/frappe/public/js/frappe/form/tab.js @@ -40,7 +40,7 @@ export default class Tab { hide = true; } - hide && this.toggle(false); + this.toggle(!hide); } toggle(show) { diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index 64960e0b09..5cfc7c75d4 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -1483,7 +1483,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { return [ filter[1], "=", - JSON.stringify([filter[2], filter[3]]), + encodeURIComponent(JSON.stringify([filter[2], filter[3]])), ].join(""); }) .join("&"); diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js index 7ba0a0228f..c226a3a458 100644 --- a/frappe/public/js/frappe/views/reports/query_report.js +++ b/frappe/public/js/frappe/views/reports/query_report.js @@ -578,6 +578,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { args: { report_name: this.report_name, filters: filters, + report_settings: this.report_settings }, callback: resolve, always: () => this.page.btn_secondary.prop('disabled', false) @@ -834,7 +835,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { let data = this.data; let columns = this.columns.filter((col) => !col.hidden); - if (this.raw_data.add_total_row) { + if (this.raw_data.add_total_row && !this.report_settings.tree) { data = data.slice(); data.splice(-1, 1); } @@ -854,7 +855,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { treeView: this.tree_report, layout: 'fixed', cellHeight: 33, - showTotalRow: this.raw_data.add_total_row, + showTotalRow: this.raw_data.add_total_row && !this.report_settings.tree, direction: frappe.utils.is_rtl() ? 'rtl' : 'ltr', hooks: { columnTotal: frappe.utils.report_column_total diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index 408d644042..68605444f1 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -3,25 +3,37 @@ # imports - standard imports import gzip +import importlib import json import os import shlex import shutil import subprocess -from typing import List import unittest +from contextlib import contextmanager +from functools import wraps from glob import glob +from typing import List, Optional from unittest.case import skipIf +from unittest.mock import patch + +# imports - third party imports +import click +from click.testing import CliRunner, Result +from click import Command # imports - module imports import frappe +import frappe.commands.site +import frappe.commands.utils import frappe.recorder from frappe.installer import add_to_installed_apps, remove_app from frappe.utils import add_to_date, get_bench_path, get_bench_relative_path, now from frappe.utils.backups import fetch_latest_backups -# imports - third party imports -import click +_result: Optional[Result] = None +TEST_SITE = "commands-site-O4PN2QKA.test" # added random string tag to avoid collisions +CLI_CONTEXT = frappe._dict(sites=[TEST_SITE]) def clean(value) -> str: @@ -76,7 +88,61 @@ def exists_in_backup(doctypes: List, file: os.PathLike) -> bool: return len(missing_doctypes) == 0 +@contextmanager +def maintain_locals(): + pre_site = frappe.local.site + pre_flags = frappe.local.flags.copy() + pre_db = frappe.local.db + + try: + yield + finally: + post_site = getattr(frappe.local, "site", None) + if not post_site or post_site != pre_site: + frappe.init(site=pre_site) + frappe.local.db = pre_db + frappe.local.flags.update(pre_flags) + + +def pass_test_context(f): + @wraps(f) + def decorated_function(*args, **kwargs): + return f(CLI_CONTEXT, *args, **kwargs) + return decorated_function + + +@contextmanager +def cli(cmd: Command, args: Optional[List] = None): + with maintain_locals(): + global _result + + patch_ctx = patch("frappe.commands.pass_context", pass_test_context) + _module = cmd.callback.__module__ + _cmd = cmd.callback.__qualname__ + + __module = importlib.import_module(_module) + patch_ctx.start() + importlib.reload(__module) + click_cmd = getattr(__module, _cmd) + + try: + _result = CliRunner().invoke(click_cmd, args=args) + _result.command = str(cmd) + yield _result + finally: + patch_ctx.stop() + __module = importlib.import_module(_module) + importlib.reload(__module) + importlib.invalidate_caches() + + class BaseTestCommands(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.setup_test_site() + return super().setUpClass() + + @classmethod def execute(self, command, kwargs=None): site = {"site": frappe.local.site} cmd_input = None @@ -102,16 +168,48 @@ class BaseTestCommands(unittest.TestCase): self.stderr = clean(self._proc.stderr) self.returncode = clean(self._proc.returncode) + @classmethod + def setup_test_site(cls): + cmd_config = { + "test_site": TEST_SITE, + "admin_password": frappe.conf.admin_password, + "root_login": frappe.conf.root_login, + "root_password": frappe.conf.root_password, + "db_type": frappe.conf.db_type, + } + + if not os.path.exists( + os.path.join(TEST_SITE, "site_config.json") + ): + cls.execute( + "bench new-site {test_site} --admin-password {admin_password} --db-type" + " {db_type}", + cmd_config, + ) + def _formatMessage(self, msg, standardMsg): output = super(BaseTestCommands, self)._formatMessage(msg, standardMsg) + + if not hasattr(self, "command") and _result: + command = _result.command + stdout = _result.stdout_bytes.decode() if _result.stdout_bytes else None + stderr = _result.stderr_bytes.decode() if _result.stderr_bytes else None + returncode = _result.exit_code + else: + command = self.command + stdout = self.stdout + stderr = self.stderr + returncode = self.returncode + cmd_execution_summary = "\n".join([ "-" * 70, "Last Command Execution Summary:", - "Command: {}".format(self.command) if self.command else "", - "Standard Output: {}".format(self.stdout) if self.stdout else "", - "Standard Error: {}".format(self.stderr) if self.stderr else "", - "Return Code: {}".format(self.returncode) if self.returncode else "", + "Command: {}".format(command) if command else "", + "Standard Output: {}".format(stdout) if stdout else "", + "Standard Error: {}".format(stderr) if stderr else "", + "Return Code: {}".format(returncode) if returncode else "", ]).strip() + return "{}\n\n{}".format(output, cmd_execution_summary) @@ -135,6 +233,7 @@ class TestCommands(BaseTestCommands): self.assertEqual(self.returncode, 0) self.assertEqual(self.stdout[1:-1], frappe.bold(text="DocType")) + @unittest.skip def test_restore(self): # step 0: create a site to run the test on global_config = { @@ -143,35 +242,30 @@ class TestCommands(BaseTestCommands): "root_password": frappe.conf.root_password, "db_type": frappe.conf.db_type, } - site_data = {"another_site": f"{frappe.local.site}-restore.test", **global_config} + site_data = {"test_site": TEST_SITE, **global_config} for key, value in global_config.items(): if value: self.execute(f"bench set-config {key} {value} -g") - self.execute( - "bench new-site {another_site} --admin-password {admin_password} --db-type" - " {db_type}", - site_data, - ) # test 1: bench restore from full backup - self.execute("bench --site {another_site} backup --ignore-backup-conf", site_data) + self.execute("bench --site {test_site} backup --ignore-backup-conf", site_data) self.execute( - "bench --site {another_site} execute frappe.utils.backups.fetch_latest_backups", + "bench --site {test_site} execute frappe.utils.backups.fetch_latest_backups", site_data, ) site_data.update({"database": json.loads(self.stdout)["database"]}) - self.execute("bench --site {another_site} restore {database}", site_data) + self.execute("bench --site {test_site} restore {database}", site_data) # test 2: restore from partial backup - self.execute("bench --site {another_site} backup --exclude 'ToDo'", site_data) + self.execute("bench --site {test_site} backup --exclude 'ToDo'", site_data) site_data.update({"kw": "\"{'partial':True}\""}) self.execute( - "bench --site {another_site} execute" + "bench --site {test_site} execute" " frappe.utils.backups.fetch_latest_backups --kwargs {kw}", site_data, ) site_data.update({"database": json.loads(self.stdout)["database"]}) - self.execute("bench --site {another_site} restore {database}", site_data) + self.execute("bench --site {test_site} restore {database}", site_data) self.assertEqual(self.returncode, 1) def test_partial_restore(self): @@ -226,7 +320,8 @@ class TestCommands(BaseTestCommands): def test_list_apps(self): # test 1: sanity check for command self.execute("bench --site all list-apps") - self.assertEqual(self.returncode, 0) + self.assertIsNotNone(self.returncode) + self.assertIsInstance(self.stdout or self.stderr, str) # test 2: bare functionality for single site self.execute("bench --site {site} list-apps") @@ -242,14 +337,12 @@ class TestCommands(BaseTestCommands): self.assertSetEqual(list_apps, installed_apps) # test 3: parse json format - self.execute("bench --site all list-apps --format json") - self.assertEqual(self.returncode, 0) - self.assertIsInstance(json.loads(self.stdout), dict) - self.execute("bench --site {site} list-apps --format json") + self.assertEqual(self.returncode, 0) self.assertIsInstance(json.loads(self.stdout), dict) self.execute("bench --site {site} list-apps -f json") + self.assertEqual(self.returncode, 0) self.assertIsInstance(json.loads(self.stdout), dict) def test_show_config(self): @@ -358,7 +451,7 @@ class TestCommands(BaseTestCommands): ) def test_bench_drop_site_should_archive_site(self): # TODO: Make this test postgres compatible - site = 'test_site.localhost' + site = TEST_SITE self.execute( f"bench new-site {site} --force --verbose " @@ -585,3 +678,18 @@ class TestRemoveApp(unittest.TestCase): # nothing to assert, if this fails rest of the test suite will crumble. remove_app("frappe", dry_run=True, yes=True, no_backup=True) + + +class TestSiteMigration(BaseTestCommands): + def test_migrate_cli(self): + with cli(frappe.commands.site.migrate) as result: + self.assertTrue(TEST_SITE in result.stdout) + self.assertEqual(result.exit_code, 0) + self.assertEqual(result.exception, None) + + +class TestBenchBuild(BaseTestCommands): + def test_build_assets(self): + with cli(frappe.commands.utils.build) as result: + self.assertEqual(result.exit_code, 0) + self.assertEqual(result.exception, None) diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index 2ef53a3343..d23804bef4 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -653,7 +653,8 @@ def get_backup_path(): @frappe.whitelist() def get_backup_encryption_key(): - return frappe.local.conf.encryption_key + frappe.only_for("System Manager") + return frappe.conf.encryption_key class Backup: def __init__(self, file_path): diff --git a/frappe/utils/minify.py b/frappe/utils/minify.py deleted file mode 100644 index 634aa11038..0000000000 --- a/frappe/utils/minify.py +++ /dev/null @@ -1,212 +0,0 @@ - -# This code is original from jsmin by Douglas Crockford, it was translated to -# Python by Baruch Even. The original code had the following copyright and -# license. -# -# /* jsmin.c -# 2007-05-22 -# -# Copyright (c) 2002 Douglas Crockford (www.crockford.com) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of -# this software and associated documentation files (the "Software"), to deal in -# the Software without restriction, including without limitation the rights to -# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -# of the Software, and to permit persons to whom the Software is furnished to do -# so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# The Software shall be used for Good, not Evil. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# */ - -from io import StringIO - -def jsmin(js): - ins = StringIO(js) - outs = StringIO() - JavascriptMinify().minify(ins, outs) - str = outs.getvalue() - if len(str) > 0 and str[0] == '\n': - str = str[1:] - return str - -def isAlphanum(c): - """return true if the character is a letter, digit, underscore, - dollar sign, or non-ASCII character. - """ - return ((c >= 'a' and c <= 'z') or (c >= '0' and c <= '9') or - (c >= 'A' and c <= 'Z') or c == '_' or c == '$' or c == '\\' or (c is not None and ord(c) > 126)); - -class UnterminatedComment(Exception): - pass - -class UnterminatedStringLiteral(Exception): - pass - -class UnterminatedRegularExpression(Exception): - pass - -class JavascriptMinify(object): - - def _outA(self): - self.outstream.write(self.theA) - def _outB(self): - self.outstream.write(self.theB) - - def _get(self): - """return the next character from stdin. Watch out for lookahead. If - the character is a control character, translate it to a space or - linefeed. - """ - c = self.theLookahead - self.theLookahead = None - if c is None: - c = self.instream.read(1) - if c >= ' ' or c == '\n': - return c - if c == '': # EOF - return '\000' - if c == '\r': - return '\n' - return ' ' - - def _peek(self): - self.theLookahead = self._get() - return self.theLookahead - - def _next(self): - """get the next character, excluding comments. peek() is used to see - if an unescaped '/' is followed by a '/' or '*'. - """ - c = self._get() - if c == '/' and self.theA != '\\': - p = self._peek() - if p == '/': - c = self._get() - while c > '\n': - c = self._get() - return c - if p == '*': - c = self._get() - while 1: - c = self._get() - if c == '*': - if self._peek() == '/': - self._get() - return ' ' - if c == '\000': - raise UnterminatedComment() - - return c - - def _action(self, action): - """do something! What you do is determined by the argument: - 1 Output A. Copy B to A. Get the next B. - 2 Copy B to A. Get the next B. (Delete A). - 3 Get the next B. (Delete B). - action treats a string as a single character. Wow! - action recognizes a regular expression if it is preceded by ( or , or =. - """ - if action <= 1: - self._outA() - - if action <= 2: - self.theA = self.theB - if self.theA == "'" or self.theA == '"': - while 1: - self._outA() - self.theA = self._get() - if self.theA == self.theB: - break - if self.theA <= '\n': - raise UnterminatedStringLiteral() - if self.theA == '\\': - self._outA() - self.theA = self._get() - - - if action <= 3: - self.theB = self._next() - if self.theB == '/' and (self.theA == '(' or self.theA == ',' or - self.theA == '=' or self.theA == ':' or - self.theA == '[' or self.theA == '?' or - self.theA == '!' or self.theA == '&' or - self.theA == '|' or self.theA == ';' or - self.theA == '{' or self.theA == '}' or - self.theA == '\n'): - self._outA() - self._outB() - while 1: - self.theA = self._get() - if self.theA == '/': - break - elif self.theA == '\\': - self._outA() - self.theA = self._get() - elif self.theA <= '\n': - raise UnterminatedRegularExpression() - self._outA() - self.theB = self._next() - - - def _jsmin(self): - """Copy the input to the output, deleting the characters which are - insignificant to JavaScript. Comments will be removed. Tabs will be - replaced with spaces. Carriage returns will be replaced with linefeeds. - Most spaces and linefeeds will be removed. - """ - self.theA = '\n' - self._action(3) - - while self.theA != '\000': - if self.theA == ' ': - if isAlphanum(self.theB): - self._action(1) - else: - self._action(2) - elif self.theA == '\n': - if self.theB in ['{', '[', '(', '+', '-']: - self._action(1) - elif self.theB == ' ': - self._action(3) - else: - if isAlphanum(self.theB): - self._action(1) - else: - self._action(2) - else: - if self.theB == ' ': - if isAlphanum(self.theA): - self._action(1) - else: - self._action(3) - elif self.theB == '\n': - if self.theA in ['}', ']', ')', '+', '-', '"', '\'']: - self._action(1) - else: - if isAlphanum(self.theA): - self._action(1) - else: - self._action(3) - else: - self._action(1) - - def minify(self, instream, outstream): - self.instream = instream - self.outstream = outstream - self.theA = '\n' - self.theB = None - self.theLookahead = None - - self._jsmin() - self.instream.close()