diff --git a/frappe/__init__.py b/frappe/__init__.py index c6cbfead43..3558603454 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -143,6 +143,8 @@ lang = local("lang") # This if block is never executed when running the code. It is only used for # telling static code analyzer where to find dynamically defined attributes. if typing.TYPE_CHECKING: + from frappe.utils.redis_wrapper import RedisWrapper + from frappe.database.mariadb.database import MariaDBDatabase from frappe.database.postgres.database import PostgresDatabase from frappe.query_builder.builder import MariaDB, Postgres @@ -150,6 +152,7 @@ if typing.TYPE_CHECKING: db: typing.Union[MariaDBDatabase, PostgresDatabase] qb: typing.Union[MariaDB, Postgres] + # end: static analysis hack def init(site, sites_path=None, new_site=False): @@ -311,9 +314,8 @@ def destroy(): release_local(local) -# memcache redis_server = None -def cache(): +def cache() -> "RedisWrapper": """Returns redis connection.""" global redis_server if not redis_server: diff --git a/frappe/client.py b/frappe/client.py index e835e7fee7..7280c29ba4 100644 --- a/frappe/client.py +++ b/frappe/client.py @@ -99,7 +99,6 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren if not filters: filters = None - if frappe.get_meta(doctype).issingle: value = frappe.db.get_values_from_single(fields, filters, doctype, as_dict=as_dict, debug=debug) else: diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 41b607b192..e3379a43aa 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -623,6 +623,7 @@ def transform_database(context, table, engine, row_format, failfast): @click.command('run-tests') @click.option('--app', help="For App") @click.option('--doctype', help="For DocType") +@click.option('--case', help="Select particular TestCase") @click.option('--doctype-list-path', help="Path to .txt file for list of doctypes. Example erpnext/tests/server/agriculture.txt") @click.option('--test', multiple=True, help="Specific test") @click.option('--ui-tests', is_flag=True, default=False, help="Run UI Tests") @@ -636,7 +637,7 @@ def transform_database(context, table, engine, row_format, failfast): @pass_context def run_tests(context, app=None, module=None, doctype=None, test=(), profile=False, coverage=False, junit_xml_output=False, ui_tests = False, doctype_list_path=None, - skip_test_records=False, skip_before_tests=False, failfast=False): + skip_test_records=False, skip_before_tests=False, failfast=False, case=None): with CodeCoverage(coverage, app): import frappe.test_runner @@ -658,7 +659,7 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal ret = frappe.test_runner.main(app, module, doctype, context.verbose, tests=tests, force=context.force, profile=profile, junit_xml_output=junit_xml_output, - ui_tests=ui_tests, doctype_list_path=doctype_list_path, failfast=failfast) + ui_tests=ui_tests, doctype_list_path=doctype_list_path, failfast=failfast, case=case) if len(ret.failures) == 0 and len(ret.errors) == 0: ret = 0 diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index 2c1042e104..3ae5b87128 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -7,9 +7,8 @@ import frappe import os import unittest from frappe import _ -from frappe.core.doctype.file.file import get_attached_images, move_file, get_files_in_folder, unzip_file +from frappe.core.doctype.file.file import File, get_attached_images, move_file, get_files_in_folder, unzip_file from frappe.utils import get_files_path -# test_records = frappe.get_test_records('File') test_content1 = 'Hello' test_content2 = 'Hello World' @@ -24,8 +23,6 @@ def make_test_doc(): class TestSimpleFile(unittest.TestCase): - - def setUp(self): self.attached_to_doctype, self.attached_to_docname = make_test_doc() self.test_content = test_content1 @@ -38,21 +35,13 @@ class TestSimpleFile(unittest.TestCase): _file.save() self.saved_file_url = _file.file_url - def test_save(self): _file = frappe.get_doc("File", {"file_url": self.saved_file_url}) content = _file.get_content() self.assertEqual(content, self.test_content) - def tearDown(self): - # File gets deleted on rollback, so blank - pass - - class TestBase64File(unittest.TestCase): - - def setUp(self): self.attached_to_doctype, self.attached_to_docname = make_test_doc() self.test_content = base64.b64encode(test_content1.encode('utf-8')) @@ -66,18 +55,12 @@ class TestBase64File(unittest.TestCase): _file.save() self.saved_file_url = _file.file_url - def test_saved_content(self): _file = frappe.get_doc("File", {"file_url": self.saved_file_url}) content = _file.get_content() self.assertEqual(content, test_content1) - def tearDown(self): - # File gets deleted on rollback, so blank - pass - - class TestSameFileName(unittest.TestCase): def test_saved_content(self): self.attached_to_doctype, self.attached_to_docname = make_test_doc() @@ -130,8 +113,6 @@ class TestSameFileName(unittest.TestCase): class TestSameContent(unittest.TestCase): - - def setUp(self): self.attached_to_doctype1, self.attached_to_docname1 = make_test_doc() self.attached_to_doctype2, self.attached_to_docname2 = make_test_doc() @@ -186,10 +167,6 @@ class TestSameContent(unittest.TestCase): limit_property.delete() frappe.clear_cache(doctype='ToDo') - def tearDown(self): - # File gets deleted on rollback, so blank - pass - class TestFile(unittest.TestCase): def setUp(self): @@ -398,7 +375,7 @@ class TestFile(unittest.TestCase): def test_make_thumbnail(self): # test web image - test_file = frappe.get_doc({ + test_file: File = frappe.get_doc({ "doctype": "File", "file_name": 'logo', "file_url": frappe.utils.get_url('/_test/assets/image.jpg'), diff --git a/frappe/database/database.py b/frappe/database/database.py index ab0a2abc72..8a6b83c5d9 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -10,19 +10,20 @@ import re import string from contextlib import contextmanager from time import time -from typing import Dict, List, Union, Tuple +from typing import Dict, List, Tuple, Union + +from pypika.terms import Criterion, NullValue, PseudoColumn import frappe import frappe.defaults import frappe.model.meta from frappe import _ -from frappe.utils import now, getdate, cast, get_datetime from frappe.model.utils.link_count import flush_local_link_count from frappe.query_builder.functions import Count -from frappe.query_builder.functions import Min, Max, Avg, Sum -from frappe.query_builder.utils import Column +from frappe.query_builder.utils import DocType +from frappe.utils import cast, get_datetime, getdate, now, sbool + from .query import Query -from pypika.terms import Criterion, PseudoColumn class Database(object): @@ -557,7 +558,21 @@ class Database(object): def get_list(*args, **kwargs): return frappe.get_list(*args, **kwargs) - def get_single_value(self, doctype, fieldname, cache=False): + def set_single_value(self, doctype, fieldname, value, *args, **kwargs): + """Set field value of Single DocType. + + :param doctype: DocType of the single object + :param fieldname: `fieldname` of the property + :param value: `value` of the property + + Example: + + # Update the `deny_multiple_sessions` field in System Settings DocType. + company = frappe.db.set_single_value("System Settings", "deny_multiple_sessions", True) + """ + return self.set_value(doctype, doctype, fieldname, value, *args, **kwargs) + + def get_single_value(self, doctype, fieldname, cache=True): """Get property of Single DocType. Cache locally by default :param doctype: DocType of the single object whose value is requested @@ -572,7 +587,7 @@ class Database(object): if not doctype in self.value_cache: self.value_cache[doctype] = {} - if fieldname in self.value_cache[doctype]: + if cache and fieldname in self.value_cache[doctype]: return self.value_cache[doctype][fieldname] val = self.query.get_sql( @@ -679,53 +694,55 @@ class Database(object): :param debug: Print the query in the developer / js console. :param for_update: Will add a row-level lock to the value that is being set so that it can be released on commit. """ - if not modified: - modified = now() - if not modified_by: - modified_by = frappe.session.user + is_single_doctype = not (dn and dt != dn) + to_update = field if isinstance(field, dict) else {field: val} - to_update = {} if update_modified: - to_update = {"modified": modified, "modified_by": modified_by} + modified = modified or now() + modified_by = modified_by or frappe.session.user + to_update.update({"modified": modified, "modified_by": modified_by}) + + if is_single_doctype: + frappe.db.delete( + "Singles", + filters={"field": ("in", tuple(to_update)), "doctype": dt}, debug=debug + ) + + singles_data = ((dt, key, sbool(value)) for key, value in to_update.items()) + query = ( + frappe.qb.into("Singles") + .columns("doctype", "field", "value") + .insert(*singles_data) + ).run(debug=debug) + frappe.clear_document_cache(dt, dt) - if isinstance(field, dict): - to_update.update(field) else: - to_update.update({field: val}) + table = DocType(dt) - if dn and dt!=dn: - # with table - set_values = [] - for key in to_update: - set_values.append('`{0}`=%({0})s'.format(key)) + if for_update: + docnames = tuple( + self.get_values(dt, dn, "name", debug=debug, for_update=for_update, pluck=True) + ) or (NullValue(),) + query = frappe.qb.update(table).where(table.name.isin(docnames)) - for name in self.get_values(dt, dn, 'name', for_update=for_update, debug=debug): - values = dict(name=name[0]) - values.update(to_update) + for docname in docnames: + frappe.clear_document_cache(dt, docname) - self.sql("""update `tab{0}` - set {1} where name=%(name)s""".format(dt, ', '.join(set_values)), - values, debug=debug) + else: + query = self.query.build_conditions(table=dt, filters=dn, update=True) + # TODO: Fix this; doesn't work rn - gavin@frappe.io + # frappe.cache().hdel_keys(dt, "document_cache") + # Workaround: clear all document caches + frappe.cache().delete_value('document_cache') - frappe.clear_document_cache(dt, values['name']) - else: - # for singles - keys = list(to_update) - self.sql(''' - delete from `tabSingles` - where field in ({0}) and - doctype=%s'''.format(', '.join(['%s']*len(keys))), - list(keys) + [dt], debug=debug) - for key, value in to_update.items(): - self.sql('''insert into `tabSingles` (doctype, field, value) values (%s, %s, %s)''', - (dt, key, value), debug=debug) - - frappe.clear_document_cache(dt, dn) + for column, value in to_update.items(): + query = query.set(column, value) + + query.run(debug=debug) if dt in self.value_cache: del self.value_cache[dt] - @staticmethod def set(doc, field, val): """Set value in document. **Avoid**""" diff --git a/frappe/query_builder/__init__.py b/frappe/query_builder/__init__.py index bf7be84c51..5b58e70c4e 100644 --- a/frappe/query_builder/__init__.py +++ b/frappe/query_builder/__init__.py @@ -1,8 +1,21 @@ -from frappe.query_builder.terms import ParameterizedValueWrapper, ParameterizedFunction -import pypika +import pypika.terms +from pypika import * +from pypika import Field +from pypika.utils import ignore_copy + +from frappe.query_builder.terms import ParameterizedFunction, ParameterizedValueWrapper +from frappe.query_builder.utils import ( + Column, + DocType, + get_query_builder, + patch_query_aggregation, + patch_query_execute, +) pypika.terms.ValueWrapper = ParameterizedValueWrapper pypika.terms.Function = ParameterizedFunction -from pypika import * -from frappe.query_builder.utils import Column, DocType, get_query_builder, patch_query_execute, patch_query_aggregation +# * Overrides the field() method and replaces it with the a `PseudoColumn` 'field' for consistency +pypika.queries.Selectable.__getattr__ = ignore_copy(lambda table, x: Field(x, table=table)) +pypika.queries.Selectable.__getitem__ = ignore_copy(lambda table, x: Field(x, table=table)) +pypika.queries.Selectable.field = pypika.terms.PseudoColumn("field") diff --git a/frappe/query_builder/builder.py b/frappe/query_builder/builder.py index a65d50fdeb..d2fdeab324 100644 --- a/frappe/query_builder/builder.py +++ b/frappe/query_builder/builder.py @@ -1,8 +1,12 @@ from pypika import MySQLQuery, Order, PostgreSQLQuery, terms -from pypika.queries import Schema, Table -from frappe.utils import get_table_name +from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder +from pypika.queries import QueryBuilder, Schema, Table from pypika.terms import Function +from frappe.query_builder.terms import ParameterizedValueWrapper +from frappe.utils import get_table_name + + class Base: terms = terms desc = Order.desc @@ -19,13 +23,13 @@ class Base: return Table(table_name, *args, **kwargs) @classmethod - def into(cls, table, *args, **kwargs): + def into(cls, table, *args, **kwargs) -> QueryBuilder: if isinstance(table, str): table = cls.DocType(table) return super().into(table, *args, **kwargs) @classmethod - def update(cls, table, *args, **kwargs): + def update(cls, table, *args, **kwargs) -> QueryBuilder: if isinstance(table, str): table = cls.DocType(table) return super().update(table, *args, **kwargs) @@ -34,6 +38,10 @@ class Base: class MariaDB(Base, MySQLQuery): Field = terms.Field + @classmethod + def _builder(cls, *args, **kwargs) -> "MySQLQueryBuilder": + return super()._builder(*args, wrapper_cls=ParameterizedValueWrapper, **kwargs) + @classmethod def from_(cls, table, *args, **kwargs): if isinstance(table, str): @@ -53,6 +61,10 @@ class Postgres(Base, PostgreSQLQuery): # they are two different objects. The quick fix used here is to replace the # Field names in the "Field" function. + @classmethod + def _builder(cls, *args, **kwargs) -> "PostgreSQLQueryBuilder": + return super()._builder(*args, wrapper_cls=ParameterizedValueWrapper, **kwargs) + @classmethod def Field(cls, field_name, *args, **kwargs): if field_name in cls.field_translation: diff --git a/frappe/query_builder/terms.py b/frappe/query_builder/terms.py index 2032cd8497..bbc316ca93 100644 --- a/frappe/query_builder/terms.py +++ b/frappe/query_builder/terms.py @@ -1,33 +1,76 @@ +from datetime import timedelta from typing import Any, Dict, Optional +from frappe.utils.data import format_timedelta from pypika.terms import Function, ValueWrapper from pypika.utils import format_alias_sql -class NamedParameterWrapper(): - def __init__(self, parameters: Dict[str, Any]): - self.parameters = parameters +class NamedParameterWrapper: + """Utility class to hold parameter values and keys""" - def update_parameters(self, param_key: Any, param_value: Any, **kwargs): + def __init__(self) -> None: + self.parameters = {} + + def get_sql(self, param_value: Any, **kwargs) -> str: + """returns SQL for a parameter, while adding the real value in a dict + + Args: + param_value (Any): Value of the parameter + + Returns: + str: parameter used in the SQL query + """ + param_key = f"%(param{len(self.parameters) + 1})s" self.parameters[param_key[2:-2]] = param_value + return param_key - def get_sql(self, **kwargs): - return f'%(param{len(self.parameters) + 1})s' + def get_parameters(self) -> Dict[str, Any]: + """get dict with parameters and values + + Returns: + Dict[str, Any]: parameter dict + """ + return self.parameters class ParameterizedValueWrapper(ValueWrapper): - def get_sql(self, quote_char: Optional[str] = None, secondary_quote_char: str = "'", param_wrapper= None, **kwargs: Any) -> str: - if param_wrapper is None: - sql = self.get_value_sql(quote_char=quote_char, secondary_quote_char=secondary_quote_char, **kwargs) - return format_alias_sql(sql, self.alias, quote_char=quote_char, **kwargs) + """ + Class to monkey patch ValueWrapper + + Adds functionality to parameterize queries when a `param wrapper` is passed in get_sql() + """ + + def get_sql( + self, + quote_char: Optional[str] = None, + secondary_quote_char: str = "'", + param_wrapper: Optional[NamedParameterWrapper] = None, + **kwargs: Any, + ) -> str: + if param_wrapper and isinstance(self.value, str): + # add quotes if it's a string value + value_sql = self.get_value_sql(quote_char=quote_char, **kwargs) + sql = param_wrapper.get_sql(param_value=value_sql, **kwargs) else: - value_sql = self.get_value_sql(quote_char=quote_char, **kwargs) if not isinstance(self.value,int) else self.value - param_sql = param_wrapper.get_sql(**kwargs) - param_wrapper.update_parameters(param_key=param_sql, param_value=value_sql, **kwargs) - return format_alias_sql(param_sql, self.alias, quote_char=quote_char, **kwargs) + # * BUG: pypika doesen't parse timedeltas + if isinstance(self.value, timedelta): + self.value = format_timedelta(self.value) + sql = self.get_value_sql( + quote_char=quote_char, + secondary_quote_char=secondary_quote_char, + **kwargs, + ) + return format_alias_sql(sql, self.alias, quote_char=quote_char, **kwargs) class ParameterizedFunction(Function): + """ + Class to monkey patch pypika.terms.Functions + + Only to pass `param_wrapper` in `get_function_sql`. + """ + def get_sql(self, **kwargs: Any) -> str: with_alias = kwargs.pop("with_alias", False) with_namespace = kwargs.pop("with_namespace", False) @@ -35,15 +78,24 @@ class ParameterizedFunction(Function): dialect = kwargs.pop("dialect", None) param_wrapper = kwargs.pop("param_wrapper", None) - function_sql = self.get_function_sql(with_namespace=with_namespace, quote_char=quote_char, param_wrapper=param_wrapper, dialect=dialect) + function_sql = self.get_function_sql( + with_namespace=with_namespace, + quote_char=quote_char, + param_wrapper=param_wrapper, + dialect=dialect, + ) if self.schema is not None: function_sql = "{schema}.{function}".format( - schema=self.schema.get_sql(quote_char=quote_char, dialect=dialect, **kwargs), + schema=self.schema.get_sql( + quote_char=quote_char, dialect=dialect, **kwargs + ), function=function_sql, ) if with_alias: - return format_alias_sql(function_sql, self.alias, quote_char=quote_char, **kwargs) + return format_alias_sql( + function_sql, self.alias, quote_char=quote_char, **kwargs + ) return function_sql diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py index 2767e90242..cbd6147e01 100644 --- a/frappe/query_builder/utils.py +++ b/frappe/query_builder/utils.py @@ -1,16 +1,16 @@ from enum import Enum -from typing import Any, Callable, Dict, Union, get_type_hints from importlib import import_module +from typing import Any, Callable, Dict, Union, get_type_hints from pypika import Query from pypika.queries import Column +from pypika.terms import PseudoColumn import frappe +from frappe.query_builder.terms import NamedParameterWrapper from .builder import MariaDB, Postgres -from pypika.terms import PseudoColumn -from frappe.query_builder.terms import NamedParameterWrapper class db_type_is(Enum): MARIADB = "mariadb" @@ -59,11 +59,11 @@ def patch_query_execute(): return frappe.db.sql(query, params, *args, **kwargs) # nosemgrep def prepare_query(query): - params = {} - query = query.get_sql(param_wrapper = NamedParameterWrapper(params)) + param_collector = NamedParameterWrapper() + query = query.get_sql(param_wrapper=param_collector) if frappe.flags.in_safe_exec and not query.lower().strip().startswith("select"): raise frappe.PermissionError('Only SELECT SQL allowed in scripting') - return query, params + return query, param_collector.get_parameters() query_class = get_attr(str(frappe.qb).split("'")[1]) builder_class = get_type_hints(query_class._builder).get('return') @@ -78,7 +78,7 @@ def patch_query_execute(): def patch_query_aggregation(): """Patch aggregation functions to frappe.qb """ - from frappe.query_builder.functions import _max, _min, _avg, _sum + from frappe.query_builder.functions import _avg, _max, _min, _sum frappe.qb.max = _max frappe.qb.min = _min diff --git a/frappe/test_runner.py b/frappe/test_runner.py index 1839f15ae8..05f1ce1cd7 100644 --- a/frappe/test_runner.py +++ b/frappe/test_runner.py @@ -30,7 +30,7 @@ def xmlrunner_wrapper(output): def main(app=None, module=None, doctype=None, verbose=False, tests=(), force=False, profile=False, junit_xml_output=None, ui_tests=False, - doctype_list_path=None, skip_test_records=False, failfast=False): + doctype_list_path=None, skip_test_records=False, failfast=False, case=None): global unittest_runner if doctype_list_path: @@ -76,7 +76,7 @@ def main(app=None, module=None, doctype=None, verbose=False, tests=(), if doctype: ret = run_tests_for_doctype(doctype, verbose, tests, force, profile, failfast=failfast, junit_xml_output=junit_xml_output) elif module: - ret = run_tests_for_module(module, verbose, tests, profile, failfast=failfast, junit_xml_output=junit_xml_output) + ret = run_tests_for_module(module, verbose, tests, profile, failfast=failfast, junit_xml_output=junit_xml_output, case=case) else: ret = run_all_tests(app, verbose, profile, ui_tests, failfast=failfast, junit_xml_output=junit_xml_output) @@ -182,16 +182,16 @@ def run_tests_for_doctype(doctypes, verbose=False, tests=(), force=False, profil return _run_unittest(modules, verbose=verbose, tests=tests, profile=profile, failfast=failfast, junit_xml_output=junit_xml_output) -def run_tests_for_module(module, verbose=False, tests=(), profile=False, failfast=False, junit_xml_output=False): +def run_tests_for_module(module, verbose=False, tests=(), profile=False, failfast=False, junit_xml_output=False, case=None): module = importlib.import_module(module) if hasattr(module, "test_dependencies"): for doctype in module.test_dependencies: make_test_records(doctype, verbose=verbose) frappe.db.commit() - return _run_unittest(module, verbose=verbose, tests=tests, profile=profile, failfast=failfast, junit_xml_output=junit_xml_output) + return _run_unittest(module, verbose=verbose, tests=tests, profile=profile, failfast=failfast, junit_xml_output=junit_xml_output, case=case) -def _run_unittest(modules, verbose=False, tests=(), profile=False, failfast=False, junit_xml_output=False): +def _run_unittest(modules, verbose=False, tests=(), profile=False, failfast=False, junit_xml_output=False, case=None): frappe.db.begin() test_suite = unittest.TestSuite() @@ -200,7 +200,10 @@ def _run_unittest(modules, verbose=False, tests=(), profile=False, failfast=Fals modules = [modules] for module in modules: - module_test_cases = unittest.TestLoader().loadTestsFromModule(module) + if case: + module_test_cases = unittest.TestLoader().loadTestsFromTestCase(getattr(module, case)) + else: + module_test_cases = unittest.TestLoader().loadTestsFromModule(module) if tests: for each in module_test_cases: for test_case in each.__dict__["_tests"]: @@ -337,7 +340,7 @@ def make_test_records_for_doctype(doctype, verbose=0, force=False): elif hasattr(test_module, "test_records"): if doctype in frappe.local.test_objects: frappe.local.test_objects[doctype] += make_test_objects(doctype, test_module.test_records, verbose, force) - else: + else: frappe.local.test_objects[doctype] = make_test_objects(doctype, test_module.test_records, verbose, force) else: diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index 885fe6ac26..6e96849b35 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -1,21 +1,21 @@ -# -*- coding: utf-8 -*- - -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import datetime +import inspect import unittest from random import choice -import datetime +from unittest.mock import patch import frappe from frappe.custom.doctype.custom_field.custom_field import create_custom_field -from frappe.utils import random_string -from frappe.utils.testutils import clear_custom_fields -from frappe.query_builder import Field from frappe.database import savepoint - -from .test_query_builder import run_only_if, db_type_is +from frappe.database.database import Database +from frappe.query_builder import Field from frappe.query_builder.functions import Concat_ws +from frappe.tests.test_query_builder import db_type_is, run_only_if +from frappe.utils import add_days, now, random_string +from frappe.utils.testutils import clear_custom_fields class TestDB(unittest.TestCase): @@ -84,20 +84,6 @@ class TestDB(unittest.TestCase): ), ) - def test_set_value(self): - todo1 = frappe.get_doc(dict(doctype='ToDo', description = 'test_set_value 1')).insert() - todo2 = frappe.get_doc(dict(doctype='ToDo', description = 'test_set_value 2')).insert() - - frappe.db.set_value('ToDo', todo1.name, 'description', 'test_set_value change 1') - self.assertEqual(frappe.db.get_value('ToDo', todo1.name, 'description'), 'test_set_value change 1') - - # multiple set-value - frappe.db.set_value('ToDo', dict(description=('like', '%test_set_value%')), - 'description', 'change 2') - - self.assertEqual(frappe.db.get_value('ToDo', todo1.name, 'description'), 'change 2') - self.assertEqual(frappe.db.get_value('ToDo', todo2.name, 'description'), 'change 2') - def test_escape(self): frappe.db.escape("香港濟生堂製藥有限公司 - IT".encode("utf-8")) @@ -246,7 +232,6 @@ class TestDB(unittest.TestCase): frappe.delete_doc(test_doctype, doc) clear_custom_fields(test_doctype) - def test_savepoints(self): frappe.db.rollback() save_point = "todonope" @@ -365,6 +350,143 @@ class TestDDLCommandsMaria(unittest.TestCase): self.assertEquals(len(indexs_in_table), 2) +class TestDBSetValue(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.todo1 = frappe.get_doc(doctype="ToDo", description="test_set_value 1").insert() + cls.todo2 = frappe.get_doc(doctype="ToDo", description="test_set_value 2").insert() + + def test_update_single_doctype_field(self): + value = frappe.db.get_single_value("System Settings", "deny_multiple_sessions") + changed_value = not value + + frappe.db.set_value("System Settings", "System Settings", "deny_multiple_sessions", changed_value) + current_value = frappe.db.get_single_value("System Settings", "deny_multiple_sessions") + self.assertEqual(current_value, changed_value) + + changed_value = not current_value + frappe.db.set_value("System Settings", None, "deny_multiple_sessions", changed_value) + current_value = frappe.db.get_single_value("System Settings", "deny_multiple_sessions") + self.assertEqual(current_value, changed_value) + + changed_value = not current_value + frappe.db.set_single_value("System Settings", "deny_multiple_sessions", changed_value) + current_value = frappe.db.get_single_value("System Settings", "deny_multiple_sessions") + self.assertEqual(current_value, changed_value) + + def test_update_single_row_single_column(self): + frappe.db.set_value("ToDo", self.todo1.name, "description", "test_set_value change 1") + updated_value = frappe.db.get_value("ToDo", self.todo1.name, "description") + self.assertEqual(updated_value, "test_set_value change 1") + + def test_update_single_row_multiple_columns(self): + description, status = "Upated by test_update_single_row_multiple_columns", "Closed" + + frappe.db.set_value("ToDo", self.todo1.name, { + "description": description, + "status": status, + }, update_modified=False) + + updated_desciption, updated_status = frappe.db.get_value("ToDo", + filters={"name": self.todo1.name}, + fieldname=["description", "status"] + ) + + self.assertEqual(description, updated_desciption) + self.assertEqual(status, updated_status) + + def test_update_multiple_rows_single_column(self): + frappe.db.set_value("ToDo", {"description": ("like", "%test_set_value%")}, "description", "change 2") + + self.assertEqual(frappe.db.get_value("ToDo", self.todo1.name, "description"), "change 2") + self.assertEqual(frappe.db.get_value("ToDo", self.todo2.name, "description"), "change 2") + + def test_update_multiple_rows_multiple_columns(self): + todos_to_update = frappe.get_all("ToDo", filters={ + "description": ("like", "%test_set_value%"), + "status": ("!=", "Closed") + }, pluck="name") + + frappe.db.set_value("ToDo", { + "description": ("like", "%test_set_value%"), + "status": ("!=", "Closed") + }, { + "status": "Closed", + "priority": "High" + }) + + test_result = frappe.get_all("ToDo", filters={"name": ("in", todos_to_update)}, fields=["status", "priority"]) + + self.assertTrue(all(x for x in test_result if x["status"] == "Closed")) + self.assertTrue(all(x for x in test_result if x["priority"] == "High")) + + def test_update_modified_options(self): + self.todo2.reload() + + todo = self.todo2 + updated_description = f"{todo.description} - by `test_update_modified_options`" + custom_modified = datetime.datetime.fromisoformat(add_days(now(), 10)) + custom_modified_by = "user_that_doesnt_exist@example.com" + + frappe.db.set_value("ToDo", todo.name, "description", updated_description, update_modified=False) + self.assertEqual(updated_description, frappe.db.get_value("ToDo", todo.name, "description")) + self.assertEqual(todo.modified, frappe.db.get_value("ToDo", todo.name, "modified")) + + frappe.db.set_value("ToDo", todo.name, "description", "test_set_value change 1", modified=custom_modified, modified_by=custom_modified_by) + self.assertTupleEqual( + (custom_modified, custom_modified_by), + frappe.db.get_value("ToDo", todo.name, ["modified", "modified_by"]) + ) + + def test_for_update(self): + self.todo1.reload() + + with patch.object(Database, "sql") as sql_called: + frappe.db.set_value( + self.todo1.doctype, + self.todo1.name, + "description", + f"{self.todo1.description}-edit by `test_for_update`" + ) + first_query = sql_called.call_args_list[0].args[0] + second_query = sql_called.call_args_list[1].args[0] + + self.assertTrue(sql_called.call_count == 2) + self.assertTrue("FOR UPDATE" in first_query) + if frappe.conf.db_type == "postgres": + from frappe.database.postgres.database import modify_query + self.assertTrue(modify_query("UPDATE `tabToDo` SET") in second_query) + if frappe.conf.db_type == "mariadb": + self.assertTrue("UPDATE `tabToDo` SET" in second_query) + + def test_cleared_cache(self): + self.todo2.reload() + + with patch.object(frappe, "clear_document_cache") as clear_cache: + frappe.db.set_value( + self.todo2.doctype, + self.todo2.name, + "description", + f"{self.todo2.description}-edit by `test_cleared_cache`" + ) + clear_cache.assert_called() + + def test_update_alias(self): + args = (self.todo1.doctype, self.todo1.name, "description", "Updated by `test_update_alias`") + kwargs = {"for_update": False, "modified": None, "modified_by": None, "update_modified": True, "debug": False} + + self.assertTrue("return self.set_value(" in inspect.getsource(frappe.db.update)) + + with patch.object(Database, "set_value") as set_value: + frappe.db.update(*args, **kwargs) + set_value.assert_called_once() + set_value.assert_called_with(*args, **kwargs) + + @classmethod + def tearDownClass(cls): + frappe.db.rollback() + + @run_only_if(db_type_is.POSTGRES) class TestDDLCommandsPost(unittest.TestCase): test_table_name = "TestNotes" diff --git a/frappe/tests/test_query_builder.py b/frappe/tests/test_query_builder.py index d2242cc6f7..bb7a883b5b 100644 --- a/frappe/tests/test_query_builder.py +++ b/frappe/tests/test_query_builder.py @@ -5,7 +5,7 @@ import frappe from frappe.query_builder.custom import ConstantColumn from frappe.query_builder.functions import Coalesce, GroupConcat, Match from frappe.query_builder.utils import db_type_is - +from frappe.query_builder import Case def run_only_if(dbtype: db_type_is) -> Callable: return unittest.skipIf( @@ -25,8 +25,14 @@ class TestCustomFunctionsMariaDB(unittest.TestCase): ) def test_constant_column(self): - query = frappe.qb.from_("DocType").select("name", ConstantColumn("John").as_("User")) - self.assertEqual(query.get_sql(), "SELECT `name`,'John' `User` FROM `tabDocType`") + query = frappe.qb.from_("DocType").select( + "name", ConstantColumn("John").as_("User") + ) + self.assertEqual( + query.get_sql(), "SELECT `name`,'John' `User` FROM `tabDocType`" + ) + + @run_only_if(db_type_is.POSTGRES) class TestCustomFunctionsPostgres(unittest.TestCase): def test_concat(self): @@ -39,8 +45,13 @@ class TestCustomFunctionsPostgres(unittest.TestCase): ) def test_constant_column(self): - query = frappe.qb.from_("DocType").select("name", ConstantColumn("John").as_("User")) - self.assertEqual(query.get_sql(), 'SELECT "name",\'John\' "User" FROM "tabDocType"') + query = frappe.qb.from_("DocType").select( + "name", ConstantColumn("John").as_("User") + ) + self.assertEqual( + query.get_sql(), 'SELECT "name",\'John\' "User" FROM "tabDocType"' + ) + class TestBuilderBase(object): def test_adding_tabs(self): @@ -55,23 +66,68 @@ class TestBuilderBase(object): self.assertIsInstance(query.run, Callable) self.assertIsInstance(data, list) - def test_walk(self): - DocType = frappe.qb.DocType('DocType') + +class TestParameterization(unittest.TestCase): + def test_where_conditions(self): + DocType = frappe.qb.DocType("DocType") query = ( frappe.qb.from_(DocType) .select(DocType.name) - .where((DocType.owner == "Administrator' --") - & (Coalesce(DocType.search_fields == "subject")) + .where((DocType.owner == "Administrator' --")) + ) + self.assertTrue("walk" in dir(query)) + query, params = query.walk() + + self.assertIn("%(param1)s", query) + self.assertIn("param1", params) + self.assertEqual(params["param1"], "Administrator' --") + + def test_set_cnoditions(self): + DocType = frappe.qb.DocType("DocType") + query = frappe.qb.update(DocType).set(DocType.value, "some_value") + + self.assertTrue("walk" in dir(query)) + query, params = query.walk() + + self.assertIn("%(param1)s", query) + self.assertIn("param1", params) + self.assertEqual(params["param1"], "some_value") + + def test_where_conditions_functions(self): + DocType = frappe.qb.DocType("DocType") + query = ( + frappe.qb.from_(DocType) + .select(DocType.name) + .where(Coalesce(DocType.search_fields == "subject")) + ) + + self.assertTrue("walk" in dir(query)) + query, params = query.walk() + + self.assertIn("%(param1)s", query) + self.assertIn("param1", params) + self.assertEqual(params["param1"], "subject") + + def test_case(self): + DocType = frappe.qb.DocType("DocType") + query = ( + frappe.qb.from_(DocType) + .select( + Case() + .when(DocType.search_fields == "value", "other_value") + .when(Coalesce(DocType.search_fields == "subject_in_function"), "true_value") ) ) + self.assertTrue("walk" in dir(query)) query, params = query.walk() self.assertIn("%(param1)s", query) - self.assertIn("%(param2)s", query) - self.assertIn("param1",params) - self.assertEqual(params["param1"],"Administrator' --") - self.assertEqual(params["param2"],"subject") + self.assertIn("param1", params) + self.assertEqual(params["param1"], "value") + self.assertEqual(params["param2"], "other_value") + self.assertEqual(params["param3"], "subject_in_function") + self.assertEqual(params["param4"], "true_value") @run_only_if(db_type_is.MARIADB) @@ -84,6 +140,7 @@ class TestBuilderMaria(unittest.TestCase, TestBuilderBase): "SELECT * FROM `__Auth`", frappe.qb.from_("__Auth").select("*").get_sql() ) + @run_only_if(db_type_is.POSTGRES) class TestBuilderPostgres(unittest.TestCase, TestBuilderBase): def test_adding_tabs_in_from(self): diff --git a/frappe/tests/test_utils.py b/frappe/tests/test_utils.py index 8ac6218b5e..507722f9e9 100644 --- a/frappe/tests/test_utils.py +++ b/frappe/tests/test_utils.py @@ -1,22 +1,28 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import unittest -import frappe -from frappe.utils import evaluate_filters, money_in_words, scrub_urls, get_url -from frappe.utils import validate_url, validate_email_address -from frappe.utils import ceil, floor -from frappe.utils.data import cast, validate_python_code -from frappe.utils.diff import get_version_diff, version_query, _get_value_from_version - -from PIL import Image -from frappe.utils.image import strip_exif_data, optimize_image import io +import json +import unittest +from datetime import date, datetime, time, timedelta +from decimal import Decimal +from enum import Enum from mimetypes import guess_type -from datetime import datetime, timedelta, date - from unittest.mock import patch +import pytz +from PIL import Image + +import frappe +from frappe.utils import ceil, evaluate_filters, floor, format_timedelta +from frappe.utils import get_url, money_in_words, parse_timedelta, scrub_urls +from frappe.utils import validate_email_address, validate_url +from frappe.utils.data import cast, validate_python_code +from frappe.utils.diff import _get_value_from_version, get_version_diff, version_query +from frappe.utils.image import optimize_image, strip_exif_data +from frappe.utils.response import json_handler + + class TestFilters(unittest.TestCase): def test_simple_dict(self): self.assertTrue(evaluate_filters({'doctype': 'User', 'status': 'Open'}, {'status': 'Open'})) @@ -273,9 +279,7 @@ class TestPythonExpressions(unittest.TestCase): for expr in invalid_expressions: self.assertRaises(frappe.ValidationError, validate_python_code, expr) - class TestDiffUtils(unittest.TestCase): - @classmethod def setUpClass(cls): cls.doc = frappe.get_doc(doctype="Client Script", dt="Client Script") @@ -330,8 +334,59 @@ class TestDateUtils(unittest.TestCase): self.assertEqual(frappe.utils.get_last_day_of_week("2020-12-28"), frappe.utils.getdate("2021-01-02")) -class TestXlsxUtils(unittest.TestCase): +class TestResponse(unittest.TestCase): + def test_json_handler(self): + class TEST(Enum): + ABC = "!@)@)!" + BCE = "ENJD" + + GOOD_OBJECT = { + "time_types": [ + date(year=2020, month=12, day=2), + datetime(year=2020, month=12, day=2, hour=23, minute=23, second=23, microsecond=23, tzinfo=pytz.utc), + time(hour=23, minute=23, second=23, microsecond=23, tzinfo=pytz.utc), + timedelta(days=10, hours=12, minutes=120, seconds=10), + ], + "float": [ + Decimal(29.21), + ], + "doc": [ + frappe.get_doc("System Settings"), + ], + "iter": [ + {1, 2, 3}, + (1, 2, 3), + "abcdef", + ], + "string": "abcdef" + } + + BAD_OBJECT = {"Enum": TEST} + + processed_object = json.loads(json.dumps(GOOD_OBJECT, default=json_handler)) + + self.assertTrue(all([isinstance(x, str) for x in processed_object["time_types"]])) + self.assertTrue(all([isinstance(x, float) for x in processed_object["float"]])) + self.assertTrue(all([isinstance(x, (list, str)) for x in processed_object["iter"]])) + self.assertIsInstance(processed_object["string"], str) + with self.assertRaises(TypeError): + json.dumps(BAD_OBJECT, default=json_handler) + +class TestTimeDeltaUtils(unittest.TestCase): + def test_format_timedelta(self): + self.assertEqual(format_timedelta(timedelta(seconds=0)), "0:00:00") + self.assertEqual(format_timedelta(timedelta(hours=10)), "10:00:00") + self.assertEqual(format_timedelta(timedelta(hours=100)), "100:00:00") + self.assertEqual(format_timedelta(timedelta(seconds=100, microseconds=129)), "0:01:40.000129") + self.assertEqual(format_timedelta(timedelta(seconds=100, microseconds=12212199129)), "3:25:12.199129") + + def test_parse_timedelta(self): + self.assertEqual(parse_timedelta("0:0:0"), timedelta(seconds=0)) + self.assertEqual(parse_timedelta("10:0:0"), timedelta(hours=10)) + self.assertEqual(parse_timedelta("7 days, 0:32:18.192221"), timedelta(days=7, seconds=1938, microseconds=192221)) + self.assertEqual(parse_timedelta("7 days, 0:32:18"), timedelta(days=7, seconds=1938)) +class TestXlsxUtils(unittest.TestCase): def test_unescape(self): from frappe.utils.xlsxutils import handle_html diff --git a/frappe/tests/test_website.py b/frappe/tests/test_website.py index e40a07c0ec..4d53fb7ba6 100644 --- a/frappe/tests/test_website.py +++ b/frappe/tests/test_website.py @@ -20,6 +20,7 @@ class TestWebsite(unittest.TestCase): doctype='User', email='test-user-for-home-page@example.com', first_name='test')).insert(ignore_if_duplicate=True) + user.reload() role = frappe.get_doc(dict( doctype = 'Role', diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index 9deec0a77c..6b93a81b6e 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import functools @@ -56,8 +56,8 @@ def get_email_address(user=None): def get_formatted_email(user, mail=None): """get Email Address of user formatted as: `John Doe `""" fullname = get_fullname(user) - method = get_hook_method('get_sender_details') + if method: sender_name, mail = method() # if method exists but sender_name is "" diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 891a55deda..23a271c77c 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1,17 +1,22 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -from typing import Optional -import frappe -import operator -import json import base64 -import re, datetime, math, time +import datetime +import json +import math +import operator +import re +import time from code import compile_command +from enum import Enum +from typing import Any, Dict, List, Optional, Tuple, Union from urllib.parse import quote, urljoin -from frappe.desk.utils import slug + from click import secho -from enum import Enum + +import frappe +from frappe.desk.utils import slug DATE_FORMAT = "%Y-%m-%d" TIME_FORMAT = "%H:%M:%S.%f" @@ -99,11 +104,17 @@ def get_timedelta(time: Optional[str] = None) -> Optional[datetime.timedelta]: datetime.timedelta: Timedelta object equivalent of the passed `time` string """ from dateutil import parser + from dateutil.parser import ParserError time = time or "0:0:0" try: - t = parser.parse(time) + try: + t = parser.parse(time) + except ParserError as e: + if "day" in e.args[1]: + from frappe.utils import parse_timedelta + return parse_timedelta(time) return datetime.timedelta( hours=t.hour, minutes=t.minute, seconds=t.second, microseconds=t.microsecond ) @@ -201,7 +212,7 @@ def get_time_zone(): return frappe.cache().get_value("time_zone", _get_time_zone) def convert_utc_to_timezone(utc_timestamp, time_zone): - from pytz import timezone, UnknownTimeZoneError + from pytz import UnknownTimeZoneError, timezone utcnow = timezone('UTC').localize(utc_timestamp) try: return utcnow.astimezone(timezone(time_zone)) @@ -327,7 +338,7 @@ def get_time(time_str): return time_str else: if isinstance(time_str, datetime.timedelta): - time_str = str(time_str) + return format_timedelta(time_str) return parser.parse(time_str).time() def get_datetime_str(datetime_obj): @@ -610,7 +621,7 @@ def cast(fieldtype, value=None): value = flt(value) elif fieldtype in ("Int", "Check"): - value = cint(value) + value = cint(sbool(value)) elif fieldtype in ("Data", "Text", "Small Text", "Long Text", "Text Editor", "Select", "Link", "Dynamic Link"): @@ -726,7 +737,7 @@ def ceil(s): def cstr(s, encoding='utf-8'): return frappe.as_unicode(s, encoding) -def sbool(x): +def sbool(x: str) -> Union[bool, Any]: """Converts str object to Boolean if possible. Example: "true" becomes True @@ -737,12 +748,15 @@ def sbool(x): x (str): String to be converted to Bool Returns: - object: Returns Boolean or type(x) + object: Returns Boolean or x """ - from distutils.util import strtobool - try: - return bool(strtobool(x)) + val = x.lower() + if val in ('true', '1'): + return True + elif val in ('false', '0'): + return False + return x except Exception: return x @@ -917,13 +931,13 @@ number_format_info = { "#.########": (".", "", 8) } -def get_number_format_info(format): +def get_number_format_info(format: str) -> Tuple[str, str, int]: return number_format_info.get(format) or (".", ",", 2) # # convert currency to words # -def money_in_words(number, main_currency = None, fraction_currency=None): +def money_in_words(number: str, main_currency: Optional[str] = None, fraction_currency: Optional[str] = None): """ Returns string in words with currency and fraction currency. """ @@ -1009,9 +1023,11 @@ def is_image(filepath): def get_thumbnail_base64_for_image(src): from os.path import exists as file_exists + from PIL import Image + + from frappe import cache, safe_decode from frappe.core.doctype.file.file import get_local_image - from frappe import safe_decode, cache if not src: frappe.throw('Invalid source for image: {0}'.format(src)) @@ -1302,7 +1318,7 @@ operator_map = { "None": lambda a, b: (not a) and True or False } -def evaluate_filters(doc, filters): +def evaluate_filters(doc, filters: Union[Dict, List, Tuple]): '''Returns true if doc matches filters''' if isinstance(filters, dict): for key, value in filters.items(): @@ -1319,7 +1335,7 @@ def evaluate_filters(doc, filters): return True -def compare(val1, condition, val2, fieldtype=None): +def compare(val1: Any, condition: str, val2: Any, fieldtype: Optional[str] = None): ret = False if fieldtype: val2 = cast(fieldtype, val2) @@ -1328,7 +1344,7 @@ def compare(val1, condition, val2, fieldtype=None): return ret -def get_filter(doctype, f, filters_config=None): +def get_filter(doctype: str, f: Union[Dict, List, Tuple], filters_config=None) -> "frappe._dict": """Returns a _dict like { @@ -1415,8 +1431,10 @@ def make_filter_dict(filters): return _filter def sanitize_column(column_name): - from frappe import _ import sqlparse + + from frappe import _ + regex = re.compile("^.*[,'();].*") column_name = sqlparse.format(column_name, strip_comments=True, keyword_case="lower") blacklisted_keywords = ['select', 'create', 'insert', 'delete', 'drop', 'update', 'case', 'and', 'or'] @@ -1492,9 +1510,10 @@ def strip(val, chars=None): return (val or "").replace("\ufeff", "").replace("\u200b", "").strip(chars) def to_markdown(html): - from html2text import html2text from html.parser import HTMLParser + from html2text import html2text + text = None try: text = html2text(html or '') @@ -1504,7 +1523,8 @@ def to_markdown(html): return text def md_to_html(markdown_text): - from markdown2 import markdown as _markdown, MarkdownError + from markdown2 import MarkdownError + from markdown2 import markdown as _markdown extras = { 'fenced-code-blocks': None, @@ -1529,14 +1549,14 @@ def md_to_html(markdown_text): def markdown(markdown_text): return md_to_html(markdown_text) -def is_subset(list_a, list_b): +def is_subset(list_a: List, list_b: List) -> bool: '''Returns whether list_a is a subset of list_b''' return len(list(set(list_a) & set(list_b))) == len(list_a) -def generate_hash(*args, **kwargs): +def generate_hash(*args, **kwargs) -> str: return frappe.generate_hash(*args, **kwargs) -def guess_date_format(date_string): +def guess_date_format(date_string: str) -> str: DATE_FORMATS = [ r"%d/%b/%y", r"%d-%m-%Y", @@ -1611,13 +1631,13 @@ def guess_date_format(date_string): if date_format and time_format: return (date_format + ' ' + time_format).strip() -def validate_json_string(string): +def validate_json_string(string: str) -> None: try: json.loads(string) except (TypeError, ValueError): raise frappe.ValidationError -def get_user_info_for_avatar(user_id): +def get_user_info_for_avatar(user_id: str) -> Dict: user_info = { "email": user_id, "image": "", @@ -1664,3 +1684,30 @@ class UnicodeWithAttrs(str): def __init__(self, text): self.toc_html = text.toc_html self.metadata = text.metadata + + +def format_timedelta(o: datetime.timedelta) -> str: + # mariadb allows a wide diff range - https://mariadb.com/kb/en/time/ + # but frappe doesnt - i think via babel : only allows 0..23 range for hour + total_seconds = o.total_seconds() + hours, remainder = divmod(total_seconds, 3600) + minutes, seconds = divmod(remainder, 60) + rounded_seconds = round(seconds, 6) + int_seconds = int(seconds) + + if rounded_seconds == int_seconds: + seconds = int_seconds + else: + seconds = rounded_seconds + + return "{:01}:{:02}:{:02}".format(int(hours), int(minutes), seconds) + + +def parse_timedelta(s: str) -> datetime.timedelta: + # ref: https://stackoverflow.com/a/21074460/10309266 + if 'day' in s: + m = re.match(r"(?P[-\d]+) day[s]*, (?P\d+):(?P\d+):(?P\d[\.\d+]*)", s) + else: + m = re.match(r"(?P\d+):(?P\d+):(?P\d[\.\d+]*)", s) + + return datetime.timedelta(**{key: float(val) for key, val in m.groupdict().items()}) diff --git a/frappe/utils/formatters.py b/frappe/utils/formatters.py index 9436dea2c2..9916853caf 100644 --- a/frappe/utils/formatters.py +++ b/frappe/utils/formatters.py @@ -3,9 +3,11 @@ import frappe import datetime -from frappe.utils import formatdate, fmt_money, flt, cstr, cint, format_datetime, format_time, format_duration +from frappe.utils import formatdate, fmt_money, flt, cstr, cint, format_datetime, format_time, format_duration, format_timedelta from frappe.model.meta import get_field_currency, get_field_precision import re +from dateutil.parser import ParserError + def format_value(value, df=None, doc=None, currency=None, translated=False, format=None): '''Format value based on given fieldtype, document reference, currency reference. @@ -47,7 +49,10 @@ def format_value(value, df=None, doc=None, currency=None, translated=False, form return format_datetime(value) elif df.get("fieldtype")=="Time": - return format_time(value) + try: + return format_time(value) + except ParserError: + return format_timedelta(value) elif value==0 and df.get("fieldtype") in ("Int", "Float", "Currency", "Percent") and df.get("print_hide_if_no_value"): # this is required to show 0 as blank in table columns diff --git a/frappe/utils/response.py b/frappe/utils/response.py index f6ad91dbd2..a852c584c6 100644 --- a/frappe/utils/response.py +++ b/frappe/utils/response.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import json @@ -16,7 +16,7 @@ from werkzeug.local import LocalProxy from werkzeug.wsgi import wrap_file from werkzeug.wrappers import Response from werkzeug.exceptions import NotFound, Forbidden -from frappe.utils import cint +from frappe.utils import cint, format_timedelta from urllib.parse import quote from frappe.core.doctype.access_log.access_log import make_access_log @@ -122,12 +122,14 @@ def make_logs(response = None): def json_handler(obj): """serialize non-serializable data for json""" - # serialize date - import collections.abc + from collections.abc import Iterable - if isinstance(obj, (datetime.date, datetime.timedelta, datetime.datetime, datetime.time)): + if isinstance(obj, (datetime.date, datetime.datetime, datetime.time)): return str(obj) + elif isinstance(obj, datetime.timedelta): + return format_timedelta(obj) + elif isinstance(obj, decimal.Decimal): return float(obj) @@ -138,7 +140,7 @@ def json_handler(obj): doc = obj.as_dict(no_nulls=True) return doc - elif isinstance(obj, collections.abc.Iterable): + elif isinstance(obj, Iterable): return list(obj) elif type(obj)==type or isinstance(obj, Exception): diff --git a/frappe/website/utils.py b/frappe/website/utils.py index fa95fbfbb0..9cf8804dd1 100644 --- a/frappe/website/utils.py +++ b/frappe/website/utils.py @@ -88,7 +88,7 @@ def get_home_page(): # portal default if not home_page: - home_page = frappe.db.get_value("Portal Settings", None, "default_portal_home") + home_page = frappe.db.get_single_value("Portal Settings", "default_portal_home") # by hooks if not home_page: @@ -96,7 +96,7 @@ def get_home_page(): # global if not home_page: - home_page = frappe.db.get_value("Website Settings", None, "home_page") + home_page = frappe.db.get_single_value("Website Settings", "home_page") if not home_page: home_page = "login" if frappe.session.user == 'Guest' else "me"