From d7789ab6ff8ee19905a34148a0022ee9b0b320b8 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Tue, 5 Jul 2022 12:37:16 +0530 Subject: [PATCH] fix(doc)!: Always cast datetime, date and time fields (#15891) ### BREAKING CHANGE #### Datetime, Date and Time fields will always be cast to respective objects in `setattr`, this will ensure uniformity while accessing the values, no more `getdate`, `get_datetime`, `to_timedelta` wrapper. - While importing data, the framework does check for `set_only_once`. - In normal case scenarios, this will work flawlessly since most date fields might not be set_only_once. - But in Subscription, the date field is set to `set_only_once` and in `after_insert`, `document.save` is called, and while doing so, `set_only_once` is checked [here](https://github.com/frappe/frappe/blob/1944a547f96dec83474390b5bce9b52371d989fd/frappe/model/document.py#L566). -This works fine if the data imported is in the correct format. - If the date's data is not in the correct format, the framework throws an error. - for eg `06-02-2022 00:00:00 != 06-02-2022` - fixes [Issue/#15370](https://github.com/frappe/frappe/issues/15370) > no-docs --- .../assignment_rule/test_assignment_rule.py | 16 +- .../doctype/auto_repeat/test_auto_repeat.py | 8 +- frappe/database/mariadb/database.py | 21 ++- frappe/database/postgres/database.py | 14 +- .../oauth_bearer_token/oauth_bearer_token.py | 4 +- frappe/model/base_document.py | 137 +++++++++++++++--- frappe/model/create_new.py | 6 - frappe/model/document.py | 121 +++++++++------- frappe/model/meta.py | 26 ++++ frappe/tests/test_document.py | 70 ++++++++- frappe/utils/data.py | 70 +++++---- .../test_personal_data_deletion_request.py | 8 +- 12 files changed, 368 insertions(+), 133 deletions(-) diff --git a/frappe/automation/doctype/assignment_rule/test_assignment_rule.py b/frappe/automation/doctype/assignment_rule/test_assignment_rule.py index 2ab2e4d263..12e7769ec0 100644 --- a/frappe/automation/doctype/assignment_rule/test_assignment_rule.py +++ b/frappe/automation/doctype/assignment_rule/test_assignment_rule.py @@ -257,13 +257,17 @@ class TestAutoAssign(unittest.TestCase): )[0] note1_todo_doc = frappe.get_doc("ToDo", note1_todo.name) - self.assertEqual(frappe.utils.get_date_str(note1_todo_doc.date), expiry_date) + self.assertEqual( + frappe.utils.get_date_str(note1_todo_doc.date), frappe.utils.get_date_str(expiry_date) + ) # due date should be updated if the reference doc's date is updated. note1.expiry_date = frappe.utils.add_days(expiry_date, 2) note1.save() note1_todo_doc.reload() - self.assertEqual(frappe.utils.get_date_str(note1_todo_doc.date), note1.expiry_date) + self.assertEqual( + frappe.utils.get_date_str(note1_todo_doc.date), frappe.utils.get_date_str(note1.expiry_date) + ) # saving one note's expiry should not update other note todo's due date note2_todo = frappe.get_all( @@ -271,8 +275,12 @@ class TestAutoAssign(unittest.TestCase): filters=dict(reference_type="Note", reference_name=note2.name, status="Open"), fields=["name", "date"], )[0] - self.assertNotEqual(frappe.utils.get_date_str(note2_todo.date), note1.expiry_date) - self.assertEqual(frappe.utils.get_date_str(note2_todo.date), expiry_date) + self.assertNotEqual( + frappe.utils.get_date_str(note2_todo.date), frappe.utils.get_date_str(note1.expiry_date) + ) + self.assertEqual( + frappe.utils.get_date_str(note2_todo.date), frappe.utils.get_date_str(expiry_date) + ) assignment_rule.delete() diff --git a/frappe/automation/doctype/auto_repeat/test_auto_repeat.py b/frappe/automation/doctype/auto_repeat/test_auto_repeat.py index ee0addf847..68fb3a1ffb 100644 --- a/frappe/automation/doctype/auto_repeat/test_auto_repeat.py +++ b/frappe/automation/doctype/auto_repeat/test_auto_repeat.py @@ -9,7 +9,7 @@ from frappe.automation.doctype.auto_repeat.auto_repeat import ( week_map, ) from frappe.custom.doctype.custom_field.custom_field import create_custom_field -from frappe.utils import add_days, add_months, getdate, today +from frappe.utils import add_days, add_months, cast, getdate, today def add_custom_fields(): @@ -39,7 +39,7 @@ class TestAutoRepeat(unittest.TestCase): ).insert() doc = make_auto_repeat(reference_document=todo.name) - self.assertEqual(doc.next_schedule_date, today()) + self.assertEqual(doc.next_schedule_date, cast("Date", today())) data = get_auto_repeat_entries(getdate(today())) create_repeated_entries(data) frappe.db.commit() @@ -67,7 +67,7 @@ class TestAutoRepeat(unittest.TestCase): start_date=add_days(today(), -7), ) - self.assertEqual(doc.next_schedule_date, today()) + self.assertEqual(doc.next_schedule_date, cast("Date", today())) data = get_auto_repeat_entries(getdate(today())) create_repeated_entries(data) frappe.db.commit() @@ -99,7 +99,7 @@ class TestAutoRepeat(unittest.TestCase): days=days, ) - self.assertEqual(doc.next_schedule_date, today()) + self.assertEqual(doc.next_schedule_date, cast("Date", today())) data = get_auto_repeat_entries(getdate(today())) create_repeated_entries(data) frappe.db.commit() diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 047039b0df..ed3ab80dc1 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -1,3 +1,5 @@ +import datetime + import pymysql from pymysql.constants import ER, FIELD_TYPE from pymysql.converters import conversions, escape_string @@ -5,7 +7,15 @@ from pymysql.converters import conversions, escape_string import frappe from frappe.database.database import Database from frappe.database.mariadb.schema import MariaDBTable -from frappe.utils import UnicodeWithAttrs, cstr, get_datetime, get_table_name +from frappe.utils import ( + UnicodeWithAttrs, + cstr, + get_date_str, + get_datetime, + get_datetime_str, + get_table_name, + get_time_str, +) class MariaDBDatabase(Database): @@ -123,6 +133,15 @@ class MariaDBDatabase(Database): # pymysql expects unicode argument to escape_string with Python 3 s = frappe.as_unicode(escape_string(frappe.as_unicode(s)), "utf-8").replace("`", "\\`") + if isinstance(s, datetime.datetime): + s = get_datetime_str(s) + + elif isinstance(s, datetime.date): + s = get_date_str(s) + + elif isinstance(s, datetime.timedelta): + s = get_time_str(s) + # NOTE separating % escape, because % escape should only be done when using LIKE operator # or when you use python format string to generate query that already has a %s # for example: sql("select name from `tabUser` where name=%s and {0}".format(conditions), something) diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 2553ebaa26..c0ba23ee01 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -1,3 +1,4 @@ +import datetime import re import psycopg2 @@ -8,7 +9,7 @@ from psycopg2.extensions import ISOLATION_LEVEL_REPEATABLE_READ import frappe from frappe.database.database import Database from frappe.database.postgres.schema import PostgresTable -from frappe.utils import cstr, get_table_name +from frappe.utils import cstr, get_date_str, get_datetime_str, get_table_name, get_time_str # cast decimals as floats DEC2FLOAT = psycopg2.extensions.new_type( @@ -95,6 +96,15 @@ class PostgresDatabase(Database): if isinstance(s, bytes): s = s.decode("utf-8") + if isinstance(s, datetime.datetime): + s = get_datetime_str(s) + + elif isinstance(s, datetime.date): + s = get_date_str(s) + + elif isinstance(s, datetime.timedelta): + s = get_time_str(s) + # MariaDB's driver treats None as an empty string # So Postgres should do the same @@ -392,7 +402,7 @@ def modify_query(query): # only find int (with/without signs), ignore decimals (with/without signs), ignore hashes (which start with numbers), # drop .0 from decimals and add quotes around them # - # >>> query = "c='abcd' , a >= 45, b = -45.0, c = 40, d=4500.0, e=3500.53, f=40psdfsd, g=9092094312, h=12.00023" + # >>> query = "c='abcd' , a >= 45, b = -45.0, c = 40, d=4500.0, e=3500.53, f=40psdfsd, g=9092094312, h=12.00023" # >>> re.sub(r"([=><]+)\s*([+-]?\d+)(\.0)?(?![a-zA-Z\.\d])", r"\1 '\2'", query) # "c='abcd' , a >= '45', b = '-45', c = '40', d= '4500', e=3500.53, f=40psdfsd, g= '9092094312', h=12.00023 diff --git a/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.py b/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.py index 2a17035571..bdae25e473 100644 --- a/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.py +++ b/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.py @@ -8,6 +8,4 @@ from frappe.model.document import Document class OAuthBearerToken(Document): def validate(self): if not self.expiration_time: - self.expiration_time = frappe.utils.datetime.datetime.strptime( - self.creation, "%Y-%m-%d %H:%M:%S.%f" - ) + frappe.utils.datetime.timedelta(seconds=self.expires_in) + self.expiration_time = self.creation + frappe.utils.datetime.timedelta(seconds=self.expires_in) diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index d3e7656d6d..238f5ab39e 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -2,6 +2,7 @@ # License: MIT. See LICENSE import datetime import json +from typing import Any import frappe from frappe import _, _dict @@ -17,7 +18,18 @@ from frappe.model.docstatus import DocStatus from frappe.model.naming import set_new_name from frappe.model.utils.link_count import notify_link_count from frappe.modules import load_doctype_module -from frappe.utils import cast_fieldtype, cint, cstr, flt, now, sanitize_html, strip_html +from frappe.utils import ( + DEFAULT_DATETIME_SHORTCUTS, + cast, + cast_fieldtype, + cint, + cstr, + flt, + now, + now_datetime, + sanitize_html, + strip_html, +) from frappe.utils.html_utils import unescape_html max_positive_value = {"smallint": 2**15, "int": 2**31, "bigint": 2**63} @@ -94,19 +106,40 @@ class BaseDocument: "_table_fieldnames", "_reserved_keywords", "dont_update_if_missing", + "_datetime_fieldnames", + "_date_fieldnames", + "_time_fieldnames", } - def __init__(self, d): - if d.get("doctype"): - self.doctype = d["doctype"] + def __init__(self, doc): + if doc.get("doctype"): + self.doctype = doc["doctype"] self._table_fieldnames = ( - d["_table_fieldnames"] # from cache - if "_table_fieldnames" in d + doc["_table_fieldnames"] + if "_table_fieldnames" in doc else {df.fieldname for df in self._get_table_fields()} ) - self.update(d) + self._datetime_fieldnames = ( + doc["_datetime_fieldnames"] + if "_datetime_fieldnames" in doc + else {df.fieldname for df in self._get_datetime_fields()} + ) + + self._date_fieldnames = ( + doc["_date_fieldnames"] + if "_date_fieldnames" in doc + else {df.fieldname for df in self._get_date_fields()} + ) + + self._time_fieldnames = ( + doc["_time_fieldnames"] + if "_time_fieldnames" in doc + else {df.fieldname for df in self._get_time_fields()} + ) + + self.update(doc) self.dont_update_if_missing = [] if hasattr(self, "__setup__"): @@ -138,14 +171,30 @@ class BaseDocument: state.pop("_meta", None) + def __setattr__(self, __name: str, __value: Any) -> None: + """ + Cast datetime/date/time/string value to datetime, date and time respectively for Datetime, Date and Time fields only. + """ + if __value in DEFAULT_DATETIME_SHORTCUTS: + __value = now_datetime() + + if hasattr(self, "_datetime_fieldnames") and __value and __name in self._datetime_fieldnames: + __value = cast("Datetime", __value) + elif hasattr(self, "_date_fieldnames") and __value and __name in self._date_fieldnames: + __value = cast("Date", __value) + elif hasattr(self, "_time_fieldnames") and __value and __name in self._time_fieldnames: + __value = cast("Time", __value) + + self.__dict__[__name] = __value + def update(self, d): """Update multiple fields of a doctype using a dictionary of key-value pairs. Example: - doc.update({ + doc.update({ "user": "admin", "balance": 42000 - }) + }) """ # set name first, as it is used a reference in child document @@ -200,6 +249,9 @@ class BaseDocument: if key in self._reserved_keywords: return + if value in DEFAULT_DATETIME_SHORTCUTS: + value = now_datetime() + if not as_value and key in self._table_fieldnames: self.__dict__[key] = [] @@ -208,6 +260,12 @@ class BaseDocument: self.extend(key, value) return + elif value and key in self._datetime_fieldnames: + value = cast("Datetime", value) + elif value and key in self._date_fieldnames: + value = cast("Date", value) + elif value and key in self._time_fieldnames: + value = cast("Time", value) self.__dict__[key] = value @@ -219,11 +277,11 @@ class BaseDocument: """Append an item to a child table. Example: - doc.append("childtable", { + doc.append("childtable", { "child_table_field": "value", "child_table_int_field": 0, ... - }) + }) """ if value is None: value = {} @@ -295,6 +353,45 @@ class BaseDocument: return self.meta.get_table_fields() + def _get_datetime_fields(self): + """ + To get datetime fields during Document init + """ + if ( + self.doctype == "DocType" + or self.doctype in DOCTYPES_FOR_DOCTYPE + or getattr(self, "parentfield", None) + ): + return () + + return self.meta.get_datetime_fields(with_standard_datetime_fields=True) + + def _get_date_fields(self): + """ + To get date fields during Document init + """ + if ( + self.doctype == "DocType" + or self.doctype in DOCTYPES_FOR_DOCTYPE + or getattr(self, "parentfield", None) + ): + return () + + return self.meta.get_date_fields() + + def _get_time_fields(self): + """ + To get time fields during Document init + """ + if ( + self.doctype == "DocType" + or self.doctype in DOCTYPES_FOR_DOCTYPE + or getattr(self, "parentfield", None) + ): + return () + + return self.meta.get_time_fields() + def get_valid_dict( self, sanitize=True, convert_dates_to_str=False, ignore_nulls=False, ignore_virtual=False ) -> dict: @@ -468,9 +565,7 @@ class BaseDocument: """INSERT the document (with valid columns) in the database. args: - ignore_if_duplicate: ignore primary key collision - at database level (postgres) - in python (mariadb) + ignore_if_duplicate: ignore primary key collision at database level (postgres) in python (mariadb) """ if not self.name: # name will be set by document class in most cases @@ -593,13 +688,13 @@ class BaseDocument: This function returns the `column_name` associated with the `key_name` passed Args: - key_name (str): The name of the database index. + key_name (str): The name of the database index. Raises: - IndexError: If the key is not found in the table. + IndexError: If the key is not found in the table. Returns: - str: The column name associated with the key. + str: The column name associated with the key. """ return frappe.db.sql( f""" @@ -620,10 +715,10 @@ class BaseDocument: """Returns the associated label for fieldname Args: - fieldname (str): The fieldname in the DocType to use to pull the label. + fieldname (str): The fieldname in the DocType to use to pull the label. Returns: - str: The label associated with the fieldname, if found, otherwise `None`. + str: The label associated with the fieldname, if found, otherwise `None`. """ df = self.meta.get_field(fieldname) if df: @@ -1110,9 +1205,9 @@ class BaseDocument: Print Hide can be set via the Print Format Builder or in the controller as a list of hidden fields. Example - class MyDoc(Document): + class MyDoc(Document): def __setup__(self): - self.print_hide = ["field1", "field2"] + self.print_hide = ["field1", "field2"] :param fieldname: Fieldname to be checked if hidden. """ diff --git a/frappe/model/create_new.py b/frappe/model/create_new.py index 51810c3e18..1bf8239c0d 100644 --- a/frappe/model/create_new.py +++ b/frappe/model/create_new.py @@ -145,12 +145,6 @@ def set_dynamic_default_values(doc, parent_doc, parentfield): if default_value is not None and not doc.get(df.fieldname): doc[df.fieldname] = default_value - elif df.fieldtype == "Datetime" and df.default.lower() == "now": - doc[df.fieldname] = now_datetime() - - if df.fieldtype == "Time": - doc[df.fieldname] = nowtime() - if parent_doc: doc["parent"] = parent_doc.name doc["parenttype"] = parent_doc.doctype diff --git a/frappe/model/document.py b/frappe/model/document.py index 9b781b1999..899a29c509 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -16,7 +16,7 @@ from frappe.model.base_document import BaseDocument, get_controller from frappe.model.docstatus import DocStatus from frappe.model.naming import set_new_name, validate_name from frappe.model.workflow import set_workflow_state_on_action, validate_workflow -from frappe.utils import cstr, date_diff, file_lock, flt, get_datetime_str, now +from frappe.utils import cast, cstr, date_diff, file_lock, flt, get_datetime_str, now from frappe.utils.data import get_absolute_url from frappe.utils.global_search import update_global_search @@ -30,23 +30,23 @@ def get_doc(*args, **kwargs): There are multiple ways to call `get_doc` - # will fetch the latest user object (with child table) from the database - user = get_doc("User", "test@example.com") + 1. will fetch the latest user object (with child table) from the database + user = get_doc("User", "test@example.com") - # create a new object - user = get_doc({ + 2. create a new object + user = get_doc({ "doctype":"User" "email_id": "test@example.com", "roles: [ - {"role": "System Manager"} + {"role": "System Manager"} ] - }) + }) - # create new object with keyword arguments - user = get_doc(doctype='User', email_id='test@example.com') + 3. create new object with keyword arguments + user = get_doc(doctype='User', email_id='test@example.com') - # select a document for update - user = get_doc("User", "test@example.com", for_update=True) + 4. select a document for update + user = get_doc("User", "test@example.com", for_update=True) """ if args: if isinstance(args[0], BaseDocument): @@ -145,7 +145,8 @@ class Document(BaseDocument): ) if not d: frappe.throw( - _("{0} {1} not found").format(_(self.doctype), self.name), frappe.DoesNotExistError + _("{0} {1} not found").format(_(self.doctype), self.name), + frappe.DoesNotExistError, ) super().__init__(d) @@ -399,22 +400,29 @@ class Document(BaseDocument): if rows: # select rows that do not match the ones in the document - deleted_rows = frappe.db.sql( - """select name from `tab{}` where parent=%s - and parenttype=%s and parentfield=%s - and name not in ({})""".format( - df.options, ",".join(["%s"] * len(rows)) - ), - [self.name, self.doctype, fieldname] + rows, + Table = frappe.qb.DocType(df.options) + + deleted_rows = ( + frappe.qb.from_(Table) + .select(Table.name) + .where( + (Table.parent == self.name) + & (Table.parenttype == self.doctype) + & (Table.parentfield == fieldname) + & (Table.name.notin(rows)) + ) + .run(pluck=True) ) - if len(deleted_rows) > 0: + + if deleted_rows: # delete rows that do not match the ones in the document - frappe.db.delete(df.options, {"name": ("in", tuple(row[0] for row in deleted_rows))}) + frappe.db.delete(df.options, {"name": ("in", deleted_rows)}) else: # no rows found, delete all rows frappe.db.delete( - df.options, {"parent": self.name, "parenttype": self.doctype, "parentfield": fieldname} + df.options, + {"parent": self.name, "parenttype": self.doctype, "parentfield": fieldname}, ) def get_doc_before_save(self): @@ -478,11 +486,10 @@ class Document(BaseDocument): frappe.db.delete("Singles", {"doctype": self.doctype}) for field, value in d.items(): if field != "doctype": - frappe.db.sql( - """insert into `tabSingles` (doctype, field, value) - values (%s, %s, %s)""", - (self.doctype, field, value), - ) + Singles = frappe.qb.DocType("Singles") + frappe.qb.into(Singles).columns(Singles.doctype, Singles.field, Singles.value).insert( + self.doctype, field, value + ).run() if self.doctype in frappe.db.value_cache: del frappe.db.value_cache[self.doctype] @@ -540,6 +547,7 @@ class Document(BaseDocument): d._extract_images_from_text_editor() d._sanitize_content() d._save_passwords() + if self.is_new(): # don't set fields like _assign, _comments for new doc for fieldname in optional_fields: @@ -656,12 +664,12 @@ class Document(BaseDocument): has_access_to = self.get_permlevel_access("read") for df in self.meta.fields: - if df.permlevel and not df.permlevel in has_access_to: + if df.permlevel and df.permlevel not in has_access_to: self.set(df.fieldname, None) for table_field in self.meta.get_table_fields(): for df in frappe.get_meta(table_field.options).fields or []: - if df.permlevel and not df.permlevel in has_access_to: + if df.permlevel and df.permlevel not in has_access_to: for child in self.get(table_field.fieldname) or []: child.set(df.fieldname, None) @@ -745,32 +753,35 @@ class Document(BaseDocument): self._action = "save" if not self.get("__islocal") and not self.meta.get("is_virtual"): if self.meta.issingle: - modified = frappe.db.sql( - """select value from tabSingles - where doctype=%s and field='modified' for update""", - self.doctype, + Singles = frappe.qb.DocType("Singles") + modified = ( + frappe.qb.from_(Singles) + .select(Singles.value) + .where((Singles.doctype == self.doctype) & (Singles.field == "modified")) + .for_update() + .run(pluck=True) ) - modified = modified and modified[0][0] - if modified and modified != cstr(self._original_modified): + modified = modified and modified[0] + + if modified and cast("Datetime", modified) != cast("Datetime", self._original_modified): conflict = True else: - tmp = frappe.db.sql( - """select modified, docstatus from `tab{}` - where name = %s for update""".format( - self.doctype - ), - self.name, - as_dict=True, + Table = frappe.qb.DocType(self.doctype) + tmp = ( + frappe.qb.from_(Table) + .select(Table.modified, Table.docstatus) + .where(Table.name == self.name) + .for_update() + .run(as_dict=True) ) if not tmp: frappe.throw(_("Record does not exist")) - else: - tmp = tmp[0] - modified = cstr(tmp.modified) + tmp = tmp[0] + modified = tmp.modified - if modified and modified != cstr(self._original_modified): + if modified and cast("Datetime", modified) != cast("Datetime", self._original_modified): conflict = True self.check_docstatus_transition(tmp.docstatus) @@ -778,7 +789,7 @@ class Document(BaseDocument): if conflict: frappe.msgprint( _("Error: Document has been modified after you have opened it") - + (f" ({modified}, {self.modified}). ") + + (f" ({cstr(modified)}, {cstr(self.modified)}). ") + _("Please refresh to get the latest document."), raise_exception=frappe.TimestampMismatchError, ) @@ -962,7 +973,7 @@ class Document(BaseDocument): return def _evaluate_alert(alert): - if not alert.name in self.flags.notifications_executed: + if alert.name not in self.flags.notifications_executed: evaluate_alert(self, alert.name, alert.event) self.flags.notifications_executed.append(alert.name) @@ -1112,7 +1123,11 @@ class Document(BaseDocument): """Clear _seen property and set current user as seen""" if getattr(self.meta, "track_seen", False): frappe.db.set_value( - self.doctype, self.name, "_seen", json.dumps([frappe.session.user]), update_modified=False + self.doctype, + self.name, + "_seen", + json.dumps([frappe.session.user]), + update_modified=False, ) def notify_update(self): @@ -1306,7 +1321,8 @@ class Document(BaseDocument): if not (isinstance(self.get(parentfield), list) and len(self.get(parentfield)) > 0): label = self.meta.get_label(parentfield) frappe.throw( - _("Table {0} cannot be empty").format(label), raise_exception or frappe.EmptyTableError + _("Table {0} cannot be empty").format(label), + raise_exception or frappe.EmptyTableError, ) def round_floats_in(self, doc, fieldnames=None): @@ -1321,7 +1337,10 @@ class Document(BaseDocument): ) for fieldname in fieldnames: - doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.get("parentfield")))) + doc.set( + fieldname, + flt(doc.get(fieldname), self.precision(fieldname, doc.get("parentfield"))), + ) def get_url(self): """Returns Desk URL for this document.""" diff --git a/frappe/model/meta.py b/frappe/model/meta.py index 014dd5faf1..9fce85d068 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -17,6 +17,7 @@ Example: import json import os from datetime import datetime +from typing import Dict, List import click @@ -99,6 +100,10 @@ class Meta(Document): frappe._dict(fieldname="creation", fieldtype="Datetime"), frappe._dict(fieldname="owner", fieldtype="Data"), ] + standard_datetime_fields = [ + frappe._dict(fieldname="creation", fieldtype="Datetime"), + frappe._dict(fieldname="modified", fieldtype="Datetime"), + ] def __init__(self, doctype): self._fields = {} @@ -654,6 +659,27 @@ class Meta(Document): def is_nested_set(self): return self.has_field("lft") and self.has_field("rgt") + def get_date_fields(self) -> list[dict[str, str]]: + if not hasattr(self, "_date_fields"): + self._date_fields = self.get("fields", {"fieldtype": "Date"}) + + return self._date_fields + + def get_time_fields(self) -> list[dict[str, str]]: + if not hasattr(self, "_time_fields"): + self._time_fields = self.get("fields", {"fieldtype": "Time"}) + + return self._time_fields + + def get_datetime_fields(self, with_standard_datetime_fields=False) -> list[dict[str, str]]: + if not hasattr(self, "_datetime_fields"): + self._datetime_fields = self.get("fields", {"fieldtype": "Datetime"}, default=[]) + + if with_standard_datetime_fields: + self._datetime_fields += self.standard_datetime_fields + + return self._datetime_fields + ####### diff --git a/frappe/tests/test_document.py b/frappe/tests/test_document.py index 454ca76983..ecfbf3dccf 100644 --- a/frappe/tests/test_document.py +++ b/frappe/tests/test_document.py @@ -2,14 +2,14 @@ # License: MIT. See LICENSE import unittest from contextlib import contextmanager -from datetime import timedelta +from datetime import datetime, timedelta from unittest.mock import patch import frappe from frappe.app import make_form_dict from frappe.desk.doctype.note.note import Note from frappe.model.naming import make_autoname, parse_naming_series, revert_series_if_last -from frappe.utils import cint, now_datetime, set_request +from frappe.utils import cast, cint, get_datetime, now_datetime, set_request from frappe.website.serve import get_response from . import update_system_settings @@ -228,7 +228,11 @@ class TestDocument(unittest.TestCase): frappe.delete_doc_if_exists("Currency", "Frappe Coin", 1) d = frappe.get_doc( - {"doctype": "Currency", "currency_name": "Frappe Coin", "smallest_currency_fraction_value": -1} + { + "doctype": "Currency", + "currency_name": "Frappe Coin", + "smallest_currency_fraction_value": -1, + } ) self.assertRaises(frappe.NonNegativeError, d.insert) @@ -247,7 +251,12 @@ class TestDocument(unittest.TestCase): "module": "Custom", "custom": 1, "fields": [ - {"label": "Currency", "fieldname": "currency", "reqd": 1, "fieldtype": "Currency"}, + { + "label": "Currency", + "fieldname": "currency", + "reqd": 1, + "fieldtype": "Currency", + }, ], } ).insert(ignore_if_duplicate=True) @@ -373,6 +382,59 @@ class TestDocument(unittest.TestCase): except Exception as e: self.fail(f"Invalid doc hook: {doctype}:{hook}\n{e}") + def test_date_casting(self): + create_time_custom_field() + + _datetime_str = "2022-02-13 12:02:33.713418" + _datetime = get_datetime(_datetime_str) + + # Check if the system parses the string for date and time + todo = frappe.get_doc( + { + "doctype": "ToDo", + "description": "test_date_and_time_casting", + "date": _datetime_str, + "time": _datetime_str, + } + ).insert() + + self.assertEqual(todo.date, cast("Date", _datetime_str)) + self.assertEqual(todo.time, cast("Time", _datetime_str)) + + # Check if the system parses the datetime object for date and time + todo.date = _datetime + todo.time = _datetime + todo.save() + + self.assertEqual(todo.date, cast("Date", _datetime)) + self.assertEqual(todo.time, cast("Time", _datetime)) + + # Check if the system parses the datetime object for date and time + todo.date = None + todo.time = None + todo.save() + + self.assertEqual(todo.date, None) + self.assertEqual(todo.time, None) + + # Check for standard datetime fields + self.assertIsInstance(todo.creation, datetime) + self.assertIsInstance(todo.modified, datetime) + + +def create_time_custom_field(): + if not frappe.db.exists({"doctype": "Custom Field", "dt": "ToDo", "fieldname": "time"}): + frappe.get_doc( + { + "doctype": "Custom Field", + "label": "Time", + "dt": "ToDo", + "fieldname": "time", + "fieldtype": "Time", + "insert_after": "date", + } + ).insert() + class TestDocumentWebView(unittest.TestCase): def get(self, path, user="Guest"): diff --git a/frappe/utils/data.py b/frappe/utils/data.py index da6f590a8f..2813dc69e0 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -130,13 +130,13 @@ def get_timedelta(time: str | None = None) -> datetime.timedelta | None: valid time format. Returns None if `time` is not a valid format Args: - time (str): A valid time representation. This string is parsed - using `dateutil.parser.parse`. Examples of valid inputs are: - '0:0:0', '17:21:00', '2012-01-19 17:21:00'. Checkout - https://dateutil.readthedocs.io/en/stable/parser.html#dateutil.parser.parse + time (str): A valid time representation. This string is parsed + using `dateutil.parser.parse`. Examples of valid inputs are: + '0:0:0', '17:21:00', '2012-01-19 17:21:00'. Checkout + https://dateutil.readthedocs.io/en/stable/parser.html#dateutil.parser.parse Returns: - datetime.timedelta: Timedelta object equivalent of the passed `time` string + datetime.timedelta: Timedelta object equivalent of the passed `time` string """ from dateutil import parser from dateutil.parser import ParserError @@ -173,6 +173,15 @@ def to_timedelta(time_str: str | datetime.time) -> datetime.timedelta: return time_str +def to_datetime( + datetime_str: str | datetime.datetime, format: str | None = None +) -> datetime.datetime: + if isinstance(datetime_str, datetime.datetime): + return datetime_str + elif isinstance(datetime_str, str): + return datetime.datetime.strptime(datetime_str, format or DATETIME_FORMAT) + + @typing.overload def add_to_date( date, @@ -840,12 +849,12 @@ def cast(fieldtype, value=None): If value can't be cast as fieldtype due to an invalid input, None will be returned. Mapping of Python types => Frappe types: - * str => ("Data", "Text", "Small Text", "Long Text", "Text Editor", "Select", "Link", "Dynamic Link") - * float => ("Currency", "Float", "Percent") - * int => ("Int", "Check") - * datetime.datetime => ("Datetime",) - * datetime.date => ("Date",) - * datetime.time => ("Time",) + * str => ("Data", "Text", "Small Text", "Long Text", "Text Editor", "Select", "Link", "Dynamic Link") + * float => ("Currency", "Float", "Percent") + * int => ("Int", "Check") + * datetime.datetime => ("Datetime",) + * datetime.date => ("Date",) + * datetime.time => ("Time",) """ if fieldtype in ("Currency", "Float", "Percent"): value = flt(value) @@ -953,13 +962,11 @@ def floor(s): Parameters ---------- - s : int or str or Decimal object - The mathematical value to be floored + s : int or str or Decimal object. The mathematical value to be floored. Returns ------- - int - number representing the largest integer less than or equal to the specified number + int: number representing the largest integer less than or equal to the specified number """ try: @@ -975,13 +982,11 @@ def ceil(s): Parameters ---------- - s : int or str or Decimal object - The mathematical value to be ceiled + s : int or str or Decimal object. The mathematical value to be ceiled Returns ------- - int - smallest integer greater than or equal to the given number + int: smallest integer greater than or equal to the given number """ try: @@ -998,15 +1003,15 @@ def cstr(s, encoding="utf-8"): def sbool(x: str) -> bool | Any: """Converts str object to Boolean if possible. Example: - "true" becomes True - "1" becomes True - "{}" remains "{}" + "true" becomes True + "1" becomes True + "{}" remains "{}" Args: - x (str): String to be converted to Bool + x (str): String to be converted to Bool Returns: - object: Returns Boolean or x + object: Returns Boolean or x """ try: val = x.lower() @@ -1700,11 +1705,11 @@ def get_filter(doctype: str, f: dict | list | tuple, filters_config=None) -> "fr """Returns a _dict like { - "doctype": - "fieldname": - "operator": - "value": - "fieldtype": + "doctype": + "fieldname": + "operator": + "value": + "fieldtype": } """ from frappe.model import child_table_fields, default_fields, optional_fields @@ -2056,8 +2061,8 @@ def validate_python_code( """Validate python code fields by using compile_command to ensure that expression is valid python. args: - fieldname: name of field being validated. - is_expression: true for validating simple single line python expression, else validated as script. + fieldname: name of field being validated. + is_expression: true for validating simple single line python expression, else validated as script. """ if not string: @@ -2114,3 +2119,6 @@ def parse_timedelta(s: str) -> datetime.timedelta: m = TIMEDELTA_BASE_PATTERN.match(s) return datetime.timedelta(**{key: float(val) for key, val in m.groupdict().items()}) + + +DEFAULT_DATETIME_SHORTCUTS = ["Today", "__today", "now", "Now"] diff --git a/frappe/website/doctype/personal_data_deletion_request/test_personal_data_deletion_request.py b/frappe/website/doctype/personal_data_deletion_request/test_personal_data_deletion_request.py index 7311291289..68d2a6056c 100644 --- a/frappe/website/doctype/personal_data_deletion_request/test_personal_data_deletion_request.py +++ b/frappe/website/doctype/personal_data_deletion_request/test_personal_data_deletion_request.py @@ -49,9 +49,7 @@ class TestPersonalDataDeletionRequest(unittest.TestCase): self.assertEqual(self.delete_request.status, "Deleted") def test_unverified_record_removal(self): - date_time_obj = datetime.strptime( - self.delete_request.creation, "%Y-%m-%d %H:%M:%S.%f" - ) + timedelta(days=-7) + date_time_obj = self.delete_request.creation + timedelta(days=-7) self.delete_request.db_set("creation", date_time_obj) self.delete_request.db_set("status", "Pending Verification") @@ -60,9 +58,7 @@ class TestPersonalDataDeletionRequest(unittest.TestCase): def test_process_auto_request(self): frappe.db.set_value("Website Settings", None, "auto_account_deletion", "1") - date_time_obj = datetime.strptime( - self.delete_request.creation, "%Y-%m-%d %H:%M:%S.%f" - ) + timedelta(hours=-2) + date_time_obj = self.delete_request.creation + timedelta(hours=-2) self.delete_request.db_set("creation", date_time_obj) self.delete_request.db_set("status", "Pending Approval")