From 6ba4baecef8fb2b2bb7079d8c38a7b6386a32cb4 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Fri, 8 May 2015 12:16:16 +0530 Subject: [PATCH 1/4] [cache-redesign] meta cache --- frappe/__init__.py | 8 ++--- frappe/desk/form/meta.py | 2 +- frappe/model/document.py | 1 - frappe/model/meta.py | 59 ++++++++++++++++++----------------- frappe/modules/__init__.py | 4 ++- frappe/utils/redis_wrapper.py | 39 ++++++++++++++--------- 6 files changed, 63 insertions(+), 50 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index af404587e2..38c2113884 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -432,10 +432,10 @@ def has_website_permission(doctype, ptype="read", doc=None, user=None, verbose=F def is_table(doctype): """Returns True if `istable` property (indicating child Table) is set for given DocType.""" - tables = cache().get_value("is_table") - if tables==None: - tables = db.sql_list("select name from tabDocType where ifnull(istable,0)=1") - cache().set_value("is_table", tables) + def get_tables(doctype): + return db.sql_list("select name from tabDocType where ifnull(istable,0)=1") + + tables = cache().get_value("is_table", get_tables) return doctype in tables def get_precision(doctype, fieldname, currency=None, doc=None): diff --git a/frappe/desk/form/meta.py b/frappe/desk/form/meta.py index 210b02a31b..7018d15b87 100644 --- a/frappe/desk/form/meta.py +++ b/frappe/desk/form/meta.py @@ -16,7 +16,7 @@ from frappe.utils.jinja import render_include def get_meta(doctype, cached=True): if cached and not frappe.conf.developer_mode: - meta = frappe.cache().get_value("form_meta:" + doctype, lambda: FormMeta(doctype)) + meta = frappe.cache().hget("form_meta", doctype, lambda: FormMeta(doctype)) else: meta = FormMeta(doctype) diff --git a/frappe/model/document.py b/frappe/model/document.py index 13ee0ced6e..4b60515c3e 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -568,7 +568,6 @@ class Document(BaseDocument): elif self._action=="update_after_submit": self.run_method("on_update_after_submit") - frappe.cache().set_value("last_modified:" + self.doctype, self.modified) self.latest = None def check_no_back_links_exist(self): diff --git a/frappe/model/meta.py b/frappe/model/meta.py index 34050d0afe..0ddb150b27 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -11,16 +11,15 @@ from frappe.model.document import Document from frappe.model.base_document import BaseDocument from frappe.model.db_schema import type_map -###### - def get_meta(doctype, cached=True): if cached: - return frappe.cache().get_value("meta:" + doctype, lambda: Meta(doctype)) + return frappe.cache().hget("meta", doctype, lambda: Meta(doctype)) else: return Meta(doctype) def get_table_columns(doctype): - return frappe.cache().get_value("table_columns:" + doctype, lambda: frappe.db.get_table_columns(doctype)) + return frappe.cache().hget("table_columns", doctype, + lambda: frappe.db.get_table_columns(doctype)) def load_doctype_from_file(doctype): fname = frappe.scrub(doctype) @@ -294,30 +293,6 @@ def get_field_precision(df, doc=None, currency=None): return precision -def clear_cache(doctype=None): - prefixes = ["meta", "form_meta", "table_columns"] - def clear_single(dt): - for p in prefixes: - frappe.cache().delete_value(p + ":" + dt) - - if doctype: - clear_single(doctype) - - # clear all parent doctypes - for dt in frappe.db.sql("""select parent from tabDocField - where fieldtype="Table" and options=%s""", (doctype,)): - clear_single(dt[0]) - - # clear all notifications - from frappe.desk.notifications import delete_notification_count_for - delete_notification_count_for(doctype) - - else: - # clear all - for p in prefixes: - frappe.cache().delete_keys(p + ":") - - frappe.cache().delete_value("is_table") def get_default_df(fieldname): if fieldname in default_fields: @@ -347,3 +322,31 @@ def trim_tables(): query = """alter table `tab{doctype}` {columns}""".format( doctype=doctype, columns=columns_to_remove) frappe.db.sql_ddl(query) + +def clear_cache(doctype=None): + frappe.cache().delete_value("is_table") + frappe.cache().delete_value("doctype_modules") + + groups = ["meta", "form_meta", "table_columns"] + + def clear_single(dt): + for name in groups: + frappe.cache().hdel(name, dt) + + if doctype: + clear_single(doctype) + + # clear all parent doctypes + for dt in frappe.db.sql("""select parent from tabDocField + where fieldtype="Table" and options=%s""", (doctype,)): + clear_single(dt[0]) + + # clear all notifications + from frappe.desk.notifications import delete_notification_count_for + delete_notification_count_for(doctype) + + else: + # clear all + for name in groups: + frappe.cache().delete_value(name) + diff --git a/frappe/modules/__init__.py b/frappe/modules/__init__.py index fefe1f4ff9..17dbc0487c 100644 --- a/frappe/modules/__init__.py +++ b/frappe/modules/__init__.py @@ -46,7 +46,9 @@ def export_doc(doctype, name, module=None): def get_doctype_module(doctype): """Returns **Module Def** name of given doctype.""" - return frappe.db.get_value('DocType', doctype, 'module') or "core" + def make_modules_dict(doctype): + return dict(frappe.db.sql("select name, module from tabDocType")) + frappe.cache().get_value("doctype_modules", make_modules_dict)[doctype] doctype_python_modules = {} def load_doctype_module(doctype, module=None, prefix=""): diff --git a/frappe/utils/redis_wrapper.py b/frappe/utils/redis_wrapper.py index 46c3f91641..7fadd9d751 100644 --- a/frappe/utils/redis_wrapper.py +++ b/frappe/utils/redis_wrapper.py @@ -14,15 +14,12 @@ class RedisWrapper(redis.Redis): key = "user:{0}:{1}".format(user, key) - return (frappe.conf.db_name + "|" + key).encode('utf-8') + return "{0}|{1}".format(frappe.conf.db_name, key).encode('utf-8') def set_value(self, key, val, user=None): """Sets cache value.""" key = self.make_key(key, user) frappe.local.cache[key] = val - if frappe.local.flags.in_install or frappe.local.flags.in_install_db: - return - try: self.set(key, pickle.dumps(val)) except redis.exceptions.ConnectionError: @@ -39,11 +36,10 @@ class RedisWrapper(redis.Redis): if key not in frappe.local.cache: val = None - if not frappe.local.flags.in_install and not frappe.local.flags.in_install_db: - try: - val = self.get(key) - except redis.exceptions.ConnectionError: - pass + try: + val = self.get(key) + except redis.exceptions.ConnectionError: + pass if val is not None: val = pickle.loads(val) if val is None and generator: @@ -87,12 +83,25 @@ class RedisWrapper(redis.Redis): if make_keys: key = self.make_key(key) - - if not frappe.local.flags.in_install and not frappe.local.flags.in_install_db: - try: - self.delete(key) - except redis.exceptions.ConnectionError: - pass + try: + self.delete(key) + except redis.exceptions.ConnectionError: + pass if key in frappe.local.cache: del frappe.local.cache[key] + + def hset(self, name, key, value): + super(redis.Redis, self).hset(self.make_key(name), key, pickle.dumps(value)) + + def hget(self, name, key, generator=None): + value = super(redis.Redis, self).hget(self.make_key(name), key) + if value: + value = pickle.loads(value) + elif generator: + value = generator + self.hset(name, key, value) + return value + + def hdel(self, name, keys): + return super(redis.Redis, self).hget(self.make_key(name), keys) From 2a9f31f1a4247a1a329cadfce1786d8de1ac98af Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Fri, 8 May 2015 13:18:39 +0530 Subject: [PATCH 2/4] [cache] fix --- frappe/__init__.py | 2 +- frappe/modules/__init__.py | 4 ++-- frappe/utils/redis_wrapper.py | 17 +++++++++++++++-- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 38c2113884..80608b580a 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -432,7 +432,7 @@ def has_website_permission(doctype, ptype="read", doc=None, user=None, verbose=F def is_table(doctype): """Returns True if `istable` property (indicating child Table) is set for given DocType.""" - def get_tables(doctype): + def get_tables(): return db.sql_list("select name from tabDocType where ifnull(istable,0)=1") tables = cache().get_value("is_table", get_tables) diff --git a/frappe/modules/__init__.py b/frappe/modules/__init__.py index 17dbc0487c..cffdf11837 100644 --- a/frappe/modules/__init__.py +++ b/frappe/modules/__init__.py @@ -46,9 +46,9 @@ def export_doc(doctype, name, module=None): def get_doctype_module(doctype): """Returns **Module Def** name of given doctype.""" - def make_modules_dict(doctype): + def make_modules_dict(): return dict(frappe.db.sql("select name, module from tabDocType")) - frappe.cache().get_value("doctype_modules", make_modules_dict)[doctype] + return frappe.cache().get_value("doctype_modules", make_modules_dict)[doctype] doctype_python_modules = {} def load_doctype_module(doctype, module=None, prefix=""): diff --git a/frappe/utils/redis_wrapper.py b/frappe/utils/redis_wrapper.py index 7fadd9d751..49087f96bc 100644 --- a/frappe/utils/redis_wrapper.py +++ b/frappe/utils/redis_wrapper.py @@ -2,7 +2,8 @@ # MIT License. See license.txt from __future__ import unicode_literals -import redis, frappe, pickle, re +import redis, frappe, re +import cPickle as pickle from frappe.utils import cstr class RedisWrapper(redis.Redis): @@ -92,16 +93,28 @@ class RedisWrapper(redis.Redis): del frappe.local.cache[key] def hset(self, name, key, value): + if not name in frappe.local.cache: + frappe.local.cache[name] = {} + frappe.local.cache[name][key] = value super(redis.Redis, self).hset(self.make_key(name), key, pickle.dumps(value)) def hget(self, name, key, generator=None): + if not name in frappe.local.cache: + frappe.local.cache[name] = {} + if key in frappe.local.cache[name]: + return frappe.local.cache[name][key] value = super(redis.Redis, self).hget(self.make_key(name), key) if value: value = pickle.loads(value) elif generator: - value = generator + value = generator() self.hset(name, key, value) return value def hdel(self, name, keys): + if name in frappe.local.cache: + del frappe.local.cache[name] return super(redis.Redis, self).hget(self.make_key(name), keys) + + def hkeys(self, name): + return super(redis.Redis, self).hkeys(self.make_key(name)) From ad100e30542b214022e527b914fb4d4b3f25745c Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Fri, 8 May 2015 18:04:13 +0530 Subject: [PATCH 3/4] [cache] fix for website, default, meta, etc --- frappe/database.py | 6 ++--- frappe/defaults.py | 19 +++++--------- frappe/desk/moduleview.py | 2 +- frappe/desk/notifications.py | 45 ++++++++++++++++++-------------- frappe/model/document.py | 2 ++ frappe/model/meta.py | 2 +- frappe/sessions.py | 33 +++++++++++++---------- frappe/tests/test_translation.py | 16 ++++++------ frappe/translate.py | 19 +++++++------- frappe/utils/data.py | 9 +------ frappe/utils/redis_wrapper.py | 3 +++ frappe/utils/user.py | 16 ++++++------ frappe/website/context.py | 10 ++++--- frappe/website/render.py | 20 +++++++++----- frappe/website/router.py | 6 +++-- frappe/website/utils.py | 14 +++++----- 16 files changed, 118 insertions(+), 104 deletions(-) diff --git a/frappe/database.py b/frappe/database.py index de22f34dde..00b85a2218 100644 --- a/frappe/database.py +++ b/frappe/database.py @@ -602,14 +602,12 @@ class Database: def set_temp(self, value): """Set a temperory value and return a key.""" key = frappe.generate_hash() - frappe.cache().set_value(key, value) + frappe.cache().hset("temp", value) return key def get_temp(self, key): """Return the temperory value and delete it.""" - value = frappe.cache().get_value(key) - frappe.cache().delete_value(key) - return value + return frappe.cache().hget("temp", key) def set_global(self, key, val, user='__global'): """Save a global key value. Global values will be automatically set if they match fieldname.""" diff --git a/frappe/defaults.py b/frappe/defaults.py index e2ea61d1e4..bba159ae75 100644 --- a/frappe/defaults.py +++ b/frappe/defaults.py @@ -31,7 +31,7 @@ def get_user_permissions(user=None): return build_user_permissions(user) def build_user_permissions(user): - out = frappe.cache().get_value("user_permissions", user=user) + out = frappe.cache().hget("user_permissions", user) if out==None: out = {} for key, value in frappe.db.sql("""select defkey, ifnull(defvalue, '') as defvalue @@ -42,7 +42,7 @@ def build_user_permissions(user): if user not in out.get("User", []): out.setdefault("User", []).append(user) - frappe.cache().set_value("user_permissions", out, user=user) + frappe.cache().hset("user_permissions", user, out) return out def get_defaults(user=None): @@ -147,7 +147,7 @@ def clear_default(key=None, value=None, parent=None, name=None, parenttype=None) def get_defaults_for(parent="__default"): """get all defaults""" - defaults = frappe.cache().get_value("__defaults:" + parent) + defaults = frappe.cache().hget("defaults", parent) if defaults==None: # sort descending because first default must get preceedence res = frappe.db.sql("""select defkey, defvalue from `tabDefaultValue` @@ -166,7 +166,7 @@ def get_defaults_for(parent="__default"): elif d.defvalue is not None: defaults[d.defkey] = d.defvalue - frappe.cache().set_value("__defaults:" + parent, defaults) + frappe.cache().hset("default", parent, defaults) return defaults @@ -181,12 +181,7 @@ def clear_cache(user=None): to_clear = [] if user: to_clear = [user] + for p in (to_clear + common_keys): + frappe.cache().hdel("default", p) elif frappe.flags.in_install!="frappe": - try: - to_clear = frappe.db.sql_list("select name from tabUser") - except Exception, e: - if e.args[0]!=1146: - # special case, in rename patch - raise - for p in (to_clear + common_keys): - frappe.cache().delete_value("__defaults:" + p) + frappe.cache().delete_key("default") diff --git a/frappe/desk/moduleview.py b/frappe/desk/moduleview.py index aae88e6143..97785b305d 100644 --- a/frappe/desk/moduleview.py +++ b/frappe/desk/moduleview.py @@ -201,7 +201,7 @@ def get_last_modified(doctype): return last_modified - last_modified = frappe.cache().get_value("last_modified:" + doctype, _get) + last_modified = frappe.cache().hget("last_modified", doctype, _get) if last_modified==-1: last_modified = None diff --git a/frappe/desk/notifications.py b/frappe/desk/notifications.py index effb4aff86..22c1d55895 100644 --- a/frappe/desk/notifications.py +++ b/frappe/desk/notifications.py @@ -12,10 +12,12 @@ def get_notifications(): return config = get_notification_config() + groups = config.get("for_doctype").keys() + config.get("for_module").keys() + cache = frappe.cache() - notification_count = frappe.cache().get_all("notification_count:" \ - + frappe.session.user + ":").iteritems() - notification_count = dict([(d.rsplit(":", 1)[1], v) for d, v in notification_count]) + notification_count = {} + for name in groups: + notification_count[name] = cache.hget("notification_count:" + name, frappe.session.user) return { "open_count_doctype": get_notifications_for_doctypes(config, notification_count), @@ -24,10 +26,9 @@ def get_notifications(): } def get_new_messages(): - cache_key = "notifications_last_update:" + frappe.session.user - last_update = frappe.cache().get_value(cache_key) + last_update = frappe.cache().hget("notifications_last_update", frappe.session.user) now_timestamp = now() - frappe.cache().set_value(cache_key, now_timestamp) + frappe.cache().hset("notifications_last_update", frappe.session.user, now_timestamp) if not last_update: return [] @@ -51,8 +52,7 @@ def get_notifications_for_modules(config, notification_count): else: open_count_module[m] = frappe.get_attr(config.for_module[m])() - frappe.cache().set_value("notification_count:" + frappe.session.user + ":" + m, - open_count_module[m]) + frappe.cache().hset("notification_count:" + m, frappe.session.user, open_count_module[m]) return open_count_module @@ -82,16 +82,20 @@ def get_notifications_for_doctypes(config, notification_count): else: open_count_doctype[d] = result - frappe.cache().set_value("notification_count:" + frappe.session.user + ":" + d, - result) + frappe.cache().hset("notification_count:" + d, frappe.session.user, result) return open_count_doctype def clear_notifications(user="*"): - frappe.cache().delete_keys("notification_count:" + (user or frappe.session.user) + ":") + if user=="*": + frappe.cache().delete_keys("notification_count:") + else: + # delete count for user + for key in frappe.cache().get_keys("notification_count:"): + frappe.cache().hdel(key, user) def delete_notification_count_for(doctype): - frappe.cache().delete_keys("notification_count:*:" + doctype) + frappe.cache().delete_key("notification_count:" + doctype) def clear_doctype_notifications(doc, method=None, *args, **kwargs): config = get_notification_config() @@ -127,10 +131,13 @@ def get_notification_info_for_boot(): return out def get_notification_config(): - config = frappe._dict() - for notification_config in frappe.get_hooks().notification_config: - nc = frappe.get_attr(notification_config)() - for key in ("for_doctype", "for_module", "for_module_doctypes"): - config.setdefault(key, {}) - config[key].update(nc.get(key, {})) - return config + def _get(): + config = frappe._dict() + for notification_config in frappe.get_hooks().notification_config: + nc = frappe.get_attr(notification_config)() + for key in ("for_doctype", "for_module", "for_module_doctypes"): + config.setdefault(key, {}) + config[key].update(nc.get(key, {})) + return config + + return frappe.cache().get_value("notification_config", _get) diff --git a/frappe/model/document.py b/frappe/model/document.py index 4b60515c3e..4905600bcd 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -568,6 +568,8 @@ class Document(BaseDocument): elif self._action=="update_after_submit": self.run_method("on_update_after_submit") + frappe.cache().hdel("last_modified", self.doctype) + self.latest = None def check_no_back_links_exist(self): diff --git a/frappe/model/meta.py b/frappe/model/meta.py index 0ddb150b27..f58b290fa0 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -327,7 +327,7 @@ def clear_cache(doctype=None): frappe.cache().delete_value("is_table") frappe.cache().delete_value("doctype_modules") - groups = ["meta", "form_meta", "table_columns"] + groups = ["meta", "form_meta", "table_columns", "last_modified"] def clear_single(dt): for name in groups: diff --git a/frappe/sessions.py b/frappe/sessions.py index d5c08bd0f1..fc88175f06 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -30,19 +30,23 @@ def clear(user=None): def clear_cache(user=None): cache = frappe.cache() + groups = ("bootinfo", "user_recent", "user_roles", "user_doc", "lang", "time_zone") + if user: + for name in groups: + cache.hdel(name, user) cache.delete_keys("user:" + user) - cache.delete_keys("user_doc:" + user) frappe.defaults.clear_cache(user) else: - cache.delete_keys("user:") - cache.delete_keys("user_doc:") + for name in groups: + cache.delete_key(name, user) clear_global_cache() frappe.defaults.clear_cache() def clear_global_cache(): frappe.model.meta.clear_cache() - frappe.cache().delete_value(["app_hooks", "installed_apps", "app_modules", "module_app", "time_zone"]) + frappe.cache().delete_value(["app_hooks", "installed_apps", + "app_modules", "module_app", "time_zone", "notification_config"]) frappe.setup_module_map() def clear_sessions(user=None, keep_current=False): @@ -58,8 +62,8 @@ def clear_sessions(user=None, keep_current=False): def delete_session(sid=None, user=None): if not user: user = hasattr(frappe.local, "session") and frappe.session.user or "Guest" - frappe.cache().delete_value("session:" + sid, user=user) - frappe.cache().delete_value("last_db_session_update:" + sid) + frappe.cache().hdel("session", sid) + frappe.cache().hdel("last_db_session_update", sid) frappe.db.sql("""delete from tabSessions where sid=%s""", sid) def clear_all_sessions(): @@ -83,17 +87,18 @@ def get(): bootinfo = None if not getattr(frappe.conf,'disable_session_cache', None): # check if cache exists - bootinfo = frappe.cache().get_value("bootinfo", user=True) + bootinfo = frappe.cache().hget("bootinfo", frappe.session.user) if bootinfo: bootinfo['from_cache'] = 1 bootinfo["notification_info"].update(get_notifications()) - bootinfo["user"]["recent"] = json.dumps(frappe.cache().get_value("recent:" + frappe.session.user)) + bootinfo["user"]["recent"] = json.dumps(\ + frappe.cache().hget("user_recent", frappe.session.user)) if not bootinfo: # if not create it bootinfo = get_bootinfo() bootinfo["notification_info"] = get_notification_info_for_boot() - frappe.cache().set_value("bootinfo", bootinfo, user=True) + frappe.cache().hset("bootinfo", frappe.session.user, bootinfo) try: frappe.cache().ping() except redis.exceptions.ConnectionError: @@ -168,7 +173,7 @@ class Session: (str(self.data['data']), self.data['user'], self.data['sid'])) # also add to memcache - frappe.cache().set_value("session:" + self.data.sid, self.data, user=self.user) + frappe.cache().hset("session", self.data.sid, self.data) def resume(self): """non-login request: load a session""" @@ -206,7 +211,7 @@ class Session: return data def get_session_data_from_cache(self): - data = frappe._dict(frappe.cache().get_value("session:" + self.sid, user=self.user) or {}) + data = frappe._dict(frappe.cache().hget("session", self.sid) or {}) if data: session_data = data.get("data", {}) self.time_diff = frappe.utils.time_diff_in_seconds(frappe.utils.now(), @@ -257,7 +262,7 @@ class Session: self.data['data']['lang'] = unicode(frappe.lang) # update session in db - last_updated = frappe.cache().get_value("last_db_session_update:" + self.sid) + last_updated = frappe.cache().hget("last_db_session_update", self.sid) time_diff = frappe.utils.time_diff_in_seconds(now, last_updated) if last_updated else None # database persistence is secondary, don't update it too often @@ -267,11 +272,11 @@ class Session: lastupdate=NOW() where sid=%s""" , (str(self.data['data']), self.data['sid'])) - frappe.cache().set_value("last_db_session_update:" + self.sid, now) + frappe.cache().hset("last_db_session_update", self.sid, now) updated_in_db = True # set in memcache - frappe.cache().set_value("session:" + self.sid, self.data, user=self.user) + frappe.cache().hset("session", self.sid, self.data) return updated_in_db diff --git a/frappe/tests/test_translation.py b/frappe/tests/test_translation.py index 5b746cc906..8c44fbeacf 100644 --- a/frappe/tests/test_translation.py +++ b/frappe/tests/test_translation.py @@ -15,7 +15,7 @@ import frappe.translate # if not messages: # messages = frappe.translate.get_messages_from_page("finder") # self.assertTrue("Finder" in messages) -# +# # def test_report(self, messages=None): # if not messages: # messages = frappe.translate.get_messages_from_report("ToDo") @@ -25,13 +25,13 @@ import frappe.translate # if not messages: # messages = frappe.translate.get_messages_from_include_files("frappe") # self.assertTrue("History" in messages) -# +# # def test_server(self, messages=None): # if not messages: # messages = frappe.translate.get_server_messages("frappe") # self.assertTrue("Login" in messages) # self.assertTrue("Did not save" in messages) -# +# # def test_all_app(self): # messages = frappe.translate.get_messages_for_app("frappe") # self.test_doctype(messages) @@ -39,14 +39,14 @@ import frappe.translate # self.test_report(messages) # self.test_include_js(messages) # self.test_server(messages) -# +# # def test_load_translations(self): # frappe.translate.clear_cache() -# self.assertFalse(frappe.cache().get_value("lang:de")) -# +# self.assertFalse(frappe.cache().hget("lang_full_dict", "de")) +# # langdict = frappe.translate.get_full_dict("de") # self.assertEquals(langdict['Row'], 'Reihe') -# +# # def test_write_csv(self): # tpath = frappe.get_pymodule_path("frappe", "translations", "de.csv") # if os.path.exists(tpath): @@ -54,7 +54,7 @@ import frappe.translate # frappe.translate.write_translations_file("frappe", "de") # self.assertTrue(os.path.exists(tpath)) # self.assertEquals(dict(frappe.translate.read_csv_file(tpath)).get("Row"), "Reihe") -# +# # def test_get_dict(self): # frappe.local.lang = "de" # self.assertEquals(frappe.get_lang_dict("doctype", "Role").get("Role"), "Rolle") diff --git a/frappe/translate.py b/frappe/translate.py index f2509a8a89..8447faa3a1 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -44,7 +44,7 @@ def get_user_lang(user=None): user = frappe.session.user # via cache - lang = frappe.cache().get_value("lang", user=user) + lang = frappe.cache().hget("lang", user) if not lang: @@ -56,7 +56,7 @@ def get_user_lang(user=None): default_lang = frappe.db.get_default("lang") lang = default_lang or frappe.local.lang - frappe.cache().set_value("lang", lang or "en", user=user) + frappe.cache().hset("lang", user, lang or "en") return lang @@ -90,9 +90,8 @@ def get_dict(fortype, name=None): """ fortype = fortype.lower() cache = frappe.cache() - cache_key = "translation_assets:" + (frappe.local.lang or "en") asset_key = fortype + ":" + (name or "-") - translation_assets = cache.get_value(cache_key) or {} + translation_assets = cache.hget("translation_assets", frappe.local.lang) or {} if not asset_key in translation_assets: if fortype=="doctype": @@ -114,7 +113,7 @@ def get_dict(fortype, name=None): translation_assets[asset_key] = make_dict_from_messages(messages) translation_assets[asset_key].update(get_dict_from_hooks(fortype, name)) - cache.set_value(cache_key, translation_assets) + cache.hset("translation_assets", frappe.local.lang, translation_assets) return translation_assets[asset_key] @@ -170,10 +169,10 @@ def get_full_dict(lang): return {} if not frappe.local.lang_full_dict: - frappe.local.lang_full_dict = frappe.cache().get_value("lang:" + lang) + frappe.local.lang_full_dict = frappe.cache().hget("lang_full_dict", lang) if not frappe.local.lang_full_dict: # cache lang - frappe.cache().set_value("lang:" + lang, frappe.local.lang_full_dict) + frappe.cache().hset("lang_full_dict", lang, frappe.local.lang_full_dict) frappe.local.lang_full_dict = load_lang(lang) return frappe.local.lang_full_dict @@ -196,9 +195,9 @@ def load_lang(lang, apps=None): def clear_cache(): """Clear all translation assets from :meth:`frappe.cache`""" cache = frappe.cache() - cache.delete_value("langinfo") - cache.delete_keys("lang:") - cache.delete_keys("translation_assets:") + cache.delete_key("langinfo") + cache.delete_key("lang_full_dict") + cache.delete_key("translation_assets") def get_messages_for_app(app): """Returns all messages (list) for a specified `app`""" diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 450079f0a6..27244ef704 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -115,14 +115,7 @@ def get_user_time_zone(): if frappe.local.flags.in_test: return _get_user_time_zone() - if getattr(frappe.local, "user_time_zone", None) is None: - frappe.local.user_time_zone = frappe.cache().get_value("time_zone") - - if not frappe.local.user_time_zone: - frappe.local.user_time_zone = _get_user_time_zone() - frappe.cache().set_value("time_zone", frappe.local.user_time_zone, user=frappe.session.user) - - return frappe.local.user_time_zone + return frappe.cache().hget("time_zone", frappe.session.user, _get_user_time_zone) def convert_utc_to_user_timezone(utc_timestamp): from pytz import timezone, UnknownTimeZoneError diff --git a/frappe/utils/redis_wrapper.py b/frappe/utils/redis_wrapper.py index 49087f96bc..5e08f64ea3 100644 --- a/frappe/utils/redis_wrapper.py +++ b/frappe/utils/redis_wrapper.py @@ -75,6 +75,9 @@ class RedisWrapper(redis.Redis): except redis.exceptions.ConnectionError: pass + def delete_key(self, *args, **kwargs): + self.delete_value(*args, **kwargs) + def delete_value(self, keys, user=None, make_keys=True): """Delete value, list of values.""" if not isinstance(keys, (list, tuple)): diff --git a/frappe/utils/user.py b/frappe/utils/user.py index 31ff16f35b..53e9660ee1 100644 --- a/frappe/utils/user.py +++ b/frappe/utils/user.py @@ -47,7 +47,7 @@ class User: return user if not frappe.flags.in_install_db and not frappe.flags.in_test: - user_doc = frappe.cache().get_value("user_doc:" + self.name, get_user_doc) + user_doc = frappe.cache().hget("user_doc", self.name, get_user_doc) if user_doc: self.doc = frappe.get_doc(user_doc) @@ -156,7 +156,7 @@ class User: # update recent documents def update_recent(self, dt, dn): - rdl = frappe.cache().get_value("recent:" + self.name) or [] + rdl = frappe.cache().hget("user_recent", self.name) or [] new_rd = [dt, dn] # clear if exists @@ -171,7 +171,7 @@ class User: rdl = [new_rd] + rdl - frappe.cache().set_value("recent:" + self.name, rdl) + frappe.cache().hset("user_recent", self.name, rdl) def _get(self, key): if not self.can_read: @@ -193,7 +193,7 @@ class User: self.build_permissions() d.name = self.name - d.recent = json.dumps(frappe.cache().get_value("recent:" + self.name) or []) + d.recent = json.dumps(frappe.cache().hget("user_recent", self.name) or []) d.roles = self.get_roles() d.defaults = self.get_defaults() @@ -267,11 +267,11 @@ def get_roles(user=None, with_standard=True): if user=='Guest': return ['Guest'] - roles = frappe.cache().get_value("roles", user=user) - if not roles: - roles = [r[0] for r in frappe.db.sql("""select role from tabUserRole + def get(): + return [r[0] for r in frappe.db.sql("""select role from tabUserRole where parent=%s and role not in ('All', 'Guest')""", (user,))] + ['All', 'Guest'] - frappe.cache().set_value("roles", roles, user=user) + + roles = frappe.cache().hget("roles", user, get) # filter standard if required if not with_standard: diff --git a/frappe/website/context.py b/frappe/website/context.py index bdb5dd19b9..0882fed340 100644 --- a/frappe/website/context.py +++ b/frappe/website/context.py @@ -11,7 +11,7 @@ from frappe.website.utils import can_cache def get_context(path): context = None - cache_key = "page_context:{0}:{1}".format(path, frappe.local.lang) + context_cache = {} def add_data_path(context): if not context.data: @@ -19,9 +19,10 @@ def get_context(path): context.data["path"] = path - # try from memcache + # try from cache if can_cache(): - context = frappe.cache().get_value(cache_key) + context_cache = frappe.cache().hget("page_context", path) or {} + context = context_cache.get(frappe.local.lang, None) if not context: context = get_route_info(path) @@ -30,7 +31,8 @@ def get_context(path): add_data_path(context) if can_cache(context.no_cache): - frappe.cache().set_value(cache_key, context) + context_cache[frappe.local.lang] = context + frappe.cache().hset("page_context", path, context_cache) else: add_data_path(context) diff --git a/frappe/website/render.py b/frappe/website/render.py index 580ad5508d..fbe42053b3 100644 --- a/frappe/website/render.py +++ b/frappe/website/render.py @@ -99,15 +99,19 @@ def build_response(path, data, http_status_code, headers=None): def render_page(path): """get page html""" - cache_key = ("page_context:{0}:{1}" if is_ajax() else "page:{0}:{1}").format(path, frappe.local.lang) - out = None - # try cache if can_cache(): - out = frappe.cache().get_value(cache_key) - if out and is_ajax(): - out = out.get("data") + if is_ajax(): + # ajax, send context + context_cache = frappe.cache().hget("page_context", path) + if context_cache and frappe.local.lang in context_cache: + out = context_cache[frappe.local.lang].get("data") + else: + # return rendered page + page_cache = frappe.cache().hget("website_page", path) + if page_cache and frappe.local.lang in page_cache: + out = page_cache[frappe.local.lang] if out: frappe.local.response.from_cache = True @@ -143,7 +147,9 @@ def build_page(path): html = frappe.get_template(context.base_template_path).render(context) if can_cache(context.no_cache): - frappe.cache().set_value("page:{0}:{1}".format(path, frappe.local.lang), html) + page_cache = frappe.cache().hget("website_page", path) or {} + page_cache[frappe.local.lang] = html + frappe.cache().hset("website_page", path, page_cache) return html diff --git a/frappe/website/router.py b/frappe/website/router.py index f28da965f5..4334bc91a4 100644 --- a/frappe/website/router.py +++ b/frappe/website/router.py @@ -12,12 +12,14 @@ def get_route_info(path): cache_key = "sitemap_options:{0}:{1}".format(path, frappe.local.lang) if can_cache(): - sitemap_options = frappe.cache().get_value(cache_key) + sitemap_options_cache = frappe.cache().hget("sitemap_options", path) or {} + sitemap_options = sitemap_options_cache.get(frappe.local.lang, None) if not sitemap_options: sitemap_options = build_route(path) if can_cache(sitemap_options.no_cache): - frappe.cache().set_value(cache_key, sitemap_options) + sitemap_options_cache[frappe.local.lang] = sitemap_options + frappe.cache().hset("sitemap_options", path, sitemap_options_cache) return sitemap_options diff --git a/frappe/website/utils.py b/frappe/website/utils.py index fe20657f68..4ac59a9230 100644 --- a/frappe/website/utils.py +++ b/frappe/website/utils.py @@ -5,12 +5,14 @@ from __future__ import unicode_literals import frappe, re, os def delete_page_cache(path): - if not path: - path = "" cache = frappe.cache() - cache.delete_keys("page:" + path) - cache.delete_keys("page_context:" + path) - cache.delete_keys("sitemap_options:" + path) + groups = ("page_context", "website_page", "sitemap_options") + if path: + for name in groups: + cache.hdel(name, path) + else: + for name in groups: + cache.delete_key(name) def find_first_image(html): m = re.finditer("""]*src\s?=\s?['"]([^'"]*)['"]""", html) @@ -49,7 +51,7 @@ def get_home_page(): return home_page - return frappe.cache().get_value("home_page", _get_home_page, user=True) + return frappe.cache().hget("home_page", frappe.session.user, _get_home_page) def is_signup_enabled(): if getattr(frappe.local, "is_signup_enabled", None) is None: From 4e6b6ebdb99d53d9677dc3ddefb39c239e6e382b Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Fri, 8 May 2015 22:30:38 +0530 Subject: [PATCH 4/4] [cache] fix --- frappe/defaults.py | 2 +- frappe/desk/notifications.py | 5 +++-- frappe/utils/redis_wrapper.py | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/frappe/defaults.py b/frappe/defaults.py index bba159ae75..5bdebf089c 100644 --- a/frappe/defaults.py +++ b/frappe/defaults.py @@ -166,7 +166,7 @@ def get_defaults_for(parent="__default"): elif d.defvalue is not None: defaults[d.defkey] = d.defvalue - frappe.cache().hset("default", parent, defaults) + frappe.cache().hset("defaults", parent, defaults) return defaults diff --git a/frappe/desk/notifications.py b/frappe/desk/notifications.py index 22c1d55895..9acd2e8b11 100644 --- a/frappe/desk/notifications.py +++ b/frappe/desk/notifications.py @@ -17,7 +17,9 @@ def get_notifications(): notification_count = {} for name in groups: - notification_count[name] = cache.hget("notification_count:" + name, frappe.session.user) + count = cache.hget("notification_count:" + name, frappe.session.user) + if count is not None: + notification_count[name] = count return { "open_count_doctype": get_notifications_for_doctypes(config, notification_count), @@ -81,7 +83,6 @@ def get_notifications_for_doctypes(config, notification_count): else: open_count_doctype[d] = result - frappe.cache().hset("notification_count:" + d, frappe.session.user, result) return open_count_doctype diff --git a/frappe/utils/redis_wrapper.py b/frappe/utils/redis_wrapper.py index 5e08f64ea3..bafda659c4 100644 --- a/frappe/utils/redis_wrapper.py +++ b/frappe/utils/redis_wrapper.py @@ -109,6 +109,7 @@ class RedisWrapper(redis.Redis): value = super(redis.Redis, self).hget(self.make_key(name), key) if value: value = pickle.loads(value) + frappe.local.cache[name][key] = value elif generator: value = generator() self.hset(name, key, value)