Browse Source

refactored reportview.py and added frappe/model/db_query.py

version-14
Rushabh Mehta 11 years ago
parent
commit
3964db5d95
5 changed files with 340 additions and 283 deletions
  1. +4
    -2
      frappe/__init__.py
  2. +287
    -0
      frappe/model/db_query.py
  3. +31
    -0
      frappe/tests/test_db_query.py
  4. +1
    -2
      frappe/website/doctype/blog_post/test_blog_post.py
  5. +17
    -279
      frappe/widgets/reportview.py

+ 4
- 2
frappe/__init__.py View File

@@ -560,10 +560,12 @@ def build_match_conditions(doctype, fields=None, as_condition=True):
def get_list(doctype, filters=None, fields=None, docstatus=None, def get_list(doctype, filters=None, fields=None, docstatus=None,
group_by=None, order_by=None, limit_start=0, limit_page_length=None, group_by=None, order_by=None, limit_start=0, limit_page_length=None,
as_list=False, debug=False): as_list=False, debug=False):
import frappe.widgets.reportview
return frappe.widgets.reportview.execute(doctype, filters=filters, fields=fields, docstatus=docstatus,
import frappe.model.db_query
return frappe.model.db_query.DatabaseQuery(doctype).execute(filters=filters, fields=fields, docstatus=docstatus,
group_by=group_by, order_by=order_by, limit_start=limit_start, limit_page_length=limit_page_length, group_by=group_by, order_by=order_by, limit_start=limit_start, limit_page_length=limit_page_length,
as_list=as_list, debug=debug) as_list=as_list, debug=debug)
run_query = get_list


def get_jenv(): def get_jenv():
if not local.jenv: if not local.jenv:


+ 287
- 0
frappe/model/db_query.py View File

@@ -0,0 +1,287 @@
# Copyright (c) 2013, Web Notes Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

from __future__ import unicode_literals
"""build query for doclistview and return results"""

import frappe, json
import frappe.defaults
import frappe.permissions
import frappe.model.doctype
from frappe.utils import cstr, flt

class DatabaseQuery(object):
def __init__(self, doctype):
self.doctype = doctype
def execute(self, query=None, filters=None, fields=None, docstatus=None,
group_by=None, order_by=None, limit_start=0, limit_page_length=20,
as_list=False, with_childnames=False, debug=False):
self.fields = fields or ["name"]
self.filters = filters or []
self.docstatus = docstatus or []
self.group_by = group_by
self.order_by = order_by
self.limit_start = limit_start
self.limit_page_length = limit_page_length
self.with_childnames = with_childnames
self.debug = debug
self.as_list = as_list
self.tables = []
self.meta = []
if query:
return self.run_custom_query(query)
else:
return self.build_and_run()

def build_and_run(self):
args = self.prepare_args()
args.limit = self.add_limit()
query = """select %(fields)s from %(tables)s where %(conditions)s
%(group_by)s order by %(order_by)s %(limit)s""" % args
return frappe.db.sql(query, as_dict=not self.as_list, debug=self.debug)

def prepare_args(self):
self.parse_args()
self.extract_tables()
self.load_metadata()
self.remove_user_tags()
self.build_conditions()
args = frappe._dict()
if self.with_childnames:
for t in self.tables:
if t != "`tab" + doctype + "`":
fields.append(t + ".name as '%s:name'" % t[4:-1])
# query dict
args.tables = ', '.join(self.tables)
args.conditions = ' and '.join(self.conditions)
args.fields = ', '.join(self.fields)
args.order_by = self.order_by or self.tables[0] + '.modified desc'
args.group_by = self.group_by and (" group by " + group_by) or ""

self.check_sort_by_table(args.order_by)
return args
def parse_args(self):
if isinstance(self.filters, basestring):
self.filters = json.loads(self.filters)
if isinstance(self.fields, basestring):
self.filters = json.loads(self.fields)
if isinstance(self.filters, dict):
fdict = self.filters
self.filters = []
for key, value in fdict.iteritems():
self.filters.append(self.make_filter_tuple(key, value))
def make_filter_tuple(self, key, value):
if isinstance(value, (list, tuple)):
return (self.doctype, key, value[0], value[1])
else:
return (self.doctype, key, "=", value)
def extract_tables(self):
"""extract tables from fields"""
self.tables = ['`tab' + self.doctype + '`']

# add tables from fields
if self.fields:
for f in self.fields:
if "." not in f: continue
table_name = f.split('.')[0]
if table_name.lower().startswith('group_concat('):
table_name = table_name[13:]
if table_name.lower().startswith('ifnull('):
table_name = table_name[7:]
if not table_name[0]=='`':
table_name = '`' + table_name + '`'
if not table_name in self.tables:
self.tables.append(table_name)
def load_metadata(self):
"""load all doctypes and roles"""
self.meta = {}

for t in self.tables:
if t.startswith('`'):
doctype = t[4:-1]
if self.meta.get(doctype):
continue
if not frappe.has_permission(doctype):
raise frappe.PermissionError, doctype
self.meta[doctype] = frappe.model.doctype.get(doctype)
def remove_user_tags(self):
"""remove column _user_tags if not in table"""
columns = frappe.db.get_table_columns(self.doctype)
to_remove = []
for fld in self.fields:
for f in ("_user_tags", "_comments"):
if f in fld and not f in columns:
to_remove.append(fld)
for fld in to_remove:
del self.fields[self.fields.index(fld)]

def build_conditions(self):
self.conditions = []
self.add_docstatus_conditions()
self.build_filter_conditions()
# join parent, child tables
for tname in self.tables[1:]:
self.conditions.append(tname + '.parent = ' + self.tables[0] + '.name')

# match conditions
match_conditions = self.build_match_conditions()
if match_conditions:
self.conditions.append(match_conditions)
def add_docstatus_conditions(self):
if self.docstatus:
self.conditions.append(self.tables[0] + '.docstatus in (' + ','.join(docstatus) + ')')
else:
self.conditions.append(self.tables[0] + '.docstatus < 2')
def build_filter_conditions(self):
"""build conditions from user filters"""
doclist = {}
for f in self.filters:
if isinstance(f, basestring):
self.conditions.append(f)
else:
f = self.get_filter_tuple(f)

tname = ('`tab' + f[0] + '`')
if not tname in self.tables:
self.tables.append(tname)

if not tname in self.meta:
self.load_metadata()
# prepare in condition
if f[2] in ['in', 'not in']:
opts = ["'" + t.strip().replace("'", "\\'") + "'" for t in f[3].split(',')]
f[3] = "(" + ', '.join(opts) + ")"
self.conditions.append('ifnull(' + tname + '.' + f[1] + ", '') " + f[2] + " " + f[3])
else:
df = self.meta[f[0]].get({"doctype": "DocField", "fieldname": f[1]})
if f[2] == "like" or (isinstance(f[3], basestring) and
(not df or df[0].fieldtype not in ["Float", "Int", "Currency", "Percent"])):
value, default_val = ("'" + f[3].replace("'", "\\'") + "'"), '""'
else:
value, default_val = flt(f[3]), 0

self.conditions.append('ifnull({tname}.{fname}, {default_val}) {operator} {value}'.format(
tname=tname, fname=f[1], default_val=default_val, operator=f[2],
value=value))
def get_filter_tuple(self, f):
if isinstance(f, dict):
key, value = f.items()[0]
f = self.make_filter_tuple(key, value)
if not isinstance(f, (list, tuple)):
frappe.throw("Filter must be a tuple or list (in a list)")

if len(f) != 4:
frappe.throw("Filter must have 4 values (doctype, fieldname, condition, value): " + str(f))
return f
def build_match_conditions(self, as_condition=True):
"""add match conditions if applicable"""
self.match_filters = {}
self.match_conditions = []
self.or_conditions = []

if not self.tables: self.extract_tables()
if not self.meta: self.load_metadata()
# explict permissions
restricted_by_user = frappe.permissions.get_user_perms(self.meta[self.doctype]).restricted
# get restrictions
restrictions = frappe.defaults.get_restrictions()
if restricted_by_user:
self.or_conditions.append('`tab{doctype}`.`owner`="{user}"'.format(doctype=self.doctype,
user=frappe.local.session.user))
self.match_filters["owner"] = frappe.session.user
if restrictions:
self.add_restrictions(restrictions)
if as_condition:
return self.build_match_condition_string()
else:
return self.match_filters
def add_restrictions(self, restrictions):
fields_to_check = self.meta[self.doctype].get_restricted_fields(restrictions.keys())
if self.doctype in restrictions:
fields_to_check.append(frappe._dict({"fieldname":"name", "options":self.doctype}))
# check in links
for df in fields_to_check:
self.match_conditions.append('`tab{doctype}`.{fieldname} in ({values})'.format(doctype=self.doctype,
fieldname=df.fieldname,
values=", ".join([('"'+v.replace('"', '\"')+'"') \
for v in restrictions[df.options]])))
self.match_filters.setdefault(df.fieldname, [])
self.match_filters[df.fieldname]= restrictions[df.options]
def build_match_condition_string(self):
conditions = " and ".join(self.match_conditions)
doctype_conditions = self.get_permission_query_conditions()
if doctype_conditions:
conditions += ' and ' + doctype_conditions if conditions else doctype_conditions
if self.or_conditions:
if conditions:
conditions = '({conditions}) or {or_conditions}'.format(conditions=conditions,
or_conditions = ' or '.join(self.or_conditions))
else:
conditions = " or ".join(self.or_conditions)
return conditions
def get_permission_query_conditions(self):
condition_methods = frappe.get_hooks("permission_query_conditions:" + self.doctype)
if condition_methods:
conditions = []
for method in condition_methods:
c = frappe.get_attr(method)()
if c:
conditions.append(c)
return " and ".join(conditions) if conditions else None
def run_custom_query(self, query):
if '%(key)s' in query:
query = query.replace('%(key)s', 'name')
return frappe.db.sql(query, as_dict = (not self.as_list))
def check_sort_by_table(self, order_by):
if "." in order_by:
tbl = order_by.split('.')[0]
if tbl not in self.tables:
if tbl.startswith('`'):
tbl = tbl[4:-1]
frappe.throw("Please select atleast 1 column from '%s' to sort" % tbl)

def add_limit(self):
if self.limit_page_length:
return 'limit %s, %s' % (self.limit_start, self.limit_page_length)
else:
return ''

+ 31
- 0
frappe/tests/test_db_query.py View File

@@ -0,0 +1,31 @@
# Copyright (c) 2013, Web Notes Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

import frappe, unittest

from frappe.model.db_query import DatabaseQuery

class TestReportview(unittest.TestCase):
def test_basic(self):
self.assertTrue({"name":"DocType"} in DatabaseQuery("DocType").execute())
def test_fields(self):
self.assertTrue({"name":"DocType", "issingle":0} \
in DatabaseQuery("DocType").execute(fields=["name", "issingle"]))

def test_filters_1(self):
self.assertFalse({"name":"DocType"} \
in DatabaseQuery("DocType").execute(filters=[["DocType", "name", "like", "J%"]]))
def test_filters_2(self):
self.assertFalse({"name":"DocType"} \
in DatabaseQuery("DocType").execute(filters=[{"name": ["like", "J%"]}]))

def test_filters_3(self):
self.assertFalse({"name":"DocType"} \
in DatabaseQuery("DocType").execute(filters={"name": ["like", "J%"]}))

def test_filters_4(self):
self.assertTrue({"name":"DocField"} \
in DatabaseQuery("DocType").execute(filters={"name": "DocField"}))

+ 1
- 2
frappe/website/doctype/blog_post/test_blog_post.py View File

@@ -69,8 +69,7 @@ class TestBlogPost(unittest.TestCase):
def test_restriction_in_report(self): def test_restriction_in_report(self):
frappe.defaults.add_default("Blog Category", "_Test Blog Category 1", "test1@example.com", frappe.defaults.add_default("Blog Category", "_Test Blog Category 1", "test1@example.com",
"Restriction") "Restriction")
frappe.local.reportview_doctypes = {}
names = [d.name for d in frappe.get_list("Blog Post", fields=["name", "blog_category"])] names = [d.name for d in frappe.get_list("Blog Post", fields=["name", "blog_category"])]


self.assertTrue("_test-blog-post-1" in names) self.assertTrue("_test-blog-post-1" in names)


+ 17
- 279
frappe/widgets/reportview.py View File

@@ -5,13 +5,19 @@ from __future__ import unicode_literals
"""build query for doclistview and return results""" """build query for doclistview and return results"""


import frappe, json import frappe, json
import frappe.defaults
import frappe.permissions import frappe.permissions
from frappe.model.db_query import DatabaseQuery


@frappe.whitelist() @frappe.whitelist()
def get(): def get():
return compress(execute(**get_form_params())) return compress(execute(**get_form_params()))


def execute(doctype, query=None, filters=None, fields=None, docstatus=None,
group_by=None, order_by=None, limit_start=0, limit_page_length=20,
as_list=False, with_childnames=False, debug=False):
return DatabaseQuery(doctype).execute(query, filters, fields, docstatus, group_by,
order_by, limit_start, limit_page_length, as_list, with_childnames, debug)

def get_form_params(): def get_form_params():
data = frappe._dict(frappe.local.form_dict) data = frappe._dict(frappe.local.form_dict)


@@ -25,63 +31,7 @@ def get_form_params():
data["docstatus"] = json.loads(data["docstatus"]) data["docstatus"] = json.loads(data["docstatus"])
return data return data
def execute(doctype, query=None, filters=None, fields=None, docstatus=None,
group_by=None, order_by=None, limit_start=0, limit_page_length=20,
as_list=False, with_childnames=False, debug=False):
"""
fields as list ["name", "owner"] or ["tabTask.name", "tabTask.owner"]
filters as list of list [["Task", "name", "=", "TASK00001"]]
"""

if query:
return run_custom_query(query)
if not filters:
filters = []
if isinstance(filters, basestring):
filters = json.loads(filters)
if not docstatus:
docstatus = []
if not fields:
fields = ["name"]
if isinstance(fields, basestring):
filters = json.loads(fields)

args = prepare_args(doctype, filters, fields, docstatus, group_by, order_by, with_childnames)
args.limit = add_limit(limit_start, limit_page_length)
query = """select %(fields)s from %(tables)s where %(conditions)s
%(group_by)s order by %(order_by)s %(limit)s""" % args
return frappe.db.sql(query, as_dict=not as_list, debug=debug)
def prepare_args(doctype, filters, fields, docstatus, group_by, order_by, with_childnames):
frappe.local.reportview_tables = get_tables(doctype, fields)
load_doctypes()
remove_user_tags(doctype, fields)
conditions = build_conditions(doctype, fields, filters, docstatus)
args = frappe._dict()
if with_childnames:
for t in frappe.local.reportview_tables:
if t != "`tab" + doctype + "`":
fields.append(t + ".name as '%s:name'" % t[4:-1])
# query dict
args.tables = ', '.join(frappe.local.reportview_tables)
args.conditions = ' and '.join(conditions)
args.fields = ', '.join(fields)
args.order_by = order_by or frappe.local.reportview_tables[0] + '.modified desc'
args.group_by = group_by and (" group by " + group_by) or ""

check_sort_by_table(args.order_by)
return args

def compress(data): def compress(data):
"""separate keys and values""" """separate keys and values"""
if not data: return data if not data: return data
@@ -97,214 +47,8 @@ def compress(data):
"keys": keys, "keys": keys,
"values": values "values": values
} }
def check_sort_by_table(sort_by):
"""check atleast 1 column selected from the sort by table """
if "." in sort_by:
tbl = sort_by.split('.')[0]
if tbl not in frappe.local.reportview_tables:
if tbl.startswith('`'):
tbl = tbl[4:-1]
frappe.msgprint("Please select atleast 1 column from '%s' to sort"\
% tbl, raise_exception=1)

def run_custom_query(query):
"""run custom query"""
if '%(key)s' in query:
query = query.replace('%(key)s', 'name')
return frappe.db.sql(query, as_dict=1)

def load_doctypes():
"""load all doctypes and roles"""
import frappe.model.doctype

if not getattr(frappe.local, "reportview_doctypes", None):
frappe.local.reportview_doctypes = {}

for t in frappe.local.reportview_tables:
if t.startswith('`'):
doctype = t[4:-1]
if frappe.local.reportview_doctypes.get(doctype):
continue
if not frappe.has_permission(doctype):
raise frappe.PermissionError, doctype
frappe.local.reportview_doctypes[doctype] = frappe.model.doctype.get(doctype)
def remove_user_tags(doctype, fields):
"""remove column _user_tags if not in table"""
columns = get_table_columns(doctype)
del_user_tags = False
del_comments = False
for fld in fields:
if '_user_tags' in fld and not "_user_tags" in columns:
del_user_tags = fld
if '_comments' in fld and not "_comments" in columns:
del_comments = fld

if del_user_tags: del fields[fields.index(del_user_tags)]
if del_comments: del fields[fields.index(del_comments)]

def add_limit(limit_start, limit_page_length):
if limit_page_length:
return 'limit %s, %s' % (limit_start, limit_page_length)
else:
return ''
def build_conditions(doctype, fields, filters, docstatus):
"""build conditions"""
if docstatus:
conditions = [frappe.local.reportview_tables[0] + '.docstatus in (' + ','.join(docstatus) + ')']
else:
# default condition
conditions = [frappe.local.reportview_tables[0] + '.docstatus < 2']
# make conditions from filters
build_filter_conditions(filters, conditions)
# join parent, child tables
for tname in frappe.local.reportview_tables[1:]:
conditions.append(tname + '.parent = ' + frappe.local.reportview_tables[0] + '.name')

# match conditions
match_conditions = build_match_conditions(doctype, fields)
if match_conditions:
conditions.append(match_conditions)
return conditions
def build_filter_conditions(filters, conditions):
"""build conditions from user filters"""
from frappe.utils import cstr, flt
if not getattr(frappe.local, "reportview_tables", None):
frappe.local.reportview_tables = []
doclist = {}
for f in filters:
if isinstance(f, basestring):
conditions.append(f)
else:
if not isinstance(f, (list, tuple)):
frappe.throw("Filter must be a tuple or list (in a list)")

if len(f) != 4:
frappe.throw("Filter must have 4 values (doctype, fieldname, condition, value): " + str(f))
tname = ('`tab' + f[0] + '`')
if not tname in frappe.local.reportview_tables:
frappe.local.reportview_tables.append(tname)
if not hasattr(frappe.local, "reportview_doctypes") \
or not frappe.local.reportview_doctypes.has_key(tname):
load_doctypes()
# prepare in condition
if f[2] in ['in', 'not in']:
opts = ["'" + t.strip().replace("'", "\\'") + "'" for t in f[3].split(',')]
f[3] = "(" + ', '.join(opts) + ")"
conditions.append('ifnull(' + tname + '.' + f[1] + ", '') " + f[2] + " " + f[3])
else:
df = frappe.local.reportview_doctypes[f[0]].get({"doctype": "DocField",
"fieldname": f[1]})
if f[2] == "like" or (isinstance(f[3], basestring) and
(not df or df[0].fieldtype not in ["Float", "Int", "Currency", "Percent"])):
value, default_val = ("'" + f[3].replace("'", "\\'") + "'"), '""'
else:
value, default_val = flt(f[3]), 0

conditions.append('ifnull({tname}.{fname}, {default_val}) {operator} {value}'.format(
tname=tname, fname=f[1], default_val=default_val, operator=f[2],
value=value))
def build_match_conditions(doctype, fields=None, as_condition=True):
"""add match conditions if applicable"""
import frappe.permissions
match_filters = {}
match_conditions = []
or_conditions = []

if not getattr(frappe.local, "reportview_tables", None):
frappe.local.reportview_tables = get_tables(doctype, fields)
load_doctypes()
# is restricted
restricted = frappe.permissions.get_user_perms(frappe.local.reportview_doctypes[doctype]).restricted
# get restrictions
restrictions = frappe.defaults.get_restrictions()
if restricted:
or_conditions.append('`tab{doctype}`.`owner`="{user}"'.format(doctype=doctype,
user=frappe.local.session.user))
match_filters["owner"] = frappe.session.user
if restrictions:
fields_to_check = frappe.local.reportview_doctypes[doctype].get_restricted_fields(restrictions.keys())
if doctype in restrictions:
fields_to_check.append(frappe._dict({"fieldname":"name", "options":doctype}))
# check in links
for df in fields_to_check:
if as_condition:
match_conditions.append('`tab{doctype}`.{fieldname} in ({values})'.format(doctype=doctype,
fieldname=df.fieldname,
values=", ".join([('"'+v.replace('"', '\"')+'"') \
for v in restrictions[df.options]])))
else:
match_filters.setdefault(df.fieldname, [])
match_filters[df.fieldname]= restrictions[df.options]
if as_condition:
conditions = " and ".join(match_conditions)
doctype_conditions = get_permission_query_conditions(doctype)
if doctype_conditions:
conditions += ' and ' + doctype_conditions if conditions else doctype_conditions
if or_conditions:
if conditions:
conditions = '({conditions}) or {or_conditions}'.format(conditions=conditions,
or_conditions = ' or '.join(or_conditions))
else:
conditions = " or ".join(or_conditions)
return conditions
else:
return match_filters
def get_permission_query_conditions(doctype):
condition_methods = frappe.get_hooks("permission_query_conditions:" + doctype)
if condition_methods:
conditions = []
for method in condition_methods:
c = frappe.get_attr(method)()
if c:
conditions.append(c)
return " and ".join(conditions) if conditions else None
def get_tables(doctype, fields):
"""extract tables from fields"""
tables = ['`tab' + doctype + '`']

# add tables from fields
if fields:
for f in fields:
if "." not in f: continue
table_name = f.split('.')[0]
if table_name.lower().startswith('group_concat('):
table_name = table_name[13:]
if table_name.lower().startswith('ifnull('):
table_name = table_name[7:]
if not table_name[0]=='`':
table_name = '`' + table_name + '`'
if not table_name in tables:
tables.append(table_name)

return tables

@frappe.whitelist() @frappe.whitelist()
def save_report(): def save_report():
"""save report""" """save report"""
@@ -359,13 +103,13 @@ def export_query():
f.seek(0) f.seek(0)
frappe.response['result'] = unicode(f.read(), 'utf-8') frappe.response['result'] = unicode(f.read(), 'utf-8')
frappe.response['type'] = 'csv' frappe.response['type'] = 'csv'
frappe.response['doctype'] = [t[4:-1] for t in frappe.local.reportview_tables][0]
frappe.response['doctype'] = [t[4:-1] for t in self.tables][0]


def get_labels(columns): def get_labels(columns):
"""get column labels based on column names""" """get column labels based on column names"""
label_dict = {} label_dict = {}
for doctype in frappe.local.reportview_doctypes:
for d in frappe.local.reportview_doctypes[doctype]:
for doctype in self.meta:
for d in self.meta[doctype]:
if d.doctype=='DocField' and d.fieldname: if d.doctype=='DocField' and d.fieldname:
label_dict[d.fieldname] = d.label label_dict[d.fieldname] = d.label
@@ -397,7 +141,7 @@ def get_stats(stats, doctype):
tags = json.loads(stats) tags = json.loads(stats)
stats = {} stats = {}
columns = get_table_columns(doctype)
columns = frappe.db.get_table_columns(doctype)
for tag in tags: for tag in tags:
if not tag in columns: continue if not tag in columns: continue
tagcount = execute(doctype, fields=[tag, "count(*)"], tagcount = execute(doctype, fields=[tag, "count(*)"],
@@ -429,16 +173,10 @@ def scrub_user_tags(tagcount):
return rlist return rlist


def get_table_columns(table):
res = frappe.db.sql("DESC `tab%s`" % table, as_dict=1)
if res: return [r['Field'] for r in res]

# used in building query in queries.py # used in building query in queries.py
def get_match_cond(doctype, searchfield = 'name'):
cond = build_match_conditions(doctype)

if cond:
cond = ' and ' + cond
else:
cond = ''
return cond
def get_match_cond(doctype):
cond = DatabaseQuery(doctype).build_match_conditions()
return (' and ' + cond) if cond else ""
def build_match_conditions(doctype, as_condition=True):
return DatabaseQuery(doctype).build_match_conditions(as_condition=as_condition)

Loading…
Cancel
Save