You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

525 rivejä
14 KiB

  1. # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
  2. # MIT License. See license.txt
  3. # util __init__.py
  4. from __future__ import unicode_literals
  5. from werkzeug.test import Client
  6. import os, re, urllib, sys, json, md5, requests, traceback
  7. import bleach, bleach_whitelist
  8. from html5lib.sanitizer import HTMLSanitizer
  9. from markdown2 import markdown as _markdown
  10. import frappe
  11. from frappe.utils.identicon import Identicon
  12. # utility functions like cint, int, flt, etc.
  13. from frappe.utils.data import *
  14. default_fields = ['doctype', 'name', 'owner', 'creation', 'modified', 'modified_by',
  15. 'parent', 'parentfield', 'parenttype', 'idx', 'docstatus']
  16. # used in import_docs.py
  17. # TODO: deprecate it
  18. def getCSVelement(v):
  19. """
  20. Returns the CSV value of `v`, For example:
  21. * apple becomes "apple"
  22. * hi"there becomes "hi""there"
  23. """
  24. v = cstr(v)
  25. if not v: return ''
  26. if (',' in v) or ('\n' in v) or ('"' in v):
  27. if '"' in v: v = v.replace('"', '""')
  28. return '"'+v+'"'
  29. else: return v or ''
  30. def get_fullname(user=None):
  31. """get the full name (first name + last name) of the user from User"""
  32. if not user:
  33. user = frappe.session.user
  34. if not hasattr(frappe.local, "fullnames"):
  35. frappe.local.fullnames = {}
  36. if not frappe.local.fullnames.get(user):
  37. p = frappe.db.get_value("User", user, ["first_name", "last_name"], as_dict=True)
  38. if p:
  39. frappe.local.fullnames[user] = " ".join(filter(None,
  40. [p.get('first_name'), p.get('last_name')])) or user
  41. else:
  42. frappe.local.fullnames[user] = user
  43. return frappe.local.fullnames.get(user)
  44. def get_formatted_email(user):
  45. """get email id of user formatted as: `John Doe <johndoe@example.com>`"""
  46. if user == "Administrator":
  47. return user
  48. from email.utils import formataddr
  49. fullname = get_fullname(user)
  50. return formataddr((fullname, user))
  51. def extract_email_id(email):
  52. """fetch only the email part of the email id"""
  53. from email.utils import parseaddr
  54. fullname, email_id = parseaddr(email)
  55. if isinstance(email_id, basestring) and not isinstance(email_id, unicode):
  56. email_id = email_id.decode("utf-8", "ignore")
  57. return email_id
  58. def validate_email_add(email_str, throw=False):
  59. """Validates the email string"""
  60. email_str = (email_str or "").strip()
  61. if not email_str:
  62. return False
  63. elif " " in email_str and "<" not in email_str:
  64. # example: "test@example.com test2@example.com" will return "test@example.comtest2" after parseaddr!!!
  65. return False
  66. email = extract_email_id(email_str.strip())
  67. match = re.match("[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?", email.lower())
  68. if not match:
  69. if throw:
  70. frappe.throw(frappe._("{0} is not a valid email id").format(email),
  71. frappe.InvalidEmailAddressError)
  72. else:
  73. return False
  74. matched = match.group(0)
  75. if match:
  76. match = matched==email.lower()
  77. if not match and throw:
  78. frappe.throw(frappe._("{0} is not a valid email id").format(email),
  79. frappe.InvalidEmailAddressError)
  80. return matched
  81. def split_emails(txt):
  82. email_list = []
  83. # emails can be separated by comma or newline
  84. for email in re.split('''[,\\n](?=(?:[^"]|"[^"]*")*$)''', cstr(txt)):
  85. email = strip(cstr(email))
  86. if email:
  87. email_list.append(email)
  88. return email_list
  89. def random_string(length):
  90. """generate a random string"""
  91. import string
  92. from random import choice
  93. return ''.join([choice(string.letters + string.digits) for i in range(length)])
  94. def has_gravatar(email):
  95. '''Returns gravatar url if user has set an avatar at gravatar.com'''
  96. if (frappe.flags.in_import
  97. or frappe.flags.in_install
  98. or frappe.flags.in_test):
  99. # no gravatar if via upload
  100. # since querying gravatar for every item will be slow
  101. return ''
  102. gravatar_url = "https://secure.gravatar.com/avatar/{hash}?d=404&s=200".format(hash=md5.md5(email).hexdigest())
  103. try:
  104. res = requests.get(gravatar_url)
  105. if res.status_code==200:
  106. return gravatar_url
  107. else:
  108. return ''
  109. except requests.exceptions.ConnectionError:
  110. return ''
  111. def get_gravatar_url(email):
  112. return "https://secure.gravatar.com/avatar/{hash}?d=mm&s=200".format(hash=md5.md5(email).hexdigest())
  113. def get_gravatar(email):
  114. gravatar_url = has_gravatar(email)
  115. if not gravatar_url:
  116. gravatar_url = Identicon(email).base64()
  117. return gravatar_url
  118. def get_traceback():
  119. """
  120. Returns the traceback of the Exception
  121. """
  122. exc_type, exc_value, exc_tb = sys.exc_info()
  123. trace_list = traceback.format_exception(exc_type, exc_value, exc_tb)
  124. body = "".join(cstr(t) for t in trace_list)
  125. return body
  126. def log(event, details):
  127. frappe.logger().info(details)
  128. def dict_to_str(args, sep='&'):
  129. """
  130. Converts a dictionary to URL
  131. """
  132. t = []
  133. for k in args.keys():
  134. t.append(str(k)+'='+urllib.quote(str(args[k] or '')))
  135. return sep.join(t)
  136. # Get Defaults
  137. # ==============================================================================
  138. def get_defaults(key=None):
  139. """
  140. Get dictionary of default values from the defaults, or a value if key is passed
  141. """
  142. return frappe.db.get_defaults(key)
  143. def set_default(key, val):
  144. """
  145. Set / add a default value to defaults`
  146. """
  147. return frappe.db.set_default(key, val)
  148. def remove_blanks(d):
  149. """
  150. Returns d with empty ('' or None) values stripped
  151. """
  152. empty_keys = []
  153. for key in d:
  154. if d[key]=='' or d[key]==None:
  155. # del d[key] raises runtime exception, using a workaround
  156. empty_keys.append(key)
  157. for key in empty_keys:
  158. del d[key]
  159. return d
  160. def strip_html_tags(text):
  161. """Remove html tags from text"""
  162. return re.sub("\<[^>]*\>", "", text)
  163. def get_file_timestamp(fn):
  164. """
  165. Returns timestamp of the given file
  166. """
  167. from frappe.utils import cint
  168. try:
  169. return str(cint(os.stat(fn).st_mtime))
  170. except OSError, e:
  171. if e.args[0]!=2:
  172. raise
  173. else:
  174. return None
  175. # to be deprecated
  176. def make_esc(esc_chars):
  177. """
  178. Function generator for Escaping special characters
  179. """
  180. return lambda s: ''.join(['\\' + c if c in esc_chars else c for c in s])
  181. # esc / unescape characters -- used for command line
  182. def esc(s, esc_chars):
  183. """
  184. Escape special characters
  185. """
  186. if not s:
  187. return ""
  188. for c in esc_chars:
  189. esc_str = '\\' + c
  190. s = s.replace(c, esc_str)
  191. return s
  192. def unesc(s, esc_chars):
  193. """
  194. UnEscape special characters
  195. """
  196. for c in esc_chars:
  197. esc_str = '\\' + c
  198. s = s.replace(esc_str, c)
  199. return s
  200. def execute_in_shell(cmd, verbose=0):
  201. # using Popen instead of os.system - as recommended by python docs
  202. from subprocess import Popen
  203. import tempfile
  204. with tempfile.TemporaryFile() as stdout:
  205. with tempfile.TemporaryFile() as stderr:
  206. p = Popen(cmd, shell=True, stdout=stdout, stderr=stderr)
  207. p.wait()
  208. stdout.seek(0)
  209. out = stdout.read()
  210. stderr.seek(0)
  211. err = stderr.read()
  212. if verbose:
  213. if err: print err
  214. if out: print out
  215. return err, out
  216. def get_path(*path, **kwargs):
  217. base = kwargs.get('base')
  218. if not base:
  219. base = frappe.local.site_path
  220. return os.path.join(base, *path)
  221. def get_site_base_path(sites_dir=None, hostname=None):
  222. return frappe.local.site_path
  223. def get_site_path(*path):
  224. return get_path(base=get_site_base_path(), *path)
  225. def get_files_path(*path, **kwargs):
  226. return get_site_path("private" if kwargs.get("is_private") else "public", "files", *path)
  227. def get_bench_path():
  228. return os.path.realpath(os.path.join(os.path.dirname(frappe.__file__), '..', '..', '..'))
  229. def get_backups_path():
  230. return get_site_path("private", "backups")
  231. def get_request_site_address(full_address=False):
  232. return get_url(full_address=full_address)
  233. def encode_dict(d, encoding="utf-8"):
  234. for key in d:
  235. if isinstance(d[key], basestring) and isinstance(d[key], unicode):
  236. d[key] = d[key].encode(encoding)
  237. return d
  238. def decode_dict(d, encoding="utf-8"):
  239. for key in d:
  240. if isinstance(d[key], basestring) and not isinstance(d[key], unicode):
  241. d[key] = d[key].decode(encoding, "ignore")
  242. return d
  243. def get_site_name(hostname):
  244. return hostname.split(':')[0]
  245. def get_disk_usage():
  246. """get disk usage of files folder"""
  247. files_path = get_files_path()
  248. if not os.path.exists(files_path):
  249. return 0
  250. err, out = execute_in_shell("du -hsm {files_path}".format(files_path=files_path))
  251. return cint(out.split("\n")[-2].split("\t")[0])
  252. def touch_file(path):
  253. with open(path, 'a'):
  254. os.utime(path, None)
  255. return True
  256. def get_test_client():
  257. from frappe.app import application
  258. return Client(application)
  259. def get_hook_method(hook_name, fallback=None):
  260. method = (frappe.get_hooks().get(hook_name))
  261. if method:
  262. method = frappe.get_attr(method[0])
  263. return method
  264. if fallback:
  265. return fallback
  266. def call_hook_method(hook, *args, **kwargs):
  267. out = None
  268. for method_name in frappe.get_hooks(hook):
  269. out = out or frappe.get_attr(method_name)(*args, **kwargs)
  270. return out
  271. def update_progress_bar(txt, i, l):
  272. if not getattr(frappe.local, 'request', None):
  273. lt = len(txt)
  274. if lt < 36:
  275. txt = txt + " "*(36-lt)
  276. complete = int(float(i+1) / l * 40)
  277. sys.stdout.write("\r{0}: [{1}{2}]".format(txt, "="*complete, " "*(40-complete)))
  278. sys.stdout.flush()
  279. def get_html_format(print_path):
  280. html_format = None
  281. if os.path.exists(print_path):
  282. with open(print_path, "r") as f:
  283. html_format = f.read()
  284. for include_directive, path in re.findall("""({% include ['"]([^'"]*)['"] %})""", html_format):
  285. for app_name in frappe.get_installed_apps():
  286. include_path = frappe.get_app_path(app_name, *path.split(os.path.sep))
  287. if os.path.exists(include_path):
  288. with open(include_path, "r") as f:
  289. html_format = html_format.replace(include_directive, f.read())
  290. break
  291. return html_format
  292. def is_markdown(text):
  293. if "<!-- markdown -->" in text:
  294. return True
  295. elif "<!-- html -->" in text:
  296. return False
  297. else:
  298. return not re.search("<p[\s]*>|<br[\s]*>", text)
  299. def get_sites(sites_path=None):
  300. import os
  301. if not sites_path:
  302. sites_path = getattr(frappe.local, 'sites_path', None) or '.'
  303. sites = []
  304. for site in os.listdir(sites_path):
  305. path = os.path.join(sites_path, site)
  306. if (os.path.isdir(path)
  307. and not os.path.islink(path)
  308. and os.path.exists(os.path.join(path, 'site_config.json'))):
  309. # is a dir and has site_config.json
  310. sites.append(site)
  311. return sites
  312. def get_request_session(max_retries=3):
  313. from requests.packages.urllib3.util import Retry
  314. session = requests.Session()
  315. session.mount("http://", requests.adapters.HTTPAdapter(max_retries=Retry(total=5, status_forcelist=[500])))
  316. session.mount("https://", requests.adapters.HTTPAdapter(max_retries=Retry(total=5, status_forcelist=[500])))
  317. return session
  318. def watch(path, handler=None, debug=True):
  319. import time
  320. from watchdog.observers import Observer
  321. from watchdog.events import FileSystemEventHandler
  322. class Handler(FileSystemEventHandler):
  323. def on_any_event(self, event):
  324. if debug:
  325. print "File {0}: {1}".format(event.event_type, event.src_path)
  326. if not handler:
  327. print "No handler specified"
  328. return
  329. handler(event.src_path, event.event_type)
  330. event_handler = Handler()
  331. observer = Observer()
  332. observer.schedule(event_handler, path, recursive=True)
  333. observer.start()
  334. try:
  335. while True:
  336. time.sleep(1)
  337. except KeyboardInterrupt:
  338. observer.stop()
  339. observer.join()
  340. def sanitize_html(html, linkify=False):
  341. """
  342. Sanitize HTML tags, attributes and style to prevent XSS attacks
  343. Based on bleach clean, bleach whitelist and HTML5lib's Sanitizer defaults
  344. Does not sanitize JSON, as it could lead to future problems
  345. """
  346. if not isinstance(html, basestring):
  347. return html
  348. elif is_json(html):
  349. return html
  350. whitelisted_tags = (HTMLSanitizer.acceptable_elements + HTMLSanitizer.svg_elements
  351. + ["html", "head", "meta", "link", "body", "iframe", "style", "o:p"])
  352. # retuns html with escaped tags, escaped orphan >, <, etc.
  353. escaped_html = bleach.clean(html,
  354. tags=whitelisted_tags,
  355. attributes={"*": HTMLSanitizer.acceptable_attributes, "svg": HTMLSanitizer.svg_attributes},
  356. styles=bleach_whitelist.all_styles,
  357. strip_comments=False)
  358. if linkify:
  359. escaped_html = bleach.linkify(escaped_html)
  360. return escaped_html
  361. def is_json(text):
  362. try:
  363. json.loads(text)
  364. except ValueError:
  365. return False
  366. else:
  367. return True
  368. def markdown(text, sanitize=True, linkify=True):
  369. html = _markdown(text)
  370. if sanitize:
  371. html = html.replace("<!-- markdown -->", "")
  372. html = sanitize_html(html, linkify=linkify)
  373. return html
  374. def sanitize_email(emails):
  375. from email.utils import parseaddr, formataddr
  376. sanitized = []
  377. for e in split_emails(emails):
  378. if not validate_email_add(e):
  379. continue
  380. fullname, email_id = parseaddr(e)
  381. sanitized.append(formataddr((fullname, email_id)))
  382. return ", ".join(sanitized)
  383. def get_site_info():
  384. from frappe.utils.user import get_system_managers
  385. from frappe.core.doctype.user.user import STANDARD_USERS
  386. from frappe.email.queue import get_emails_sent_this_month
  387. # only get system users
  388. users = frappe.get_all('User', filters={'user_type': 'System User', 'name': ('not in', STANDARD_USERS)},
  389. fields=['name', 'first_name', 'last_name', 'enabled',
  390. 'last_login', 'last_active', 'language', 'time_zone'])
  391. system_managers = get_system_managers(only_name=True)
  392. for u in users:
  393. # tag system managers
  394. u.is_system_manager = 1 if u.name in system_managers else 0
  395. u.full_name = get_fullname(u.name)
  396. system_settings = frappe.db.get_singles_dict('System Settings')
  397. space_usage = frappe._dict((frappe.local.conf.limits or {}).get('space_usage', {}))
  398. site_info = {
  399. 'users': users,
  400. 'country': system_settings.country,
  401. 'language': system_settings.language or 'english',
  402. 'time_zone': system_settings.time_zone,
  403. 'setup_complete': cint(system_settings.setup_complete),
  404. # usage
  405. 'emails_sent': get_emails_sent_this_month(),
  406. 'space_used': flt((space_usage.total or 0) / 1024.0, 2),
  407. 'database_size': space_usage.database_size,
  408. 'backup_size': space_usage.backup_size,
  409. 'files_size': space_usage.files_size
  410. }
  411. # from other apps
  412. for method_name in frappe.get_hooks('get_site_info'):
  413. site_info.update(frappe.get_attr(method_name)(site_info) or {})
  414. # dumps -> loads to prevent datatype conflicts
  415. return json.loads(frappe.as_json(site_info))