diff --git a/frappe/cli.py b/frappe/cli.py index cbe4a2a4c1..42a822474e 100755 --- a/frappe/cli.py +++ b/frappe/cli.py @@ -448,9 +448,9 @@ def build_sitemap(): @cmd def sync_statics(): - from frappe.website.doctype.web_page import web_page + from frappe.website import statics frappe.connect() - web_page.sync_statics() + statics.sync_statics() frappe.conn.commit() frappe.destroy() diff --git a/frappe/website/doctype/web_page/web_page.py b/frappe/website/doctype/web_page/web_page.py index f4ae972dd2..6bd7cd98e5 100644 --- a/frappe/website/doctype/web_page/web_page.py +++ b/frappe/website/doctype/web_page/web_page.py @@ -4,9 +4,6 @@ from __future__ import unicode_literals import frappe, os, time from frappe.website.website_generator import WebsiteGenerator -from frappe import _ -from frappe.utils import cint -from markdown2 import markdown class DocType(WebsiteGenerator): def validate(self): @@ -36,149 +33,3 @@ class DocType(WebsiteGenerator): from frappe.website.render import clear_cache clear_cache() -def sync_statics(): - while True: - _sync_statics() - frappe.conn.commit() - print "sync complete" - time.sleep(2) - -def _sync_statics(): - synced = [] - to_insert = [] - - def sync_file(fname, fpath, statics_path, priority=0): - url = os.path.relpath(fpath, statics_path).rsplit(".", 1)[0] - - if fname.rsplit(".", 1)[0]=="index" and os.path.dirname(fpath) != statics_path: - url = os.path.dirname(url) - - if url in synced: - return - - parent_website_route = os.path.dirname(url) - page_name = os.path.basename(url) - - try: - sitemap = frappe.bean("Website Route", url) - - except frappe.DoesNotExistError: - title, content = get_static_content(fpath) - if not title: - title = page_name.replace("-", " ").replace("_", " ").title() - page = frappe.bean({ - "doctype":"Web Page", - "idx": priority, - "title": title, - "page_name": page_name, - "main_section": content, - "published": 1, - "parent_website_route": parent_website_route - }) - - page.insert() - - # update timestamp - sitemap = frappe.bean("Website Route", {"ref_doctype": "Web Page", - "docname": page.doc.name}) - sitemap.doc.static_file_timestamp = os.path.getmtime(fpath) - sitemap.save() - - synced.append(url) - else: - if str(os.path.getmtime(fpath))!=sitemap.doc.static_file_timestamp \ - or cint(sitemap.doc.idx) != cint(priority): - - page = frappe.bean("Web Page", sitemap.doc.docname) - title, content = get_static_content(fpath) - page.doc.main_section = content - page.doc.idx = priority - if not title: - title = page_name.replace("-", " ").replace("_", " ").title() - page.doc.title = title - page.save() - - sitemap = frappe.bean("Website Route", url) - sitemap.doc.static_file_timestamp = os.path.getmtime(fpath) - sitemap.save() - - synced.append(url) - - for app in frappe.get_installed_apps(): - statics_path = frappe.get_app_path(app, "templates", "statics") - - if os.path.exists(frappe.get_app_path(app, "templates", "statics")): - for basepath, folders, files in os.walk(statics_path): - # index file first! - index = [] - has_index = False - if "index.txt" in files: - with open(os.path.join(basepath, "index.txt"), "r") as indexfile: - index = indexfile.read().splitlines() - - if basepath!=statics_path: - for fname in files: - page_name = fname.rsplit(".", 1)[0] - if page_name=="index" and fname!="index.txt": - sync_file(fname, os.path.join(basepath, fname), statics_path) - has_index = True - break - - if not has_index: - continue - - if index: - # index.txt given - for i, page_name in enumerate(index): - if page_name in folders: - # for folder, sync inner index first (so that idx is set) - for extn in ("md", "html"): - path = os.path.join(basepath, page_name, "index." + extn) - if os.path.exists(path): - sync_file("index." + extn, path, statics_path, i) - break - - # other files - for extn in ("md", "html"): - path = os.path.join(basepath, page_name + "." + extn) - if page_name + "." + extn in files: - sync_file(page_name + "." + extn, path, statics_path, i) - break - - else: - for fname in files: - page_name = fname.rsplit(".", 1)[0] - if not (page_name=="index" and basepath!=statics_path): - sync_file(fname, os.path.join(basepath, fname), statics_path, None) - - # delete not synced - if synced: - frappe.delete_doc("Web Page", frappe.conn.sql_list("""select docname from `tabWebsite Route` - where ifnull(static_file_timestamp,'')!='' and name not in ({}) - order by (rgt-lft) asc""".format(', '.join(["%s"]*len(synced))), tuple(synced))) - else: - frappe.delete_doc("Web Page", frappe.conn.sql_list("""select docname from `tabWebsite Route` - where ifnull(static_file_timestamp,'')!='' order by (rgt-lft) asc""")) - - -def get_static_content(fpath): - with open(fpath, "r") as contentfile: - title = None - content = unicode(contentfile.read(), 'utf-8') - - if fpath.endswith(".md"): - if content: - lines = content.splitlines() - first_line = lines[0].strip() - - if first_line.startswith("# "): - title = first_line[2:] - content = "\n".join(lines[1:]) - - content = markdown(content) - - content = unicode(content.encode("utf-8"), 'utf-8') - - return title, content - - diff --git a/frappe/website/statics.py b/frappe/website/statics.py new file mode 100644 index 0000000000..f4cb082f98 --- /dev/null +++ b/frappe/website/statics.py @@ -0,0 +1,199 @@ +# Copyright (c) 2013, Web Notes Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals +import frappe, os, time + +from frappe import _ +from frappe.utils import cint +from markdown2 import markdown + +def sync_statics(): + s = sync() + while True: + s.start() + frappe.conn.commit() + time.sleep(2) + +class sync(object): + def __init__(self): + self.synced = [] + + def start(self): + for app in frappe.get_installed_apps(): + self.sync_for_app(app) + + self.cleanup() + print "sync complete" + + def sync_for_app(self, app): + self.statics_path = frappe.get_app_path(app, "templates", "statics") + if os.path.exists(self.statics_path): + for basepath, folders, files in os.walk(self.statics_path): + self.sync_folder(basepath, folders, files) + + + def sync_folder(self, basepath, folders, files): + self.has_index = False + self.get_index_txt(basepath, files) + + if basepath!=self.statics_path: + self.sync_index_page(basepath, files) + + if not self.has_index: # index.md or index.html is required, else skip + return + + if self.index: + self.sync_using_given_index(basepath, folders, files) + + else: + self.sync_alphabetically(basepath, folders, files) + + + def get_index_txt(self, basepath, files): + self.index = [] + if "index.txt" in files: + with open(os.path.join(basepath, "index.txt"), "r") as indexfile: + self.index = indexfile.read().splitlines() + + def sync_index_page(self, basepath, files): + for extn in ("md", "html"): + fname = "index." + extn + if fname in files: + self.sync_file(fname, os.path.join(basepath, fname), None) + self.has_index = True + return + + def sync_using_given_index(self, basepath, folders, files): + for i, page_name in enumerate(self.index): + if page_name in folders: + # for folder, sync inner index first (so that idx is set) + for extn in ("md", "html"): + path = os.path.join(basepath, page_name, "index." + extn) + if os.path.exists(path): + self.sync_file("index." + extn, path, i) + break + + # other files + for extn in ("md", "html"): + fname = page_name + "." + extn + if fname in files: + self.sync_file(fname, os.path.join(basepath, fname), i) + break + elif page_name not in folders: + print page_name + " not found in " + basepath + + def sync_alphabetically(self, basepath, folders, files): + files.sort() + for fname in files: + page_name = fname.rsplit(".", 1)[0] + if not (page_name=="index" and basepath!=self.statics_path): + self.sync_file(fname, os.path.join(basepath, fname), None) + + def sync_file(self, fname, fpath, priority): + route = os.path.relpath(fpath, self.statics_path).rsplit(".", 1)[0] + + if fname.rsplit(".", 1)[0]=="index" and os.path.dirname(fpath) != self.statics_path: + route = os.path.dirname(route) + + if route in self.synced: + return + + parent_website_route = os.path.dirname(route) + page_name = os.path.basename(route) + + try: + route_bean = frappe.bean("Website Route", route) + except frappe.DoesNotExistError: + # Route does not exist, new page + self.insert_web_page(route, fpath, page_name, priority, parent_website_route) + else: + self.update_web_page(route, route_bean, fpath, priority) + return None + + def insert_web_page(self, route, fpath, page_name, priority, parent_website_route): + title, content = get_static_content(fpath) + if not title: + title = page_name.replace("-", " ").replace("_", " ").title() + page = frappe.bean({ + "doctype":"Web Page", + "idx": priority, + "title": title, + "page_name": page_name, + "main_section": content, + "published": 1, + "parent_website_route": parent_website_route + }) + + try: + page.insert() + except NameError: + # page exists, if deleted static, delete it and try again + old_route = frappe.doc("Website Route", {"ref_doctype":"Web Page", + "docname": page.doc.name}) + if old_route.static_file_timestamp and not os.path.exists(os.path.join(self.statics_path, + old_route.name)): + + frappe.delete_doc("Web Page", page.doc.name) + page.insert() # retry + + # update timestamp + route_bean = frappe.bean("Website Route", {"ref_doctype": "Web Page", + "docname": page.doc.name}) + route_bean.doc.static_file_timestamp = os.path.getmtime(fpath) + route_bean.save() + + self.synced.append(route) + + def update_web_page(self, route, route_bean, fpath, priority): + if str(os.path.getmtime(fpath))!=route_bean.doc.static_file_timestamp \ + or cint(route_bean.doc.idx) != cint(priority): + + page = frappe.bean("Web Page", route_bean.doc.docname) + title, content = get_static_content(fpath) + page.doc.main_section = content + page.doc.idx = priority + if not title: + title = route_bean.doc.docname.replace("-", " ").replace("_", " ").title() + page.doc.title = title + page.save() + + route_bean = frappe.bean("Website Route", route) + route_bean.doc.static_file_timestamp = os.path.getmtime(fpath) + route_bean.save() + + self.synced.append(route) + + def cleanup(self): + if self.synced: + frappe.delete_doc("Web Page", frappe.conn.sql_list("""select docname + from `tabWebsite Route` + where ifnull(static_file_timestamp,'')!='' and name not in ({}) + order by (rgt-lft) asc""".format(', '.join(["%s"]*len(self.synced))), + tuple(self.synced))) + else: + frappe.delete_doc("Web Page", frappe.conn.sql_list("""select docname + from `tabWebsite Route` + where ifnull(static_file_timestamp,'')!='' + order by (rgt-lft) asc""")) + + +def get_static_content(fpath): + with open(fpath, "r") as contentfile: + title = None + content = unicode(contentfile.read(), 'utf-8') + + if fpath.endswith(".md"): + if content: + lines = content.splitlines() + first_line = lines[0].strip() + + if first_line.startswith("# "): + title = first_line[2:] + content = "\n".join(lines[1:]) + + content = markdown(content) + + content = unicode(content.encode("utf-8"), 'utf-8') + + return title, content