From da36fac49337f5f057a2b99c43a3a6ad11b9c02c Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Mon, 18 Nov 2013 19:27:17 +0530 Subject: [PATCH] [website] [sitemap] cleanup --- public/js/wn/ui/appframe.js | 6 +- public/js/wn/views/doclistview.js | 2 +- webnotes/build.py | 13 +- webnotes/db.py | 2 + webnotes/webutils.py | 278 ++++++++++++++----------- website/doctype/blog_post/blog_post.py | 8 +- website/doctype/web_page/web_page.py | 10 +- website/templates/pages/sitemap.py | 12 +- wnf.py | 9 + 9 files changed, 190 insertions(+), 150 deletions(-) diff --git a/public/js/wn/ui/appframe.js b/public/js/wn/ui/appframe.js index 872e20e25b..7978e87d45 100644 --- a/public/js/wn/ui/appframe.js +++ b/public/js/wn/ui/appframe.js @@ -63,7 +63,7 @@ wn.ui.AppFrame = Class.extend({ this.title_right = $("") .html((icon ? ' ' : "") + txt) .click(click) - .prependTo($right); + .appendTo($right); return this.title_right; } else { $right.empty(); @@ -79,8 +79,8 @@ wn.ui.AppFrame = Class.extend({ var $right = this.parent.find(".titlebar-item.text-right"); this.primary_action = $("") .html(wn._("Actions") + " ") - .css({"margin-left":"15px", "display":"inline-block"}) - .appendTo($right); + .css({"margin-right":"15px", "display":"inline-block"}) + .prependTo($right); } var id = "dropdown-" + wn.dom.set_unique_id(); diff --git a/public/js/wn/views/doclistview.js b/public/js/wn/views/doclistview.js index c036dd329b..236045f824 100644 --- a/public/js/wn/views/doclistview.js +++ b/public/js/wn/views/doclistview.js @@ -64,7 +64,7 @@ wn.views.DocListView = wn.ui.Listing.extend({ this.$page.find(".layout-main-section") .css({"border-right":"1px solid #d7d7d7"}) - .parent().css({"margin-top":"-20px"}); + .parent().css({"margin-top":"-15px"}); this.appframe = this.page.appframe; var module = locals.DocType[this.doctype].module; diff --git a/webnotes/build.py b/webnotes/build.py index 5fedc68e17..a93b2d28fb 100644 --- a/webnotes/build.py +++ b/webnotes/build.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from webnotes.utils.minify import JavascriptMinify +from webnotes.webutils import build_website_sitemap_config """ Build the `public` folders and setup languages @@ -27,9 +28,7 @@ def bundle(no_compress, cms_make=True): on_build() except ImportError, e: pass - - clear_pyc_files() - + def watch(no_compress): """watch and rebuild if necessary""" import time @@ -50,14 +49,6 @@ def check_lang(): from webnotes.translate import update_translations update_translations() -def clear_pyc_files(): - from webnotes.utils import get_base_path - for path, folders, files in os.walk(get_base_path()): - if 'locale' in folders: folders.remove('locale') - for f in files: - if f.decode("utf-8").endswith(".pyc"): - os.remove(os.path.join(path, f)) - class Bundle: """ Concatenate, compress and mix (if required) js+css files from build.json diff --git a/webnotes/db.py b/webnotes/db.py index 60a581e81a..beb67af346 100644 --- a/webnotes/db.py +++ b/webnotes/db.py @@ -357,6 +357,8 @@ class Database: fl = ", ".join(fields) else: fl = fields + if fields=="*": + as_dict = True conditions, filters = self.build_conditions(filters) diff --git a/webnotes/webutils.py b/webnotes/webutils.py index 2d61a006d4..bb0616c59b 100644 --- a/webnotes/webutils.py +++ b/webnotes/webutils.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals from webnotes import conf import webnotes -import json +import json, os, time from webnotes import _ import webnotes.utils import mimetypes @@ -27,7 +27,8 @@ def render_page(page_name): """get page html""" set_content_type(page_name) - page_name = scrub_page_name(page_name) + if page_name.endswith('.html'): + page_name = page_name[:-5] html = '' if not conf.auto_cache_clear: @@ -61,38 +62,25 @@ def build_page(page_name): if not webnotes.conn: webnotes.connect() - sitemap = get_website_sitemap() - page_options = sitemap.get(page_name) + + sitemap_options = webnotes.doc("Website Sitemap", page_name).fields + page_options = webnotes.doc("Website Sitemap Config", + sitemap_options.get("website_sitemap_config")).fields.update(sitemap_options) + if not page_options: - if page_name=="index": - # page not found, try home page - home_page = get_home_page() - page_options = sitemap.get(home_page) - if not page_options: - raise PageNotFoundError - page_options["page_name"] = home_page - else: - raise PageNotFoundError + raise PageNotFoundError else: page_options["page_name"] = page_name basepath = webnotes.utils.get_base_path() - module = None - no_cache = False + no_cache = page_options.get("no_cache") - if page_options.get("controller"): - module = webnotes.get_module(page_options["controller"]) - no_cache = getattr(module, "no_cache", False) # if generator, then load bean, pass arguments - if page_options.get("is_generator"): - if not module: - raise Exception("Generator controller not defined") - - name = webnotes.conn.get_value(module.doctype, { - page_options.get("page_name_field", "page_name"): page_options["page_name"]}) - obj = webnotes.get_obj(module.doctype, name, with_children=True) + if page_options.get("page_or_generator")=="Generator": + doctype = page_options.get("ref_doctype") + obj = webnotes.get_obj(doctype, page_options["docname"], with_children=True) if hasattr(obj, 'get_context'): obj.get_context() @@ -102,58 +90,54 @@ def build_page(page_name): else: # page context = webnotes._dict({ 'name': page_name }) - if module and hasattr(module, "get_context"): - context.update(module.get_context()) + if page_options.get("controller"): + module = webnotes.get_module(page_options.get("controller")) + if module and hasattr(module, "get_context"): + context.update(module.get_context()) context.update(get_website_settings()) jenv = webnotes.get_jenv() context["base_template"] = jenv.get_template(webnotes.get_config().get("base_template")) - template_name = page_options['template'] + template_name = page_options['template_path'] html = jenv.get_template(template_name).render(context) if not no_cache: webnotes.cache().set_value("page:" + page_name, html) return html -def build_sitemap(): - sitemap = {} - config = webnotes.cache().get_value("website_sitemap_config", build_website_sitemap_config) - sitemap.update(config["pages"]) - - # pages - for p in config["pages"].values(): - if p.get("controller"): - module = webnotes.get_module(p["controller"]) - p["no_cache"] = getattr(module, "no_cache", False) - p["no_sitemap"] = getattr(module, "no_sitemap", False) or p["no_cache"] - - # generators - for g in config["generators"].values(): - g["is_generator"] = True - module = webnotes.get_module(g["controller"]) - - condition = "" - page_name_field = "page_name" - if hasattr(module, "page_name_field"): - page_name_field = module.page_name_field - if hasattr(module, "condition_field"): - condition = " where ifnull(%s, 0)=1" % module.condition_field - - for page_name, name, modified in webnotes.conn.sql("""select %s, name, modified from - `tab%s` %s""" % (page_name_field, module.doctype, condition)): - opts = g.copy() - opts["doctype"] = module.doctype - opts["no_cache"] = getattr(module, "no_cache", False) - opts["page_name"] = page_name - if page_name_field != "page_name": - opts["page_name_field"] = page_name_field - opts["docname"] = name - opts["lastmod"] = modified.strftime("%Y-%m-%d %H:%M:%S") - sitemap[page_name] = opts - - return sitemap +def build_sitemap(): + webnotes.conn.sql("""delete from `tabWebsite Sitemap Config`""") + webnotes.conn.sql("""delete from `tabWebsite Sitemap`""") + webnotes.conn.commit() + build_website_sitemap_config() + + for config in webnotes.conn.sql("""select * from `tabWebsite Sitemap Config`""", as_dict=True): + if config.page_or_generator == "Page": + config.page_name = config.link_name + add_to_sitemap(config) + else: + module = webnotes.get_module(config.controller) + + condition = "" + if hasattr(module, "condition_field"): + condition = " where ifnull(%s, 0)=1" % module.condition_field + + page_name_field = getattr(module, "page_name_field", "page_name") + + for name in webnotes.conn.sql_list("""select name from `tab%s` %s""" \ + % (module.doctype, condition)): + webnotes.bean(module.doctype, name).run_method("on_update") + +def add_to_sitemap(options): + doc = webnotes.doc({"doctype":"Website Sitemap"}) + for key in ("page_name", "ref_doctype", "docname", "page_or_generator", "lastmod"): + doc.fields[key] = options.get(key) + doc.name = options.page_name + doc.website_sitemap_config = options.link_name + doc.insert() + webnotes.conn.commit() def get_home_page(): if not webnotes.conn: @@ -166,49 +150,64 @@ def get_home_page(): return page_name -def build_website_sitemap_config(): - import os, time - +def build_website_sitemap_config(): config = {"pages": {}, "generators":{}} basepath = webnotes.utils.get_base_path() - - def get_options(path, fname): - name = fname - if fname.endswith(".html"): - name = fname[:-5] - template_path = os.path.relpath(os.path.join(path, fname), basepath) - - options = webnotes._dict({ - "link_name": name, - "template": template_path, - "lastmod": time.ctime(os.path.getmtime(template_path)) - }) - - controller_name = fname.split(".")[0].replace("-", "_") + ".py" - controller_path = os.path.join(path, controller_name) - if os.path.exists(controller_path): - options.controller = os.path.relpath(controller_path[:-3], basepath).replace(os.path.sep, ".") - options.controller = ".".join(options.controller.split(".")[1:]) - - return options - for path, folders, files in os.walk(basepath, followlinks=True): - if 'locale' in folders: folders.remove('locale') + if 'locale' in folders: + folders.remove('locale') + + # utility - remove pyc files + for f in files: + if f.decode("utf-8").endswith(".pyc"): + os.remove(os.path.join(path, f)) + if os.path.basename(path)=="pages" and os.path.basename(os.path.dirname(path))=="templates": for fname in files: fname = webnotes.utils.cstr(fname) if fname.split(".")[-1] in ("html", "xml", "js", "css"): - options = get_options(path, fname) - config["pages"][options.link_name] = options + add_website_sitemap_config("Page", path, fname) if os.path.basename(path)=="generators" and os.path.basename(os.path.dirname(path))=="templates": for fname in files: if fname.endswith(".html"): - options = get_options(path, fname) - config["generators"][fname] = options + add_website_sitemap_config("Generator", path, fname) + + webnotes.conn.commit() + +def add_website_sitemap_config(page_or_generator, path, fname): + basepath = webnotes.utils.get_base_path() + name = fname + if fname.endswith(".html"): + name = fname[:-5] + + template_path = os.path.relpath(os.path.join(path, fname), basepath) + + options = webnotes._dict({ + "doctype": "Website Sitemap Config", + "page_or_generator": page_or_generator, + "link_name": name, + "template_path": template_path, + "lastmod": time.ctime(os.path.getmtime(template_path)) + }) + + controller_name = fname.split(".")[0].replace("-", "_") + ".py" + controller_path = os.path.join(path, controller_name) + if os.path.exists(controller_path): + options.controller = os.path.relpath(controller_path[:-3], basepath).replace(os.path.sep, ".") + options.controller = ".".join(options.controller.split(".")[1:]) + + if options.controller: + module = webnotes.get_module(options.controller) + options.no_cache = getattr(module, "no_cache", 0) + options.no_sitemap = options.no_cache or getattr(module, "no_sitemap", 0) + options.ref_doctype = getattr(module, "doctype", None) - return config + webnotes.doc(options).insert() + + return options + def get_website_settings(): from webnotes.utils import get_request_site_address, encode, cint @@ -276,20 +275,13 @@ def clear_cache(page_name=None): delete_page_cache(page_name) else: cache = webnotes.cache() - for p in get_all_pages(): + for p in webnotes.conn.sql_list("""select name from `tabWebsite Sitemap`"""): if p is not None: cache.delete_value("page:" + p) cache.delete_value("page:index") cache.delete_value("website_sitemap") cache.delete_value("website_sitemap_config") - -def get_website_sitemap(): - return webnotes.cache().get_value("website_sitemap", build_sitemap) - -def get_all_pages(): - return get_website_sitemap().keys() - def delete_page_cache(page_name): if page_name: cache = webnotes.cache() @@ -321,12 +313,6 @@ def get_hex_shade(color, percent): return p(r) + p(g) + p(b) -def scrub_page_name(page_name): - if page_name.endswith('.html'): - page_name = page_name[:-5] - - return page_name - def is_signup_enabled(): if getattr(webnotes.local, "is_signup_enabled", None) is None: webnotes.local.is_signup_enabled = True @@ -336,21 +322,71 @@ def is_signup_enabled(): return webnotes.local.is_signup_enabled -def update_page_name(doc, title): - """set page_name and check if it is unique""" - new_page_name = page_name(title) - sitemap = get_website_sitemap() +class WebsiteGenerator(object): + def setup_generator(self): + self._website_config = webnotes.conn.get_values("Website Sitemap Config", {"ref_doctype": self.doc.doctype}, "*")[0] + self._website_module = webnotes.get_module(self._website_config.controller) + + self._page_name_field = getattr(self._website_module, "page_name_field", "page_name") + self._condition_field = getattr(self._website_module, "condition_field", "") + + def on_update(self, page_name=None): + self.setup_generator() + if self._condition_field: + if not self.doc.fields[self._condition_field]: + remove_page(self.doc.fields[self._page_name_field]) + return + + if not page_name: + new_page_name = cleanup_page_name(self.get_page_title() \ + if hasattr(self, "get_page_title") else (self.doc.title or self.doc.name)) + self.check_if_page_name_is_unique(new_page_name) - if new_page_name in sitemap and \ - not (sitemap[new_page_name].doctype == doc.doctype and sitemap[new_page_name].docname == doc.name): - webnotes.throw("%s: %s. %s: %s" % (new_page_name, _("Page already exists"), - _("Please change the value"), title)) + remove_page(self.doc.page_name) + add_generator_to_sitemap(self.doc.doctype, self.doc.name, new_page_name, self.doc.modified, + self._website_config, self._website_module) + + if self.doc.fields[self._page_name_field]!=new_page_name: + webnotes.conn.set(self.doc, self._page_name_field, new_page_name) + else: + add_generator_to_sitemap(self.doc.doctype, self.doc.name, page_name, self.doc.modified, + self._website_config, self._website_module) + + delete_page_cache(self.doc.page_name) + + def check_if_page_name_is_unique(self, new_page_name): + if webnotes.conn.sql("""select name from `tabWebsite Sitemap` where name=%s + and ref_doctype!=%s and docname!=%s""", (new_page_name, self.doc.doctype, self.doc.name)): + webnotes.throw("%s: %s. %s: %s" % (new_page_name, _("Page already exists"), + _("Please change the value"), title)) + + def on_trash(self): + self.setup_generator() + remove_page(self.doc.fields[self._page_name_field]) + +def add_generator_to_sitemap(ref_doctype, docname, page_name, modified, config=None, module=None): + if not config: + config = webnotes.conn.get_values("Website Sitemap Config", {"ref_doctype": ref_doctype}, "*")[0] + if not module: + module = webnotes.get_module(config.controller) - if doc.page_name: delete_page_cache(doc.page_name) - webnotes.conn.set(doc, "page_name", new_page_name) - delete_page_cache(doc.page_name) + page_name_field = getattr(module, "page_name_field", "page_name") + + opts = config.copy() + opts["page_name"] = page_name + if page_name_field != "page_name": + opts["page_name_field"] = page_name_field + opts["docname"] = docname + opts["lastmod"] = modified + add_to_sitemap(opts) + + +def remove_page(page_name): + if page_name: + delete_page_cache(page_name) + webnotes.conn.sql("delete from `tabWebsite Sitemap` where name=%s", page_name) -def page_name(title): +def cleanup_page_name(title): """make page name from title""" import re name = title.lower() diff --git a/website/doctype/blog_post/blog_post.py b/website/doctype/blog_post/blog_post.py index deda9f9f60..addfafc341 100644 --- a/website/doctype/blog_post/blog_post.py +++ b/website/doctype/blog_post/blog_post.py @@ -5,16 +5,16 @@ from __future__ import unicode_literals import webnotes import webnotes.webutils +from webnotes.webutils import WebsiteGenerator, cleanup_page_name from webnotes import _ from webnotes.utils import today -class DocType: +class DocType(WebsiteGenerator): def __init__(self, d, dl): self.doc, self.doclist = d, dl def autoname(self): - from webnotes.webutils import page_name - self.doc.name = page_name(self.doc.title) + self.doc.name = cleanup_page_name(self.doc.title) def validate(self): if self.doc.blog_intro: @@ -29,7 +29,7 @@ class DocType: where name=%s""", self.doc.blogger) def on_update(self): - webnotes.webutils.update_page_name(self.doc, self.doc.title) + WebsiteGenerator.on_update(self) webnotes.webutils.delete_page_cache("writers") def get_context(self): diff --git a/website/doctype/web_page/web_page.py b/website/doctype/web_page/web_page.py index d070af9fc1..5e5d59a4fe 100644 --- a/website/doctype/web_page/web_page.py +++ b/website/doctype/web_page/web_page.py @@ -3,8 +3,9 @@ from __future__ import unicode_literals import webnotes +from webnotes.webutils import WebsiteGenerator -class DocType(): +class DocType(WebsiteGenerator): def __init__(self, d, dl): self.doc, self.doclist = d, dl @@ -13,8 +14,7 @@ class DocType(): self.doc.name = page_name(self.doc.title) def on_update(self): - from webnotes.webutils import update_page_name - update_page_name(self.doc, self.doc.title) + WebsiteGenerator.on_update(self) self.if_home_clear_cache() # clear all cache if it has toc @@ -24,6 +24,8 @@ class DocType(): def on_trash(self): # delete entry from Table of Contents of other pages + WebsiteGenerator.on_trash(self) + webnotes.conn.sql("""delete from `tabTable of Contents` where web_page=%s""", self.doc.name) @@ -35,6 +37,8 @@ class DocType(): def if_home_clear_cache(self): """if home page, clear cache""" if webnotes.conn.get_value("Website Settings", None, "home_page")==self.doc.name: + WebsiteGenerator.on_update(self, page_name="index") + from webnotes.sessions import clear_cache clear_cache('Guest') diff --git a/website/templates/pages/sitemap.py b/website/templates/pages/sitemap.py index b2f445567c..784cc5c419 100644 --- a/website/templates/pages/sitemap.py +++ b/website/templates/pages/sitemap.py @@ -10,13 +10,11 @@ from webnotes.utils import get_request_site_address def get_context(): """generate the sitemap XML""" - links = webnotes.webutils.get_website_sitemap().items() host = get_request_site_address() + links = [] + for l in webnotes.conn.sql("""select * from `tabWebsite Sitemap` where ifnull(no_sitemap, 0)=0""", + as_dict=True): + links.append({"loc": urllib.basejoin(host, urllib.quote(l.page_name.encode("utf-8")))}) - for l in links: - l[1]["loc"] = urllib.basejoin(host, urllib.quote(l[1].get("page_name", l[1]["link_name"]).encode("utf-8"))) - - return { - "links": [l[1] for l in links if not l[1].get("no_sitemap")] - } + return links \ No newline at end of file diff --git a/wnf.py b/wnf.py index 56f165ab52..000439a1ad 100755 --- a/wnf.py +++ b/wnf.py @@ -150,6 +150,8 @@ def setup_utilities(parser): # clear parser.add_argument("--clear_web", default=False, action="store_true", help="Clear website cache") + parser.add_argument("--build_sitemap", default=False, action="store_true", + help="Build Website Sitemap") parser.add_argument("--clear_cache", default=False, action="store_true", help="Clear cache, doctype cache and defaults") parser.add_argument("--reset_perms", default=False, action="store_true", @@ -419,6 +421,13 @@ def clear_web(site=None): webnotes.webutils.clear_cache() webnotes.destroy() +@cmd +def build_sitemap(site=None): + import webnotes.webutils + webnotes.connect(site=site) + webnotes.webutils.build_sitemap() + webnotes.destroy() + @cmd def reset_perms(site=None): webnotes.connect(site=site)