[website] [sitemap] cleanup

public/js/wn/ui/appframe.js

@@ -63,7 +63,7 @@ wn.ui.AppFrame = Class.extend({
 			this.title_right = $("<a>")
 				.html((icon ? '<i class="'+icon+'"></i> ' : "") + txt)
 				.click(click)
-				.prependTo($right);
+				.appendTo($right);
 			return this.title_right;
 		} else {
 			$right.empty();
@@ -79,8 +79,8 @@ wn.ui.AppFrame = Class.extend({
 				var $right = this.parent.find(".titlebar-item.text-right");
 				this.primary_action = $("<a>")
 					.html(wn._("Actions") + " <i class='icon-caret-down'></i>")
-					.css({"margin-left":"15px", "display":"inline-block"})
-					.appendTo($right);
+					.css({"margin-right":"15px", "display":"inline-block"})
+					.prependTo($right);
 			}
 			
 			var id = "dropdown-" + wn.dom.set_unique_id();

public/js/wn/views/doclistview.js

@@ -64,7 +64,7 @@ wn.views.DocListView = wn.ui.Listing.extend({
 		
 		this.$page.find(".layout-main-section")
 			.css({"border-right":"1px solid #d7d7d7"})
-			.parent().css({"margin-top":"-20px"});
+			.parent().css({"margin-top":"-15px"});
 		this.appframe = this.page.appframe;
 		var module = locals.DocType[this.doctype].module;
 		

webnotes/build.py

@@ -3,6 +3,7 @@
 
 from __future__ import unicode_literals
 from webnotes.utils.minify import JavascriptMinify
+from webnotes.webutils import build_website_sitemap_config
 
 """
 Build the `public` folders and setup languages
@@ -27,9 +28,7 @@ def bundle(no_compress, cms_make=True):
 			on_build()
 		except ImportError, e:
 			pass
-			
-	clear_pyc_files()
-	
+				
 def watch(no_compress):
 	"""watch and rebuild if necessary"""
 	import time
@@ -50,14 +49,6 @@ def check_lang():
 	from webnotes.translate import update_translations
 	update_translations()
 	
-def clear_pyc_files():
-	from webnotes.utils import get_base_path
-	for path, folders, files in os.walk(get_base_path()):
-		if 'locale' in folders: folders.remove('locale')
-		for f in files:
-			if f.decode("utf-8").endswith(".pyc"):
-				os.remove(os.path.join(path, f))
-	
 class Bundle:
 	"""
 		Concatenate, compress and mix (if required) js+css files from build.json

webnotes/db.py

@@ -357,6 +357,8 @@ class Database:
 			fl = ", ".join(fields)
 		else:
 			fl = fields
+			if fields=="*":
+				as_dict = True
 
 		conditions, filters = self.build_conditions(filters)
 	

webnotes/webutils.py

@@ -5,7 +5,7 @@ from __future__ import unicode_literals
 
 from webnotes import conf
 import webnotes
-import json
+import json, os, time
 from webnotes import _
 import webnotes.utils
 import mimetypes
@@ -27,7 +27,8 @@ def render_page(page_name):
 	"""get page html"""
 	set_content_type(page_name)
 	
-	page_name = scrub_page_name(page_name)
+	if page_name.endswith('.html'):
+		page_name = page_name[:-5]
 	html = ''
 		
 	if not conf.auto_cache_clear:
@@ -61,38 +62,25 @@ def build_page(page_name):
 	if not webnotes.conn:
 		webnotes.connect()
 
-	sitemap = get_website_sitemap()
-	page_options = sitemap.get(page_name)
+
+	sitemap_options = webnotes.doc("Website Sitemap", page_name).fields
 	
+	page_options = webnotes.doc("Website Sitemap Config", 
+		sitemap_options.get("website_sitemap_config")).fields.update(sitemap_options)
+		
 	if not page_options:
-		if page_name=="index":
-			# page not found, try home page
-			home_page = get_home_page()
-			page_options = sitemap.get(home_page)
-			if not page_options:
-				raise PageNotFoundError
-			page_options["page_name"] = home_page
-		else:
-			raise PageNotFoundError
+		raise PageNotFoundError
 	else:
 		page_options["page_name"] = page_name
 	
 	basepath = webnotes.utils.get_base_path()
-	module = None
-	no_cache = False
+	no_cache = page_options.get("no_cache")
 	
-	if page_options.get("controller"):
-		module = webnotes.get_module(page_options["controller"])
-		no_cache = getattr(module, "no_cache", False)
 
 	# if generator, then load bean, pass arguments
-	if page_options.get("is_generator"):
-		if not module:
-			raise Exception("Generator controller not defined")
-		
-		name = webnotes.conn.get_value(module.doctype, {
-			page_options.get("page_name_field", "page_name"): page_options["page_name"]})
-		obj = webnotes.get_obj(module.doctype, name, with_children=True)
+	if page_options.get("page_or_generator")=="Generator":
+		doctype = page_options.get("ref_doctype")
+		obj = webnotes.get_obj(doctype, page_options["docname"], with_children=True)
 
 		if hasattr(obj, 'get_context'):
 			obj.get_context()
@@ -102,58 +90,54 @@ def build_page(page_name):
 	else:
 		# page
 		context = webnotes._dict({ 'name': page_name })
-		if module and hasattr(module, "get_context"):
-			context.update(module.get_context())
+		if page_options.get("controller"):
+			module = webnotes.get_module(page_options.get("controller"))
+			if module and hasattr(module, "get_context"):
+				context.update(module.get_context())
 	
 	context.update(get_website_settings())
 
 	jenv = webnotes.get_jenv()
 	context["base_template"] = jenv.get_template(webnotes.get_config().get("base_template"))
 	
-	template_name = page_options['template']	
+	template_name = page_options['template_path']	
 	html = jenv.get_template(template_name).render(context)
 	
 	if not no_cache:
 		webnotes.cache().set_value("page:" + page_name, html)
 	return html
 	
-def build_sitemap():
-	sitemap = {}
-	config = webnotes.cache().get_value("website_sitemap_config", build_website_sitemap_config)
- 	sitemap.update(config["pages"])
-	
-	# pages
-	for p in config["pages"].values():
-		if p.get("controller"):
-			module = webnotes.get_module(p["controller"])
-			p["no_cache"] = getattr(module, "no_cache", False)
-			p["no_sitemap"] = getattr(module, "no_sitemap", False) or p["no_cache"]
-
-	# generators
-	for g in config["generators"].values():
-		g["is_generator"] = True
-		module = webnotes.get_module(g["controller"])
-
-		condition = ""
-		page_name_field = "page_name"
-		if hasattr(module, "page_name_field"):
-			page_name_field = module.page_name_field
-		if hasattr(module, "condition_field"):
-			condition = " where ifnull(%s, 0)=1" % module.condition_field
-
-		for page_name, name, modified in webnotes.conn.sql("""select %s, name, modified from 
-			`tab%s` %s""" % (page_name_field, module.doctype, condition)):
-			opts = g.copy()
-			opts["doctype"] = module.doctype
-			opts["no_cache"] = getattr(module, "no_cache", False)
-			opts["page_name"] = page_name
-			if page_name_field != "page_name":
-				opts["page_name_field"] = page_name_field
-			opts["docname"] = name
-			opts["lastmod"] = modified.strftime("%Y-%m-%d %H:%M:%S")
-			sitemap[page_name] = opts
-		
-	return sitemap
+def build_sitemap():	
+	webnotes.conn.sql("""delete from `tabWebsite Sitemap Config`""")
+	webnotes.conn.sql("""delete from `tabWebsite Sitemap`""")
+	webnotes.conn.commit()
+	build_website_sitemap_config()
+
+	for config in webnotes.conn.sql("""select * from `tabWebsite Sitemap Config`""", as_dict=True):
+		if config.page_or_generator == "Page":
+			config.page_name = config.link_name
+			add_to_sitemap(config)
+		else:
+			module = webnotes.get_module(config.controller)
+
+			condition = ""
+			if hasattr(module, "condition_field"):
+				condition = " where ifnull(%s, 0)=1" % module.condition_field
+
+			page_name_field = getattr(module, "page_name_field", "page_name")
+			
+			for name in webnotes.conn.sql_list("""select name from `tab%s` %s""" \
+				% (module.doctype, condition)):
+				webnotes.bean(module.doctype, name).run_method("on_update")
+
+def add_to_sitemap(options):
+	doc = webnotes.doc({"doctype":"Website Sitemap"})
+	for key in ("page_name", "ref_doctype", "docname", "page_or_generator", "lastmod"):
+		doc.fields[key] = options.get(key)
+	doc.name = options.page_name
+	doc.website_sitemap_config = options.link_name
+	doc.insert()
+	webnotes.conn.commit()
 	
 def get_home_page():
 	if not webnotes.conn:
@@ -166,49 +150,64 @@ def get_home_page():
 
 	return page_name
 		
-def build_website_sitemap_config():
-	import os, time
-	
+def build_website_sitemap_config():		
 	config = {"pages": {}, "generators":{}}
 	basepath = webnotes.utils.get_base_path()
-	
-	def get_options(path, fname):
-		name = fname
-		if fname.endswith(".html"):
-			name = fname[:-5]
 		
-		template_path = os.path.relpath(os.path.join(path, fname), basepath)
-		
-		options = webnotes._dict({
-			"link_name": name,
-			"template": template_path,
-			"lastmod": time.ctime(os.path.getmtime(template_path))
-		})
-
-		controller_name = fname.split(".")[0].replace("-", "_") + ".py"
-		controller_path = os.path.join(path, controller_name)
-		if os.path.exists(controller_path):
-			options.controller = os.path.relpath(controller_path[:-3], basepath).replace(os.path.sep, ".")
-			options.controller = ".".join(options.controller.split(".")[1:])
-
-		return options
-	
 	for path, folders, files in os.walk(basepath, followlinks=True):
-		if 'locale' in folders: folders.remove('locale')
+		if 'locale' in folders: 
+			folders.remove('locale')
+
+		# utility - remove pyc files
+		for f in files:
+			if f.decode("utf-8").endswith(".pyc"):
+				os.remove(os.path.join(path, f))
+
 		if os.path.basename(path)=="pages" and os.path.basename(os.path.dirname(path))=="templates":
 			for fname in files:
 				fname = webnotes.utils.cstr(fname)
 				if fname.split(".")[-1] in ("html", "xml", "js", "css"):
-					options = get_options(path, fname)
-					config["pages"][options.link_name] = options
+					add_website_sitemap_config("Page", path, fname)
 
 		if os.path.basename(path)=="generators" and os.path.basename(os.path.dirname(path))=="templates":
 			for fname in files:
 				if fname.endswith(".html"):
-					options = get_options(path, fname)
-					config["generators"][fname] = options
+					add_website_sitemap_config("Generator", path, fname)
+	
+	webnotes.conn.commit()
+		
+def add_website_sitemap_config(page_or_generator, path, fname):
+	basepath = webnotes.utils.get_base_path()
+	name = fname
+	if fname.endswith(".html"):
+		name = fname[:-5]
+	
+	template_path = os.path.relpath(os.path.join(path, fname), basepath)
+	
+	options = webnotes._dict({
+		"doctype": "Website Sitemap Config",
+		"page_or_generator": page_or_generator,
+		"link_name": name,
+		"template_path": template_path,
+		"lastmod": time.ctime(os.path.getmtime(template_path))
+	})
+
+	controller_name = fname.split(".")[0].replace("-", "_") + ".py"
+	controller_path = os.path.join(path, controller_name)
+	if os.path.exists(controller_path):
+		options.controller = os.path.relpath(controller_path[:-3], basepath).replace(os.path.sep, ".")
+		options.controller = ".".join(options.controller.split(".")[1:])
+
+	if options.controller:
+		module = webnotes.get_module(options.controller)
+		options.no_cache = getattr(module, "no_cache", 0)
+		options.no_sitemap = options.no_cache or getattr(module, "no_sitemap", 0)
+		options.ref_doctype = getattr(module, "doctype", None)
 		
-	return config
+	webnotes.doc(options).insert()
+
+	return options
+	
 
 def get_website_settings():
 	from webnotes.utils import get_request_site_address, encode, cint
@@ -276,20 +275,13 @@ def clear_cache(page_name=None):
 		delete_page_cache(page_name)
 	else:
 		cache = webnotes.cache()
-		for p in get_all_pages():
+		for p in webnotes.conn.sql_list("""select name from `tabWebsite Sitemap`"""):
 			if p is not None:
 				cache.delete_value("page:" + p)
 		cache.delete_value("page:index")
 		cache.delete_value("website_sitemap")
 		cache.delete_value("website_sitemap_config")
 		
-		
-def get_website_sitemap():
-	return webnotes.cache().get_value("website_sitemap", build_sitemap)
-
-def get_all_pages():
-	return get_website_sitemap().keys()
-
 def delete_page_cache(page_name):
 	if page_name:
 		cache = webnotes.cache()
@@ -321,12 +313,6 @@ def get_hex_shade(color, percent):
 	
 	return p(r) + p(g) + p(b)
 
-def scrub_page_name(page_name):
-	if page_name.endswith('.html'):
-		page_name = page_name[:-5]
-
-	return page_name
-
 def is_signup_enabled():
 	if getattr(webnotes.local, "is_signup_enabled", None) is None:
 		webnotes.local.is_signup_enabled = True
@@ -336,21 +322,71 @@ def is_signup_enabled():
 		
 	return webnotes.local.is_signup_enabled
 
-def update_page_name(doc, title):
-	"""set page_name and check if it is unique"""
-	new_page_name = page_name(title)
-	sitemap = get_website_sitemap()
+class WebsiteGenerator(object):
+	def setup_generator(self):
+		self._website_config = webnotes.conn.get_values("Website Sitemap Config", {"ref_doctype": self.doc.doctype}, "*")[0]
+		self._website_module = webnotes.get_module(self._website_config.controller)
+		
+		self._page_name_field = getattr(self._website_module, "page_name_field", "page_name")
+		self._condition_field = getattr(self._website_module, "condition_field", "")
+		
+	def on_update(self, page_name=None):
+		self.setup_generator()
+		if self._condition_field:
+			if not self.doc.fields[self._condition_field]:
+				remove_page(self.doc.fields[self._page_name_field])
+				return
+
+		if not page_name:
+			new_page_name = cleanup_page_name(self.get_page_title() \
+				if hasattr(self, "get_page_title") else (self.doc.title or self.doc.name))
+			self.check_if_page_name_is_unique(new_page_name)
 	
-	if new_page_name in sitemap and \
-		not (sitemap[new_page_name].doctype == doc.doctype and sitemap[new_page_name].docname == doc.name):
-			webnotes.throw("%s: %s. %s: %s" % (new_page_name, _("Page already exists"),
-				_("Please change the value"), title))
+			remove_page(self.doc.page_name)
+			add_generator_to_sitemap(self.doc.doctype, self.doc.name, new_page_name, self.doc.modified, 
+				self._website_config, self._website_module)
+
+			if self.doc.fields[self._page_name_field]!=new_page_name: 
+				webnotes.conn.set(self.doc, self._page_name_field, new_page_name)	
+		else:
+			add_generator_to_sitemap(self.doc.doctype, self.doc.name, page_name, self.doc.modified, 
+				self._website_config, self._website_module)
+			
+		delete_page_cache(self.doc.page_name)
+
+	def check_if_page_name_is_unique(self, new_page_name):
+		if webnotes.conn.sql("""select name from `tabWebsite Sitemap` where name=%s 
+			and ref_doctype!=%s and docname!=%s""", (new_page_name, self.doc.doctype, self.doc.name)):
+				webnotes.throw("%s: %s. %s: <b>%s<b>" % (new_page_name, _("Page already exists"),
+					_("Please change the value"), title))
+		
+	def on_trash(self):
+		self.setup_generator()
+		remove_page(self.doc.fields[self._page_name_field])
+
+def add_generator_to_sitemap(ref_doctype, docname, page_name, modified, config=None, module=None):
+	if not config:
+		config = webnotes.conn.get_values("Website Sitemap Config", {"ref_doctype": ref_doctype}, "*")[0]
+	if not module:
+		module = webnotes.get_module(config.controller)
 	
-	if doc.page_name: delete_page_cache(doc.page_name)
-	webnotes.conn.set(doc, "page_name", new_page_name)
-	delete_page_cache(doc.page_name)
+	page_name_field = getattr(module, "page_name_field", "page_name")
+		
+	opts = config.copy()
+	opts["page_name"] = page_name
+	if page_name_field != "page_name":
+		opts["page_name_field"] = page_name_field
+	opts["docname"] = docname
+	opts["lastmod"] = modified
+	add_to_sitemap(opts)
+
+
+def remove_page(page_name):
+	if page_name:
+		delete_page_cache(page_name)
+		webnotes.conn.sql("delete from `tabWebsite Sitemap` where name=%s", page_name)	
 
-def page_name(title):
+def cleanup_page_name(title):
 	"""make page name from title"""
 	import re
 	name = title.lower()

website/doctype/blog_post/blog_post.py

@@ -5,16 +5,16 @@ from __future__ import unicode_literals
 
 import webnotes
 import webnotes.webutils
+from webnotes.webutils import WebsiteGenerator, cleanup_page_name
 from webnotes import _
 from webnotes.utils import today
 
-class DocType:
+class DocType(WebsiteGenerator):
 	def __init__(self, d, dl):
 		self.doc, self.doclist = d, dl
 
 	def autoname(self):
-		from webnotes.webutils import page_name
-		self.doc.name = page_name(self.doc.title)
+		self.doc.name = cleanup_page_name(self.doc.title)
 
 	def validate(self):
 		if self.doc.blog_intro:
@@ -29,7 +29,7 @@ class DocType:
 			where name=%s""", self.doc.blogger)
 
 	def on_update(self):
-		webnotes.webutils.update_page_name(self.doc, self.doc.title)
+		WebsiteGenerator.on_update(self)
 		webnotes.webutils.delete_page_cache("writers")
 
 	def get_context(self):

website/doctype/web_page/web_page.py

@@ -3,8 +3,9 @@
 
 from __future__ import unicode_literals
 import webnotes
+from webnotes.webutils import WebsiteGenerator
 
-class DocType():
+class DocType(WebsiteGenerator):
 	def __init__(self, d, dl):
 		self.doc, self.doclist = d, dl
 
@@ -13,8 +14,7 @@ class DocType():
 		self.doc.name = page_name(self.doc.title)
 
 	def on_update(self):
-		from webnotes.webutils import update_page_name
-		update_page_name(self.doc, self.doc.title)
+		WebsiteGenerator.on_update(self)
 		self.if_home_clear_cache()
 		
 		# clear all cache if it has toc
@@ -24,6 +24,8 @@ class DocType():
 			
 	def on_trash(self):
 		# delete entry from Table of Contents of other pages
+		WebsiteGenerator.on_trash(self)
+		
 		webnotes.conn.sql("""delete from `tabTable of Contents`
 			where web_page=%s""", self.doc.name)
 		
@@ -35,6 +37,8 @@ class DocType():
 	def if_home_clear_cache(self):
 		"""if home page, clear cache"""
 		if webnotes.conn.get_value("Website Settings", None, "home_page")==self.doc.name:
+			WebsiteGenerator.on_update(self, page_name="index")
+
 			from webnotes.sessions import clear_cache
 			clear_cache('Guest')
 			

website/templates/pages/sitemap.py

@@ -10,13 +10,11 @@ from webnotes.utils import get_request_site_address
 
 def get_context():
 	"""generate the sitemap XML"""
-	links = webnotes.webutils.get_website_sitemap().items()
 	host = get_request_site_address()
+	links = []
+	for l in webnotes.conn.sql("""select * from `tabWebsite Sitemap` where ifnull(no_sitemap, 0)=0""", 
+		as_dict=True):
+		links.append({"loc": urllib.basejoin(host, urllib.quote(l.page_name.encode("utf-8")))})
 	
-	for l in links:
-		l[1]["loc"] = urllib.basejoin(host, urllib.quote(l[1].get("page_name", l[1]["link_name"]).encode("utf-8")))
-	
-	return {
-		"links": [l[1] for l in links if not l[1].get("no_sitemap")]
-	}
+	return links
 	

wnf.py

@@ -150,6 +150,8 @@ def setup_utilities(parser):
 	# clear
 	parser.add_argument("--clear_web", default=False, action="store_true",
 		help="Clear website cache")
+	parser.add_argument("--build_sitemap", default=False, action="store_true",
+		help="Build Website Sitemap")
 	parser.add_argument("--clear_cache", default=False, action="store_true",
 		help="Clear cache, doctype cache and defaults")
 	parser.add_argument("--reset_perms", default=False, action="store_true",
@@ -419,6 +421,13 @@ def clear_web(site=None):
 	webnotes.webutils.clear_cache()
 	webnotes.destroy()
 
+@cmd
+def build_sitemap(site=None):
+	import webnotes.webutils
+	webnotes.connect(site=site)
+	webnotes.webutils.build_sitemap()
+	webnotes.destroy()
+
 @cmd
 def reset_perms(site=None):
 	webnotes.connect(site=site)