[website sitemap] [minor] cleanup code and trigger build

webnotes/build.py

@@ -3,7 +3,6 @@
 
 from __future__ import unicode_literals
 from webnotes.utils.minify import JavascriptMinify
-from webnotes.webutils import build_website_sitemap_config
 
 """
 Build the `public` folders and setup languages

webnotes/model/bean.py

@@ -302,6 +302,8 @@ class Bean:
 			self.save_main()
 			self.save_children()
 			self.run_method('on_update')
+			if perm_to_check=="create":
+				self.run_method("after_insert")
 		else:
 			self.no_permission_to(_(perm_to_check.title()))
 		

webnotes/webutils.py

@@ -10,6 +10,8 @@ from webnotes import _
 import webnotes.utils
 import mimetypes
 
+from website.doctype.website_sitemap.website_sitemap import add_to_sitemap
+
 class PageNotFoundError(Exception): pass
 
 def render(page_name):
@@ -62,11 +64,13 @@ def build_page(page_name):
 	if not webnotes.conn:
 		webnotes.connect()
 
-
 	sitemap_options = webnotes.doc("Website Sitemap", page_name).fields
 	
 	page_options = webnotes.doc("Website Sitemap Config", 
-		sitemap_options.get("website_sitemap_config")).fields.update(sitemap_options)
+		sitemap_options.get("website_sitemap_config")).fields.update({
+			"page_name":sitemap_options.page_name,
+			"docname":sitemap_options.docname
+		})
 		
 	if not page_options:
 		raise PageNotFoundError
@@ -106,39 +110,7 @@ def build_page(page_name):
 	if not no_cache:
 		webnotes.cache().set_value("page:" + page_name, html)
 	return html
-	
-def build_sitemap():	
-	webnotes.conn.sql("""delete from `tabWebsite Sitemap Config`""")
-	webnotes.conn.sql("""delete from `tabWebsite Sitemap`""")
-	webnotes.conn.commit()
-	build_website_sitemap_config()
-
-	for config in webnotes.conn.sql("""select * from `tabWebsite Sitemap Config`""", as_dict=True):
-		if config.page_or_generator == "Page":
-			config.page_name = config.link_name
-			add_to_sitemap(config)
-		else:
-			module = webnotes.get_module(config.controller)
-
-			condition = ""
-			if hasattr(module, "condition_field"):
-				condition = " where ifnull(%s, 0)=1" % module.condition_field
-
-			page_name_field = getattr(module, "page_name_field", "page_name")
-			
-			for name in webnotes.conn.sql_list("""select name from `tab%s` %s""" \
-				% (module.doctype, condition)):
-				webnotes.bean(module.doctype, name).run_method("on_update")
-
-def add_to_sitemap(options):
-	doc = webnotes.doc({"doctype":"Website Sitemap"})
-	for key in ("page_name", "ref_doctype", "docname", "page_or_generator", "lastmod"):
-		doc.fields[key] = options.get(key)
-	doc.name = options.page_name
-	doc.website_sitemap_config = options.link_name
-	doc.insert()
-	webnotes.conn.commit()
-	
+		
 def get_home_page():
 	if not webnotes.conn:
 		webnotes.connect()
@@ -149,65 +121,6 @@ def get_home_page():
 		page_name = 'login'
 
 	return page_name
-		
-def build_website_sitemap_config():		
-	config = {"pages": {}, "generators":{}}
-	basepath = webnotes.utils.get_base_path()
-		
-	for path, folders, files in os.walk(basepath, followlinks=True):
-		if 'locale' in folders: 
-			folders.remove('locale')
-
-		# utility - remove pyc files
-		for f in files:
-			if f.decode("utf-8").endswith(".pyc"):
-				os.remove(os.path.join(path, f))
-
-		if os.path.basename(path)=="pages" and os.path.basename(os.path.dirname(path))=="templates":
-			for fname in files:
-				fname = webnotes.utils.cstr(fname)
-				if fname.split(".")[-1] in ("html", "xml", "js", "css"):
-					add_website_sitemap_config("Page", path, fname)
-
-		if os.path.basename(path)=="generators" and os.path.basename(os.path.dirname(path))=="templates":
-			for fname in files:
-				if fname.endswith(".html"):
-					add_website_sitemap_config("Generator", path, fname)
-	
-	webnotes.conn.commit()
-		
-def add_website_sitemap_config(page_or_generator, path, fname):
-	basepath = webnotes.utils.get_base_path()
-	name = fname
-	if fname.endswith(".html"):
-		name = fname[:-5]
-	
-	template_path = os.path.relpath(os.path.join(path, fname), basepath)
-	
-	options = webnotes._dict({
-		"doctype": "Website Sitemap Config",
-		"page_or_generator": page_or_generator,
-		"link_name": name,
-		"template_path": template_path,
-		"lastmod": time.ctime(os.path.getmtime(template_path))
-	})
-
-	controller_name = fname.split(".")[0].replace("-", "_") + ".py"
-	controller_path = os.path.join(path, controller_name)
-	if os.path.exists(controller_path):
-		options.controller = os.path.relpath(controller_path[:-3], basepath).replace(os.path.sep, ".")
-		options.controller = ".".join(options.controller.split(".")[1:])
-
-	if options.controller:
-		module = webnotes.get_module(options.controller)
-		options.no_cache = getattr(module, "no_cache", 0)
-		options.no_sitemap = options.no_cache or getattr(module, "no_sitemap", 0)
-		options.ref_doctype = getattr(module, "doctype", None)
-		
-	webnotes.doc(options).insert()
-
-	return options
-	
 
 def get_website_settings():
 	from webnotes.utils import get_request_site_address, encode, cint
@@ -288,31 +201,6 @@ def delete_page_cache(page_name):
 		cache.delete_value("page:" + page_name)
 		cache.delete_value("website_sitemap")
 			
-def get_hex_shade(color, percent):
-	def p(c):
-		v = int(c, 16) + int(int('ff', 16) * (float(percent)/100))
-		if v < 0: 
-			v=0
-		if v > 255: 
-			v=255
-		h = hex(v)[2:]
-		if len(h) < 2:
-			h = "0" + h
-		return h
-		
-	r, g, b = color[0:2], color[2:4], color[4:6]
-	
-	avg = (float(int(r, 16) + int(g, 16) + int(b, 16)) / 3)
-	# switch dark and light shades
-	if avg > 128:
-		percent = -percent
-
-	# stronger diff for darker shades
-	if percent < 25 and avg < 64:
-		percent = percent * 2
-	
-	return p(r) + p(g) + p(b)
-
 def is_signup_enabled():
 	if getattr(webnotes.local, "is_signup_enabled", None) is None:
 		webnotes.local.is_signup_enabled = True
@@ -324,17 +212,15 @@ def is_signup_enabled():
 
 class WebsiteGenerator(object):
 	def setup_generator(self):
-		self._website_config = webnotes.conn.get_values("Website Sitemap Config", {"ref_doctype": self.doc.doctype}, "*")[0]
-		self._website_module = webnotes.get_module(self._website_config.controller)
-		
-		self._page_name_field = getattr(self._website_module, "page_name_field", "page_name")
-		self._condition_field = getattr(self._website_module, "condition_field", "")
+		self._website_config = webnotes.conn.get_values("Website Sitemap Config", 
+			{"ref_doctype": self.doc.doctype}, "*")[0]
 		
 	def on_update(self, page_name=None):
 		self.setup_generator()
-		if self._condition_field:
-			if not self.doc.fields[self._condition_field]:
-				remove_page(self.doc.fields[self._page_name_field])
+		if self._website_config.condition_field:
+			if not self.doc.fields[self._website_config.condition_field]:
+				# condition field failed, return
+				remove_page(self.doc.fields[self._website_config.page_name_field])
 				return
 
 		if not page_name:
@@ -343,17 +229,19 @@ class WebsiteGenerator(object):
 			self.check_if_page_name_is_unique(new_page_name)
 	
 			remove_page(self.doc.page_name)
-			add_generator_to_sitemap(self.doc.doctype, self.doc.name, new_page_name, self.doc.modified, 
-				self._website_config, self._website_module)
 
-			if self.doc.fields[self._page_name_field]!=new_page_name: 
-				webnotes.conn.set(self.doc, self._page_name_field, new_page_name)	
-		else:
-			add_generator_to_sitemap(self.doc.doctype, self.doc.name, page_name, self.doc.modified, 
-				self._website_config, self._website_module)
+			if self.doc.fields[self._website_config.page_name_field]!=new_page_name: 
+				webnotes.conn.set(self.doc, self._website_config.page_name_field, new_page_name)
+				
+			page_name = new_page_name
+
+		add_to_sitemap(webnotes._dict({
+			"ref_doctype":self.doc.doctype, 
+			"docname": self.doc.name, 
+			"page_name": page_name,
+			"link_name": self._website_config.name,
+			"lastmod": self.doc.modified}))
 			
-		delete_page_cache(self.doc.page_name)
-
 	def check_if_page_name_is_unique(self, new_page_name):
 		if webnotes.conn.sql("""select name from `tabWebsite Sitemap` where name=%s 
 			and ref_doctype!=%s and docname!=%s""", (new_page_name, self.doc.doctype, self.doc.name)):
@@ -364,23 +252,6 @@ class WebsiteGenerator(object):
 		self.setup_generator()
 		remove_page(self.doc.fields[self._page_name_field])
 
-def add_generator_to_sitemap(ref_doctype, docname, page_name, modified, config=None, module=None):
-	if not config:
-		config = webnotes.conn.get_values("Website Sitemap Config", {"ref_doctype": ref_doctype}, "*")[0]
-	if not module:
-		module = webnotes.get_module(config.controller)
-	
-	page_name_field = getattr(module, "page_name_field", "page_name")
-		
-	opts = config.copy()
-	opts["page_name"] = page_name
-	if page_name_field != "page_name":
-		opts["page_name_field"] = page_name_field
-	opts["docname"] = docname
-	opts["lastmod"] = modified
-	add_to_sitemap(opts)
-
-
 def remove_page(page_name):
 	if page_name:
 		delete_page_cache(page_name)
@@ -399,3 +270,28 @@ def cleanup_page_name(title):
 	name = re.sub(r"(-)\1+", r"\1", name)
 	
 	return name
+
+def get_hex_shade(color, percent):
+	def p(c):
+		v = int(c, 16) + int(int('ff', 16) * (float(percent)/100))
+		if v < 0: 
+			v=0
+		if v > 255: 
+			v=255
+		h = hex(v)[2:]
+		if len(h) < 2:
+			h = "0" + h
+		return h
+		
+	r, g, b = color[0:2], color[2:4], color[4:6]
+	
+	avg = (float(int(r, 16) + int(g, 16) + int(b, 16)) / 3)
+	# switch dark and light shades
+	if avg > 128:
+		percent = -percent
+
+	# stronger diff for darker shades
+	if percent < 25 and avg < 64:
+		percent = percent * 2
+	
+	return p(r) + p(g) + p(b)

website/doctype/website_sitemap/website_sitemap.py

@@ -8,4 +8,15 @@ import webnotes
 
 class DocType:
 	def __init__(self, d, dl):
-		self.doc, self.doclist = d, dl
+		self.doc, self.doclist = d, dl
+		
+def add_to_sitemap(options):
+	doc = webnotes.doc({"doctype":"Website Sitemap"})
+	for key in ("page_name", "ref_doctype", "docname", "page_or_generator", "lastmod"):
+		doc.fields[key] = options.get(key)
+	if not doc.page_name:
+		doc.page_name = options.link_name
+	doc.name = doc.page_name
+	doc.website_sitemap_config = options.link_name
+	doc.insert()
+	webnotes.conn.commit()

website/doctype/website_sitemap_config/website_sitemap_config.py

@@ -5,7 +5,106 @@
 
 from __future__ import unicode_literals
 import webnotes
+import webnotes.utils
+
+from website.doctype.website_sitemap.website_sitemap import add_to_sitemap
 
 class DocType:
 	def __init__(self, d, dl):
-		self.doc, self.doclist = d, dl
+		self.doc, self.doclist = d, dl
+
+	def after_insert(self):
+		if self.doc.page_or_generator == "Page":
+			add_to_sitemap(self.doc.fields)
+		else:
+			condition = ""
+			if self.doc.condition_field:
+				condition = " where ifnull(%s, 0)=1" % self.doc.condition_field
+			
+			for name in webnotes.conn.sql_list("""select name from `tab%s` %s""" \
+				% (self.doc.ref_doctype, condition)):
+				webnotes.bean(self.doc.ref_doctype, name).run_method("on_update")
+
+	def on_trash(self):
+		webnotes.conn.sql("""delete from `tabWebsite Sitemap` 
+			where website_sitemap_config = %s""", self.doc.name)
+
+import os, time
+
+def rebuild_website_sitemap_config():
+	webnotes.conn.sql("""delete from `tabWebsite Sitemap Config`""")
+	webnotes.conn.sql("""delete from `tabWebsite Sitemap`""")
+	webnotes.conn.commit()
+	build_website_sitemap_config()
+
+def build_website_sitemap_config():		
+	config = {"pages": {}, "generators":{}}
+	basepath = webnotes.utils.get_base_path()
+	
+	existing_configs = dict(webnotes.conn.sql("""select name, lastmod from `tabWebsite Sitemap Config`"""))
+		
+	for path, folders, files in os.walk(basepath, followlinks=True):
+		if 'locale' in folders: 
+			folders.remove('locale')
+
+		# utility - remove pyc files
+		for f in files:
+			if f.decode("utf-8").endswith(".pyc"):
+				os.remove(os.path.join(path, f))
+
+		if os.path.basename(path)=="pages" and os.path.basename(os.path.dirname(path))=="templates":
+			for fname in files:
+				fname = webnotes.utils.cstr(fname)
+				if fname.split(".")[-1] in ("html", "xml", "js", "css"):
+					name = add_website_sitemap_config("Page", path, fname, existing_configs)
+					if name in existing_configs: del existing_configs[name]
+
+		if os.path.basename(path)=="generators" and os.path.basename(os.path.dirname(path))=="templates":
+			for fname in files:
+				if fname.endswith(".html"):
+					name = add_website_sitemap_config("Generator", path, fname, existing_configs)
+					if name in existing_configs: del existing_configs[name]
+					
+	for name in existing_configs:
+		webnotes.delete_doc("Website Sitemap Config", name)
+		
+def add_website_sitemap_config(page_or_generator, path, fname, existing_configs):
+	basepath = webnotes.utils.get_base_path()
+	template_path = os.path.relpath(os.path.join(path, fname), basepath)
+	lastmod = time.ctime(os.path.getmtime(template_path))
+
+	name = fname[:-5] if fname.endswith(".html") else fname
+	
+	config_lastmod = existing_configs.get(name)
+	if config_lastmod != lastmod:
+		webnotes.delete_doc("Website Sitemap Config", name)
+	else:
+		return name
+		# already exists and safe
+	
+	wsc = webnotes._dict({
+		"doctype": "Website Sitemap Config",
+		"page_or_generator": page_or_generator,
+		"link_name": name,
+		"template_path": template_path,
+		"lastmod": lastmod
+	})
+	
+	controller_name = fname.split(".")[0].replace("-", "_") + ".py"
+	controller_path = os.path.join(path, controller_name)
+	if os.path.exists(controller_path):
+		wsc.controller = os.path.relpath(controller_path[:-3], basepath).replace(os.path.sep, ".")
+		wsc.controller = ".".join(wsc.controller.split(".")[1:])
+
+	if wsc.controller:
+		module = webnotes.get_module(wsc.controller)
+		wsc.no_cache = getattr(module, "no_cache", 0)
+		wsc.no_sitemap = wsc.no_cache or getattr(module, "no_sitemap", 0)
+		wsc.ref_doctype = getattr(module, "doctype", None)
+		wsc.page_name_field = getattr(module, "page_name_field", "page_name")
+		wsc.condition_field = getattr(module, "condition_field", None)
+		
+	webnotes.bean(wsc).insert()
+	webnotes.conn.commit()
+	
+	return name

website/doctype/website_sitemap_config/website_sitemap_config.txt

@@ -2,7 +2,7 @@
  {
   "creation": "2013-11-18 15:35:00", 
   "docstatus": 0, 
-  "modified": "2013-11-18 18:36:01", 
+  "modified": "2013-11-19 11:37:52", 
   "modified_by": "Administrator", 
   "owner": "Administrator"
  }, 
@@ -81,5 +81,19 @@
   "fieldtype": "Check", 
   "label": "No Sitemap", 
   "read_only": 1
+ }, 
+ {
+  "doctype": "DocField", 
+  "fieldname": "page_name_field", 
+  "fieldtype": "Data", 
+  "label": "Page Name Field", 
+  "read_only": 1
+ }, 
+ {
+  "doctype": "DocField", 
+  "fieldname": "condition_field", 
+  "fieldtype": "Data", 
+  "label": "Condition Field", 
+  "read_only": 1
  }
 ]

wnf.py

@@ -277,6 +277,7 @@ def latest(site=None, verbose=True):
 	import webnotes.modules.patch_handler
 	import webnotes.model.sync
 	import webnotes.plugins
+	from website.doctype.website_sitemap_config.website_sitemap_config import build_website_sitemap_config
 	
 	webnotes.connect(site=site)
 	
@@ -293,6 +294,9 @@ def latest(site=None, verbose=True):
 		# remove __init__.py from plugins
 		webnotes.plugins.remove_init_files()
 		
+		# build website config if any changes in templates etc.
+		build_website_sitemap_config()
+		
 	except webnotes.modules.patch_handler.PatchError, e:
 		print "\n".join(webnotes.local.patch_log_list)
 		raise
@@ -423,9 +427,9 @@ def clear_web(site=None):
 
 @cmd
 def build_sitemap(site=None):
-	import webnotes.webutils
+	from website.doctype.website_sitemap_config.website_sitemap_config import build_website_sitemap_config
 	webnotes.connect(site=site)
-	webnotes.webutils.build_sitemap()
+	build_website_sitemap_config()
 	webnotes.destroy()
 
 @cmd