From f81632b9d91791a43cdc4ceb87b26d9ad2db31f1 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Wed, 22 Jun 2016 17:49:04 +0530
Subject: [PATCH] [redesign] removed website statics, www served like
 templates/pages

---
 frappe/commands/utils.py                    |  31 ---
 frappe/installer.py                         |   3 +-
 frappe/migrate.py                           |   6 +-
 frappe/patches.txt                          |   3 +-
 frappe/patches/v5_0/force_sync_website.py   |   3 +-
 frappe/templates/includes/static_index.html |   2 +-
 frappe/templates/pages/404.html             |   2 +-
 frappe/templates/pages/about.html           |   2 +-
 frappe/templates/pages/contact.html         |   2 +-
 frappe/templates/pages/edit-profile.html    |   2 +-
 frappe/templates/pages/error.html           |   2 +-
 frappe/templates/pages/login.html           |   2 +-
 frappe/templates/pages/message.html         |   2 +-
 frappe/templates/pages/update-password.html |   2 +-
 frappe/website/context.py                   |   9 +-
 frappe/website/doctype/web_page/web_page.py |  92 ++-------
 frappe/website/render.py                    |   6 +-
 frappe/website/router.py                    | 155 ++++++++++++---
 frappe/website/statics.py                   | 200 --------------------
 19 files changed, 167 insertions(+), 359 deletions(-)
 delete mode 100644 frappe/website/statics.py

diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py
index 986c0ffec5..51ac94ac70 100644
--- a/frappe/commands/utils.py
+++ b/frappe/commands/utils.py
@@ -65,35 +65,6 @@ def destroy_all_sessions(context):
 		finally:
 			frappe.destroy()
 
-@click.command('sync-www')
-@click.option('--force', help='Rebuild all pages', is_flag=True, default=False)
-@pass_context
-def sync_www(context, force=False):
-	"Sync files from static pages from www directory to Web Pages"
-	from frappe.website import statics
-	for site in context.sites:
-		try:
-			frappe.init(site=site)
-			frappe.connect()
-			statics.sync_statics(rebuild=force)
-			frappe.db.commit()
-		finally:
-			frappe.destroy()
-
-@click.command('build-website')
-@pass_context
-def build_website(context):
-	"Sync statics and clear cache"
-	from frappe.website import render, statics
-	for site in context.sites:
-		try:
-			frappe.init(site=site)
-			frappe.connect()
-			render.clear_cache()
-			statics.sync(verbose=context.verbose).start(rebuild=True)
-			frappe.db.commit()
-		finally:
-			frappe.destroy()
 
 @click.command('reset-perms')
 @pass_context
@@ -409,7 +380,6 @@ def get_version():
 
 commands = [
 	build,
-	build_website,
 	clear_cache,
 	clear_website_cache,
 	console,
@@ -429,7 +399,6 @@ commands = [
 	run_tests,
 	serve,
 	set_config,
-	sync_www,
 	watch,
 	_bulk_rename,
 	add_to_email_queue,
diff --git a/frappe/installer.py b/frappe/installer.py
index bd0fb42781..fe1500313c 100755
--- a/frappe/installer.py
+++ b/frappe/installer.py
@@ -14,7 +14,7 @@ import importlib
 from frappe.model.db_schema import DbManager
 from frappe.model.sync import sync_for
 from frappe.utils.fixtures import sync_fixtures
-from frappe.website import render, statics
+from frappe.website import render
 from frappe.desk.doctype.desktop_icon.desktop_icon import sync_from_app
 from frappe.utils.password import create_auth_table
 
@@ -208,7 +208,6 @@ def remove_app(app_name, dry_run=False):
 def post_install(rebuild_website=False):
 	if rebuild_website:
 		render.clear_cache()
-		statics.sync().start()
 
 	init_singles()
 	frappe.db.commit()
diff --git a/frappe/migrate.py b/frappe/migrate.py
index 33d20e8ffc..ef88944b7e 100644
--- a/frappe/migrate.py
+++ b/frappe/migrate.py
@@ -10,7 +10,7 @@ import frappe.model.sync
 from frappe.utils.fixtures import sync_fixtures
 from frappe.sessions import clear_global_cache
 from frappe.desk.notifications import clear_notifications
-from frappe.website import statics, render
+from frappe.website import render
 from frappe.desk.doctype.desktop_icon.desktop_icon import sync_desktop_icons
 
 def migrate(verbose=True, rebuild_website=False):
@@ -34,10 +34,6 @@ def migrate(verbose=True, rebuild_website=False):
 
 	# syncs statics
 	render.clear_cache()
-	if rebuild_website:
-		statics.sync(verbose=verbose).start(True)
-	else:
-		statics.sync_statics()
 
 	frappe.db.commit()
 
diff --git a/frappe/patches.txt b/frappe/patches.txt
index 72e1011205..36e26d03b7 100644
--- a/frappe/patches.txt
+++ b/frappe/patches.txt
@@ -129,4 +129,5 @@ frappe.patches.v7_0.desktop_icons_hidden_by_admin_as_blocked
 frappe.patches.v7_0.add_communication_in_doc
 frappe.patches.v7_0.update_send_after_in_bulk_email
 frappe.patches.v7_0.setup_list_settings
-execute:frappe.db.sql('''delete from `tabSingles` where doctype="Email Settings"''') # 2016-06-13
\ No newline at end of file
+execute:frappe.db.sql('''delete from `tabSingles` where doctype="Email Settings"''') # 2016-06-13
+execute:frappe.db.sql("delete from `tabWeb Page` where ifnull(template_path, '')!=''")
diff --git a/frappe/patches/v5_0/force_sync_website.py b/frappe/patches/v5_0/force_sync_website.py
index 652ba9d159..5dcd0d79c7 100644
--- a/frappe/patches/v5_0/force_sync_website.py
+++ b/frappe/patches/v5_0/force_sync_website.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 import frappe
-from frappe.website import statics
 
 def execute():
-	statics.sync_statics(rebuild=True)
+	pass
diff --git a/frappe/templates/includes/static_index.html b/frappe/templates/includes/static_index.html
index c61bb001a6..73a08fd752 100644
--- a/frappe/templates/includes/static_index.html
+++ b/frappe/templates/includes/static_index.html
@@ -1,4 +1,4 @@
-<!-- no-sidebar -->
+
 <div class="row">
 	<div class="col-md-8">
 		<ol>
diff --git a/frappe/templates/pages/404.html b/frappe/templates/pages/404.html
index 70164412a4..2ebd230288 100644
--- a/frappe/templates/pages/404.html
+++ b/frappe/templates/pages/404.html
@@ -7,7 +7,7 @@
 {%- endblock -%}
 
 {% block page_content %}
-<!-- no-sidebar -->
+
 <div class="404-content" style="margin-bottom: 100px;">
 	<p>{{_("We are very sorry for this, but the page you are looking for is missing (this could be because of a typo in the address) or moved.")}}</p>
 </div>
diff --git a/frappe/templates/pages/about.html b/frappe/templates/pages/about.html
index f65fc6b9ac..14ac0a216d 100644
--- a/frappe/templates/pages/about.html
+++ b/frappe/templates/pages/about.html
@@ -5,7 +5,7 @@
 {% include "templates/pages/web_sidebar.html" %}
 {% endblock %}
 {% block page_content %}
-<!-- no-sidebar -->
+
 <article class="about-content">
 	{{ doc.company_introduction or """<p>Some Introduction about your company that you would
 			like your website visitor to know.
diff --git a/frappe/templates/pages/contact.html b/frappe/templates/pages/contact.html
index 18a17b4a0c..5f6f0f7980 100644
--- a/frappe/templates/pages/contact.html
+++ b/frappe/templates/pages/contact.html
@@ -3,7 +3,7 @@
 {% block title %}{{ heading or "Contact Us"}}{% endblock %}
 
 {% block page_content %}
-<!-- no-sidebar -->
+
 <div class="contact-content">
 	<div class="row">
 		<div class="col-md-8">
diff --git a/frappe/templates/pages/edit-profile.html b/frappe/templates/pages/edit-profile.html
index 2e8756f3e9..f2b3ef4c83 100644
--- a/frappe/templates/pages/edit-profile.html
+++ b/frappe/templates/pages/edit-profile.html
@@ -41,6 +41,6 @@ frappe.ready(function() {
 	})
 })
 </script>
-<!-- no-sidebar -->
+
 {% endblock %}
 
diff --git a/frappe/templates/pages/error.html b/frappe/templates/pages/error.html
index 1a86d8e4d7..267c035fee 100644
--- a/frappe/templates/pages/error.html
+++ b/frappe/templates/pages/error.html
@@ -9,7 +9,7 @@
 {% endblock %}
 
 {% block page_content %}
-<!-- no-sidebar -->
+
 <div class="error-content">
 	<pre><code>{{ error }}</code></pre>
 </div>
diff --git a/frappe/templates/pages/login.html b/frappe/templates/pages/login.html
index a6c2a630fa..0da1fe3ac1 100644
--- a/frappe/templates/pages/login.html
+++ b/frappe/templates/pages/login.html
@@ -8,7 +8,7 @@
 
 {% block page_content %}
 <!-- no-header -->
-<!-- no-sidebar -->
+
 <div class="login-content">
 	<form class="form-signin form-login" role="form">
 
diff --git a/frappe/templates/pages/message.html b/frappe/templates/pages/message.html
index 1e9b82cd42..80fbe6f88a 100644
--- a/frappe/templates/pages/message.html
+++ b/frappe/templates/pages/message.html
@@ -9,7 +9,7 @@
 {% endblock %}
 
 {% block page_content %}
-<!-- no-sidebar -->
+
 <div class="message-content" style="min-height: 200px;">
 	<div>
 		{{ message }}
diff --git a/frappe/templates/pages/update-password.html b/frappe/templates/pages/update-password.html
index fb02197871..95c0364a4e 100644
--- a/frappe/templates/pages/update-password.html
+++ b/frappe/templates/pages/update-password.html
@@ -4,7 +4,7 @@
 
 {% block header %}<h1>{{_("Reset Password")}}</h1>{% endblock %}
 {% block page_content %}
-<!-- no-sidebar -->
+
 <div class="row" style="margin-top: 40px; margin-bottom: 20px">
 	<div class="col-sm-6">
         <form id="reset-password">
diff --git a/frappe/website/context.py b/frappe/website/context.py
index 423f727462..d36a67e9f1 100644
--- a/frappe/website/context.py
+++ b/frappe/website/context.py
@@ -26,13 +26,18 @@ def get_context(path, args=None):
 	if hasattr(frappe.local, 'response') and frappe.local.response.get('context'):
 		context.update(frappe.local.response.context)
 
+	# print frappe.as_json(context)
+
 	return context
 
 def build_context(context):
-	"""get_context method of doc or module is supposed to render content templates and push it into context"""
+	"""get_context method of doc or module is supposed to render
+		content templates and push it into context"""
 	context = frappe._dict(context)
+
 	if not "url_prefix" in context:
 		context.url_prefix = ""
+
 	context.update(get_website_settings())
 	context.update(frappe.local.conf.get("website_context") or {})
 
@@ -92,7 +97,7 @@ def add_sidebar_data(context):
 		if item.reference_doctype:
 			try:
 				item.count = len(frappe.templates.pages.list.get(item.reference_doctype).get('result'))
-			
+
 			except frappe.PermissionError:
 				pass
 
diff --git a/frappe/website/doctype/web_page/web_page.py b/frappe/website/doctype/web_page/web_page.py
index 2ba1006273..be0649155f 100644
--- a/frappe/website/doctype/web_page/web_page.py
+++ b/frappe/website/doctype/web_page/web_page.py
@@ -2,9 +2,9 @@
 # MIT License. See license.txt
 
 from __future__ import unicode_literals
-import frappe, re, os, json, imp
+import frappe, re
 import requests, requests.exceptions
-from frappe.utils import strip_html, markdown
+from frappe.utils import strip_html
 from frappe.website.website_generator import WebsiteGenerator
 from frappe.website.router import resolve_route
 from frappe.website.doctype.website_slideshow.website_slideshow import get_slideshow
@@ -26,9 +26,6 @@ class WebPage(WebsiteGenerator):
 		return self.title
 
 	def validate(self):
-		if self.template_path and not getattr(self, "from_website_sync"):
-			frappe.throw(frappe._("Cannot edit templated page"))
-
 		# avoid recursive parent_web_page.
 		if self.parent_web_page == self.page_name:
 			self.parent_web_page = ""
@@ -48,33 +45,19 @@ class WebPage(WebsiteGenerator):
 		context.children = self.get_children()
 		context.parents = self.get_parents(context)
 
-		if self.template_path:
-			# render dynamic context (if .py file exists)
-
-			# get absolute template path considering first fragment as app name
-			if not self.template_path.startswith(os.sep):
-				split_path = self.template_path.split(os.sep)
-
-				self.template_path = os.path.join(frappe.get_app_path(split_path[0]), *split_path[1:])
-
-			context = self.get_dynamic_context(frappe._dict(context))
+		context.update({
+			"style": self.css or "",
+			"script": self.javascript or "",
+			"header": self.header,
+			"title": self.title,
+			"text_align": self.text_align,
+		})
 
-			# load content from template
-			self.get_static_content(context)
-		else:
-			context.update({
-				"style": self.css or "",
-				"script": self.javascript or "",
-				"header": self.header,
-				"title": self.title,
-				"text_align": self.text_align,
-			})
+		if self.description:
+			context.setdefault("metatags", {})["description"] = self.description
 
-			if self.description:
-				context.setdefault("metatags", {})["description"] = self.description
-
-			if not self.show_title:
-				context["no_header"] = 1
+		if not self.show_title:
+			context["no_header"] = 1
 
 		self.set_metatags(context)
 		self.set_breadcrumbs(context)
@@ -183,61 +166,12 @@ class WebPage(WebsiteGenerator):
 			context.main_section = parts1[0] + parts2[1]
 			context.hero = parts2[0]
 
-	def get_static_content(self, context):
-		with open(self.template_path, "r") as contentfile:
-			context.main_section = unicode(contentfile.read(), 'utf-8')
-
-			self.check_for_redirect(context)
-
-			if not context.title:
-				context.title = self.name.replace("-", " ").replace("_", " ").title()
-
-			self.render_dynamic(context)
-
-			if self.template_path.endswith(".md"):
-				if context.main_section:
-					lines = context.main_section.splitlines()
-					first_line = lines[0].strip()
-
-					if first_line.startswith("# "):
-						context.title = first_line[2:]
-						context.main_section = "\n".join(lines[1:])
-
-					context.main_section = markdown(context.main_section, sanitize=False)
-
-		for extn in ("js", "css"):
-			fpath = self.template_path.rsplit(".", 1)[0] + "." + extn
-			if os.path.exists(fpath):
-				with open(fpath, "r") as f:
-					context["style" if extn=="css" else "script"] = f.read()
-
 	def check_for_redirect(self, context):
 		if "<!-- redirect:" in context.main_section:
 			frappe.local.flags.redirect_location = \
 				context.main_section.split("<!-- redirect:")[1].split("-->")[0].strip()
 			raise frappe.Redirect
 
-	def get_dynamic_context(self, context):
-		"update context from `.py` and load sidebar from `_sidebar.json` if either exists"
-		basename = os.path.basename(self.template_path).rsplit(".", 1)[0]
-		module_path = os.path.join(os.path.dirname(self.template_path),
-			frappe.scrub(basename) + ".py")
-
-		if os.path.exists(module_path):
-			module = imp.load_source(basename, module_path)
-			if hasattr(module, "get_context"):
-				ret = module.get_context(context)
-				if ret:
-					context = ret
-
-		# sidebar?
-		sidebar_path = os.path.join(os.path.dirname(self.template_path), "_sidebar.json")
-		if os.path.exists(sidebar_path):
-			with open(sidebar_path, "r") as f:
-				context.children = json.loads(f.read())
-
-		return context
-
 	def set_metatags(self, context):
 		context.metatags = {
 			"name": context.title,
diff --git a/frappe/website/render.py b/frappe/website/render.py
index c44b7de63a..ee3cbb279d 100644
--- a/frappe/website/render.py
+++ b/frappe/website/render.py
@@ -125,7 +125,11 @@ def build_page(path):
 		frappe.local.path = path
 
 	context = get_context(path)
-	html = frappe.get_template(context.template).render(context)
+
+	if context.source:
+		html = frappe.render_template(context.source, context)
+	elif context.template:
+		html = frappe.get_template(context.template).render(context)
 
 	# html = frappe.get_template(context.base_template_path).render(context)
 
diff --git a/frappe/website/router.py b/frappe/website/router.py
index 2bf79a3663..354a4790e6 100644
--- a/frappe/website/router.py
+++ b/frappe/website/router.py
@@ -89,42 +89,143 @@ def get_page_context_from_doctypes():
 	return routes
 
 def get_pages():
-	pages = frappe.cache().get_value("_website_pages")
+	pages = frappe.cache().get_value("_website_pages") if can_cache() else []
+
 	if not pages:
 		pages = []
 		for app in frappe.get_installed_apps():
 			app_path = frappe.get_app_path(app)
+
+			# old
 			path = os.path.join(app_path, "templates", "pages")
-			if os.path.exists(path):
-				for fname in os.listdir(path):
-					fname = frappe.utils.cstr(fname)
-					page_name, extn = fname.rsplit(".", 1)
-					if extn in ("html", "xml", "js", "css"):
-						route_page_name = page_name if extn=="html" else fname
-
-						# add website route
-						route = frappe._dict()
-						route.page_or_generator = "Page"
-						route.template = os.path.relpath(os.path.join(path, fname), app_path)
-						route.name = route.page_name = route_page_name
-						controller_path = os.path.join(path, page_name.replace("-", "_") + ".py")
-
-						if os.path.exists(controller_path):
-							controller = app + "." + os.path.relpath(controller_path,
-								app_path).replace(os.path.sep, ".")[:-3]
-							route.controller = controller
-
-							for fieldname in ("page_title", "no_sitemap"):
-								try:
-									route[fieldname] = frappe.get_attr(controller + "." + fieldname)
-								except AttributeError:
-									pass
-
-						pages.append(route)
+			pages += get_pages_from_path(path, app, app_path)
+
+			# new
+			path = os.path.join(app_path, "www")
+			pages += get_pages_from_path(path, app, app_path)
 
 		frappe.cache().set_value("_website_pages", pages)
 	return pages
 
+def get_pages_from_path(path, app, app_path):
+	pages = []
+	if os.path.exists(path):
+		for basepath, folders, files in os.walk(path):
+			# add missing __init__.py
+			if not '__init__.py' in files:
+				open(os.path.join(basepath, '__init__.py'), 'a').close()
+
+			for fname in files:
+				fname = frappe.utils.cstr(fname)
+				page_name, extn = fname.rsplit(".", 1)
+				if extn in ('js', 'css') and os.path.exists(os.path.join(basepath, fname + '.html')):
+					# js, css is linked to html, skip
+					continue
+
+				if extn in ("html", "xml", "js", "css", "md"):
+					pages.append(get_page_info(path, basepath, app, app_path, fname))
+					# print frappe.as_json(pages[-1])
+
+	return pages
+
+def get_page_info(path, basepath, app, app_path, fname):
+	'''Load page info'''
+	page_name, extn = fname.rsplit(".", 1)
+
+	# add website route
+	page_info = frappe._dict()
+
+	page_info.basename = page_name if extn in ('html', 'md') else fname
+	page_info.page_or_generator = "Page"
+
+	page_info.template = os.path.relpath(os.path.join(basepath, fname), app_path)
+
+	if page_info.basename == 'index' and basepath != path:
+		page_info.basename = ''
+
+	page_info.name = page_info.page_name = os.path.join(os.path.relpath(basepath, path),
+		page_info.basename).strip('/').strip('.').strip('/')
+
+	page_info.controller_path = os.path.join(basepath, page_name.replace("-", "_") + ".py")
+
+	# get the source
+	page_info.source = get_source(page_info, basepath)
+
+	# extract properties from HTML comments
+	if page_info.only_content:
+		load_properties(page_info)
+
+	# controller
+	controller = app + "." + os.path.relpath(page_info.controller_path,
+		app_path).replace(os.path.sep, ".")[:-3]
+	page_info.controller = controller
+
+	return page_info
+
+
+def get_source(page_info, basepath):
+	'''Get the HTML source of the template'''
+	from markdown2 import markdown
+	jenv = frappe.get_jenv()
+	source = jenv.loader.get_source(jenv, page_info.template)[0]
+	html = ''
+
+	if page_info.template.endswith('.md'):
+		source = markdown(source)
+
+	# if only content
+	if page_info.template.endswith('.html') or page_info.template.endswith('.md'):
+		if ('</body>' not in source) and ('{% block' not in source):
+			page_info.only_content = True
+			js, css = '', ''
+
+			js_path = os.path.join(basepath, page_info.basename + '.js')
+			if os.path.exists(js_path):
+				js = unicode(open(js_path, 'r').read(), 'utf-8')
+
+			css_path = os.path.join(basepath, page_info.basename + '.css')
+			if os.path.exists(css_path):
+				js = unicode(open(css_path, 'r').read(), 'utf-8')
+
+			html = '{% extends "templates/web.html" %}'
+
+			if css:
+				html += '\n{% block style %}\n<style>\n' + css + '\n</style>\n{% endblock %}'
+
+			html += '\n{% block page_content %}\n' + source + '\n{% endblock %}'
+
+			if js:
+				html += '\n{% block script %}<script>' + js + '\n</script>\n{% endblock %}'
+		else:
+			html = source
+
+	return html
+
+def load_properties(page_info):
+	'''Load properties like no_cache, title from raw'''
+	import re
+	if "<!-- title:" in page_info.source:
+		page_info.title = re.findall('<!-- title:([^>]*) -->', page_info.source)[0].strip()
+	else:
+		page_info.title = os.path.basename(page_info.name).replace('_', ' ').replace('-', ' ').title()
+
+	if not '{% block title %}' in page_info.source:
+		page_info.source += '\n{% block title %}' + page_info.title + '{% endblock %}'
+
+	if "<!-- no-breadcrumbs -->" in page_info.source:
+		page_info.no_breadcrumbs = 1
+
+	if "<!-- no-header -->" in page_info.source:
+		page_info.no_header = 1
+	else:
+		# every page needs a header
+		# add missing header if there is no <h1> tag
+		if (not '{% block header %}' in page_info.source) and (not '<h1' in page_info.source):
+			page_info.source += '\n{% block header %}<h1>' + page_info.title + '</h1>{% endblock %}'
+
+	if "<!-- no-cache -->" in page_info.source:
+		page_info.no_cache = 1
+
 def process_generators(func):
 	for app in frappe.get_installed_apps():
 		for doctype in frappe.get_hooks("website_generators", app_name = app):
diff --git a/frappe/website/statics.py b/frappe/website/statics.py
deleted file mode 100644
index 66ba02bd5d..0000000000
--- a/frappe/website/statics.py
+++ /dev/null
@@ -1,200 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-from __future__ import unicode_literals
-import frappe, os
-import json
-from frappe.utils import call_hook_method
-
-def sync_statics(rebuild=False):
-	s = sync()
-	s.verbose = True
-	s.start(rebuild)
-	frappe.db.commit()
-
-	# while True:
-	# 	s.start(rebuild)
-	# 	frappe.db.commit()
-	# 	time.sleep(2)
-	# 	rebuild = False
-
-class sync(object):
-	def __init__(self, verbose=False, path=None):
-		self.verbose = verbose
-
-	def start(self, rebuild=False, path="www", apps=None):
-		self.path = path
-		self.synced = []
-		self.synced_paths = []
-		self.updated = 0
-		if rebuild:
-			frappe.db.sql("delete from `tabWeb Page` where ifnull(template_path, '')!=''")
-
-		for app in apps or frappe.get_installed_apps():
-			# print "Syncing for {0}".format(app)
-			self.sync_for_app(app)
-		self.cleanup()
-
-	def sync_for_app(self, app):
-		self.statics_path = frappe.get_app_path(app, self.path)
-		if os.path.exists(self.statics_path):
-			for basepath, folders, files in os.walk(self.statics_path):
-				self.sync_folder(basepath, folders, files, app)
-
-	def sync_folder(self, basepath, folders, files, app):
-		self.get_index_txt(basepath, files)
-		index_found = self.sync_index_page(basepath, files, app)
-
-		if not index_found and basepath!=self.statics_path:
-			# not synced either by generator or by index.html
-			return
-
-		if self.index:
-			self.sync_using_given_index(basepath, folders, files, app)
-		else:
-			self.sync_alphabetically(basepath, folders, [filename for filename in files if filename.endswith('html') or filename.endswith('md')], app)
-
-	def get_index_txt(self, basepath, files):
-		self.index = []
-		if "index.txt" in files:
-			with open(os.path.join(basepath, "index.txt"), "r") as indexfile:
-				self.index = indexfile.read().splitlines()
-
-	def sync_index_page(self, basepath, files, app):
-		for extn in ("md", "html", "json"):
-			fname = "index." + extn
-			if fname in files:
-				self.sync_file(fname, os.path.join(basepath, fname), None, app)
-				return True
-
-	def sync_using_given_index(self, basepath, folders, files, app):
-		for i, page_name in enumerate(self.index):
-			if page_name in folders:
-				# for folder, sync inner index first (so that idx is set)
-				for extn in ("md", "html", "json"):
-					path = os.path.join(basepath, page_name, "index." + extn)
-					if os.path.exists(path):
-						self.sync_file("index." + extn, path, i, app)
-						break
-
-			# other files
-			if page_name + ".md" in files:
-				self.sync_file(page_name + ".md", os.path.join(basepath, page_name + ".md"), i, app)
-			elif page_name + ".html" in files:
-				self.sync_file(page_name + ".html", os.path.join(basepath, page_name + ".html"), i, app)
-			elif page_name + ".json" in files:
-				self.sync_file(page_name + ".json", os.path.join(basepath, page_name + ".json"), i, app)
-			else:
-				if page_name not in folders:
-					print page_name + " not found in " + basepath
-
-	def sync_alphabetically(self, basepath, folders, files, app):
-		files.sort()
-		for fname in files:
-			page_name = fname.rsplit(".", 1)[0]
-			if not (page_name=="index" and basepath!=self.statics_path):
-				self.sync_file(fname, os.path.join(basepath, fname), None, app)
-
-	def sync_file(self, fname, template_path, priority, app):
-		'''sync file into Web Page'''
-		title = None
-		route = os.path.relpath(template_path, self.statics_path).rsplit(".", 1)[0]
-		generated = False
-
-		if fname.rsplit(".", 1)[0]=="index" and \
-			os.path.dirname(template_path) != self.statics_path:
-			route = os.path.dirname(route)
-
-		parent_web_page = frappe.db.sql("""select name from `tabWeb Page` where
-			page_name=%s and ifnull(parent_website_route, '')=ifnull(%s, '')""",
-				(os.path.basename(os.path.dirname(route)), os.path.dirname(os.path.dirname(route))))
-
-		parent_web_page = parent_web_page and parent_web_page[0][0] or ""
-
-		page_name = os.path.basename(route)
-
-		published = 1
-		idx = priority
-
-		if (parent_web_page, page_name) in self.synced:
-			return
-
-		with open(template_path, "r") as f:
-			content = unicode(f.read().strip(), "utf-8")
-			if template_path.endswith('.json'):
-				content = json.loads(content)
-				title = content.get('title')
-				content['page_name'] = page_name
-				content = call_hook_method('build_json_page', content)
-				generated = True
-
-		if not title:
-			title = self.get_title(template_path, content)
-
-		relative_template_path = os.path.join(app, os.path.relpath(template_path, frappe.get_app_path(app)))
-		if not frappe.db.get_value("Web Page", {"template_path":relative_template_path}):
-			web_page = frappe.new_doc("Web Page")
-			web_page.page_name = page_name
-			web_page.parent_web_page = parent_web_page
-			if not generated:
-				web_page.template_path = relative_template_path
-			web_page.main_section = content
-			web_page.title = title
-			web_page.published = published
-			web_page.idx = idx
-			web_page.from_website_sync = True
-			web_page.insert()
-			if self.verbose: print "Inserted: " + web_page.name
-
-		else:
-			web_page = frappe.get_doc("Web Page", {"template_path":relative_template_path})
-			dirty = False
-			for key in ("parent_web_page", "title", "published", "idx"):
-				if web_page.get(key) != locals().get(key):
-					web_page.set(key, locals().get(key))
-					dirty = True
-
-			if web_page.template_path != relative_template_path:
-				web_page.template_path = relative_template_path
-				dirty = True
-
-			if dirty:
-				web_page.from_website_sync = True
-				web_page.save()
-				if self.verbose: print "Updated: " + web_page.name
-
-		self.synced.append((parent_web_page, page_name))
-
-	def get_title(self, fpath, content):
-		if isinstance(content, dict):
-			# content is json
-			return content.get('title')
-
-		title = os.path.basename(fpath).rsplit(".", 1)[0]
-		if title =="index":
-			title = os.path.basename(os.path.dirname(fpath))
-
-		title = title.replace("-", " ").replace("_", " ").title()
-
-		if content.startswith("# "):
-			title = content.splitlines()[0][2:]
-
-		if "<!-- title:" in content:
-			title = content.split("<!-- title:", 1)[1].split("-->", 1)[0].strip()
-
-		return title
-
-	def cleanup(self):
-		if self.synced:
-			# delete static web pages that are not in immediate list
-			for static_page in frappe.db.sql("""select name, page_name, parent_web_page
-				from `tabWeb Page` where ifnull(template_path,'')!=''""", as_dict=1):
-				if (static_page.parent_web_page, static_page.page_name) not in self.synced:
-					frappe.delete_doc("Web Page", static_page.name, force=1)
-		else:
-			# delete all static web pages
-			frappe.delete_doc("Web Page", frappe.db.sql_list("""select name
-				from `tabWeb Page`
-				where ifnull(template_path,'')!=''"""), force=1)
-
-