cleaned up verbosity

frappe/cli.py

@@ -557,6 +557,7 @@ def build_website():
 	import frappe.website.sync
 	frappe.connect()
 	frappe.website.sync.sync()
+	frappe.db.commit()
 	frappe.destroy()
 
 @cmd

frappe/installer.py

@@ -10,8 +10,8 @@ import os, json
 import frappe
 import frappe.database
 import getpass
-from frappe import _
 from frappe.model.db_schema import DbManager
+import frappe.website.sync
 from frappe.model.sync import sync_for
 from frappe.utils.fixtures import sync_fixtures
 
@@ -116,7 +116,8 @@ def install_app(name, verbose=False, set_as_patched=True):
 	for after_install in app_hooks.after_install or []:
 		frappe.get_attr(after_install)()
 
-	sync_fixtures()
+	print "Installing Fixtures..."
+	sync_fixtures(name)
 
 	frappe.flags.in_install_app = False
 
@@ -128,9 +129,10 @@ def add_to_installed_apps(app_name, rebuild_website=True):
 		frappe.db.commit()
 
 		if rebuild_website:
-			import frappe.website.sync
 			frappe.website.sync.sync()
 
+		frappe.db.commit()
+
 		frappe.clear_cache()
 
 def set_all_patches_as_completed(app):

frappe/model/sync.py

@@ -9,7 +9,6 @@ from __future__ import unicode_literals
 import frappe
 import os, sys
 from frappe.modules.import_file import import_file_by_path
-from frappe.utils import get_path, cstr
 from frappe.modules.patch_handler import block_user
 
 def sync_all(force=0, verbose=False):
@@ -23,40 +22,37 @@ def sync_all(force=0, verbose=False):
 	frappe.clear_cache()
 
 def sync_for(app_name, force=0, sync_everything = False, verbose=False):
+	files = []
 	for module_name in frappe.local.app_modules.get(app_name) or []:
 		folder = os.path.dirname(frappe.get_module(app_name + "." + module_name).__file__)
-		walk_and_sync(folder, force, sync_everything, verbose=verbose)
+		files += get_doc_files(folder, force, sync_everything, verbose=verbose)
 
-def walk_and_sync(start_path, force=0, sync_everything = False, verbose=False):
-	"""walk and sync all doctypes and pages"""
-
-	modules = []
-
-	document_type = ['doctype', 'page', 'report', 'print_format']
-
-	for path, folders, files in os.walk(start_path):
-		# sort folders so that doctypes are synced before pages or reports
+	l = len(files)
+	if l:
+		for i, doc_path in enumerate(files):
+			if import_file_by_path(doc_path, force=force) and verbose:
+				complete = int(float(i+1) / l * 40)
+				sys.stdout.write("\rSyncing {0}: [{1}{2}]".format(app_name, "="*complete, " "*(40-complete)))
+				sys.stdout.flush()
+				#print module_name + ' | ' + doctype + ' | ' + name
 
-		for dontwalk in (".git", "locale", "public"):
-			if dontwalk in folders:
-				folders.remove(dontwalk)
+			frappe.db.commit()
 
-		folders.sort()
+		print ""
 
-		if sync_everything or (os.path.basename(os.path.dirname(path)) in document_type):
-			for f in files:
-				f = cstr(f)
-				if f.endswith(".json"):
-					doc_name = f.split(".json")[0]
-					if doc_name == os.path.basename(path):
 
-						module_name = path.split(os.sep)[-3]
-						doctype = path.split(os.sep)[-2]
-						name = path.split(os.sep)[-1]
-
-						if import_file_by_path(os.path.join(path, f), force=force) and verbose:
-							print module_name + ' | ' + doctype + ' | ' + name
-
-						frappe.db.commit()
+def get_doc_files(start_path, force=0, sync_everything = False, verbose=False):
+	"""walk and sync all doctypes and pages"""
 
-	return modules
+	out = []
+	document_type = ['doctype', 'page', 'report', 'print_format']
+	for doctype in document_type:
+		doctype_path = os.path.join(start_path, doctype)
+		if os.path.exists(doctype_path):
+			for docname in os.listdir(doctype_path):
+				if os.path.isdir(os.path.join(doctype_path, docname)):
+					doc_path = os.path.join(doctype_path, docname, docname) + ".json"
+					if os.path.exists(doc_path):
+						out.append(doc_path)
+
+	return out

frappe/templates/includes/static_index.html

@@ -7,7 +7,7 @@
 				<div class="row">
 					<div class="col-xs-1 text-right" style="padding-right: 0px;"><b>{{ loop.index }}.</b></div>
 					<div class="col-xs-11">
-						<a href="{{ item.name }}">{{ item.page_title }}</a>
+						<a href="/{{ item.name }}">{{ item.page_title }}</a>
 					</div>
 				</div>
 			</li>

frappe/utils/fixtures.py

@@ -6,8 +6,12 @@ from __future__ import unicode_literals
 import frappe, os
 from frappe.core.page.data_import_tool.data_import_tool import import_doc, export_fixture, export_csv
 
-def sync_fixtures():
-	for app in frappe.get_installed_apps():
+def sync_fixtures(app=None):
+	if app:
+		apps = [app]
+	else:
+		apps = frappe.get_installed_apps()
+	for app in apps:
 		if os.path.exists(frappe.get_app_path(app, "fixtures")):
 			for fname in os.listdir(frappe.get_app_path(app, "fixtures")):
 				if fname.endswith(".json") or fname.endswith(".csv"):

frappe/website/statics.py

@@ -2,15 +2,15 @@
 # MIT License. See license.txt
 
 from __future__ import unicode_literals
-import frappe, os, time
+import frappe, os, time, sys
 
-from frappe import _
 from frappe.utils import cint
 from markdown2 import markdown
-from frappe.website.sitemap import get_route_children, get_next
+# from frappe.website.sitemap import get_route_children, get_next
 
 def sync_statics(rebuild=False):
 	s = sync()
+	s.verbose = True
 	while True:
 		s.start(rebuild)
 		frappe.db.commit()
@@ -19,17 +19,19 @@ def sync_statics(rebuild=False):
 
 class sync(object):
 	def start(self, rebuild=False):
+		self.verbose = False
 		self.synced = []
+		self.synced_paths = []
+		self.to_insert = []
+		self.to_update = []
 		self.updated = 0
 		self.rebuild = rebuild
 		for app in frappe.get_installed_apps():
 			self.sync_for_app(app)
 
+		self.insert_and_update()
 		self.cleanup()
 
-		if self.updated:
-			print str(self.updated) + " files updated"
-
 	def sync_for_app(self, app):
 		self.statics_path = frappe.get_app_path(app, "templates", "statics")
 		if os.path.exists(self.statics_path):
@@ -38,17 +40,15 @@ class sync(object):
 
 
 	def sync_folder(self, basepath, folders, files):
-		folder_route = os.path.relpath(basepath, self.statics_path)
 		self.get_index_txt(basepath, files)
-		self.sync_index_page(basepath, files)
+		index_found = self.sync_index_page(basepath, files)
 
-		if not frappe.db.exists("Website Route", folder_route) and basepath!=self.statics_path:
+		if not index_found and basepath!=self.statics_path:
 			# not synced either by generator or by index.html
 			return
 
 		if self.index:
 			self.sync_using_given_index(basepath, folders, files)
-
 		else:
 			self.sync_alphabetically(basepath, folders, [filename for filename in files if filename.endswith('html') or filename.endswith('md')])
 
@@ -64,7 +64,7 @@ class sync(object):
 			fname = "index." + extn
 			if fname in files:
 				self.sync_file(fname, os.path.join(basepath, fname), None)
-				return
+				return True
 
 	def sync_using_given_index(self, basepath, folders, files):
 		for i, page_name in enumerate(self.index):
@@ -108,12 +108,42 @@ class sync(object):
 			["name", "idx", "static_file_timestamp", "docname"], as_dict=True)
 
 		if route_details:
-			self.update_web_page(route_details, fpath, priority, parent_website_route)
+			page = self.get_route_details_for_update(route_details, fpath,
+				priority, parent_website_route)
+			if page:
+				self.to_update.append(page)
 		else:
 			# Route does not exist, new page
-			self.insert_web_page(route, fpath, page_name, priority, parent_website_route)
+			page = self.get_web_page_for_insert(route, fpath, page_name,
+				priority, parent_website_route)
+			self.to_insert.append(page)
 
-	def insert_web_page(self, route, fpath, page_name, priority, parent_website_route):
+		self.synced.append(route)
+
+	def insert_and_update(self):
+		if self.to_insert:
+			for i, page in enumerate(self.to_insert):
+				if self.verbose:
+					print "Inserting " + page.route
+				else:
+					sys.stdout.write("\rInserting statics {0}/{1}".format(i+1, len(self.to_insert)))
+					sys.stdout.flush()
+
+				self.insert_web_page(page)
+			if not self.verbose: print ""
+
+		if self.to_update:
+			for i, route_details in enumerate(self.to_update):
+				if self.verbose:
+					print "Updating " + route_details.name
+				else:
+					sys.stdout.write("\rUpdating statics {0}/{1}".format(i+1, len(self.to_update)))
+					sys.stdout.flush()
+
+				self.update_web_page(route_details)
+			if not self.verbose: print ""
+
+	def get_web_page_for_insert(self, route, fpath, page_name, priority, parent_website_route):
 		page = frappe.get_doc({
 			"doctype":"Web Page",
 			"idx": priority,
@@ -122,61 +152,66 @@ class sync(object):
 			"parent_website_route": parent_website_route
 		})
 
+		page.fpath = fpath
+		page.route = route
 		page.update(get_static_content(fpath, page_name, route))
+		return page
 
+	def insert_web_page(self, page):
 		try:
 			page.insert()
-		except frappe.NameError:
+		except frappe.NameError, e:
+			print e
 			# page exists, if deleted static, delete it and try again
 			old_route = frappe.get_doc("Website Route", {"ref_doctype":"Web Page",
 				"docname": page.name})
-			if old_route.static_file_timestamp and not os.path.exists(os.path.join(self.statics_path,
-				old_route.name)):
-
+			if old_route.static_file_timestamp and \
+				not os.path.exists(os.path.join(self.statics_path, old_route.name)):
 				frappe.delete_doc("Web Page", page.name)
 				page.insert() # retry
 
-
 		# update timestamp
 		route_doc = frappe.get_doc("Website Route", {"ref_doctype": "Web Page",
 			"docname": page.name})
-		route_doc.static_file_timestamp = cint(os.path.getmtime(fpath))
+		route_doc.static_file_timestamp = cint(os.path.getmtime(page.fpath))
 		route_doc.save()
 
-		self.updated += 1
-		print route_doc.name + " inserted"
-		self.synced.append(route)
-
-	def update_web_page(self, route_details, fpath, priority, parent_website_route):
+	def get_route_details_for_update(self, route_details, fpath, priority, parent_website_route):
+		out = None
 		if not route_details.docname:
 			print "Ignoring {0} because page found".format(route_details.name)
 			return
+
 		if str(cint(os.path.getmtime(fpath)))!= route_details.static_file_timestamp \
 			or (cint(route_details.idx) != cint(priority) and (priority is not None) \
 			or self.rebuild):
 
-			page = frappe.get_doc("Web Page", route_details.docname)
-			page.update(get_static_content(fpath, route_details.docname, route_details.name))
-			page.idx = priority
-			page.save()
+			out = route_details
+			out.idx = priority
+			out.fpath = fpath
 
-			route_doc = frappe.get_doc("Website Route", route_details.name)
-			route_doc.static_file_timestamp = cint(os.path.getmtime(fpath))
-			route_doc.save()
+		return out
 
-			print route_doc.name + " updated"
-			self.updated += 1
+	def update_web_page(self, route_details):
+		page = frappe.get_doc("Web Page", route_details.docname)
+		page.update(get_static_content(route_details.fpath,
+			route_details.docname, route_details.name))
+		page.save()
 
-		self.synced.append(route_details.name)
+		route_doc = frappe.get_doc("Website Route", route_details.name)
+		route_doc.static_file_timestamp = cint(os.path.getmtime(route_details.fpath))
+		route_doc.save()
 
 	def cleanup(self):
 		if self.synced:
+			# delete static web pages that are not in immediate list
 			frappe.delete_doc("Web Page", frappe.db.sql_list("""select docname
 				from `tabWebsite Route`
 				where ifnull(static_file_timestamp,'')!='' and name not in ({})
 				order by (rgt-lft) asc""".format(', '.join(["%s"]*len(self.synced))),
 					tuple(self.synced)))
 		else:
+			# delete all static web pages
 			frappe.delete_doc("Web Page", frappe.db.sql_list("""select docname
 				from `tabWebsite Route`
 				where ifnull(static_file_timestamp,'')!=''
@@ -201,21 +236,21 @@ def get_static_content(fpath, docname, route):
 					d.title = first_line[2:]
 					content = "\n".join(lines[1:])
 
-				if "{index}" in content:
-					children = get_route_children(route)
-					html = frappe.get_template("templates/includes/static_index.html").render({
-							"items":children})
-					content = content.replace("{index}", html)
-
-				if "{next}" in content:
-					next_item = get_next(route)
-					html = ""
-					if next_item:
-						html = '''<p>
-							<br><a href="{name}" class="btn btn-primary">
-								{page_title} <i class="icon-chevron-right"></i></a>
-						</p>'''.format(**next_item)
-					content = content.replace("{next}", html)
+				# if "{index}" in content:
+				# 	children = get_route_children(route)
+				# 	html = frappe.get_template("templates/includes/static_index.html").render({
+				# 			"items":children})
+				# 	content = content.replace("{index}", html)
+				#
+				# if "{next}" in content:
+				# 	next_item = get_next(route)
+				# 	html = ""
+				# 	if next_item:
+				# 		html = '''<p>
+				# 			<br><a href="{name}" class="btn btn-primary">
+				# 				{page_title} <i class="icon-chevron-right"></i></a>
+				# 		</p>'''.format(**next_item)
+				# 	content = content.replace("{next}", html)
 
 				content = markdown(content)
 

frappe/website/sync.py

@@ -2,7 +2,7 @@
 # MIT License. See license.txt
 
 from __future__ import unicode_literals
-import frappe, os
+import frappe, os, sys
 from frappe.modules import load_doctype_module
 import statics, render
 
@@ -21,29 +21,48 @@ def sync(app=None):
 	# delete all routes (resetting)
 	frappe.db.sql("delete from `tabWebsite Route`")
 
+	print "Finding routes..."
+	routes, generators = [], []
 	for app in apps:
-		print "Importing pages from " + app
-		sync_pages(app)
-		print "Importing generators from " + app
-		sync_generators(app)
+		routes += get_sync_pages(app)
+		generators += get_sync_generators(app)
+
+	sync_pages(routes)
+	sync_generators(generators)
 
 	# sync statics
-	print "Importing statics"
 	statics_sync = statics.sync()
 	statics_sync.start()
 
-	# rebuild for new inserts
-	statics_sync.start(rebuild=True)
+def sync_pages(routes):
+	l = len(routes)
+	if l:
+		for i, r in enumerate(routes):
+			r.insert(ignore_permissions=True)
+			sys.stdout.write("\rUpdating pages {0}/{1}".format(i+1, l))
+			sys.stdout.flush()
+		print ""
+
+def sync_generators(generators):
+	l = len(generators)
+	if l:
+		for i, g in enumerate(generators):
+			doc = frappe.get_doc(g[0], g[1])
+			doc.save(ignore_permissions=True)
+			sys.stdout.write("\rUpdating generators {0}/{1}".format(i+1, l))
+			sys.stdout.flush()
+		print ""
 
-def sync_pages(app):
+def get_sync_pages(app):
 	app_path = frappe.get_app_path(app)
+	pages = []
 
 	path = os.path.join(app_path, "templates", "pages")
 	if os.path.exists(path):
 		for fname in os.listdir(path):
 			fname = frappe.utils.cstr(fname)
-			page_name = fname.rsplit(".", 1)[0]
-			if fname.split(".")[-1] in ("html", "xml", "js", "css"):
+			page_name, extn = fname.rsplit(".", 1)
+			if extn in ("html", "xml", "js", "css"):
 				# add website route
 				route = frappe.new_doc("Website Route")
 				route.page_or_generator = "Page"
@@ -56,9 +75,12 @@ def sync_pages(app):
 						app_path).replace(os.path.sep, ".")[:-3]
 					route.controller = controller
 
-				route.insert(ignore_permissions=True)
+				pages.append(route)
 
-def sync_generators(app):
+	return pages
+
+def get_sync_generators(app):
+	generators = []
 	for doctype in frappe.get_hooks("website_generators", app_name = app):
 		condition, order_by = "", "name asc"
 		module = load_doctype_module(doctype)
@@ -68,6 +90,6 @@ def sync_generators(app):
 			order_by = "{0} {1}".format(module.sort_by, module.sort_order)
 		for name in frappe.db.sql_list("select name from `tab{0}` {1} order by {2}".format(doctype,
 			condition, order_by)):
-			doc = frappe.get_doc(doctype, name)
-			print doctype, name
-			doc.save(ignore_permissions=True)
+			generators.append((doctype, name))
+
+	return generators

frappe/website/template.py

@@ -10,6 +10,8 @@ from jinja2.utils import concat
 from jinja2 import meta
 import re
 
+from sitemap import get_next
+
 def render_blocks(context):
 	"""returns a dict of block name and its rendered content"""
 
@@ -52,6 +54,20 @@ def render_blocks(context):
 	if "<!-- title:" in out.get("content", ""):
 		out["title"] = re.findall('<!-- title:([^>]*) -->', out.get("content"))[0].strip()
 
+	if "{index}" in out.get("content", "") and context.get("children"):
+		html = frappe.get_template("templates/includes/static_index.html").render({
+				"items": context["children"]})
+		out["content"] = out["content"].replace("{index}", html)
+
+	if "{next}" in out.get("content", ""):
+		next_item = get_next(context["pathname"])
+		if next_item:
+			if next_item.name[0]!="/": next_item.name = "/" + next_item.name
+			html = '''<p><br><a href="{name}" class="btn btn-primary">
+				{page_title} <i class="icon-chevron-right"></i></a>
+			</p>'''.format(**next_item)
+			out["content"] = out["content"].replace("{next}", html)
+
 	if "sidebar" not in out and not out.get("no_sidebar"):
 		out["sidebar"] = scrub_relative_urls(
 			frappe.get_template("templates/includes/sidebar.html").render(context))