New Build System using NodeJS (#3117)

* added build.js

* js babelify and concat, css concat working

* less wip

* live reload css working

* css reload on less change working

* added watch_js

* setup file watcher only in dev mode, don't compile variables.less

* Minify js files using babili

- Add --minify flag

* Set minify to false as default

* [minor] Remove redundant code

* Used subprocess instead of os.system

- Also added experimental flag

frappe/build.js

@@ -0,0 +1,301 @@
+const path = require('path');
+const fs = require('fs');
+const babel = require('babel-core');
+const less = require('less');
+const chokidar = require('chokidar');
+
+// for file watcher
+const app = require('express')();
+const http = require('http').Server(app);
+const io = require('socket.io')(http);
+const file_watcher_port = 6787;
+
+const p = path.resolve;
+
+// basic setup
+const sites_path = p(__dirname, '..', '..', '..', 'sites');
+const apps_path = p(__dirname, '..', '..', '..', 'apps'); // the apps folder
+const apps_contents = fs.readFileSync(p(sites_path, 'apps.txt'), 'utf8');
+const apps = apps_contents.split('\n');
+const app_paths = apps.map(app => p(apps_path, app, app)) // base_path of each app
+const assets_path = p(sites_path, 'assets');
+const build_map = make_build_map();
+
+// command line args
+const action = process.argv[2] || '--build';
+
+if (!['--build', '--watch'].includes(action)) {
+	console.log('Invalid argument: ', action);
+	return;
+}
+
+if (action === '--build') {
+	const minify = process.argv[3] === '--minify' ? true : false;
+	build({ minify });
+}
+
+if (action === '--watch') {
+	watch();
+}
+
+function build({ minify=false } = {}) {
+	for (const output_path in build_map) {
+		pack(output_path, build_map[output_path], minify);
+	}
+}
+
+let socket_connection = false;
+
+function watch() {
+	http.listen(file_watcher_port, function () {
+		console.log('file watching on *:', file_watcher_port);
+	});
+
+	compile_less().then(() => {
+		build();
+		watch_less(function (filename) {
+			if(socket_connection) {
+				io.emit('reload_css', filename);
+			}
+		});
+		watch_js(function (filename) {
+			if(socket_connection) {
+				io.emit('reload_js', filename);
+			}
+		});
+	});
+
+	io.on('connection', function (socket) {
+		socket_connection = true;
+
+		socket.on('disconnect', function() {
+			socket_connection = false;
+		})
+	});
+}
+
+function pack(output_path, inputs, minify) {
+	const output_type = output_path.split('.').pop();
+
+	let output_txt = '';
+	for (const file of inputs) {
+
+		if (!fs.existsSync(file)) {
+			console.log('File not found: ', file);
+			continue;
+		}
+
+		let file_content = fs.readFileSync(file, 'utf-8');
+
+		if (file.endsWith('.html') && output_type === 'js') {
+			file_content = html_to_js_template(file, file_content);
+		}
+
+		if(file.endsWith('class.js')) {
+			file_content = minify_js(file_content, file);
+		}
+
+		if (file.endsWith('.js') && !file.includes('/lib/') && output_type === 'js' && !file.endsWith('class.js')) {
+			file_content = babelify(file_content, file, minify);
+		}
+
+		if(!minify) {
+			output_txt += `\n/*\n *\t${file}\n */\n`
+		}
+		output_txt += file_content;
+	}
+
+	const target = p(assets_path, output_path);
+
+	try {
+		fs.writeFileSync(target, output_txt);
+		console.log(`Wrote ${output_path} - ${get_file_size(target)}`);
+		return target;
+	} catch (e) {
+		console.log('Error writing to file', output_path);
+		console.log(e);
+	}
+}
+
+function babelify(content, path, minify) {
+	let presets = ['es2015', 'es2016'];
+	if(minify) {
+		presets.push('babili'); // new babel minifier
+	}
+	try {
+		return babel.transform(content, {
+			presets: presets,
+			comments: false
+		}).code;
+	} catch (e) {
+		console.log('Cannot babelify', path);
+		console.log(e);
+		return content;
+	}
+}
+
+function minify_js(content, path) {
+	try {
+		return babel.transform(content, {
+			comments: false
+		}).code;
+	} catch (e) {
+		console.log('Cannot minify', path);
+		console.log(e);
+		return content;
+	}
+}
+
+function make_build_map() {
+	const build_map = {};
+	for (const app_path of app_paths) {
+		const build_json_path = p(app_path, 'public', 'build.json');
+		if (!fs.existsSync(build_json_path)) continue;
+
+		let build_json = fs.readFileSync(build_json_path);
+		try {
+			build_json = JSON.parse(build_json);
+		} catch (e) {
+			console.log(e);
+			continue;
+		}
+
+		for (const target in build_json) {
+			const sources = build_json[target];
+
+			const new_sources = [];
+			for (const source of sources) {
+				const s = p(app_path, source);
+				new_sources.push(s);
+			}
+
+			if (new_sources.length)
+				build_json[target] = new_sources;
+			else
+				delete build_json[target];
+		}
+
+		Object.assign(build_map, build_json);
+	}
+	return build_map;
+}
+
+function compile_less() {
+	return new Promise(function (resolve) {
+		const promises = [];
+		for (const app_path of app_paths) {
+			const public_path = p(app_path, 'public');
+			const less_path = p(public_path, 'less');
+			if (!fs.existsSync(less_path)) continue;
+
+			const files = fs.readdirSync(less_path);
+			for (const file of files) {
+				if(file.includes('variables.less')) continue;
+				promises.push(compile_less_file(file, less_path, public_path))
+			}
+		}
+
+		Promise.all(promises).then(() => {
+			console.log('Less files compiled');
+			resolve();
+		});
+	});
+}
+
+function compile_less_file(file, less_path, public_path) {
+	const file_content = fs.readFileSync(p(less_path, file), 'utf8');
+	const output_file = file.split('.')[0] + '.css';
+	console.log('compiling', file);
+
+	return less.render(file_content, {
+		paths: [less_path],
+		filename: file,
+		sourceMap: false
+	}).then(output => {
+		const out_css = p(public_path, 'css', output_file);
+		fs.writeFileSync(out_css, output.css);
+		return out_css;
+	}).catch(e => {
+		console.log('Error compiling ', file);
+		console.log(e);
+	});
+}
+
+function watch_less(ondirty) {
+	const less_paths = app_paths.map(path => p(path, 'public', 'less'));
+
+	const to_watch = [];
+	for (const less_path of less_paths) {
+		if (!fs.existsSync(less_path)) continue;
+		to_watch.push(less_path);
+	}
+	chokidar.watch(to_watch).on('change', (filename, stats) => {
+		console.log(filename, 'dirty');
+		var last_index = filename.lastIndexOf('/');
+		const less_path = filename.slice(0, last_index);
+		const public_path = p(less_path, '..');
+		filename = filename.split('/').pop();
+
+		compile_less_file(filename, less_path, public_path)
+		.then(css_file_path => {
+			// build the target css file for which this css file is input
+			for (const target in build_map) {
+				const sources = build_map[target];
+				if (sources.includes(css_file_path)) {
+					pack(target, sources);
+					ondirty && ondirty(target);
+					break;
+				}
+			}
+		})
+	});
+}
+
+function watch_js(ondirty) {
+	const js_paths = app_paths.map(path => p(path, 'public', 'js'));
+
+	const to_watch = [];
+	for (const js_path of js_paths) {
+		if (!fs.existsSync(js_path)) continue;
+		to_watch.push(js_path);
+	}
+	chokidar.watch(to_watch).on('change', (filename, stats) => {
+		console.log(filename, 'dirty');
+		var last_index = filename.lastIndexOf('/');
+		const js_path = filename.slice(0, last_index);
+		const public_path = p(js_path, '..');
+
+		// build the target js file for which this js/html file is input
+		for (const target in build_map) {
+			const sources = build_map[target];
+			if (sources.includes(filename)) {
+				pack(target, sources);
+				ondirty && ondirty(target);
+				break;
+			}
+		}
+	});
+}
+
+function html_to_js_template(path, content) {
+	let key = path.split('/');
+	key = key[key.length - 1];
+	key = key.split('.')[0];
+
+	content = scrub_html_template(content);
+	return `frappe.templates['${key}'] = '${content}';\n`;
+}
+
+function scrub_html_template(content) {
+	content = content.replace(/\s/g, ' ');
+	content = content.replace(/(<!--.*?-->)/g, '');
+	return content.replace("'", "\'");
+}
+
+function get_file_size(filepath) {
+	const stats = fs.statSync(filepath);
+	const size = stats.size;
+	// convert it to humanly readable format.
+	const i = Math.floor(Math.log(size) / Math.log(1024));
+	return (size / Math.pow(1024, i)).toFixed(2) * 1 + ' ' + ['B', 'KB', 'MB', 'GB', 'TB'][i];
+}

frappe/build.py

@@ -3,6 +3,7 @@
 
 from __future__ import unicode_literals
 from frappe.utils.minify import JavascriptMinify
+import subprocess
 
 """
 Build the `public` folders and setup languages
@@ -22,16 +23,30 @@ def setup():
 		except ImportError: pass
 	app_paths = [os.path.dirname(pymodule.__file__) for pymodule in pymodules]
 
-def bundle(no_compress, make_copy=False, verbose=False):
+def bundle(no_compress, make_copy=False, verbose=False, experimental=False):
 	"""concat / minify js files"""
 	# build js files
 	setup()
 
 	make_asset_dirs(make_copy=make_copy)
+
+	if experimental:
+		command = 'node ../apps/frappe/frappe/build.js --build'
+		if not no_compress:
+			command += ' --minify'
+		subprocess.call(command.split(' '))
+		return
+
 	build(no_compress, verbose)
 
-def watch(no_compress):
+def watch(no_compress, experimental=False):
 	"""watch and rebuild if necessary"""
+
+	if experimental:
+		command = 'node ../apps/frappe/frappe/build.js --watch'
+		subprocess.Popen(command.split(' '))
+		return
+
 	setup()
 
 	import time
@@ -101,7 +116,6 @@ def get_build_maps():
 				except ValueError, e:
 					print path
 					print 'JSON syntax error {0}'.format(str(e))
-
 	return build_maps
 
 timestamps = {}

frappe/commands/utils.py

@@ -8,19 +8,21 @@ from frappe.commands import pass_context, get_site
 @click.command('build')
 @click.option('--make-copy', is_flag=True, default=False, help='Copy the files instead of symlinking')
 @click.option('--verbose', is_flag=True, default=False, help='Verbose')
-def build(make_copy=False, verbose=False):
+@click.option('--experimental', is_flag=True, default=False, help='Use the new NodeJS build system')
+def build(make_copy=False, verbose=False, experimental=False):
 	"Minify + concatenate JS and CSS files, build translations"
 	import frappe.build
 	import frappe
 	frappe.init('')
-	frappe.build.bundle(False, make_copy=make_copy, verbose=verbose)
+	frappe.build.bundle(False, make_copy=make_copy, verbose=verbose, experimental=experimental)
 
 @click.command('watch')
-def watch():
+@click.option('--experimental', is_flag=True, default=False, help='Use the new NodeJS build system')
+def watch(experimental=False):
 	"Watch and concatenate JS and CSS files as and when they change"
 	import frappe.build
 	frappe.init('')
-	frappe.build.watch(True)
+	frappe.build.watch(True, experimental=experimental)
 
 @click.command('clear-cache')
 @pass_context

frappe/public/build.json

@@ -6,6 +6,7 @@
 		"public/css/avatar.css"
 	],
 	"js/frappe-web.min.js": [
+		"public/js/frappe/class.js",
 		"public/js/lib/bootstrap.min.js",
 		"public/js/lib/md5.min.js",
 		"public/js/frappe/provide.js",
@@ -16,7 +17,6 @@
 		"public/js/frappe/misc/pretty_date.js",
 		"public/js/lib/moment/moment.min.js",
 		"public/js/lib/highlight.pack.js",
-		"public/js/frappe/class.js",
 		"public/js/lib/microtemplate.js",
 		"public/js/frappe/query_string.js",
 		"website/js/website.js",
@@ -75,8 +75,8 @@
 		"public/js/lib/datepicker/locale-all.js"
 	],
 	"js/desk.min.js": [
-		"public/js/frappe/provide.js",
 		"public/js/frappe/class.js",
+		"public/js/frappe/provide.js",
 		"public/js/frappe/assets.js",
 		"public/js/frappe/format.js",
 		"public/js/frappe/form/formatters.js",

frappe/public/js/frappe/router.js

@@ -116,10 +116,11 @@ frappe.get_route_str = function(route) {
 }
 
 frappe.set_route = function() {
-	if(arguments.length===1 && $.isArray(arguments[0])) {
-		arguments = arguments[0];
+	var params = arguments;
+	if(params.length===1 && $.isArray(params[0])) {
+		params = params[0];
 	}
-	route = $.map(arguments, function(a) {
+	route = $.map(params, function(a) {
 		if($.isPlainObject(a)) {
 			frappe.route_options = a;
 			return null;

frappe/public/js/frappe/socketio_client.js

@@ -11,6 +11,11 @@ frappe.socket = {
 			return;
 		}
 
+		if (frappe.boot.developer_mode) {
+			// File watchers for development
+			frappe.socket.setup_file_watchers();
+		}
+
 		//Enable secure option when using HTTPS
 		if (window.location.protocol == "https:") {
    			frappe.socket.socket = io.connect(frappe.socket.get_host(), {secure: true});
@@ -186,7 +191,38 @@ frappe.socket = {
 				}
 			}, 5000);
 		});
+	},
+	setup_file_watchers: function() {
+		var host = window.location.origin;
+		var port = '6787';
+		// remove the port number from string
+		host = host.split(':').slice(0, -1).join(":");
+		host = host + ':' + port;
 
+		frappe.socket.file_watcher = io.connect(host);
+		// css files auto reload
+		frappe.socket.file_watcher.on('reload_css', function(filename) {
+			let abs_file_path = "assets/" + filename;
+			const link = $(`link[href*="${abs_file_path}"]`);
+			abs_file_path = abs_file_path.split('?')[0] + '?v=' + +moment();
+			link.attr('href', abs_file_path);
+			frappe.show_alert({
+				indicator: 'orange',
+				message: filename + ' reloaded'
+			}, 5);
+		});
+		// js files show alert
+		frappe.socket.file_watcher.on('reload_js', function(filename) {
+			filename = "assets/" + filename;
+			var msg = $(`
+				<span>${filename} changed <a data-action="reload">Click to Reload</a></span>
+			`)
+			msg.find('a').click(frappe.ui.toolbar.clear_cache);
+			frappe.show_alert({
+				indicator: 'orange',
+				message: msg
+			}, 5);
+		});
 	},
 	process_response: function(data, method) {
 		if(!data) {

frappe/public/js/frappe/ui/messages.js

@@ -250,7 +250,7 @@ frappe.hide_progress = function() {
 }
 
 // Floating Message
-frappe.show_alert = function(message, seconds) {
+frappe.show_alert = function(message, seconds=7) {
 	if(typeof message==='string') {
 		message = {
 			message: message
@@ -261,14 +261,19 @@ frappe.show_alert = function(message, seconds) {
 	}
 
 	if(message.indicator) {
-		message_html = '<span class="indicator ' + message.indicator + '">' + message.message + '</span>';
+		message_html = $('<span class="indicator ' + message.indicator + '"></span>').append(message.message);
 	} else {
 		message_html = message.message;
 	}
 
-	var div = $(repl('<div class="alert desk-alert" style="display: none;">'
-			+ '<span class="alert-message">%(txt)s</span><a class="close">&times;</a>'
-		+ '</div>', {txt: message_html}))
+	var div = $(`
+		<div class="alert desk-alert">
+			<span class="alert-message"></span><a class="close">&times;</a>
+		</div>`);
+
+	div.find('.alert-message').append(message_html);
+
+	div.hide()
 		.appendTo("#alert-container")
 		.fadeIn(300);
 
@@ -277,7 +282,7 @@ frappe.show_alert = function(message, seconds) {
 		return false;
 	});
 
-	div.delay(seconds ? seconds * 1000 : 7000).fadeOut(300);
+	div.delay(seconds * 1000).fadeOut(300);
 	return div;
 }
 

frappe/public/js/frappe/views/image/image_view.js

@@ -68,8 +68,7 @@ frappe.views.ImageView = frappe.views.ListRenderer.extend({
 			gallery.show(name);
 			return false;
 		});
-	},
-	refresh: this.render_view
+	}
 });
 
 frappe.views.GalleryView = Class.extend({

frappe/public/js/legacy/globals.js

@@ -39,7 +39,7 @@ var user_img = {};
 var _f = {};
 var _p = {};
 var _r = {};
-var FILTER_SEP = '\1';
+// var FILTER_SEP = '\1';
 
 // API globals
 var frms={};

socketio.js

@@ -293,3 +293,5 @@ function get_conf() {
 
 	return conf;
 }
+
+