Merge pull request #4100 from rmehta/file-upload-socket

[feature] File upload using Socket.io by making chunks

frappe/boot.py

@@ -30,6 +30,7 @@ def get_bootinfo():
 	get_user(bootinfo)
 
 	# system info
+	bootinfo.sitename = frappe.local.site
 	bootinfo.sysdefaults = frappe.defaults.get_defaults()
 	bootinfo.user_permissions = get_user_permissions()
 	bootinfo.server_date = frappe.utils.nowdate()

frappe/core/page/data_import_tool/importer.py

@@ -217,8 +217,8 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 
 	# header
 	if not rows:
-		from frappe.utils.file_manager import save_uploaded
-		file_doc = save_uploaded(dt=None, dn="Data Import", folder='Home', is_private=1)
+		from frappe.utils.file_manager import get_file_doc
+		file_doc = get_file_doc(dt='', dn="Data Import", folder='Home', is_private=1)
 		filename, file_extension = os.path.splitext(file_doc.file_name)
 
 		if file_extension == '.xlsx' and from_data_import == 'Yes':

frappe/public/css/sidebar.css

@@ -94,9 +94,6 @@ body[data-route^="Module"] .main-menu .form-sidebar {
   position: absolute;
   right: 5px;
 }
-.form-sidebar .attachment-row a.close {
-  margin-top: -5px;
-}
 .form-sidebar .form-shared .share-doc-btn,
 .form-sidebar .form-viewers .share-doc-btn {
   cursor: pointer;

frappe/public/js/frappe/desk.js

@@ -29,7 +29,7 @@ frappe.Application = Class.extend({
 		this.startup();
 	},
 	startup: function() {
-		frappe.socket.init();
+		frappe.socketio.init();
 		frappe.model.init();
 
 		if(frappe.boot.status==='failed') {

frappe/public/js/frappe/form/controls/text_editor.js

@@ -137,17 +137,17 @@ frappe.ui.form.ControlTextEditor = frappe.ui.form.ControlCode.extend({
 			});
 	},
 	get_image: function (fileobj, callback) {
-		var freader = new FileReader();
+		var reader = new FileReader();
 
-		freader.onload = function() {
-			var dataurl = freader.result;
+		reader.onload = function() {
+			var dataurl = reader.result;
 			// add filename to dataurl
 			var parts = dataurl.split(",");
 			parts[0] += ";filename=" + fileobj.name;
 			dataurl = parts[0] + ',' + parts[1];
 			callback(dataurl);
 		};
-		freader.readAsDataURL(fileobj);
+		reader.readAsDataURL(fileobj);
 	},
 	hide_elements_on_mobile: function() {
 		this.note_editor.find('.note-btn-underline,\

frappe/public/js/frappe/request.js

@@ -34,7 +34,7 @@ frappe.call = function(opts) {
 	var callback = function(data, response_text) {
 		if(data.task_id) {
 			// async call, subscribe
-			frappe.socket.subscribe(data.task_id, opts);
+			frappe.socketio.subscribe(data.task_id, opts);
 
 			if(opts.queued) {
 				opts.queued(data);

frappe/public/js/frappe/socketio_client.js

@@ -1,4 +1,4 @@
-frappe.socket = {
+frappe.socketio = {
 	open_tasks: {},
 	open_docs: [],
 	emit_queue: [],
@@ -7,40 +7,40 @@ frappe.socket = {
 			return;
 		}
 
-		if (frappe.socket.socket) {
+		if (frappe.socketio.socket) {
 			return;
 		}
 
 		if (frappe.boot.developer_mode) {
 			// File watchers for development
-			frappe.socket.setup_file_watchers();
+			frappe.socketio.setup_file_watchers();
 		}
 
 		//Enable secure option when using HTTPS
 		if (window.location.protocol == "https:") {
-			frappe.socket.socket = io.connect(frappe.socket.get_host(), {secure: true});
+			frappe.socketio.socket = io.connect(frappe.socketio.get_host(), {secure: true});
 		}
 		else if (window.location.protocol == "http:") {
-			frappe.socket.socket = io.connect(frappe.socket.get_host());
+			frappe.socketio.socket = io.connect(frappe.socketio.get_host());
 		}
 		else if (window.location.protocol == "file:") {
-			frappe.socket.socket = io.connect(window.localStorage.server);
+			frappe.socketio.socket = io.connect(window.localStorage.server);
 		}
 
-		if (!frappe.socket.socket) {
-			console.log("Unable to connect to " + frappe.socket.get_host());
+		if (!frappe.socketio.socket) {
+			console.log("Unable to connect to " + frappe.socketio.get_host());
 			return;
 		}
 
-		frappe.socket.socket.on('msgprint', function(message) {
+		frappe.socketio.socket.on('msgprint', function(message) {
 			frappe.msgprint(message);
 		});
 
-		frappe.socket.socket.on('eval_js', function(message) {
+		frappe.socketio.socket.on('eval_js', function(message) {
 			eval(message);
 		});
 
-		frappe.socket.socket.on('progress', function(data) {
+		frappe.socketio.socket.on('progress', function(data) {
 			if(data.progress) {
 				data.percent = flt(data.progress[0]) / data.progress[1] * 100;
 			}
@@ -53,23 +53,24 @@ frappe.socket = {
 			}
 		});
 
-		frappe.socket.setup_listeners();
-		frappe.socket.setup_reconnect();
+		frappe.socketio.setup_listeners();
+		frappe.socketio.setup_reconnect();
+		frappe.socketio.uploader = new frappe.socketio.SocketIOUploader();
 
 		$(document).on('form-load form-rename', function(e, frm) {
 			if (frm.is_new()) {
 				return;
 			}
 
-			for (var i=0, l=frappe.socket.open_docs.length; i<l; i++) {
-				var d = frappe.socket.open_docs[i];
+			for (var i=0, l=frappe.socketio.open_docs.length; i<l; i++) {
+				var d = frappe.socketio.open_docs[i];
 				if (frm.doctype==d.doctype && frm.docname==d.name) {
 					// already subscribed
 					return false;
 				}
 			}
 
-			frappe.socket.doc_subscribe(frm.doctype, frm.docname);
+			frappe.socketio.doc_subscribe(frm.doctype, frm.docname);
 		});
 
 		$(document).on("form_refresh", function(e, frm) {
@@ -77,7 +78,7 @@ frappe.socket = {
 				return;
 			}
 
-			frappe.socket.doc_open(frm.doctype, frm.docname);
+			frappe.socketio.doc_open(frm.doctype, frm.docname);
 		});
 
 		$(document).on('form-unload', function(e, frm) {
@@ -85,8 +86,8 @@ frappe.socket = {
 				return;
 			}
 
-			// frappe.socket.doc_unsubscribe(frm.doctype, frm.docname);
-			frappe.socket.doc_close(frm.doctype, frm.docname);
+			// frappe.socketio.doc_unsubscribe(frm.doctype, frm.docname);
+			frappe.socketio.doc_close(frm.doctype, frm.docname);
 		});
 
 		window.onbeforeunload = function() {
@@ -96,7 +97,7 @@ frappe.socket = {
 
 			// if tab/window is closed, notify other users
 			if (cur_frm.doc) {
-				frappe.socket.doc_close(cur_frm.doctype, cur_frm.docname);
+				frappe.socketio.doc_close(cur_frm.doctype, cur_frm.docname);
 			}
 		}
 	},
@@ -115,16 +116,16 @@ frappe.socket = {
 	subscribe: function(task_id, opts) {
 		// TODO DEPRECATE
 
-		frappe.socket.socket.emit('task_subscribe', task_id);
-		frappe.socket.socket.emit('progress_subscribe', task_id);
+		frappe.socketio.socket.emit('task_subscribe', task_id);
+		frappe.socketio.socket.emit('progress_subscribe', task_id);
 
-		frappe.socket.open_tasks[task_id] = opts;
+		frappe.socketio.open_tasks[task_id] = opts;
 	},
 	task_subscribe: function(task_id) {
-		frappe.socket.socket.emit('task_subscribe', task_id);
+		frappe.socketio.socket.emit('task_subscribe', task_id);
 	},
 	task_unsubscribe: function(task_id) {
-		frappe.socket.socket.emit('task_unsubscribe', task_id);
+		frappe.socketio.socket.emit('task_unsubscribe', task_id);
 	},
 	doc_subscribe: function(doctype, docname) {
 		if (frappe.flags.doc_subscribe) {
@@ -137,12 +138,12 @@ frappe.socket = {
 		// throttle to 1 per sec
 		setTimeout(function() { frappe.flags.doc_subscribe = false }, 1000);
 
-		frappe.socket.socket.emit('doc_subscribe', doctype, docname);
-		frappe.socket.open_docs.push({doctype: doctype, docname: docname});
+		frappe.socketio.socket.emit('doc_subscribe', doctype, docname);
+		frappe.socketio.open_docs.push({doctype: doctype, docname: docname});
 	},
 	doc_unsubscribe: function(doctype, docname) {
-		frappe.socket.socket.emit('doc_unsubscribe', doctype, docname);
-		frappe.socket.open_docs = $.filter(frappe.socket.open_docs, function(d) {
+		frappe.socketio.socket.emit('doc_unsubscribe', doctype, docname);
+		frappe.socketio.open_docs = $.filter(frappe.socketio.open_docs, function(d) {
 			if(d.doctype===doctype && d.name===docname) {
 				return null;
 			} else {
@@ -152,44 +153,44 @@ frappe.socket = {
 	},
 	doc_open: function(doctype, docname) {
 		// notify that the user has opened this doc, if not already notified
-		if(!frappe.socket.last_doc
-			|| (frappe.socket.last_doc[0]!=doctype && frappe.socket.last_doc[0]!=docname)) {
-			frappe.socket.socket.emit('doc_open', doctype, docname);
+		if(!frappe.socketio.last_doc
+			|| (frappe.socketio.last_doc[0]!=doctype && frappe.socketio.last_doc[0]!=docname)) {
+			frappe.socketio.socket.emit('doc_open', doctype, docname);
 		}
-		frappe.socket.last_doc = [doctype, docname];
+		frappe.socketio.last_doc = [doctype, docname];
 	},
 	doc_close: function(doctype, docname) {
 		// notify that the user has closed this doc
-		frappe.socket.socket.emit('doc_close', doctype, docname);
+		frappe.socketio.socket.emit('doc_close', doctype, docname);
 	},
 	setup_listeners: function() {
-		frappe.socket.socket.on('task_status_change', function(data) {
-			frappe.socket.process_response(data, data.status.toLowerCase());
+		frappe.socketio.socket.on('task_status_change', function(data) {
+			frappe.socketio.process_response(data, data.status.toLowerCase());
 		});
-		frappe.socket.socket.on('task_progress', function(data) {
-			frappe.socket.process_response(data, "progress");
+		frappe.socketio.socket.on('task_progress', function(data) {
+			frappe.socketio.process_response(data, "progress");
 		});
 	},
 	setup_reconnect: function() {
 		// subscribe again to open_tasks
-		frappe.socket.socket.on("connect", function() {
+		frappe.socketio.socket.on("connect", function() {
 			// wait for 5 seconds before subscribing again
 			// because it takes more time to start python server than nodejs server
 			// and we use validation requests to python server for subscribing
 			setTimeout(function() {
-				$.each(frappe.socket.open_tasks, function(task_id, opts) {
-					frappe.socket.subscribe(task_id, opts);
+				$.each(frappe.socketio.open_tasks, function(task_id, opts) {
+					frappe.socketio.subscribe(task_id, opts);
 				});
 
 				// re-connect open docs
-				$.each(frappe.socket.open_docs, function(d) {
+				$.each(frappe.socketio.open_docs, function(d) {
 					if(locals[d.doctype] && locals[d.doctype][d.name]) {
-						frappe.socket.doc_subscribe(d.doctype, d.name);
+						frappe.socketio.doc_subscribe(d.doctype, d.name);
 					}
 				});
 
 				if (cur_frm && cur_frm.doc) {
-					frappe.socket.doc_open(cur_frm.doc.doctype, cur_frm.doc.name);
+					frappe.socketio.doc_open(cur_frm.doc.doctype, cur_frm.doc.name);
 				}
 			}, 5000);
 		});
@@ -208,9 +209,9 @@ frappe.socket = {
 		}
 		host = host + ':' + port;
 
-		frappe.socket.file_watcher = io.connect(host);
+		frappe.socketio.file_watcher = io.connect(host);
 		// css files auto reload
-		frappe.socket.file_watcher.on('reload_css', function(filename) {
+		frappe.socketio.file_watcher.on('reload_css', function(filename) {
 			let abs_file_path = "assets/" + filename;
 			const link = $(`link[href*="${abs_file_path}"]`);
 			abs_file_path = abs_file_path.split('?')[0] + '?v='+ moment();
@@ -221,7 +222,7 @@ frappe.socket = {
 			}, 5);
 		});
 		// js files show alert
-		frappe.socket.file_watcher.on('reload_js', function(filename) {
+		frappe.socketio.file_watcher.on('reload_js', function(filename) {
 			filename = "assets/" + filename;
 			var msg = $(`
 				<span>${filename} changed <a data-action="reload">Click to Reload</a></span>
@@ -239,7 +240,7 @@ frappe.socket = {
 		}
 
 		// success
-		var opts = frappe.socket.open_tasks[data.task_id];
+		var opts = frappe.socketio.open_tasks[data.task_id];
 		if(opts[method]) {
 			opts[method](data);
 		}
@@ -264,15 +265,108 @@ frappe.socket = {
 
 frappe.provide("frappe.realtime");
 frappe.realtime.on = function(event, callback) {
-	frappe.socket.socket && frappe.socket.socket.on(event, callback);
+	frappe.socketio.socket && frappe.socketio.socket.on(event, callback);
 };
 
 frappe.realtime.off = function(event, callback) {
-	frappe.socket.socket && frappe.socket.socket.off(event, callback);
+	frappe.socketio.socket && frappe.socketio.socket.off(event, callback);
 }
 
 frappe.realtime.publish = function(event, message) {
-	if(frappe.socket.socket) {
-		frappe.socket.socket.emit(event, message);
+	if(frappe.socketio.socket) {
+		frappe.socketio.socket.emit(event, message);
 	}
 }
+
+frappe.socketio.SocketIOUploader = class SocketIOUploader {
+	constructor() {
+		frappe.socketio.socket.on('upload-request-slice', (data) => {
+			var place = data.currentSlice * this.chunk_size,
+				slice = this.file.slice(place,
+					place + Math.min(this.chunk_size, this.file.size - place));
+
+			if (this.on_progress) {
+				// update progress
+				this.on_progress(place / this.file.size * 100);
+			}
+
+			this.reader.readAsArrayBuffer(slice);
+			this.keep_alive();
+		});
+
+		frappe.socketio.socket.on('upload-end', (data) => {
+			if (data.file_url.substr(0, 7)==='/public') {
+				data.file_url = data.file_url.substr(7);
+			}
+			this.callback(data);
+			this.reader = null;
+			this.file = null;
+		});
+
+		frappe.socketio.socket.on('upload-error', (data) => {
+			this.disconnect(false);
+			frappe.msgprint({
+				title: __('Upload Failed'),
+				message: data.error,
+				indicator: 'red'
+			});
+		});
+
+		frappe.socketio.socket.on('disconnect', () => {
+			this.disconnect();
+		});
+	}
+
+	start({file=null, is_private=0, filename='', callback=null, on_progress=null,
+		chunk_size=100000} = {}) {
+
+		if (this.reader) {
+			frappe.throw(__('File Upload in Progress. Please try again in a few moments.'));
+		}
+
+		this.reader = new FileReader();
+		this.file = file;
+		this.chunk_size = chunk_size;
+		this.callback = callback;
+		this.on_progress = on_progress;
+
+		this.reader.onload = () => {
+			frappe.socketio.socket.emit('upload-accept-slice', {
+				is_private: is_private,
+				name: filename,
+				type: this.file.type,
+				size: this.file.size,
+				data: this.reader.result
+			});
+			this.keep_alive();
+		};
+
+		var slice = file.slice(0, this.chunk_size);
+		this.reader.readAsArrayBuffer(slice);
+	}
+
+	keep_alive() {
+		if (this.next_check) {
+			clearTimeout (this.next_check);
+		}
+		this.next_check = setTimeout (() => {
+			this.disconnect();
+		}, 3000);
+	}
+
+	disconnect(with_message = true) {
+		if (this.reader) {
+			this.reader = null;
+			this.file = null;
+			frappe.hide_progress();
+			if (with_message) {
+				frappe.msgprint({
+					title: __('File Upload'),
+					message: __('File Upload Disconnected. Please try again.'),
+					indicator: 'red'
+				});
+			}
+		}
+	}
+
+}

frappe/public/js/frappe/ui/messages.js

@@ -234,7 +234,7 @@ frappe.verify_password = function(callback) {
 	}, __("Verify Password"), __("Verify"))
 }
 
-frappe.show_progress = function(title, count, total) {
+frappe.show_progress = function(title, count, total=100, description) {
 	if(frappe.cur_progress && frappe.cur_progress.title === title
 			&& frappe.cur_progress.$wrapper.is(":visible")) {
 		var dialog = frappe.cur_progress;
@@ -242,7 +242,10 @@ frappe.show_progress = function(title, count, total) {
 		var dialog = new frappe.ui.Dialog({
 			title: title,
 		});
-		dialog.progress = $('<div class="progress"><div class="progress-bar"></div></div>')
+		dialog.progress = $(`<div class="progress">
+			<div class="progress-bar"></div>
+			<p class="description text-muted small"></p>
+		</div>`)
 			.appendTo(dialog.body);
 		dialog.progress_bar = dialog.progress.css({"margin-top": "10px"})
 			.find(".progress-bar");
@@ -250,7 +253,12 @@ frappe.show_progress = function(title, count, total) {
 		dialog.show();
 		frappe.cur_progress = dialog;
 	}
-	dialog.progress_bar.css({"width": cint(flt(count) * 100 / total) + "%" });
+	if (description) {
+		dialog.progress.find('.description').text(description);
+	}
+	dialog.percent = cint(flt(count) * 100 / total);
+	dialog.progress_bar.css({"width": dialog.percent + "%" });
+	return dialog;
 }
 
 frappe.hide_progress = function() {

frappe/public/js/frappe/upload.js

@@ -187,7 +187,7 @@ frappe.upload = {
 	},
 	upload_multiple_files: function(files /*FileData array*/, args, opts) {
 		var i = -1;
-
+		frappe.upload.total_files = files ? files.length : 0;
 		// upload the first file
 		upload_next();
 		// subsequent files will be uploaded after
@@ -200,7 +200,7 @@ frappe.upload = {
 				var file = files[i];
 				args.is_private = file.is_private;
 				if(!opts.progress) {
-					frappe.show_progress(__('Uploading'), i+1, files.length);
+					frappe.show_progress(__('Uploading'), i, files.length);
 				}
 			}
 			frappe.upload.upload_file(file, args, opts);
@@ -225,20 +225,21 @@ frappe.upload = {
 			return;
 		}
 
-		if(args.file_url) {
-			frappe.upload._upload_file(fileobj, args, opts);
-		} else {
+		if(fileobj) {
 			frappe.upload.read_file(fileobj, args, opts);
+		} else {
+			// with file_url
+			frappe.upload._upload_file(fileobj, args, opts);
 		}
 	},
 
-	_upload_file: function(fileobj, args, opts, dataurl) {
+	_upload_file: function(fileobj, args, opts) {
 		if (args.file_size) {
 			frappe.upload.validate_max_file_size(args.file_size);
 		}
 
 		if(opts.on_attach) {
-			opts.on_attach(args, dataurl)
+			opts.on_attach(args)
 		} else {
 			if (opts.confirm_is_private) {
 				frappe.prompt({
@@ -248,55 +249,53 @@ frappe.upload = {
 					"default": 1
 				}, function(values) {
 					args["is_private"] = values.is_private;
-					frappe.upload.upload_to_server(fileobj, args, opts, dataurl);
+					frappe.upload.upload_to_server(fileobj, args, opts);
 				}, __("Private or Public?"));
 			} else {
 				if ("is_private" in opts) {
 					args["is_private"] = opts.is_private;
 				}
 
-				frappe.upload.upload_to_server(fileobj, args, opts, dataurl);
+				frappe.upload.upload_to_server(fileobj, args, opts);
 			}
 
 		}
 	},
 
 	read_file: function(fileobj, args, opts) {
-		var freader = new FileReader();
+		args.filename = fileobj.name.split(' ').join('_');
+		args.file_url = null;
 
-		freader.onload = function() {
-			args.filename = fileobj.name.split(' ').join('_');
-			if(opts.options && opts.options.toLowerCase()=="image") {
-				if(!frappe.utils.is_image_file(args.filename)) {
-					frappe.msgprint(__("Only image extensions (.gif, .jpg, .jpeg, .tiff, .png, .svg) allowed"));
-					return;
-				}
+		if(opts.options && opts.options.toLowerCase()=="image") {
+			if(!frappe.utils.is_image_file(args.filename)) {
+				frappe.msgprint(__("Only image extensions (.gif, .jpg, .jpeg, .tiff, .png, .svg) allowed"));
+				return;
 			}
+		}
 
-			if((opts.max_width || opts.max_height) && frappe.utils.is_image_file(args.filename)) {
-				frappe.utils.resize_image(freader, function(_dataurl) {
-					var dataurl = _dataurl;
-					args.filedata = _dataurl.split(",")[1];
-					args.file_size = Math.round(args.filedata.length * 3 / 4);
-					console.log("resized!")
-					frappe.upload._upload_file(fileobj, args, opts, dataurl);
-				})
-			} else {
-				var dataurl = freader.result;
-				args.filedata = freader.result.split(",")[1];
-				args.file_size = fileobj.size;
-				frappe.upload._upload_file(fileobj, args, opts, dataurl);
-			}
-		};
+		let start_complete = frappe.cur_progress ? frappe.cur_progress.percent : 0;
 
-		freader.readAsDataURL(fileobj);
+		frappe.socketio.uploader.start({
+			file: fileobj,
+			filename: args.filename,
+			is_private: args.is_private,
+			callback: (data) => {
+				args.file_url = data.file_url;
+				frappe.upload._upload_file(fileobj, args, opts);
+			},
+			on_progress: (percent_complete) => {
+				let increment = (flt(percent_complete) / frappe.upload.total_files);
+				frappe.show_progress(__('Uploading'),
+					start_complete + increment);
+			}
+		});
 	},
 
-	upload_to_server: function(fileobj, args, opts, dataurl) {
-		// var msgbox =	frappe.msgprint(__("Uploading..."));
+	upload_to_server: function(file, args, opts) {
 		if(opts.start) {
 			opts.start();
 		}
+
 		var ajax_args = {
 			"method": "uploadfile",
 			args: args,
@@ -368,7 +367,7 @@ frappe.upload = {
 				d.hide();
 				opts.loopcallback = function (){
 					if (i < j) {
-						args.is_private = d.fields_dict[fileobjs[i].name + "_is_private"].get_value()
+						args.is_private = d.fields_dict[fileobjs[i].name + "_is_private"].get_value();
 						frappe.upload.upload_file(fileobjs[i], args, opts);
 						i++;
 					}

frappe/public/less/sidebar.less

@@ -130,15 +130,6 @@ body[data-route^="Module"] .main-menu {
 		right: 5px;
 	}
 
-	// .attachment-row .icon-lock {
-	// 	color: @text-warning;
-	// 	display: inline-block;
-	// 	margin-top: 1px;
-	// }
-
-	.attachment-row a.close {
-		margin-top: -5px;
-	}
 
 	.form-shared, .form-viewers {
 		.share-doc-btn {

frappe/utils/file_manager.py

@@ -15,6 +15,7 @@ from six import text_type
 
 class MaxFileSizeReachedError(frappe.ValidationError): pass
 
+
 def get_file_url(file_data_name):
 	data = frappe.db.get_value("File", file_data_name, ["file_name", "file_url"], as_dict=True)
 	return data.file_url or data.file_name
@@ -23,38 +24,51 @@ def upload():
 	# get record details
 	dt = frappe.form_dict.doctype
 	dn = frappe.form_dict.docname
-	folder = frappe.form_dict.folder
 	file_url = frappe.form_dict.file_url
 	filename = frappe.form_dict.filename
-	is_private = cint(frappe.form_dict.is_private)
+	frappe.form_dict.is_private = cint(frappe.form_dict.is_private)
 
 	if not filename and not file_url:
 		frappe.msgprint(_("Please select a file or url"),
 			raise_exception=True)
 
-	# save
-	if frappe.form_dict.filedata:
-		filedata = save_uploaded(dt, dn, folder, is_private)
-	elif file_url:
-		filedata = save_url(file_url, filename, dt, dn, folder, is_private)
+	file_doc = get_file_doc()
 
 	comment = {}
 	if dt and dn:
 		comment = frappe.get_doc(dt, dn).add_comment("Attachment",
 			_("added {0}").format("<a href='{file_url}' target='_blank'>{file_name}</a>{icon}".format(**{
-				"icon": ' <i class="fa fa-lock text-warning"></i>' if filedata.is_private else "",
-				"file_url": filedata.file_url.replace("#", "%23") if filedata.file_name else filedata.file_url,
-				"file_name": filedata.file_name or filedata.file_url
+				"icon": ' <i class="fa fa-lock text-warning"></i>' \
+					if file_doc.is_private else "",
+				"file_url": file_doc.file_url.replace("#", "%23") \
+					if file_doc.file_name else file_doc.file_url,
+				"file_name": file_doc.file_name or file_doc.file_url
 			})))
 
 	return {
-		"name": filedata.name,
-		"file_name": filedata.file_name,
-		"file_url": filedata.file_url,
-		"is_private": filedata.is_private,
+		"name": file_doc.name,
+		"file_name": file_doc.file_name,
+		"file_url": file_doc.file_url,
+		"is_private": file_doc.is_private,
 		"comment": comment.as_dict() if comment else {}
 	}
 
+def get_file_doc(dt=None, dn=None, folder=None, is_private=None):
+	'''returns File object (Document) from given parameters or form_dict'''
+	r = frappe.form_dict
+
+	if dt is None: dt = r.doctype
+	if dn is None: dn = r.docname
+	if folder is None: folder = r.folder
+	if is_private is None: is_private = r.is_private
+
+	if r.filedata:
+		file_doc = save_uploaded(dt, dn, folder, is_private)
+	elif r.file_url:
+		file_doc = save_url(r.file_url, r.filename, dt, dn, folder, is_private)
+
+	return file_doc
+
 def save_uploaded(dt, dn, folder, is_private):
 	fname, content = get_uploaded_content()
 	if content:
@@ -97,55 +111,6 @@ def get_uploaded_content():
 		frappe.msgprint(_('No file attached'))
 		return None, None
 
-def extract_images_from_doc(doc, fieldname):
-	content = doc.get(fieldname)
-	content = extract_images_from_html(doc, content)
-	if frappe.flags.has_dataurl:
-		doc.set(fieldname, content)
-
-def extract_images_from_html(doc, content):
-	frappe.flags.has_dataurl = False
-
-	def _save_file(match):
-		data = match.group(1)
-		data = data.split("data:")[1]
-		headers, content = data.split(",")
-
-		if "filename=" in headers:
-			filename = headers.split("filename=")[-1]
-
-			# decode filename
-			if not isinstance(filename, text_type):
-				filename = text_type(filename, 'utf-8')
-		else:
-			mtype = headers.split(";")[0]
-			filename = get_random_filename(content_type=mtype)
-
-		doctype = doc.parenttype if doc.parent else doc.doctype
-		name = doc.parent or doc.name
-
-		# TODO fix this
-		file_url = save_file(filename, content, doctype, name, decode=True).get("file_url")
-		if not frappe.flags.has_dataurl:
-			frappe.flags.has_dataurl = True
-
-		return '<img src="{file_url}"'.format(file_url=file_url)
-
-	if content:
-		content = re.sub('<img[^>]*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content)
-
-	return content
-
-def get_random_filename(extn=None, content_type=None):
-	if extn:
-		if not extn.startswith("."):
-			extn = "." + extn
-
-	elif content_type:
-		extn = mimetypes.guess_extension(content_type)
-
-	return random_string(7) + (extn or "")
-
 def save_file(fname, content, dt, dn, folder=None, decode=False, is_private=0):
 	if decode:
 		if isinstance(content, text_type):
@@ -370,3 +335,52 @@ def download_file(file_url):
 	frappe.local.response.filename = file_url[file_url.rfind("/")+1:]
 	frappe.local.response.filecontent = filedata
 	frappe.local.response.type = "download"
+
+def extract_images_from_doc(doc, fieldname):
+	content = doc.get(fieldname)
+	content = extract_images_from_html(doc, content)
+	if frappe.flags.has_dataurl:
+		doc.set(fieldname, content)
+
+def extract_images_from_html(doc, content):
+	frappe.flags.has_dataurl = False
+
+	def _save_file(match):
+		data = match.group(1)
+		data = data.split("data:")[1]
+		headers, content = data.split(",")
+
+		if "filename=" in headers:
+			filename = headers.split("filename=")[-1]
+
+			# decode filename
+			if not isinstance(filename, text_type):
+				filename = text_type(filename, 'utf-8')
+		else:
+			mtype = headers.split(";")[0]
+			filename = get_random_filename(content_type=mtype)
+
+		doctype = doc.parenttype if doc.parent else doc.doctype
+		name = doc.parent or doc.name
+
+		# TODO fix this
+		file_url = save_file(filename, content, doctype, name, decode=True).get("file_url")
+		if not frappe.flags.has_dataurl:
+			frappe.flags.has_dataurl = True
+
+		return '<img src="{file_url}"'.format(file_url=file_url)
+
+	if content:
+		content = re.sub('<img[^>]*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content)
+
+	return content
+
+def get_random_filename(extn=None, content_type=None):
+	if extn:
+		if not extn.startswith("."):
+			extn = "." + extn
+
+	elif content_type:
+		extn = mimetypes.guess_extension(content_type)
+
+	return random_string(7) + (extn or "")

socketio.js

@@ -3,11 +3,22 @@ var http = require('http').Server(app);
 var io = require('socket.io')(http);
 var cookie = require('cookie')
 var fs = require('fs');
+var path = require('path');
 var redis = require("redis");
 var request = require('superagent');
 
 var conf = get_conf();
 var flags = {};
+var files_struct = {
+	name: null,
+	type: null,
+	size: 0,
+	data: [],
+	slice: 0,
+	site_name: null,
+	is_private: 0
+};
+
 var subscriber = redis.createClient(conf.redis_socketio || conf.redis_async_broker_port);
 
 // serve socketio
@@ -21,7 +32,7 @@ app.get('/', function(req, res) {
 });
 
 // on socket connection
-io.on('connection', function(socket){
+io.on('connection', function(socket) {
 	if (get_hostname(socket.request.headers.host) != get_hostname(socket.request.headers.origin)) {
 		return;
 	}
@@ -41,6 +52,7 @@ io.on('connection', function(socket){
 	setTimeout(function() { flags[sid] = null; }, 10000);
 
 	socket.user = cookie.parse(socket.request.headers.cookie).user_id;
+	socket.files = {};
 
 	// console.log("firing get_user_info");
 	request.get(get_url(socket, '/api/method/frappe.async.get_user_info'))
@@ -61,6 +73,10 @@ io.on('connection', function(socket){
 			}
 		});
 
+	socket.on('disconnect', function() {
+		delete socket.files;
+	})
+
 	socket.on('task_subscribe', function(task_id) {
 		var room = get_task_room(socket, task_id);
 		socket.join(room);
@@ -134,9 +150,46 @@ io.on('connection', function(socket){
 		});
 	});
 
-	// socket.on('disconnect', function (arguments) {
-	// 	console.log("user disconnected", arguments);
-	// });
+	socket.on('upload-accept-slice', (data) => {
+		try {
+			if (!socket.files[data.name]) {
+				socket.files[data.name] = Object.assign({}, files_struct, data);
+				socket.files[data.name].data = [];
+			}
+
+			//convert the ArrayBuffer to Buffer
+			data.data = new Buffer(new Uint8Array(data.data));
+			//save the data
+			socket.files[data.name].data.push(data.data);
+			socket.files[data.name].slice++;
+
+			if (socket.files[data.name].slice * 100000 >= socket.files[data.name].size) {
+				// do something with the data
+				var fileBuffer = Buffer.concat(socket.files[data.name].data);
+
+				const file_url = path.join((socket.files[data.name].is_private ? 'private' : 'public'),
+					'files', data.name);
+				const file_path = path.join('sites', get_site_name(socket), file_url);
+
+				fs.writeFile(file_path, fileBuffer, (err) => {
+					delete socket.files[data.name];
+					if (err) return socket.emit('upload error');
+					socket.emit('upload-end', {
+						file_url: '/' + file_url
+					});
+				});
+			} else {
+				socket.emit('upload-request-slice', {
+					currentSlice: socket.files[data.name].slice
+				});
+			}
+		} catch (e) {
+			console.log(e);
+			socket.emit('upload-error', {
+				error: e.message
+			});
+		}
+	});
 });
 
 subscriber.on("message", function(channel, message) {