diff --git a/frappe/boot.py b/frappe/boot.py index 5d042f82ce..9f0c05a6ce 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -30,6 +30,7 @@ def get_bootinfo(): get_user(bootinfo) # system info + bootinfo.sitename = frappe.local.site bootinfo.sysdefaults = frappe.defaults.get_defaults() bootinfo.user_permissions = get_user_permissions() bootinfo.server_date = frappe.utils.nowdate() diff --git a/frappe/core/page/data_import_tool/importer.py b/frappe/core/page/data_import_tool/importer.py index aa16802c5d..d26dcfd2b1 100644 --- a/frappe/core/page/data_import_tool/importer.py +++ b/frappe/core/page/data_import_tool/importer.py @@ -217,8 +217,8 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, # header if not rows: - from frappe.utils.file_manager import save_uploaded - file_doc = save_uploaded(dt=None, dn="Data Import", folder='Home', is_private=1) + from frappe.utils.file_manager import get_file_doc + file_doc = get_file_doc(dt='', dn="Data Import", folder='Home', is_private=1) filename, file_extension = os.path.splitext(file_doc.file_name) if file_extension == '.xlsx' and from_data_import == 'Yes': diff --git a/frappe/public/css/sidebar.css b/frappe/public/css/sidebar.css index 1bea165047..d5b07cc422 100644 --- a/frappe/public/css/sidebar.css +++ b/frappe/public/css/sidebar.css @@ -94,9 +94,6 @@ body[data-route^="Module"] .main-menu .form-sidebar { position: absolute; right: 5px; } -.form-sidebar .attachment-row a.close { - margin-top: -5px; -} .form-sidebar .form-shared .share-doc-btn, .form-sidebar .form-viewers .share-doc-btn { cursor: pointer; diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index 68c63dae10..0610e6b551 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -29,7 +29,7 @@ frappe.Application = Class.extend({ this.startup(); }, startup: function() { - frappe.socket.init(); + frappe.socketio.init(); frappe.model.init(); if(frappe.boot.status==='failed') { diff --git a/frappe/public/js/frappe/form/controls/text_editor.js b/frappe/public/js/frappe/form/controls/text_editor.js index 1110fee7fc..00af468754 100644 --- a/frappe/public/js/frappe/form/controls/text_editor.js +++ b/frappe/public/js/frappe/form/controls/text_editor.js @@ -137,17 +137,17 @@ frappe.ui.form.ControlTextEditor = frappe.ui.form.ControlCode.extend({ }); }, get_image: function (fileobj, callback) { - var freader = new FileReader(); + var reader = new FileReader(); - freader.onload = function() { - var dataurl = freader.result; + reader.onload = function() { + var dataurl = reader.result; // add filename to dataurl var parts = dataurl.split(","); parts[0] += ";filename=" + fileobj.name; dataurl = parts[0] + ',' + parts[1]; callback(dataurl); }; - freader.readAsDataURL(fileobj); + reader.readAsDataURL(fileobj); }, hide_elements_on_mobile: function() { this.note_editor.find('.note-btn-underline,\ diff --git a/frappe/public/js/frappe/request.js b/frappe/public/js/frappe/request.js index 19b622e8ee..bde19db1f3 100644 --- a/frappe/public/js/frappe/request.js +++ b/frappe/public/js/frappe/request.js @@ -34,7 +34,7 @@ frappe.call = function(opts) { var callback = function(data, response_text) { if(data.task_id) { // async call, subscribe - frappe.socket.subscribe(data.task_id, opts); + frappe.socketio.subscribe(data.task_id, opts); if(opts.queued) { opts.queued(data); diff --git a/frappe/public/js/frappe/socketio_client.js b/frappe/public/js/frappe/socketio_client.js index c8af253bed..d058b8a3f4 100644 --- a/frappe/public/js/frappe/socketio_client.js +++ b/frappe/public/js/frappe/socketio_client.js @@ -1,4 +1,4 @@ -frappe.socket = { +frappe.socketio = { open_tasks: {}, open_docs: [], emit_queue: [], @@ -7,40 +7,40 @@ frappe.socket = { return; } - if (frappe.socket.socket) { + if (frappe.socketio.socket) { return; } if (frappe.boot.developer_mode) { // File watchers for development - frappe.socket.setup_file_watchers(); + frappe.socketio.setup_file_watchers(); } //Enable secure option when using HTTPS if (window.location.protocol == "https:") { - frappe.socket.socket = io.connect(frappe.socket.get_host(), {secure: true}); + frappe.socketio.socket = io.connect(frappe.socketio.get_host(), {secure: true}); } else if (window.location.protocol == "http:") { - frappe.socket.socket = io.connect(frappe.socket.get_host()); + frappe.socketio.socket = io.connect(frappe.socketio.get_host()); } else if (window.location.protocol == "file:") { - frappe.socket.socket = io.connect(window.localStorage.server); + frappe.socketio.socket = io.connect(window.localStorage.server); } - if (!frappe.socket.socket) { - console.log("Unable to connect to " + frappe.socket.get_host()); + if (!frappe.socketio.socket) { + console.log("Unable to connect to " + frappe.socketio.get_host()); return; } - frappe.socket.socket.on('msgprint', function(message) { + frappe.socketio.socket.on('msgprint', function(message) { frappe.msgprint(message); }); - frappe.socket.socket.on('eval_js', function(message) { + frappe.socketio.socket.on('eval_js', function(message) { eval(message); }); - frappe.socket.socket.on('progress', function(data) { + frappe.socketio.socket.on('progress', function(data) { if(data.progress) { data.percent = flt(data.progress[0]) / data.progress[1] * 100; } @@ -53,23 +53,24 @@ frappe.socket = { } }); - frappe.socket.setup_listeners(); - frappe.socket.setup_reconnect(); + frappe.socketio.setup_listeners(); + frappe.socketio.setup_reconnect(); + frappe.socketio.uploader = new frappe.socketio.SocketIOUploader(); $(document).on('form-load form-rename', function(e, frm) { if (frm.is_new()) { return; } - for (var i=0, l=frappe.socket.open_docs.length; i${filename} changed Click to Reload @@ -239,7 +240,7 @@ frappe.socket = { } // success - var opts = frappe.socket.open_tasks[data.task_id]; + var opts = frappe.socketio.open_tasks[data.task_id]; if(opts[method]) { opts[method](data); } @@ -264,15 +265,108 @@ frappe.socket = { frappe.provide("frappe.realtime"); frappe.realtime.on = function(event, callback) { - frappe.socket.socket && frappe.socket.socket.on(event, callback); + frappe.socketio.socket && frappe.socketio.socket.on(event, callback); }; frappe.realtime.off = function(event, callback) { - frappe.socket.socket && frappe.socket.socket.off(event, callback); + frappe.socketio.socket && frappe.socketio.socket.off(event, callback); } frappe.realtime.publish = function(event, message) { - if(frappe.socket.socket) { - frappe.socket.socket.emit(event, message); + if(frappe.socketio.socket) { + frappe.socketio.socket.emit(event, message); } } + +frappe.socketio.SocketIOUploader = class SocketIOUploader { + constructor() { + frappe.socketio.socket.on('upload-request-slice', (data) => { + var place = data.currentSlice * this.chunk_size, + slice = this.file.slice(place, + place + Math.min(this.chunk_size, this.file.size - place)); + + if (this.on_progress) { + // update progress + this.on_progress(place / this.file.size * 100); + } + + this.reader.readAsArrayBuffer(slice); + this.keep_alive(); + }); + + frappe.socketio.socket.on('upload-end', (data) => { + if (data.file_url.substr(0, 7)==='/public') { + data.file_url = data.file_url.substr(7); + } + this.callback(data); + this.reader = null; + this.file = null; + }); + + frappe.socketio.socket.on('upload-error', (data) => { + this.disconnect(false); + frappe.msgprint({ + title: __('Upload Failed'), + message: data.error, + indicator: 'red' + }); + }); + + frappe.socketio.socket.on('disconnect', () => { + this.disconnect(); + }); + } + + start({file=null, is_private=0, filename='', callback=null, on_progress=null, + chunk_size=100000} = {}) { + + if (this.reader) { + frappe.throw(__('File Upload in Progress. Please try again in a few moments.')); + } + + this.reader = new FileReader(); + this.file = file; + this.chunk_size = chunk_size; + this.callback = callback; + this.on_progress = on_progress; + + this.reader.onload = () => { + frappe.socketio.socket.emit('upload-accept-slice', { + is_private: is_private, + name: filename, + type: this.file.type, + size: this.file.size, + data: this.reader.result + }); + this.keep_alive(); + }; + + var slice = file.slice(0, this.chunk_size); + this.reader.readAsArrayBuffer(slice); + } + + keep_alive() { + if (this.next_check) { + clearTimeout (this.next_check); + } + this.next_check = setTimeout (() => { + this.disconnect(); + }, 3000); + } + + disconnect(with_message = true) { + if (this.reader) { + this.reader = null; + this.file = null; + frappe.hide_progress(); + if (with_message) { + frappe.msgprint({ + title: __('File Upload'), + message: __('File Upload Disconnected. Please try again.'), + indicator: 'red' + }); + } + } + } + +} \ No newline at end of file diff --git a/frappe/public/js/frappe/ui/messages.js b/frappe/public/js/frappe/ui/messages.js index 79c1345cfa..b4d4244bbf 100644 --- a/frappe/public/js/frappe/ui/messages.js +++ b/frappe/public/js/frappe/ui/messages.js @@ -234,7 +234,7 @@ frappe.verify_password = function(callback) { }, __("Verify Password"), __("Verify")) } -frappe.show_progress = function(title, count, total) { +frappe.show_progress = function(title, count, total=100, description) { if(frappe.cur_progress && frappe.cur_progress.title === title && frappe.cur_progress.$wrapper.is(":visible")) { var dialog = frappe.cur_progress; @@ -242,7 +242,10 @@ frappe.show_progress = function(title, count, total) { var dialog = new frappe.ui.Dialog({ title: title, }); - dialog.progress = $('
') + dialog.progress = $(`
+
+

+
`) .appendTo(dialog.body); dialog.progress_bar = dialog.progress.css({"margin-top": "10px"}) .find(".progress-bar"); @@ -250,7 +253,12 @@ frappe.show_progress = function(title, count, total) { dialog.show(); frappe.cur_progress = dialog; } - dialog.progress_bar.css({"width": cint(flt(count) * 100 / total) + "%" }); + if (description) { + dialog.progress.find('.description').text(description); + } + dialog.percent = cint(flt(count) * 100 / total); + dialog.progress_bar.css({"width": dialog.percent + "%" }); + return dialog; } frappe.hide_progress = function() { diff --git a/frappe/public/js/frappe/upload.js b/frappe/public/js/frappe/upload.js index 319fc61971..5914cb9c79 100644 --- a/frappe/public/js/frappe/upload.js +++ b/frappe/public/js/frappe/upload.js @@ -187,7 +187,7 @@ frappe.upload = { }, upload_multiple_files: function(files /*FileData array*/, args, opts) { var i = -1; - + frappe.upload.total_files = files ? files.length : 0; // upload the first file upload_next(); // subsequent files will be uploaded after @@ -200,7 +200,7 @@ frappe.upload = { var file = files[i]; args.is_private = file.is_private; if(!opts.progress) { - frappe.show_progress(__('Uploading'), i+1, files.length); + frappe.show_progress(__('Uploading'), i, files.length); } } frappe.upload.upload_file(file, args, opts); @@ -225,20 +225,21 @@ frappe.upload = { return; } - if(args.file_url) { - frappe.upload._upload_file(fileobj, args, opts); - } else { + if(fileobj) { frappe.upload.read_file(fileobj, args, opts); + } else { + // with file_url + frappe.upload._upload_file(fileobj, args, opts); } }, - _upload_file: function(fileobj, args, opts, dataurl) { + _upload_file: function(fileobj, args, opts) { if (args.file_size) { frappe.upload.validate_max_file_size(args.file_size); } if(opts.on_attach) { - opts.on_attach(args, dataurl) + opts.on_attach(args) } else { if (opts.confirm_is_private) { frappe.prompt({ @@ -248,55 +249,53 @@ frappe.upload = { "default": 1 }, function(values) { args["is_private"] = values.is_private; - frappe.upload.upload_to_server(fileobj, args, opts, dataurl); + frappe.upload.upload_to_server(fileobj, args, opts); }, __("Private or Public?")); } else { if ("is_private" in opts) { args["is_private"] = opts.is_private; } - frappe.upload.upload_to_server(fileobj, args, opts, dataurl); + frappe.upload.upload_to_server(fileobj, args, opts); } } }, read_file: function(fileobj, args, opts) { - var freader = new FileReader(); + args.filename = fileobj.name.split(' ').join('_'); + args.file_url = null; - freader.onload = function() { - args.filename = fileobj.name.split(' ').join('_'); - if(opts.options && opts.options.toLowerCase()=="image") { - if(!frappe.utils.is_image_file(args.filename)) { - frappe.msgprint(__("Only image extensions (.gif, .jpg, .jpeg, .tiff, .png, .svg) allowed")); - return; - } + if(opts.options && opts.options.toLowerCase()=="image") { + if(!frappe.utils.is_image_file(args.filename)) { + frappe.msgprint(__("Only image extensions (.gif, .jpg, .jpeg, .tiff, .png, .svg) allowed")); + return; } + } - if((opts.max_width || opts.max_height) && frappe.utils.is_image_file(args.filename)) { - frappe.utils.resize_image(freader, function(_dataurl) { - var dataurl = _dataurl; - args.filedata = _dataurl.split(",")[1]; - args.file_size = Math.round(args.filedata.length * 3 / 4); - console.log("resized!") - frappe.upload._upload_file(fileobj, args, opts, dataurl); - }) - } else { - var dataurl = freader.result; - args.filedata = freader.result.split(",")[1]; - args.file_size = fileobj.size; - frappe.upload._upload_file(fileobj, args, opts, dataurl); - } - }; + let start_complete = frappe.cur_progress ? frappe.cur_progress.percent : 0; - freader.readAsDataURL(fileobj); + frappe.socketio.uploader.start({ + file: fileobj, + filename: args.filename, + is_private: args.is_private, + callback: (data) => { + args.file_url = data.file_url; + frappe.upload._upload_file(fileobj, args, opts); + }, + on_progress: (percent_complete) => { + let increment = (flt(percent_complete) / frappe.upload.total_files); + frappe.show_progress(__('Uploading'), + start_complete + increment); + } + }); }, - upload_to_server: function(fileobj, args, opts, dataurl) { - // var msgbox = frappe.msgprint(__("Uploading...")); + upload_to_server: function(file, args, opts) { if(opts.start) { opts.start(); } + var ajax_args = { "method": "uploadfile", args: args, @@ -368,7 +367,7 @@ frappe.upload = { d.hide(); opts.loopcallback = function (){ if (i < j) { - args.is_private = d.fields_dict[fileobjs[i].name + "_is_private"].get_value() + args.is_private = d.fields_dict[fileobjs[i].name + "_is_private"].get_value(); frappe.upload.upload_file(fileobjs[i], args, opts); i++; } diff --git a/frappe/public/less/sidebar.less b/frappe/public/less/sidebar.less index aacb83b17e..e45cc91bd7 100644 --- a/frappe/public/less/sidebar.less +++ b/frappe/public/less/sidebar.less @@ -130,15 +130,6 @@ body[data-route^="Module"] .main-menu { right: 5px; } - // .attachment-row .icon-lock { - // color: @text-warning; - // display: inline-block; - // margin-top: 1px; - // } - - .attachment-row a.close { - margin-top: -5px; - } .form-shared, .form-viewers { .share-doc-btn { diff --git a/frappe/utils/file_manager.py b/frappe/utils/file_manager.py index 2d32215d6a..de0ebdf12f 100644 --- a/frappe/utils/file_manager.py +++ b/frappe/utils/file_manager.py @@ -15,6 +15,7 @@ from six import text_type class MaxFileSizeReachedError(frappe.ValidationError): pass + def get_file_url(file_data_name): data = frappe.db.get_value("File", file_data_name, ["file_name", "file_url"], as_dict=True) return data.file_url or data.file_name @@ -23,38 +24,51 @@ def upload(): # get record details dt = frappe.form_dict.doctype dn = frappe.form_dict.docname - folder = frappe.form_dict.folder file_url = frappe.form_dict.file_url filename = frappe.form_dict.filename - is_private = cint(frappe.form_dict.is_private) + frappe.form_dict.is_private = cint(frappe.form_dict.is_private) if not filename and not file_url: frappe.msgprint(_("Please select a file or url"), raise_exception=True) - # save - if frappe.form_dict.filedata: - filedata = save_uploaded(dt, dn, folder, is_private) - elif file_url: - filedata = save_url(file_url, filename, dt, dn, folder, is_private) + file_doc = get_file_doc() comment = {} if dt and dn: comment = frappe.get_doc(dt, dn).add_comment("Attachment", _("added {0}").format("{file_name}{icon}".format(**{ - "icon": ' ' if filedata.is_private else "", - "file_url": filedata.file_url.replace("#", "%23") if filedata.file_name else filedata.file_url, - "file_name": filedata.file_name or filedata.file_url + "icon": ' ' \ + if file_doc.is_private else "", + "file_url": file_doc.file_url.replace("#", "%23") \ + if file_doc.file_name else file_doc.file_url, + "file_name": file_doc.file_name or file_doc.file_url }))) return { - "name": filedata.name, - "file_name": filedata.file_name, - "file_url": filedata.file_url, - "is_private": filedata.is_private, + "name": file_doc.name, + "file_name": file_doc.file_name, + "file_url": file_doc.file_url, + "is_private": file_doc.is_private, "comment": comment.as_dict() if comment else {} } +def get_file_doc(dt=None, dn=None, folder=None, is_private=None): + '''returns File object (Document) from given parameters or form_dict''' + r = frappe.form_dict + + if dt is None: dt = r.doctype + if dn is None: dn = r.docname + if folder is None: folder = r.folder + if is_private is None: is_private = r.is_private + + if r.filedata: + file_doc = save_uploaded(dt, dn, folder, is_private) + elif r.file_url: + file_doc = save_url(r.file_url, r.filename, dt, dn, folder, is_private) + + return file_doc + def save_uploaded(dt, dn, folder, is_private): fname, content = get_uploaded_content() if content: @@ -97,55 +111,6 @@ def get_uploaded_content(): frappe.msgprint(_('No file attached')) return None, None -def extract_images_from_doc(doc, fieldname): - content = doc.get(fieldname) - content = extract_images_from_html(doc, content) - if frappe.flags.has_dataurl: - doc.set(fieldname, content) - -def extract_images_from_html(doc, content): - frappe.flags.has_dataurl = False - - def _save_file(match): - data = match.group(1) - data = data.split("data:")[1] - headers, content = data.split(",") - - if "filename=" in headers: - filename = headers.split("filename=")[-1] - - # decode filename - if not isinstance(filename, text_type): - filename = text_type(filename, 'utf-8') - else: - mtype = headers.split(";")[0] - filename = get_random_filename(content_type=mtype) - - doctype = doc.parenttype if doc.parent else doc.doctype - name = doc.parent or doc.name - - # TODO fix this - file_url = save_file(filename, content, doctype, name, decode=True).get("file_url") - if not frappe.flags.has_dataurl: - frappe.flags.has_dataurl = True - - return ']*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content) - - return content - -def get_random_filename(extn=None, content_type=None): - if extn: - if not extn.startswith("."): - extn = "." + extn - - elif content_type: - extn = mimetypes.guess_extension(content_type) - - return random_string(7) + (extn or "") - def save_file(fname, content, dt, dn, folder=None, decode=False, is_private=0): if decode: if isinstance(content, text_type): @@ -370,3 +335,52 @@ def download_file(file_url): frappe.local.response.filename = file_url[file_url.rfind("/")+1:] frappe.local.response.filecontent = filedata frappe.local.response.type = "download" + +def extract_images_from_doc(doc, fieldname): + content = doc.get(fieldname) + content = extract_images_from_html(doc, content) + if frappe.flags.has_dataurl: + doc.set(fieldname, content) + +def extract_images_from_html(doc, content): + frappe.flags.has_dataurl = False + + def _save_file(match): + data = match.group(1) + data = data.split("data:")[1] + headers, content = data.split(",") + + if "filename=" in headers: + filename = headers.split("filename=")[-1] + + # decode filename + if not isinstance(filename, text_type): + filename = text_type(filename, 'utf-8') + else: + mtype = headers.split(";")[0] + filename = get_random_filename(content_type=mtype) + + doctype = doc.parenttype if doc.parent else doc.doctype + name = doc.parent or doc.name + + # TODO fix this + file_url = save_file(filename, content, doctype, name, decode=True).get("file_url") + if not frappe.flags.has_dataurl: + frappe.flags.has_dataurl = True + + return ']*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content) + + return content + +def get_random_filename(extn=None, content_type=None): + if extn: + if not extn.startswith("."): + extn = "." + extn + + elif content_type: + extn = mimetypes.guess_extension(content_type) + + return random_string(7) + (extn or "") diff --git a/socketio.js b/socketio.js index cd3eb64aa8..15e9263398 100644 --- a/socketio.js +++ b/socketio.js @@ -3,11 +3,22 @@ var http = require('http').Server(app); var io = require('socket.io')(http); var cookie = require('cookie') var fs = require('fs'); +var path = require('path'); var redis = require("redis"); var request = require('superagent'); var conf = get_conf(); var flags = {}; +var files_struct = { + name: null, + type: null, + size: 0, + data: [], + slice: 0, + site_name: null, + is_private: 0 +}; + var subscriber = redis.createClient(conf.redis_socketio || conf.redis_async_broker_port); // serve socketio @@ -21,7 +32,7 @@ app.get('/', function(req, res) { }); // on socket connection -io.on('connection', function(socket){ +io.on('connection', function(socket) { if (get_hostname(socket.request.headers.host) != get_hostname(socket.request.headers.origin)) { return; } @@ -41,6 +52,7 @@ io.on('connection', function(socket){ setTimeout(function() { flags[sid] = null; }, 10000); socket.user = cookie.parse(socket.request.headers.cookie).user_id; + socket.files = {}; // console.log("firing get_user_info"); request.get(get_url(socket, '/api/method/frappe.async.get_user_info')) @@ -61,6 +73,10 @@ io.on('connection', function(socket){ } }); + socket.on('disconnect', function() { + delete socket.files; + }) + socket.on('task_subscribe', function(task_id) { var room = get_task_room(socket, task_id); socket.join(room); @@ -134,9 +150,46 @@ io.on('connection', function(socket){ }); }); - // socket.on('disconnect', function (arguments) { - // console.log("user disconnected", arguments); - // }); + socket.on('upload-accept-slice', (data) => { + try { + if (!socket.files[data.name]) { + socket.files[data.name] = Object.assign({}, files_struct, data); + socket.files[data.name].data = []; + } + + //convert the ArrayBuffer to Buffer + data.data = new Buffer(new Uint8Array(data.data)); + //save the data + socket.files[data.name].data.push(data.data); + socket.files[data.name].slice++; + + if (socket.files[data.name].slice * 100000 >= socket.files[data.name].size) { + // do something with the data + var fileBuffer = Buffer.concat(socket.files[data.name].data); + + const file_url = path.join((socket.files[data.name].is_private ? 'private' : 'public'), + 'files', data.name); + const file_path = path.join('sites', get_site_name(socket), file_url); + + fs.writeFile(file_path, fileBuffer, (err) => { + delete socket.files[data.name]; + if (err) return socket.emit('upload error'); + socket.emit('upload-end', { + file_url: '/' + file_url + }); + }); + } else { + socket.emit('upload-request-slice', { + currentSlice: socket.files[data.name].slice + }); + } + } catch (e) { + console.log(e); + socket.emit('upload-error', { + error: e.message + }); + } + }); }); subscriber.on("message", function(channel, message) {