anoopmb
/
frappe
1
0
Forka 0
Codice Problemi 0 Pull Requests 0 Rilasci 0 Wiki Attività
Sfoglia il codice sorgente

[wip] file upload with socketio

version-14
Rushabh Mehta 7 anni fa
parent
563db38e40
commit
fd7e8eda62
9 ha cambiato i file con 402 aggiunte e 104 eliminazioni
Visualizzazione separata
Opzioni Diff
Mostra statistiche Scarica il file Patch Scarica il file Diff
  1. +2
    -1
      .eslintrc
  2. +1
    -0
      frappe/boot.py
  3. +1
    -0
      frappe/public/build.json
  4. +1
    -1
      frappe/public/js/frappe/desk.js
  5. +1
    -1
      frappe/public/js/frappe/request.js
  6. +59
    -52
      frappe/public/js/frappe/socketio_client.js
  7. +259
    -0
      frappe/public/js/lib/socket.io-file-client.js
  8. +50
    -49
      frappe/utils/file_manager.py
  9. +28
    -0
      socketio.js

+ 2
- 1
.eslintrc Vedi File

@@ -119,6 +119,7 @@
"get_url_arg": true,
"QUnit": true,
"Snap": true,
"mina": true
"mina": true,
"SocketIOFileClient"
}
}

+ 1
- 0
frappe/boot.py Vedi File

@@ -30,6 +30,7 @@ def get_bootinfo():
get_user(bootinfo)

# system info
bootinfo.sitename = frappe.local.site
bootinfo.sysdefaults = frappe.defaults.get_defaults()
bootinfo.user_permissions = get_user_permissions()
bootinfo.server_date = frappe.utils.nowdate()


+ 1
- 0
frappe/public/build.json Vedi File

@@ -126,6 +126,7 @@
"public/js/lib/moment/moment-with-locales.min.js",
"public/js/lib/moment/moment-timezone-with-data.min.js",
"public/js/lib/socket.io.min.js",
"public/js/lib/socket.io-file-client.js",
"public/js/lib/markdown.js",
"public/js/lib/jSignature.min.js",
"public/js/frappe/translate.js",


+ 1
- 1
frappe/public/js/frappe/desk.js Vedi File

@@ -29,7 +29,7 @@ frappe.Application = Class.extend({
this.startup();
},
startup: function() {
frappe.socket.init();
frappe.socketio.init();
frappe.model.init();

if(frappe.boot.status==='failed') {


+ 1
- 1
frappe/public/js/frappe/request.js Vedi File

@@ -34,7 +34,7 @@ frappe.call = function(opts) {
var callback = function(data, response_text) {
if(data.task_id) {
// async call, subscribe
frappe.socket.subscribe(data.task_id, opts);
frappe.socketio.subscribe(data.task_id, opts);

if(opts.queued) {
opts.queued(data);


+ 59
- 52
frappe/public/js/frappe/socketio_client.js Vedi File

@@ -1,4 +1,4 @@
frappe.socket = {
frappe.socketio = {
open_tasks: {},
open_docs: [],
emit_queue: [],
@@ -7,40 +7,40 @@ frappe.socket = {
return;
}

if (frappe.socket.socket) {
if (frappe.socketio.socket) {
return;
}

if (frappe.boot.developer_mode) {
// File watchers for development
frappe.socket.setup_file_watchers();
frappe.socketio.setup_file_watchers();
}

//Enable secure option when using HTTPS
if (window.location.protocol == "https:") {
frappe.socket.socket = io.connect(frappe.socket.get_host(), {secure: true});
frappe.socketio.socket = io.connect(frappe.socketio.get_host(), {secure: true});
}
else if (window.location.protocol == "http:") {
frappe.socket.socket = io.connect(frappe.socket.get_host());
frappe.socketio.socket = io.connect(frappe.socketio.get_host());
}
else if (window.location.protocol == "file:") {
frappe.socket.socket = io.connect(window.localStorage.server);
frappe.socketio.socket = io.connect(window.localStorage.server);
}

if (!frappe.socket.socket) {
console.log("Unable to connect to " + frappe.socket.get_host());
if (!frappe.socketio.socket) {
console.log("Unable to connect to " + frappe.socketio.get_host());
return;
}

frappe.socket.socket.on('msgprint', function(message) {
frappe.socketio.socket.on('msgprint', function(message) {
frappe.msgprint(message);
});

frappe.socket.socket.on('eval_js', function(message) {
frappe.socketio.socket.on('eval_js', function(message) {
eval(message);
});

frappe.socket.socket.on('progress', function(data) {
frappe.socketio.socket.on('progress', function(data) {
if(data.progress) {
data.percent = flt(data.progress[0]) / data.progress[1] * 100;
}
@@ -53,23 +53,24 @@ frappe.socket = {
}
});

frappe.socket.setup_listeners();
frappe.socket.setup_reconnect();
frappe.socketio.setup_listeners();
frappe.socketio.setup_reconnect();
frappe.socketio.setup_fileupload();

$(document).on('form-load form-rename', function(e, frm) {
if (frm.is_new()) {
return;
}

for (var i=0, l=frappe.socket.open_docs.length; i<l; i++) {
var d = frappe.socket.open_docs[i];
for (var i=0, l=frappe.socketio.open_docs.length; i<l; i++) {
var d = frappe.socketio.open_docs[i];
if (frm.doctype==d.doctype && frm.docname==d.name) {
// already subscribed
return false;
}
}

frappe.socket.doc_subscribe(frm.doctype, frm.docname);
frappe.socketio.doc_subscribe(frm.doctype, frm.docname);
});

$(document).on("form_refresh", function(e, frm) {
@@ -77,7 +78,7 @@ frappe.socket = {
return;
}

frappe.socket.doc_open(frm.doctype, frm.docname);
frappe.socketio.doc_open(frm.doctype, frm.docname);
});

$(document).on('form-unload', function(e, frm) {
@@ -85,8 +86,8 @@ frappe.socket = {
return;
}

// frappe.socket.doc_unsubscribe(frm.doctype, frm.docname);
frappe.socket.doc_close(frm.doctype, frm.docname);
// frappe.socketio.doc_unsubscribe(frm.doctype, frm.docname);
frappe.socketio.doc_close(frm.doctype, frm.docname);
});

window.onbeforeunload = function() {
@@ -96,7 +97,7 @@ frappe.socket = {

// if tab/window is closed, notify other users
if (cur_frm.doc) {
frappe.socket.doc_close(cur_frm.doctype, cur_frm.docname);
frappe.socketio.doc_close(cur_frm.doctype, cur_frm.docname);
}
}
},
@@ -115,16 +116,16 @@ frappe.socket = {
subscribe: function(task_id, opts) {
// TODO DEPRECATE

frappe.socket.socket.emit('task_subscribe', task_id);
frappe.socket.socket.emit('progress_subscribe', task_id);
frappe.socketio.socket.emit('task_subscribe', task_id);
frappe.socketio.socket.emit('progress_subscribe', task_id);

frappe.socket.open_tasks[task_id] = opts;
frappe.socketio.open_tasks[task_id] = opts;
},
task_subscribe: function(task_id) {
frappe.socket.socket.emit('task_subscribe', task_id);
frappe.socketio.socket.emit('task_subscribe', task_id);
},
task_unsubscribe: function(task_id) {
frappe.socket.socket.emit('task_unsubscribe', task_id);
frappe.socketio.socket.emit('task_unsubscribe', task_id);
},
doc_subscribe: function(doctype, docname) {
if (frappe.flags.doc_subscribe) {
@@ -137,12 +138,12 @@ frappe.socket = {
// throttle to 1 per sec
setTimeout(function() { frappe.flags.doc_subscribe = false }, 1000);

frappe.socket.socket.emit('doc_subscribe', doctype, docname);
frappe.socket.open_docs.push({doctype: doctype, docname: docname});
frappe.socketio.socket.emit('doc_subscribe', doctype, docname);
frappe.socketio.open_docs.push({doctype: doctype, docname: docname});
},
doc_unsubscribe: function(doctype, docname) {
frappe.socket.socket.emit('doc_unsubscribe', doctype, docname);
frappe.socket.open_docs = $.filter(frappe.socket.open_docs, function(d) {
frappe.socketio.socket.emit('doc_unsubscribe', doctype, docname);
frappe.socketio.open_docs = $.filter(frappe.socketio.open_docs, function(d) {
if(d.doctype===doctype && d.name===docname) {
return null;
} else {
@@ -152,44 +153,50 @@ frappe.socket = {
},
doc_open: function(doctype, docname) {
// notify that the user has opened this doc, if not already notified
if(!frappe.socket.last_doc
|| (frappe.socket.last_doc[0]!=doctype && frappe.socket.last_doc[0]!=docname)) {
frappe.socket.socket.emit('doc_open', doctype, docname);
if(!frappe.socketio.last_doc
|| (frappe.socketio.last_doc[0]!=doctype && frappe.socketio.last_doc[0]!=docname)) {
frappe.socketio.socket.emit('doc_open', doctype, docname);
}
frappe.socket.last_doc = [doctype, docname];
frappe.socketio.last_doc = [doctype, docname];
},
doc_close: function(doctype, docname) {
// notify that the user has closed this doc
frappe.socket.socket.emit('doc_close', doctype, docname);
frappe.socketio.socket.emit('doc_close', doctype, docname);
},
setup_fileupload: function() {
frappe.socketio.uploader = new SocketIOFileClient(frappe.socketio.socket, {
rename: function(filename) {
return `${frappe.boot.sitename}_${filename}`;
});
},
setup_listeners: function() {
frappe.socket.socket.on('task_status_change', function(data) {
frappe.socket.process_response(data, data.status.toLowerCase());
frappe.socketio.socket.on('task_status_change', function(data) {
frappe.socketio.process_response(data, data.status.toLowerCase());
});
frappe.socket.socket.on('task_progress', function(data) {
frappe.socket.process_response(data, "progress");
frappe.socketio.socket.on('task_progress', function(data) {
frappe.socketio.process_response(data, "progress");
});
},
setup_reconnect: function() {
// subscribe again to open_tasks
frappe.socket.socket.on("connect", function() {
frappe.socketio.socket.on("connect", function() {
// wait for 5 seconds before subscribing again
// because it takes more time to start python server than nodejs server
// and we use validation requests to python server for subscribing
setTimeout(function() {
$.each(frappe.socket.open_tasks, function(task_id, opts) {
frappe.socket.subscribe(task_id, opts);
$.each(frappe.socketio.open_tasks, function(task_id, opts) {
frappe.socketio.subscribe(task_id, opts);
});

// re-connect open docs
$.each(frappe.socket.open_docs, function(d) {
$.each(frappe.socketio.open_docs, function(d) {
if(locals[d.doctype] && locals[d.doctype][d.name]) {
frappe.socket.doc_subscribe(d.doctype, d.name);
frappe.socketio.doc_subscribe(d.doctype, d.name);
}
});

if (cur_frm && cur_frm.doc) {
frappe.socket.doc_open(cur_frm.doc.doctype, cur_frm.doc.name);
frappe.socketio.doc_open(cur_frm.doc.doctype, cur_frm.doc.name);
}
}, 5000);
});
@@ -208,9 +215,9 @@ frappe.socket = {
}
host = host + ':' + port;

frappe.socket.file_watcher = io.connect(host);
frappe.socketio.file_watcher = io.connect(host);
// css files auto reload
frappe.socket.file_watcher.on('reload_css', function(filename) {
frappe.socketio.file_watcher.on('reload_css', function(filename) {
let abs_file_path = "assets/" + filename;
const link = $(`link[href*="${abs_file_path}"]`);
abs_file_path = abs_file_path.split('?')[0] + '?v='+ moment();
@@ -221,7 +228,7 @@ frappe.socket = {
}, 5);
});
// js files show alert
frappe.socket.file_watcher.on('reload_js', function(filename) {
frappe.socketio.file_watcher.on('reload_js', function(filename) {
filename = "assets/" + filename;
var msg = $(`
<span>${filename} changed <a data-action="reload">Click to Reload</a></span>
@@ -239,7 +246,7 @@ frappe.socket = {
}

// success
var opts = frappe.socket.open_tasks[data.task_id];
var opts = frappe.socketio.open_tasks[data.task_id];
if(opts[method]) {
opts[method](data);
}
@@ -264,15 +271,15 @@ frappe.socket = {

frappe.provide("frappe.realtime");
frappe.realtime.on = function(event, callback) {
frappe.socket.socket && frappe.socket.socket.on(event, callback);
frappe.socketio.socket && frappe.socketio.socket.on(event, callback);
};

frappe.realtime.off = function(event, callback) {
frappe.socket.socket && frappe.socket.socket.off(event, callback);
frappe.socketio.socket && frappe.socketio.socket.off(event, callback);
}

frappe.realtime.publish = function(event, message) {
if(frappe.socket.socket) {
frappe.socket.socket.emit(event, message);
if(frappe.socketio.socket) {
frappe.socketio.socket.emit(event, message);
}
}

+ 259
- 0
frappe/public/js/lib/socket.io-file-client.js Vedi File

@@ -0,0 +1,259 @@
"use strict";
(function() {

var instanceId = 0;
function getInstanceId() {
return instanceId++;
}
// note that this function invoked from call/apply, which has "this" binded
function _upload(file, options) {
options = options || {};

var self = this;
var socket = this.socket;
var chunkSize = this.chunkSize;
var transmissionDelay = this.transmissionDelay;
var uploadId = file.uploadId;
var uploadTo = options.uploadTo || '';
var fileInfo = {
id: uploadId,
name: file.name,
size: file.size,
chunkSize: chunkSize,
sent: 0
};

uploadTo && (fileInfo.uploadTo = uploadTo);

// read file
var fileReader = new FileReader();
fileReader.onloadend = function() {
var buffer = fileReader.result;

// check file mime type if exists
if(self.accepts && self.accepts.length > 0) {
var found = false;

for(var i = 0; i < self.accepts.length; i++) {
var accept = self.accepts[i];

if(file.type === accept) {
found = true;
break;
}
}

if(!found) {
return self.emit('error', new Error('Not Acceptable file type ' + file.type + ' of ' + file.name + '. Type must be one of these: ' + self.accepts.join(', ')));
}
}

// check file size
if(self.maxFileSize && self.maxFileSize > 0) {
if(file.size > +self.maxFileSize) {
return self.emit('error', new Error('Max Uploading File size must be under ' + self.maxFileSize + ' byte(s).'));
}
}

// put into uploadingFiles list
self.uploadingFiles[uploadId] = fileInfo;

// request the server to make a file
self.emit('start', {
name: fileInfo.name,
size: fileInfo.size,
uploadTo: uploadTo
});
socket.emit('socket.io-file::createFile', fileInfo);

function sendChunk() {
if(fileInfo.aborted) {
return;
}

if(fileInfo.sent >= buffer.byteLength) {
socket.emit('socket.io-file::done::' + uploadId);
return;
}

var chunk = buffer.slice(fileInfo.sent, fileInfo.sent + chunkSize);

self.emit('stream', {
name: fileInfo.name,
size: fileInfo.size,
sent: fileInfo.sent,
uploadTo: uploadTo
});
socket.once('socket.io-file::request::' + uploadId, sendChunk);
socket.emit('socket.io-file::stream::' + uploadId, chunk);

fileInfo.sent += chunk.byteLength;
self.uploadingFiles[uploadId] = fileInfo;
}
socket.once('socket.io-file::request::' + uploadId, sendChunk);
socket.on('socket.io-file::complete::' + uploadId, function(info) {
self.emit('complete', info);
socket.removeAllListeners('socket.io-file::abort::' + uploadId);
socket.removeAllListeners('socket.io-file::error::' + uploadId);
socket.removeAllListeners('socket.io-file::complete::' + uploadId);

// remove from uploadingFiles list
delete self.uploadingFiles[uploadId];
});
socket.on('socket.io-file::abort::' + uploadId, function(info) {
fileInfo.aborted = true;
self.emit('abort', {
name: fileInfo.name,
size: fileInfo.size,
sent: fileInfo.sent,
wrote: info.wrote,
uploadTo: uploadTo
});
});
socket.on('socket.io-file::error::' + uploadId, function(err) {
self.emit('error', new Error(err.message));
});
};
fileReader.readAsArrayBuffer(file);
}

function SocketIOFileClient(socket, options) {
if(!socket) {
return this.emit('error', new Error('SocketIOFile requires Socket.'));
}

this.instanceId = getInstanceId(); // using for identifying multiple file upload from SocketIOFileClient objects
this.uploadId = 0; // using for identifying each uploading
this.ev = {}; // event handlers
this.options = options || {};
this.accepts = [];
this.maxFileSize = undefined;
this.socket = socket;
this.uploadingFiles = {};

var self = this;

socket.once('socket.io-file::recvSync', function(settings) {
self.maxFileSize = settings.maxFileSize || undefined;
self.accepts = settings.accepts || [];
self.chunkSize = settings.chunkSize || 10240;
self.transmissionDelay = settings.transmissionDelay || 0;

self.emit('ready');
});
socket.emit('socket.io-file::reqSync');
}
SocketIOFileClient.prototype.getUploadId = function() {
return 'u_' + this.uploadId++;
}
SocketIOFileClient.prototype.upload = function(fileEl, options) {
if(!fileEl ||
(fileEl.files && fileEl.files.length <= 0) ||
fileEl.length <= 0
) {
this.emit('error', new Error('No file(s) to upload.'));
return [];
}

var self = this;
var uploadIds = [];

var files = fileEl.files ? fileEl.files : fileEl;
var loaded = 0;

for(var i = 0; i < files.length; i++) {
var file = files[i];
var uploadId = this.getUploadId();
uploadIds.push(uploadId);

file.uploadId = uploadId;

_upload.call(self, file, options);
}
return uploadIds;
};
SocketIOFileClient.prototype.on = function(evName, fn) {
if(!this.ev[evName]) {
this.ev[evName] = [];
}

this.ev[evName].push(fn);
return this;
};
SocketIOFileClient.prototype.off = function(evName, fn) {
if(typeof evName === 'undefined') {
this.ev = [];
}
else if(typeof fn === 'undefined') {
if(this.ev[evName]) {
delete this.ev[evName];
}
}
else {
var evList = this.ev[evName] || [];

for(var i = 0; i < evList.length; i++) {
if(evList[i] === fn) {
evList = evList.splice(i, 1);
break;
}
}
}

return this;
};
SocketIOFileClient.prototype.emit = function(evName, args) {
var evList = this.ev[evName] || [];

for(var i = 0; i < evList.length; i++) {
evList[i](args);
}

return this;
};
SocketIOFileClient.prototype.abort = function(id) {
var socket = this.socket;
socket.emit('socket.io-file::abort::' + id);
};
SocketIOFileClient.prototype.destroy = function() {
var uploadingFiles = this.uploadingFiles;

for(var key in uploadingFiles) {
this.abort(key);
}

this.socket = null;
this.uploadingFiles = null;
this.ev = null;
};
SocketIOFileClient.prototype.getUploadInfo = function() {
return JSON.parse(JSON.stringify(this.uploadingFiles));
};

// module export
// CommonJS
if (typeof exports === "object" && typeof module !== "undefined") {
module.exports = SocketIOFileClient;
}
// RequireJS
else if (typeof define === "function" && define.amd) {
define(['SocketIOFileClient'], SocketIOFileClient);
}
else {
var g;

if (typeof window !== "undefined") {
g = window;
}
else if (typeof global !== "undefined") {
g = global;
}
else if (typeof self !== "undefined") {
g = self;
}

g.SocketIOFileClient = SocketIOFileClient;
}
})();

+ 50
- 49
frappe/utils/file_manager.py Vedi File

@@ -15,6 +15,7 @@ from six import text_type

class MaxFileSizeReachedError(frappe.ValidationError): pass


def get_file_url(file_data_name):
data = frappe.db.get_value("File", file_data_name, ["file_name", "file_url"], as_dict=True)
return data.file_url or data.file_name
@@ -97,55 +98,6 @@ def get_uploaded_content():
frappe.msgprint(_('No file attached'))
return None, None

def extract_images_from_doc(doc, fieldname):
content = doc.get(fieldname)
content = extract_images_from_html(doc, content)
if frappe.flags.has_dataurl:
doc.set(fieldname, content)

def extract_images_from_html(doc, content):
frappe.flags.has_dataurl = False

def _save_file(match):
data = match.group(1)
data = data.split("data:")[1]
headers, content = data.split(",")

if "filename=" in headers:
filename = headers.split("filename=")[-1]

# decode filename
if not isinstance(filename, text_type):
filename = text_type(filename, 'utf-8')
else:
mtype = headers.split(";")[0]
filename = get_random_filename(content_type=mtype)

doctype = doc.parenttype if doc.parent else doc.doctype
name = doc.parent or doc.name

# TODO fix this
file_url = save_file(filename, content, doctype, name, decode=True).get("file_url")
if not frappe.flags.has_dataurl:
frappe.flags.has_dataurl = True

return '<img src="{file_url}"'.format(file_url=file_url)

if content:
content = re.sub('<img[^>]*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content)

return content

def get_random_filename(extn=None, content_type=None):
if extn:
if not extn.startswith("."):
extn = "." + extn

elif content_type:
extn = mimetypes.guess_extension(content_type)

return random_string(7) + (extn or "")

def save_file(fname, content, dt, dn, folder=None, decode=False, is_private=0):
if decode:
if isinstance(content, text_type):
@@ -370,3 +322,52 @@ def download_file(file_url):
frappe.local.response.filename = file_url[file_url.rfind("/")+1:]
frappe.local.response.filecontent = filedata
frappe.local.response.type = "download"

def extract_images_from_doc(doc, fieldname):
content = doc.get(fieldname)
content = extract_images_from_html(doc, content)
if frappe.flags.has_dataurl:
doc.set(fieldname, content)

def extract_images_from_html(doc, content):
frappe.flags.has_dataurl = False

def _save_file(match):
data = match.group(1)
data = data.split("data:")[1]
headers, content = data.split(",")

if "filename=" in headers:
filename = headers.split("filename=")[-1]

# decode filename
if not isinstance(filename, text_type):
filename = text_type(filename, 'utf-8')
else:
mtype = headers.split(";")[0]
filename = get_random_filename(content_type=mtype)

doctype = doc.parenttype if doc.parent else doc.doctype
name = doc.parent or doc.name

# TODO fix this
file_url = save_file(filename, content, doctype, name, decode=True).get("file_url")
if not frappe.flags.has_dataurl:
frappe.flags.has_dataurl = True

return '<img src="{file_url}"'.format(file_url=file_url)

if content:
content = re.sub('<img[^>]*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content)

return content

def get_random_filename(extn=None, content_type=None):
if extn:
if not extn.startswith("."):
extn = "." + extn

elif content_type:
extn = mimetypes.guess_extension(content_type)

return random_string(7) + (extn or "")

+ 28
- 0
socketio.js Vedi File

@@ -134,6 +134,34 @@ io.on('connection', function(socket){
});
});

var uploader = new SocketIOFile(socket, {
// uploadDir: { // multiple directories
// music: 'data/music',
// document: 'data/document'
// },
uploadDir: 'sites/uploads',
// maxFileSize: 4194304, // 4 MB. default is undefined(no limit)
chunkSize: 10240, // default is 10240(1KB)
overwrite: true // overwrite file if exists, default is true.
});
uploader.on('start', (fileInfo) => {
console.log('Start uploading');
console.log(fileInfo);
});
uploader.on('stream', (fileInfo) => {
console.log(`${fileInfo.wrote} / ${fileInfo.size} byte(s)`);
});
uploader.on('complete', (fileInfo) => {
console.log('Upload Complete.');
console.log(fileInfo);
});
uploader.on('error', (err) => {
console.log('Error!', err);
});
uploader.on('abort', (fileInfo) => {
console.log('Aborted: ', fileInfo);
});

// socket.on('disconnect', function (arguments) {
// console.log("user disconnected", arguments);
// });


Scrivi Anteprima
Caricamento…
Annulla
Salva