瀏覽代碼

data import tool and cleanup of files, handler and other minor

version-14
Rushabh Mehta 13 年之前
父節點
當前提交
b35719f1fa
共有 11 個文件被更改,包括 256 次插入157 次删除
  1. +1
    -0
      css/legacy/body.css
  2. +1
    -1
      js/wn/app.js
  3. +1
    -1
      js/wn/upload.js
  4. +1
    -1
      py/core/doctype/doctype_mapper/doctype_mapper.py
  5. +68
    -11
      py/core/page/data_import_tool/data_import_tool.js
  6. +128
    -37
      py/core/page/data_import_tool/data_import_tool.py
  7. +15
    -2
      py/webnotes/__init__.py
  8. +16
    -53
      py/webnotes/handler.py
  9. +2
    -2
      py/webnotes/model/code.py
  10. +12
    -29
      py/webnotes/model/doc.py
  11. +11
    -20
      py/webnotes/utils/file_manager.py

+ 1
- 0
css/legacy/body.css 查看文件

@@ -92,6 +92,7 @@ li {
}

hr {
clear: both;
margin: 18px 0;
border: 0;
border-top: 1px solid #e5e5e5;


+ 1
- 1
js/wn/app.js 查看文件

@@ -38,7 +38,7 @@ wn.Application = Class.extend({
})
} else {
// clear sid cookie
document.cookie = "sid=Guest;expires=Thu, 01 Jan 1970 00:00:00 GMT; path=/"
//document.cookie = "sid=Guest;expires=Thu, 01 Jan 1970 00:00:00 GMT; path=/"
this.startup();
//wn.views.pageview.show(window.home_page);
}


+ 1
- 1
js/wn/upload.js 查看文件

@@ -6,7 +6,7 @@ wn.upload = {
style="width:0px; height:0px; border:0px"></iframe>\
<form method="POST" enctype="multipart/form-data" \
action="%(action)s" target="%(id)s">\
<input type="file" name="filedata" /><br>\
<input type="file" name="filedata" /><br><br>\
<input type="submit" class="btn btn-small" value="Upload" />\
</form>', {
id: id,


+ 1
- 1
py/core/doctype/doctype_mapper/doctype_mapper.py 查看文件

@@ -25,7 +25,7 @@ import webnotes

from webnotes.utils import cint, cstr, default_fields, flt, formatdate, get_defaults, getdate, now, nowdate, replace_newlines, set_default
from webnotes.model import db_exists, default_fields
from webnotes.model.doc import Document, addchild, removechild, getchildren, make_autoname, SuperDocType
from webnotes.model.doc import Document, addchild, getchildren, make_autoname
from webnotes.model.doclist import getlist
from webnotes.model.code import get_obj
from webnotes import session, form, msgprint, errprint


+ 68
- 11
py/core/page/data_import_tool/data_import_tool.js 查看文件

@@ -6,17 +6,40 @@ wn.pages['data-import-tool'].onload = function(wrapper) {
$(wrapper).find('.layout-main-section').append('<h3>1. Download Template</h3>\
<div style="min-height: 150px">\
<p class="help">Download a template for importing a table</p>\
<p class="float-column"><select style="width: 200px" name="dit-doctype">\
</select></p>\
<p class="float-column dit-download"></p>\
<p class="help">Download a template for importing a table.</p>\
<p class="float-column">\
<select style="width: 200px" name="dit-doctype">\
</select><br><br>\
<input type="checkbox" name="dit-with-data">\
<span> Download with data</span>\
</p>\
<p class="float-column" id="dit-download"></p>\
</div>\
<hr>\
<h3>2. Import Data</h3>\
<p class="help">Attach file to import data</p>\
<p class="help">Attach .csv file to import data</p>\
<div id="dit-upload-area"></div><br>\
<p id="dit-output"></p>\
');
$(wrapper).find('.layout-side-section').append('<h4>Help</h4><br>\
<p><b>Date Format:</b></p>\
<p>Dates must be in format "YYYY-MM-DD", for example, \
31st Jan 2012 must be "2012-01-31"</p>\
<p><b>Importing non-English data:</b></p>\
<p>While uploading non English files ensure that the encoding is UTF-8.</p>\
<p>Microsoft Excel Users:\
<ol>\
<li>In Excel, save the file in CSV (Comma Delimited) format</li>\
<li>Open this saved file in Notepad</li>\
<li>Click on File -&gt; Save As</li>\
<li>File Name: &lt;your filename&gt;.csv<br />\
Save as type: Text Documents (*.txt)<br />\
Encoding: UTF-8\
</li>\
<li>Click on Save</li>\
</ol>\
</p>')
$select = $(wrapper).find('[name="dit-doctype"]');
@@ -32,20 +55,37 @@ wn.pages['data-import-tool'].onload = function(wrapper) {
$select.change(function() {
var val = $(this).val()
if(val!='Select...') {
$('.dit-download').empty();
$('#dit-download').empty();
// get options
wn.call({
method:'core.page.data_import_tool.data_import_tool.get_doctype_options',
args: {doctype: val},
callback: function(r) {
$('<h4>Select Template:</h4>').appendTo('.dit-download');
$('<h4>Select Template:</h4>').appendTo('#dit-download');
var with_data = $('[name="dit-with-data"]:checked').length ? 'Yes' : 'No';
// download link
$.each(r.message, function(i, v) {
$('<a href="index.cgi?cmd=core.page.data_import_tool.data_import_tool.get_template&doctype='
+v+'&parent_doctype='+val+'">'+v+'</a><br>')
.appendTo('.dit-download');
if(i==0)
$('<span>Main Table:</span><br>').appendTo('#dit-download');
if(i==1)
$('<br><span>Child Tables:</span><br>').appendTo('#dit-download');
$('<a style="cursor: pointer">')
.html(v)
.data('doctype', v)
.click(function() {
window.location.href = repl(wn.request.url
+ '?cmd=%(cmd)s&doctype=%(doctype)s'
+ '&parent_doctype=%(parent_doctype)s&with_data=%(with_data)s',
{
cmd: 'core.page.data_import_tool.data_import_tool.get_template',
doctype: $(this).data('doctype'),
parent_doctype: $('[name="dit-doctype"]').val(),
with_data: $('[name="dit-with-data"]:checked').length ? 'Yes' : 'No'
});
})
.appendTo('#dit-download');
$('#dit-download').append('<br>');
})
}
})
@@ -66,8 +106,25 @@ wn.pages['data-import-tool'].onload = function(wrapper) {
if(v.substr(0,5)=='Error') {
$p.css('color', 'red');
}
if(v.substr(0,8)=='Inserted') {
$p.css('color', 'green');
}
if(v.substr(0,7)=='Updated') {
$p.css('color', 'green');
}
});
}
});
// add overwrite option
$('<input type="checkbox" name="overwrite"><span> Overwrite</span><br><br>')
.insertBefore('#dit-upload-area form input[type="submit"]')
// rename button
$('#dit-upload-area form input[type="submit"]')
.attr('value', 'Upload and Import')
.click(function() {
$('#dit-output').html('Performing hardcore import process....')
});
}

+ 128
- 37
py/core/page/data_import_tool/data_import_tool.py 查看文件

@@ -12,25 +12,24 @@ def get_doctype_options():
import webnotes.model.doctype
return [doctype] + filter(None, map(lambda d: \
d.doctype=='DocField' and d.fieldtype=='Table' and d.options or None,
webnotes.model.doctype.get(doctype)))
webnotes.model.doctype.get(doctype, form=0)))

data_separator = '----Start entering data below this line----'

@webnotes.whitelist()
doctype_dl = None

@webnotes.whitelist(allow_roles=['System Manager', 'Administrator'])
def get_template():
import webnotes, csv
from cStringIO import StringIO
import webnotes.model.doctype
global doctype_dl

doctype = webnotes.form_dict['doctype']
parentdoctype = webnotes.form_dict.get('parent_doctype')
doclist = webnotes.model.doctype.get(doctype)
tablefields = [f[0] for f in webnotes.conn.sql('desc `tab%s`' % doctype)]

def getdocfield(fieldname):
l = [d for d in doclist if d.doctype=='DocField' and d.fieldname==fieldname]
return l and l[0] or None
doctype_dl = webnotes.model.doctype.get(doctype)
tablecolumns = [f[0] for f in webnotes.conn.sql('desc `tab%s`' % doctype)]

def getinforow(docfield):
"""make info comment"""
@@ -53,7 +52,7 @@ def get_template():
w.writerow(['Upload Template for: %s' % doctype])
if parentdoctype != doctype:
w.writerow(['This is a child table for %s' % parentdoctype])
w.writerow(['This is a child table for: %s' % parentdoctype])
key = 'parent'
else:
w.writerow([''])
@@ -64,25 +63,26 @@ def get_template():
mandatoryrow = ['Mandatory:', 'Yes']
typerow = ['Type:', 'Data (text)']
inforow = ['Info:', 'ID']
# get all mandatory fields
for t in tablefields:
columns = [key]
def append_row(t, mandatory):
docfield = getdocfield(t)
if docfield and docfield.reqd:
if docfield and ((mandatory and docfield.reqd) or (not mandatory and not docfield.reqd)) \
and (t not in ('parenttype', 'trash_reason')):
fieldrow.append(t)
mandatoryrow.append('Yes')
typerow.append(docfield.fieldtype)
mandatoryrow.append(docfield.reqd and 'Yes' or 'No')
typerow.append(docfield.fieldtype)
inforow.append(getinforow(docfield))
columns.append(t)
# get all mandatory fields
for t in tablecolumns:
append_row(t, True)

# all non mandatory fields
for t in tablefields:
docfield = getdocfield(t)
if docfield and not docfield.reqd:
fieldrow.append(t)
mandatoryrow.append('No')
typerow.append(docfield.fieldtype)
inforow.append(getinforow(docfield))
for t in tablecolumns:
append_row(t, False)

w.writerow(fieldrow)
w.writerow(mandatoryrow)
w.writerow(typerow)
@@ -90,43 +90,134 @@ def get_template():
w.writerow([data_separator])

if webnotes.form_dict.get('with_data')=='Yes':
data = webnotes.conn.sql("""select * from `tab%s` where docstatus<2""" % doctype, as_dict=1)
for d in data:
row = ['']
for c in columns:
val = d.get(c, '')
if type(val) is unicode:
val = val.encode('utf-8')
row.append(val)
w.writerow(row)

# write out response as a type csv
webnotes.response['result'] = tobj.getvalue()
webnotes.response['type'] = 'csv'
webnotes.response['doctype'] = doctype
@webnotes.whitelist()

def getdocfield(fieldname):
"""get docfield from doclist of doctype"""
l = [d for d in doctype_dl if d.doctype=='DocField' and d.fieldname==fieldname]
return l and l[0] or None

@webnotes.whitelist(allow_roles=['System Manager', 'Administrator'])
def upload():
"""upload data"""
import csv
global doctype_dl
from webnotes.utils.file_manager import get_uploaded_content
import webnotes.model.doctype
from webnotes.model.doc import Document
from webnotes.model.doclist import DocList

fname, fcontent = get_uploaded_content()
overwrite = webnotes.form_dict.get('overwrite')

ret, rows = [], []
for row in csv.reader(fcontent.splitlines()):
rows.append([c.strip() for c in row])
try:
reader = csv.reader(fcontent.splitlines())
# decode everything
for row in reader:
rows.append([unicode(c.strip(), 'utf-8') for c in row])
except Exception, e:
webnotes.msgprint("Not a valid Comma Separated Value (CSV File)")
raise e

# doctype
doctype = rows[0][0].split(':')[1].strip()
doctype_dl = webnotes.model.doctype.get(doctype, form=0)
parentdoctype = None
if len(rows[1]) > 0 and ':' in rows[1][0]:
parentdoctype = rows[1][0].split(':')[1].strip()
# columns
columns = rows[3][1:]
if parentdoctype and overwrite:
delete_child_rows(rows, doctype)
for row in rows[8:]:
d = dict(zip(columns, row[1:]))
d['doctype'] = doctype
try:
if not webnotes.conn.exists(doctype, d['name']):
d['__islocal'] = 1
DocList([Document(fielddata = d)]).save()
ret.append('Uploaded ' + row[1])
try:
check_record(d, parentdoctype)
if parentdoctype:
# child doc
doc = Document(doctype)
doc.fields.update(d)
doc.save()
ret.append('Inserted row for %s at #%s' % (getlink(parentdoctype, doc.parent),
str(doc.idx)))
else:
ret.append(import_doc(d, doctype, overwrite))
except Exception, e:
ret.append('Error for ' + row[1] + ': ' + str(e))
webnotes.errprint(webnotes.getTraceback())
return ret

def check_record(d, parentdoctype):
"""check for mandatory, select options, dates. these should ideally be in doclist"""
if parentdoctype and not d.get('parent'):
raise Exception, "parent is required."

for key in d:
docfield = getdocfield(key)
val = d[key]
if docfield:
if docfield.reqd and (val=='' or val==None):
raise Exception, "%s is mandatory." % key

if docfield.fieldtype=='Select':
if docfield.options.startswith('link:'):
if val:
link_doctype = docfield.options.split(':')[1]
if not webnotes.conn.exists(link_doctype, val):
raise Exception, "%s must be a valid %s" % (key, link_doctype)
else:
if val not in docfield.options.split('\n'):
raise Exception, "%s must be one of:" % key
if docfield.fieldtype=='Date':
import datetime
datetime.datetime.strptime(val, '%Y-%m-%d')

def getlink(doctype, name):
return '<a href="#Form/%(doctype)s/%(name)s">%(name)s</a>' % locals()

def delete_child_rows(rows, doctype):
"""delete child rows for all parents"""
import webnotes
for p in list(set([r[1] for r in rows[8:]])):
webnotes.conn.sql("""delete from `tab%s` where parent=%s""" % (doctype, '%s'), p)

def import_doc(d, doctype, overwrite):
"""import main (non child) document"""
import webnotes
from webnotes.model.doc import Document
from webnotes.model.doclist import DocList

if webnotes.conn.exists(doctype, d['name']):
if overwrite:
doc = Document(doctype, d['name'])
doc.fields.update(d)
DocList([doc]).save()
return 'Updated ' + getlink(doctype, d['name'])
else:
return 'Ignored ' + getlink(doctype, d['name']) + ' (exists)'
else:
d['__islocal'] = 1
DocList([Document(fielddata = d)]).save()
return 'Inserted ' + getlink(doctype, d['name'])

+ 15
- 2
py/webnotes/__init__.py 查看文件

@@ -95,7 +95,7 @@ def msgprint(msg, small=0, raise_exception=0, as_table=False):
message_log.append((small and '__small:' or '')+cstr(msg or ''))
if raise_exception:
raise ValidationError
raise ValidationError, msg

def is_apache_user():
import os
@@ -190,12 +190,14 @@ def get_db_password(db_name):

whitelisted = []
guest_methods = []
def whitelist(allow_guest=False):
def whitelist(allow_guest=False, allow_roles=[]):
"""
decorator for whitelisting a function
Note: if the function is allowed to be accessed by a guest user,
it must explicitly be marked as allow_guest=True
for specific roles, set allow_roles = ['Administrator'] etc.
"""
def innerfn(fn):
global whitelisted, guest_methods
@@ -204,6 +206,17 @@ def whitelist(allow_guest=False):
if allow_guest:
guest_methods.append(fn)

if allow_roles:
roles = get_roles()
allowed = False
for role in allow_roles:
if role in roles:
allowed = True
break
if not allowed:
raise PermissionError, "Method not allowed"

return fn

return innerfn


+ 16
- 53
py/webnotes/handler.py 查看文件

@@ -154,58 +154,8 @@ def uploadfile():
if not webnotes.response.get('result'):
webnotes.response['result'] = """<script>
window.parent.wn.upload.callback("%s", %s);
var messages = %s;
if(messages.length) {
for(var i in messages)
window.parent.msgprint(messages[i]);
};
var errors = %s;
if(errors.length) {
for(var i in errors)
window.parent.console.log(errors[i]);
}
</script>""" % (webnotes.form_dict.get('_id'),
json.dumps(ret),
json.dumps(webnotes.message_log),
json.dumps(webnotes.debug_log))
# File upload (from scripts)
# ------------------------------------------------------------------------------------

@webnotes.whitelist()
def upload_many():
from webnotes.model.code import get_obj

# pass it on to upload_many method in Control Panel
cp = get_obj('Control Panel')
cp.upload_many(webnotes.form)
webnotes.response['result'] = """
<script type='text/javascript'>
%s
</script>
%s
%s""" % (cp.upload_callback(webnotes.form), '\n----\n'.join(webnotes.message_log).replace("'", "\'"), '\n----\n'.join(webnotes.debug_log).replace("'", "\'").replace("\n","<br>"))
webnotes.response['type'] = 'iframe'


@webnotes.whitelist()
def get_file():
import webnotes
import webnotes.utils.file_manager
form = webnotes.form

res = webnotes.utils.file_manager.get_file(form.getvalue('fname'))
if res:
webnotes.response['type'] = 'download'
webnotes.response['filename'] = res[0]
if hasattr(res[1], 'tostring'):
webnotes.response['filecontent'] = res[1].tostring()
else:
webnotes.response['filecontent'] = res[1]
else:
webnotes.msgprint('[get_file] Unknown file name')
</script>""" % (webnotes.form_dict.get('_id'),
json.dumps(ret))

@webnotes.whitelist(allow_guest=True)
def reset_password():
@@ -318,12 +268,25 @@ def print_csv():
print webnotes.response['result']

def print_iframe():
import json
print "Content-Type: text/html"
print
if webnotes.response.get('result'):
print webnotes.response['result']
if webnotes.debug_log:
print '''<script type='text/javascript'>alert("%s");</script>''' % ('-------'.join(webnotes.debug_log).replace('"', '').replace('\n',''))
print """
<script>
var messages = %s;
if(messages.length) {
for(var i in messages)
window.parent.msgprint(messages[i]);
};
var errors = %s;
if(errors.length) {
for(var i in errors)
window.parent.console.log(errors[i]);
}
</script>""" % (json.dumps(webnotes.message_log), json.dumps(webnotes.debug_log))

def print_raw():
import mimetypes


+ 2
- 2
py/webnotes/model/code.py 查看文件

@@ -39,7 +39,7 @@ import webnotes

from webnotes.utils import add_days, add_months, add_years, cint, cstr, date_diff, default_fields, flt, fmt_money, formatdate, generate_hash, getTraceback, get_defaults, get_first_day, get_last_day, getdate, has_common, month_name, now, nowdate, replace_newlines, sendmail, set_default, str_esc_quote, user_format, validate_email_add
from webnotes.model import db_exists
from webnotes.model.doc import Document, addchild, removechild, getchildren, make_autoname, SuperDocType
from webnotes.model.doc import Document, addchild, getchildren, make_autoname
from webnotes.model.utils import getlist
from webnotes.model.code import get_obj, get_server_obj, run_server_obj, updatedb, check_syntax
from webnotes import session, form, is_testing, msgprint, errprint
@@ -70,7 +70,7 @@ def execute(code, doc=None, doclist=[]):
# --------------------------------------------------
from webnotes.utils import add_days, add_months, add_years, cint, cstr, date_diff, default_fields, flt, fmt_money, formatdate, generate_hash, getTraceback, get_defaults, get_first_day, get_last_day, getdate, has_common, month_name, now, nowdate, replace_newlines, sendmail, set_default, str_esc_quote, user_format, validate_email_add
from webnotes.model import db_exists
from webnotes.model.doc import Document, addchild, removechild, getchildren
from webnotes.model.doc import Document, addchild, getchildren
from webnotes.model.utils import getlist
from webnotes import session, form, msgprint, errprint



+ 12
- 29
py/webnotes/model/doc.py 查看文件

@@ -29,19 +29,6 @@ import webnotes.model.meta

from webnotes.utils import *

# actually should be "BaseDocType" - deprecated. Only for v160
class SuperDocType:
def __init__(self):
pass
def __getattr__(self, name):
if self.__dict__.has_key(name):
return self.__dict__[name]
elif self.super and hasattr(self.super, name):
return getattr(self.super, name)
else:
raise AttributeError, 'BaseDocType Attribute Error'

class Document:
"""
The wn(meta-data)framework equivalent of a Database Record.
@@ -400,6 +387,9 @@ class Document:
# add missing parentinfo (if reqd)
if self.parent and not (self.parenttype and self.parentfield):
self.update_parentinfo()
if self.parent and not self.idx:
self.set_idx()

# if required, make new
if new or (not new and self.fields.get('__islocal')) and (not res.get('issingle')):
@@ -430,6 +420,12 @@ class Document:
self.parenttype = tmp[0][0]
self.parentfield = tmp[0][1]

def set_idx(self):
"""set idx"""
self.idx = (webnotes.conn.sql("""select max(idx) from `tab%s`
where parent=%s and parentfield=%s""" % (self.doctype, '%s', '%s'),
(self.parent, self.parentfield))[0][0] or 0) + 1

# check permissions
# ---------------------------------------------------------------------------

@@ -559,20 +555,6 @@ def addchild(parent, fieldname, childtype = '', local=0, doclist=None):
d.save(1)
"""
return parent.addchild(fieldname, childtype, local, doclist)

# Remove Child
# ------------

def removechild(d, is_local = 0):
"""
Sets the docstatus of the object d to 2 (deleted) and appends an 'old_parent:' to the parent name
"""
if not is_local:
set(d, 'docstatus', 2)
set(d, 'parent', 'old_parent:' + d.parent)
else:
d.parent = 'old_parent:' + d.parent
d.docstatus = 2
# Naming
# ------
@@ -644,11 +626,12 @@ def getchildren(name, childtype, field='', parenttype='', from_doctype=0, prefix
if field:
tmp = ' and parentfield="%s" ' % field
if parenttype:
if parenttype:
tmp = ' and parenttype="%s" ' % parenttype

try:
dataset = webnotes.conn.sql("select * from `%s%s` where parent='%s' %s order by idx" % (prefix, childtype, name, tmp))
dataset = webnotes.conn.sql("select * from `%s%s` where parent='%s' %s order by idx" \
% (prefix, childtype, name, tmp))
desc = webnotes.conn.get_description()
except Exception, e:
if prefix=='arc' and e.args[0]==1146:


+ 11
- 20
py/webnotes/utils/file_manager.py 查看文件

@@ -70,26 +70,17 @@ def add_file_list(dt, dn, fname, fid):
"""
udpate file_list attribute of the record
"""
try:
# get the old file_list
fl = webnotes.conn.get_value(dt, dn, 'file_list') or ''
if fl:
fl += '\n'
# add new file id
fl += fname + ',' + fid
# save
webnotes.conn.set_value(dt, dn, 'file_list', fl)
return True
fl = webnotes.conn.get_value(dt, dn, 'file_list') or ''
if fl:
fl += '\n'
# add new file id
fl += fname + ',' + fid

except Exception, e:
webnotes.response['result'] = """
<script type='text/javascript'>
window.parent.msgprint("Error while uploading: %s");
</script>""" % str(e)
# save
webnotes.conn.set_value(dt, dn, 'file_list', fl)

return True

def remove_all(dt, dn):
"""remove all files in a transaction"""
@@ -141,10 +132,10 @@ def get_uploaded_content():
webnotes.uploaded_filename, webnotes.uploaded_content = i.filename, i.file.read()
return webnotes.uploaded_filename, webnotes.uploaded_content
else:
webnotes.response['result'] = """<script type='text/javascript'>window.parent.msgprint("No file"); %s</script>""" % js_fail
webnotes.msgprint('No File');
return None, None

def save_uploaded(js_okay='window.parent.msgprint("File Upload Successful")', js_fail=''):
def save_uploaded():
import webnotes.utils
webnotes.response['type'] = 'iframe'


Loading…
取消
儲存