fixes in data import tool - restrict import upto 500 rows, commit only if import is successful, else rollback

core/page/data_import_tool/data_import_tool.js

@@ -19,6 +19,7 @@ wn.pages['data-import-tool'].onload = function(wrapper) {
 		<h3>2. Import Data</h3>\
 		<p class="help">Attach .csv file to import data</p>\
 		<div id="dit-upload-area"></div><br>\
+		<div class="dit-progress-area" style="display: None"></div>\
 		<p id="dit-output"></p>\
 		');
 		
@@ -43,6 +44,9 @@ wn.pages['data-import-tool'].onload = function(wrapper) {
 	
 	$select = $(wrapper).find('[name="dit-doctype"]');
 	
+	wn.messages.waiting($(wrapper).find(".dit-progress-area").toggle(false), 
+		"Performing hardcore import process....", 100);
+	
 	// load doctypes
 	wn.call({
 		method:'core.page.data_import_tool.data_import_tool.get_doctypes',
@@ -99,14 +103,33 @@ wn.pages['data-import-tool'].onload = function(wrapper) {
 			method: 'core.page.data_import_tool.data_import_tool.upload'
 		},
 		callback: function(r) {
-			$('#dit-output').empty();
+			$(wrapper).find(".dit-progress-area").toggle(false);
 			
-			$.each(r, function(i, v) {
+			// replace links if error has occured
+			if(r.error) {
+				r.messages = $.map(r.messages, function(v) {
+					var msg = v.replace("Inserted", "Valid").split("<");
+					if (msg.length > 1) {
+						v = msg[0] + (msg[1].split(">").slice(-1)[0]);
+					} else {
+						v = msg[0];
+					}
+					return v;
+				});
+				
+				r.messages = ["<h4 style='color:red'>Import Failed!</h4>"]
+					.concat(r.messages)
+			} else {
+				r.messages = ["<h4 style='color:green'>Import Successful!</h4>"].
+					concat(r.messages)
+			}
+			
+			$.each(r.messages, function(i, v) {
 				var $p = $('<p>').html(v).appendTo('#dit-output');
 				if(v.substr(0,5)=='Error') {
 					$p.css('color', 'red');
 				}
-				if(v.substr(0,8)=='Inserted') {
+				if(v.substr(0,8)=='Inserted' || v.substr(0,5)=='Valid') {
 					$p.css('color', 'green');
 				}
 				if(v.substr(0,7)=='Updated') {
@@ -137,6 +160,7 @@ wn.pages['data-import-tool'].onload = function(wrapper) {
 	$('#dit-upload-area form input[type="submit"]')
 		.attr('value', 'Upload and Import')
 		.click(function() {
-			$('#dit-output').html('Performing hardcore import process....')
+			$('#dit-output').empty();
+			$(wrapper).find(".dit-progress-area").toggle(true);
 		});
 }

core/page/data_import_tool/data_import_tool.py

@@ -101,7 +101,7 @@ def get_template():
 			w.writerow(row)
 
 	# write out response as a type csv
-	webnotes.response['result'] = cstr(w.queue.getvalue())
+	webnotes.response['result'] = cstr(w.getvalue())
 	webnotes.response['type'] = 'csv'
 	webnotes.response['doctype'] = doctype
 
@@ -118,44 +118,36 @@ def upload():
 	from webnotes.model.doc import Document
 	from webnotes.utils.datautils import read_csv_content_from_uploaded_file
 	
-	overwrite = webnotes.form_dict.get('overwrite')
-
-	ret = []
-	
 	rows = read_csv_content_from_uploaded_file()
-	
-	# doctype
 	doctype = rows[0][0].split(':')[1].strip()
-	doctype_dl = webnotes.model.doctype.get(doctype, form=0)
 	
-	if doctype in ['Customer', 'Supplier'] and len(rows[8:]) > 100:
-		webnotes.msgprint("Please upload only upto 100 %ss at a time" % doctype)
+	# allow limit rows to be uploaded
+	max_rows = 500
+	if len(rows[8:]) > max_rows:
+		webnotes.msgprint("Please upload only upto %d %ss at a time" % \
+			(max_rows, doctype))
 		raise Exception
+
+	webnotes.conn.begin()
 	
-	parenttype, parentfield = None, None
-	if len(rows[1]) > 0 and ':' in rows[1][0]:
-		parenttype = rows[1][0].split(':')[1].strip()
-	
-	# get parentfield
-	if parenttype:
-		import webnotes.model.doctype
-		for d in webnotes.model.doctype.get(parenttype):
-			if d.fieldtype=='Table' and d.options==doctype:
-				parentfield = d.fieldname
-				break
-	
-		if not parentfield:
-			webnotes.msgprint("Did not find parentfield for %s (%s)" % (parenttype, doctype),
-				raise_exception=1)
-			
-	# columns
+	overwrite = webnotes.form_dict.get('overwrite')
+	doctype_dl = webnotes.model.doctype.get(doctype, form=0)
 	columns = rows[3][1:]
 	
+	# parent details
+	parenttype, parentfield = get_parent_details(rows)
 	if parenttype and overwrite:
 		delete_child_rows(rows, doctype)
-		
-	for i, row in enumerate(rows[8:]):
+
+	ret = []
+	error = False
+	start_row = 8
+	for i, row in enumerate(rows[start_row:]):
+		# bypass empty rows
 		if not row: continue
+		
+		row_idx = (i + 1) + start_row
+		
 		d = dict(zip(columns, row[1:]))
 		d['doctype'] = doctype
 				
@@ -169,14 +161,43 @@ def upload():
 					doc.parenttype = parenttype
 					doc.parentfield = parentfield
 				doc.save()
-				ret.append('Inserted row for %s at #%s' % (getlink(parenttype, doc.parent), 
-					unicode(doc.idx)))
+				ret.append('Inserted row for %s at #%s' % (getlink(parenttype,
+					doc.parent), unicode(doc.idx)))
 			else:
-				ret.append(import_doc(d, doctype, overwrite))
+				ret.append(import_doc(d, doctype, overwrite, row_idx))
 		except Exception, e:
-			ret.append('Error for row (#%d) %s : %s' % (i+9, row[1], cstr(e)))
+			error = True
+			ret.append('Error for row (#%d) %s : %s' % (row_idx, row[1], cstr(e)))
 			webnotes.errprint(webnotes.getTraceback())
-	return ret
+	
+	if error:
+		webnotes.conn.rollback()		
+	else:
+		webnotes.conn.commit()
+	
+	return {"messages": ret, "error": error}
+	
+def get_parent_details(rows):
+	parenttype, parentfield = None, None
+	
+	# get parenttype
+	if len(rows[1]) > 0 and ':' in rows[1][0]:
+		parenttype = rows[1][0].split(':')[1].strip()
+		
+	# get parentfield
+	if parenttype:
+		import webnotes.model.doctype
+		for d in webnotes.model.doctype.get(parenttype):
+			if d.fieldtype=='Table' and d.options==doctype:
+				parentfield = d.fieldname
+				break
+	
+		if not parentfield:
+			webnotes.msgprint("Did not find parentfield for %s (%s)" % \
+				(parenttype, doctype))
+			raise Exception
+	
+	return parenttype, parentfield
 	
 def check_record(d, parenttype):
 	"""check for mandatory, select options, dates. these should ideally be in doclist"""
@@ -193,22 +214,19 @@ def check_record(d, parenttype):
 			if docfield.reqd and (val=='' or val==None):
 				raise Exception, "%s is mandatory." % key
 
-			if docfield.fieldtype=='Select':
+			if docfield.fieldtype=='Select' and val:
 				if docfield.options and docfield.options.startswith('link:'):
-					if val:
-						link_doctype = docfield.options.split(':')[1]
-						if not webnotes.conn.exists(link_doctype, val):
-							raise Exception, "%s must be a valid %s" % (key, link_doctype)
-				else:
-					if val:
-						if not docfield.options:
-							raise Exception, "Select options are missing for %s"
-						elif val not in docfield.options.split('\n'):
-							raise Exception, "%s must be one of: %s" % (key, 
-								", ".join(filter(None, docfield.options.split("\n"))))
-						
-			if docfield.fieldtype=='Date' and val:
-				d[key] = user_to_str(val, webnotes.form_dict['date_format'])
+					link_doctype = docfield.options.split(':')[1]
+					if not webnotes.conn.exists(link_doctype, val):
+						raise Exception, "%s must be a valid %s" % (key, link_doctype)
+				elif not docfield.options:
+					raise Exception, "Select options are missing for %s"
+				elif val not in docfield.options.split('\n'):
+					raise Exception, "%s must be one of: %s" % (key, 
+						", ".join(filter(None, docfield.options.split("\n"))))
+					
+			if val and docfield.fieldtype=='Date':
+				d[key] = parse_date(val, webnotes.form_dict['date_format'])
 
 def getlink(doctype, name):
 	return '<a href="#Form/%(doctype)s/%(name)s">%(name)s</a>' % locals()
@@ -218,8 +236,35 @@ def delete_child_rows(rows, doctype):
 	import webnotes
 	for p in list(set([r[1] for r in rows[8:]])):
 		webnotes.conn.sql("""delete from `tab%s` where parent=%s""" % (doctype, '%s'), p)
+		
+def parse_date(val, format):
+	from webnotes.utils.dateutils import user_to_str
+	
+	parsed_val = None
+	check_formats = [
+		format, # the one mentioned in the form
+		None, # users default date format
+		"yyyy-mm-dd", # system date format
+		"dd-mmm-yyyy", # numbers app format
+		"mm/dd/yyyy", # excel app format
+	]
+	
+	for f in check_formats:
+		try:
+			parsed_val = user_to_str(val, f, verbose=0)
+			if parsed_val:
+				webnotes.errprint([f, parsed_val])
+				break
+		except ValueError, e:
+			pass
+	
+	if not parsed_val:
+		webnotes.msgprint("Cannot identify date")
+		raise Exception
+		
+	return parsed_val
 
-def import_doc(d, doctype, overwrite):
+def import_doc(d, doctype, overwrite, row_idx):
 	"""import main (non child) document"""
 	import webnotes
 	import webnotes.model.doc
@@ -230,13 +275,16 @@ def import_doc(d, doctype, overwrite):
 			doclist = webnotes.model.doc.get(doctype, d['name'])
 			doclist[0].fields.update(d)
 			DocList(doclist).save()
-			return 'Updated ' + getlink(doctype, d['name'])
+			return 'Updated row (#%d) %s' % (row_idx, getlink(doctype, d['name']))
 		else:
-			return 'Ignored ' + getlink(doctype, d['name']) + ' (exists)'
+			return 'Ignored row (#%d) %s (exists)' % (row_idx, 
+				getlink(doctype, d['name']))
 	else:
 		d['__islocal'] = 1
-		DocList([webnotes.model.doc.Document(fielddata = d)]).save()
-		return 'Inserted ' + getlink(doctype, d['name'])
+		dl = DocList([webnotes.model.doc.Document(fielddata = d)])
+		dl.save()
+		return 'Inserted row (#%d) %s' % (row_idx, getlink(doctype,
+			dl.doc.fields['name']))
 
 import csv, cStringIO
 from webnotes.utils import encode
@@ -249,4 +297,6 @@ class UnicodeWriter:
 	def writerow(self, row):
 		row = encode(row, self.encoding)
 		self.writer.writerow(row)
-	
+	
+	def getvalue(self):
+		return self.queue.getvalue()

public/js/wn/upload.js

@@ -27,6 +27,7 @@ wn.upload = {
 		}
 		
 		$('#' + id).get(0).callback = opts.callback
+		
 	},
 	callback: function(id, file_id, args) {
 		$('#' + id).get(0).callback(file_id, args);

webnotes/db.py

@@ -264,13 +264,12 @@ class Database:
 		"""Get a single / multiple value from a record. 
 		For Single DocType, let filters be = None"""
 		if filters is not None and (filters!=doctype or filters=='DocType'):
-			
 			fl = isinstance(fieldname, basestring) and fieldname or "`, `".join(fieldname)
 			conditions, filters = self.build_conditions(filters)
 			
 			try:
-				r = self.sql("select `%s` from `tab%s` where %s" % (fl, doctype, conditions), 
-					filters, as_dict)
+				r = self.sql("select `%s` from `tab%s` where %s" % (fl, doctype,
+					conditions), filters, as_dict)
 			except Exception, e:
 				if e.args[0]==1054 and ignore:
 					return None

webnotes/utils/dateutils.py

@@ -22,32 +22,37 @@
 
 from __future__ import unicode_literals
 import webnotes
+import datetime
 
+# global values -- used for caching
 user_date_format = None
+dateformats = {
+	'yyyy-mm-dd': '%Y-%m-%d',
+	'dd/mm/yyyy': '%d/%m/%Y',
+	'mm/dd/yyyy': '%m/%d/%Y',
+	'dd-mm-yyyy': '%d-%m-%Y',
+	'dd-mmm-yyyy': '%d-%b-%Y', # numbers app format
+	'yyyy-mm-dd': '%Y-%m-%d',
+}
 
-def user_to_str(date, date_format = None):
+def user_to_str(date, date_format=None, verbose=1):
 	if not date: return date
 	
-	import datetime
-	
 	if not date_format:
-		if not user_date_format:
-			global user_date_format
-			user_date_format = webnotes.conn.get_value("Control Panel", None, "date_format")
+		date_format = get_user_date_format()
 
-		date_format = user_date_format
-		
-	dateformats = {
-		'yyyy-mm-dd':'%Y-%m-%d',
-		'dd/mm/yyyy':'%d/%m/%Y',
-		'mm/dd/yyyy':'%m/%d/%Y',
-		'dd-mm-yyyy':'%d-%m-%Y'
-	}
 	try:
 		return datetime.datetime.strptime(date, 
 			dateformats[date_format]).strftime('%Y-%m-%d')
 	except ValueError, e:
-		webnotes.msgprint("Date %s must be in format %s" % (date, date_format), raise_exception=True)
-
-
-	
+		if verbose:
+			webnotes.msgprint("Date %s must be in format %s" % (date, date_format),
+				raise_exception=True)
+		else:
+			raise e
+		
+def get_user_date_format():
+	if not user_date_format:
+		global user_date_format
+		user_date_format = webnotes.conn.get_value("Control Panel", None, "date_format")
+	return user_date_format