introducing model 3.0, refactored import module

core/doctype/customize_form/customize_form.py

@@ -139,7 +139,7 @@ class DocType:
 
 	def post(self):
 		"""
-			Save diff between Customize Form DocList and DocType DocList as property setter entries
+			Save diff between Customize Form ModelWrapper and DocType ModelWrapper as property setter entries
 		"""
 		if self.doc.doc_type:
 			from webnotes.model import doc

core/doctype/doctype/doctype.py

@@ -86,7 +86,7 @@ class DocType:
 		self.change_modified_of_parent()
 		
 		import conf
-		from webnotes.utils.transfer import in_transfer
+		from webnotes.modules.import_merge import in_transfer
 
 		if (not in_transfer) and getattr(conf,'developer_mode', 0):
 			self.export_doc()
@@ -96,7 +96,7 @@ class DocType:
 
 		
 	def export_doc(self):
-		from webnotes.modules.export_module import export_to_files
+		from webnotes.modules.export_file import export_to_files
 		export_to_files(record_list=[['DocType', self.doc.name]])
 		
 	def import_doc(self):

core/doctype/doctype_mapper/doctype_mapper.py

@@ -27,7 +27,7 @@ import webnotes
 from webnotes.utils import cint, cstr, default_fields, flt, formatdate, get_defaults, getdate, now, nowdate, replace_newlines, set_default
 from webnotes.model import db_exists, default_fields
 from webnotes.model.doc import Document, addchild, getchildren, make_autoname
-from webnotes.model.doclist import getlist
+from webnotes.model.wrapper import getlist
 from webnotes.model.code import get_obj
 from webnotes import session, form, msgprint, errprint
 from webnotes.model.doctype import get
@@ -308,5 +308,5 @@ class DocType:
 		"""
 		import conf
 		if hasattr(conf, 'developer_mode') and conf.developer_mode:
-			from webnotes.modules.export_module import export_to_files
+			from webnotes.modules.export_file import export_to_files
 			export_to_files(record_list=[[self.doc.doctype, self.doc.name]])		

core/doctype/page/page.py

@@ -67,9 +67,9 @@ class DocType:
 			it will write out a .html file
 		"""
 		import conf
-		from webnotes.utils.transfer import in_transfer
+		from webnotes.modules.import_merge import in_transfer
 		if not in_transfer and getattr(conf,'developer_mode', 0) and self.doc.standard=='Yes':
-			from webnotes.modules.export_module import export_to_files
+			from webnotes.modules.export_file import export_to_files
 			from webnotes.modules import get_module_path, scrub
 			import os
 			export_to_files(record_list=[['Page', self.doc.name]])

core/doctype/report/report.py

@@ -12,4 +12,15 @@ class DocType:
 		"""only administrator can save standard report"""
 		if self.doc.is_standard == "Yes" and webnotes.session.user!="Administrator":
 			webnotes.msgprint("""Only Administrator can save a standard report.
-			Please rename and save.""", raise_exception=True)
+			Please rename and save.""", raise_exception=True)
+
+	def on_update(self):
+		self.export_doc()
+	
+	def export_doc(self):
+		# export
+		import conf
+		if self.doc.is_standard == 'Yes' and getattr(conf, 'developer_mode', 0) == 1:
+			from webnotes.modules.export_file import export_to_files
+			export_to_files(record_list=[['Report', self.doc.name]], 
+				record_module=webnotes.conn.get_value("DocType", self.doc.ref_doctype, "module"))	

core/doctype/report/report.txt

@@ -5,7 +5,7 @@
 	{
 		u'creation': '2012-10-04 18:45:27',
 		u'docstatus': 0,
-		u'modified': '2012-11-12 12:53:08',
+		u'modified': '2012-11-12 12:53:09',
 		u'modified_by': u'Administrator',
 		u'owner': u'Administrator'
 	},

core/doctype/search_criteria/search_criteria.py

@@ -61,7 +61,7 @@ class DocType:
 		# export
 		import conf
 		if self.doc.standard == 'Yes' and getattr(conf, 'developer_mode', 0) == 1:
-			from webnotes.modules.export_module import export_to_files
+			from webnotes.modules.export_file import export_to_files
 			export_to_files(record_list=[['Search Criteria', self.doc.name]])
 
 	# patch to rename search criteria from old style numerical to

core/page/data_import_tool/data_import_tool.py

@@ -282,20 +282,20 @@ def delete_child_rows(rows, doctype):
 		
 def import_doc(d, doctype, overwrite, row_idx):
 	"""import main (non child) document"""
-	from webnotes.model.doclist import DocList
+	from webnotes.model.wrapper import ModelWrapper
 
 	if webnotes.conn.exists(doctype, d['name']):
 		if overwrite:
 			doclist = webnotes.model.doc.get(doctype, d['name'])
 			doclist[0].fields.update(d)
-			DocList(doclist).save()
+			ModelWrapper(doclist).save()
 			return 'Updated row (#%d) %s' % (row_idx, getlink(doctype, d['name']))
 		else:
 			return 'Ignored row (#%d) %s (exists)' % (row_idx, 
 				getlink(doctype, d['name']))
 	else:
 		d['__islocal'] = 1
-		dl = DocList([webnotes.model.doc.Document(fielddata = d)])
+		dl = ModelWrapper([webnotes.model.doc.Document(fielddata = d)])
 		dl.save()
 		return 'Inserted row (#%d) %s' % (row_idx, getlink(doctype,
 			dl.doc.fields['name']))

webnotes/__init__.py

@@ -262,5 +262,5 @@ def generate_hash():
 	return hashlib.sha224(str(time.time())).hexdigest()
 
 def model_wrapper(doctype, name=None):
-	from webnotes.model.doclist import DocList
-	return DocList(doctype, name)
+	from webnotes.model.wrapper import ModelWrapper
+	return ModelWrapper(doctype, name)

webnotes/client.py

@@ -29,12 +29,12 @@ def save():
 	"""insert or update from form query"""
 	doclist = json.loads(webnotes.form_dict.doclist)
 	
-	from webnotes.model.doclist import DocList
+	from webnotes.model.wrapper import ModelWrapper
 	
 	if not webnotes.has_permission(doclist[0]["doctype"], "write"):
 		webnotes.msgprint("No Write Permission", raise_exception=True)
 
-	doclistobj = DocList(doclist)
+	doclistobj = ModelWrapper(doclist)
 	doclistobj.save()
 	
 	return [d.fields for d in doclist]

webnotes/model/__init__.py

@@ -30,10 +30,10 @@ default_fields = ['doctype','name','owner','creation','modified','modified_by','
 #=================================================================================
 
 def insert(doclist):
-	from webnotes.model.doclist import DocList
+	from webnotes.model.wrapper import ModelWrapper
 	for d in doclist:
 		d["__islocal"] = 1
-	dl = DocList(doclist)
+	dl = ModelWrapper(doclist)
 	dl.save()
 	
 	return dl

webnotes/model/code.py

@@ -32,7 +32,7 @@ methods in following modules are imported for backward compatibility
 	* webnotes.*
 	* webnotes.utils.*
 	* webnotes.model.doc.*
-	* webnotes.model.doclist.*
+	* webnotes.model.wrapper.*
 """
 custom_class = '''
 # Please edit this list and import only required elements

webnotes/model/controller.py

@@ -22,7 +22,7 @@
 
 from __future__ import unicode_literals
 """
-Transactions are defined as collection of classes, a DocList represents collection of Document
+Transactions are defined as collection of classes, a ModelWrapper represents collection of Document
 objects for a transaction with main and children.
 
 Group actions like save, etc are performed on doclists
@@ -32,7 +32,7 @@ import webnotes
 import json, os
 import webnotes.model
 import webnotes.model.doc
-import webnotes.model.doclist
+import webnotes.model.wrapper
 import webnotes.utils.cache
 
 from webnotes import _
@@ -79,12 +79,12 @@ def get_controller_class(module_name, doctype, module_path):
 		for attr in dir(module):
 			attrobj = getattr(module, attr)
 			if inspect.isclass(attrobj) and attr.startswith(doctype.replace(' ', '').replace('-', '')) \
-				and issubclass(attrobj, DocListController):
+				and issubclass(attrobj, ModelWrapperController):
 				return attrobj
 		
-	return DocListController
+	return ModelWrapperController
 			
-class DocListController(object):
+class ModelWrapperController(object):
 	"""
 	Collection of Documents with one parent and multiple children
 	"""
@@ -104,11 +104,11 @@ class DocListController(object):
 		self.set_doclist(self.load_doclist(doctype, name))
 		
 	def load_doclist(self, doctype, name):
-		return webnotes.model.doclist.load(doctype, name)
+		return webnotes.model.wrapper.load(doctype, name)
 	
 	def set_doclist(self, doclist):
-		if not isinstance(doclist, webnotes.model.doclist.DocList):
-			self.doclist = webnotes.model.doclist.objectify(doclist)
+		if not isinstance(doclist, webnotes.model.wrapper.ModelWrapper):
+			self.doclist = webnotes.model.wrapper.objectify(doclist)
 		else:
 			self.doclist = doclist
 		self.doc = self.doclist[0]
@@ -125,7 +125,7 @@ class DocListController(object):
 		from webnotes.model.doctype import get_property
 		if get_property(self.doc.doctype, "document_type") in ["Master", "Transaction"]\
 			and not self.doc.get("__islocal"):
-			from webnotes.model.doclist import load
+			from webnotes.model.wrapper import load
 			# get the old doclist
 			try:
 				oldlist = load(self.doc.doctype, self.doc.name)

webnotes/model/doc.py

@@ -632,6 +632,7 @@ def getseries(key, digits, doctype=''):
 
 def getchildren(name, childtype, field='', parenttype='', from_doctype=0, prefix='tab'):
 	import webnotes
+	from webnotes.model.doclist import DocList
 	
 	tmp = ''
 	
@@ -640,17 +641,11 @@ def getchildren(name, childtype, field='', parenttype='', from_doctype=0, prefix
 	if parenttype:
 		tmp = ' and parenttype="%s" ' % parenttype
 
-	try:
-		dataset = webnotes.conn.sql("select * from `%s%s` where parent='%s' %s order by idx" \
-			% (prefix, childtype, name, tmp))
-		desc = webnotes.conn.get_description()
-	except Exception, e:
-		if prefix=='arc' and e.args[0]==1146:
-			return []
-		else:
-			raise e
+	dataset = webnotes.conn.sql("select * from `%s%s` where parent='%s' %s order by idx" \
+		% (prefix, childtype, name, tmp))
+	desc = webnotes.conn.get_description()
 
-	l = []
+	l = DocList()
 	
 	for i in dataset:
 		d = Document()
@@ -692,7 +687,8 @@ def get(dt, dn='', with_children = 1, from_get_obj = 0, prefix = 'tab'):
 	"""	
 	import webnotes
 	import webnotes.model
-
+	from webnotes.model.doclist import DocList
+	
 	dn = dn or dt
 
 	# load the main doc
@@ -709,13 +705,13 @@ def get(dt, dn='', with_children = 1, from_get_obj = 0, prefix = 'tab'):
 
 	if not with_children:
 		# done
-		return [doc,]
+		return DocList([doc,])
 	
 	# get all children types
 	tablefields = webnotes.model.meta.get_table_fields(dt)
 
 	# load chilren
-	doclist = [doc,]
+	doclist = DocList([doc,])
 	for t in tablefields:
 		doclist += getchildren(doc.name, t[0], t[1], dt, prefix=prefix)
 

webnotes/model/doclist.py

@@ -20,324 +20,142 @@
 # OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 # 
 
-from __future__ import unicode_literals
-"""
-Transactions are defined as collection of classes, a DocList represents collection of Document
-objects for a transaction with main and children.
-
-Group actions like save, etc are performed on doclists
-"""
-
 import webnotes
-from webnotes.utils import cint
+import webnotes.model
 from webnotes.model.doc import Document
 
-class DocList:
-	"""
-	Collection of Documents with one parent and multiple children
-	"""
-	def __init__(self, dt=None, dn=None):
-		self.docs = []
-		self.obj = None
-		self.to_docstatus = 0
-		if dt and dn:
-			self.load_from_db(dt, dn)
-		if type(dt) is list:
-			self.set_doclist(dt)
-
-	def load_from_db(self, dt=None, dn=None, prefix='tab'):
-		"""
-			Load doclist from dt
-		"""
-		from webnotes.model.doc import Document, getchildren
-		if not dt: dt = self.doc.doctype
-		if not dn: dn = self.doc.name
-
-		doc = Document(dt, dn, prefix=prefix)
-
-		# get all children types
-		tablefields = webnotes.model.meta.get_table_fields(dt)
-
-		# load chilren
-		doclist = [doc,]
-		for t in tablefields:
-			doclist += getchildren(doc.name, t[0], t[1], dt, prefix=prefix)
-
-		self.set_doclist(doclist)
-
-	def __iter__(self):
-		"""
-			Make this iterable
-		"""
-		return self.docs.__iter__()
-
-	def from_compressed(self, data, docname):
-		"""
-			Expand called from client
-		"""
-		from webnotes.model.utils import expand
-		self.docs = expand(data)
-		self.set_doclist(self.docs)
+class DocList(list):
+	"""DocList object as a wrapper around a list"""	
+	def get(self, filters, limit=0):
+		"""pass filters as:
+			{"key": "val", "key": ["!=", "val"],
+			"key": ["in", "val"], "key": ["not in", "val"], "key": "^val"}"""
+		# map reverse operations to set add = False
+		import operator
+		ops_map = {
+			"!=": lambda (a, b): operator.ne(a, b),
+			"in": lambda (a, b): operator.contains(b, a),
+			"not in": lambda (a, b): not operator.contains(b, a)
+		}
+			
+		out = []
 		
-	def set_doclist(self, docs):
-		for i, d in enumerate(docs):
-			if isinstance(d, dict):
-				docs[i] = Document(fielddata=d)
+		for doc in self:
+			d = isinstance(getattr(doc, "fields", None), dict) and doc.fields or doc
+			add = True
+			for f in filters:
+				fval = filters[f]
+					
+				if isinstance(fval, list):
+					if fval[0] in ops_map and not ops_map[fval[0]]((d.get(f), fval[1])):
+						add = False
+						break
+				elif isinstance(fval, basestring) and fval.startswith("^"):
+					if not (d.get(f) or "").startswith(fval[1:]):
+						add = False
+						break
+				elif d.get(f)!=fval:
+					add = False
+					break
+
+			if add:
+				out.append(doc)
+				if limit and (len(out)-1)==limit:
+					break
 		
-		self.docs = self.doclist = docs
-		self.doc, self.children = docs[0], docs[1:]
-		if self.obj:
-			self.obj.doclist = self.doclist
-			self.obj.doc = self.doc
-
-	def make_obj(self):
-		"""
-			Create a DocType object
-		"""
-		if self.obj: return self.obj
-
-		from webnotes.model.code import get_obj
-		self.obj = get_obj(doc=self.doc, doclist=self.children)
-		return self.obj
-
-	def to_dict(self):
-		"""
-			return as a list of dictionaries
-		"""
-		return [d.fields for d in self.docs]
-
-	def check_if_latest(self):
-		"""
-			Raises exception if the modified time is not the same as in the database
-		"""
-		from webnotes.model.meta import is_single
-
-		if (not is_single(self.doc.doctype)) and (not cint(self.doc.fields.get('__islocal'))):
-			tmp = webnotes.conn.sql("""
-				SELECT modified FROM `tab%s` WHERE name="%s" for update"""
-				% (self.doc.doctype, self.doc.name))
-
-			if tmp and str(tmp[0][0]) != str(self.doc.modified):
-				webnotes.msgprint("""
-				Document has been modified after you have opened it.
-				To maintain the integrity of the data, you will not be able to save your changes.
-				Please refresh this document. [%s/%s]""" % (tmp[0][0], self.doc.modified), raise_exception=1)
-
-	def check_permission(self):
-		"""
-			Raises exception if permission is not valid
-		"""
-		if not self.doc.check_perm(verbose=1):
-			webnotes.msgprint("Not enough permission to save %s" % self.doc.doctype, raise_exception=1)
-
-	def check_links(self):
-		"""
-			Checks integrity of links (throws exception if links are invalid)
-		"""
-		ref, err_list = {}, []
-		for d in self.docs:
-			if not ref.get(d.doctype):
-				ref[d.doctype] = d.make_link_list()
-
-			err_list += d.validate_links(ref[d.doctype])
-
-		if err_list:
-			webnotes.msgprint("""[Link Validation] Could not find the following values: %s.
-			Please correct and resave. Document Not Saved.""" % ', '.join(err_list), raise_exception=1)
-
-	def update_timestamps_and_docstatus(self):
-		"""
-			Update owner, creation, modified_by, modified, docstatus
-		"""
-		from webnotes.utils import now
-		ts = now()
-		user = webnotes.__dict__.get('session', {}).get('user') or 'Administrator'
-
-		for d in self.docs:
-			if self.doc.fields.get('__islocal'):
-				d.owner = user
-				d.creation = ts
-
-			d.modified_by = user
-			d.modified = ts
-			if d.docstatus != 2: # don't update deleted
-				d.docstatus = self.to_docstatus
-
-	def prepare_for_save(self, check_links):
-		"""
-			Set owner, modified etc before saving
-		"""
-		self.check_if_latest()
-		self.check_permission()
-		if check_links:
-			self.check_links()
-		self.update_timestamps_and_docstatus()
+		return DocList(out)
 
-	def run_method(self, method):
-		"""
-		Run a method and custom_method
-		"""
-		self.make_obj()
-		if hasattr(self.obj, method):
-			getattr(self.obj, method)()
-		if hasattr(self.obj, 'custom_' + method):
-			getattr(self.obj, 'custom_' + method)()
+	def getone(self, filters):
+		return self.get(filters, limit=1)[0]
 
-		trigger(method, self.obj.doc)
+	def extend(self, n):
+		list.extend(self, n)
+		return self
 		
-		self.set_doclist([self.obj.doc] + self.obj.doclist)
-
-	def save_main(self):
-		"""
-			Save the main doc
-		"""
-		try:
-			self.doc.save(cint(self.doc.fields.get('__islocal')))
-		except NameError, e:
-			webnotes.msgprint('%s "%s" already exists' % (self.doc.doctype, self.doc.name))
-
-			# prompt if cancelled
-			if webnotes.conn.get_value(self.doc.doctype, self.doc.name, 'docstatus')==2:
-				webnotes.msgprint('[%s "%s" has been cancelled]' % (self.doc.doctype, self.doc.name))
-			webnotes.errprint(webnotes.utils.getTraceback())
-			raise e
-
-	def save_children(self):
-		"""
-			Save Children, with the new parent name
-		"""
-		child_map = {}
-		for d in self.children:
-			if (d.fields.has_key('parent') and d.fields.get('parent')) or \
-					(d.fields.has_key("parentfield") and d.fields.get("parentfield")):
-				# if d.parent:
-				d.parent = self.doc.name # rename if reqd
-				d.parenttype = self.doc.doctype
+	def copy(self):
+		out = []
+		for d in self:
+			out.append(Document(d))
+		return DocList(out)
+		
+	def filter_valid_fields(self):
+		import webnotes.model
+		fieldnames = {}
+		for d in self:
+			remove = []
+			for f in d:
+				if f not in fieldnames.setdefault(d.doctype,
+						webnotes.model.get_fieldnames(d.doctype)):
+					remove.append(f)
+			for f in remove:
+				del d[f]
 				
-				d.save(new = cint(d.fields.get('__islocal')))
+	def append(self, doc):
+		if not isinstance(doc, Document):
+			doc = Document(doc)
 			
-			child_map.setdefault(d.doctype, []).append(d.name)
+		self._prepare_doc(doc)
+
+		super(DocList, self).append(doc)
 		
-		# delete all children in database that are not in the child_map
+	def extend(self, doclist):
+		doclist = objectify(doclist)
+		for doc in doclist:
+			self._prepare_doc(doc)
 		
-		# get all children types
-		tablefields = webnotes.model.meta.get_table_fields(self.doc.doctype)
-				
-		for dt in tablefields:
-			cnames = child_map.get(dt[0]) or []
-			if cnames:
-				webnotes.conn.sql("""delete from `tab%s` where parent=%s and parenttype=%s and
-					name not in (%s)""" % (dt[0], '%s', '%s', ','.join(['%s'] * len(cnames))), 
-						tuple([self.doc.name, self.doc.doctype] + cnames))
-			else:
-				webnotes.conn.sql("""delete from `tab%s` where parent=%s and parenttype=%s""" \
-					% (dt[0], '%s', '%s'), (self.doc.name, self.doc.doctype))
-
-	def save(self, check_links=1):
-		"""
-			Save the list
-		"""
-		self.prepare_for_save(check_links)
-		self.run_method('validate')
-		self.save_main()
-		self.save_children()
-		self.run_method('on_update')
-
-	def submit(self):
-		"""
-			Save & Submit - set docstatus = 1, run "on_submit"
-		"""
-		if self.doc.docstatus != 0:
-			webnotes.msgprint("Only draft can be submitted", raise_exception=1)
-		self.to_docstatus = 1
-		self.save()
-		self.run_method('on_submit')
-
-	def cancel(self):
-		"""
-			Cancel - set docstatus 2, run "on_cancel"
-		"""
-		if self.doc.docstatus != 1:
-			webnotes.msgprint("Only submitted can be cancelled", raise_exception=1)
-		self.to_docstatus = 2
-		self.prepare_for_save(1)
-		self.save_main()
-		self.save_children()
-		self.run_method('on_cancel')
-
-	def update_after_submit(self):
-		"""
-			Update after submit - some values changed after submit
-		"""
-		if self.doc.docstatus != 1:
-			webnotes.msgprint("Only to called after submit", raise_exception=1)
-		self.to_docstatus = 1
-		self.prepare_for_save(1)
-		self.save_main()
-		self.save_children()
-		self.run_method('on_update_after_submit')
-
-# clone
-
-def clone(source_doclist):
-	""" Copy previous invoice and change dates"""
-	from webnotes.model.doc import Document
-	new_doclist = []
-	new_parent = Document(fielddata = source_doclist.doc.fields.copy())
-	new_parent.name = 'Temp/001'
-	new_parent.fields['__islocal'] = 1
-	new_parent.fields['docstatus'] = 0
-
-	if new_parent.fields.has_key('amended_from'):
-		new_parent.fields['amended_from'] = None
-		new_parent.fields['amendment_date'] = None
-
-	new_parent.save(1)
-
-	new_doclist.append(new_parent)
-
-	for d in source_doclist.doclist[1:]:
-		newd = Document(fielddata = d.fields.copy())
-		newd.name = None
-		newd.fields['__islocal'] = 1
-		newd.fields['docstatus'] = 0
-		newd.parent = new_parent.name
-		new_doclist.append(newd)
+		super(DocList, self).extend(doclist)
+		
+	def _prepare_doc(self, doc):
+		if not doc.name:
+			doc.fields["__islocal"] = 1
+		if doc.parentfield:
+			if not doc.parenttype:
+				doc.parenttype = self[0].doctype
+			if not doc.parent:
+				doc.parent = self[0].name
+
+def load(doctype, name):	
+	# load main doc
+	return objectify(load_doclist(doctype, name))
+
+def load_doclist(doctype, name):
+	doclist = DocList([load_main(doctype, name)])
 	
-	doclistobj = DocList()
-	doclistobj.docs = new_doclist
-	doclistobj.doc = new_doclist[0]
-	doclistobj.doclist = new_doclist
-	doclistobj.children = new_doclist[1:]
-	doclistobj.save()
-	return doclistobj
+	# load children
+	table_fields = map(lambda f: (f.options, name, f.fieldname, doctype),
+		webnotes.conn.get_table_fields(doctype))
 
-
-def trigger(method, doc):
-	"""trigger doctype events"""
-	try:
-		import startup.event_handlers
-	except ImportError:
-		return
+	for args in table_fields:
+		children = load_children(*args)
+		if children: doclist += children
 		
-	if hasattr(startup.event_handlers, method):
-		getattr(startup.event_handlers, method)(doc)
+	return doclist
+
+def load_main(doctype, name):
+	"""retrieves doc from database"""
+	if webnotes.conn.is_single(doctype):
+		doc = webnotes.conn.sql("""select field, value from `tabSingles`
+			where doctype=%s""", doctype, as_list=1)
+		doc = dict(doc)
+		doc["name"] = doctype
+	else:
+		doc  = webnotes.conn.sql("""select * from `tab%s` where name = %s""" % \
+			(doctype, "%s"), name, as_dict=1)
+		if not doc:
+			raise NameError, """%s: "%s" does not exist""" % (doctype, name)
+		doc = doc[0]
+
+	doc["doctype"] = doctype
+	return doc
+
+def load_children(options, parent, parentfield, parenttype):
+	"""load children based on options, parentfield, parenttype and parent"""
+	options = options.split("\n")[0].strip()
 		
-	if hasattr(startup.event_handlers, 'doclist_all'):
-		startup.event_handlers.doclist_all(doc, method)
-
-# for bc
-def getlist(doclist, parentfield):
-	"""
-		Return child records of a particular type
-	"""
-	import webnotes.model.utils
-	return webnotes.model.utils.getlist(doclist, parentfield)
-
-def copy_doclist(doclist, no_copy = []):
-	"""
-		Make a copy of the doclist
-	"""
-	import webnotes.model.utils
-	return webnotes.model.utils.copy_doclist(doclist, no_copy)
-
+	return webnotes.conn.sql("""select *, "%s" as doctype from `tab%s` where parent = %s 
+		and parentfield = %s and parenttype = %s order by idx""" % (options, options, "%s", "%s", "%s"),
+		(parent, parentfield, parenttype), as_dict=1)
+		
+def objectify(doclist):
+	from webnotes.model.doc import Document
+	return map(lambda d: isinstance(d, Document) and d or Document(d), doclist)

webnotes/model/doctype.py

@@ -77,7 +77,8 @@ class _DocType:
 		"""
 			Gets a list of custom field docs masked as type DocField
 		"""
-		custom_doclist = []
+		from webnotes.model.doclist import DocList
+		custom_doclist = DocList()
 		res = webnotes.conn.sql("""SELECT * FROM `tabCustom Field`
 			WHERE dt = %s AND docstatus < 2""", doc_type, as_dict=1)
 		for r in res:

webnotes/model/sync.py

@@ -21,10 +21,10 @@ def sync_all(force=0):
 
 def sync_core_doctypes(force=0):
 	# doctypes
-	return walk_and_sync(os.path.join(os.path.dirname(conf.__file__), 'lib'), force)
+	return walk_and_sync(os.path.join(os.path.dirname(os.path.abspath(conf.__file__)), 'lib'), force)
 
 def sync_modules(force=0):
-	return walk_and_sync(os.path.join(os.path.dirname(conf.__file__), 'app'), force)
+	return walk_and_sync(os.path.join(os.path.dirname(os.path.abspath(conf.__file__)), 'app'), force)
 
 def walk_and_sync(start_path, force=0):
 	"""walk and sync all doctypes and pages"""
@@ -32,8 +32,9 @@ def walk_and_sync(start_path, force=0):
 
 	modules = []
 
+	document_type = ['page', 'workflow', 'module_def', 'workflow_state', 'workflow_action']
 	for path, folders, files in os.walk(start_path):
-		if os.path.basename(os.path.dirname(path)) in ('doctype', 'page'):
+		if os.path.basename(os.path.dirname(path)) in (['doctype'] + document_type):
 			for f in files:				
 				if f.endswith(".txt"):
 					# great grand-parent folder is module_name
@@ -49,7 +50,7 @@ def walk_and_sync(start_path, force=0):
 				
 					if doctype == 'doctype':
 						sync(module_name, name, force)
-					elif doctype in ['page']:#, 'search_criteria', 'Print Format', 'DocType Mapper']:
+					elif doctype in document_type:
 						if reload_doc(module_name, doctype, name):
 							print module_name + ' | ' + doctype + ' | ' + name
 					
@@ -60,14 +61,18 @@ def walk_and_sync(start_path, force=0):
 def sync(module_name, docname, force=0):
 	"""sync doctype from file if modified"""
 	with open(get_file_path(module_name, docname), 'r') as f:
-		from webnotes.model.utils import peval_doclist
+		from webnotes.modules.utils import peval_doclist
 		doclist = peval_doclist(f.read())
 		modified = doclist[0]['modified']
 		if not doclist:
-			raise Exception('DocList could not be evaluated')
-		if modified == str(webnotes.conn.get_value(doclist[0].get('doctype'), 
-			doclist[0].get('name'), 'modified')) and not force:
+			raise Exception('ModelWrapper could not be evaluated')
+
+		db_modified = str(webnotes.conn.get_value(doclist[0].get('doctype'),
+			doclist[0].get('name'), 'modified'))
+			
+		if modified == db_modified and not force:
 			return
+
 		webnotes.conn.begin()
 		
 		delete_doctype_docfields(doclist)
@@ -162,47 +167,3 @@ def create_doc(data):
 	d.fields.update(data)
 	d.save()
 	print 'Created %(doctype)s %(name)s' % d.fields
-
-import unittest
-class TestSync(unittest.TestCase):
-	def setUp(self):
-		self.test_case = {
-			'module_name': 'selling',
-			'docname': 'sales_order'
-		}
-		webnotes.conn.begin()
-
-	def tearDown(self):
-		pass
-		#from webnotes.utils import cstr
-		#webnotes.conn.rollback()
-
-	def test_sync(self):
-		#old_doctype, old_docfields = self.query('Profile')
-		#self.parent = sync(self.test_case.get('module_name'), self.test_case.get('docname'))
-		#new_doctype, new_docfields = self.query(self.parent)
-		#self.assertNotEqual(old_doctype, new_doctype)
-		#self.assertNotEqual(old_docfields, new_docfields)
-
-		#from webnotes.utils import cstr
-		#print new_doctype[0]
-		#print
-		#print "\n".join((cstr(d) for d in new_docfields))
-		#print "\n\n"
-		pass
-
-	def test_sync_all(self):
-		sync_all()
-
-	def query(self, parent):
-		doctype = webnotes.conn.sql("SELECT name FROM `tabDocType` \
-			WHERE name=%s", parent)
-		docfields = webnotes.conn.sql("SELECT idx, fieldname, label, fieldtype FROM `tabDocField` \
-			WHERE parent=%s order by idx", parent)
-		#doctype = webnotes.conn.sql("SELECT * FROM `tabDocType` \
-		#	WHERE name=%s", parent)
-		#docfields = webnotes.conn.sql("SELECT * FROM `tabDocField` \
-		#	WHERE parent=%s order by idx", parent)
-		return doctype, docfields
-
-

webnotes/model/wrapper.py

@@ -0,0 +1,344 @@
+# Copyright (c) 2012 Web Notes Technologies Pvt Ltd (http://erpnext.com)
+# 
+# MIT License (MIT)
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a 
+# copy of this software and associated documentation files (the "Software"), 
+# to deal in the Software without restriction, including without limitation 
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+# and/or sell copies of the Software, and to permit persons to whom the 
+# Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in 
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 
+# PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF 
+# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE 
+# OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# 
+
+from __future__ import unicode_literals
+"""
+Transactions are defined as collection of classes, a ModelWrapper represents collection of Document
+objects for a transaction with main and children.
+
+Group actions like save, etc are performed on doclists
+"""
+
+import webnotes
+from webnotes.utils import cint
+from webnotes.model.doc import Document
+
+class ModelWrapper:
+	"""
+	Collection of Documents with one parent and multiple children
+	"""
+	def __init__(self, dt=None, dn=None):
+		self.docs = []
+		self.obj = None
+		self.to_docstatus = 0
+		if dt and dn:
+			self.load_from_db(dt, dn)
+		if type(dt) is list:
+			self.set_doclist(dt)
+
+	def load_from_db(self, dt=None, dn=None, prefix='tab'):
+		"""
+			Load doclist from dt
+		"""
+		from webnotes.model.doc import Document, getchildren
+		from webnotes.model.doclist import DocList
+		if not dt: dt = self.doc.doctype
+		if not dn: dn = self.doc.name
+
+		doc = Document(dt, dn, prefix=prefix)
+
+		# get all children types
+		tablefields = webnotes.model.meta.get_table_fields(dt)
+
+		# load chilren
+		doclist = DocList([doc,])
+		for t in tablefields:
+			doclist += getchildren(doc.name, t[0], t[1], dt, prefix=prefix)
+
+		self.set_doclist(doclist)
+
+	def __iter__(self):
+		"""
+			Make this iterable
+		"""
+		return self.docs.__iter__()
+
+	def from_compressed(self, data, docname):
+		"""
+			Expand called from client
+		"""
+		from webnotes.model.utils import expand
+		self.docs = expand(data)
+		self.set_doclist(self.docs)
+		
+	def set_doclist(self, docs):
+		for i, d in enumerate(docs):
+			if isinstance(d, dict):
+				docs[i] = Document(fielddata=d)
+		
+		self.docs = self.doclist = docs
+		self.doc, self.children = docs[0], docs[1:]
+		if self.obj:
+			self.obj.doclist = self.doclist
+			self.obj.doc = self.doc
+
+	def make_obj(self):
+		"""
+			Create a DocType object
+		"""
+		if self.obj: return self.obj
+
+		from webnotes.model.code import get_obj
+		self.obj = get_obj(doc=self.doc, doclist=self.children)
+		return self.obj
+
+	def to_dict(self):
+		"""
+			return as a list of dictionaries
+		"""
+		return [d.fields for d in self.docs]
+
+	def check_if_latest(self):
+		"""
+			Raises exception if the modified time is not the same as in the database
+		"""
+		from webnotes.model.meta import is_single
+
+		if (not is_single(self.doc.doctype)) and (not cint(self.doc.fields.get('__islocal'))):
+			tmp = webnotes.conn.sql("""
+				SELECT modified FROM `tab%s` WHERE name="%s" for update"""
+				% (self.doc.doctype, self.doc.name))
+
+			if tmp and str(tmp[0][0]) != str(self.doc.modified):
+				webnotes.msgprint("""
+				Document has been modified after you have opened it.
+				To maintain the integrity of the data, you will not be able to save your changes.
+				Please refresh this document. [%s/%s]""" % (tmp[0][0], self.doc.modified), raise_exception=1)
+
+	def check_permission(self):
+		"""
+			Raises exception if permission is not valid
+		"""
+		if not self.doc.check_perm(verbose=1):
+			webnotes.msgprint("Not enough permission to save %s" % self.doc.doctype, raise_exception=1)
+
+	def check_links(self):
+		"""
+			Checks integrity of links (throws exception if links are invalid)
+		"""
+		ref, err_list = {}, []
+		for d in self.docs:
+			if not ref.get(d.doctype):
+				ref[d.doctype] = d.make_link_list()
+
+			err_list += d.validate_links(ref[d.doctype])
+
+		if err_list:
+			webnotes.msgprint("""[Link Validation] Could not find the following values: %s.
+			Please correct and resave. Document Not Saved.""" % ', '.join(err_list), raise_exception=1)
+
+	def update_timestamps_and_docstatus(self):
+		"""
+			Update owner, creation, modified_by, modified, docstatus
+		"""
+		from webnotes.utils import now
+		ts = now()
+		user = webnotes.__dict__.get('session', {}).get('user') or 'Administrator'
+
+		for d in self.docs:
+			if self.doc.fields.get('__islocal'):
+				d.owner = user
+				d.creation = ts
+
+			d.modified_by = user
+			d.modified = ts
+			if d.docstatus != 2: # don't update deleted
+				d.docstatus = self.to_docstatus
+
+	def prepare_for_save(self, check_links):
+		"""
+			Set owner, modified etc before saving
+		"""
+		self.check_if_latest()
+		self.check_permission()
+		if check_links:
+			self.check_links()
+		self.update_timestamps_and_docstatus()
+
+	def run_method(self, method):
+		"""
+		Run a method and custom_method
+		"""
+		self.make_obj()
+		if hasattr(self.obj, method):
+			getattr(self.obj, method)()
+		if hasattr(self.obj, 'custom_' + method):
+			getattr(self.obj, 'custom_' + method)()
+
+		trigger(method, self.obj.doc)
+		
+		self.set_doclist([self.obj.doc] + self.obj.doclist)
+
+	def save_main(self):
+		"""
+			Save the main doc
+		"""
+		try:
+			self.doc.save(cint(self.doc.fields.get('__islocal')))
+		except NameError, e:
+			webnotes.msgprint('%s "%s" already exists' % (self.doc.doctype, self.doc.name))
+
+			# prompt if cancelled
+			if webnotes.conn.get_value(self.doc.doctype, self.doc.name, 'docstatus')==2:
+				webnotes.msgprint('[%s "%s" has been cancelled]' % (self.doc.doctype, self.doc.name))
+			webnotes.errprint(webnotes.utils.getTraceback())
+			raise e
+
+	def save_children(self):
+		"""
+			Save Children, with the new parent name
+		"""
+		child_map = {}
+		for d in self.children:
+			if (d.fields.has_key('parent') and d.fields.get('parent')) or \
+					(d.fields.has_key("parentfield") and d.fields.get("parentfield")):
+				# if d.parent:
+				d.parent = self.doc.name # rename if reqd
+				d.parenttype = self.doc.doctype
+				
+				d.save(new = cint(d.fields.get('__islocal')))
+			
+			child_map.setdefault(d.doctype, []).append(d.name)
+		
+		# delete all children in database that are not in the child_map
+		
+		# get all children types
+		tablefields = webnotes.model.meta.get_table_fields(self.doc.doctype)
+				
+		for dt in tablefields:
+			cnames = child_map.get(dt[0]) or []
+			if cnames:
+				webnotes.conn.sql("""delete from `tab%s` where parent=%s and parenttype=%s and
+					name not in (%s)""" % (dt[0], '%s', '%s', ','.join(['%s'] * len(cnames))), 
+						tuple([self.doc.name, self.doc.doctype] + cnames))
+			else:
+				webnotes.conn.sql("""delete from `tab%s` where parent=%s and parenttype=%s""" \
+					% (dt[0], '%s', '%s'), (self.doc.name, self.doc.doctype))
+
+	def save(self, check_links=1):
+		"""
+			Save the list
+		"""
+		self.prepare_for_save(check_links)
+		self.run_method('validate')
+		self.save_main()
+		self.save_children()
+		self.run_method('on_update')
+
+	def submit(self):
+		"""
+			Save & Submit - set docstatus = 1, run "on_submit"
+		"""
+		if self.doc.docstatus != 0:
+			webnotes.msgprint("Only draft can be submitted", raise_exception=1)
+		self.to_docstatus = 1
+		self.save()
+		self.run_method('on_submit')
+
+	def cancel(self):
+		"""
+			Cancel - set docstatus 2, run "on_cancel"
+		"""
+		if self.doc.docstatus != 1:
+			webnotes.msgprint("Only submitted can be cancelled", raise_exception=1)
+		self.to_docstatus = 2
+		self.prepare_for_save(1)
+		self.save_main()
+		self.save_children()
+		self.run_method('on_cancel')
+
+	def update_after_submit(self):
+		"""
+			Update after submit - some values changed after submit
+		"""
+		if self.doc.docstatus != 1:
+			webnotes.msgprint("Only to called after submit", raise_exception=1)
+		self.to_docstatus = 1
+		self.prepare_for_save(1)
+		self.save_main()
+		self.save_children()
+		self.run_method('on_update_after_submit')
+
+# clone
+
+def clone(source_doclist):
+	""" Copy previous invoice and change dates"""
+	from webnotes.model.doc import Document
+	new_doclist = []
+	new_parent = Document(fielddata = source_doclist.doc.fields.copy())
+	new_parent.name = 'Temp/001'
+	new_parent.fields['__islocal'] = 1
+	new_parent.fields['docstatus'] = 0
+
+	if new_parent.fields.has_key('amended_from'):
+		new_parent.fields['amended_from'] = None
+		new_parent.fields['amendment_date'] = None
+
+	new_parent.save(1)
+
+	new_doclist.append(new_parent)
+
+	for d in source_doclist.doclist[1:]:
+		newd = Document(fielddata = d.fields.copy())
+		newd.name = None
+		newd.fields['__islocal'] = 1
+		newd.fields['docstatus'] = 0
+		newd.parent = new_parent.name
+		new_doclist.append(newd)
+	
+	doclistobj = ModelWrapper()
+	doclistobj.docs = new_doclist
+	doclistobj.doc = new_doclist[0]
+	doclistobj.doclist = new_doclist
+	doclistobj.children = new_doclist[1:]
+	doclistobj.save()
+	return doclistobj
+
+
+def trigger(method, doc):
+	"""trigger doctype events"""
+	try:
+		import startup.event_handlers
+	except ImportError:
+		return
+		
+	if hasattr(startup.event_handlers, method):
+		getattr(startup.event_handlers, method)(doc)
+		
+	if hasattr(startup.event_handlers, 'doclist_all'):
+		startup.event_handlers.doclist_all(doc, method)
+
+# for bc
+def getlist(doclist, parentfield):
+	"""
+		Return child records of a particular type
+	"""
+	import webnotes.model.utils
+	return webnotes.model.utils.getlist(doclist, parentfield)
+
+def copy_doclist(doclist, no_copy = []):
+	"""
+		Make a copy of the doclist
+	"""
+	import webnotes.model.utils
+	return webnotes.model.utils.copy_doclist(doclist, no_copy)
+

webnotes/modules/__init__.py

@@ -24,9 +24,23 @@ from __future__ import unicode_literals
 """
 	Utilities for using modules
 """
-import webnotes
+import webnotes, os, conf
 
-transfer_types = ['Role', 'Print Format','DocType','Page','DocType Mapper','GL Mapper','Search Criteria', 'Patch', 'Report']
+transfer_types = ['Role', 'Print Format','DocType','Page','DocType Mapper',
+	'GL Mapper','Search Criteria', 'Patch', 'Report']
+	
+lower_case_files_for = ['DocType', 'Page', 'Search Criteria', 'Report', 
+	"Workflow", 'Module Def', 'Desktop Item', 'Workflow State', 'Workflow Action']
+
+code_fields_dict = {
+	'Page':[('script', 'js'), ('content', 'html'), ('style', 'css'), ('static_content', 'html'), ('server_code', 'py')],
+	'DocType':[('server_code_core', 'py'), ('client_script_core', 'js')],
+	'Search Criteria':[('report_script', 'js'), ('server_script', 'py'), ('custom_query', 'sql')],
+	'Patch':[('patch_code', 'py')],
+	'Stylesheet':['stylesheet', 'css'],
+	'Page Template':['template', 'html'],
+	'Control Panel':[('startup_code', 'js'), ('startup_css', 'css')]
+}
 
 def scrub(txt):
 	return txt.replace(' ','_').replace('-', '_').replace('/', '_').lower()
@@ -34,14 +48,13 @@ def scrub(txt):
 def scrub_dt_dn(dt, dn):
 	"""Returns in lowercase and code friendly names of doctype and name for certain types"""
 	ndt, ndn = dt, dn
-	if dt.lower() in ('doctype', 'search criteria', 'page', 'report'):
+	if dt in lower_case_files_for:
 		ndt, ndn = scrub(dt), scrub(dn)
 
 	return ndt, ndn
 			
 def get_module_path(module):
 	"""Returns path of the given module"""
-	import os, conf
 	m = scrub(module)
 	
 	app_path = os.path.dirname(conf.__file__)
@@ -50,56 +63,18 @@ def get_module_path(module):
 		return os.path.join(app_path, 'lib', 'core')
 	else:
 		return os.path.join(app_path, 'app', m)
-	
-
-def reload_doc(module, dt=None, dn=None):
-	"""reload single / list of records"""
-
-	if type(module) is list:
-		for m in module:
-			reload_single_doc(m[0], m[1], m[2])
-	else:
-		reload_single_doc(module, dt, dn)
-		
-def reload_single_doc(module, dt, dn, force=False):
-	"""Sync a file from txt if modifed, return false if not updated"""
-	if dt.lower() == 'doctype':
-		return
-		
-	import os
-	dt, dn = scrub_dt_dn(dt, dn)
-
-	path = os.path.join(get_module_path(module), 
-		os.path.join(dt, dn, dn + '.txt'))
-		
-	if os.path.exists(path):
-		from webnotes.model.utils import peval_doclist
-		
-		with open(path, 'r') as f:
-			doclist = peval_doclist(f.read())
-			
-		if doclist:
-			doc = doclist[0]
-			if not force:
-				# check if timestamps match
-				if doc['modified']== str(webnotes.conn.get_value(doc['doctype'], doc['name'], 'modified')):
-					return False
-			
-			from webnotes.utils.transfer import set_doc
-			set_doc(doclist, 1, 1, 1)
 
-			# since there is a new timestamp on the file, update timestamp in
-			webnotes.conn.sql("update `tab%s` set modified=%s where name=%s" % \
-				(doc['doctype'], '%s', '%s'), 
-				(doc['modified'],doc['name']))
-			return True
-	else:
-		raise Exception, '%s missing' % path
+def get_doc_path(module, doctype, name):
+	dt, dn = scrub_dt_dn(doctype, name)
+	return os.path.join(get_module_path(module), dt, dn)
 
+def reload_doc(module, dt=None, dn=None):
+	from webnotes.modules.import_file import import_files
+	return import_files(module, dt, dn)
 
 def export_doc(doctype, name, module=None):
 	"""write out a doc"""
-	from webnotes.modules.export_module import write_document_file
+	from webnotes.modules.export_file import write_document_file
 	import webnotes.model.doc
 	if not module: module = webnotes.conn.get_value(doctype, name, 'module')
 	doclist = [d.fields for d in webnotes.model.doc.get(doctype, name)]

webnotes/modules/export_module.py → webnotes/modules/export_file.py

@@ -21,113 +21,111 @@
 # 
 
 from __future__ import unicode_literals
-"""
-	Export files to modules
-"""
 
-from webnotes.modules import scrub, get_module_path
+import webnotes, os
+import webnotes.model.doc
+from webnotes.modules import scrub, get_module_path, lower_case_files_for, \
+	code_fields_dict, scrub_dt_dn
+
+def export_doc(doc):
+	export_to_files([[doc.doctype, doc.name]])
+
 def export_to_files(record_list=[], record_module=None, verbose=0):
 	"""
 		Export record_list to files. record_list is a list of lists ([doctype],[docname] )  ,
 	"""
-	import webnotes.model.doc
+	from webnotes.modules.import_merge import in_transfer
+	if in_transfer:
+		return
+			
 	module_doclist =[]
 	if record_list:
 		for record in record_list:
-			doclist = [d.fields for d in webnotes.model.doc.get(record[0], record[1])]
-			write_document_file(doclist, record_module)
+			write_document_file(webnotes.model.doc.get(record[0], record[1]), 
+				record_module)
+	
+def write_document_file(doclist, record_module=None):
+	from webnotes.modules.utils import pprint_doclist
 
-def create_init_py(module_path, dt, dn):
-	"""
-		Creates __init__.py in the module directory structure
-	"""
-	import os
+	doclist = [filter_fields(d.fields) for d in doclist]
 
-	def create_if_not_exists(path):
-		initpy = os.path.join(path, '__init__.py')
-		if not os.path.exists(initpy):
-			open(initpy, 'w').close()
-	
-	create_if_not_exists(os.path.join(module_path))
-	create_if_not_exists(os.path.join(module_path, dt))
-	create_if_not_exists(os.path.join(module_path, dt, dn))
-	
-def create_folder(module, dt, dn):
-	"""
-		Creates directories for module and their __init__.py
-	"""
-	import webnotes, os
-	
-	# get module path by importing the module
-	module_path = get_module_path(module)
-			
-	code_type = dt in ['DocType', 'Page', 'Search Criteria', 'Report']
+	module = record_module or get_module_name(doclist)
+	code_type = doclist[0]['doctype'] in lower_case_files_for
 	
 	# create folder
-	folder = os.path.join(module_path, code_type and scrub(dt) or dt, code_type and scrub(dn) or dn)
+	folder = create_folder(module, doclist[0]['doctype'], doclist[0]['name'], code_type)
 	
-	webnotes.create_folder(folder)
+	# separate code files
+	clear_code_fields(doclist, folder, code_type)
+		
+	# write the data file	
+	fname = (code_type and scrub(doclist[0]['name'])) or doclist[0]['name']
+	with open(os.path.join(folder, fname +'.txt'),'w+') as txtfile:
+		txtfile.write(pprint_doclist(doclist))
+
+def filter_fields(doc):
+	from webnotes.model.doctype import get
+	from webnotes.model import default_fields
+
+	doctypelist = get(doc.doctype, False)
+	valid_fields = [d.fieldname for d in doctypelist.get({"parent":doc.doctype,
+		"doctype":"DocField"})]
+	to_remove = []
 	
-	# create init_py_files
-	if code_type:
-		create_init_py(module_path, scrub(dt), scrub(dn))
+	for key in doc:
+		if (not key in default_fields) and (not key in valid_fields):
+			to_remove.append(key)
+		elif doc[key]==None:
+			to_remove.append(key)
+			
+	for key in to_remove:
+		del doc[key]
 	
-	return folder
+	return doc
 
-def get_module_name(doclist, record_module=None):
-	"""
-		Returns the module-name of a doclist
-	"""
-	# module name
+def get_module_name(doclist):
 	if doclist[0]['doctype'] == 'Module Def':
 		module = doclist[0]['name']
 	elif doclist[0]['doctype']=='Control Panel':
 		module = 'Core'
-	elif record_module:
-		module = record_module
+	elif doclist[0]['doctype']=="Workflow":
+		module = webnotes.conn.get_value("DocType", doclist[0]["document_type"], "module")
 	else:
 		module = doclist[0]['module']
 
 	return module
-	
-def write_document_file(doclist, record_module=None):
-	"""
-		Write a doclist to file, can optionally specify module name
-	"""
-	import os
-	from webnotes.model.utils import pprint_doclist
-
-	module = get_module_name(doclist, record_module)
+		
+def create_folder(module, dt, dn, code_type):
+	# get module path by importing the module
+	module_path = get_module_path(module)
 
-	# create the folder
-	code_type = doclist[0]['doctype'] in ['DocType','Page','Search Criteria', 'Report']
+	dt, dn = scrub_dt_dn(dt, dn)
 	
 	# create folder
-	folder = create_folder(module, doclist[0]['doctype'], doclist[0]['name'])
+	folder = os.path.join(module_path, dt, dn)
 	
-	# separate code files
-	clear_code_fields(doclist, folder, code_type)
-		
-	# write the data file	
-	fname = (code_type and scrub(doclist[0]['name'])) or doclist[0]['name']
-	txtfile = open(os.path.join(folder, fname +'.txt'),'w+')
-	txtfile.write(pprint_doclist(doclist))
-	#dict_list = [pprint_dict(d) for d in doclist]	
-	#txtfile.write('[\n' + ',\n'.join(dict_list) + '\n]')
-	txtfile.close()
-
-def clear_code_fields(doclist, folder, code_type):
-	"""
-		Removes code from the doc
-	"""
+	webnotes.create_folder(folder)
 	
-	import os
-	import webnotes
-	# code will be in the parent only
-	code_fields = webnotes.code_fields_dict.get(doclist[0]['doctype'], [])
+	# create init_py_files
+	if code_type:
+		create_init_py(module_path, dt, dn)
+	
+	return folder
+
+def create_init_py(module_path, dt, dn):
+	def create_if_not_exists(path):
+		initpy = os.path.join(path, '__init__.py')
+		if not os.path.exists(initpy):
+			open(initpy, 'w').close()
 	
+	create_if_not_exists(os.path.join(module_path))
+	create_if_not_exists(os.path.join(module_path, dt))
+	create_if_not_exists(os.path.join(module_path, dt, dn))
+
+
+def clear_code_fields(doclist, folder, code_type):
+	code_fields = code_fields_dict.get(doclist[0]['doctype'], [])
 	for code_field in code_fields:
 		if doclist[0].get(code_field[0]):
-
 			doclist[0][code_field[0]] = None
 

webnotes/modules/import_file.py

@@ -0,0 +1,71 @@
+# Copyright (c) 2012 Web Notes Technologies Pvt Ltd (http://erpnext.com)
+# 
+# MIT License (MIT)
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a 
+# copy of this software and associated documentation files (the "Software"), 
+# to deal in the Software without restriction, including without limitation 
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+# and/or sell copies of the Software, and to permit persons to whom the 
+# Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in 
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 
+# PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF 
+# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE 
+# OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# 
+
+from __future__ import unicode_literals
+
+import webnotes, os
+from webnotes.modules import scrub, get_module_path, scrub_dt_dn
+
+def import_files(module, dt=None, dn=None):
+	if type(module) is list:
+		for m in module:
+			import_file(m[0], m[1], m[2])
+	else:
+		import_file(module, dt, dn)
+		
+def import_file(module, dt, dn, force=False):
+	"""Sync a file from txt if modifed, return false if not updated"""
+	if dt.lower() == 'doctype':
+		return
+		
+	dt, dn = scrub_dt_dn(dt, dn)
+
+	path = os.path.join(get_module_path(module), 
+		os.path.join(dt, dn, dn + '.txt'))
+		
+	import_file_by_path(path, force)
+
+def import_file_by_path(path, force=False):
+	if os.path.exists(path):
+		from webnotes.modules.utils import peval_doclist
+		
+		with open(path, 'r') as f:
+			doclist = peval_doclist(f.read())
+			
+		if doclist:
+			doc = doclist[0]
+			if not force:
+				# check if timestamps match
+				if doc['modified']== str(webnotes.conn.get_value(doc['doctype'], doc['name'], 'modified')):
+					return False
+			
+			from webnotes.modules.import_merge import set_doc
+			set_doc(doclist, 1, 1, 1)
+
+			# since there is a new timestamp on the file, update timestamp in
+			webnotes.conn.sql("update `tab%s` set modified=%s where name=%s" % \
+				(doc['doctype'], '%s', '%s'), 
+				(doc['modified'],doc['name']))
+			return True
+	else:
+		raise Exception, '%s missing' % path
+

webnotes/utils/transfer.py → webnotes/modules/import_merge.py

@@ -121,7 +121,7 @@ class UpdateDocument:
 		
 	def save(self):
 		# parent
-		self.doc.save(new = 1, ignore_fields = 1, check_links=0)
+		self.doc.save(new = 1, check_links=0)
 		self.doclist = [self.doc]
 		self.save_children()
 			
@@ -131,12 +131,12 @@ class UpdateDocument:
 			
 	def save_one_doc(self, df, as_new=1):
 		d = Document(fielddata = df)
-		d.save(new = as_new, ignore_fields = 1, check_links=0)
+		d.save(new = as_new, check_links=0)
 		self.doclist.append(d)
 
 	def run_on_update(self):
-		from webnotes.model.code import get_server_obj
-		so = get_server_obj(self.doc, self.doclist)
+		from webnotes.model.controller import get_obj
+		so = get_obj(doc=self.doc, doclist=self.doclist)
 		if hasattr(so, 'on_update'):
 			so.on_update()
 
@@ -165,7 +165,7 @@ class UpdateDocumentMerge(UpdateDocument):
 		if self.exists:
 			# save main doc
 			self.keep_values(self.doc)
-			self.doc.save(ignore_fields = 1, check_links=0)
+			self.doc.save(check_links=0)
 			self.doclist.append(self.doc)
 			self.save_children()
 			self.on_save()
@@ -186,10 +186,10 @@ class UpdateDocumentMerge(UpdateDocument):
 					# is it new?
 					if self.child_exists(d):
 						self.keep_values(d)
-						d.save(ignore_fields = 1, check_links=0)
+						d.save(check_links=0)
 						self.log.append('updated %s, %s' % (d.doctype, d.name))
 					else:
-						d.save(1, ignore_fields = 1, check_links=0)
+						d.save(1, check_links=0)
 						self.log.append('new %s' % d.doctype)
 			self.doclist.append(d)
 
@@ -272,7 +272,7 @@ class UpdateDocType(UpdateDocumentMerge):
 					tmp = Document(fielddata = d)
 					tmp.fieldname = ''
 					tmp.name = None
-					tmp.save(1, ignore_fields = 1, check_links=0)
+					tmp.save(1, check_links=0)
 				else:
 					webnotes.conn.sql("update tabDocField set idx=%s where %s=%s and parent=%s" % \
 						('%s', d.get('fieldname') and 'fieldname' or 'label', '%s', '%s'), (d.get('idx'), d.get('fieldname') or d.get('label'), self.doc.name))
@@ -294,8 +294,8 @@ class UpdateDocType(UpdateDocumentMerge):
 					('%s', e[0] and 'fieldname' or 'label', '%s', '%s'), (idx+1, e[0] or e[1], self.doc.name))
 
 	def run_on_update(self):
-		from webnotes.model.code import get_server_obj
-		so = get_server_obj(self.doc, self.doclist)
+		from webnotes.model.controller import get_obj
+		so = get_obj(doc=self.doc, doclist=self.doclist)
 		if hasattr(so, 'on_update'):
 			so.on_update()
 

webnotes/modules/utils.py

@@ -58,3 +58,119 @@ def switch_module(dt, dn, to, frm=None, export=None):
 
 			for ext in ('py','js','html','css'):
 				os.system('cp %s %s')
+
+def commonify_doclist(doclist, with_comments=1):
+	"""
+		Makes a doclist more readable by extracting common properties.
+		This is used for printing Documents in files
+	"""
+	from webnotes.utils import get_common_dict, get_diff_dict
+
+	def make_common(doclist):
+		c = {}
+		if with_comments:
+			c['##comment'] = 'These values are common in all dictionaries'
+		for k in common_keys:
+			c[k] = doclist[0][k]
+		return c
+
+	def strip_common_and_idx(d):
+		for k in common_keys:
+			if k in d: del d[k]
+			
+		if 'idx' in d: del d['idx']
+		return d
+
+	def make_common_dicts(doclist):
+
+		common_dict = {} # one per doctype
+
+		# make common dicts for all records
+		for d in doclist:
+			if not d['doctype'] in common_dict:
+				d1 = d.copy()
+				del d1['name']
+				common_dict[d['doctype']] = d1
+			else:
+				common_dict[d['doctype']] = get_common_dict(common_dict[d['doctype']], d)
+		return common_dict
+
+	common_keys = ['owner','docstatus','creation','modified','modified_by']
+	common_dict = make_common_dicts(doclist)
+
+	# make docs
+	final = []
+	for d in doclist:
+		f = strip_common_and_idx(get_diff_dict(common_dict[d['doctype']], d))
+		f['doctype'] = d['doctype'] # keep doctype!
+
+		# strip name for child records (only an auto generated number!)
+		if f['doctype'] != doclist[0]['doctype']:
+			del f['name']
+
+		if with_comments:
+			f['##comment'] = d['doctype'] + ('name' in f and (', ' + f['name']) or '')
+		final.append(f)
+
+	# add commons
+	commons = []
+	for d in common_dict.values():
+		d['name']='__common__'
+		if with_comments:
+			d['##comment'] = 'These values are common for all ' + d['doctype']
+		commons.append(strip_common_and_idx(d))
+
+	common_values = make_common(doclist)
+	return [common_values]+commons+final
+
+def uncommonify_doclist(dl):
+	"""
+		Expands an commonified doclist
+	"""
+	# first one has common values
+	common_values = dl[0]
+	common_dict = {}
+	final = []
+	idx_dict = {}
+
+	for d in dl[1:]:
+		if 'name' in d and d['name']=='__common__':
+			# common for a doctype - 
+			del d['name']
+			common_dict[d['doctype']] = d
+		else:
+			dt = d['doctype']
+			if not dt in idx_dict: idx_dict[dt] = 1;
+			d1 = common_values.copy()
+
+			# update from common and global
+			d1.update(common_dict[dt])
+			d1.update(d)
+
+			# idx by sequence
+			d1['idx'] = idx_dict[dt]
+			
+			# increment idx
+			idx_dict[dt] += 1
+
+			final.append(d1)
+
+	return final
+
+def pprint_doclist(doclist, with_comments = 1):
+	"""
+		Pretty Prints a doclist with common keys separated and comments
+	"""
+	from json import dumps
+
+	return dumps(commonify_doclist(doclist, False), indent=1)
+
+def peval_doclist(txt):
+	"""
+		Restore a pretty printed doclist
+	"""
+	from json import loads
+	if txt.startswith("#"):
+		return uncommonify_doclist(eval(txt))
+	else:
+		return uncommonify_doclist(loads(txt))

webnotes/utils/nestedset.py

@@ -33,7 +33,7 @@ from __future__ import unicode_literals
 
 import webnotes, unittest
 from webnotes import msgprint
-from webnotes.model.doclist import DocList
+from webnotes.model.wrapper import ModelWrapper
 from webnotes.model.doc import Document
 
 class TestNSM(unittest.TestCase):
@@ -53,7 +53,7 @@ class TestNSM(unittest.TestCase):
 		]
 		
 		for d in self.data:
-			self.__dict__[d[0]] = DocList([Document(fielddata = {
+			self.__dict__[d[0]] = ModelWrapper([Document(fielddata = {
 				"doctype": "Item Group", "item_group_name": d[0], "parent_item_group": d[1],
 				"__islocal": 1
 			})])

webnotes/widgets/form/run_method.py

@@ -29,7 +29,7 @@ def runserverobj():
 		Run server objects
 	"""
 	import webnotes.model.code
-	from webnotes.model.doclist import DocList
+	from webnotes.model.wrapper import ModelWrapper
 	from webnotes.utils import cint
 
 	doclist = None
@@ -43,7 +43,7 @@ def runserverobj():
 		so = webnotes.model.code.get_obj(dt, dn)
 
 	else:
-		doclist = DocList()
+		doclist = ModelWrapper()
 		doclist.from_compressed(webnotes.form_dict.get('docs'), dn)
 		so = doclist.make_obj()
 		doclist.check_if_latest()

webnotes/widgets/form/save.py

@@ -27,10 +27,10 @@ import webnotes
 def savedocs():
 	"""save / submit / cancel / update doclist"""
 	try:
-		from webnotes.model.doclist import DocList
+		from webnotes.model.wrapper import ModelWrapper
 		form = webnotes.form_dict
 
-		doclist = DocList()
+		doclist = ModelWrapper()
 		doclist.from_compressed(form.get('docs'), form.get('docname'))
 
 		# action

webnotes/widgets/query_builder.py

@@ -131,7 +131,7 @@ def exec_report(code, res, colnames=[], colwidths=[], coltypes=[], coloptions=[]
 	from webnotes import *
 	from webnotes.utils import *
 	from webnotes.model.doc import *
-	from webnotes.model.doclist import getlist
+	from webnotes.model.wrapper import getlist
 	from webnotes.model.db_schema import updatedb
 	from webnotes.model.code import get_obj