frappe/py/webnotes/modules/diff.py

# Copyright (c) 2012 Web Notes Technologies Pvt Ltd (http://erpnext.com)
# 
# MIT License (MIT)
# 
# Permission is hereby granted, free of charge, to any person obtaining a 
# copy of this software and associated documentation files (the "Software"), 
# to deal in the Software without restriction, including without limitation 
# the rights to use, copy, modify, merge, publish, distribute, sublicense, 
# and/or sell copies of the Software, and to permit persons to whom the 
# Software is furnished to do so, subject to the following conditions:
# 
# The above copyright notice and this permission notice shall be included in 
# all copies or substantial portions of the Software.
# 
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 
# PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF 
# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE 
# OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# 

"""get diff bettween txt files and database records"""

import webnotes
import os, conf
from webnotes.model.utils import peval_doclist

dt_map = {
	'DocType': {
		'DocField': ['fieldname', 'label']
	},
	'Search Criteria': {},
	'Page': {}
}

ignore_fields = ('modified', 'creation', 'owner', 'modified_by', 
	'_last_update', 'version', 'idx', 'name')

missing = property_diff = 0
property_count = {}


def diff_ref_file():
	"""get diff using .txt files as reference"""
	global missing, property_diff, property_count
	missing = property_diff = 0
	property_count = {}

	get_diff(conf.modules_path)
	get_diff(os.path.join(os.getcwd(), 'lib', 'py', 'core'))
	
	print_stats()


def get_diff(path):
	for wt in os.walk(path):
		for fname in wt[2]:
			if fname.endswith('.txt'):
				path = os.path.join(wt[0], fname)
				with open(path, 'r') as txtfile:
					doclist_diff(peval_doclist(txtfile.read()))

	
def diff_ref_db():
	"""get diff using database as reference"""
	from webnotes.modules import scrub
	for dt in dt_map:
		# get all main docs
		for doc in webnotes.conn.sql("""select * from `tab%s`""" % dt, as_dict=1):
			# get file for this doc
			doc['doctype'] = dt
			#print doc['name'], doc['doctype'], doc['module']
			path = os.path.join(conf.modules_path, scrub(doc['module']), \
				scrub(dt), scrub(doc['name']), scrub(doc['name']) + '.txt')	
			path_core = os.path.join(os.getcwd(), 'lib', 'py', 'core', \
				scrub(dt), scrub(doc['name']), scrub(doc['name']) + '.txt')

			if os.path.exists(path):
				with open(path, 'r') as txtfile:
					target = peval_doclist(txtfile.read())					
			elif os.path.exists(path_core):
				with open(path_core, 'r') as txtfile:
					target = peval_doclist(txtfile.read())					
			else:
				target = [None,]
			
			doc_diff(doc, target[0])
			
			# do diff for child records
			if target[0] and dt_map[dt].keys():
				for child_dt in dt_map[dt]:					

					# for each child type, we need to create
					# a key (e.g. fieldname, label) based mapping of child records in 
					# txt files
					child_key_map = {}
					keys = dt_map[dt][child_dt]
					for target_d in target:
						if target_d['doctype'] == child_dt:
							for key in keys:
								if target_d.get(key):
									child_key_map[target_d.get(key)] = target_d
									break

					for d in webnotes.conn.sql("""select * from `tab%s` where 
						parent=%s and docstatus<2""" % (child_dt, '%s'), doc['name'], as_dict=1):

						source_key = None
						d['doctype'] = child_dt
						for key in keys:
							if d.get(key):
								source_key = d.get(key)
								break
						
						# only if a key is found
						if source_key:
							doc_diff(d, child_key_map.get(source_key), source_key)
				

	print_stats()


def doclist_diff(doclist):
	# main doc
	doc = doclist[0]
	if doc['doctype'] in dt_map.keys():
		# do for main
		target = webnotes.conn.sql("""select * from `tab%s` 
			where name=%s""" % (doc['doctype'], '%s'), doc['name'], as_dict=1)
		doc_diff(doc, target and target[0] or None)
		
		if not target:
			# no parent, no children!
			return
			
		for d in doclist[1:]:
			# if child
			if d['doctype'] in dt_map[doc['doctype']].keys():
				child_keys = dt_map[doc['doctype']][d['doctype']]
				
				# find the key on which a child is unique
				child_key = child_key_value = None
				for key in child_keys:
					if d.get(key):
						child_key = key
						child_key_value = d[key]
						break
					
				# incoming child record has a uniquely
				# identifiable key
				if child_key:
					target = webnotes.conn.sql("""select * from `tab%s`
						where `%s`=%s and parent=%s""" % (d['doctype'], child_key, '%s', '%s'),
						(child_key_value, d['parent']), as_dict=1)
					
					doc_diff(d, target and target[0] or None, child_key_value)
	

def doc_diff(source, target, child_key_value=None):
	from termcolor import colored
	global property_diff, property_count, missing

	docname = source.get('parent') and (source.get('parent') + '.' + child_key_value) \
		or source['name']

	if not target:
		missing += 1
		print(colored((source['doctype'] + ' -> ' + docname), 'red') + ' missing')
		
	else:
		target['doctype'] = source['doctype']
		for key in source:
			if (key not in ignore_fields) \
				and (target.get(key) != source[key]) \
				and (not (target.get(key)==None and source[key]==0)):
				
				prefix = source['doctype']+' -> ' + docname + ' -> '+key+' = '
				in_db = colored(str(target.get(key)), 'red')
				in_file = colored(str(source[key]), 'green')
				
				print(prefix + in_db + ' | ' + in_file)
				
				property_diff += 1
				if not key in property_count:
					property_count[key] = 0
				property_count[key] += 1
				
def print_stats():
	print
	print "Missing records: " + str(missing)
	print "Property mismatch: " + str(property_diff)
	for key in property_count:
		print "- " + key + ": " + str(property_count[key] or 0)
	print