Still more search fixes (#3054)

* [fix] awesome bar translation frappe/erpnext#8279 frappe/erpnext#8306

* [fix] frappe/erpnext#8348

* unescape and remove html entities

* ellipsify long field values

* [fix] test

* Add global search in custom field, Email Inbox searchable

* remove beautiful soup, make_field test case

* [fix] test

* Patch to update existing record in global search

* restore update_published patch

* more specific test cases

* Code descriptions for complex result ellipsifying

frappe/patches.txt

@@ -12,6 +12,7 @@ execute:frappe.reload_doc('core', 'doctype', 'docfield', force=True) #2017-03-03
 execute:frappe.reload_doc('core', 'doctype', 'docperm') #2017-03-03
 frappe.patches.v8_0.drop_is_custom_from_docperm
 frappe.patches.v8_0.drop_in_dialog
+frappe.patches.v8_0.update_records_in_global_search
 frappe.patches.v8_0.update_published_in_global_search
 execute:frappe.reload_doc('core', 'doctype', 'custom_docperm')
 execute:frappe.reload_doc('core', 'doctype', 'deleted_document')

frappe/patches/v8_0/update_published_in_global_search.py

@@ -4,4 +4,3 @@ def execute():
 
 	for doctype in get_doctypes_with_web_view():
 		rebuild_for_doctype(doctype)
-

frappe/patches/v8_0/update_records_in_global_search.py

@@ -0,0 +1,5 @@
+def execute():
+	from frappe.utils.global_search import get_doctypes_with_global_search, rebuild_for_doctype
+
+	for doctype in get_doctypes_with_global_search():
+		rebuild_for_doctype(doctype)

frappe/public/js/frappe/ui/toolbar/awesome_bar.js

@@ -72,7 +72,7 @@ frappe.search.AwesomeBar = Class.extend({
 				}
 				me.add_help();
 
-				awesomplete.list = me.options;
+				awesomplete.list = me.deduplicate(me.options);
 			}, 100));
 
 		});

frappe/public/js/frappe/ui/toolbar/search_utils.js

@@ -87,8 +87,8 @@ frappe.search.utils = {
                     if(level) {
                         out.push({
                             type: "In List",
-                            prefix: "Find '" + __(parts[0]).bold() + "' in ",
-                            label: __(me.bolden_match_part(item, parts[1])),
+                            prefix: __("Find {0} in ", [__(parts[0]).bold()]),
+                            label: me.bolden_match_part(__(item), parts[1]),
                             value: __('Find {0} in {1}', [__(parts[0]), __(item)]),
                             route_options: {"name": ["like", "%" + parts[0] + "%"]},
                             index: 1 + level,
@@ -111,8 +111,8 @@ frappe.search.utils = {
 					out.push({
 						type: "New",
 						prefix: "New ",
-						label: __(me.bolden_match_part(item, keywords.substr(4))),
-						value: __("New {0}", [item]),
+						label: me.bolden_match_part(__(item), keywords.substr(4)),
+						value: __("New {0}", [__(item)]),
 						index: 1 + level,
 						match: item,
 						onclick: function() { frappe.new_doc(item, true); }
@@ -131,8 +131,8 @@ frappe.search.utils = {
 		var option = function(type, route, order) {
 			return {
 				type: type,
-                label: __("{0}" + " " + type, [__(me.bolden_match_part(target, keywords))]),
-				value: __(target + " " + type),
+                label: __("{0}" + " " + type, [me.bolden_match_part(__(target), keywords)]),
+				value: __(__(target) + " " + type),
 				index: level + order,
 				match: target,
 				route: route,
@@ -151,7 +151,7 @@ frappe.search.utils = {
                             var match = item;
                             out.push({
                                 type: "New",
-                                label: __("New {0}", [__(me.bolden_match_part(item, keywords))]),
+                                label: __("New {0}", [me.bolden_match_part(__(item), keywords)]),
                                 value: __("New {0}", [__(item)]),
                                 index: level + 0.01,
                                 match: item,
@@ -191,8 +191,8 @@ frappe.search.utils = {
 				out.push({
                     type: "Report",
                     prefix: "Report ",
-					label: __(me.bolden_match_part(item, keywords)),
-					value: __("Report {0}" , [item]),
+					label: me.bolden_match_part(__(item), keywords),
+					value: __("Report {0}" , [__(item)]),
 					index: level,
 					route: route
 				});
@@ -216,7 +216,7 @@ frappe.search.utils = {
 				out.push({
 					type: "Page",
 					prefix: "Open ",
-					label: __(me.bolden_match_part(me.unscrub_and_titlecase(item), keywords)),
+					label: me.bolden_match_part(__(item), keywords),
 					value: __("Open {0}", [__(item)]),
 					match: item,
 					index: level,
@@ -236,6 +236,17 @@ frappe.search.utils = {
 				route: ['List', 'Event', target],
 			});
 		}
+        if(__('email inbox').indexOf(keywords.toLowerCase()) === 0) {
+			out.push({
+				type: "Inbox",
+				prefix: "Open ",
+				label: __('Email Inbox'),
+				value: __("Open {0}", [__('Email Inbox')]),
+				index: me.fuzzy_search(keywords, 'email inbox'),
+				match: target,
+				route: ['List', 'Communication', 'Inbox'],
+			});
+		}
 		return out;
     },
 
@@ -250,7 +261,7 @@ frappe.search.utils = {
 				ret = {
 					type: "Module",
 					prefix: "Open ",
-					label: __(me.bolden_match_part(item, keywords)),
+					label: me.bolden_match_part(__(item), keywords),
 					value: __("Open {0}", [__(item)]),
 					index: level,
 				}
@@ -276,42 +287,61 @@ frappe.search.utils = {
             }
 
             function make_description(content, doc_name) {
-                parts = content.split("|||");
-                content_length = 300;
-                fields = [];
-                current_length = 0;
+                var parts = content.split(" ||| ");
+                var result_max_length = 300;
+                var field_length = 120;
+                var fields = [];
+                var result_current_length = 0;
                 var field_text = "";
                 for(var i = 0; i < parts.length; i++) {
                     part = parts[i];
                     if(part.toLowerCase().indexOf(keywords) !== -1) {
-                        if(part.indexOf('&&&') !== -1) {
-                            var colon_index = part.indexOf('&&&');
-                            var field_value = part.slice(colon_index + 3);
+                        // If the field contains the keyword
+                        if(part.indexOf(' &&& ') !== -1) {
+                            var colon_index = part.indexOf(' &&& ');
+                            var field_value = part.slice(colon_index + 5);
                         } else {
-                            var colon_index = part.indexOf(':');
-                            var field_value = part.slice(colon_index + 1);
+                            var colon_index = part.indexOf(' : ');
+                            var field_value = part.slice(colon_index + 3);
+                        }
+                        if(field_value.length > field_length) {
+                            // If field value exceeds field_length, find the keyword in it
+                            // and trim field value by half the field_length at both sides
+                            // ellipsify if necessary
+                            var field_data = "";
+                            var index = field_value.indexOf(keywords);
+                            field_data += index < field_length/2 ? field_value.slice(0, index)
+                                : '...' + field_value.slice(index - field_length/2, index)
+                            field_data += field_value.slice(index, index + field_length/2);
+                            field_data += index + field_length/2 < field_value.length ? "..." : "";
+                            field_value = field_data;
                         }
                         var field_name = part.slice(0, colon_index);
 
-                        var remaining_length = content_length - current_length;
-                        current_length += field_name.length + field_value.length + 2;
-                        if(current_length < content_length) {
+                        // Find remaining result_length and add field length to result_current_length
+                        var remaining_length = result_max_length - result_current_length;
+                        result_current_length += field_name.length + field_value.length + 2;
+                        if(result_current_length < result_max_length) {
+                            // We have room, push the entire field
                             field_text = '<span class="field-name text-muted">' +
-                                me.bolden_match_part(field_name, keywords) + ':' + '</span>' +
+                                me.bolden_match_part(field_name, keywords) + ': </span> ' +
                                 me.bolden_match_part(field_value, keywords);
                             if(fields.indexOf(field_text) === -1 && doc_name !== field_value) {
                                 fields.push(field_text);
                             }
                         } else {
+                            // Not enough room
                             if(field_name.length < remaining_length){
+                                // Ellipsify (trim at word end) and push
                                 remaining_length -= field_name.length;
                                 field_text = '<span class="field-name text-muted">' +
-                                    me.bolden_match_part(field_name, keywords) + ':' + '</span>';
+                                    me.bolden_match_part(field_name, keywords) + ': </span> ';
                                 field_value = field_value.slice(0, remaining_length);
                                 field_value = field_value.slice(0, field_value.lastIndexOf(' ')) + ' ...';
                                 field_text += me.bolden_match_part(field_value, keywords);
                                 fields.push(field_text);
                             } else {
+                                // No room for even the field name, skip
                                 fields.push('...');
                             }
                             break;

frappe/tests/test_global_search.py

@@ -109,4 +109,69 @@ class TestGlobalSearch(unittest.TestCase):
 			})
 			doc.insert()
 
-		frappe.db.commit()
+		frappe.db.commit()
+
+	def test_get_field_value(self):
+		cases = [
+			{
+				"case_type": "generic",
+				"data": '''
+					<style type="text/css"> p.p1 {margin: 0.0px 0.0px 0.0px 0.0px; font: 14.0px 'Open Sans';
+					-webkit-text-stroke: #000000} span.s1 {font-kerning: none} </style>
+					<script>
+					var options = {
+						foo: "bar"
+					}
+					</script>
+					<p class="p1"><span class="s1">Contrary to popular belief, Lorem Ipsum is not simply random text. It has
+					roots in a piece of classical Latin literature from 45 BC, making it over 2000 years old. Richard McClintock,
+					a Latin professor at Hampden-Sydney College in Virginia, looked up one of the more obscure Latin words, consectetur,
+					from a Lorem Ipsum passage, and going through the cites of the word in classical literature, discovered the undoubtable source.
+					Lorem Ipsum comes from sections 1.10.32 and 1.10.33 of "de Finibus Bonorum et Malorum" (The Extremes of Good and Evil) by Cicero,
+					written in 45 BC. This book is a treatise on the theory of ethics, very popular during the Renaissance. The first line of Lorem Ipsum,
+					"Lorem ipsum dolor sit amet..", comes from a line in section 1.10.32.</span></p>
+					''',
+				"result": ('Description : Contrary to popular belief, Lorem Ipsum is not simply random text. It has roots in a piece of classical '
+					'Latin literature from 45 BC, making it over 2000 years old. Richard McClintock, a Latin professor at Hampden-Sydney College in Virginia, '
+					'looked up one of the more obscure Latin words, consectetur, from a Lorem Ipsum passage, and going through the cites of the word '
+					'in classical literature, discovered the undoubtable source. Lorem Ipsum comes from sections 1.10.32 and 1.10.33 of "de Finibus Bonorum '
+					'et Malorum" (The Extremes of Good and Evil) by Cicero, written in 45 BC. This book is a treatise on the theory of ethics, very popular '
+					'during the Renaissance. The first line of Lorem Ipsum, "Lorem ipsum dolor sit amet..", comes from a line in section 1.10.32.')
+
+			},
+			{
+				"case_type": "with_style",
+				"data": '''
+					<style type="text/css"> p.p1 {margin: 0.0px 0.0px 0.0px 0.0px; font: 14.0px 'Open Sans';
+					-webkit-text-stroke: #000000} span.s1 {font-kerning: none} </style>Lorem Ipsum Dolor Sit Amet
+					''',
+				"result": "Description : Lorem Ipsum Dolor Sit Amet"
+			},
+			{
+				"case_type": "with_script",
+				"data": '''
+					<script>
+					var options = {
+						foo: "bar"
+					}
+					</script>
+					Lorem Ipsum Dolor Sit Amet
+					''',
+				"result": "Description : Lorem Ipsum Dolor Sit Amet"
+			}
+		]
+
+		for case in cases:
+			doc = frappe.get_doc({
+				'doctype':'Event',
+				'subject': 'Lorem Ipsum',
+				'starts_on': frappe.utils.now_datetime(),
+				'description': case["data"]
+			})
+
+			field_as_text = ''
+			for field in doc.meta.fields:
+				if field.fieldname == 'description':
+					field_as_text = global_search.get_field_value(doc, field)
+
+			self.assertEquals(case["result"], field_as_text)

frappe/utils/global_search.py

@@ -4,6 +4,7 @@
 from __future__ import unicode_literals
 
 import frappe
+import re
 from frappe.utils import cint, strip_html_tags
 
 def setup_global_search_table():
@@ -26,6 +27,23 @@ def reset():
 	'''Deletes all data in __global_search'''
 	frappe.db.sql('delete from __global_search')
 
+def get_doctypes_with_global_search():
+	'''Return doctypes with global search fields'''
+	def _get():
+		global_search_doctypes = []
+		for d in frappe.get_all('DocType', 'name, module'):
+			meta = frappe.get_meta(d.name)
+			if len(meta.get_global_search_fields()) > 0:
+				global_search_doctypes.append(d)
+
+		installed_apps = frappe.get_installed_apps()
+
+		doctypes = [d.name for d in global_search_doctypes
+			if frappe.local.module_app[frappe.scrub(d.module)] in installed_apps]
+		return doctypes
+
+	return frappe.cache().get_value('doctypes_with_global_search', _get)
+
 def update_global_search(doc):
 	'''Add values marked with `in_global_search` to
 		`frappe.flags.update_global_search` from given doc
@@ -52,9 +70,9 @@ def update_global_search(doc):
 				  	if d.parent == doc.name:
 				  		for field in d.meta.get_global_search_fields():
 				  			if d.get(field.fieldname):
-				  				content.append(field.label + "&&& " + strip_html_tags(unicode(d.get(field.fieldname))))
+								content.append(get_field_value(d, field))
 			else:
-				content.append(field.label + "&&& " + strip_html_tags(unicode(doc.get(field.fieldname))))
+				content.append(get_field_value(doc, field))
 
 	if content:
 		published = 0
@@ -62,9 +80,22 @@ def update_global_search(doc):
 			published = 1 if doc.is_website_published() else 0
 
 		frappe.flags.update_global_search.append(
-			dict(doctype=doc.doctype, name=doc.name, content='|||'.join(content or ''),
+			dict(doctype=doc.doctype, name=doc.name, content=' ||| '.join(content or ''),
 				published=published, title=doc.get_title(), route=doc.get('route')))
 
+def get_field_value(doc, field):
+	'''Prepare field from raw data'''
+
+	from HTMLParser import HTMLParser
+
+	value = doc.get(field.fieldname)
+	if(getattr(field, 'fieldtype', None) in ["Text", "Text Editor"]):
+		h = HTMLParser()
+		value = h.unescape(value)
+		value = (re.subn(r'<[\s]*(script|style).*?</\1>(?s)', '', unicode(value))[0])
+		value = ' '.join(value.split())
+	return field.label + " : " + strip_html_tags(unicode(value))
+
 def sync_global_search():
 	'''Add values from `frappe.flags.update_global_search` to __global_search.
 		This is called internally at the end of the request.'''