Merge pull request #16409 from ankush/translate_newlines

frappe/tests/test_translate.py

@@ -36,7 +36,18 @@ class TestTranslate(unittest.TestCase):
 
 	def test_extract_message_from_file(self):
 		data = frappe.translate.get_messages_from_file(translation_string_file)
-		self.assertListEqual(data, expected_output)
+		exp_filename = "apps/frappe/frappe/tests/translation_test_file.txt"
+
+		self.assertEqual(len(data), len(expected_output),
+				msg=f"Mismatched output:\nExpected: {expected_output}\nFound: {data}")
+
+		for extracted, expected in zip(data, expected_output):
+			ext_filename, ext_message, ext_context, ext_line  = extracted
+			exp_message, exp_context, exp_line  = expected
+			self.assertEqual(ext_filename, exp_filename)
+			self.assertEqual(ext_message, exp_message)
+			self.assertEqual(ext_context, exp_context)
+			self.assertEqual(ext_line, exp_line)
 
 	def test_translation_with_context(self):
 		try:
@@ -107,13 +118,16 @@ class TestTranslate(unittest.TestCase):
 
 
 expected_output = [
-	('apps/frappe/frappe/tests/translation_test_file.txt', 'Warning: Unable to find {0} in any table related to {1}', 'This is some context', 2),
-	('apps/frappe/frappe/tests/translation_test_file.txt', 'Warning: Unable to find {0} in any table related to {1}', None, 4),
-	('apps/frappe/frappe/tests/translation_test_file.txt', "You don't have any messages yet.", None, 6),
-	('apps/frappe/frappe/tests/translation_test_file.txt', 'Submit', 'Some DocType', 8),
-	('apps/frappe/frappe/tests/translation_test_file.txt', 'Warning: Unable to find {0} in any table related to {1}', 'This is some context', 15),
-	('apps/frappe/frappe/tests/translation_test_file.txt', 'Submit', 'Some DocType', 17),
-	('apps/frappe/frappe/tests/translation_test_file.txt', "You don't have any messages yet.", None, 19),
-	('apps/frappe/frappe/tests/translation_test_file.txt', "You don't have any messages yet.", None, 21)
+	('Warning: Unable to find {0} in any table related to {1}', 'This is some context', 2),
+	('Warning: Unable to find {0} in any table related to {1}', None, 4),
+	("You don't have any messages yet.", None, 6),
+	('Submit', 'Some DocType', 8),
+	('Warning: Unable to find {0} in any table related to {1}', 'This is some context', 15),
+	('Submit', 'Some DocType', 17),
+	("You don't have any messages yet.", None, 19),
+	("You don't have any messages yet.", None, 21),
+	("Long string that needs its own line because of black formatting.", None, 24),
+	("Long string with", "context", 28),
+	("Long string with", "context on newline", 32),
 ]
 

frappe/tests/translation_test_file.txt

@@ -18,4 +18,18 @@ _('Submit', context="Some DocType")
 
 _("""You don't have any messages yet.""")
 
-_('''You don't have any messages yet.''')
+_('''You don't have any messages yet.''')
+
+// allow newline in beginning
+_(
+"""Long string that needs its own line because of black formatting."""
+).format("blah")
+
+_(
+"Long string with", context="context"
+).format("blah")
+
+_(
+    "Long string with",
+    context="context on newline"
+).format("blah")

frappe/translate.py

@@ -23,6 +23,35 @@ from frappe.utils import get_bench_path, is_html, strip, strip_html_tags
 from frappe.query_builder import Field, DocType
 from pypika.terms import PseudoColumn
 
+TRANSLATE_PATTERN = re.compile(
+	r"_\([\s\n]*"  # starts with literal `_(`, ignore following whitespace/newlines
+
+	# BEGIN: message search
+	r"([\"']{,3})"  # start of message string identifier - allows: ', ", """, '''; 1st capture group
+		r"(?P<message>((?!\1).)*)" # Keep matching until string closing identifier is met which is same as 1st capture group
+	r"\1" # match exact string closing identifier
+	# END: message search
+
+	# BEGIN: python context search
+	r"([\s\n]*,[\s\n]*context\s*=\s*"  # capture `context=` with ignoring whitespace
+		r"([\"'])"  # start of context string identifier; 5th capture group
+		r"(?P<py_context>((?!\5).)*)" # capture context string till closing id is found
+		r"\5"  # match context string closure
+	r")?"  # match 0 or 1 context strings
+	# END: python context search
+
+	# BEGIN: JS context search
+	r"(\s*,\s*(.)*?\s*(,\s*" # skip message format replacements: ["format", ...] | null | []
+		r"([\"'])"  # start of context string; 11th capture group
+		r"(?P<js_context>((?!\11).)*)" # capture context string till closing id is found
+		r"\11"  # match context string closure
+		r")*"
+	r")*"  # match one or more context string
+	# END: JS context search
+
+	r"[\s\n]*\)"  # Closing function call ignore leading whitespace/newlines
+)
+
 
 def get_language(lang_list: List = None) -> str:
 	"""Set `frappe.local.lang` from HTTP headers at beginning of request
@@ -651,9 +680,8 @@ def extract_messages_from_code(code):
 			frappe.clear_last_message()
 
 	messages = []
-	pattern = r"_\(([\"']{,3})(?P<message>((?!\1).)*)\1(\s*,\s*context\s*=\s*([\"'])(?P<py_context>((?!\5).)*)\5)*(\s*,\s*(.)*?\s*(,\s*([\"'])(?P<js_context>((?!\11).)*)\11)*)*\)"
 
-	for m in re.compile(pattern).finditer(code):
+	for m in TRANSLATE_PATTERN.finditer(code):
 		message = m.group('message')
 		context = m.group('py_context') or m.group('js_context')
 		pos = m.start()