From 25ad8ef16ce7f95ae80f49961bd1cac8be092916 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Thu, 13 Jul 2017 15:17:12 +0530 Subject: [PATCH 01/24] Test Runner - Refactored (#3679) * [tests] test_runner to run one JS test at a time * [tests] test_runner to run one JS test at a time * [test] ignore failing tests * [test] comment test_calendar_view * [test] add timeout * [test] add timeout --- .travis.yml | 2 +- frappe/commands/utils.py | 28 +++----- .../core/doctype/test_runner/test_runner.js | 15 +++- .../core/doctype/test_runner/test_runner.json | 4 +- .../core/doctype/test_runner/test_runner.py | 52 ++++++++++---- frappe/desk/doctype/todo/todo.json | 4 +- .../guides/automated-testing/qunit-testing.md | 68 +++++++++++++------ frappe/test_runner.py | 9 +++ ...alendar_view.js => _test_calendar_view.js} | 0 .../ui/{test_desktop.js => _test_desktop.js} | 0 ...test_gantt_view.js => _test_gantt_view.js} | 0 frappe/tests/ui/{ => data}/test_lib.js | 19 +++--- frappe/tests/ui/test_list.js | 9 ++- frappe/tests/ui/test_test_runner.py | 59 +++++++++++++--- frappe/utils/selenium_testdriver.py | 18 +++-- 15 files changed, 200 insertions(+), 87 deletions(-) rename frappe/tests/ui/{test_calendar_view.js => _test_calendar_view.js} (100%) rename frappe/tests/ui/{test_desktop.js => _test_desktop.js} (100%) rename frappe/tests/ui/{test_gantt_view.js => _test_gantt_view.js} (100%) rename frappe/tests/ui/{ => data}/test_lib.js (93%) diff --git a/.travis.yml b/.travis.yml index ea584e38c0..30b21ecf18 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,4 +51,4 @@ script: - set -e - bench --verbose run-tests - sleep 5 - - bench --verbose run-tests --ui-tests + - bench --verbose run-ui-tests --app frappe diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 9cabd36c75..74da084beb 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -323,30 +323,24 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), @click.command('run-ui-tests') @click.option('--app', help="App to run tests on, leave blank for all apps") -@click.option('--ci', is_flag=True, default=False, help="Run in CI environment") +@click.option('--test', help="File name of the test you want to run") +@click.option('--profile', is_flag=True, default=False) @pass_context -def run_ui_tests(context, app=None, ci=False): +def run_ui_tests(context, app=None, test=False, profile=False): "Run UI tests" - import subprocess + import frappe.test_runner site = get_site(context) frappe.init(site=site) + frappe.connect() - if app is None: - app = ",".join(frappe.get_installed_apps()) - - cmd = [ - './node_modules/.bin/nightwatch', - '--config', './apps/frappe/frappe/nightwatch.js', - '--app', app, - '--site', site - ] - - if ci: - cmd.extend(['--env', 'ci_server']) + ret = frappe.test_runner.run_ui_tests(app=app, test=test, verbose=context.verbose, + profile=profile) + if len(ret.failures) == 0 and len(ret.errors) == 0: + ret = 0 - bench_path = frappe.utils.get_bench_path() - subprocess.call(cmd, cwd=bench_path) + if os.environ.get('CI'): + sys.exit(ret) @click.command('serve') @click.option('--port', default=8000) diff --git a/frappe/core/doctype/test_runner/test_runner.js b/frappe/core/doctype/test_runner/test_runner.js index 477d8903de..f7d4128c50 100644 --- a/frappe/core/doctype/test_runner/test_runner.js +++ b/frappe/core/doctype/test_runner/test_runner.js @@ -11,7 +11,7 @@ frappe.ui.form.on('Test Runner', { // all tests frappe.call({ - method: 'frappe.core.doctype.test_runner.test_runner.get_all_tests' + method: 'frappe.core.doctype.test_runner.test_runner.get_test_js' }).always((data) => { $("
").appendTo(wrapper.empty()); frm.events.run_tests(frm, data.message); @@ -54,7 +54,18 @@ frappe.ui.form.on('Test Runner', { console.log(JSON.stringify(result, null, 2)); }); QUnit.load(); - QUnit.done(() => { + + QUnit.done(({ total, failed, passed, runtime }) => { + // flag for selenium that test is done + $('
').appendTo($('body')); + + console.log( `Total: ${total}, Failed: ${failed}, Passed: ${passed}, Runtime: ${runtime}` ); // eslint-disable-line + + if(failed) { + console.log('Tests Failed'); // eslint-disable-line + } else { + console.log('Tests Passed'); // eslint-disable-line + } frappe.set_route('Form', 'Test Runner', 'Test Runner'); }); }); diff --git a/frappe/core/doctype/test_runner/test_runner.json b/frappe/core/doctype/test_runner/test_runner.json index 0094d6c659..8396d5df43 100644 --- a/frappe/core/doctype/test_runner/test_runner.json +++ b/frappe/core/doctype/test_runner/test_runner.json @@ -83,7 +83,7 @@ "issingle": 1, "istable": 0, "max_attachments": 0, - "modified": "2017-06-26 10:57:19.976624", + "modified": "2017-07-12 23:16:15.910891", "modified_by": "Administrator", "module": "Core", "name": "Test Runner", @@ -104,7 +104,7 @@ "print": 1, "read": 1, "report": 0, - "role": "System Manager", + "role": "Administrator", "set_user_permissions": 0, "share": 1, "submit": 0, diff --git a/frappe/core/doctype/test_runner/test_runner.py b/frappe/core/doctype/test_runner/test_runner.py index 2d66622955..a59ddc69a5 100644 --- a/frappe/core/doctype/test_runner/test_runner.py +++ b/frappe/core/doctype/test_runner/test_runner.py @@ -10,18 +10,40 @@ class TestRunner(Document): pass @frappe.whitelist() -def get_all_tests(): - tests = [] - for app in frappe.get_installed_apps(): - tests_path = frappe.get_app_path(app, 'tests', 'ui') - if os.path.exists(tests_path): - for basepath, folders, files in os.walk(tests_path): # pylint: disable=unused-variable - for fname in files: - if fname.startswith('test') and fname.endswith('.js'): - path = os.path.join(basepath, fname) - with open(path, 'r') as fileobj: - tests.append(dict( - path = os.path.relpath(frappe.get_app_path(app), path), - script = fileobj.read() - )) - return tests +def get_test_js(): + '''Get test + data for app, example: app/tests/ui/test_name.js''' + test_path = frappe.db.get_single_value('Test Runner', 'module_path') + + # split + app, test_path = test_path.split(os.path.sep, 1) + test_js = get_test_data(app) + + # full path + test_path = frappe.get_app_path(app, test_path) + + with open(test_path, 'r') as fileobj: + test_js.append(dict( + script = fileobj.read() + )) + return test_js + +def get_test_data(app): + '''Get the test fixtures from all js files in app/tests/ui/data''' + test_js = [] + + def add_file(path): + with open(path, 'r') as fileobj: + test_js.append(dict( + script = fileobj.read() + )) + + data_path = frappe.get_app_path(app, 'tests', 'ui', 'data') + if os.path.exists(data_path): + for fname in os.listdir(data_path): + if fname.endswith('.js'): + add_file(os.path.join(data_path, fname)) + + if app != 'frappe': + add_file(frappe.get_app_path('frappe', 'tests', 'ui', 'data', 'test_lib.js')) + + return test_js diff --git a/frappe/desk/doctype/todo/todo.json b/frappe/desk/doctype/todo/todo.json index ebd2489e40..d62248b550 100644 --- a/frappe/desk/doctype/todo/todo.json +++ b/frappe/desk/doctype/todo/todo.json @@ -4,7 +4,7 @@ "allow_import": 0, "allow_rename": 0, "autoname": "hash", - "beta": 0, + "beta": 1, "creation": "2012-07-03 13:30:35", "custom": 0, "docstatus": 0, @@ -514,7 +514,7 @@ "issingle": 0, "istable": 0, "max_attachments": 0, - "modified": "2017-07-06 10:23:39.656033", + "modified": "2017-07-12 19:08:23.760631", "modified_by": "Administrator", "module": "Desk", "name": "ToDo", diff --git a/frappe/docs/user/en/guides/automated-testing/qunit-testing.md b/frappe/docs/user/en/guides/automated-testing/qunit-testing.md index a8e35607d5..d66dde2782 100644 --- a/frappe/docs/user/en/guides/automated-testing/qunit-testing.md +++ b/frappe/docs/user/en/guides/automated-testing/qunit-testing.md @@ -14,32 +14,56 @@ In the CI, all QUnit tests are run by the **Test Runner** using `frappe/tests/te +### Running Tests + +To run a Test Runner based test, use the `run-ui-tests` bench command by passing the name of the file you want to run. + + bench run-ui-tests --test frappe/tests/ui/test_list.js + +This will pass the filename to `test_test_runner.py` that will load the required JS in the browser and execute the tests + +### Adding Fixtures / Test Data + +You can also add data that you require for all tests in the `tests/ui/data` folder of your app. All the files in this folder will be loaded in the browser before running the test. + +The file `frappe/tests/ui/data/test_lib.js`, which contains library functions for testing is always loaded. + +### Running All UI Tests + +To run all UI tests together for your app run + + bench run-ui-tests --app [app_name] + +This will run all the files in your `tests/ui` folder one by one. + ### Example QUnit Test Here is the example of the To Do test in QUnit - QUnit.test("test quick entry", function(assert) { - assert.expect(2); - let done = assert.async(); - let random = frappe.utils.get_random(10); - - frappe.set_route('List', 'ToDo') - .then(() => { - return frappe.new_doc('ToDo'); - }) - .then(() => { - frappe.quick_entry.dialog.set_value('description', random); - return frappe.quick_entry.insert(); - }) - .then((doc) => { - assert.ok(doc && !doc.__islocal); - return frappe.set_route('Form', 'ToDo', doc.name); - }) - .then(() => { - assert.ok(cur_frm.doc.description.includes(random)); - done(); - }); - }); + QUnit.test("Test quick entry", function(assert) { + assert.expect(2); + let done = assert.async(); + let random_text = frappe.utils.get_random(10); + + frappe.run_serially([ + () => frappe.set_route('List', 'ToDo'), + () => frappe.new_doc('ToDo'), + () => frappe.quick_entry.dialog.set_value('description', random_text), + () => frappe.quick_entry.insert(), + (doc) => { + assert.ok(doc && !doc.__islocal); + return frappe.set_route('Form', 'ToDo', doc.name); + }, + () => assert.ok(cur_frm.doc.description.includes(random_text)), + + // Delete the created ToDo + () => frappe.tests.click_page_head_item('Menu'), + () => frappe.tests.click_dropdown_item('Delete'), + () => frappe.tests.click_page_head_item('Yes'), + + () => done() + ]); + }); ### Writing Test Friendly Code with Promises diff --git a/frappe/test_runner.py b/frappe/test_runner.py index edefaaca11..1c0dfb4180 100644 --- a/frappe/test_runner.py +++ b/frappe/test_runner.py @@ -139,6 +139,13 @@ def run_tests_for_module(module, verbose=False, tests=(), profile=False): return _run_unittest(module=module, verbose=verbose, tests=tests, profile=profile) +def run_ui_tests(app=None, test=None, verbose=False, profile=False): + '''Run a single unit test for UI using test_test_runner''' + module = importlib.import_module('frappe.tests.ui.test_test_runner') + frappe.flags.ui_test_app = app + frappe.flags.ui_test_path = test + return _run_unittest(module=module, verbose=verbose, tests=(), profile=profile) + def _run_unittest(module, verbose=False, tests=(), profile=False): test_suite = unittest.TestSuite() module_test_cases = unittest.TestLoader().loadTestsFromModule(module) @@ -154,6 +161,8 @@ def _run_unittest(module, verbose=False, tests=(), profile=False): pr = cProfile.Profile() pr.enable() + frappe.flags.tests_verbose = verbose + out = unittest_runner(verbosity=1+(verbose and 1 or 0)).run(test_suite) if profile: diff --git a/frappe/tests/ui/test_calendar_view.js b/frappe/tests/ui/_test_calendar_view.js similarity index 100% rename from frappe/tests/ui/test_calendar_view.js rename to frappe/tests/ui/_test_calendar_view.js diff --git a/frappe/tests/ui/test_desktop.js b/frappe/tests/ui/_test_desktop.js similarity index 100% rename from frappe/tests/ui/test_desktop.js rename to frappe/tests/ui/_test_desktop.js diff --git a/frappe/tests/ui/test_gantt_view.js b/frappe/tests/ui/_test_gantt_view.js similarity index 100% rename from frappe/tests/ui/test_gantt_view.js rename to frappe/tests/ui/_test_gantt_view.js diff --git a/frappe/tests/ui/test_lib.js b/frappe/tests/ui/data/test_lib.js similarity index 93% rename from frappe/tests/ui/test_lib.js rename to frappe/tests/ui/data/test_lib.js index ede4a3449f..5023eb451d 100644 --- a/frappe/tests/ui/test_lib.js +++ b/frappe/tests/ui/data/test_lib.js @@ -67,8 +67,10 @@ frappe.tests = { return frappe.run_serially(grid_row_tasks); }, setup_doctype: (doctype) => { - return frappe.set_route('List', doctype) - .then(() => { + return frappe.run_serially([ + () => frappe.set_route('List', doctype), + () => frappe.timeout(1), + () => { frappe.tests.data[doctype] = []; let expected = frappe.tests.get_fixture_names(doctype); cur_list.data.forEach((d) => { @@ -88,10 +90,11 @@ frappe.tests = { }); return frappe.run_serially(tasks); - }); + } + ]); }, click_page_head_item: (text) => { - // Method to items present on the page header like New, Save, Delete etc. + // Method to items present on the page header like New, Save, Delete etc. let possible_texts = ["New", "Delete", "Save", "Yes"]; return frappe.run_serially([ () => { @@ -107,7 +110,7 @@ frappe.tests = { ]); }, click_dropdown_item: (text) => { - // Method to click dropdown elements + // Method to click dropdown elements return frappe.run_serially([ () => { let li = $(".dropdown-menu li:contains("+text+"):visible").get(0); @@ -117,7 +120,7 @@ frappe.tests = { ]); }, click_navbar_item: (text) => { - // Method to click an elements present on the navbar + // Method to click an elements present on the navbar return frappe.run_serially([ () => { if (text == "Help"){ @@ -137,14 +140,14 @@ frappe.tests = { ]); }, click_generic_text: (text, tag='a') => { - // Method to click an element by its name + // Method to click an element by its name return frappe.run_serially([ () => $(tag+":contains("+text+"):visible")[0].click(), () => frappe.timeout(0.3) ]); }, click_desktop_icon: (text) => { - // Method to click the desktop icons on the Desk, by their name + // Method to click the desktop icons on the Desk, by their name return frappe.run_serially([ () => $("#icon-grid > div > div.app-icon[title="+text+"]").click(), () => frappe.timeout(0.3) diff --git a/frappe/tests/ui/test_list.js b/frappe/tests/ui/test_list.js index f154b45c01..5000794014 100644 --- a/frappe/tests/ui/test_list.js +++ b/frappe/tests/ui/test_list.js @@ -28,10 +28,13 @@ QUnit.test("Test quick entry", function(assert) { QUnit.test("Test list values", function(assert) { assert.expect(2); let done = assert.async(); - frappe.set_route('List', 'DocType') - .then(() => { + frappe.run_serially([ + () => frappe.set_route('List', 'DocType'), + () => frappe.timeout(1), + () => { assert.deepEqual(['List', 'DocType', 'List'], frappe.get_route()); assert.ok($('.list-item:visible').length > 10); done(); - }); + } + ]); }); diff --git a/frappe/tests/ui/test_test_runner.py b/frappe/tests/ui/test_test_runner.py index 7cd6529b3d..185d8f1af5 100644 --- a/frappe/tests/ui/test_test_runner.py +++ b/frappe/tests/ui/test_test_runner.py @@ -1,17 +1,54 @@ from __future__ import print_function from frappe.utils.selenium_testdriver import TestDriver -import unittest +import unittest, os, frappe, time class TestLogin(unittest.TestCase): - def setUp(self): - self.driver = TestDriver() - def test_test_runner(self): - self.driver.login() - self.driver.set_route('Form', 'Test Runner') - self.driver.click_primary_action() - self.driver.wait_for('#qunit-testresult-display', timeout=60) - self.driver.print_console() + for test in get_tests(): + print('Running {0}...'.format(test)) + frappe.db.set_value('Test Runner', None, 'module_path', test) + frappe.db.commit() + driver = TestDriver() + driver.login() + driver.set_route('Form', 'Test Runner') + driver.click_primary_action() + driver.wait_for('#frappe-qunit-done', timeout=60) + console = driver.get_console() + if frappe.flags.tests_verbose: + for line in console: + print(line) + print('-' * 40) + print('Checking if passed "{0}"'.format(test)) + self.assertTrue('Tests Passed' in console) + driver.close() + time.sleep(1) + +def get_tests(): + '''Get tests base on flag''' + if frappe.flags.ui_test_app: + return get_tests_for(frappe.flags.ui_test_app) + elif frappe.flags.ui_test_path: + return (frappe.flags.ui_test_path,) + else: + tests = [] + for app in frappe.get_installed_apps(): + tests.extend(get_tests_for(app)) + return tests + +def get_tests_for(app): + '''Get all tests for a particular app''' + tests = [] + tests_path = frappe.get_app_path(app, 'tests', 'ui') + if os.path.exists(tests_path): + for basepath, folders, files in os.walk(tests_path): # pylint: disable=unused-variable + if os.path.join('ui', 'data') in basepath: + continue + + for fname in files: + if fname.startswith('test') and fname.endswith('.js'): + path = os.path.join(basepath, fname) + path = os.path.relpath(path, frappe.get_app_path(app)) + tests.append(os.path.join(app, path)) + + return tests - def tearDown(self): - self.driver.close() diff --git a/frappe/utils/selenium_testdriver.py b/frappe/utils/selenium_testdriver.py index b39bd93d23..3f497d312a 100644 --- a/frappe/utils/selenium_testdriver.py +++ b/frappe/utils/selenium_testdriver.py @@ -55,6 +55,9 @@ class TestDriver(object): sys.exit(0) signal.signal(signal.SIGINT, signal_handler) + def refresh(self): + self.driver.refresh() + def close(self): if self.driver: self.driver.quit() @@ -118,7 +121,8 @@ class TestDriver(object): self.print_console() raise e - def print_console(self): + def get_console(self): + out = [] for entry in self.driver.get_log('browser'): source, line_no, message = entry.get('message').split(' ', 2) @@ -126,9 +130,15 @@ class TestDriver(object): # message is a quoted/escaped string message = literal_eval(message) - print(source + ' ' + line_no) - print(message) - print('-'*40) + out.append(source + ' ' + line_no) + out.append(message) + out.append('-'*40) + + return out + + def print_console(self): + for line in self.get_console(): + print(line) def get_wait(self, timeout=20): return WebDriverWait(self.driver, timeout) From 7b4957d1f3631d86171e5db54815222dce956fd9 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Thu, 13 Jul 2017 15:26:11 +0530 Subject: [PATCH 02/24] [tests] more timeouts --- frappe/tests/ui/test_list.js | 7 ++++--- frappe/tests/ui/test_test_runner.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/frappe/tests/ui/test_list.js b/frappe/tests/ui/test_list.js index 5000794014..6d04c07a7f 100644 --- a/frappe/tests/ui/test_list.js +++ b/frappe/tests/ui/test_list.js @@ -20,6 +20,7 @@ QUnit.test("Test quick entry", function(assert) { () => frappe.tests.click_page_head_item('Menu'), () => frappe.tests.click_dropdown_item('Delete'), () => frappe.tests.click_page_head_item('Yes'), + () => frappe.timeout(2), () => done() ]); @@ -30,11 +31,11 @@ QUnit.test("Test list values", function(assert) { let done = assert.async(); frappe.run_serially([ () => frappe.set_route('List', 'DocType'), - () => frappe.timeout(1), + () => frappe.timeout(2), () => { assert.deepEqual(['List', 'DocType', 'List'], frappe.get_route()); assert.ok($('.list-item:visible').length > 10); - done(); - } + }, + () => done() ]); }); diff --git a/frappe/tests/ui/test_test_runner.py b/frappe/tests/ui/test_test_runner.py index 185d8f1af5..a2587ee3c6 100644 --- a/frappe/tests/ui/test_test_runner.py +++ b/frappe/tests/ui/test_test_runner.py @@ -14,7 +14,7 @@ class TestLogin(unittest.TestCase): driver.click_primary_action() driver.wait_for('#frappe-qunit-done', timeout=60) console = driver.get_console() - if frappe.flags.tests_verbose: + if frappe.flags.tests_verbose or True: for line in console: print(line) print('-' * 40) From ad7911fb96947135f065f0d4b10f5a0fe5ed72ed Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Thu, 13 Jul 2017 17:45:20 +0530 Subject: [PATCH 03/24] [minor] better message for beta docs --- frappe/desk/doctype/todo/todo.json | 4 ++-- frappe/public/js/frappe/dom.js | 3 +++ frappe/public/js/frappe/form/templates/form_sidebar.html | 6 +++++- frappe/test_runner.py | 4 ++-- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/frappe/desk/doctype/todo/todo.json b/frappe/desk/doctype/todo/todo.json index d62248b550..487dcbd3d8 100644 --- a/frappe/desk/doctype/todo/todo.json +++ b/frappe/desk/doctype/todo/todo.json @@ -4,7 +4,7 @@ "allow_import": 0, "allow_rename": 0, "autoname": "hash", - "beta": 1, + "beta": 0, "creation": "2012-07-03 13:30:35", "custom": 0, "docstatus": 0, @@ -514,7 +514,7 @@ "issingle": 0, "istable": 0, "max_attachments": 0, - "modified": "2017-07-12 19:08:23.760631", + "modified": "2017-07-13 17:44:54.369254", "modified_by": "Administrator", "module": "Desk", "name": "ToDo", diff --git a/frappe/public/js/frappe/dom.js b/frappe/public/js/frappe/dom.js index 5f35dc83c9..c66841ceee 100644 --- a/frappe/public/js/frappe/dom.js +++ b/frappe/public/js/frappe/dom.js @@ -211,6 +211,9 @@ frappe.timeout = seconds => { }); }; +frappe.scrub = function(text) { + return text.replace(/ /g, "_").toLowerCase(); +}; frappe.get_modal = function(title, content) { return $(frappe.render_template("modal", {title:title, content:content})).appendTo(document.body); diff --git a/frappe/public/js/frappe/form/templates/form_sidebar.html b/frappe/public/js/frappe/form/templates/form_sidebar.html index e44a55cedc..398d6a50d1 100644 --- a/frappe/public/js/frappe/form/templates/form_sidebar.html +++ b/frappe/public/js/frappe/form/templates/form_sidebar.html @@ -10,7 +10,11 @@ {% if frm.meta.beta %} {% endif %}