From 765a255a009dfd5dcbc3ee46ebb23415ce6d6b5d Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 31 Aug 2021 18:26:15 +0530 Subject: [PATCH 1/4] ci: use ubuntu-latest for all jobs --- .github/workflows/patch-mariadb-tests.yml | 2 +- .github/workflows/server-mariadb-tests.yml | 4 ++-- .github/workflows/server-postgres-tests.yml | 2 +- .github/workflows/translation_linter.yml | 22 --------------------- .github/workflows/ui-tests.yml | 2 +- 5 files changed, 5 insertions(+), 27 deletions(-) delete mode 100644 .github/workflows/translation_linter.yml diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index 0dd4cd51d8..3ac5cfa349 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -9,7 +9,7 @@ concurrency: jobs: test: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest name: Patch Test diff --git a/.github/workflows/server-mariadb-tests.yml b/.github/workflows/server-mariadb-tests.yml index fb6e56037c..9187aa3c30 100644 --- a/.github/workflows/server-mariadb-tests.yml +++ b/.github/workflows/server-mariadb-tests.yml @@ -13,7 +13,7 @@ concurrency: jobs: test: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest strategy: fail-fast: false @@ -146,7 +146,7 @@ jobs: name: Coverage Wrap Up needs: test container: python:3-slim - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest steps: - uses: actions/download-artifact@v1 with: diff --git a/.github/workflows/server-postgres-tests.yml b/.github/workflows/server-postgres-tests.yml index 1539e8c2d5..539cf53f65 100644 --- a/.github/workflows/server-postgres-tests.yml +++ b/.github/workflows/server-postgres-tests.yml @@ -10,7 +10,7 @@ concurrency: jobs: test: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/translation_linter.yml b/.github/workflows/translation_linter.yml deleted file mode 100644 index 4becaebd6b..0000000000 --- a/.github/workflows/translation_linter.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Frappe Linter -on: - pull_request: - branches: - - develop - - version-12-hotfix - - version-11-hotfix -jobs: - check_translation: - name: Translation Syntax Check - runs-on: ubuntu-18.04 - steps: - - uses: actions/checkout@v2 - - name: Setup python3 - uses: actions/setup-python@v1 - with: - python-version: 3.6 - - name: Validating Translation Syntax - run: | - git fetch origin $GITHUB_BASE_REF:$GITHUB_BASE_REF -q - files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF) - python $GITHUB_WORKSPACE/.github/helper/translation.py $files diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 2a55546ec4..0727b06043 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -12,7 +12,7 @@ concurrency: jobs: test: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest strategy: fail-fast: false From c663ab7d4493be0ad2e738519f4135114f0f2a7a Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 31 Aug 2021 21:59:06 +0530 Subject: [PATCH 2/4] test: improve test failure message --- frappe/tests/test_commands.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index 1797698a11..564ddafe54 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -63,7 +63,7 @@ def clean(value): return value -def exists_in_backup(doctypes, file): +def missing_in_backup(doctypes, file): """Checks if the list of doctypes exist in the database.sql.gz file supplied Args: @@ -71,7 +71,7 @@ def exists_in_backup(doctypes, file): file (str): Path of the database file Returns: - bool: True if all tables exist + doctypes(list): doctypes that are missing in backup """ predicate = ( 'COPY public."tab{}"' @@ -79,8 +79,24 @@ def exists_in_backup(doctypes, file): else "CREATE TABLE `tab{}`" ) with gzip.open(file, "rb") as f: - content = f.read().decode("utf8") - return all(predicate.format(doctype).lower() in content.lower() for doctype in doctypes) + content = f.read().decode("utf8").lower() + + return [doctype for doctype in doctypes + if predicate.format(doctype).lower() not in content] + + +def exists_in_backup(doctypes, file): + """Checks if the list of doctypes exist in the database.sql.gz file supplied + + Args: + doctypes (list): List of DocTypes to be checked + file (str): Path of the database file + + Returns: + bool: True if all tables exist + """ + missing_doctypes = missing_in_backup(doctypes, file) + return len(missing_doctypes) == 0 class BaseTestCommands(unittest.TestCase): @@ -222,7 +238,7 @@ class TestCommands(BaseTestCommands): self.execute("bench --site {site} backup --verbose") self.assertEqual(self.returncode, 0) database = fetch_latest_backups(partial=True)["database"] - self.assertTrue(exists_in_backup(backup["includes"]["includes"], database)) + self.assertEqual([], missing_in_backup(backup["includes"]["includes"], database)) # test 8: take a backup with frappe.conf.backup.excludes self.execute( @@ -233,7 +249,7 @@ class TestCommands(BaseTestCommands): self.assertEqual(self.returncode, 0) database = fetch_latest_backups(partial=True)["database"] self.assertFalse(exists_in_backup(backup["excludes"]["excludes"], database)) - self.assertTrue(exists_in_backup(backup["includes"]["includes"], database)) + self.assertEqual([], missing_in_backup(backup["includes"]["includes"], database)) # test 9: take a backup with --include (with frappe.conf.excludes still set) self.execute( @@ -242,7 +258,7 @@ class TestCommands(BaseTestCommands): ) self.assertEqual(self.returncode, 0) database = fetch_latest_backups(partial=True)["database"] - self.assertTrue(exists_in_backup(backup["includes"]["includes"], database)) + self.assertEqual([], missing_in_backup(backup["includes"]["includes"], database)) # test 10: take a backup with --exclude self.execute( @@ -257,7 +273,7 @@ class TestCommands(BaseTestCommands): self.execute("bench --site {site} backup --ignore-backup-conf") self.assertEqual(self.returncode, 0) database = fetch_latest_backups()["database"] - self.assertTrue(exists_in_backup(backup["excludes"]["excludes"], database)) + self.assertEqual([], missing_in_backup(backup["excludes"]["excludes"], database)) def test_restore(self): # step 0: create a site to run the test on From eb4e94f52d6474e75ad1ba4c70c7b8c6e1f0b87b Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 12 Sep 2021 18:10:14 +0530 Subject: [PATCH 3/4] fix: install mariadb client GitHub action's Ubuntu 20.04 image has mysql8 tooling which is not compatible with mariadb. --- .github/helper/install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/helper/install.sh b/.github/helper/install.sh index f6f0cad31a..93189d2b1f 100644 --- a/.github/helper/install.sh +++ b/.github/helper/install.sh @@ -17,6 +17,7 @@ if [ "$TYPE" == "server" ]; then fi if [ "$DB" == "mariadb" ];then + sudo apt install mariadb-client-10.3 mysql --host 127.0.0.1 --port 3306 -u root -e "SET GLOBAL character_set_server = 'utf8mb4'"; mysql --host 127.0.0.1 --port 3306 -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"; @@ -58,4 +59,4 @@ cd ../.. bench start & bench --site test_site reinstall --yes if [ "$TYPE" == "server" ]; then bench --site test_site_producer reinstall --yes; fi -bench build --app frappe \ No newline at end of file +bench build --app frappe From ae5cf9c32c5acfcf90f990d40757b8440cd50799 Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Sun, 12 Sep 2021 19:09:05 +0530 Subject: [PATCH 4/4] chore: change `missing_in_backup` function docstring --- frappe/tests/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index 564ddafe54..ee184843ad 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -64,7 +64,7 @@ def clean(value): def missing_in_backup(doctypes, file): - """Checks if the list of doctypes exist in the database.sql.gz file supplied + """Returns list of missing doctypes in the backup. Args: doctypes (list): List of DocTypes to be checked