Merge pull request #17605 from gavindsouza/ci-actions-refactor

ci: Actions Cleanup

.github/helper/documentation.py

@@ -30,7 +30,7 @@ def docs_link_exists(body):
 
 if __name__ == "__main__":
 	pr = sys.argv[1]
-	response = requests.get("https://api.github.com/repos/frappe/frappe/pulls/{}".format(pr))
+	response = requests.get(f"https://api.github.com/repos/frappe/frappe/pulls/{pr}")
 
 	if response.ok:
 		payload = response.json()

.github/helper/install.sh

@@ -5,55 +5,63 @@ cd ~ || exit
 echo "Setting Up Bench..."
 
 pip install frappe-bench
-
 bench -v init frappe-bench --skip-assets --python "$(which python)" --frappe-path "${GITHUB_WORKSPACE}"
+cd ./frappe-bench || exit
+
+bench -v setup requirements --dev
+if [ "$TYPE" == "ui" ]; then
+  bench -v setup requirements --node;
+fi
+
+echo "Setting Up Sites & Database..."
 
 mkdir ~/frappe-bench/sites/test_site
 cp "${GITHUB_WORKSPACE}/.github/helper/consumer_db/$DB.json" ~/frappe-bench/sites/test_site/site_config.json
 
 if [ "$TYPE" == "server" ]; then
-      mkdir ~/frappe-bench/sites/test_site_producer;
-      cp "${GITHUB_WORKSPACE}/.github/helper/producer_db/$DB.json" ~/frappe-bench/sites/test_site_producer/site_config.json;
+  mkdir ~/frappe-bench/sites/test_site_producer;
+  cp "${GITHUB_WORKSPACE}/.github/helper/producer_db/$DB.json" ~/frappe-bench/sites/test_site_producer/site_config.json;
 fi
-
 if [ "$DB" == "mariadb" ];then
-    mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL character_set_server = 'utf8mb4'";
-    mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'";
+  mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL character_set_server = 'utf8mb4'";
+  mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'";
 
-    mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe_consumer";
-    mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe_consumer'@'localhost' IDENTIFIED BY 'test_frappe_consumer'";
-    mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe_consumer\`.* TO 'test_frappe_consumer'@'localhost'";
+  mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe_consumer";
+  mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe_consumer'@'localhost' IDENTIFIED BY 'test_frappe_consumer'";
+  mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe_consumer\`.* TO 'test_frappe_consumer'@'localhost'";
 
-    mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe_producer";
-    mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe_producer'@'localhost' IDENTIFIED BY 'test_frappe_producer'";
-    mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe_producer\`.* TO 'test_frappe_producer'@'localhost'";
-
-    mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "FLUSH PRIVILEGES";
-  fi
+  mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe_producer";
+  mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe_producer'@'localhost' IDENTIFIED BY 'test_frappe_producer'";
+  mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe_producer\`.* TO 'test_frappe_producer'@'localhost'";
 
+  mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "FLUSH PRIVILEGES";
+fi
 if [ "$DB" == "postgres" ];then
-    echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_consumer" -U postgres;
-    echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_consumer WITH PASSWORD 'test_frappe'" -U postgres;
+  echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_consumer" -U postgres;
+  echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_consumer WITH PASSWORD 'test_frappe'" -U postgres;
 
-    echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_producer" -U postgres;
-    echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_producer WITH PASSWORD 'test_frappe'" -U postgres;
+  echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_producer" -U postgres;
+  echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_producer WITH PASSWORD 'test_frappe'" -U postgres;
 fi
 
-cd ./frappe-bench || exit
+echo "Setting Up Procfile..."
 
 sed -i 's/^watch:/# watch:/g' Procfile
 sed -i 's/^schedule:/# schedule:/g' Procfile
+if [ "$TYPE" == "server" ]; then
+  sed -i 's/^socketio:/# socketio:/g' Procfile;
+  sed -i 's/^redis_socketio:/# redis_socketio:/g' Procfile;
+fi
+if [ "$TYPE" == "ui" ]; then
+  sed -i 's/^web: bench serve/web: bench serve --with-coverage/g' Procfile;
+fi
 
-if [ "$TYPE" == "server" ]; then sed -i 's/^socketio:/# socketio:/g' Procfile; fi
-if [ "$TYPE" == "server" ]; then sed -i 's/^redis_socketio:/# redis_socketio:/g' Procfile; fi
-
-if [ "$TYPE" == "ui" ]; then bench -v setup requirements --node; fi
-bench -v setup requirements --dev
-
-if [ "$TYPE" == "ui" ]; then sed -i 's/^web: bench serve/web: bench serve --with-coverage/g' Procfile; fi
+echo "Starting Bench..."
 
 bench start &> bench_start.log &
 bench --site test_site reinstall --yes
 
-if [ "$TYPE" == "server" ]; then bench --site test_site_producer reinstall --yes; fi
-if [ "$TYPE" == "server" ]; then CI=Yes bench build --app frappe; fi
+if [ "$TYPE" == "server" ]; then
+  bench --site test_site_producer reinstall --yes;
+  CI=Yes bench build --app frappe;
+fi

.github/helper/install_dependencies.sh

@@ -3,8 +3,11 @@ set -e
 
 echo "Setting Up System Dependencies..."
 
-wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb
-sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb
+install_wkhtmltopdf() {
+  wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb
+  sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb
+}
+install_wkhtmltopdf &
 
 curl -LsS -O https://downloads.mariadb.com/MariaDB/mariadb_repo_setup
 sudo bash mariadb_repo_setup --mariadb-server-version=10.6

.github/helper/translation.py

@@ -20,19 +20,12 @@ for _file in files_to_scan:
 			if 'frappe-lint: disable-translate' in line:
 				continue
 
-			start_matches = start_pattern.search(line)
-			if start_matches:
-				starts_with_f = starts_with_f_pattern.search(line)
-
-				if starts_with_f:
-					has_f_string = f_string_pattern.search(line)
-					if has_f_string:
+			if start_matches := start_pattern.search(line):
+				if starts_with_f := starts_with_f_pattern.search(line):
+					if has_f_string := f_string_pattern.search(line):
 						errors_encounter += 1
 						print(f'\nF-strings are not supported for translations at line number {line_number}\n{line.strip()[:100]}')
-						continue
-					else:
-						continue
-
+					continue
 				match = pattern.search(line)
 				error_found = False
 

.github/workflows/deps-checker.yml

@@ -1,22 +0,0 @@
-name: 'Python Dependency Check'
-on:
-  pull_request:
-  workflow_dispatch:
-  push:
-    branches: [ develop ]
-
-permissions:
-  contents: read
-
-jobs:
-  deps-vulnerable-check:
-    name: 'Vulnerable Dependency'
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-      - uses: actions/checkout@v3
-      - run: pip install pip-audit
-      - run: pip-audit ${GITHUB_WORKSPACE}

.github/workflows/docker-release.yml

@@ -1,20 +0,0 @@
-name: 'Trigger Docker build on release'
-on:
-  release:
-    types: [released]
-permissions:
-  contents: read
-
-jobs:
-  curl:
-    permissions:
-      contents: none
-    name: 'Trigger Docker build on release'
-    runs-on: ubuntu-latest
-    container:
-      image: alpine:latest
-    steps:
-    - name: curl
-      run: |
-        apk add curl bash
-        curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${{ secrets.CI_PAT }}" https://api.github.com/repos/frappe/frappe_docker/actions/workflows/build_stable.yml/dispatches -d '{"ref":"main"}'

.github/workflows/docs-checker.yml

@@ -1,28 +0,0 @@
-name: 'Documentation Check'
-on:
-  pull_request:
-    types: [ opened, synchronize, reopened, edited ]
-
-permissions:
-  contents: read
-
-jobs:
-  docs-required:
-    name: 'Documentation Required'
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: 'Setup Environment'
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-
-      - name: 'Clone repo'
-        uses: actions/checkout@v3
-
-      - name: Validate Docs
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          pip install requests --quiet
-          python $GITHUB_WORKSPACE/.github/helper/documentation.py $PR_NUMBER

.github/workflows/linters.yml

@@ -1,29 +1,84 @@
 name: Linters
 
 on:
-  pull_request: { }
+  pull_request:
+  workflow_dispatch:
+  push:
+    branches: [ develop ]
 
-jobs:
+permissions:
+  contents: read
+
+concurrency:
+  group: commitcheck-frappe-${{ github.event.number }}
+  cancel-in-progress: true
 
-  linters:
-    name: Frappe Linter
+jobs:
+  commit-lint:
+    name: 'Semantic Commits'
     runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+
     steps:
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 200
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
+          check-latest: true
+
+      - name: Check commit titles
+        run: |
+          npm install @commitlint/cli @commitlint/config-conventional
+          npx commitlint --verbose --from ${{ github.event.pull_request.base.sha }} --to ${{ github.event.pull_request.head.sha }}
 
-      - name: Set up Python
+  docs-required:
+    name: 'Documentation Required'
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+
+    steps:
+      - name: 'Setup Environment'
         uses: actions/setup-python@v4
         with:
           python-version: '3.10'
+      - uses: actions/checkout@v3
 
-      - name: Install and Run Pre-commit
-        uses: pre-commit/action@v3.0.0
+      - name: Validate Docs
+        env:
+          PR_NUMBER: ${{ github.event.number }}
+        run: |
+          pip install requests --quiet
+          python $GITHUB_WORKSPACE/.github/helper/documentation.py $PR_NUMBER
+
+  linter:
+    name: 'Frappe Linter'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - uses: pre-commit/action@v3.0.0
 
       - name: Download Semgrep rules
         run: git clone --depth 1 https://github.com/frappe/semgrep-rules.git frappe-semgrep-rules
 
-      - name: Download semgrep
-        run: pip install semgrep==0.97.0
-
       - name: Run Semgrep rules
-        run: semgrep ci --config ./frappe-semgrep-rules/rules --config r/python.lang.correctness
+        run: |
+          pip install semgrep==0.97.0
+          semgrep ci --config ./frappe-semgrep-rules/rules --config r/python.lang.correctness
+
+  deps-vulnerable-check:
+    name: 'Vulnerable Dependency Check'
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - uses: actions/checkout@v3
+      - run: |
+          pip install pip-audit
+          pip-audit ${GITHUB_WORKSPACE}

.github/workflows/publish-assets-releases.yml → .github/workflows/on_release.yml

@@ -1,8 +1,11 @@
-name: 'Frappe Assets'
+name: 'Release'
 
 on:
   release:
-    types: [ created ]
+    types: [released]
+
+permissions:
+  contents: read
 
 env:
   GITHUB_TOKEN: ${{ github.token }}
@@ -47,3 +50,16 @@ jobs:
           asset_path: build/assets.tar.gz
           asset_name: assets.tar.gz
           asset_content_type: application/octet-stream
+
+  docker-release:
+    name: 'Trigger Docker build on release'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: none
+    container:
+      image: alpine:latest
+    steps:
+    - name: curl
+      run: |
+        apk add curl bash
+        curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${{ secrets.CI_PAT }}" https://api.github.com/repos/frappe/frappe_docker/actions/workflows/build_stable.yml/dispatches -d '{"ref":"main"}'

.github/workflows/patch-mariadb-tests.yml

@@ -1,7 +1,8 @@
-name: Patch
-
-on: [pull_request, workflow_dispatch]
+name: Server (MariaDB)
 
+on:
+  pull_request:
+  workflow_dispatch:
 
 concurrency:
   group: patch-mariadb-develop-${{ github.event.number }}
@@ -12,11 +13,10 @@ permissions:
 
 jobs:
   test:
+    name: Patch
     runs-on: ubuntu-latest
     timeout-minutes: 60
 
-    name: Patch Test
-
     services:
       mariadb:
         image: mariadb:10.6

.github/workflows/publish-assets-develop.yml

@@ -1,6 +1,7 @@
 name: 'Frappe Assets'
 
 on:
+  workflow_dispatch:
   push:
     branches: [ develop ]
 

.github/workflows/semantic-commits.yml

@@ -1,30 +0,0 @@
-name: Semantic Commits
-
-on:
-  pull_request: {}
-
-permissions:
-  contents: read
-
-concurrency:
-  group: commitcheck-frappe-${{ github.event.number }}
-  cancel-in-progress: true
-
-jobs:
-  commitlint:
-    name: Check Commit Titles
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 200
-
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 16
-          check-latest: true
-
-      - name: Check commit titles
-        run: |
-          npm install @commitlint/cli @commitlint/config-conventional
-          npx commitlint --verbose --from ${{ github.event.pull_request.base.sha }} --to ${{ github.event.pull_request.head.sha }}

.github/workflows/server-mariadb-tests.yml

@@ -1,4 +1,4 @@
-name: Server
+name: Server (MariaDB)
 
 on:
   pull_request:
@@ -16,6 +16,7 @@ permissions:
 
 jobs:
   test:
+    name: Unit Tests
     runs-on: ubuntu-latest
     timeout-minutes: 60
 
@@ -24,8 +25,6 @@ jobs:
       matrix:
        container: [1, 2]
 
-    name: Python Unit Tests (MariaDB)
-
     services:
       mariadb:
         image: mariadb:10.6

.github/workflows/server-postgres-tests.yml

@@ -1,4 +1,4 @@
-name: Server
+name: Server (Postgres)
 
 on:
   pull_request:
@@ -15,6 +15,7 @@ permissions:
 
 jobs:
   test:
+    name: Unit Tests
     runs-on: ubuntu-latest
     timeout-minutes: 60
 
@@ -23,8 +24,6 @@ jobs:
       matrix:
        container: [1, 2]
 
-    name: Python Unit Tests (Postgres)
-
     services:
       postgres:
         image: postgres:12.4