Преглед на файлове

Merge pull request #17605 from gavindsouza/ci-actions-refactor

ci: Actions Cleanup
version-14
gavin преди 2 години
committed by GitHub
родител
ревизия
2eccdad19d
No known key found for this signature in database GPG ключ ID: 4AEE18F83AFDEB23
променени са 15 файла, в които са добавени 141 реда и са изтрити 167 реда
  1. +1
    -1
      .github/helper/documentation.py
  2. +37
    -29
      .github/helper/install.sh
  3. +5
    -2
      .github/helper/install_dependencies.sh
  4. +4
    -11
      .github/helper/translation.py
  5. +0
    -0
      .github/workflows/create-release.yml
  6. +0
    -22
      .github/workflows/deps-checker.yml
  7. +0
    -20
      .github/workflows/docker-release.yml
  8. +0
    -28
      .github/workflows/docs-checker.yml
  9. +66
    -11
      .github/workflows/linters.yml
  10. +18
    -2
      .github/workflows/on_release.yml
  11. +5
    -5
      .github/workflows/patch-mariadb-tests.yml
  12. +1
    -0
      .github/workflows/publish-assets-develop.yml
  13. +0
    -30
      .github/workflows/semantic-commits.yml
  14. +2
    -3
      .github/workflows/server-mariadb-tests.yml
  15. +2
    -3
      .github/workflows/server-postgres-tests.yml

+ 1
- 1
.github/helper/documentation.py Целия файл

@@ -30,7 +30,7 @@ def docs_link_exists(body):

if __name__ == "__main__":
pr = sys.argv[1]
response = requests.get("https://api.github.com/repos/frappe/frappe/pulls/{}".format(pr))
response = requests.get(f"https://api.github.com/repos/frappe/frappe/pulls/{pr}")

if response.ok:
payload = response.json()


+ 37
- 29
.github/helper/install.sh Целия файл

@@ -5,55 +5,63 @@ cd ~ || exit
echo "Setting Up Bench..."

pip install frappe-bench

bench -v init frappe-bench --skip-assets --python "$(which python)" --frappe-path "${GITHUB_WORKSPACE}"
cd ./frappe-bench || exit

bench -v setup requirements --dev
if [ "$TYPE" == "ui" ]; then
bench -v setup requirements --node;
fi

echo "Setting Up Sites & Database..."

mkdir ~/frappe-bench/sites/test_site
cp "${GITHUB_WORKSPACE}/.github/helper/consumer_db/$DB.json" ~/frappe-bench/sites/test_site/site_config.json

if [ "$TYPE" == "server" ]; then
mkdir ~/frappe-bench/sites/test_site_producer;
cp "${GITHUB_WORKSPACE}/.github/helper/producer_db/$DB.json" ~/frappe-bench/sites/test_site_producer/site_config.json;
mkdir ~/frappe-bench/sites/test_site_producer;
cp "${GITHUB_WORKSPACE}/.github/helper/producer_db/$DB.json" ~/frappe-bench/sites/test_site_producer/site_config.json;
fi

if [ "$DB" == "mariadb" ];then
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL character_set_server = 'utf8mb4'";
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'";
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL character_set_server = 'utf8mb4'";
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'";

mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe_consumer";
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe_consumer'@'localhost' IDENTIFIED BY 'test_frappe_consumer'";
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe_consumer\`.* TO 'test_frappe_consumer'@'localhost'";
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe_consumer";
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe_consumer'@'localhost' IDENTIFIED BY 'test_frappe_consumer'";
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe_consumer\`.* TO 'test_frappe_consumer'@'localhost'";

mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe_producer";
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe_producer'@'localhost' IDENTIFIED BY 'test_frappe_producer'";
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe_producer\`.* TO 'test_frappe_producer'@'localhost'";

mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "FLUSH PRIVILEGES";
fi
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe_producer";
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe_producer'@'localhost' IDENTIFIED BY 'test_frappe_producer'";
mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe_producer\`.* TO 'test_frappe_producer'@'localhost'";

mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "FLUSH PRIVILEGES";
fi
if [ "$DB" == "postgres" ];then
echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_consumer" -U postgres;
echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_consumer WITH PASSWORD 'test_frappe'" -U postgres;
echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_consumer" -U postgres;
echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_consumer WITH PASSWORD 'test_frappe'" -U postgres;

echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_producer" -U postgres;
echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_producer WITH PASSWORD 'test_frappe'" -U postgres;
echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_producer" -U postgres;
echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_producer WITH PASSWORD 'test_frappe'" -U postgres;
fi

cd ./frappe-bench || exit
echo "Setting Up Procfile..."

sed -i 's/^watch:/# watch:/g' Procfile
sed -i 's/^schedule:/# schedule:/g' Procfile
if [ "$TYPE" == "server" ]; then
sed -i 's/^socketio:/# socketio:/g' Procfile;
sed -i 's/^redis_socketio:/# redis_socketio:/g' Procfile;
fi
if [ "$TYPE" == "ui" ]; then
sed -i 's/^web: bench serve/web: bench serve --with-coverage/g' Procfile;
fi

if [ "$TYPE" == "server" ]; then sed -i 's/^socketio:/# socketio:/g' Procfile; fi
if [ "$TYPE" == "server" ]; then sed -i 's/^redis_socketio:/# redis_socketio:/g' Procfile; fi

if [ "$TYPE" == "ui" ]; then bench -v setup requirements --node; fi
bench -v setup requirements --dev

if [ "$TYPE" == "ui" ]; then sed -i 's/^web: bench serve/web: bench serve --with-coverage/g' Procfile; fi
echo "Starting Bench..."

bench start &> bench_start.log &
bench --site test_site reinstall --yes

if [ "$TYPE" == "server" ]; then bench --site test_site_producer reinstall --yes; fi
if [ "$TYPE" == "server" ]; then CI=Yes bench build --app frappe; fi
if [ "$TYPE" == "server" ]; then
bench --site test_site_producer reinstall --yes;
CI=Yes bench build --app frappe;
fi

+ 5
- 2
.github/helper/install_dependencies.sh Целия файл

@@ -3,8 +3,11 @@ set -e

echo "Setting Up System Dependencies..."

wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb
sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb
install_wkhtmltopdf() {
wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb
sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb
}
install_wkhtmltopdf &

curl -LsS -O https://downloads.mariadb.com/MariaDB/mariadb_repo_setup
sudo bash mariadb_repo_setup --mariadb-server-version=10.6


+ 4
- 11
.github/helper/translation.py Целия файл

@@ -20,19 +20,12 @@ for _file in files_to_scan:
if 'frappe-lint: disable-translate' in line:
continue

start_matches = start_pattern.search(line)
if start_matches:
starts_with_f = starts_with_f_pattern.search(line)

if starts_with_f:
has_f_string = f_string_pattern.search(line)
if has_f_string:
if start_matches := start_pattern.search(line):
if starts_with_f := starts_with_f_pattern.search(line):
if has_f_string := f_string_pattern.search(line):
errors_encounter += 1
print(f'\nF-strings are not supported for translations at line number {line_number}\n{line.strip()[:100]}')
continue
else:
continue

continue
match = pattern.search(line)
error_found = False



.github/workflows/release.yml → .github/workflows/create-release.yml Целия файл


+ 0
- 22
.github/workflows/deps-checker.yml Целия файл

@@ -1,22 +0,0 @@
name: 'Python Dependency Check'
on:
pull_request:
workflow_dispatch:
push:
branches: [ develop ]

permissions:
contents: read

jobs:
deps-vulnerable-check:
name: 'Vulnerable Dependency'
runs-on: ubuntu-latest

steps:
- uses: actions/setup-python@v4
with:
python-version: '3.10'
- uses: actions/checkout@v3
- run: pip install pip-audit
- run: pip-audit ${GITHUB_WORKSPACE}

+ 0
- 20
.github/workflows/docker-release.yml Целия файл

@@ -1,20 +0,0 @@
name: 'Trigger Docker build on release'
on:
release:
types: [released]
permissions:
contents: read

jobs:
curl:
permissions:
contents: none
name: 'Trigger Docker build on release'
runs-on: ubuntu-latest
container:
image: alpine:latest
steps:
- name: curl
run: |
apk add curl bash
curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${{ secrets.CI_PAT }}" https://api.github.com/repos/frappe/frappe_docker/actions/workflows/build_stable.yml/dispatches -d '{"ref":"main"}'

+ 0
- 28
.github/workflows/docs-checker.yml Целия файл

@@ -1,28 +0,0 @@
name: 'Documentation Check'
on:
pull_request:
types: [ opened, synchronize, reopened, edited ]

permissions:
contents: read

jobs:
docs-required:
name: 'Documentation Required'
runs-on: ubuntu-latest

steps:
- name: 'Setup Environment'
uses: actions/setup-python@v4
with:
python-version: '3.10'

- name: 'Clone repo'
uses: actions/checkout@v3

- name: Validate Docs
env:
PR_NUMBER: ${{ github.event.number }}
run: |
pip install requests --quiet
python $GITHUB_WORKSPACE/.github/helper/documentation.py $PR_NUMBER

+ 66
- 11
.github/workflows/linters.yml Целия файл

@@ -1,29 +1,84 @@
name: Linters

on:
pull_request: { }
pull_request:
workflow_dispatch:
push:
branches: [ develop ]

jobs:
permissions:
contents: read

concurrency:
group: commitcheck-frappe-${{ github.event.number }}
cancel-in-progress: true

linters:
name: Frappe Linter
jobs:
commit-lint:
name: 'Semantic Commits'
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'

steps:
- uses: actions/checkout@v3
with:
fetch-depth: 200
- uses: actions/setup-node@v3
with:
node-version: 16
check-latest: true

- name: Check commit titles
run: |
npm install @commitlint/cli @commitlint/config-conventional
npx commitlint --verbose --from ${{ github.event.pull_request.base.sha }} --to ${{ github.event.pull_request.head.sha }}

- name: Set up Python
docs-required:
name: 'Documentation Required'
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'

steps:
- name: 'Setup Environment'
uses: actions/setup-python@v4
with:
python-version: '3.10'
- uses: actions/checkout@v3

- name: Install and Run Pre-commit
uses: pre-commit/action@v3.0.0
- name: Validate Docs
env:
PR_NUMBER: ${{ github.event.number }}
run: |
pip install requests --quiet
python $GITHUB_WORKSPACE/.github/helper/documentation.py $PR_NUMBER

linter:
name: 'Frappe Linter'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.10'
- uses: pre-commit/action@v3.0.0

- name: Download Semgrep rules
run: git clone --depth 1 https://github.com/frappe/semgrep-rules.git frappe-semgrep-rules

- name: Download semgrep
run: pip install semgrep==0.97.0

- name: Run Semgrep rules
run: semgrep ci --config ./frappe-semgrep-rules/rules --config r/python.lang.correctness
run: |
pip install semgrep==0.97.0
semgrep ci --config ./frappe-semgrep-rules/rules --config r/python.lang.correctness

deps-vulnerable-check:
name: 'Vulnerable Dependency Check'
runs-on: ubuntu-latest

steps:
- uses: actions/setup-python@v4
with:
python-version: '3.10'
- uses: actions/checkout@v3
- run: |
pip install pip-audit
pip-audit ${GITHUB_WORKSPACE}

.github/workflows/publish-assets-releases.yml → .github/workflows/on_release.yml Целия файл

@@ -1,8 +1,11 @@
name: 'Frappe Assets'
name: 'Release'

on:
release:
types: [ created ]
types: [released]

permissions:
contents: read

env:
GITHUB_TOKEN: ${{ github.token }}
@@ -47,3 +50,16 @@ jobs:
asset_path: build/assets.tar.gz
asset_name: assets.tar.gz
asset_content_type: application/octet-stream

docker-release:
name: 'Trigger Docker build on release'
runs-on: ubuntu-latest
permissions:
contents: none
container:
image: alpine:latest
steps:
- name: curl
run: |
apk add curl bash
curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${{ secrets.CI_PAT }}" https://api.github.com/repos/frappe/frappe_docker/actions/workflows/build_stable.yml/dispatches -d '{"ref":"main"}'

+ 5
- 5
.github/workflows/patch-mariadb-tests.yml Целия файл

@@ -1,7 +1,8 @@
name: Patch

on: [pull_request, workflow_dispatch]
name: Server (MariaDB)

on:
pull_request:
workflow_dispatch:

concurrency:
group: patch-mariadb-develop-${{ github.event.number }}
@@ -12,11 +13,10 @@ permissions:

jobs:
test:
name: Patch
runs-on: ubuntu-latest
timeout-minutes: 60

name: Patch Test

services:
mariadb:
image: mariadb:10.6


+ 1
- 0
.github/workflows/publish-assets-develop.yml Целия файл

@@ -1,6 +1,7 @@
name: 'Frappe Assets'

on:
workflow_dispatch:
push:
branches: [ develop ]



+ 0
- 30
.github/workflows/semantic-commits.yml Целия файл

@@ -1,30 +0,0 @@
name: Semantic Commits

on:
pull_request: {}

permissions:
contents: read

concurrency:
group: commitcheck-frappe-${{ github.event.number }}
cancel-in-progress: true

jobs:
commitlint:
name: Check Commit Titles
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 200

- uses: actions/setup-node@v3
with:
node-version: 16
check-latest: true

- name: Check commit titles
run: |
npm install @commitlint/cli @commitlint/config-conventional
npx commitlint --verbose --from ${{ github.event.pull_request.base.sha }} --to ${{ github.event.pull_request.head.sha }}

+ 2
- 3
.github/workflows/server-mariadb-tests.yml Целия файл

@@ -1,4 +1,4 @@
name: Server
name: Server (MariaDB)

on:
pull_request:
@@ -16,6 +16,7 @@ permissions:

jobs:
test:
name: Unit Tests
runs-on: ubuntu-latest
timeout-minutes: 60

@@ -24,8 +25,6 @@ jobs:
matrix:
container: [1, 2]

name: Python Unit Tests (MariaDB)

services:
mariadb:
image: mariadb:10.6


+ 2
- 3
.github/workflows/server-postgres-tests.yml Целия файл

@@ -1,4 +1,4 @@
name: Server
name: Server (Postgres)

on:
pull_request:
@@ -15,6 +15,7 @@ permissions:

jobs:
test:
name: Unit Tests
runs-on: ubuntu-latest
timeout-minutes: 60

@@ -23,8 +24,6 @@ jobs:
matrix:
container: [1, 2]

name: Python Unit Tests (Postgres)

services:
postgres:
image: postgres:12.4


Зареждане…
Отказ
Запис