feat: Add support for build combined coverage file

- from parallel tests
- Update workflow files

.github/workflows/ci-tests.yml

@@ -1,171 +0,0 @@
-name: CI
-
-on:
-  pull_request:
-    types: [opened, synchronize, reopened, labeled, unlabeled]
-  workflow_dispatch:
-  push:
-
-jobs:
-  test:
-    runs-on: ubuntu-18.04
-
-    strategy:
-      fail-fast: false
-      matrix:
-       include:
-        - DB: "mariadb"
-          TYPE: "server"
-          JOB_NAME: "Python MariaDB"
-          RUN_COMMAND: bench --site test_site run-tests --coverage
-
-        - DB: "postgres"
-          TYPE: "server"
-          JOB_NAME: "Python PostgreSQL"
-          RUN_COMMAND: bench --site test_site run-tests --coverage
-
-        - DB: "mariadb"
-          TYPE: "ui"
-          JOB_NAME: "UI MariaDB"
-          RUN_COMMAND:  bench --site test_site run-ui-tests frappe --headless
-
-    name: ${{ matrix.JOB_NAME }}
-
-    services:
-      mysql:
-        image: mariadb:10.3
-        env:
-          MYSQL_ALLOW_EMPTY_PASSWORD: YES
-        ports:
-          - 3306:3306
-        options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
-
-      postgres:
-        image: postgres:12.4
-        env:
-          POSTGRES_PASSWORD: travis
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
-
-    steps:
-      - name: Clone
-        uses: actions/checkout@v2
-
-      - name: Setup Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.7
-
-      - uses: actions/setup-node@v2
-        with:
-          node-version: '12'
-          check-latest: true
-
-      - name: Add to Hosts
-        run: |
-          echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
-          echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
-
-      - name: Cache pip
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-            ${{ runner.os }}-
-
-      - name: Cache node modules
-        uses: actions/cache@v2
-        env:
-          cache-name: cache-node-modules
-        with:
-          path: ~/.npm
-          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-build-${{ env.cache-name }}-
-            ${{ runner.os }}-build-
-            ${{ runner.os }}-
-
-      - name: Get yarn cache directory path
-        id: yarn-cache-dir-path
-        run: echo "::set-output name=dir::$(yarn cache dir)"
-
-      - uses: actions/cache@v2
-        id: yarn-cache
-        with:
-          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-yarn-
-
-      - name: Cache cypress binary
-        if: matrix.TYPE == 'ui'
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache
-          key: ${{ runner.os }}-cypress-
-          restore-keys: |
-            ${{ runner.os }}-cypress-
-            ${{ runner.os }}-
-
-      - name: Install Dependencies
-        run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
-        env:
-          BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
-          AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
-          TYPE: ${{ matrix.TYPE }}
-
-      - name: Install
-        run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
-        env:
-          DB: ${{ matrix.DB }}
-          TYPE: ${{ matrix.TYPE }}
-
-      - name: Run Set-Up
-        if: matrix.TYPE == 'ui'
-        run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard
-        env:
-          DB: ${{ matrix.DB }}
-          TYPE: ${{ matrix.TYPE }}
-
-      - name: Setup tmate session
-        if: contains(github.event.pull_request.labels.*.name, 'debug-gha')
-        uses: mxschmitt/action-tmate@v3
-
-      - name: Run Tests
-        run: cd ~/frappe-bench/ && ${{ matrix.RUN_COMMAND }}
-        env:
-          DB: ${{ matrix.DB }}
-          TYPE: ${{ matrix.TYPE }}
-
-      - name: Coverage - Pull Request
-        if: matrix.TYPE == 'server' && github.event_name == 'pull_request'
-        run: |
-          cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE}
-          cd ${GITHUB_WORKSPACE}
-          pip install coveralls==2.2.0
-          pip install coverage==4.5.4
-          coveralls --service=github
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
-          COVERALLS_SERVICE_NAME: github
-          
-      - name: Coverage - Push
-        if: matrix.TYPE == 'server' && github.event_name == 'push'
-        run: |
-          cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE}
-          cd ${GITHUB_WORKSPACE}
-          pip install coveralls==2.2.0
-          pip install coverage==4.5.4
-          coveralls --service=github-actions
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
-          COVERALLS_SERVICE_NAME: github-actions

.github/workflows/server-mariadb-tests.yml

@@ -0,0 +1,123 @@
+name: Server Mariadb tests
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-18.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+       containers: [1, 2]
+
+    name: Server Mariadb
+
+    services:
+      mysql:
+        image: mariadb:10.3
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: YES
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+          check-latest: true
+
+      - name: Add to Hosts
+        run: |
+          echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
+          echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
+
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+
+      - uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Install Dependencies
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
+        env:
+          BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
+          AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
+          TYPE: server
+
+      - name: Install
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
+        env:
+          DB: mariadb
+          TYPE: server
+
+      - name: Setup tmate session
+        if: contains(github.event.pull_request.labels.*.name, 'debug-gha')
+        uses: mxschmitt/action-tmate@v3
+
+      - name: Run Tests
+        run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --with-coverage --ci-build-id $GITHUB_RUN_NUMBER
+
+      # - name: Coverage - Pull Request
+      #   if: github.event_name == 'pull_request'
+      #   run: |
+      #     cp ~/frappe-bench/sites/combined_coverage ${GITHUB_WORKSPACE}
+      #     cd ${GITHUB_WORKSPACE}
+      #     if [ -f "./combined_coverage" ]; pip install coveralls==2.2.0 && pip install coverage==4.5.4; fi
+      #     if [ -f "./combined_coverage" ]; coveralls --service=github --source ./combined_coverage; fi
+      #     coveralls --service=github
+      #   env:
+      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      #     COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
+      #     COVERALLS_SERVICE_NAME: github
+
+      # - name: Coverage - Push
+      #   if: github.event_name == 'push'
+      #   run: |
+      #     cp ~/frappe-bench/sites/combined_coverage ${GITHUB_WORKSPACE}/.coverage
+      #     cd ${GITHUB_WORKSPACE}
+      #     pip install coveralls==2.2.0
+      #     pip install coverage==4.5.4
+      #     coveralls --service=github-actions
+      #   env:
+      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      #     COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
+      #     COVERALLS_SERVICE_NAME: github-actions

.github/workflows/server-postgres-tests.yml

@@ -0,0 +1,97 @@
+name: Server PostgreSQL tests
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-18.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+       containers: [1, 2]
+
+    name: Server PostgreSQL
+
+    services:
+      postgres:
+        image: postgres:12.4
+        env:
+          POSTGRES_PASSWORD: travis
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+          check-latest: true
+
+      - name: Add to Hosts
+        run: |
+          echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
+          echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
+
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+
+      - uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Install Dependencies
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
+        env:
+          BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
+          AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
+          TYPE: server
+
+      - name: Install
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
+        env:
+          DB: postgres
+          TYPE: server
+
+      - name: Run Tests
+        run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --ci-build-id $GITHUB_RUN_NUMBER

frappe/commands/utils.py

@@ -554,12 +554,13 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal
 
 @click.command('run-parallel-tests')
 @click.option('--app', help="For App", default='frappe')
-@click.option('--build-id', help="For App")
+@click.option('--ci-build-id', help="CI Build ID")
+@click.option('--with-coverage', is_flag=True, help="Build coverage file")
 @pass_context
-def run_parallel_tests(context, app, build_id):
+def run_parallel_tests(context, app, ci_build_id, with_coverage):
 	from frappe.test_runner import ParallelTestRunner
 	site = get_site(context)
-	ParallelTestRunner(app, site=site, build_id=build_id)
+	ParallelTestRunner(app, site=site, ci_build_id=ci_build_id, with_coverage=with_coverage)
 
 @click.command('run-ui-tests')
 @click.argument('app')

frappe/test_runner.py

@@ -492,13 +492,14 @@ def get_all_tests():
 	return test_file_list
 
 class ParallelTestRunner():
-	def __init__(self, app, site, build_id, instance_id=None):
+	def __init__(self, app, site, ci_build_id, ci_instance_id=None, with_coverage=False):
 		self.app = app
 		self.site = site
-		self.orchestrator_url = 'http://localhost:3000'
-		self.build_id = build_id or '12321'
+		self.orchestrator_url = 'https://8b52f89a8c13.ngrok.io'
+		self.ci_build_id = ci_build_id
+		self.with_coverage = with_coverage
 		self.setup_test_site()
-		self.instance_id = instance_id or frappe.generate_hash(length=10)
+		self.ci_instance_id = ci_instance_id or frappe.generate_hash(length=10)
 		frappe.flags.in_test = True
 		self.start_test()
 
@@ -520,16 +521,20 @@ class ParallelTestRunner():
 		self.test_result = PrettyPrintResult(stream=Writeln(sys.stderr), descriptions=True, verbosity=2)
 		self.test_status = 'ongoing'
 
+		self.setup_coverage()
 		while self.test_status == 'ongoing':
 			self.run_tests_for_file(self.get_next_test())
 
 		self.print_result()
+		self.call_orchestrator('test-completed')
+		self.submit_coverage()
 
 	def register_instance(self):
 		test_spec_list = get_all_tests()
-		self.call_orchestrator('init-test', data={
+		response_data = self.call_orchestrator('init-test', data={
 			'test_spec_list': test_spec_list
 		})
+		self.is_master = response_data.get('is_master')
 
 	def get_next_test(self):
 		response_data = self.call_orchestrator('get-next-test')
@@ -570,47 +575,88 @@ class ParallelTestRunner():
 		self.test_result.printErrors()
 		click.echo(self.test_result)
 
-	def call_orchestrator(self, endpoint, data={}):
+	def call_orchestrator(self, endpoint, data={}, files={}):
 		# add repo token header
 		# build id in header
-		res = requests.get(f'{self.orchestrator_url}/{endpoint}', data=data, headers={
-			'CI-BUILD-ID': self.build_id,
-			'CI-INSTANCE-ID': self.instance_id,
+		headers = {
+			'CI-BUILD-ID': self.ci_build_id,
+			'CI-INSTANCE-ID': self.ci_instance_id,
 			'REPO-TOKEN': '2948288382838DE'
-		})
+		}
+		url = f'{self.orchestrator_url}/{endpoint}'
+
+		if files:
+			res = requests.post(url, headers=headers, files=files)
+		else:
+			res = requests.get(url, data=data, headers=headers)
+		print(self.ci_build_id, self.ci_instance_id, endpoint)
 		res.raise_for_status()
-		return res.json() if 'application/json' in res.headers.get('content-type') else {}
-
-	# def setup_coverage(self):
-	# 	if self.with_coverage:
-	# 		from coverage import Coverage
-	# 		from frappe.utils import get_bench_path
-
-	# 		# Generate coverage report only for app that is being tested
-	# 		source_path = os.path.join(get_bench_path(), 'apps', self.app)
-	# 		omit=[
-	# 			'*.html',
-	# 			'*.js',
-	# 			'*.xml',
-	# 			'*.css',
-	# 			'*.less',
-	# 			'*.scss',
-	# 			'*.vue',
-	# 			'*/doctype/*/*_dashboard.py',
-	# 			'*/patches/*'
-	# 		]
-
-	# 		if self.app == 'frappe':
-	# 			omit.append('*/commands/*')
-
-	# 		self.coverage = Coverage(source=[source_path], omit=omit, data_file='coverage_report')
-	# 		self.cov.start()
-
-	# def submit_coverage(self):
-	# 	if self.with_coverage:
-	# 		self.cov.stop()
-
-	# 	if self.is_master:
-	# 		pass
-
-# [] coverage
+		response_data = {}
+		if 'application/json' in res.headers.get('content-type'):
+			response_data = res.json()
+		elif 'application/zip' in res.headers.get('content-type'):
+			response_data = res.content
+
+		return response_data
+
+	def setup_coverage(self):
+		if self.with_coverage:
+			from coverage import Coverage
+			from frappe.utils import get_bench_path
+
+			# Generate coverage report only for app that is being tested
+			source_path = os.path.join(get_bench_path(), 'apps', self.app)
+			omit=[
+				'*.html',
+				'*.js',
+				'*.xml',
+				'*.css',
+				'*.less',
+				'*.scss',
+				'*.vue',
+				'*/doctype/*/*_dashboard.py',
+				'*/patches/*'
+			]
+
+			if self.app == 'frappe':
+				omit.append('*/commands/*')
+
+			self.coverage = Coverage(
+				source=[source_path],
+				omit=omit,
+				data_file='coverage_data',
+				data_suffix=self.ci_instance_id
+			)
+			self.coverage.start()
+
+	def submit_coverage(self):
+		if self.with_coverage:
+			self.coverage.stop()
+			self.coverage.save()
+
+		if self.is_master:
+			self.build_coverage_file()
+		else:
+			self.upload_coverage_file()
+
+
+	def upload_coverage_file(self):
+		files = {'upload_file': open(f'coverage_data.{self.ci_instance_id}','rb')}
+		self.call_orchestrator('upload-coverage-file', files=files)
+
+
+	def build_coverage_file(self):
+		import time
+		import zipfile
+		import io
+		click.echo()
+		while self.call_orchestrator('test-status')['test_status'] == 'ongoing':
+			click.echo('Waiting for tests to complete...')
+			time.sleep(5)
+
+		res = self.call_orchestrator('download-coverage-files')
+		z = zipfile.ZipFile(io.BytesIO(res))
+		z.extractall("./coverage_files")
+		file_list = os.listdir('./coverage_files')
+
+		self.coverage.combine(data_paths=file_list)