From e515794901596e0c5afef8d8ffd326e793322dc2 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Sat, 1 Nov 2025 22:47:54 +1100 Subject: [PATCH] [#1560] Moved post build test to PHPUnit. --- .circleci/config.yml | 16 +- .vortex/tests/README.md | 4 - .vortex/tests/bats/e2e/README.md | 7 - .vortex/tests/bats/e2e/circleci.bats | 72 ------- .../phpunit/Functional/PostBuildTest.php | 126 +++++++++++ .../tests/phpunit/Traits/CircleCiTrait.php | 201 ++++++++++++++++++ .vortex/tests/test.postbuild.sh | 31 --- 7 files changed, 337 insertions(+), 120 deletions(-) delete mode 100644 .vortex/tests/bats/e2e/README.md delete mode 100644 .vortex/tests/bats/e2e/circleci.bats create mode 100644 .vortex/tests/phpunit/Functional/PostBuildTest.php create mode 100644 .vortex/tests/phpunit/Traits/CircleCiTrait.php delete mode 100755 .vortex/tests/test.postbuild.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 0e5ecce99..8002ac87e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -537,8 +537,16 @@ jobs: ahoy --version - run: - name: Run CircleCI tests (long) - command: VORTEX_DEV_VOLUMES_SKIP_MOUNT=1 VORTEX_DEV_TEST_COVERAGE_DIR=/tmp/artifacts/coverage .vortex/tests/test.postbuild.sh + name: Install test dependencies + command: | + cd .vortex/tests + composer install --no-interaction --prefer-dist + + - run: + name: Run CircleCI post-build tests + command: | + cd .vortex/tests + ./vendor/bin/phpunit --group=postbuild - store_test_results: path: *test_results @@ -546,10 +554,6 @@ jobs: - store_artifacts: path: *artifacts - - run: - name: Upload code coverage reports to Codecov - command: codecov -Z -s /tmp/artifacts/coverage - #----------------------------------------------------------------------------- # Launching and testing databases stored within Docker data image. #----------------------------------------------------------------------------- diff --git a/.vortex/tests/README.md b/.vortex/tests/README.md index 0544d4ed2..7b8217f87 100644 --- a/.vortex/tests/README.md +++ b/.vortex/tests/README.md @@ -71,9 +71,6 @@ For parallel execution, tests can be run across multiple CI nodes using the convenience script wrappers: - [`test.common.sh]`(test.common.sh) - Common tests for all environments -- [`test.deployment.sh`](test.deployment.sh) - Deployment tests -- [`test.postbuild.sh`](test.postbuild.sh) - Post-build tests -- [`test.workflow.sh`](test.workflow.sh) - Workflow tests - [`lint.scripts.sh`](lint.scripts.sh) - Linting for shell scripts - [`lint.dockerfiles.sh`](lint.dockerfiles.sh) - Linting for Dockerfiles @@ -112,4 +109,3 @@ phpunit/ ├── SubtestAhoyTrait.php # Steps and assertions for testing Ahoy-based workflows └── SubtestDockerComposeTrait.php # Steps and assertions for Docker Compose-based workflows ``` - diff --git a/.vortex/tests/bats/e2e/README.md b/.vortex/tests/bats/e2e/README.md deleted file mode 100644 index 7c06e9174..000000000 --- a/.vortex/tests/bats/e2e/README.md +++ /dev/null @@ -1,7 +0,0 @@ -Tests in this directory are end-to-end tests for Vortex, written using the Bats -testing framework. - -These are being ported to PHPUnit tests. -See https://github.com/drevops/vortex/issues/1560 - -BATS will be used only for unit testing of scripts. diff --git a/.vortex/tests/bats/e2e/circleci.bats b/.vortex/tests/bats/e2e/circleci.bats deleted file mode 100644 index 5e530ce7b..000000000 --- a/.vortex/tests/bats/e2e/circleci.bats +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bats -# -# Test for CircleCI lifecycle. -# -# shellcheck disable=SC2030,SC2031,SC2129 - -load ../_helper.bash -load ../_helper.circleci.bash - -@test "CircleCI artifacts are saved" { - if [ -z "${CIRCLECI-}" ]; then - skip "This test is only run on CircleCI" - fi - - export TEST_CIRCLECI_TOKEN="${TEST_CIRCLECI_TOKEN?CircleCI token is not set}" - export CIRCLE_PROJECT_REPONAME="${CIRCLE_PROJECT_REPONAME?CircleCI project repo name is not set}" - export CIRCLE_PROJECT_USERNAME="${CIRCLE_PROJECT_USERNAME?CircleCI project username is not set}" - export CIRCLE_BUILD_NUM="${CIRCLE_BUILD_NUM?CircleCI build number is not set}" - - previous_job_numbers="$(circleci_get_previous_job_numbers "${CIRCLE_BUILD_NUM}")" - - for previous_job_number in ${previous_job_numbers}; do - artifacts_data="$(circleci_get_job_artifacts "${previous_job_number}")" - - artifact_path_runner_0="$(echo "${artifacts_data}" | jq -r '.items | map(select(.node_index == 0).path) | join("\n")')" - assert_contains "coverage/phpunit/cobertura.xml" "${artifact_path_runner_0}" - assert_contains "coverage/phpunit/.coverage-html/index.html" "${artifact_path_runner_0}" - - assert_contains "homepage.feature" "${artifact_path_runner_0}" - assert_contains "login.feature" "${artifact_path_runner_0}" - assert_contains "clamav.feature" "${artifact_path_runner_0}" - assert_not_contains "search.feature" "${artifact_path_runner_0}" - - artifact_path_runner_1="$(echo "${artifacts_data}" | jq -r '.items | map(select(.node_index == 1).path) | join("\n")')" - assert_contains "coverage/phpunit/cobertura.xml" "${artifact_path_runner_1}" - assert_contains "coverage/phpunit/.coverage-html/index.html" "${artifact_path_runner_1}" - - assert_contains "homepage.feature" "${artifact_path_runner_1}" - assert_contains "login.feature" "${artifact_path_runner_1}" - assert_not_contains "clamav.feature" "${artifact_path_runner_1}" - assert_contains "search.feature" "${artifact_path_runner_1}" - done -} - -@test "CircleCI test results are saved" { - if [ -z "${CIRCLECI-}" ]; then - skip "This test is only run on CircleCI" - fi - - export TEST_CIRCLECI_TOKEN="${TEST_CIRCLECI_TOKEN?CircleCI token is not set}" - export CIRCLE_PROJECT_REPONAME="${CIRCLE_PROJECT_REPONAME?CircleCI project repo name is not set}" - export CIRCLE_PROJECT_USERNAME="${CIRCLE_PROJECT_USERNAME?CircleCI project username is not set}" - export CIRCLE_BUILD_NUM="${CIRCLE_BUILD_NUM?CircleCI build number is not set}" - - previous_job_numbers="$(circleci_get_previous_job_numbers "${CIRCLE_BUILD_NUM}")" - - for previous_job_number in ${previous_job_numbers}; do - tests_data="$(circleci_get_job_test_metadata "${previous_job_number}")" - assert_contains "tests/phpunit/CircleCiConfigTest.php" "${tests_data}" - assert_contains "tests/phpunit/Drupal/DatabaseSettingsTest.php" "${tests_data}" - assert_contains "tests/phpunit/Drupal/EnvironmentSettingsTest.php" "${tests_data}" - assert_contains "tests/phpunit/Drupal/SwitchableSettingsTest.php" "${tests_data}" - assert_contains "web/modules/custom/ys_base/tests/src/Functional/ExampleTest.php" "${tests_data}" - assert_contains "web/modules/custom/ys_base/tests/src/Kernel/ExampleTest.php" "${tests_data}" - assert_contains "web/modules/custom/ys_base/tests/src/Unit/ExampleTest.php" "${tests_data}" - - assert_contains "homepage.feature" "${tests_data}" - assert_contains "login.feature" "${tests_data}" - assert_contains "clamav.feature" "${tests_data}" - assert_contains "search.feature" "${tests_data}" - done -} diff --git a/.vortex/tests/phpunit/Functional/PostBuildTest.php b/.vortex/tests/phpunit/Functional/PostBuildTest.php new file mode 100644 index 000000000..f6ff87931 --- /dev/null +++ b/.vortex/tests/phpunit/Functional/PostBuildTest.php @@ -0,0 +1,126 @@ +markTestSkipped('This test is only run on CircleCI'); + } + + // Verify required environment variables are set. + $this->assertNotEmpty(getenv('TEST_CIRCLECI_TOKEN'), 'CircleCI token is not set'); + $this->assertNotEmpty(getenv('CIRCLE_PROJECT_REPONAME'), 'CircleCI project repo name is not set'); + $this->assertNotEmpty(getenv('CIRCLE_PROJECT_USERNAME'), 'CircleCI project username is not set'); + $this->assertNotEmpty(getenv('CIRCLE_BUILD_NUM'), 'CircleCI build number is not set'); + + // Skip the parent setUp as we don't need to prepare SUT for these tests. + } + + /** + * {@inheritdoc} + */ + protected function tearDown(): void { + // Skip parent tearDown as we didn't set up SUT. + } + + /** + * Test that CircleCI artifacts are saved correctly. + * + * Verifies that: + * - PHPUnit coverage reports are generated for both parallel runners + * - Behat feature files are saved for both parallel runners + * - Feature files are split correctly between runners (e.g., clamav.feature + * on runner 0 but not runner 1, search.feature on runner 1 but not + * runner 0) + */ + #[Group('postbuild')] + public function testCircleCiArtifactsAreSaved(): void { + $currentJobNumber = (int) getenv('CIRCLE_BUILD_NUM'); + $previousJobNumbers = $this->circleCiGetPreviousJobNumbers($currentJobNumber); + + $this->assertNotEmpty($previousJobNumbers, 'No previous job numbers found'); + + foreach ($previousJobNumbers as $previousJobNumber) { + $artifactsData = $this->circleCiGetJobArtifacts($previousJobNumber); + + // Verify runner 0 artifacts. + $artifactPathsRunner0 = $this->circleCiExtractArtifactPaths($artifactsData, 0); + $artifactPathsRunner0Str = implode("\n", $artifactPathsRunner0); + + $this->assertStringContainsString('coverage/phpunit/cobertura.xml', $artifactPathsRunner0Str, 'Runner 0 should have PHPUnit cobertura coverage'); + $this->assertStringContainsString('coverage/phpunit/.coverage-html/index.html', $artifactPathsRunner0Str, 'Runner 0 should have PHPUnit HTML coverage'); + + $this->assertStringContainsString('homepage.feature', $artifactPathsRunner0Str, 'Runner 0 should have homepage.feature'); + $this->assertStringContainsString('login.feature', $artifactPathsRunner0Str, 'Runner 0 should have login.feature'); + $this->assertStringContainsString('clamav.feature', $artifactPathsRunner0Str, 'Runner 0 should have clamav.feature'); + $this->assertStringNotContainsString('search.feature', $artifactPathsRunner0Str, 'Runner 0 should NOT have search.feature'); + + // Verify runner 1 artifacts. + $artifactPathsRunner1 = $this->circleCiExtractArtifactPaths($artifactsData, 1); + $artifactPathsRunner1Str = implode("\n", $artifactPathsRunner1); + + $this->assertStringContainsString('coverage/phpunit/cobertura.xml', $artifactPathsRunner1Str, 'Runner 1 should have PHPUnit cobertura coverage'); + $this->assertStringContainsString('coverage/phpunit/.coverage-html/index.html', $artifactPathsRunner1Str, 'Runner 1 should have PHPUnit HTML coverage'); + + $this->assertStringContainsString('homepage.feature', $artifactPathsRunner1Str, 'Runner 1 should have homepage.feature'); + $this->assertStringContainsString('login.feature', $artifactPathsRunner1Str, 'Runner 1 should have login.feature'); + $this->assertStringNotContainsString('clamav.feature', $artifactPathsRunner1Str, 'Runner 1 should NOT have clamav.feature'); + $this->assertStringContainsString('search.feature', $artifactPathsRunner1Str, 'Runner 1 should have search.feature'); + } + } + + /** + * Test that CircleCI test results are saved correctly. + * + * Verifies that: + * - PHPUnit test results from various test suites are recorded + * - Behat feature test results are recorded. + */ + #[Group('postbuild')] + public function testCircleCiTestResultsAreSaved(): void { + $currentJobNumber = (int) getenv('CIRCLE_BUILD_NUM'); + $previousJobNumbers = $this->circleCiGetPreviousJobNumbers($currentJobNumber); + + $this->assertNotEmpty($previousJobNumbers, 'No previous job numbers found'); + + foreach ($previousJobNumbers as $previousJobNumber) { + $testsData = $this->circleCiGetJobTestMetadata($previousJobNumber); + $testPaths = $this->circleCiExtractTestPaths($testsData); + $testPathsStr = implode("\n", $testPaths); + + // Verify PHPUnit test results. + $this->assertStringContainsString('tests/phpunit/CircleCiConfigTest.php', $testPathsStr, 'Should have CircleCiConfigTest results'); + $this->assertStringContainsString('tests/phpunit/Drupal/DatabaseSettingsTest.php', $testPathsStr, 'Should have DatabaseSettingsTest results'); + $this->assertStringContainsString('tests/phpunit/Drupal/EnvironmentSettingsTest.php', $testPathsStr, 'Should have EnvironmentSettingsTest results'); + $this->assertStringContainsString('tests/phpunit/Drupal/SwitchableSettingsTest.php', $testPathsStr, 'Should have SwitchableSettingsTest results'); + $this->assertStringContainsString('web/modules/custom/ys_base/tests/src/Functional/ExampleTest.php', $testPathsStr, 'Should have custom module Functional test results'); + $this->assertStringContainsString('web/modules/custom/ys_base/tests/src/Kernel/ExampleTest.php', $testPathsStr, 'Should have custom module Kernel test results'); + $this->assertStringContainsString('web/modules/custom/ys_base/tests/src/Unit/ExampleTest.php', $testPathsStr, 'Should have custom module Unit test results'); + + // Verify Behat test results. + $this->assertStringContainsString('homepage.feature', $testPathsStr, 'Should have homepage.feature results'); + $this->assertStringContainsString('login.feature', $testPathsStr, 'Should have login.feature results'); + $this->assertStringContainsString('clamav.feature', $testPathsStr, 'Should have clamav.feature results'); + $this->assertStringContainsString('search.feature', $testPathsStr, 'Should have search.feature results'); + } + } + +} diff --git a/.vortex/tests/phpunit/Traits/CircleCiTrait.php b/.vortex/tests/phpunit/Traits/CircleCiTrait.php new file mode 100644 index 000000000..4d89404e1 --- /dev/null +++ b/.vortex/tests/phpunit/Traits/CircleCiTrait.php @@ -0,0 +1,201 @@ +circleCiApiRequest($url, $token); + $data = json_decode($response, TRUE, 512, JSON_THROW_ON_ERROR); + + return $data['latest_workflow']['id']; + } + + /** + * Get numbers of previous jobs that current job depends on. + * + * @param int $currentJobNumber + * The current job number. + * + * @return array + * Array of previous job numbers. + */ + protected function circleCiGetPreviousJobNumbers(int $currentJobNumber): array { + $token = getenv('TEST_CIRCLECI_TOKEN'); + $workflowId = $this->circleCiGetWorkflowIdFromJobNumber($currentJobNumber); + + $url = sprintf('https://circleci.com/api/v2/workflow/%s/job', $workflowId); + $response = $this->circleCiApiRequest($url, $token); + $workflowData = json_decode($response, TRUE, 512, JSON_THROW_ON_ERROR); + + // Find the current job and get its dependencies. + $dependenciesJobIds = []; + foreach ($workflowData['items'] as $item) { + if ($item['job_number'] == $currentJobNumber) { + $dependenciesJobIds = $item['dependencies'] ?? []; + break; + } + } + + // Map dependency IDs to job numbers. + $previousJobNumbers = []; + foreach ($dependenciesJobIds as $dependencyId) { + foreach ($workflowData['items'] as $item) { + if ($item['id'] === $dependencyId) { + $previousJobNumbers[] = $item['job_number']; + break; + } + } + } + + return $previousJobNumbers; + } + + /** + * Get artifacts for a job. + * + * @param int $jobNumber + * The job number. + * + * @return array + * Array of artifacts data. + */ + protected function circleCiGetJobArtifacts(int $jobNumber): array { + $token = getenv('TEST_CIRCLECI_TOKEN'); + $username = getenv('CIRCLE_PROJECT_USERNAME'); + $reponame = getenv('CIRCLE_PROJECT_REPONAME'); + + $url = sprintf( + 'https://circleci.com/api/v2/project/gh/%s/%s/%d/artifacts', + $username, + $reponame, + $jobNumber + ); + + $response = $this->circleCiApiRequest($url, $token); + + return json_decode($response, TRUE, 512, JSON_THROW_ON_ERROR); + } + + /** + * Get test metadata for a job. + * + * @param int $jobNumber + * The job number. + * + * @return array + * Array of test metadata. + */ + protected function circleCiGetJobTestMetadata(int $jobNumber): array { + $token = getenv('TEST_CIRCLECI_TOKEN'); + $username = getenv('CIRCLE_PROJECT_USERNAME'); + $reponame = getenv('CIRCLE_PROJECT_REPONAME'); + + $url = sprintf( + 'https://circleci.com/api/v2/project/gh/%s/%s/%d/tests', + $username, + $reponame, + $jobNumber + ); + + $response = $this->circleCiApiRequest($url, $token); + + return json_decode($response, TRUE, 512, JSON_THROW_ON_ERROR); + } + + /** + * Make a CircleCI API request. + * + * @param string $url + * The API URL. + * @param string $token + * The CircleCI API token. + * + * @return string + * The response body. + */ + protected function circleCiApiRequest(string $url, string $token): string { + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE); + curl_setopt($ch, CURLOPT_HTTPHEADER, [ + 'Circle-Token: ' . $token, + ]); + + $response = curl_exec($ch); + $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + curl_close($ch); + + if ($httpCode !== 200) { + throw new \RuntimeException(sprintf('CircleCI API request failed with HTTP code %d: %s', $httpCode, $response)); + } + + return $response; + } + + /** + * Extract artifact paths for a specific node index. + * + * @param array $artifactsData + * The artifacts data from CircleCI API. + * @param int $nodeIndex + * The node index (for parallel jobs). + * + * @return array + * Array of artifact paths. + */ + protected function circleCiExtractArtifactPaths(array $artifactsData, int $nodeIndex): array { + $paths = []; + foreach ($artifactsData['items'] as $item) { + if ($item['node_index'] === $nodeIndex) { + $paths[] = $item['path']; + } + } + + return $paths; + } + + /** + * Extract test file paths from test metadata. + * + * @param array $testsData + * The test metadata from CircleCI API. + * + * @return array + * Array of test file paths. + */ + protected function circleCiExtractTestPaths(array $testsData): array { + $paths = []; + foreach ($testsData['items'] as $item) { + $paths[] = $item['file']; + } + + return $paths; + } + +} diff --git a/.vortex/tests/test.postbuild.sh b/.vortex/tests/test.postbuild.sh deleted file mode 100755 index 02d475daa..000000000 --- a/.vortex/tests/test.postbuild.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -## -# Run Vortex CI post-build tests. -# -# LCOV_EXCL_START - -set -eu -[ "${VORTEX_DEBUG-}" = "1" ] && set -x - -ROOT_DIR="$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)")" - -SCRIPTS_DIR="${ROOT_DIR}/scripts/vortex" - -TEST_DIR="${ROOT_DIR}/.vortex/tests" - -# ------------------------------------------------------------------------------ - -[ ! -d "${TEST_DIR}/node_modules" ] && echo " > Install test Node dependencies." && yarn --cwd="${TEST_DIR}" install --frozen-lockfile - -bats() { - pushd "${ROOT_DIR}" >/dev/null || exit 1 - if [ -n "${VORTEX_DEV_TEST_COVERAGE_DIR:-}" ]; then - mkdir -p "${VORTEX_DEV_TEST_COVERAGE_DIR}" - kcov --include-pattern=.sh,.bash --bash-parse-files-in-dir="${SCRIPTS_DIR}","${TEST_DIR}" --exclude-pattern=vendor,node_modules "${VORTEX_DEV_TEST_COVERAGE_DIR}" "${TEST_DIR}/node_modules/.bin/bats" "$@" - else - "${TEST_DIR}/node_modules/.bin/bats" "$@" - fi - popd >/dev/null || exit 1 -} - -bats "${TEST_DIR}/bats/e2e/circleci.bats"