diff --git a/.github/workflows/pack_publish.yml b/.github/workflows/pack_publish.yml index 6cb32d7008..e65db89037 100644 --- a/.github/workflows/pack_publish.yml +++ b/.github/workflows/pack_publish.yml @@ -34,6 +34,8 @@ jobs: check_branch: name: Check branch existence runs-on: ubuntu-22.04 + needs: [pack, check_deps] + if: needs.pack.outputs.should_continue == 'true' outputs: columnar_locator: ${{ steps.set_locator.outputs.columnar_locator }} steps: @@ -185,487 +187,10 @@ jobs: - name: Run dependency check script run: | bash dist/check_deps_in_repos.sh --target ${{ needs.pack.outputs.target }} - - pack_debian_ubuntu: - name: Debian/Ubuntu packages - uses: ./.github/workflows/build_template.yml - needs: [pack, check_branch, check_deps] - if: needs.pack.outputs.should_continue == 'true' - strategy: - fail-fast: false - matrix: - DISTR: [bionic, focal, jammy, bullseye, bookworm] - arch: [x86_64, aarch64] - with: - COLUMNAR_LOCATOR: ${{ needs.check_branch.outputs.columnar_locator }} - DISTR: ${{ matrix.DISTR }} - arch: ${{ matrix.arch }} - cmake_command: | - mkdir build - cd build - export BUNDLE_REPO_TYPE=${{ needs.pack.outputs.target == 'release' && 'release_candidate' || 'dev' }} - cmake -DPACK=1 -DPACK_BUNDLE=1 .. - cmake --build . --target package - cache_key: pack_${{ matrix.DISTR }}_${{ matrix.arch }} - artifact_list: "build/manticore*deb" - version: ${{ needs.pack.outputs.version }} - build_tag: ${{ needs.pack.outputs.target == 'dev' && 'dev' || '' }} - - pack_rhel: - name: RHEL packages - uses: ./.github/workflows/build_template.yml - needs: [pack, check_branch, check_deps] - if: needs.pack.outputs.should_continue == 'true' - strategy: - fail-fast: false - matrix: - DISTR: [rhel8, rhel9, rhel10] - arch: [x86_64, aarch64] - with: - COLUMNAR_LOCATOR: ${{ needs.check_branch.outputs.columnar_locator }} - DISTR: ${{ matrix.DISTR }} - arch: ${{ matrix.arch }} - boost_url_key: boost_rhel_feb17 - cmake_command: | - ln -s $(pwd) /builds_manticoresearch_dev_usr_src_debug_manticore_component_src_0_0 - cd /builds_manticoresearch_dev_usr_src_debug_manticore_component_src_0_0 - mkdir build - cd build - export BUNDLE_REPO_TYPE=${{ needs.pack.outputs.target == 'release' && 'release_candidate' || 'dev' }} - cmake -DPACK=1 -DPACK_BUNDLE=1 .. - cmake --build . --target package - cache_key: pack_${{ matrix.DISTR }}_${{ matrix.arch }} - artifact_list: "build/manticore*rpm" - version: ${{ needs.pack.outputs.version }} - build_tag: ${{ needs.pack.outputs.target == 'dev' && 'dev' || '' }} - - pack_macos: - name: MacOS packages - uses: ./.github/workflows/build_template.yml - needs: [pack, check_branch, check_deps] - if: needs.pack.outputs.should_continue == 'true' - strategy: - fail-fast: false - matrix: - DISTR: [macos] - arch: [x86_64, arm64] - with: - COLUMNAR_LOCATOR: ${{ needs.check_branch.outputs.columnar_locator }} - DISTR: ${{ matrix.DISTR }} - arch: ${{ matrix.arch }} - HOMEBREW_PREFIX: ${{ matrix.arch == 'arm64' && '/opt/homebrew' || '/usr/local' }} - cmake_command: | - mkdir build - cd build - export BUNDLE_REPO_TYPE=${{ needs.pack.outputs.target == 'release' && 'release_candidate' || 'dev' }} - cmake -DPACK=1 -DPACK_BUNDLE=1 -DDL_CURL=1 \ - -DCURL_LIBRARY=${HOMEBREW_PREFIX}/opt/curl/lib/libcurl.dylib \ - -DCURL_LIBRARY_RELEASE=${HOMEBREW_PREFIX}/opt/curl/lib/libcurl.dylib .. - cmake --build . --target package - cache_key: pack_${{ matrix.DISTR }}_${{ matrix.arch }} - artifact_list: "build/manticore*tar.gz" - version: ${{ needs.pack.outputs.version }} - build_tag: ${{ needs.pack.outputs.target == 'dev' && 'dev' || '' }} - - pack_windows: - name: Windows x64 package - uses: ./.github/workflows/build_template.yml - needs: [pack, check_branch, check_deps] - if: needs.pack.outputs.should_continue == 'true' - with: - COLUMNAR_LOCATOR: ${{ needs.check_branch.outputs.columnar_locator }} - DISTR: windows - arch: x64 - sysroot_url_key: roots_mysql83_jan17 - boost_url_key: boost_80 - cmake_command: | - mkdir build - cd build - export BUNDLE_REPO_TYPE=${{ needs.pack.outputs.target == 'release' && 'release_candidate' || 'dev' }} - cmake -DPACK=1 -DPACK_BUNDLE=1 .. - cmake --build . --target package - cache_key: pack_windows_x64 - artifact_list: "build/manticore*exe build/manticore*zip" - version: ${{ needs.pack.outputs.version }} - build_tag: ${{ needs.pack.outputs.target == 'dev' && 'dev' || '' }} - - build_nsis: - name: Making Windows NSIS installer - needs: [pack_windows, pack] - if: needs.pack.outputs.should_continue == 'true' - runs-on: ubuntu-22.04 - container: - image: manticoresearch/build_nsis:1.0.0 - env: - CI_COMMIT_SHA: ${{ github.sha }} - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - name: Patch version - run: | - sed -i "s/0\.0\.0/${{ needs.pack.outputs.version }}/g" src/sphinxversion.h.in - - name: Initialization - # without adding the safe.directory the script fails to do git show ... - run: git config --global --add safe.directory /__w/manticoresearch/manticoresearch - - name: Make installer - # TODO: remove the hardcoded paths /builds/ below - run: | - mkdir -p /builds/manticoresearch/dev/build/ - /bin/bash dist/build_dockers/nsis/nsis_src_parser.sh ${{ needs.pack.outputs.target }} - shell: bash - - run: mv /builds/manticoresearch/dev/build build - - name: Upload artifact - uses: manticoresoftware/upload_artifact_with_retries@main - with: - name: win_installer - path: build/manticore-*.exe - - # virtual job to simplify the CI - publish: - name: OK to publish? - runs-on: ubuntu-22.04 - # This job depends on all the package preparation jobs that have to pass before we can start publishing packages - needs: [pack_debian_ubuntu, pack_rhel, pack_macos, build_nsis] - if: | - (github.repository == 'manticoresoftware/manticoresearch') - && ( - (github.event_name == 'pull_request' && (contains(github.event.pull_request.labels.*.name, 'publish'))) - || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') - || (github.event_name == 'push' && startsWith(github.ref, 'refs/heads/manticore-')) - || (github.event_name == 'push' && contains(github.ref, 'refs/tags/pack_publish')) - ) - steps: - - run: echo "Ready to publish" - - publish_debian_ubuntu: - needs: [publish, pack] - strategy: - fail-fast: true - matrix: - DISTR: [bionic, focal, jammy, bullseye, bookworm] - arch: [x86_64, aarch64] - runs-on: ubuntu-22.04 - name: ${{ matrix.DISTR }} ${{ matrix.arch }} publishing - steps: - - uses: manticoresoftware/publish_to_repo@main - with: - ssh_key: ${{ secrets.REPO_SSH_KEY }} - distr: ${{ matrix.DISTR }} - arch: ${{ matrix.arch }} - artifact: build_${{ matrix.DISTR }}_RelWithDebInfo_${{ matrix.arch }} - type: deb - delimiter: "-" - target: ${{ needs.pack.outputs.target }} - - publish_rhel: - needs: [publish, pack] - strategy: - fail-fast: true - matrix: - DISTR: [8, 9, 10] - arch: [x86_64, aarch64] - runs-on: ubuntu-22.04 - name: RHEL ${{ matrix.DISTR }} ${{ matrix.arch }} publishing - steps: - - uses: manticoresoftware/publish_to_repo@main - with: - ssh_key: ${{ secrets.REPO_SSH_KEY }} - distr: ${{ matrix.DISTR }} - arch: ${{ matrix.arch }} - artifact: build_rhel${{ matrix.DISTR }}_RelWithDebInfo_${{ matrix.arch }} - type: rpm - delimiter: "_" - target: ${{ needs.pack.outputs.target }} - - publish_macos: - name: Publishing MacOS - needs: [publish, pack] - strategy: - fail-fast: true - matrix: - arch: [x86_64, arm64] - runs-on: ubuntu-22.04 - steps: - - uses: manticoresoftware/publish_to_repo@main - with: - ssh_key: ${{ secrets.REPO_SSH_KEY }} - distr: macos - arch: ${{ matrix.arch }} - artifact: build_macos_RelWithDebInfo_${{ matrix.arch }} - type: arc - delimiter: "-" - target: ${{ needs.pack.outputs.target }} - - publish_windows: - name: Publishing Windows packages to repo.manticoresearch.com - needs: [publish, pack] - runs-on: ubuntu-22.04 - steps: - - uses: manticoresoftware/publish_to_repo@main - with: - ssh_key: ${{ secrets.REPO_SSH_KEY }} - distr: windows - arch: x64 - artifact: build_windows_RelWithDebInfo_x64 - type: arc - delimiter: "-" - target: ${{ needs.pack.outputs.target }} - - # Test Windows installer without Docker/Buddy on GitHub Actions - test_windows_installer: - name: Test Windows NSIS installer - needs: [publish_windows, pack] - runs-on: windows-latest - steps: - - name: Download Windows installer artifact - uses: actions/download-artifact@v4 - with: - name: win_installer - path: ./installer - - - name: Extract and Install Manticore - timeout-minutes: 5 - run: | - echo "=== EXTRACTING INSTALLER ===" - tar -xf ./installer/artifact.tar -C ./installer - - $installer = "./installer/build/manticore-*.exe" - $installerFile = Get-ChildItem $installer | Select-Object -First 1 - - if (-not $installerFile) { - echo "ERROR: Installer not found!" - Get-ChildItem ./installer -Recurse - exit 1 - } - - echo "=== INSTALLING MANTICORE ===" - echo "Installing from: $($installerFile.FullName)" - - $process = Start-Process -FilePath $installerFile.FullName -ArgumentList "/S /NOEXECUTOR" -Wait -PassThru -NoNewWindow - echo "Installation process exited with code: $($process.ExitCode)" - - # Wait for installation to complete - Start-Sleep -Seconds 10 - shell: pwsh - - - name: Test Manticore Service - run: | - echo "=== VERIFYING INSTALLATION ===" - - $installPath = "C:\Program Files (x86)\Manticore" - if (-not (Test-Path $installPath)) { - echo "ERROR: Manticore not installed at expected path: $installPath" - exit 1 - } - - echo "Manticore installed at: $installPath" - - $searchdPath = "$installPath\bin\searchd.exe" - $indexerPath = "$installPath\bin\indexer.exe" - - if (-not (Test-Path $searchdPath)) { - echo "ERROR: searchd.exe not found at: $searchdPath" - exit 1 - } - - if (-not (Test-Path $indexerPath)) { - echo "ERROR: indexer.exe not found at: $indexerPath" - exit 1 - } - - echo "searchd.exe found at: $searchdPath" - echo "indexer.exe found at: $indexerPath" - - & $searchdPath --version - - # Disable Buddy before starting Manticore - $configPath = "$installPath\etc\manticoresearch\manticore.conf" - if (Test-Path $configPath) { - $config = Get-Content $configPath -Raw - # Add buddy_path with empty value to disable it - $config = $config -replace "(searchd\s*\{)", "`$1`n buddy_path = " - Set-Content -Path $configPath -Value $config -Force - } - - echo "=== STARTING MANTICORE SERVICE ===" - - Push-Location $installPath - - $process = Start-Process -FilePath $searchdPath -PassThru -NoNewWindow - - # Smart wait for daemon to be ready - echo "Waiting for Manticore daemon to be ready..." - $maxAttempts = 30 - $attempt = 0 - $allPortsReady = $false - - while ($attempt -lt $maxAttempts -and -not $allPortsReady) { - Start-Sleep -Seconds 1 - $attempt++ - - # Check if process is still running - $manticoreProcess = Get-Process -Name "searchd" -ErrorAction SilentlyContinue - if (-not $manticoreProcess) { - echo "ERROR: Manticore searchd process died during startup" - exit 1 - } - - # Check all three ports - $port9306 = Test-NetConnection -ComputerName localhost -Port 9306 -WarningAction SilentlyContinue -InformationLevel Quiet - $port9308 = Test-NetConnection -ComputerName localhost -Port 9308 -WarningAction SilentlyContinue -InformationLevel Quiet - $port9312 = Test-NetConnection -ComputerName localhost -Port 9312 -WarningAction SilentlyContinue -InformationLevel Quiet - - if ($port9306 -and $port9308 -and $port9312) { - $allPortsReady = $true - echo "All ports are ready after $attempt seconds" - } - } - - if (-not $allPortsReady) { - echo "ERROR: Manticore did not start properly within $maxAttempts seconds" - echo "Port 9306 status: $port9306" - echo "Port 9308 status: $port9308" - echo "Port 9312 status: $port9312" - exit 1 - } - - $manticoreProcess = Get-Process -Name "searchd" -ErrorAction SilentlyContinue - echo "Manticore searchd is running (PID: $($manticoreProcess.Id))" - - Pop-Location - - echo "=== TESTING MANTICORE SERVICE WITH SIMPLE STATUS CHECKS ===" - - # Test 1: Check HTTP API port 9308 - echo "`nTest 1: Checking HTTP API (port 9308)..." - try { - $response = Invoke-WebRequest -Uri "http://localhost:9308/" -UseBasicParsing - if ($response.StatusCode -eq 200) { - echo "✓ HTTP API responds with status 200" - echo "✓ Response length: $($response.Content.Length) bytes" - } else { - echo "ERROR: Unexpected status code: $($response.StatusCode)" - exit 1 - } - } catch { - echo "ERROR: Failed to access HTTP API: $_" - exit 1 - } - - # Test 2: Check Sphinx/HTTP port 9312 - echo "`nTest 2: Checking Sphinx/HTTP port (9312)..." - try { - $response = Invoke-WebRequest -Uri "http://localhost:9312/" -UseBasicParsing - if ($response.StatusCode -eq 200) { - echo "✓ Sphinx/HTTP port responds with status 200" - # Check for expected content in response - if ($response.Content -like '*"version":*') { - echo "✓ Response contains version information" - } - } else { - echo "ERROR: Unexpected status code: $($response.StatusCode)" - exit 1 - } - } catch { - echo "ERROR: Failed to access port 9312: $_" - exit 1 - } - - # Test 3: Verify all three ports are listening - echo "`nTest 3: Verifying all service ports are listening..." - $ports = @(9306, 9308, 9312) - $allPortsOk = $true - - foreach ($port in $ports) { - $connection = Test-NetConnection -ComputerName localhost -Port $port -WarningAction SilentlyContinue -InformationLevel Quiet - if ($connection) { - echo "✓ Port $port is listening" - } else { - echo "✗ Port $port is NOT listening" - $allPortsOk = $false - } - } - - if (-not $allPortsOk) { - echo "ERROR: Not all required ports are listening" - exit 1 - } - - echo "" - echo "=== ALL BASIC TESTS COMPLETED SUCCESSFULLY ===" - echo "✓ Installation verified" - echo "✓ Daemon is running" - echo "✓ All ports (9306, 9308, 9312) are accessible" - echo "✓ HTTP endpoints respond with status 200" - echo "✓ Manticore service is operational" - shell: pwsh - - publish_nsis: - name: Publishing Windows NSIS installer - needs: [test_windows_installer, publish, pack] - runs-on: ubuntu-22.04 - steps: - - uses: manticoresoftware/publish_to_repo@main - with: - ssh_key: ${{ secrets.REPO_SSH_KEY }} - distr: - arch: - artifact: win_installer - type: arc - delimiter: "-" - target: ${{ needs.pack.outputs.target }} - - build_docker: - name: Building and pushing docker - needs: [publish_debian_ubuntu, pack] - runs-on: ubuntu-22.04 - env: - DOCKER_USER: ${{ secrets.DOCKER_USER }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - GHCR_USER: ${{ secrets.GHCR_USER }} - GHCR_PASSWORD: ${{ secrets.GHCR_PASSWORD }} - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - name: Patch version - run: | - sed -i "s/0\.0\.0/${{ needs.pack.outputs.version }}/g" src/sphinxversion.h.in - - name: Calculate short commit hash - id: sha - run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - image: tonistiigi/binfmt:qemu-v7.0.0-28 - - name: Building docker - run: CI_COMMIT_SHORT_SHA=${{ steps.sha.outputs.sha_short }} /bin/bash dist/dockerhub_deploy.sh - - clt_rhel_dev_installation: - name: Testing RHEL dev packages installation - needs: publish_rhel - strategy: - fail-fast: false - matrix: - image: [ "almalinux:8", "almalinux:9", "almalinux:10", "oraclelinux:9", "amazonlinux:latest" ] - runs-on: ubuntu-22.04 - steps: - - uses: manticoresoftware/clt@0.7.8 - with: - image: ${{ matrix.image }} - test_prefix: test/clt-tests/installation/rhel-dev- - run_args: -e TELEMETRY=0 - - clt_deb_dev_installation: - name: Testing DEB dev packages installation - needs: publish_debian_ubuntu - strategy: - fail-fast: false - matrix: - image: [ "ubuntu:bionic", "ubuntu:focal", "ubuntu:jammy", "debian:bullseye", "debian:bookworm" ] - runs-on: ubuntu-22.04 - steps: - - uses: manticoresoftware/clt@0.7.8 + - name: Upload dependency check debug artifact + if: failure() + uses: actions/upload-artifact@v4 with: - image: ${{ matrix.image }} - test_prefix: test/clt-tests/installation/deb-dev- - run_args: -e TELEMETRY=0 + name: check_deps_debug + path: check_deps_debug_artifact + if-no-files-found: ignore diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 1ce84c3a4f..0000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,643 +0,0 @@ -name: 🔬 Test -run-name: 🔬 Test ${{ github.sha }} - -#on: workflow_call - -on: - push: - branches: - - master - - manticore-* - paths-ignore: - - 'manual/**' - - '!manual/References.md' - - '.translation-cache/**' - - 'cmake/GetGALERA.cmake' - - 'galera_packaging/**' - pull_request: - branches: [ master, update-buddy-version ] - paths-ignore: - - 'manual/**' - - '!manual/References.md' - - '.translation-cache/**' - - 'cmake/GetGALERA.cmake' - - 'galera_packaging/**' - types: [opened, synchronize, reopened, labeled, unlabeled] - -# cancels the previous workflow run when a new one appears in the same branch (e.g. master or a PR's branch) -concurrency: - group: test_${{ github.ref }} - cancel-in-progress: true - -# Note: Many build jobs skip execution for PRs targeting 'update-buddy-version' branch -# as these PRs only update dependency versions and don't require full builds/tests -jobs: - meta: - name: Meta - runs-on: ubuntu-22.04 - outputs: - columnar_locator: ${{ steps.set_locator.outputs.columnar_locator }} - branch_name: ${{ steps.check_branch.outputs.branch_name }} - branch_tag: ${{ steps.sanitize_branch.outputs.branch_tag }} - source_hash: ${{ steps.source_hash.outputs.source_hash }} - image_exists: ${{ steps.image_check.outputs.image_exists }} - image_tag: ${{ steps.image_check.outputs.image_tag }} - image_commit: ${{ steps.image_check.outputs.image_commit }} - image_source_hash: ${{ steps.image_check.outputs.image_source_hash }} - source: ${{ steps.detect.outputs.source }} - test: ${{ steps.detect.outputs.test }} - clt: ${{ steps.detect.outputs.clt }} - changes_detected: ${{ steps.calc.outputs.changes_detected }} # any relevant change (source/test/clt) - skip_builds: ${{ steps.calc.outputs.skip_builds }} # only CLT tests changed and cached test-kit image exists - run_builds: ${{ steps.calc.outputs.run_builds }} # build/ubertest jobs - run_image_build: ${{ steps.calc.outputs.run_image_build }} # build test-kit image - run_clt: ${{ steps.calc.outputs.run_clt }} # run CLT workflow - run_image_push: ${{ steps.calc.outputs.run_image_push }} # push test-kit image tags - skip_clt_reason: ${{ steps.related_prs.outputs.skip_clt_reason }} # explanation for CLT skip - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Commit info - run: | - echo "# Automated Tests of commit ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY - echo "* Commit URL: [${{ github.sha }}](/${{ github.repository }}/commit/${{ github.sha }})" >> $GITHUB_STEP_SUMMARY - echo "* Initiated by: [@${{ github.actor }}](https://github.com/${{ github.actor }})" >> $GITHUB_STEP_SUMMARY - echo "* Ref: ${{ github.ref_type }} \"${{ github.ref_name }}\"" >> $GITHUB_STEP_SUMMARY - echo "* Attempt: ${{ github.run_attempt }}" >> $GITHUB_STEP_SUMMARY - - name: Check if branch exists in manticoresoftware/columnar - id: check_branch - run: | - # Extract the actual branch name for pull requests - if [[ "${{ github.event_name }}" == "pull_request" ]]; then - BRANCH_NAME="${{ github.event.pull_request.head.ref }}" - else - BRANCH_NAME="${{ github.ref_name }}" - fi - - HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" https://api.github.com/repos/manticoresoftware/columnar/branches/$BRANCH_NAME) - if [ "$HTTP_STATUS" -eq "200" ]; then - echo "branch_exists=true" >> $GITHUB_OUTPUT - echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT - else - echo "branch_exists=false" >> $GITHUB_OUTPUT - echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT - fi - - name: Sanitize branch name for tags - id: sanitize_branch - run: | - sanitize_tag() { - local name=$1 - name=$(echo "$name" | tr '/' '_') - name=$(echo "$name" | sed 's/[^a-zA-Z0-9_.-]//g') - echo "$name" - } - branch_name="${{ steps.check_branch.outputs.branch_name }}" - branch_tag=$(sanitize_tag "$branch_name") - echo "branch_tag=$branch_tag" >> $GITHUB_OUTPUT - - name: Calculate source hash - id: source_hash - run: | - set -euo pipefail - if [[ "${{ steps.check_branch.outputs.branch_exists }}" == "true" ]]; then - git clone --depth 1 --branch "${{ steps.check_branch.outputs.branch_name }}" https://github.com/manticoresoftware/columnar.git columnar - rm -rf columnar/.git - fi - HASH=$(find . -type f ! -path "./test/clt-tests/*" ! -path "./columnar/test/clt-tests/*" ! -path "./.git/*" -print0 | sort -z | xargs -0 md5sum | md5sum | awk '{print $1}') - echo "source_hash=$HASH" >> $GITHUB_OUTPUT - - name: Check for existing test kit image - id: image_check - env: - REPO_OWNER: ${{ github.repository_owner }} - run: | - set -euo pipefail - TAG="test-kit-${{ steps.sanitize_branch.outputs.branch_tag }}-${{ steps.source_hash.outputs.source_hash }}" - if docker manifest inspect "ghcr.io/${REPO_OWNER}/manticoresearch:${TAG}" > /dev/null 2>&1; then - echo "image_exists=true" >> $GITHUB_OUTPUT - echo "* Reusing cached test-kit image tag: ${TAG}." >> $GITHUB_STEP_SUMMARY - docker pull "ghcr.io/${REPO_OWNER}/manticoresearch:${TAG}" > /dev/null - image_commit=$(docker inspect -f '{{ index .Config.Labels "org.manticore.testkit.commit" }}' "ghcr.io/${REPO_OWNER}/manticoresearch:${TAG}" || true) - image_source_hash=$(docker inspect -f '{{ index .Config.Labels "org.manticore.testkit.source_hash" }}' "ghcr.io/${REPO_OWNER}/manticoresearch:${TAG}" || true) - echo "image_commit=${image_commit}" >> $GITHUB_OUTPUT - echo "image_source_hash=${image_source_hash}" >> $GITHUB_OUTPUT - else - echo "image_exists=false" >> $GITHUB_OUTPUT - echo "image_commit=" >> $GITHUB_OUTPUT - echo "image_source_hash=" >> $GITHUB_OUTPUT - fi - echo "image_tag=${TAG}" >> $GITHUB_OUTPUT - - name: Set Columnar Locator - id: set_locator - run: | - if [[ "${{ github.ref_name }}" != "master" && "${{ steps.check_branch.outputs.branch_exists }}" == "true" ]]; then - echo "columnar_locator=GIT_REPOSITORY https://github.com/manticoresoftware/columnar.git GIT_TAG ${{ steps.check_branch.outputs.branch_name }}" >> $GITHUB_OUTPUT - else - echo "columnar_locator=" >> $GITHUB_OUTPUT - fi - - name: Detect changed files and categories - id: detect - shell: bash - run: | - set -euo pipefail - head="${{ github.sha }}" - base="" - if [[ -n "${{ steps.image_check.outputs.image_commit }}" ]]; then - base="${{ steps.image_check.outputs.image_commit }}" - elif [[ "${{ github.event_name }}" == "pull_request" ]]; then - base="${{ github.event.pull_request.base.sha }}" - head="${{ github.event.pull_request.head.sha }}" - else - base="${{ github.event.before }}" - fi - - files="" - if [[ -z "$base" || "$base" == "0000000000000000000000000000000000000000" ]]; then - files=$(git ls-tree -r --name-only "$head") - else - if ! git cat-file -e "$base^{commit}" 2>/dev/null; then - git fetch --no-tags --depth=1 origin "$base" - fi - files=$(git diff --name-only "$base" "$head") - fi - - { - echo "### Changed files" - echo "* Base: ${base:-'(none)'}" - echo "* Head: ${head}" - if [[ -n "${{ steps.image_check.outputs.image_commit }}" ]]; then - echo "* Using image commit as base" - fi - } >> "$GITHUB_STEP_SUMMARY" - if [[ -z "$files" && "$base" == "$head" ]]; then - echo "* No diff (base == head); forcing full run" >> "$GITHUB_STEP_SUMMARY" - files="__force_all__" - fi - if [[ -n "$files" ]]; then - echo "$files" | sed 's/^/* /' >> "$GITHUB_STEP_SUMMARY" - else - echo "* (no files)" >> "$GITHUB_STEP_SUMMARY" - fi - - export FILES="$files" - - python3 - <<'PY' - import os - - files = os.environ.get("FILES", "").splitlines() - source = False - test = False - clt = False - for path in files: - if not path: - continue - if path == "__force_all__": - source = True - elif path.startswith("test/clt-tests/"): - clt = True - elif path.startswith("test/"): - test = True - elif path.startswith("manual/") or path.startswith("doc/"): - pass - else: - source = True - output = os.environ["GITHUB_OUTPUT"] - with open(output, "a", encoding="utf-8") as fh: - fh.write(f"source={'true' if source else 'false'}\n") - fh.write(f"test={'true' if test else 'false'}\n") - fh.write(f"clt={'true' if clt else 'false'}\n") - PY - - name: Check related PRs for branch - id: related_prs - run: | - set -euo pipefail - reason="" - if [[ "${{ github.event_name }}" == "pull_request" ]]; then - for repo in columnar manticoresearch-buddy; do - response="$( (curl -sSL -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository_owner }}/$repo/pulls?state=open&head=${{ github.repository_owner }}:${{ github.event.pull_request.head.ref }}&per_page=1" || true) | tr -d '[:space:]')" - if [[ "$response" != "[]" && "$response" != "" ]]; then - reason+="$repo: open PR for ${{ github.repository_owner }}:${{ github.event.pull_request.head.ref }} | " - fi - done - - reason="${reason%% | }" - fi - echo "skip_clt_reason=$reason" >> "$GITHUB_OUTPUT" - - name: Calculate gating - id: calc - run: | - set -euo pipefail - changes_detected=false - if [[ "${{ steps.detect.outputs.source }}" == "true" || "${{ steps.detect.outputs.test }}" == "true" || "${{ steps.detect.outputs.clt }}" == "true" ]]; then - changes_detected=true - fi - - skip_builds=false - if [[ "${{ steps.detect.outputs.clt }}" == "true" && "${{ steps.detect.outputs.source }}" != "true" && "${{ steps.detect.outputs.test }}" != "true" && "${{ steps.image_check.outputs.image_exists }}" == "true" ]]; then - skip_builds=true - fi - - run_builds=false - if [[ "$changes_detected" == "true" && "${{ github.event_name }}" == "pull_request" && "${{ github.event.pull_request.base.ref }}" == "update-buddy-version" ]]; then - run_builds=false - elif [[ "$changes_detected" == "true" && "$skip_builds" == "false" ]]; then - run_builds=true - fi - - run_image_build="$changes_detected" - run_clt="$changes_detected" - skip_clt_tag=false - if git tag --points-at HEAD | grep -qx "skip_clt"; then - skip_clt_tag=true - run_clt=false - fi - if [[ -n "${{ steps.related_prs.outputs.skip_clt_reason }}" ]]; then - run_clt=false - fi - run_image_push=false - if [[ "$changes_detected" == "true" && "${{ steps.image_check.outputs.image_exists }}" != "true" ]]; then - run_image_push=true - fi - - echo "changes_detected=$changes_detected" >> "$GITHUB_OUTPUT" - echo "skip_builds=$skip_builds" >> "$GITHUB_OUTPUT" - echo "run_builds=$run_builds" >> "$GITHUB_OUTPUT" - echo "run_image_build=$run_image_build" >> "$GITHUB_OUTPUT" - echo "run_clt=$run_clt" >> "$GITHUB_OUTPUT" - echo "run_image_push=$run_image_push" >> "$GITHUB_OUTPUT" - echo "skip_clt_tag=$skip_clt_tag" >> "$GITHUB_OUTPUT" - - name: Summarize gating - run: | - { - echo "### Gate decisions" - echo "* Changes detected (source/test/CLT): ${{ steps.calc.outputs.changes_detected }}" - echo "* Skip builds/ubertests (only CLT changes + cached image): ${{ steps.calc.outputs.skip_builds }}" - echo "* Run build/ubertest jobs: ${{ steps.calc.outputs.run_builds }}" - echo "* Build test-kit image: ${{ steps.calc.outputs.run_image_build }}" - echo "* Run CLT tests: ${{ steps.calc.outputs.run_clt }}" - echo "* Push test-kit image tags: ${{ steps.calc.outputs.run_image_push }}" - if [[ "${{ steps.calc.outputs.skip_clt_tag }}" == "true" ]]; then - echo "* CLT skip reason: skip_clt tag on commit" - fi - if [[ -n "${{ steps.related_prs.outputs.skip_clt_reason }}" ]]; then - echo "* CLT skip reason: ${{ steps.related_prs.outputs.skip_clt_reason }}" - fi - } >> "$GITHUB_STEP_SUMMARY" - - win_bundle: - # Skip build jobs for PRs targeting 'update-buddy-version' branch (buddy version updates don't need full builds) - if: needs.meta.outputs.run_builds == 'true' - needs: [meta] - name: Windows supplementary files preparation - runs-on: ubuntu-22.04 - steps: - - name: Check out cache - id: cache - uses: actions/cache@v4 - with: - path: | - bundle - boost_1_75_0 - enableCrossOsArchive: true - key: win_bundle - lookup-only: true - - name: Extract Windows bundle from Windows sysroot - if: steps.cache.outputs.cache-hit != 'true' - run: | - wget https://repo.manticoresearch.com/repository/sysroots/roots_apr15/sysroot_windows_x64.tar.xz - tar -xvf sysroot_windows_x64.tar.xz - mv diskc/winbundle bundle - - name: Extract Boost to put it to the cache - if: steps.cache.outputs.cache-hit != 'true' - run: | - wget https://repo.manticoresearch.com/repository/ci/boost_1_75_0.tgz - tar -xf boost_1_75_0.tgz - - build_linux_debug: - if: needs.meta.outputs.run_builds == 'true' - needs: [meta] - name: Linux debug build - uses: ./.github/workflows/build_template.yml - with: - CTEST_CONFIGURATION_TYPE: "Debug" - COLUMNAR_LOCATOR: ${{ needs.meta.outputs.columnar_locator }} - artifact_name: debug_build - cache_key: build_linux_debug_x86_64 - cmake_command: | - export CMAKE_TOOLCHAIN_FILE=$(pwd)/dist/build_dockers/cross/linux.cmake - ctest -VV -S misc/ctest/gltest.cmake --no-compress-output - - test_linux_debug: - if: (github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'update-buddy-version') != true - name: Linux debug mode tests - needs: [build_linux_debug, meta] - uses: ./.github/workflows/test_template.yml - with: - COLUMNAR_LOCATOR: ${{ needs.meta.outputs.columnar_locator }} - build_artifact_name: debug_build - artifact_name: debug_test_results - results_name: "Linux debug test results" - timeout: 10 - - build_linux_release: - if: needs.meta.outputs.run_builds == 'true' - needs: [meta] - name: Linux release build - uses: ./.github/workflows/build_template.yml - with: - artifact_name: release_build - COLUMNAR_LOCATOR: ${{ needs.meta.outputs.columnar_locator }} - cache_key: build_linux_release_x86_64 - artifact_list: "build/xml build/CMakeFiles/CMake*.log build/api/libsphinxclient/testcli build/src/indexer build/src/indextool build/src/searchd build/src/tests build/src/gtests/gmanticoretest build/config/*.c build/config/*.h build/**/*.cxx build/**/*.gcno build/manticore*deb" - cmake_command: | - export CMAKE_TOOLCHAIN_FILE=$(pwd)/dist/build_dockers/cross/linux.cmake - export PACK=1 - export PACK_KEEP_TESTS=1 - ctest -VV -S misc/ctest/gltest.cmake --no-compress-output - cmake --build build --target package - - test_linux_release_mcl: - if: (github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'update-buddy-version') != true - name: Linux release mode tests (MCL) - needs: [build_linux_release, meta] - uses: ./.github/workflows/test_template.yml - with: - COLUMNAR_LOCATOR: ${{ needs.meta.outputs.columnar_locator }} - build_artifact_name: release_build - artifact_name: release_mcl_test_results - results_name: "Linux release MCL test results" - timeout: 10 - INSTALL_MCL_FROM_DEPS: true - FLOAT_TOLERANCE: true - CTEST_EXCLUDE_REGEX: "rt_298|rt_306|rt_481" - - test_linux_release: - if: (github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'update-buddy-version') != true - name: Linux release mode tests - needs: [build_linux_release, meta] - uses: ./.github/workflows/test_template.yml - with: - COLUMNAR_LOCATOR: ${{ needs.meta.outputs.columnar_locator }} - build_artifact_name: release_build - artifact_name: release_test_results - results_name: "Linux release test results" - timeout: 10 - - - # When only CLT changed and a cached image exists, build jobs are skipped. A job that needs - # build_linux_release would then be skipped too (GitHub skips dependents of skipped jobs), - # so we use a separate reuse-only job that only needs meta. That way CLT always runs when - # run_clt is true and we have a test-kit image (either built or reused). - test_kit_docker_reuse: - name: Test Kit docker image (reuse cache) - needs: [meta] - if: needs.meta.outputs.run_image_build == 'true' && needs.meta.outputs.image_exists == 'true' - runs-on: ubuntu-22.04 - steps: - - name: Checkout repository # We have to checkout to access .github/workflows/ in further steps - uses: actions/checkout@v3 - - name: Reuse cached test kit image - env: - REPO_OWNER: ${{ github.repository_owner }} - IMAGE_TAG: ${{ needs.meta.outputs.image_tag }} - run: | - set -euo pipefail - IMAGE="ghcr.io/${REPO_OWNER}/manticoresearch:${IMAGE_TAG}" - docker pull "$IMAGE" - docker tag "$IMAGE" test-kit:img - docker create --name manticore-test-kit-src --entrypoint /bin/sh test-kit:img -c "true" - docker export manticore-test-kit-src > manticore_test_kit.img - docker rm manticore-test-kit-src - - name: Upload docker image artifact - uses: manticoresoftware/upload_artifact_with_retries@v4 - with: - name: manticore_test_kit.img - path: manticore_test_kit.img - - test_kit_docker_build: - name: Test Kit docker image - needs: - - build_linux_release - - meta - if: needs.meta.outputs.run_image_build == 'true' && needs.meta.outputs.image_exists != 'true' - runs-on: ubuntu-22.04 - outputs: - out-build: ${{ steps.build.outputs.build_image }} - steps: - - name: Checkout repository # We have to checkout to access .github/workflows/ in further steps - uses: actions/checkout@v3 - - name: Download built x86_64 Ubuntu Jammy packages - uses: manticoresoftware/download_artifact_with_retries@v3 - with: - name: release_build - path: . - # Uncomment this shortcut for debug to save time by not preparing the packages in the pack_jammy job - # - run: | - # wget http://dev2.manticoresearch.com/build_jammy_RelWithDebInfo_x86_64.zip - # unzip build_jammy_RelWithDebInfo_x86_64.zip - # tar -xvf artifact.tar - - name: Calculate short commit hash - id: sha - run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - name: Building docker - id: build - run: | - BUILD_COMMIT=${{ steps.sha.outputs.sha_short }} /bin/bash dist/test_kit_docker_build.sh - echo "build_image=ghcr.io/$REPO_OWNER/manticoresearch:test-kit-${{ steps.sha.outputs.sha_short }}" >> $GITHUB_OUTPUT - - name: Upload docker image artifact - uses: manticoresoftware/upload_artifact_with_retries@v4 - with: - name: manticore_test_kit.img - path: manticore_test_kit.img - - clt: - if: always() && needs.meta.outputs.run_clt == 'true' && (needs.test_kit_docker_build.result == 'success' || needs.test_kit_docker_reuse.result == 'success') - name: CLT - needs: [test_kit_docker_build, test_kit_docker_reuse, meta] - uses: ./.github/workflows/clt_tests.yml - secrets: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }} - JINA_API_KEY: ${{ secrets.JINA_API_KEY }} - with: - docker_image: test-kit:img - artifact_name: manticore_test_kit.img - repository: ${{ github.repository }} - ref: ${{ github.sha }} - - test_kit_docker_push: - if: always() && needs.meta.outputs.run_image_push == 'true' - needs: - - clt - - meta - - test_kit_docker_build - - test_kit_docker_reuse - - test_linux_debug - - test_linux_release - - test_linux_release_mcl - - test_windows - name: Push Test Kit docker image to repo - runs-on: ubuntu-22.04 - env: - GHCR_USER: ${{ vars.GHCR_USER }} - GHCR_PASSWORD: ${{ secrets.GHCR_PASSWORD }} - REPO_OWNER: ${{ github.repository_owner }} - SOURCE_HASH: ${{ needs.meta.outputs.source_hash }} - BRANCH_TAG: ${{ needs.meta.outputs.branch_tag }} - HASH_TAG_OK: ${{ needs.test_linux_debug.result == 'success' && needs.test_linux_release.result == 'success' && needs.test_linux_release_mcl.result == 'success' && needs.test_windows.result == 'success' }} - IMAGE_BUILD_COMMIT: ${{ github.sha }} - steps: - - name: Checkout repository # We have to checkout to access .github/workflows/ in further steps - uses: actions/checkout@v3 - - name: Download artifact - uses: manticoresoftware/download_artifact_with_retries@main - with: - name: manticore_test_kit.img - path: . - - name: Calculate short commit hash - id: sha - run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - name: Pushing docker image to repo - id: test-kit-push - run: | - TEST_RESULT=${{ needs.clt.result }} BUILD_COMMIT=${{ steps.sha.outputs.sha_short }} /bin/bash dist/test_kit_docker_push.sh - - build_aarch64: - if: needs.meta.outputs.run_builds == 'true' - needs: [meta] - name: Linux aarch64 build - uses: ./.github/workflows/build_template.yml - # Use -VV instead of -V below for more verbose output - with: - arch: aarch64 - COLUMNAR_LOCATOR: ${{ needs.meta.outputs.columnar_locator }} - cmake_command: | - mkdir build - cd build - export CMAKE_TOOLCHAIN_FILE=$(pwd)/../dist/build_dockers/cross/linux.cmake - ctest -V -S ../misc/ctest/justbuild.cmake -DCTEST_SOURCE_DIRECTORY=.. --no-compress-output - cache_key: build_jammy_aarch64 - - build_freebsd: - if: needs.meta.outputs.run_builds == 'true' - needs: [meta] - name: FreeBSD x86_64 build - uses: ./.github/workflows/build_template.yml - with: - DISTR: freebsd13 - COLUMNAR_LOCATOR: ${{ needs.meta.outputs.columnar_locator }} - boost_url_key: none - cmake_command: | - mkdir build - cd build - export CMAKE_TOOLCHAIN_FILE=$(pwd)/../dist/build_dockers/cross/freebsd.cmake - ctest -VV -S ../misc/ctest/justbuild.cmake -DCTEST_SOURCE_DIRECTORY=.. --no-compress-output - cache_key: build_freebsd_x86_64 - - build_windows: - if: needs.meta.outputs.run_builds == 'true' - needs: [meta] - name: Windows x64 build - uses: ./.github/workflows/build_template.yml - with: - DISTR: windows - arch: x64 - sysroot_url_key: roots_mysql83_jan17 - boost_url_key: boost_80 - CTEST_CMAKE_GENERATOR: "Ninja Multi-Config" - CTEST_CONFIGURATION_TYPE: Debug - cache_key: build_windows_x64 - artifact_list: "build/xml build/src/Debug/indexer.exe build/src/Debug/indexer.pdb build/src/Debug/searchd.exe build/src/Debug/searchd.pdb build/src/gtests/Debug/gmanticoretest.exe build/src/gtests/Debug/gmanticoretest.pdb build/src/Debug/*.dll build/src/gtests/Debug/*.dll build/config/*.c build/config/*.h" - COLUMNAR_LOCATOR: ${{ needs.meta.outputs.columnar_locator }} - - test_windows: - if: (github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'update-buddy-version') != true - name: Windows tests - needs: [build_windows, win_bundle, meta] - uses: ./.github/workflows/win_test_template.yml - with: - CTEST_START: 1 - CTEST_END: 999999 - COLUMNAR_LOCATOR: ${{ needs.meta.outputs.columnar_locator }} - artifact_name: windows_test_results - - final_status: - name: Final status - needs: - - build_linux_debug - - test_linux_debug - - build_linux_release - - test_linux_release - - test_linux_release_mcl - - build_windows - - test_windows - - build_freebsd - - build_aarch64 - - clt - - meta - if: always() - runs-on: ubuntu-22.04 - steps: - - name: Fail if any required job failed - run: | - set -euo pipefail - failed=0 - results=( - "build_linux_debug:${{ needs.build_linux_debug.result }}" - "test_linux_debug:${{ needs.test_linux_debug.result }}:${{ needs.test_linux_debug.outputs.test_outcome }}" - "build_linux_release:${{ needs.build_linux_release.result }}" - "test_linux_release:${{ needs.test_linux_release.result }}:${{ needs.test_linux_release.outputs.test_outcome }}" - "test_linux_release_mcl:${{ needs.test_linux_release_mcl.result }}:${{ needs.test_linux_release_mcl.outputs.test_outcome }}" - "build_windows:${{ needs.build_windows.result }}" - "test_windows:${{ needs.test_windows.result }}:${{ needs.test_windows.outputs.test_outcome }}" - "build_freebsd:${{ needs.build_freebsd.result }}" - "build_aarch64:${{ needs.build_aarch64.result }}" - ) - { - echo "### Final status" - for entry in "${results[@]}"; do - name="${entry%%:*}" - rest="${entry#*:}" - result="${rest%%:*}" - outcome="${rest#*:}" - if [[ "$name" == test_* ]]; then - if [[ "$outcome" != "$result" && -n "$outcome" ]]; then - echo "* $name: $result (ctest: $outcome)" - else - echo "* $name: $result" - fi - else - echo "* $name: $result" - fi - done - if [[ "${{ needs.meta.outputs.run_clt }}" == "true" ]]; then - echo "* clt: ${{ needs.clt.result }}" - fi - } >> "$GITHUB_STEP_SUMMARY" - for entry in "${results[@]}"; do - name="${entry%%:*}" - rest="${entry#*:}" - result="${rest%%:*}" - outcome="${rest#*:}" - if [[ "$name" == test_* && "$result" == "success" && "$outcome" == "failure" ]]; then - echo "$name: ctest failed" - failed=1 - continue - fi - if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then - echo "$name: $result" - failed=1 - fi - done - if [[ "${{ needs.meta.outputs.run_clt }}" == "true" ]]; then - result="${{ needs.clt.result }}" - if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then - echo "clt: $result" - failed=1 - fi - fi - if [[ "$failed" -ne 0 ]]; then - echo "* Overall: failure" >> "$GITHUB_STEP_SUMMARY" - echo "One or more build/test jobs failed" - exit 1 - fi - echo "* Overall: success" >> "$GITHUB_STEP_SUMMARY" diff --git a/dist/check_deps_in_repos.sh b/dist/check_deps_in_repos.sh index 4ffa4adcfc..67a9df5f0c 100755 --- a/dist/check_deps_in_repos.sh +++ b/dist/check_deps_in_repos.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -euo pipefail # Parse command line arguments REPO_TYPE="dev" # Default value @@ -22,6 +22,8 @@ done SCRIPT_DIR="$(dirname "$0")" REPO_ROOT="$(realpath "$SCRIPT_DIR/..")" TEMP_DIR=$(mktemp -d) +ARTIFACT_DIR="$REPO_ROOT/check_deps_debug_artifact" +PRESERVE_DEBUG=0 echo "Current directory: $(pwd)" echo "Script location: $SCRIPT_DIR" @@ -29,6 +31,31 @@ echo "Expected repository root: $REPO_ROOT" echo "Temporary directory: $TEMP_DIR" echo "Using repository type: $REPO_TYPE" +cleanup() { + if [ "$PRESERVE_DEBUG" -eq 0 ] && [ -d "$TEMP_DIR" ]; then + rm -rf "$TEMP_DIR" + fi +} + +trap cleanup EXIT + +prepare_debug_bundle() { + local reason="$1" + PRESERVE_DEBUG=1 + rm -rf "$ARTIFACT_DIR" + mkdir -p "$ARTIFACT_DIR" + cp -R "$TEMP_DIR"/. "$ARTIFACT_DIR"/ + { + echo "reason=$reason" + echo "repo_type=$REPO_TYPE" + echo "cwd=$(pwd)" + echo "script_dir=$SCRIPT_DIR" + echo "repo_root=$REPO_ROOT" + echo "temp_dir=$TEMP_DIR" + echo "artifact_dir=$ARTIFACT_DIR" + } > "$ARTIFACT_DIR/context.txt" +} + # List of HTML pages to download declare -a urls=( "bionic-amd.html https://repo.manticoresearch.com/repository/manticoresearch_bionic_${REPO_TYPE}/dists/bionic/main/binary-amd64/" @@ -73,22 +100,58 @@ should_skip() { } # Download HTML files into TEMP_DIR +mkdir -p "$TEMP_DIR/wget_logs" pids=() for pair in "${urls[@]}"; do name="${pair%% *}" url="${pair#* }" echo "Starting download for $name..." - wget -q -O "$TEMP_DIR/$name" "$url" & + wget -c --tries=10 --retry-connrefused --waitretry=5 --timeout=60 --server-response --output-file="$TEMP_DIR/wget_logs/${name}.log" -O "$TEMP_DIR/$name" "$url" >/dev/null 2>&1 & pids+=($!) done # Wait for all download processes to complete echo "Waiting for all downloads to complete..." for pid in "${pids[@]}"; do - wait $pid || { echo "Error: One of the downloads failed"; exit 1; } + if ! wait "$pid"; then + echo "Error: One of the downloads failed" + prepare_debug_bundle "download_failed" + echo "Debug artifact prepared in $ARTIFACT_DIR" + exit 1 + fi done echo "All HTML pages were successfully downloaded to $TEMP_DIR." +download_summary="$TEMP_DIR/download_summary.txt" +printf "repo\tbytes\tstatus\tcontent_type\tpackage_links\ttitle\n" > "$download_summary" + +invalid_downloads=() +for html_file in "$TEMP_DIR"/*.html; do + repo_name="$(basename "$html_file")" + log_file="$TEMP_DIR/wget_logs/${repo_name}.log" + size_bytes=$(wc -c < "$html_file" | tr -d ' ') + status_code=$( (grep -Eo 'HTTP/[0-9.]+ [0-9]+' "$log_file" || true) | tail -1 | awk '{print $2}' ) + content_type=$( (grep -i '^ Content-Type:' "$log_file" || true) | tail -1 | sed 's/^ Content-Type: //; s/\r$//' ) + package_links=$( (grep -Eio '\.(deb|rpm|tar\.gz|zip)(\"|<|$)' "$html_file" || true) | wc -l | tr -d ' ' ) + title=$( { tr '\n' ' ' < "$html_file" | sed -n 's:.*\(.*\).*:\1:Ip' | sed 's/[[:space:]]\+/ /g; s/^ //; s/ $//'; } || true ) + + printf "%s\t%s\t%s\t%s\t%s\t%s\n" \ + "$repo_name" "${size_bytes:-0}" "${status_code:-unknown}" "${content_type:-unknown}" "${package_links:-0}" "${title:-n/a}" \ + >> "$download_summary" + + if [ "${size_bytes:-0}" -eq 0 ] || [ "${package_links:-0}" -eq 0 ]; then + invalid_downloads+=("$repo_name|size=${size_bytes:-0}|status=${status_code:-unknown}|content_type=${content_type:-unknown}|links=${package_links:-0}|title=${title:-n/a}") + fi +done + +if [ ${#invalid_downloads[@]} -gt 0 ]; then + echo "Warning: Some downloaded pages do not look like package indexes." + for entry in "${invalid_downloads[@]}"; do + IFS='|' read -r repo_name size_info status_info content_type_info links_info title_info <<< "$entry" + echo " - $repo_name ($size_info, $status_info, $content_type_info, $links_info, $title_info)" + done +fi + # Read deps.txt file DEPS_FILE="$REPO_ROOT/deps.txt" if [ ! -f "$DEPS_FILE" ]; then @@ -97,6 +160,14 @@ if [ ! -f "$DEPS_FILE" ]; then fi missing_packages=() +declare -A repo_issue_details=() + +if [ ${#invalid_downloads[@]} -gt 0 ]; then + for entry in "${invalid_downloads[@]}"; do + IFS='|' read -r repo_name size_info status_info content_type_info links_info title_info <<< "$entry" + repo_issue_details["$repo_name"]="$size_info, $status_info, $content_type_info, $links_info, $title_info" + done +fi while IFS=" " read -r package version_string date hash suffix || [ -n "$package" ]; do [[ -z "$package" ]] && continue @@ -155,7 +226,12 @@ while IFS=" " read -r package version_string date hash suffix || [ -n "$package" if [ ${#missing_in[@]} -gt 0 ]; then echo "❌ Package $package (real name: $real_name) is missing in:" for repo in "${missing_in[@]}"; do - echo " - $repo" + extra_info="${repo_issue_details[$repo]:-}" + if [ -n "$extra_info" ]; then + echo " - $repo [$extra_info]" + else + echo " - $repo" + fi done missing_packages+=("$package $version $search_substring") else @@ -170,10 +246,11 @@ if [ ${#missing_packages[@]} -gt 0 ]; then for m in "${missing_packages[@]}"; do echo " - $m" done - rm -rf "$TEMP_DIR" + prepare_debug_bundle "packages_missing_or_unparseable_pages" + cp "$download_summary" "$ARTIFACT_DIR/download_summary.txt" + echo "Debug artifact prepared in $ARTIFACT_DIR" exit 1 else echo "" echo "🎉 All packages listed in deps.txt were successfully found!" - rm -rf "$TEMP_DIR" fi