diff --git a/.cirrus.yml b/.cirrus.yml deleted file mode 100644 index 77ed2a7265..0000000000 --- a/.cirrus.yml +++ /dev/null @@ -1,338 +0,0 @@ ---- - -# Main collection of env. vars to set for all tasks and scripts. -env: - #### - #### Global variables used for all tasks - #### - # Overrides default location (/tmp/cirrus) for repo clone - CIRRUS_WORKING_DIR: "/var/tmp/go/src/github.com/containers/container-libs" - # Shell used to execute all script commands - CIRRUS_SHELL: "/bin/bash" - # Automation script path relative to $CIRRUS_WORKING_DIR) - SCRIPT_BASE: "./storage/contrib/cirrus" - IMAGE_SCRIPT_BASE: "./image/contrib/cirrus" - # No need to go crazy, but grab enough to cover most PRs - CIRRUS_CLONE_DEPTH: 50 - DEST_BRANCH: "main" - GOPATH: "/var/tmp/go" - GOBIN: "${GOPATH}/bin" - GOCACHE: "${GOPATH}/cache" - SKOPEO_PATH: "/var/tmp/go/src/github.com/containers/skopeo" - IMAGE_WORKING_DIR: "/var/tmp/go/src/github.com/containers/image" - - - #### - #### Cache-image names to test with (double-quotes around names are critical) - ### - FEDORA_NAME: "fedora-42" - DEBIAN_NAME: "debian-13" - - # GCE project where images live - IMAGE_PROJECT: "libpod-218412" - # VM Image built in containers/automation_images - # If you are updating IMAGE_SUFFIX: We are currently using rawhide for - # the containers_image_sequoia tests because the rust-podman-sequoia - # package is not available in earlier releases; once we update to a future - # Fedora release (or if the package is backported), switch back from Rawhide - # to the latest Fedora release. - IMAGE_SUFFIX: "c20250910t092246z-f42f41d13" - RAWHIDE_CACHE_IMAGE_NAME: "rawhide-${IMAGE_SUFFIX}" - - FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}" - DEBIAN_CACHE_IMAGE_NAME: "debian-${IMAGE_SUFFIX}" - FEDORA_CONTAINER_FQIN: "quay.io/libpod/fedora_podman:${IMAGE_SUFFIX}" - - - #### - #### Command variables to help avoid duplication - #### - # Command to prefix every output line with a timestamp - # (can't do inline awk script, Cirrus-CI or YAML mangles quoting) - _TIMESTAMP: 'awk --file ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/timestamp.awk' - _DFCMD: 'df -lhTx tmpfs' - _RAUDITCMD: 'cat /var/log/audit/audit.log' - _UAUDITCMD: 'cat /var/log/kern.log' - _JOURNALCMD: 'journalctl -b' - -gcp_credentials: ENCRYPTED[f448b0e5ed38a52ba93550305f9e38c9b0c03852b9fc3115930f231fc7124b7423e8c912b4d9e002cc34b3a7ac3c145b] - -# Default timeout for each task -timeout_in: 20m - -# Default VM to use unless set or modified by task -gce_instance: - image_project: "${IMAGE_PROJECT}" - zone: "us-central1-b" # Required by Cirrus for the time being - cpu: 4 - memory: "4Gb" - disk: 200 - image_name: "${FEDORA_CACHE_IMAGE_NAME}" - - -storage_linux_testing: &linux_testing - gce_instance: # Only need to specify differences from defaults (above) - image_name: "${VM_IMAGE}" - - # Go module cache that is shared for all tasks - modules_cache: &modules_cache - fingerprint_script: cat **/go.sum - folder: $GOPATH/pkg/mod - # Note the cache is populated the first time it is used with the given deps. - # But because each task might not download all he same deps force make tidy - # which downloads all deps for all modules. This is only run once when the - # cache is created so it should not slow things down much. - populate_script: make tidy - - # Separate scripts for separate outputs, makes debugging easier. - setup_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/setup.sh |& ${_TIMESTAMP}' - build_and_test_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/build_and_test.sh |& ${_TIMESTAMP}' - - always: - df_script: '${_DFCMD} || true' - rh_audit_log_script: '${_RAUDITCMD} || true' - debian_audit_log_script: '${_UAUDITCMD} || true' - journal_log_script: '${_JOURNALCMD} || true' - - -storage_fedora_testing_task: - <<: *linux_testing - alias: storage_fedora_testing - only_if: &only_if_storage >- - $CIRRUS_PR == '' || - changesInclude('.cirrus.yml', 'go.work', 'go.work.sum') || - changesInclude('storage/**') - name: &std_test_name "Storage: ${OS_NAME} ${TEST_DRIVER}" - env: - OS_NAME: "${FEDORA_NAME}" - VM_IMAGE: "${FEDORA_CACHE_IMAGE_NAME}" - # Not all $TEST_DRIVER combinations valid for all $VM_IMAGE types. - matrix: &test_matrix - - env: - TEST_DRIVER: "vfs" - - env: - TEST_DRIVER: "overlay" - - env: - TEST_DRIVER: "overlay-transient" - - env: - TEST_DRIVER: "fuse-overlay" - - env: - TEST_DRIVER: "fuse-overlay-whiteout" - - env: - TEST_DRIVER: "btrfs" - - -# aufs was dropped between 20.04 and 22.04, can't test it -storage_debian_testing_task: - <<: *linux_testing - alias: storage_debian_testing - name: *std_test_name - only_if: *only_if_storage - env: - OS_NAME: "${DEBIAN_NAME}" - VM_IMAGE: "${DEBIAN_CACHE_IMAGE_NAME}" - # /tmp is a tmpfs, and as of 2025-09-11 we are using Debian images with Linux 6.1, where tmpfs does not support extended attributes. - # That prevents testing various graph drivers; setting TMPDIR changes where graph driver tests place their roots. - TMPDIR: "/var/tmp" - # Not all $TEST_DRIVER combinations valid for all $VM_IMAGE types. - matrix: - - env: - TEST_DRIVER: "vfs" - - env: - TEST_DRIVER: "overlay" - - env: - TEST_DRIVER: "fuse-overlay" - - env: - TEST_DRIVER: "fuse-overlay-whiteout" - - env: - TEST_DRIVER: "btrfs" - -# Update metadata on VM images referenced by this repository state -images_timestamp_update_task: - alias: images_timestamp_update - name: "VM Images Timestamp Update" - - container: - image: "quay.io/libpod/imgts:latest" - cpu: 1 - memory: 1 - - env: - # Space-separated list of images used by this repository state - IMGNAMES: |- - ${FEDORA_CACHE_IMAGE_NAME} - ${DEBIAN_CACHE_IMAGE_NAME} - ${RAWHIDE_CACHE_IMAGE_NAME} - BUILDID: "${CIRRUS_BUILD_ID}" - REPOREF: "${CIRRUS_CHANGE_IN_REPO}" - GCPJSON: ENCRYPTED[84d8f13981b019af7dfffbe13129ed035aa09ac167c55f2b4ebaccc4f91e8fca0b9c805e2bcf8f18da5b964b35c68203] - GCPNAME: ENCRYPTED[abc17b496b2a96dbf232a97e95a3696e71241f6addc82815ee230fcb67ce2908d4a572f13203cc5797ec24176ea40301] - GCPPROJECT: ENCRYPTED[dc578f8953d01e33fe6b329886b8b088c7a909246e169e640d621ac3f7f0ac251ea4f41353e162b1e5e6586cfb01520d] - CIRRUS_CLONE_DEPTH: 1 # source not used - - script: '/usr/local/bin/entrypoint.sh |& ${_TIMESTAMP}' - - -image_cross_task: - alias: image_cross - name: "Image: Cross" - only_if: &only_if_image >- - $CIRRUS_PR == '' || - changesInclude('.cirrus.yml', 'go.work', 'go.work.sum') || - changesInclude('storage/**', 'image/**') - gce_instance: &fedora_vm - image_project: libpod-218412 - zone: "us-central1-f" - cpu: 2 - memory: "4Gb" - # Required to be 200gig, do not modify - has i/o performance impact - # according to gcloud CLI tool warning messages. - disk: 200 - # Eventually, hard-code FEDORA_CACHE_IMAGE_NAME here again and remove the - # VM_IMAGE_NAME parameter. - image_name: ${VM_IMAGE_NAME} - modules_cache: *modules_cache - env: - HOME: "/root" # default unset, needed by golangci-lint. - VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME} - script: | - cd image - make cross - - -image_test_task: - alias: image_test - only_if: *only_if_image - gce_instance: *fedora_vm - modules_cache: *modules_cache - env: - HOME: "/root" # default unset, needed by golangci-lint. - GOSRC: "${CIRRUS_WORKING_DIR}/image" - GOPATH: "/var/tmp/go" - # Required for consistency with containers/skopeo CI - matrix: - - name: "Image: Test" - env: - BUILDTAGS: '' - VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME} - - name: "Image: Test w/ opengpg" - env: - BUILDTAGS: &withopengpg 'containers_image_openpgp' - VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME} - - name: "Test w/ Sequoia (currently Rawhide)" - env: - BUILDTAGS: &withsequoia 'containers_image_sequoia' - VM_IMAGE_NAME: ${RAWHIDE_CACHE_IMAGE_NAME} - script: | - cd image - ls -l ${CIRRUS_WORKING_DIR} - echo "${GOPATH}" - ${CIRRUS_WORKING_DIR}/${IMAGE_SCRIPT_BASE}/runner.sh image_tests - - -##### -##### NOTE: This task is subtantially duplicated in the containers/skopeo -##### repository's `.cirrus.yml`. Changes made here should be fully merged -##### prior to being manually duplicated and maintained in containers/skopeo. -##### -image_test_skopeo_task: - alias: image_test_skopeo - only_if: *only_if_image - gce_instance: *fedora_vm - env: - HOME: "/root" # default unset, needed by golangci-lint. - GOSRC: "${CIRRUS_WORKING_DIR}" - GOPATH: "/var/tmp/go" - SKOPEO_CI_BRANCH: "release-1.21" - SCRIPT_BASE: "./contrib/cirrus" - # Built along with the standard PR-based workflow in c/automation_images - SKOPEO_CIDEV_CONTAINER_FQIN: "quay.io/libpod/skopeo_cidev:${IMAGE_SUFFIX}" - modules_cache: *modules_cache - matrix: - - name: "Image: Skopeo Test" - env: - BUILDTAGS: '' - VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME} - - name: "Image: Skopeo Test w/ opengpg" - env: - BUILDTAGS: *withopengpg - VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME} - - name: "Skopeo Test w/ Sequoia (currently Rawhide)" - env: - BUILDTAGS: *withsequoia - VM_IMAGE_NAME: ${RAWHIDE_CACHE_IMAGE_NAME} - setup_script: >- - "${CIRRUS_WORKING_DIR}/${IMAGE_SCRIPT_BASE}/runner.sh" setup - vendor_script: >- - "${SKOPEO_PATH}/contrib/cirrus/runner.sh" vendor - build_script: >- - "${SKOPEO_PATH}/contrib/cirrus/runner.sh" build - unit_script: >- - "${SKOPEO_PATH}/contrib/cirrus/runner.sh" unit - integration_script: >- - "${SKOPEO_PATH}/contrib/cirrus/runner.sh" integration - system_script: > - "${SKOPEO_PATH}/contrib/cirrus/runner.sh" system - - - -common_testing_task: - alias: common_testing - name: "Common: Test" - modules_cache: *modules_cache - env: - NETAVARK_BINARY: "/usr/libexec/podman/netavark" - test_script: | - cd common - make build - make build-cross - make test - -storage_cross_task: - only_if: *only_if_storage - alias: storage_cross - name: "Storage: Cross" - container: - image: golang:1.24 - modules_cache: *modules_cache - build_script: | - cd storage - make cross - -storage_gofix_task: - only_if: *only_if_storage - alias: storage_gofix - name: "Storage: gofix" - container: - image: golang:1.24 - build_script: | - cd storage - go fix ./... - test_script: | - # The git diff complains about broken symlinks in some test files. - # We therefore filter-out these files, since they are not changed by - # gofix anyway. - git diff --diff-filter=M --exit-code -- . ":(exclude)*.crt" ":(exclude)*.key" ":(exclude)*.cert" - -# Status aggregator for all tests. This task simply ensures a defined -# set of tasks all passed, and allows confirming that based on the status -# of this task. -success_task: - alias: success - # N/B: The prow merge-bot (tide) is sensitized to this exact name, DO NOT CHANGE IT. - # Ref: https://github.com/openshift/release/pull/49820 - name: "Total Success" - depends_on: - - storage_fedora_testing - - storage_debian_testing - - images_timestamp_update - - storage_cross - - storage_gofix - - common_testing - - image_cross - - image_test - - image_test_skopeo - container: - image: golang:1.24 - clone_script: 'mkdir -p "$CIRRUS_WORKING_DIR"' # Source code not needed - script: /bin/true diff --git a/.github/filters.yaml b/.github/filters.yaml new file mode 100644 index 0000000000..ba698e8f94 --- /dev/null +++ b/.github/filters.yaml @@ -0,0 +1,22 @@ +all: + - '.github/workflows/ci.yml' + - '.github/workflows/lima.yml' + - '.github/filters.yaml' + - 'hack/ci/**' + - 'Makefile' + - 'go.work' + - 'go.work.sum' + +storage: + - 'storage/**' + +image: + - 'image/**' + # image depends on storage, so any storage change must trigger image tests as well + - 'storage/**' + +common: + - 'common/**' + # see comment above, common depends on both image and storage + - 'image/**' + - 'storage/**' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..26d40c2e91 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,158 @@ +name: ci + +on: + push: + branches: + - main + - podman-* + pull_request: + branches: + - main + - podman-* + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + path-filter: + runs-on: ubuntu-24.04 + outputs: + all: ${{ steps.filter.outputs.all }} + storage: ${{ steps.filter.outputs.storage }} + image: ${{ steps.filter.outputs.image }} + common: ${{ steps.filter.outputs.common }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2 + id: filter + with: + filters: .github/filters.yaml + + validate: + uses: ./.github/workflows/validate.yml + + storage-test: + needs: [path-filter, validate] + if: | + github.event_name != 'pull_request' || + needs.path-filter.outputs.all == 'true' || + needs.path-filter.outputs.storage == 'true' + name: "storage ${{ matrix.distro }} ${{ matrix.driver }}" + strategy: + fail-fast: false + matrix: + distro: [fedora-current, debian-sid] + driver: [vfs, overlay, overlay-transient, fuse-overlay, fuse-overlay-whiteout, btrfs] + exclude: + - distro: debian-sid + driver: overlay-transient + uses: ./.github/workflows/lima.yml + with: + runner: cncf-ubuntu-2-8-x86 + module: storage + distro: ${{ matrix.distro }} + variant: ${{ matrix.driver }} + timeout: 20 + + storage-cross: + needs: [path-filter, validate] + if: | + github.event_name != 'pull_request' || + needs.path-filter.outputs.all == 'true' || + needs.path-filter.outputs.storage == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + name: "Storage: Cross" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: stable + cache-dependency-path: "./storage/go.sum" + - name: Cross-compile + working-directory: ./storage + run: make cross + + image-cross: + needs: [path-filter, validate] + if: | + github.event_name != 'pull_request' || + needs.path-filter.outputs.all == 'true' || + needs.path-filter.outputs.image == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + name: "Image: Cross" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: stable + cache-dependency-path: "./image/go.sum" + - name: Install dependencies + run: | + sudo apt-get -qq update + sudo apt-get -qq install -y libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev + - name: Cross-compile + working-directory: ./image + run: make cross + + image-test: + needs: [path-filter, validate] + if: | + github.event_name != 'pull_request' || + needs.path-filter.outputs.all == 'true' || + needs.path-filter.outputs.image == 'true' + name: "${{ matrix.module }} fedora-current ${{ matrix.variant }}" + strategy: + fail-fast: false + matrix: + variant: [default, openpgp, sequoia] + module: [image, image-skopeo] + uses: ./.github/workflows/lima.yml + with: + runner: cncf-ubuntu-2-8-x86 + module: ${{ matrix.module }} + distro: fedora-current + variant: ${{ matrix.variant }} + timeout: 20 + + common-test: + needs: [path-filter, validate] + if: | + github.event_name != 'pull_request' || + needs.path-filter.outputs.all == 'true' || + needs.path-filter.outputs.common == 'true' + name: "common fedora-current" + uses: ./.github/workflows/lima.yml + with: + runner: cncf-ubuntu-2-8-x86 + module: common + distro: fedora-current + timeout: 20 + + # N/B: GitHub merge protection is configured for this exact job name, DO NOT CHANGE IT. + success: + name: "Total Success" + if: always() + needs: + - path-filter + - validate + - storage-test + - storage-cross + - image-cross + - image-test + - common-test + runs-on: ubuntu-24.04 + steps: + - name: Check all required jobs + run: | + if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]] || \ + [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then + echo "One or more required jobs failed or were cancelled" + exit 1 + fi + echo "All required jobs passed or were skipped" diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 26936d4521..7d7161f9a6 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -10,4 +10,4 @@ jobs: pull-requests: write runs-on: ubuntu-latest steps: - - uses: actions/labeler@v6 + - uses: actions/labeler@bf12e9b00b37c5c0ca2b87b79b2daf7891dbda13 # v7.0.0 diff --git a/.github/workflows/lima.yml b/.github/workflows/lima.yml new file mode 100644 index 0000000000..849690f66b --- /dev/null +++ b/.github/workflows/lima.yml @@ -0,0 +1,66 @@ +name: lima + +on: + workflow_call: + inputs: + module: + required: true + type: string + distro: + required: true + type: string + variant: + required: false + type: string + runner: + required: true + type: string + timeout: + required: false + type: number + +permissions: + contents: read + +env: + # CI automation repo release for the VM image downloads in ci.sh. + # ENV is managed by renovate, do not change the name without + # updating the renovate config in the automation repo. + AUTOMATION_RELEASE: 20260722t094115z + +jobs: + lima: + name: lima # main name is set by who spawns this job + runs-on: ${{ inputs.runner }} + timeout-minutes: ${{ inputs.timeout || 20 }} # default to 20m timeout if non specified + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: lima-vm/lima-actions/setup@55627e31b78637bf254a8b2a14da8ea7d12564e5 # v1.1.0 + id: lima + with: + version: v2.1.1 + + # NOTE, we are not using a cache here as documented in the lima action. + # This is because our VM images are above 5GB and the total github limit is 10 GB which means + # it will clean out the other image basically right away making the cache useless for other tasks. + # Also most importantly while testing it the cache was slower, the VM images are hosted in the + # oracle cloud bucket which is already local to the oracle CI runners unlike the cache which is + # stored by github somewhere else. + + - name: Run test on lima + run: | # zizmor: ignore[template-injection] + ./hack/ci/ci.sh ${{ inputs.module }} ${{ inputs.distro }} ${{ inputs.variant }} + + - name: Upload logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: "logs-${{ inputs.module }}-${{ inputs.distro }}-${{ inputs.variant }}" + path: | + ./hack/ci/journal.log + ./hack/ci/audit.log + ./hack/ci/df.log + if-no-files-found: ignore diff --git a/.github/workflows/opendownstream-pr.yml b/.github/workflows/opendownstream-pr.yml deleted file mode 100644 index 3f31eac31b..0000000000 --- a/.github/workflows/opendownstream-pr.yml +++ /dev/null @@ -1,152 +0,0 @@ -name: 'Open downstream PRs' - -on: - pull_request_target: - branches: - - 'main' - paths: - - '**/*.go' - - '!vendor/**' - - '!**/*_test.go' - -jobs: - sync: - runs-on: ubuntu-latest - steps: - - name: 'Checkout Self' - uses: actions/checkout@v5 - # This checks out the code from the PR branch itself - - - name: 'Setup Go' - uses: actions/setup-go@v6 - with: - go-version: 'stable' - - - name: 'Checkout forked buildah' - uses: actions/checkout@v5 - with: - repository: 'podmanbot/buildah' # The target repository - path: 'buildah' # Checkout into a sub-directory - fetch-depth: '1' - token: ${{ secrets.VENDOR_TOKEN_PODMANBOT }} # We need to push into pobmanbot/buildah - - - name: 'Vendor Code from this repo to buildah' - run: | - # Get the current commit SHA from the PR - COMMIT_SHA="${{ github.event.pull_request.head.sha }}" - echo "Using commit SHA: $COMMIT_SHA" - - cd buildah - # Create a unique branch name based on the container-libs PR number - BRANCH_NAME="sync/container-libs-${{ github.event.pull_request.number }}" - git switch -c $BRANCH_NAME - git remote add upstream https://github.com/containers/buildah.git - git fetch upstream - git rebase upstream/main - - # Function to update module and verify - update_module() { - local module=$1 - echo "Updating module: $module" - go mod edit -replace ${module}=github.com/${{ github.event.pull_request.head.repo.full_name }}/${module#go.podman.io/}@${COMMIT_SHA} - GOWORK=off go mod tidy - } - - # Update all required modules - update_module "go.podman.io/common" - update_module "go.podman.io/storage" - update_module "go.podman.io/image/v5" - GOWORK=off go mod vendor - GOWORK=off go mod verify - - echo "Updated go.mod:" - cat go.mod - - - name: 'Commit and Push to buildah' - run: | - cd buildah - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - BRANCH_NAME="sync/container-libs-${{ github.event.pull_request.number }}" - git switch $BRANCH_NAME - - # Check if there are any changes to commit - if git diff --quiet && git diff --cached --quiet; then - echo "No changes to commit, skipping..." - echo "SKIP_PR=true" >> $GITHUB_ENV - exit 0 - fi - - git add . - git commit -m "dnm: Vendor changes from containers/container-libs#${{ github.event.pull_request.number }}" - - # Force push to update the branch if the action re-runs on 'synchronize' - git push origin $BRANCH_NAME --force - - # Verify the branch exists remotely - if ! git ls-remote --heads origin $BRANCH_NAME | grep -q $BRANCH_NAME; then - echo "Failed to push branch $BRANCH_NAME" - exit 1 - fi - - echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV - - - name: 'Create or Update Pull Request in Buildah' - id: create_pr - if: env.SKIP_PR != 'true' - env: - GH_TOKEN: ${{ secrets.VENDOR_TOKEN_PODMANBOT }} - SELF_REPO_PR_NUMBER: ${{ github.event.pull_request.number }} - SELF_REPO_PR_URL: ${{ github.event.pull_request.html_url }} - SELF_REPO_PR_TITLE: ${{ github.event.pull_request.title }} - run: | - cd buildah - - BRANCH_NAME="sync/container-libs-${{ github.event.pull_request.number }}" - PR_TITLE="Sync: ${SELF_REPO_PR_TITLE}" - PR_BODY="This PR automatically vendors changes from [container-libs#${SELF_REPO_PR_NUMBER}](${SELF_REPO_PR_URL}).\n This PR is intended solely for testing purposes and should not be merged. Ideally, upstream maintainers should review any failing tests here and close this PR once the upstream PR is merged. Maintainers of this repository do not need to take any action, but may review the changes and notify upstream maintainers of any breaking issues if they wish. \n\nThis PR is generated from a github actions workflow, please report any issues on https://github.com/containers/container-libs" - - # Check if PR already exists for this branch - echo "Searching for existing PR with branch: $BRANCH_NAME" - - EXISTING_PR_URL=$(gh pr list --repo containers/buildah --head "$BRANCH_NAME" --json url --jq '.[0].url // empty' 2>/dev/null || echo "") - - if [ -n "$EXISTING_PR_URL" ]; then - echo "Found existing PR: $EXISTING_PR_URL" - # Update existing PR title and body - gh pr edit $EXISTING_PR_URL \ - --title "$PR_TITLE" \ - --body "$PR_BODY" - echo "Updated existing PR: $EXISTING_PR_URL" - echo "pr_url=$EXISTING_PR_URL" >> $GITHUB_OUTPUT - echo "pr_action=updated" >> $GITHUB_OUTPUT - else - # Create new PR - echo "Creating new PR with head: podmanbot:$BRANCH_NAME" - NEW_PR_URL=$(gh pr create \ - --repo containers/buildah \ - --draft \ - --base main \ - --head "podmanbot:$BRANCH_NAME" \ - --title "$PR_TITLE" \ - --body "$PR_BODY") - echo "Created new PR: $NEW_PR_URL" - echo "pr_url=$NEW_PR_URL" >> $GITHUB_OUTPUT - echo "pr_action=created" >> $GITHUB_OUTPUT - fi - - - name: 'Comment on container-libs PR with the link to buildah PR' - if: env.SKIP_PR != 'true' - env: - GH_TOKEN: ${{ secrets.VENDOR_TOKEN_PODMANBOT }} - SELF_REPO_PR_NUMBER: ${{ github.event.pull_request.number }} - TARGET_REPO_PR_URL: ${{ steps.create_pr.outputs.pr_url }} - PR_ACTION: ${{ steps.create_pr.outputs.pr_action }} - run: | - if [ "${{ env.PR_ACTION }}" = "created" ]; then - COMMENT_BODY="✅ A new PR has been created in buildah to vendor these changes: **${{ env.TARGET_REPO_PR_URL }}**" - gh pr comment ${{ env.SELF_REPO_PR_NUMBER }} \ - --repo ${{ github.repository }} \ - --body "$COMMENT_BODY" - fi diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 231caa1986..adf3e6ce1d 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -1,41 +1,39 @@ name: validate on: - push: - branches: - - main - - podman-* - pull_request: - branches: - - main - - podman-* + workflow_call: -permissions: read-all +permissions: + contents: read env: LINT_VERSION: v2.5.0 jobs: - codespell: + basic: runs-on: ubuntu-24.04 + timeout-minutes: 10 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: install deps # Version of codespell bundled with Ubuntu is way old, so use pip. run: pip install --break-system-packages codespell==v2.4.1 - name: run codespell run: make codespell + - name: check-ci-yaml + run: make check-ci-yaml lint: runs-on: ubuntu-24.04 + timeout-minutes: 10 defaults: run: working-directory: ./common steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 2 - - uses: actions/setup-go@v6 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: 1.25.x # By default the go cache will only use go.sum in the root which we don't have, @@ -47,19 +45,19 @@ jobs: sudo apt-get -qq update sudo apt-get -qq install libseccomp-dev libgpgme-dev libbtrfs-dev libsubid-dev - name: lint-common - uses: golangci/golangci-lint-action@v8 + uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 with: version: "${{ env.LINT_VERSION }}" args: --verbose working-directory: ./common - name: lint-image - uses: golangci/golangci-lint-action@v8 + uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 with: version: "${{ env.LINT_VERSION }}" args: --verbose working-directory: ./image - name: lint-storage - uses: golangci/golangci-lint-action@v8 + uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 with: version: "${{ env.LINT_VERSION }}" args: --verbose @@ -72,19 +70,36 @@ jobs: # only run this on PRs if: github.event_name == 'pull_request' runs-on: ubuntu-24.04 + timeout-minutes: 10 + env: + # Base commit sha that we use the lint the commit from + EPOCH_TEST_COMMIT: "${{ github.event.pull_request.base.sha }}" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # By default github actions creates a merge commit which fails the validation, # we only must validate the actual commits of the author. ref: ${{ github.event.pull_request.head.sha }} fetch-depth: ${{ github.event.pull_request.commits }} - - uses: actions/setup-go@v6 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: 1.25.x # See comment on lint task cache-dependency-path: "**/go.sum" - name: run git-validation - # We validate all commits as we only fetched the number of commits in the PR above, - # by default git-validation has some special github action handling but that seems broken. - run: make .install.gitvalidation && git-validation -no-github + run: make .install.gitvalidation && make git-validation + + go-tidy: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: 1.25.x + # See comment on lint task + cache-dependency-path: "**/go.sum" + - name: tidy + run: make tidy + - name: check if tree is clean + run: hack/tree_status.sh diff --git a/Makefile b/Makefile index 4fe5e092a0..17a8f8eca5 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,11 @@ export PATH := $(PATH):${GOBIN} EPOCH_TEST_COMMIT ?= $(shell git merge-base $${DEST_BRANCH:-main} HEAD) -validate: codespell git-validation lint +validate: codespell git-validation lint check-ci-yaml + +.PHONY: check-ci-yaml +check-ci-yaml: + hack/ci/ci_yaml_test.py .PHONY: codespell codespell: @@ -41,7 +45,10 @@ install.tools: .install.gitvalidation .install.golangci-lint .install.md2man .PHONY: git-validation git-validation: .install.gitvalidation - git-validation -q -run DCO,short-subject,dangling-whitespace -range "$(EPOCH_TEST_COMMIT)..HEAD" +ifndef EPOCH_TEST_COMMIT + $(error EPOCH_TEST_COMMIT is empty) +endif + GIT_CHECK_EXCLUDE="./vendor" git-validation $(if $(CI),,-q) -run DCO,short-subject,dangling-whitespace -range "$(EPOCH_TEST_COMMIT)..HEAD" .PHONY: lint lint: .install.golangci-lint diff --git a/common/Makefile b/common/Makefile index 9d7a1c3f5b..7e2a39d2bf 100644 --- a/common/Makefile +++ b/common/Makefile @@ -102,7 +102,6 @@ test: test-unit test-unit: netavark-testplugin go test --tags seccomp,$(BUILDTAGS) -v ./... go test --tags remote,$(BUILDTAGS) -v ./pkg/config - go test --tags cni,$(BUILDTAGS) -v ./libnetwork/cni clean: ## Clean artifacts $(MAKE) -C docs clean diff --git a/common/libnetwork/netavark/run_test.go b/common/libnetwork/netavark/run_test.go index e6573953c1..7c9ebf5669 100644 --- a/common/libnetwork/netavark/run_test.go +++ b/common/libnetwork/netavark/run_test.go @@ -99,9 +99,9 @@ var _ = Describe("run netavark", func() { _ = netns.UnmountNS(netNSContainer.Path()) _ = netNSContainer.Close() }) - // Force iptables driver, firewalld is broken inside the extra + // Force nftables driver, firewalld is broken inside the extra // namespace because it still connects to firewalld on the host. - t.Setenv("NETAVARK_FW", "iptables") + t.Setenv("NETAVARK_FW", "nftables") }) JustBeforeEach(func() { diff --git a/hack/ci/ci.sh b/hack/ci/ci.sh new file mode 100755 index 0000000000..ec08c34cc9 --- /dev/null +++ b/hack/ci/ci.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +set -eo pipefail + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" && pwd ) + +source "$SCRIPT_DIR/lib.sh" + +AUTOMATION_RELEASE="${AUTOMATION_RELEASE:-$(get_automation_release)}" +LIMA_VM_NAME=container-libs-ci + +MODULE=${1:?must give module as first argument} + +REPO_DIR="$SCRIPT_DIR/../.." + +parse_args "$@" + +IMAGE="$DISTRO_NAME.x86_64.qcow2.zst" + +IMAGE_URL="https://objectstorage.us-ashburn-1.oraclecloud.com/n/id0lmbbwgcdv/b/podman-ci-vm-images/o/releases/$AUTOMATION_RELEASE/$IMAGE" + +trap 'limactl delete --force $LIMA_VM_NAME' EXIT + +echo "::group::Starting VM" +limactl --yes start --plain --name=$LIMA_VM_NAME --cpus $(nproc) --memory 4 --nested-virt \ + --set ".images=[{\"location\":\"$IMAGE_URL\", \"arch\": \"x86_64\"}]" \ + "$SCRIPT_DIR/template.lima.yml" + +limactl copy "$REPO_DIR" "$LIMA_VM_NAME:/var/tmp/container-libs" +echo "::endgroup::" + +set +e + +limactl shell --workdir /var/tmp/container-libs $LIMA_VM_NAME ./hack/ci/runner.sh "${@}" +rc=$? + +echo "::group::Collecting logs" +limactl shell --workdir /var/tmp/container-libs $LIMA_VM_NAME sudo hack/ci/logcollector.sh journal &> "$SCRIPT_DIR/journal.log" +limactl shell --workdir /var/tmp/container-libs $LIMA_VM_NAME sudo hack/ci/logcollector.sh audit &> "$SCRIPT_DIR/audit.log" +limactl shell --workdir /var/tmp/container-libs $LIMA_VM_NAME sudo hack/ci/logcollector.sh df &> "$SCRIPT_DIR/df.log" +echo "::endgroup::" + +exit $rc diff --git a/hack/ci/ci_yaml_test.py b/hack/ci/ci_yaml_test.py new file mode 100755 index 0000000000..6635257b78 --- /dev/null +++ b/hack/ci/ci_yaml_test.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 + +""" +Verify contents of .github/workflows/ci.yml meets some basic expectations +""" + +import sys +import os +import unittest +import yaml +import re + +# Assumes directory structure of this file relative to repo. +SCRIPT_DIRPATH = os.path.dirname(os.path.realpath(__file__)) +REPO_ROOT = os.path.realpath(os.path.join(SCRIPT_DIRPATH, '..', '..')) + + +class TestCase(unittest.TestCase): + + CI_YAML = None + + def setUp(self): + with open(os.path.join(REPO_ROOT, '.github/workflows/ci.yml')) as ci_yaml: + self.CI_YAML = yaml.safe_load(ci_yaml.read()) + + # Critical for the merge protection to work as we only block on this task. + def test_success_deps(self): + """success task depends on all others""" + all_tasks = list(self.CI_YAML['jobs'].keys()) + # need to remove success from the list as it cannot depend on itself + all_tasks.remove('success') + needs = self.CI_YAML['jobs']['success']['needs'] + self.assertCountEqual(needs, all_tasks) + + def test_gh_actions_are_pinned_by(self): + """ensure all actions are pinned by digest and have version comment""" + # Note local paths are allowed, i.e. uses: ./.github/workflows/lima.yml + pattern = re.compile(r"uses:\s+(?:(\./[\w./-]+)(?:\s+#.*)?|([\w.-]+/[\w./-]+)@([a-f0-9]{40})\s+#\s*(.+))$") + dir = os.path.join(REPO_ROOT, '.github/workflows') + for name in os.listdir(dir): + with open(os.path.join(dir, name)) as file: + for i, line in enumerate(file, 1): + if 'uses:' in line: + self.assertRegex(line, pattern, msg=f"Action must be pinned with a version number comment, file: .github/workflows/{name}:{i}") + +if __name__ == "__main__": + unittest.main() diff --git a/hack/ci/lib.sh b/hack/ci/lib.sh new file mode 100644 index 0000000000..839588c96f --- /dev/null +++ b/hack/ci/lib.sh @@ -0,0 +1,76 @@ +# This must be sourced from other scripts to work. + +OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | tr -d '.')" +OS_RELEASE_ID="$(source /etc/os-release; echo $ID)" +OS_REL_VER="$OS_RELEASE_ID-$OS_RELEASE_VER" + +function die() { + echo "$1" >&2 + exit 1 +} + +function parse_args() { + # module name: storage, image, image-skopeo, common + MODULE=${1:?must give module as first argument} + # distro: fedora-current, debian-sid + DISTRO_NAME=${2:?must give distro as second argument} + # variant: driver for storage, buildtag for image, unused for common + VARIANT=${3:-} + + validate_module "$MODULE" + validate_distro "$DISTRO_NAME" + validate_variant "$MODULE" "$VARIANT" +} + +function validate_module() { + case "$1" in + "storage"|"image"|"image-skopeo"|"common") + ;; + *) + die "Unknown MODULE '$1', expected: storage, image, image-skopeo, common" + ;; + esac +} + +function validate_distro() { + case "$1" in + "fedora-current"|"debian-sid") + ;; + *) + die "Unknown DISTRO_NAME '$1', expected: fedora-current, debian-sid" + ;; + esac +} + +function validate_variant() { + local module="$1" + local variant="$2" + case "$module" in + storage) + case "$variant" in + "vfs"|"overlay"|"overlay-transient"|"fuse-overlay"|"fuse-overlay-whiteout"|"btrfs") + ;; + *) + die "Unknown storage variant '$variant', expected: vfs, overlay, overlay-transient, fuse-overlay, fuse-overlay-whiteout, btrfs" + ;; + esac + ;; + image|image-skopeo) + case "$variant" in + ""|"default"|"openpgp"|"sequoia") + ;; + *) + die "Unknown image variant '$variant', expected: default, openpgp, sequoia" + ;; + esac + ;; + common) + ;; + esac +} + +# Reads and prints the AUTOMATION_RELEASE value from the workflow file. +# Used for local runs where ci.sh is not triggered by gh actions. +function get_automation_release() { + sed -En 's/.*AUTOMATION_RELEASE:\s(.*)/\1/p' .github/workflows/lima.yml +} diff --git a/hack/ci/logcollector.sh b/hack/ci/logcollector.sh new file mode 100755 index 0000000000..0a303edc35 --- /dev/null +++ b/hack/ci/logcollector.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +set -e + +SCRIPT_DIR=$(dirname $0) + +# shellcheck source=hack/ci/lib.sh +source "$SCRIPT_DIR/lib.sh" + +# Assume there are other log collection commands to follow - Don't +# let one break another that may be useful, but also keep any +# actual script-problems fatal so they are noticed right away. +showrun() { + echo '+ '$(printf " %q" "$@") + set +e + echo '------------------------------------------------------------' + "$@" + local status=$? + [[ $status -eq 0 ]] || \ + echo "[ rc = $status -- proceeding anyway ]" + echo '------------------------------------------------------------' + set -e +} + +bad_os_id_ver() { + die "Unknown OS '$OS_RELEASE_ID'" +} + +case $1 in + audit) + case $OS_RELEASE_ID in + debian) showrun cat /var/log/kern.log ;; + fedora) showrun cat /var/log/audit/audit.log ;; + *) bad_os_id_ver ;; + esac + ;; + df) showrun df -lhTx tmpfs ;; + journal) showrun journalctl -b ;; + packages) + PKG_NAMES=(\ + golang + podman + skopeo + btrfs-progs + fuse-overlayfs + ) + case $OS_RELEASE_ID in + fedora) + cat /etc/fedora-release + PKG_LST_CMD='rpm -q --qf=%{N}-%{V}-%{R}-%{ARCH}\n' + PKG_NAMES+=(\ + gpgme-devel + device-mapper-devel + libseccomp-devel + ) + ;; + debian) + cat /etc/issue + PKG_LST_CMD='dpkg-query --show --showformat=${Package}-${Version}-${Architecture}\n' + PKG_NAMES+=(\ + libgpgme-dev + libdevmapper-dev + libseccomp-dev + ) + ;; + *) bad_os_id_ver ;; + esac + echo "Kernel: " $(uname -r) + echo "Cgroups: " $(stat -f -c %T /sys/fs/cgroup) + # Any not-present packages will be listed as such + $PKG_LST_CMD "${PKG_NAMES[@]}" | sort -u + ;; + ip) showrun sh -c "ip addr && ip route && ip -6 route" ;; + *) die "Warning, $(basename $0) doesn't know how to handle the parameter '$1'" +esac diff --git a/hack/ci/runner.sh b/hack/ci/runner.sh new file mode 100755 index 0000000000..b071285212 --- /dev/null +++ b/hack/ci/runner.sh @@ -0,0 +1,213 @@ +#!/usr/bin/env bash + +SKOPEO_CI_BRANCH=release-1.22 + +# This script is only intended to be run inside the Lima VM to configure it and start the tests. +# Do not run locally. + +set -eo pipefail + +export PATH="/usr/sbin:/usr/local/sbin:$PATH" + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" && pwd ) + +source "$SCRIPT_DIR/lib.sh" + +MODULE=${1:?must give module as first argument} + +parse_args "$@" + +############################################################################### +# Environment preparation +############################################################################### + +prepare_storage_env() { + for i in $(seq 0 1023); do + [ -e /dev/loop$i ] || sudo mknod /dev/loop$i b 7 $i 2>/dev/null || true + done +} + +############################################################################### +# Test runners +############################################################################### + +run_storage() { + cd storage + make local-binary + + SUDO="sudo -E env PATH=$PATH GOPATH=$(go env GOPATH) HOME=$HOME" + + case "$VARIANT" in + overlay) + $SUDO make STORAGE_DRIVER=overlay local-test-integration local-test-unit + ;; + overlay-transient) + $SUDO make STORAGE_DRIVER=overlay STORAGE_TRANSIENT=1 local-test-integration local-test-unit + ;; + fuse-overlay) + $SUDO make STORAGE_DRIVER=overlay STORAGE_OPTION=overlay.mount_program=/usr/bin/fuse-overlayfs local-test-integration local-test-unit + ;; + fuse-overlay-whiteout) + $SUDO FUSE_OVERLAYFS_DISABLE_OVL_WHITEOUT=1 make STORAGE_DRIVER=overlay STORAGE_OPTION=overlay.mount_program=/usr/bin/fuse-overlayfs local-test-integration local-test-unit + ;; + vfs) + $SUDO make STORAGE_DRIVER=vfs local-test-integration local-test-unit + ;; + btrfs) + if [[ "$(./hack/btrfs_tag.sh)" =~ exclude_graphdriver_btrfs ]]; then + echo "Built without btrfs, so we can't test it" + exit 1 + fi + if ! grep -q " btrfs$" /proc/filesystems; then + sudo modprobe btrfs || true + if ! grep -q " btrfs$" /proc/filesystems; then + echo "Kernel does not support btrfs" + exit 1 + fi + fi + if ! command -v mkfs.btrfs &> /dev/null; then + echo "mkfs.btrfs not installed" + exit 1 + fi + tmpdir=$(mktemp -d) + trap "sudo umount -l $tmpdir; rm -f btrfs.img" EXIT + truncate -s 0 btrfs.img + fallocate -l 1G btrfs.img + sudo mkfs.btrfs btrfs.img + sudo mount -o loop btrfs.img $tmpdir + $SUDO TMPDIR="$tmpdir" make STORAGE_DRIVER=btrfs local-test-integration local-test-unit + ;; + *) + die "Unknown storage variant: $VARIANT" + ;; + esac +} + +run_image() { + cd image + + local BUILDTAGS="" + case "$VARIANT" in + default|"") BUILDTAGS="" ;; + openpgp) BUILDTAGS="containers_image_openpgp" ;; + sequoia) BUILDTAGS="containers_image_sequoia" ;; + esac + + GOPATH_DIR="$(go env GOPATH)" + GOROOT_DIR="$(go env GOROOT)" + GOSRC="$(cd .. && pwd)" + + git config --global --add safe.directory "$GOSRC" + + # Run root tests for storage-dependent tests + + # Hacky solution to find test that must be run as root. + # This looks for the ensureTestCanCreateImages() test function call and gets the + # function name where it is called via git grep, + # then trims the line to only show the actual function name and add "^$" around it + # since go test commands only accepts a single regex. + # Then join all names with "|" with paste to again build up a single regex string + # that matches all these names. + # + # test_filter must have the $ duplicated because make expands the value + # (and there seems to be no trivial way to avoid that while defining the variable + # as an argument?!) + test_filter=$(git grep -h --show-function ensureTestCanCreateImages ./storage | + sed -n 's/func \(Test[[:alnum:]]*\)(.*/^\1$$/p' | + paste -sd "|" -) + if [ -n "$test_filter" ]; then + sudo -E env "PATH=$PATH" "GOPATH=$GOPATH_DIR" \ + make test "BUILDTAGS=$BUILDTAGS" "TESTFLAGS=-v -run '$test_filter'" TEST_PACKAGES=./storage + fi + + # Restore permissions + sudo chown -R $(id -u):$(id -g) "$GOPATH_DIR" + + # Run rootless tests + cleanup() { + $GOSRC/image/signature/sigstore/rekor/testdata/start-rekor.sh ci remove || true + } + trap cleanup EXIT + + # start custom rekor which is needed by the tests + $GOSRC/image/signature/sigstore/rekor/testdata/start-rekor.sh ci + make test BUILDTAGS='$BUILDTAGS' TESTFLAGS=-v REKOR_SERVER_URL='http://127.0.0.1:3000' +} + +run_image_skopeo() { + local BUILDTAGS="" + case "$VARIANT" in + default|"") BUILDTAGS="" ;; + openpgp) BUILDTAGS="containers_image_openpgp" ;; + sequoia) BUILDTAGS="containers_image_sequoia" ;; + esac + + GOSRC="$(pwd)" + SKOPEO_PATH="/var/tmp/skopeo" + AUTOMATION_RELEASE="${AUTOMATION_RELEASE:-$(get_automation_release)}" + SKOPEO_CIDEV_CONTAINER_FQIN="ghcr.io/podman-container-tools/skopeo_cidev:$AUTOMATION_RELEASE" + + sudo podman pull --quiet "$SKOPEO_CIDEV_CONTAINER_FQIN" + ctr_id=$(sudo podman create "$SKOPEO_CIDEV_CONTAINER_FQIN") + mnt=$(sudo podman mount "$ctr_id") + sudo cp -a "$mnt/usr/local/bin/." /usr/local/bin/ + sudo mkdir -p /registry + sudo cp -a "$mnt/atomic-registry-config.yml" / + sudo podman umount --latest + sudo podman rm --latest + + git clone -b "$SKOPEO_CI_BRANCH" https://github.com/podman-container-tools/skopeo.git "$SKOPEO_PATH" + cd "$SKOPEO_PATH" + go mod edit -replace "go.podman.io/storage=$GOSRC/storage" + go mod edit -replace "go.podman.io/image/v5=$GOSRC/image" + go mod edit -replace "go.podman.io/common=$GOSRC/common" + make vendor + + make bin/skopeo "BUILDTAGS=$BUILDTAGS" + sudo make install PREFIX=/usr/local "BUILDTAGS=$BUILDTAGS" + + make test-unit-local "BUILDTAGS=$BUILDTAGS" + + sudo podman system reset --force + export SKOPEO_CONTAINER_TESTS=1 + sudo -E env "PATH=/usr/local/bin:$PATH" "GOPATH=$(go env GOPATH)" "SKOPEO_CONTAINER_TESTS=$SKOPEO_CONTAINER_TESTS" \ + make test-integration-local "BUILDTAGS=$BUILDTAGS" + + sudo podman system reset --force + sudo -E env "PATH=/usr/local/bin:$PATH" "GOPATH=$(go env GOPATH)" "SKOPEO_CONTAINER_TESTS=$SKOPEO_CONTAINER_TESTS" \ + make test-system-local "BUILDTAGS=$BUILDTAGS" +} + +run_common() { + cd common + NETAVARK_BINARY=/usr/libexec/podman/netavark + export NETAVARK_BINARY + + make build + make build-cross + + sudo -E env "PATH=$PATH" "GOPATH=$(go env GOPATH)" "HOME=$HOME" \ + make test +} + +############################################################################### +# Main dispatch +############################################################################### + + +# Normalize module name for function dispatch (image-skopeo -> image_skopeo) +MODULE_FUNC="${MODULE//-/_}" + +if type -t prepare_${MODULE_FUNC}_env &>/dev/null; then + echo "::group::Preparing environment for $MODULE" + prepare_${MODULE_FUNC}_env + echo "::endgroup::" +fi + +echo "::group::Logging system info" +"$SCRIPT_DIR/logcollector.sh" packages +"$SCRIPT_DIR/logcollector.sh" ip +echo "::endgroup::" + +echo "Starting tests: $MODULE $VARIANT" +run_${MODULE_FUNC} diff --git a/hack/ci/template.lima.yml b/hack/ci/template.lima.yml new file mode 100644 index 0000000000..3b82607142 --- /dev/null +++ b/hack/ci/template.lima.yml @@ -0,0 +1,4 @@ + +# images is empty and should be override on the cli +# --set '.images=[{"location":"...", "arch": "x86_64"}]' +images: diff --git a/common/hack/tree_status.sh b/hack/tree_status.sh similarity index 80% rename from common/hack/tree_status.sh rename to hack/tree_status.sh index 8c9c2abe8f..899a7e733f 100755 --- a/common/hack/tree_status.sh +++ b/hack/tree_status.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -SUGGESTION="${SUGGESTION:-run \"make vendor\" and commit all changes.}" +SUGGESTION="${SUGGESTION:-run \"make tidy\" and commit all changes.}" STATUS=$(git status --porcelain) if [[ -z $STATUS ]] diff --git a/image/contrib/cirrus/runner.sh b/image/contrib/cirrus/runner.sh deleted file mode 100755 index 2b802f64f9..0000000000 --- a/image/contrib/cirrus/runner.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash - -# This script is intended to be executed by automation or humans -# under a hack/get_ci_vm.sh context. Use under any other circumstances -# is unlikely to function. - -set -e - -if [[ -r "/etc/automation_environment" ]]; then - source /etc/automation_environment - source $AUTOMATION_LIB_PATH/common_lib.sh -else - ( - echo "WARNING: It does not appear that containers/automation was installed." - echo " Functionality of most of ${BASH_SOURCE[0]} will be negatively" - echo " impacted." - ) > /dev/stderr -fi - -export "PATH=$PATH:$GOPATH/bin" - -_run_setup() { - req_env_vars SKOPEO_PATH SKOPEO_CI_BRANCH GOSRC - - rm -rf "${SKOPEO_PATH}" - git clone -b ${SKOPEO_CI_BRANCH} \ - https://github.com/containers/skopeo.git ${SKOPEO_PATH} - - cd "${SKOPEO_PATH}" - if [[ -n "$SKOPEO_PR" ]] && [[ $SKOPEO_PR -gt 1000 ]]; then - warn "Fetching and checking out code from skopeo pull-request #$SKOPEO_PR" - git fetch origin "+refs/pull/$SKOPEO_PR/head" - git checkout FETCH_HEAD - fi - - go mod edit -replace go.podman.io/storage="$GOSRC/storage" - go mod edit -replace go.podman.io/image/v5="$GOSRC/image" - go mod edit -replace go.podman.io/common="$GOSRC/common" - - "${SKOPEO_PATH}/${SCRIPT_BASE}/runner.sh" setup -} - -_run_image_tests() { - req_env_vars GOPATH GOSRC - - # Hacky solution to find test that must be run as root. - # This looks for the ensureTestCanCreateImages() test function call and gets the - # function name where it is called via git grep, - # then trims the line to only show the actual function name and add "^$" around it - # since go test commands only accepts a single regex. - # Then join all names with "|" with paste to again build up a single regex string - # that matches all these names. - # With that we don't have to run everything twice and can just run the ones that - # actually need to be root. - # Note we must run git before we switch/chown to the user because it will error - # out otherwise since the file ownership doesn't match. - test_filter=$(git grep -h --show-function ensureTestCanCreateImages ./storage | - sed -n 's/func \(Test[[:alnum:]]*\)(.*/^\1\$\$/p' | - paste -sd "|" -) - showrun make test "BUILDTAGS='$BUILDTAGS'" "TESTFLAGS=-v -run '$test_filter'" TEST_PACKAGES=./storage - - # Most tests in this repo are intended to run as a regular user. - ROOTLESS_USER="testuser$RANDOM" - msg "Setting up rootless user '$ROOTLESS_USER'" - cd $GOSRC || exit 1 - # Guarantee independence from specific values - rootless_uid=$((RANDOM+1000)) - rootless_gid=$((RANDOM+1000)) - msg "Creating $rootless_uid:$rootless_gid $ROOTLESS_USER user" - groupadd -g $rootless_gid $ROOTLESS_USER - useradd -g $rootless_gid -u $rootless_uid --no-user-group --create-home $ROOTLESS_USER - - msg "Setting ownership of $GOPATH and $GOSRC" - chown -R $ROOTLESS_USER:$ROOTLESS_USER "$GOPATH" "$GOSRC" - - msg "Creating ssh key pairs" - mkdir -p "/root/.ssh" "/home/$ROOTLESS_USER/.ssh" - ssh-keygen -t ed25519 -P "" -f "/root/.ssh/id_ed25519" - - msg "Setup authorized_keys" - cat /root/.ssh/*.pub >> /home/$ROOTLESS_USER/.ssh/authorized_keys - - msg "Configure ssh file permissions" - chmod -R 700 "/root/.ssh" - chmod -R 700 "/home/$ROOTLESS_USER/.ssh" - chown -R $ROOTLESS_USER:$ROOTLESS_USER "/home/$ROOTLESS_USER/.ssh" - - msg "Ensure the ssh daemon is up and running within 5 minutes" - systemctl is-active sshd || \ - systemctl start sshd - - msg "Setup known_hosts for root" - ssh-keyscan localhost > /root/.ssh/known_hosts \ - - msg "Start rekor server as $ROOTLESS_USER" - showrun ssh $ROOTLESS_USER@localhost $GOSRC/signature/sigstore/rekor/testdata/start-rekor.sh ci - # remove rekor server on function exit - trap "ssh $ROOTLESS_USER@localhost $GOSRC/signature/sigstore/rekor/testdata/start-rekor.sh ci remove" RETURN - - msg "Executing tests as $ROOTLESS_USER" - showrun ssh $ROOTLESS_USER@localhost make -C $GOSRC test "BUILDTAGS='$BUILDTAGS'" "TESTFLAGS=-v" "REKOR_SERVER_URL='http://127.0.0.1:3000'" -} - -req_env_vars GOSRC - -handler="_run_${1}" -if [ "$(type -t $handler)" != "function" ]; then - die "Unknown/Unsupported command-line argument '$1'" -fi - -msg "************************************************************" -msg "Runner executing $1 on $OS_REL_VER" -msg "************************************************************" - -cd "$GOSRC" -$handler diff --git a/storage/contrib/cirrus/build_and_test.sh b/storage/contrib/cirrus/build_and_test.sh deleted file mode 100755 index a1742facc4..0000000000 --- a/storage/contrib/cirrus/build_and_test.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source $(dirname $0)/lib.sh - -cd $GOSRC/storage -pwd -ls -l -showrun make local-binary - -case $TEST_DRIVER in - overlay) - showrun make STORAGE_DRIVER=overlay local-test-integration local-test-unit - ;; - overlay-transient) - showrun make STORAGE_DRIVER=overlay STORAGE_TRANSIENT=1 local-test-integration local-test-unit - ;; - fuse-overlay) - showrun make STORAGE_DRIVER=overlay STORAGE_OPTION=overlay.mount_program=/usr/bin/fuse-overlayfs local-test-integration local-test-unit - ;; - fuse-overlay-whiteout) - showrun make STORAGE_DRIVER=overlay FUSE_OVERLAYFS_DISABLE_OVL_WHITEOUT=1 STORAGE_OPTION=overlay.mount_program=/usr/bin/fuse-overlayfs local-test-integration local-test-unit - ;; - vfs) - showrun make STORAGE_DRIVER=vfs local-test-integration local-test-unit - ;; - btrfs) - # Fedora: Needs btrfs-progs, btrfs-progs-devel - # Debian: Needs btrfs-progs, libbtrfs-dev - if [[ "$(./hack/btrfs_tag.sh)" =~ exclude_graphdriver_btrfs ]]; then - die "Built without btrfs, so we can't test it" - fi - if ! check_filesystem_supported $TEST_DRIVER ; then - die "This CI VM does not support $TEST_DRIVER in its kernel" - fi - if test -z "$(which mkfs.btrfs 2> /dev/null)" ; then - die "This CI VM does not have mkfs.btrfs installed" - fi - tmpdir=$(mktemp -d) - if [ -z "$tmpdir" ]; then - die "Error creating temporary directory" - fi - trap "umount -l $tmpdir; rm -f $GOSRC/$TEST_DRIVER.img" EXIT - truncate -s 0 $GOSRC/$TEST_DRIVER.img - fallocate -l 1G $GOSRC/$TEST_DRIVER.img - mkfs.btrfs $GOSRC/$TEST_DRIVER.img - mount -o loop $GOSRC/$TEST_DRIVER.img $tmpdir - TMPDIR="$tmpdir" showrun make STORAGE_DRIVER=$TEST_DRIVER local-test-integration local-test-unit - ;; - zfs) - # Debian: Needs zfsutils - if ! check_filesystem_supported $TEST_DRIVER ; then - die "This CI VM does not support $TEST_DRIVER in its kernel" - fi - if test -z "$(which zpool 2> /dev/null)" ; then - die "This CI VM does not have zpool installed" - fi - if test -z "$(which zfs 2> /dev/null)" ; then - die "This CI VM does not have zfs installed" - fi - tmpfile=$(mktemp -p $GOSRC) - truncate -s 0 $tmpfile - fallocate -l 1G $tmpfile - zpool=$(basename $tmpfile) - zpool create $zpool $tmpfile - trap "zfs destroy -Rf $zpool/tmp; zpool destroy -f $zpool; rm -f $tmpfile" EXIT - zfs create $zpool/tmp - TMPDIR="/$zpool/tmp" showrun make STORAGE_DRIVER=$TEST_DRIVER local-test-integration local-test-unit - # Ensure no datasets are held open prior to `zfs destroy` trap. - datasets=$(lsns -J -t mnt --output-all | jq '.namespaces[]|select(.command=="sleep 1000s").pid') - if [[ -n "$datasets" ]]; then - kill $datasets - fi - ;; - *) - die "Unknown/Unsupported \$TEST_DRIVER=$TEST_DRIVER (see .cirrus.yml and $(basename $0))" - ;; -esac diff --git a/storage/contrib/cirrus/lib.sh b/storage/contrib/cirrus/lib.sh deleted file mode 100755 index fc121d7e7d..0000000000 --- a/storage/contrib/cirrus/lib.sh +++ /dev/null @@ -1,128 +0,0 @@ - - -# Library of common, shared utility functions. This file is intended -# to be sourced by other scripts, not called directly. - -# BEGIN Global export of all variables -set -a - -# Due to differences across platforms and runtime execution environments, -# handling of the (otherwise) default shell setup is non-uniform. Rather -# than attempt to workaround differences, simply force-load/set required -# items every time this library is utilized. -USER="$(whoami)" -HOME="$(getent passwd $USER | cut -d : -f 6)" -# Some platforms set and make this read-only -[[ -n "$UID" ]] || \ - UID=$(getent passwd $USER | cut -d : -f 3) - -# Automation library installed at image-build time, -# defining $AUTOMATION_LIB_PATH in this file. -if [[ -r "/etc/automation_environment" ]]; then - source /etc/automation_environment -fi -# shellcheck disable=SC2154 -if [[ -n "$AUTOMATION_LIB_PATH" ]]; then - # shellcheck source=/usr/share/automation/lib/common_lib.sh - source $AUTOMATION_LIB_PATH/common_lib.sh -else - ( - echo "WARNING: It does not appear that containers/automation was installed." - echo " Functionality of most of this library will be negatively impacted" - echo " This ${BASH_SOURCE[0]} was loaded by ${BASH_SOURCE[1]}" - ) > /dev/stderr -fi - -# Essential default paths, many are overridden when executing under Cirrus-CI -# others are duplicated here, to assist in debugging. -GOPATH="${GOPATH:-/var/tmp/go}" -GOCACHE="${GOCACHE:-$GOPATH/cache/go-build}" -# called processes like `make` and other tools need these vars. -eval "$(go env)" -CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-$GOPATH/src/github.com/containers/storage}" -GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}" -PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH" -SCRIPT_BASE=${GOSRC}/contrib/cirrus - -CI="${CI:-false}" -CIRRUS_CI="${CIRRUS_CI:-false}" -DEST_BRANCH="${DEST_BRANCH:-main}" -CONTINUOUS_INTEGRATION="${CONTINUOUS_INTEGRATION:-false}" -CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-storage} -# Cirrus only sets $CIRRUS_BASE_SHA properly for PRs, but $EPOCH_TEST_COMMIT -# needs to be set from this value in order for `make validate` to run properly. -# When running get_ci_vm.sh, most $CIRRUS_xyz variables are empty. Attempt -# to accommodate both branch and get_ci_vm.sh testing by discovering the base -# branch SHA value. -if [[ -z "$CIRRUS_BASE_SHA" ]] && [[ -z "$CIRRUS_TAG" ]] -then # Operating on a branch, or under `get_ci_vm.sh` - CIRRUS_BASE_SHA=$(git rev-parse ${UPSTREAM_REMOTE:-origin}/$DEST_BRANCH) -elif [[ -z "$CIRRUS_BASE_SHA" ]] -then # Operating on a tag - CIRRUS_BASE_SHA=$(git rev-parse HEAD) -fi -# The starting place for linting and code validation -EPOCH_TEST_COMMIT="$CIRRUS_BASE_SHA" - -# Unsafe env. vars for display -SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(^GC[EP]..+)|(SSH)' - -# Working with dnf + timeout/retry -SHORT_DNFY='lilto dnf -y' -LONG_DNFY='bigto dnf -y' -# Working with apt under Debian/Ubuntu automation is a PITA, make it easy -# Avoid some ways of getting stuck waiting for user input -DEBIAN_FRONTEND=noninteractive -# Short-cut for frequently used base command -SUDOAPTGET='sudo -E apt-get -q --yes' -# Short list of packages or quick-running command -SHORT_APTGET="lilto $SUDOAPTGET" -# Long list / long-running command -LONG_APTGET="bigto $SUDOAPTGET" - -# Packages in generic VM images that conflict with containers/storage testing -RPMS_CONFLICTING="gcc-go" -DEBS_CONFLICTING="" - -# END Global export of all variables -set +a - -bad_os_id_ver() { - die "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $(basename $0)" -} - -lilto() { err_retry 8 1000 "" "$@"; } # just over 4 minutes max -bigto() { err_retry 7 5670 "" "$@"; } # 12 minutes max - -install_fuse_overlayfs_from_git(){ - wd=$(pwd) - DEST="$GOPATH/src/github.com/containers/fuse-overlayfs" - rm -rf "$DEST" - ooe.sh git clone https://github.com/containers/fuse-overlayfs.git "$DEST" - cd "$DEST" - ooe.sh git fetch origin --tags - ooe.sh ./autogen.sh - ooe.sh ./configure - ooe.sh make - sudo make install prefix=/usr - cd $wd -} - -install_bats_from_git(){ - git clone https://github.com/bats-core/bats-core --depth=1 - sudo ./bats-core/install.sh /usr - rm -rf bats-core - mkdir -p ~/.parallel - touch ~/.parallel/will-cite -} - -check_filesystem_supported(){ - if ! grep -q " $1\$" /proc/filesystems ; then - modprobe $1 > /dev/null 2> /dev/null || :en - if ! grep -q " $1\$" /proc/filesystems ; then - echo "This CI VM does not support $TEST_DRIVER in its kernel" - false - fi - fi - true -} diff --git a/storage/contrib/cirrus/setup.sh b/storage/contrib/cirrus/setup.sh deleted file mode 100755 index 97e79b8521..0000000000 --- a/storage/contrib/cirrus/setup.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source $(dirname $0)/lib.sh - -req_env_vars GOSRC OS_RELEASE_ID OS_RELEASE_VER SHORT_APTGET TEST_DRIVER - -show_env_vars - -cd $GOSRC -msg "Setting up $OS_RELEASE_ID $OS_RELEASE_VER" -case "$OS_RELEASE_ID" in - fedora) - [[ -z "$RPMS_CONFLICTING" ]] || \ - $SHORT_DNFY remove $RPMS_CONFLICTING - ;; - debian) - [[ -z "$DEBS_CONFLICTING" ]] || \ - $SHORT_APTGET -q remove $DEBS_CONFLICTING - ;; - *) - bad_os_id_ver - ;; -esac - -install_fuse_overlayfs_from_git -install_bats_from_git diff --git a/storage/contrib/cirrus/timestamp.awk b/storage/contrib/cirrus/timestamp.awk deleted file mode 100644 index 95b312e515..0000000000 --- a/storage/contrib/cirrus/timestamp.awk +++ /dev/null @@ -1,20 +0,0 @@ - - -# This script is intended to be piped into by automation, in order to -# mark output lines with timing information. For example: -# /path/to/command |& awk --file timestamp.awk - -BEGIN { - STARTTIME=systime() - printf "[%s] START", strftime("%T") - printf " - All [+xxxx] lines that follow are relative to right now.\n" -} - -{ - printf "[%+05ds] %s\n", systime()-STARTTIME, $0 -} - -END { - printf "[%s] END", strftime("%T") - printf " - [%+05ds] total duration since START\n", systime()-STARTTIME -} diff --git a/storage/hack/tree_status.sh b/storage/hack/tree_status.sh deleted file mode 100755 index 2889a7fc8e..0000000000 --- a/storage/hack/tree_status.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -set -e - -STATUS=$(git status --porcelain) -if [[ -z $STATUS ]] -then - echo "tree is clean" -else - echo "tree is dirty, please commit all changes" - echo "" - echo "$STATUS" - exit 1 -fi