ci.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

---
name: CI

concurrency:
  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

# trigger for all PRs that touch certain files and changes to main
on:
  push:
    branches:
      - main
  pull_request:

jobs:
  clippy:
    name: Clippy
    runs-on: ubuntu-latest
    container:
      image: amd64/rust
    steps:
      - uses: actions/checkout@v7
      - name: Setup Clippy
        run: rustup component add clippy
      - name: Verify base features do not enable TLS/signing crypto crates
        # `--prefix none -f '{p}'` prints one `name vX.Y.Z` per line, so the
        # crate name can be anchored with `^`. Avoid `\b`, which is not
        # portable (e.g. BSD grep) and would also substring-match crates like
        # `iri-string`.
        run: |
          found=$(cargo tree --no-default-features --features aws-base,azure-base,gcp-base,http-base --edges normal --prefix none -f '{p}' \
            | grep -E '^(aws-lc-rs|aws-lc-sys|native-tls|openssl|ring|rustls) v' | sort -u || true)
          if [ -n "$found" ]; then
            echo "❌ disallowed crate(s) found:"
            echo "$found"
            exit 1
          fi
          echo "✅ no TLS/signing crypto crates"
      - name: Verify base features do not enable reqwest
        run: |
          found=$(cargo tree --no-default-features --features aws-base,azure-base,gcp-base,http-base --edges normal --prefix none -f '{p}' \
            | grep -E '^reqwest v' | sort -u || true)
          if [ -n "$found" ]; then
            echo "❌ reqwest must not be enabled by *-base features:"
            echo "$found"
            exit 1
          fi
          echo "✅ *-base features do not enable reqwest"
      - name: Check ring build does not pull aws-lc-rs
        # A `ring` build (with a non-aws-lc-rs reqwest TLS backend) must be free
        # of aws-lc-rs entirely, so users can opt out of it on every target.
        run: |
          found=$(cargo tree --no-default-features --features aws-base,azure-base,gcp-base,http-base,reqwest,reqwest/native-tls,ring --edges normal --prefix none -f '{p}' \
            | grep -E '^(aws-lc-rs|aws-lc-sys) v' | sort -u || true)
          if [ -n "$found" ]; then
            echo "❌ aws-lc-rs pulled into a ring-only build:"
            echo "$found"
            exit 1
          fi
          echo "✅ ring build is free of aws-lc-rs"
      - name: Check batteries-included features do not pull ring
        # The batteries-included features use reqwest/rustls, which uses aws-lc-rs
        # for TLS by default. Keep object_store's default bundled crypto provider
        # aligned with that choice by ensuring these features don't pull ring.
        run: |
          found=$(cargo tree --no-default-features --features aws,azure,gcp,http --edges normal --prefix none -f '{p}' \
            | grep -E '^ring v' | sort -u || true)
          if [ -n "$found" ]; then
            echo "❌ batteries-included features pulled ring:"
            echo "$found"
            exit 1
          fi
          echo "✅ batteries-included features are free of ring"
      # Run different tests for the library on its own as well as
      # all targets to ensure that it still works in the absence of
      # features that might be enabled by dev-dependencies of other
      # targets.
      - name: Run clippy with default features
        run: cargo clippy -- -D warnings
      - name: Run clippy without default features
        run: cargo clippy --no-default-features -- -D warnings
      - name: Run clippy with fs features
        run: cargo clippy --no-default-features --features fs -- -D warnings
      - name: Run clippy with aws feature
        run: cargo clippy --features aws -- -D warnings
      - name: Run clippy with gcp feature
        run: cargo clippy --features gcp -- -D warnings
      - name: Run clippy with azure feature
        run: cargo clippy --features azure -- -D warnings
      - name: Run clippy with http feature
        run: cargo clippy --features http -- -D warnings
      - name: Run clippy with aws-base feature
        run: cargo clippy --no-default-features --features aws-base -- -D warnings
      - name: Run clippy with azure-base feature
        run: cargo clippy --no-default-features --features azure-base -- -D warnings
      - name: Run clippy with gcp-base feature
        run: cargo clippy --no-default-features --features gcp-base -- -D warnings
      - name: Run clippy with http-base feature
        run: cargo clippy --no-default-features --features http-base -- -D warnings
      # Testing matrix of HTTP and crypto providers
      #
      # Improvements tracked in  https://github.com/apache/arrow-rs-object-store/issues/759
      #
      # Notes:
      # Direct `reqwest` needs a TLS backend. Use `rustls-no-provider` so TLS
      # does not add AWS-LC to ring/custom CryptoProvider checks.
      - name: Run clippy with base features, reqwest and ring CryptoProvider
        run: cargo clippy --no-default-features --features aws-base,azure-base,gcp-base,http-base,reqwest,reqwest/rustls-no-provider,ring -- -D warnings
      - name: Run clippy with base features, reqwest and no bundled CryptoProvider
        run: cargo clippy --no-default-features --features aws-base,azure-base,gcp-base,http-base,reqwest,reqwest/rustls-no-provider -- -D warnings
      - name: Run clippy with base features, custom HTTP and aws-lc-rs CryptoProvider
        run: cargo clippy --no-default-features --features aws-base,azure-base,gcp-base,http-base,aws-lc-rs,aws-lc-rs/aws-lc-sys -- -D warnings
      - name: Run clippy with base features, custom HTTP and ring CryptoProvider
        run: cargo clippy --no-default-features --features aws-base,azure-base,gcp-base,http-base,ring -- -D warnings
      - name: Run clippy with base features, custom HTTP and no bundled CryptoProvider
        run: cargo clippy --no-default-features --features aws-base,azure-base,gcp-base,http-base -- -D warnings
      - name: Run clippy with integration feature
        run: cargo clippy --no-default-features --features integration -- -D warnings
      - name: Run clippy with all features
        run: cargo clippy --all-features -- -D warnings
      - name: Run clippy with all features and all targets
        run: cargo clippy --all-features --all-targets -- -D warnings

  # test doc links still work
  docs:
    name: Rustdocs
    runs-on: ubuntu-latest
    env:
      RUSTDOCFLAGS: "-Dwarnings"
    steps:
      - uses: actions/checkout@v7
      - name: Run cargo doc
        run: cargo doc --document-private-items --no-deps --all-features

  # test the crate
  # This runs outside a container to workaround lack of support for passing arguments
  # to service containers - https://github.com/orgs/community/discussions/26688
  linux-test:
    name: Emulator Tests
    runs-on: ubuntu-latest
    env:
      # Disable full debug symbol generation to speed up CI build and keep memory down
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      RUST_BACKTRACE: "1"
      # Run integration tests
      TEST_INTEGRATION: 1
      EC2_METADATA_ENDPOINT: http://localhost:1338
      AZURE_CONTAINER_NAME: test-bucket
      AZURE_STORAGE_USE_EMULATOR: "1"
      AZURITE_BLOB_STORAGE_URL: "http://localhost:10000"
      AZURITE_QUEUE_STORAGE_URL: "http://localhost:10001"
      AWS_BUCKET: test-bucket
      AWS_DEFAULT_REGION: "us-east-1"
      AWS_ACCESS_KEY_ID: test
      AWS_SECRET_ACCESS_KEY: test
      AWS_ENDPOINT: http://localhost:4566
      AWS_ALLOW_HTTP: true
      AWS_COPY_IF_NOT_EXISTS: multipart
      AWS_CONDITIONAL_PUT: etag
      AWS_SERVER_SIDE_ENCRYPTION: aws:kms
      HTTP_URL: "http://localhost:8080"
      GOOGLE_BUCKET: test-bucket
      GOOGLE_SERVICE_ACCOUNT: "/tmp/gcs.json"

    steps:
      - uses: actions/checkout@v7

      # We are forced to use docker commands instead of service containers as we need to override the entrypoints
      # which is currently not supported - https://github.com/actions/runner/discussions/1872
      - name: Configure Fake GCS Server (GCP emulation)
        # Custom image - see fsouza/fake-gcs-server#1164
        run: |
          GCS_CONTAINER=$(docker run -d -p 4443:4443 tustvold/fake-gcs-server -scheme http -backend memory -public-host localhost:4443)
          echo "GCS_CONTAINER=$GCS_CONTAINER" >> $GITHUB_ENV
          # Give the container a moment to start up prior to configuring it
          sleep 1
          curl -v -X POST --data-binary '{"name":"test-bucket"}' -H "Content-Type: application/json" "http://localhost:4443/storage/v1/b"
          echo '{"gcs_base_url": "http://localhost:4443", "disable_oauth": true, "client_email": "", "private_key": "", "private_key_id": ""}' > "$GOOGLE_SERVICE_ACCOUNT"

      - name: Setup WebDav
        run: docker run -d -p 8080:80 rclone/rclone serve webdav /data --addr :80

      - name: Setup LocalStack (AWS emulation)
        run: |
          LOCALSTACK_CONTAINER=$(docker run -d -p 4566:4566 localstack/localstack:4.11.1)
          echo "LOCALSTACK_CONTAINER=$LOCALSTACK_CONTAINER" >> $GITHUB_ENV
          EC2_METADATA_CONTAINER=$(docker run -d -p 1338:1338 amazon/amazon-ec2-metadata-mock:v1.9.2 --imdsv2)
          echo "EC2_METADATA_CONTAINER=$EC2_METADATA_CONTAINER" >> $GITHUB_ENV
          aws --endpoint-url=http://localhost:4566 s3 mb s3://test-bucket
          aws --endpoint-url=http://localhost:4566 s3 mb s3://test-bucket-for-spawn
          aws --endpoint-url=http://localhost:4566 s3 mb s3://test-bucket-for-checksum
          aws --endpoint-url=http://localhost:4566 s3 mb s3://test-bucket-for-copy-if-not-exists
          aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket test-object-lock --object-lock-enabled-for-bucket

          KMS_KEY=$(aws --endpoint-url=http://localhost:4566 kms create-key --description "test key")
          AWS_SSE_KMS_KEY_ID=$(jq -r .KeyMetadata.KeyId <<< "$KMS_KEY")
          echo "AWS_SSE_KMS_KEY_ID=$AWS_SSE_KMS_KEY_ID" >> $GITHUB_ENV

      - name: Configure Azurite (Azure emulation)
        # the magical connection string is from
        # https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings
        # We skip the API version check to prevent breaks related to differences between Azurite, Azure and the azure-cli,
        # see https://github.com/Azure/Azurite/issues/2623
        run: |
          AZURITE_CONTAINER=$(docker run -d -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite azurite -l /data --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 --skipApiVersionCheck)
          echo "AZURITE_CONTAINER=$AZURITE_CONTAINER" >> $GITHUB_ENV
          az storage container create -n test-bucket --connection-string 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;'

      - name: Setup Rust toolchain
        run: |
          rustup toolchain install stable
          rustup default stable

      - name: Run object_store tests
        run: cargo test --features=aws,azure,gcp,http

      # Don't rerun doc tests (some of them rely on features other than aws)
      - name: Run object_store tests (AWS native conditional put)
        run: cargo test --lib --tests --features=aws
        env:
          AWS_CONDITIONAL_PUT: etag
          AWS_COPY_IF_NOT_EXISTS: multipart

      # Exercise the `ring` crypto provider at runtime (the other jobs only
      # compile it)
      #
      # Notes:
      # * Use `reqwest/native-tls` to avoid aws-lc-rs anywhere in the tree
      #
      # * use `env -u TEST_INTEGRATION` to run only the (crypto) unit tests
      #   as the integration tests already ran above.
      - name: Run object_store tests with ring crypto provider
        run: env -u TEST_INTEGRATION cargo test --lib --tests --no-default-features --features fs,aws-base,azure-base,gcp-base,http-base,reqwest,reqwest/native-tls,ring

      - name: GCS Output
        if: ${{ !cancelled() }}
        run: docker logs $GCS_CONTAINER

      - name: LocalStack Output
        if: ${{ !cancelled() }}
        run: docker logs $LOCALSTACK_CONTAINER

      - name: EC2 Metadata Output
        if: ${{ !cancelled() }}
        run: docker logs $EC2_METADATA_CONTAINER

      - name: Azurite Output
        if: ${{ !cancelled() }}
        run: docker logs $AZURITE_CONTAINER

  # test the object_store crate builds against wasm32 in stable rust
  wasm32-build:
    name: Build wasm32
    runs-on: ubuntu-latest
    container:
      image: amd64/rust
    steps:
      - uses: actions/checkout@v7
        with:
          submodules: true
      - name: Install clang (needed for ring)
        run: apt-get update && apt-get install -y clang
      - name: Install wasm32-unknown-unknown
        run: rustup target add wasm32-unknown-unknown
      - name: Build wasm32-unknown-unknown
        run: cargo build --target wasm32-unknown-unknown
      # Note: we don't `cargo build` the cloud features for wasm32-unknown-unknown.
      # They pull in `rand` -> `getrandom`, which on this target only compiles with
      # its `wasm_js` feature enabled. We can't enable that via `--features` because
      # `getrandom` is a transitive dependency (pulled in by `rand`), not direct
      - name: Install wasm32-wasip1
        run: rustup target add wasm32-wasip1
      - name: Use reqwest 0.13.2 for wasm32-wasip1
        # TODO: reqwest 0.13.3+ doesn't compile against wasm32-wasip1
        run: cargo update -p reqwest --precise 0.13.2
      - name: Build wasm32-wasip1
        # aws-lc-rs does not build for wasm32-wasip1, so use the ring crypto provider here
        run: cargo build --no-default-features --features aws-base,gcp-base,azure-base,http-base,reqwest,ring --target wasm32-wasip1
      - name: Build wasm32-wasip1 without reqwest
        run: cargo build --no-default-features --features aws-base --target wasm32-wasip1
      - name: Install wasm-pack
        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
      - uses: actions/setup-node@v6
        with:
          node-version: 20
      - name: Run wasm32-unknown-unknown tests (via Node)
        run: wasm-pack test --node --features http-base,reqwest,ring --no-default-features

  windows:
    name: cargo test LocalFileSystem (win64)
    runs-on: windows-latest
    steps:
      - uses: actions/checkout@v7
        with:
          submodules: true
      - name: Run LocalFileSystem tests
        run: cargo test local::tests