Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 31 additions & 3 deletions .github/workflows/pull-request-main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@ jobs:
# disable the checkptr runtime check due a false positive in github.com/xssnick/tonutils-go
# causing tests in ci to fail "fatal error: checkptr: pointer arithmetic result points to invalid allocation"
# https://github.com/xssnick/tonutils-go/issues/310
# Exclude provider packages which use Docker containers
go-test-cmd: go test -race -gcflags=all=-d=checkptr=0 -coverprofile=coverage.txt $(go list ./... | grep -v '/provider')
# Exclude provider packages which use Docker containers and remote catalog tests
go-test-cmd: go test -race -gcflags=all=-d=checkptr=0 -coverprofile=coverage.txt $(go list ./... | grep -v '/provider' | grep -v '/catalog/remote')
use-go-cache: true
artifact-name: unit-tests

Expand All @@ -71,11 +71,39 @@ jobs:
use-go-cache: true
artifact-name: provider-tests

ci-test-catalog-remote:
name: Catalog Remote Tests
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
id-token: write
contents: read
actions: read
env:
CATALOG_SERVICE_IMAGE: ${{ secrets.AWS_ACCOUNT_NUMBER_PROD }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/op-catalog-service:latest
steps:
- name: Pull Catalog Service ECR Image
uses: smartcontractkit/.github/actions/pull-private-ecr-image@2f8f0baf38e46140c6a119eb551a56eaaabcc09e # pull-private-ecr-image@1.0.0
with:
aws-account-number: ${{ secrets.AWS_ACCOUNT_NUMBER_PROD }}
aws-region: ${{ secrets.AWS_REGION }}
aws-role-arn: ${{ secrets.ECR_READ_ROLE_ARN }}
ecr-repository: "op-catalog-service"
image-tag: "latest"

- name: Run Catalog Remote Integration Tests
uses: smartcontractkit/.github/actions/ci-test-go@dfcba48f05933158428bce867d790e3d5a9baa6b # ci-test-go@1.1.0
with:
# Must cd into datastore/catalog/remote because TestMain only runs from package directory
go-test-cmd: cd datastore/catalog/remote && go test -v -race -timeout 10m -gcflags=all=-d=checkptr=0 -coverprofile=../../../coverage.txt
use-go-cache: true
artifact-name: catalog-remote-tests

sonarqube:
name: Sonar Scan
if: github.event_name == 'pull_request'
runs-on: ubuntu-24.04
needs: [ci-test, ci-test-provider, ci-lint-misc, ci-lint]
needs: [ci-test, ci-test-provider, ci-test-catalog-remote, ci-lint-misc, ci-lint]
permissions:
contents: read
actions: read
Expand Down
56 changes: 51 additions & 5 deletions .github/workflows/schedule-main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,19 +40,65 @@ jobs:
actions: read
steps:
- name: Build and test
uses: smartcontractkit/.github/actions/ci-test-go@eeb76b5870e3c17856d5a60fd064a053c023b5f5 # ci-test-go@1.0.0
uses: smartcontractkit/.github/actions/ci-test-go@dfcba48f05933158428bce867d790e3d5a9baa6b # ci-test-go@1.1.0
with:
# disable the checkptr runtime check due a false positive in github.com/xssnick/tonutils-go
# Exclude provider packages which use Docker containers and remote catalog tests
go-test-cmd: go test -race -gcflags=all=-d=checkptr=0 -coverprofile=coverage.txt $(go list ./... | grep -v '/provider' | grep -v '/catalog/remote')
use-go-cache: true
artifact-name: unit-tests

ci-test-provider:
name: Provider Tests
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
id-token: write
contents: read
actions: read
steps:
- name: Build and test provider packages
uses: smartcontractkit/.github/actions/ci-test-go@dfcba48f05933158428bce867d790e3d5a9baa6b # ci-test-go@1.1.0
with:
# disable the checkptr runtime check due a false positive in github.com/xssnick/tonutils-go
# causing tests in ci to fail "fatal error: checkptr: pointer arithmetic result points to invalid allocation"
# https://github.com/xssnick/tonutils-go/issues/310
# -p 2 -parallel 3 = 2 packages, 3 tests max = 6 containers max
go-test-cmd: go test -race -gcflags=all=-d=checkptr=0 -p 2 -parallel 3 -coverprofile=coverage.txt $(go list ./...)
# Only run provider packages which use Docker containers
go-test-cmd: go test -race -gcflags=all=-d=checkptr=0 -p 2 -parallel 3 -coverprofile=coverage.txt $(go list ./... | grep '/provider')
use-go-cache: true
artifact-name: provider-tests

ci-test-catalog-remote:
name: Catalog Remote Tests
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
id-token: write
contents: read
actions: read
env:
CATALOG_SERVICE_IMAGE: ${{ secrets.AWS_ACCOUNT_NUMBER_PROD }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/op-catalog-service:latest
steps:
- name: Pull Catalog Service ECR Image
uses: smartcontractkit/.github/actions/pull-private-ecr-image@2f8f0baf38e46140c6a119eb551a56eaaabcc09e # pull-private-ecr-image@1.0.0
with:
aws-account-number: ${{ secrets.AWS_ACCOUNT_NUMBER_PROD }}
aws-region: ${{ secrets.AWS_REGION }}
aws-role-arn: ${{ secrets.ECR_READ_ROLE_ARN }}
ecr-repository: "op-catalog-service"
image-tag: "latest"

- name: Run Catalog Remote Integration Tests
uses: smartcontractkit/.github/actions/ci-test-go@dfcba48f05933158428bce867d790e3d5a9baa6b # ci-test-go@1.1.0
with:
# Must cd into datastore/catalog/remote because TestMain only runs from package directory
go-test-cmd: cd datastore/catalog/remote && go test -v -race -timeout 10m -gcflags=all=-d=checkptr=0 -coverprofile=../../../coverage.txt
Comment thread
DimitriosNaikopoulos marked this conversation as resolved.
use-go-cache: true
artifact-name: catalog-remote-tests

sonarqube:
name: Sonar Scan
runs-on: ubuntu-24.04
needs: [ci-test, ci-lint-misc, ci-lint]
needs: [ci-test, ci-test-provider, ci-test-catalog-remote, ci-lint-misc, ci-lint]
permissions:
contents: read
actions: read
Expand Down
89 changes: 88 additions & 1 deletion datastore/catalog/remote/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,91 @@ state to be maintained through that stream connection. If the stream closes,
normal cleanup will rollback the transaction. The [Catalog service APIs] include
message to begin, commit, and roll-back a transaction.

[Catalog service APIs]: http://github.com/smartcontractkit/chainlink-catalog
[Catalog service APIs]: http://github.com/smartcontractkit/op-catalog

## Running Tests

The tests in this package use **testcontainers** to automatically start the required services:
- PostgreSQL database
- Catalog service (from ECR or local image)
- Database migration

**Important:** Testcontainers uses images from your local Docker daemon. You must either:
1. Build the image locally from the `op-catalog` directory:
```bash
cd op-catalog
docker build -t op-catalog-service:latest .
```
2. Pull from ECR with a specific tag: `docker pull 123123123123.dkr.ecr.us-east-1.amazonaws.com/op-catalog-service:TAG`
- Use `aws ecr describe-images` to find available tags
- Common patterns: version tags (v0.0.1)

### Quick Start (Local Build)

```bash
# Build the catalog service image first
cd op-catalog
docker build -t op-catalog-service:latest .

# Run the tests (must be in the remote directory)
cd ../chainlink-deployments-framework/datastore/catalog/remote
go test -v
```

## Configuration Options

### Environment Variables

- `CATALOG_SERVICE_IMAGE`: The Docker image to use for the catalog service
- Default: `op-catalog-service:latest`
- CI: Set to the ECR image URL
- Example: `export CATALOG_SERVICE_IMAGE="123456789.dkr.ecr.us-east-1.amazonaws.com/op-catalog-service:latest"`

- `CATALOG_GRPC_ADDRESS`: Connect to an existing catalog service instead of starting containers
- Example: `export CATALOG_GRPC_ADDRESS="localhost:8080"`

## Running Without Testcontainers

If you prefer to manage the services manually:

### Option 1: Docker Compose

```bash
cd op-catalog
docker-compose up -d

# Wait for the service to be ready
sleep 5

# Run tests pointing to the local service
cd ../chainlink-deployments-framework/datastore/catalog/remote
export CATALOG_GRPC_ADDRESS="localhost:8080"
go test -v
```

## CI/CD Integration

The CI workflow automatically:
1. Downloads the latest Catalog service image from ECR
2. Runs the tests with testcontainers using the downloaded image
3. Generates coverage reports

See `.github/workflows/pull-request-main.yml` for the full configuration.

## Test Workflow

The test setup follows this flow:

1. **TestMain** (`main_test.go`) - Entry point that:
- Checks for `CATALOG_GRPC_ADDRESS` environment variable
- If not set, starts testcontainers automatically
- Sets up PostgreSQL and Catalog service
- Runs all tests
- Cleans up containers

2. **TestContainerSetup** (`testcontainer_setup.go`) - Manages:
- Docker network creation
- PostgreSQL container startup
- Database migration via catalog service
- Catalog service container startup
- Port mapping and connection details
73 changes: 73 additions & 0 deletions datastore/catalog/remote/main_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package remote

import (
"context"
"log"
"os"
"testing"
"time"
)

var (
// globalTestSetup is the shared testcontainer setup for all remote tests
globalTestSetup *TestContainerSetup
// catalogGRPCAddress is the address of the catalog service for tests
catalogGRPCAddress string
)

// TestMain is the entry point for all tests in this package
// It sets up testcontainers once for all tests and tears them down at the end
func TestMain(m *testing.M) {
// Check if we should use an existing catalog service instead of testcontainers
existingAddr := os.Getenv("CATALOG_GRPC_ADDRESS")
if existingAddr != "" {
log.Printf("Using existing catalog service at: %s", existingAddr)
catalogGRPCAddress = existingAddr
// Run tests and exit
os.Exit(m.Run())
}

// Setup context with timeout for initialization
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()

log.Println("========================================")
log.Println("Setting up testcontainers for catalog remote tests...")
log.Println("========================================")

// Setup testcontainers
setup, err := SetupCatalogTestContainers(ctx)
if err != nil {
log.Fatalf("Failed to setup testcontainers: %v", err)
}
globalTestSetup = setup
catalogGRPCAddress = setup.GetCatalogGRPCAddress()

log.Println("========================================")
log.Println("Testcontainers setup complete!")
log.Printf("PostgreSQL DSN: %s", setup.PostgresDSN)
log.Printf("Catalog gRPC Address: %s", catalogGRPCAddress)
log.Println("========================================")

// Set the environment variable so tests can find the service
os.Setenv("CATALOG_GRPC_ADDRESS", catalogGRPCAddress)

// Run all tests
exitCode := m.Run()

// Cleanup
log.Println("========================================")
log.Println("Cleaning up testcontainers...")
log.Println("========================================")

// Use a fresh context for cleanup to avoid timeout issues
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cleanupCancel()

if err := globalTestSetup.Teardown(cleanupCtx); err != nil {
log.Printf("Warning: Failed to teardown testcontainers: %v", err)
}

log.Println("Cleanup complete!")
os.Exit(exitCode)
}
Loading
Loading