Skip to content

Commit b632883

Browse files
sudo-bmitchsudo-bmitch
committed
Conformance redesign
This reworks the conformance tests to verify the various APIs with different sets of data. The configuration allows various APIs and types of data to be disabled. The configuration will default to the tests appropriate for a specific version of the spec. However, new types of data may still be pushed when testing older versions of the spec. The tests are also designed to be run with `go build` rather than `go test`. This will allow the conformance tests themselves to be tested in the future. Attempts to run the tests with `go test` and the previous configuration variables should still work, but support for this is deprecated and warning will be included in the output to update. The results now include a new results.yaml including the configuration and the result of each API and type of data tested. Processing this on the oci-conformance repo requires a separate PR there. This redesign was necessary to allow the conformance tests to scale with the different types of data we are seeing. This includes data with a subject, artifacts, and new digest algorithms. Signed-off-by: Brandon Mitchell <git@bmitch.net>
1 parent 5e57cc0 commit b632883

32 files changed

Lines changed: 6276 additions & 3698 deletions

‎.github/workflows/build-pr.yml‎

Lines changed: 0 additions & 51 deletions
This file was deleted.

‎.github/workflows/build.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,17 @@ on:
44
push:
55
branches:
66
- main
7+
pull_request:
8+
branches:
9+
- main
710

811
jobs:
912
run:
1013
runs-on: ubuntu-latest
1114

1215
strategy:
1316
matrix:
14-
go: ['1.21', '1.22']
17+
go: ['stable', 'oldstable']
1518

1619
steps:
1720
- name: checkout source code
@@ -47,5 +50,6 @@ jobs:
4750
with:
4851
name: oci-conformance-results-${{ matrix.go }}
4952
path: |
53+
./results.yaml
5054
./report.html
5155
./junit.xml

‎.github/workflows/conformance-action-pr.yml‎

Lines changed: 0 additions & 26 deletions
This file was deleted.

‎.github/workflows/conformance-action.yml‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,23 +4,27 @@ on:
44
push:
55
branches:
66
- main
7+
pull_request:
8+
branches:
9+
- main
710

811
jobs:
912
run:
1013
runs-on: ubuntu-latest
1114
steps:
1215
- name: checkout source code
1316
uses: actions/checkout@v4
14-
- name: Start a test registry (zot)
17+
- name: Start a test registry
18+
id: setup
1519
run: |
1620
set -x
1721
make registry-ci
22+
echo "port=$(docker port oci-conformance-olareg 5000| head -1 | cut -f2 -d:)" >>$GITHUB_OUTPUT
1823
- name: Run OCI distribution-spec conformance
1924
env:
20-
OCI_ROOT_URL: http://localhost:5000
21-
OCI_NAMESPACE: myorg/myrepo
22-
OCI_TEST_PULL: 1
23-
OCI_TEST_PUSH: 1
24-
OCI_TEST_CONTENT_DISCOVERY: 1
25-
OCI_TEST_CONTENT_MANAGEMENT: 1
25+
OCI_REGISTRY: "localhost:${{ steps.setup.outputs.port }}"
26+
OCI_TLS: "disabled"
27+
OCI_REPO1: "myorg/myrepo"
28+
OCI_REPO2: "myorg/myrepo2"
29+
OCI_RESULTS_DIR: "."
2630
uses: ./

‎.github/workflows/release.yml‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,6 @@ jobs:
2626
fi
2727
VERSION=$(echo "${VERSION}" | sed -r 's#/+#-#g')
2828
TAGS="${IMAGE}:${VERSION}"
29-
if [[ $VERSION == "${{ github.event.repository.default_branch }}" ]]; then
30-
GITSHA="$(git rev-parse --short HEAD)"
31-
TAGS="${TAGS},${IMAGE}:${GITSHA}"
32-
fi
3329
if [[ $VERSION =~ ^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
3430
MINOR=${VERSION%.*}
3531
MAJOR=${MINOR%.*}

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,4 @@ go.mod
77
go.sum
88
junit.xml
99
report.html
10+
results.yaml

‎Makefile‎

Lines changed: 53 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,13 @@ ifeq "$(strip $(PANDOC))" ''
2424
endif
2525
endif
2626

27-
GOLANGCILINT_CONTAINER ?= ghcr.io/opencontainers/golangci-lint:v1.52.1@sha256:d3d3d56f9706ebe843c1b06686c385877ba65b33f39507cdbeb22f482adce65a
27+
GOLANGCILINT_CONTAINER ?= docker.io/golangci/golangci-lint:v2.11.3@sha256:e838e8ab68aaefe83e2408691510867ade9329c0e0b895a3fb35eb93d1c2a4ba
2828
ifeq "$(strip $(GOLANGCILINT))" ''
2929
ifneq "$(strip $(DOCKER))" ''
3030
GOLANGCILINT = $(DOCKER) run \
3131
--rm \
3232
-v $(shell pwd)/:/input:ro \
3333
-e GOCACHE=/tmp/.cache \
34-
-e GO111MODULE=on \
3534
-e GOLANGCI_LINT_CACHE=/tmp/.cache \
3635
--entrypoint /bin/bash \
3736
-u $(shell id -u) \
@@ -85,35 +84,70 @@ install.tools: .install.gitvalidation
8584
.install.gitvalidation:
8685
go install github.com/vbatts/git-validation@latest
8786

88-
conformance: conformance-test conformance-binary
87+
conformance: conformance-test conformance-cmd
8988

9089
conformance-test:
9190
$(GOLANGCILINT) -c 'cd conformance && golangci-lint run -v'
9291

9392
conformance-binary: $(OUTPUT_DIRNAME)/conformance.test
9493

95-
TEST_REGISTRY_CONTAINER ?= ghcr.io/project-zot/zot-minimal-linux-amd64:v2.1.7@sha256:2114797f00696011f38cc94c72f5773c84b1036562df5034d05ea19075179ad1
96-
registry-ci:
97-
docker rm -f oci-conformance && \
94+
conformance-cmd: $(OUTPUT_DIRNAME)/conformance
95+
96+
registry-ci: registry-ci-olareg
97+
98+
TEST_REGISTRY_IMAGE_OLAREG ?= ghcr.io/olareg/olareg:edge
99+
registry-ci-olareg:
100+
docker rm -f oci-conformance-olareg && \
101+
docker run --rm -d \
102+
--name=oci-conformance-olareg \
103+
-p 5000 \
104+
$(TEST_REGISTRY_IMAGE_OLAREG) serve --store-type mem --api-delete --api-blob-delete --api-sparse-image --api-sparse-index && \
105+
sleep 2
106+
107+
TEST_REGISTRY_IMAGE_ZOT ?= ghcr.io/project-zot/zot-minimal-linux-amd64:v2.1.7@sha256:2114797f00696011f38cc94c72f5773c84b1036562df5034d05ea19075179ad1
108+
registry-ci-zot:
109+
docker rm -f oci-conformance-zot && \
98110
mkdir -p $(OUTPUT_DIRNAME) && \
99111
echo '{"distSpecVersion":"1.1.0-dev","storage":{"rootDirectory":"/tmp/zot","gc":false,"dedupe":false},"http":{"address":"0.0.0.0","port":"5000"},"log":{"level":"debug"}}' > $(shell pwd)/$(OUTPUT_DIRNAME)/zot-config.json
100-
docker run -d \
112+
docker run --rm -d \
101113
-v $(shell pwd)/$(OUTPUT_DIRNAME)/zot-config.json:/etc/zot/config.json \
102-
--name=oci-conformance \
103-
-p 5000:5000 \
104-
$(TEST_REGISTRY_CONTAINER) && \
114+
--name=oci-conformance-zot \
115+
-p 5000 \
116+
$(TEST_REGISTRY_IMAGE_ZOT) && \
105117
sleep 5
106118

107-
conformance-ci:
108-
export OCI_ROOT_URL="http://localhost:5000" && \
109-
export OCI_NAMESPACE="myorg/myrepo" && \
110-
export OCI_TEST_PULL=1 && \
111-
export OCI_TEST_PUSH=1 && \
112-
export OCI_TEST_CONTENT_DISCOVERY=1 && \
113-
export OCI_TEST_CONTENT_MANAGEMENT=1 && \
114-
$(shell pwd)/$(OUTPUT_DIRNAME)/conformance.test
119+
conformance-ci: conformance-ci-olareg
120+
121+
conformance-ci-olareg: $(OUTPUT_DIRNAME)/conformance
122+
export OCI_VERSION="dev" && \
123+
export OCI_REGISTRY="localhost:$$(docker port oci-conformance-olareg 5000| head -1 | cut -f2 -d:)" && \
124+
export OCI_TLS="disabled" && \
125+
export OCI_REPO1="myorg/myrepo" && \
126+
export OCI_REPO2="myorg/myrepo2" && \
127+
export OCI_RESULTS_DIR="." && \
128+
export OCI_DATA_SPARSE=true && \
129+
$(shell pwd)/$(OUTPUT_DIRNAME)/conformance
130+
131+
conformance-ci-zot: $(OUTPUT_DIRNAME)/conformance
132+
export OCI_REGISTRY="localhost:$$(docker port oci-conformance-zot 5000| head -1 | cut -f2 -d:)" && \
133+
export OCI_TLS="disabled" && \
134+
export OCI_REPO1="myorg/myrepo" && \
135+
export OCI_REPO2="myorg/myrepo2" && \
136+
export OCI_RESULTS_DIR="." && \
137+
$(shell pwd)/$(OUTPUT_DIRNAME)/conformance
138+
139+
clean-ci:
140+
docker rm -f oci-conformance-olareg oci-conformance-zot
141+
142+
$(OUTPUT_DIRNAME)/conformance: conformance/*.go conformance/go.mod
143+
cd conformance && \
144+
CGO_ENABLED=0 go build -o $(shell pwd)/$(OUTPUT_DIRNAME)/conformance \
145+
--ldflags="-X github.com/opencontainers/distribution-spec/conformance.Version=$(CONFORMANCE_VERSION)"
115146

116-
$(OUTPUT_DIRNAME)/conformance.test:
147+
$(OUTPUT_DIRNAME)/conformance.test: conformance/*.go conformance/go.mod
117148
cd conformance && \
118149
CGO_ENABLED=0 go test -c -o $(shell pwd)/$(OUTPUT_DIRNAME)/conformance.test \
119150
--ldflags="-X github.com/opencontainers/distribution-spec/conformance.Version=$(CONFORMANCE_VERSION)"
151+
152+
clean: clean-ci
153+
rm -rf header.html junit.xml report.html results.yaml output conformance/results

‎action.yml‎

Lines changed: 37 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,16 +14,39 @@ runs:
1414
id: check-conformance
1515
run: |
1616
set -x
17+
required_env_vars=(
18+
"OCI_REGISTRY"
19+
"OCI_REPO1"
20+
)
21+
missing=false
22+
for v in ${required_env_vars[@]}; do
23+
if [[ "${!v}" == "" ]]; then
24+
echo "::error title=Missing variable::Missing required variable ${v}"
25+
missing=true
26+
fi
27+
done
28+
if [[ "${missing}" == "false" ]]; then
29+
exit 0
30+
fi
31+
# fallback to previous variables
1732
required_env_vars=(
1833
"OCI_ROOT_URL"
1934
"OCI_NAMESPACE"
2035
)
36+
missing=false
2137
for v in ${required_env_vars[@]}; do
2238
if [[ "${!v}" == "" ]]; then
23-
echo "Error: the following environment variable is required: ${v}"
24-
exit 1
39+
echo "::error title=Missing fallback variable::Missing fallback variable ${v}"
40+
missing=true
41+
else
42+
echo "::warning title=Deprecated fallback variable::Variable ${v} has been deprecated"
2543
fi
2644
done
45+
if [[ "${missing}" == "false" ]]; then
46+
exit 0
47+
else
48+
exit 1
49+
fi
2750
2851
- name: Build OCI distribution-spec conformance binary
2952
shell: bash
@@ -34,14 +57,16 @@ runs:
3457
# Enter the directory containing the checkout of this action which is surpisingly hard to do (but we did it... #OCI)
3558
cd "$(dirname $(find $(find ~/work/_actions -name distribution-spec -print -quit) -name Makefile -print -quit))"
3659
37-
# The .git folder is not present, but the dirname is the requested action ref, so use this as the conformance version
38-
conformance_version="$(basename "${PWD}")"
39-
echo "conformance-version=${conformance_version}" >> $GITHUB_OUTPUT
60+
# The .git folder is not present, but the dirname is the requested action ref, so use this as the commit version
61+
commit_version="$(basename "${PWD}")"
4062
4163
# Build the conformance binary
42-
CONFORMANCE_VERSION="${conformance_version}" OUTPUT_DIRNAME=bin make conformance-binary
64+
CONFORMANCE_VERSION="${commit_version}" OUTPUT_DIRNAME=bin make conformance-cmd
65+
66+
# The spec version is independent of the conformance git commit
67+
echo "oci-spec-version=${OCI_VERSION:-stable}" >> $GITHUB_OUTPUT
4368
44-
# Add bin to the PATH so we can just run "conformance.test"
69+
# Add bin to the PATH so we can just run "conformance"
4570
echo "${PWD}/bin" >> $GITHUB_PATH
4671
4772
- name: Run OCI distribution-spec conformance binary
@@ -50,7 +75,8 @@ runs:
5075
run: |
5176
set -x
5277
set +e
53-
conformance.test
78+
OCI_RESULTS_DIR=${OCI_RESULTS_DIR:-.}
79+
conformance
5480
conformance_rc="$?"
5581
set -e
5682
if [[ -f report.html ]]; then
@@ -62,9 +88,10 @@ runs:
6288
6389
- name: Upload OCI distribution-spec conformance results as build artifact
6490
if: always() && steps.run-conformance.outputs.has-report == 'true'
65-
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
91+
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
6692
with:
67-
name: oci-distribution-spec-conformance-results-${{ steps.build-conformance.outputs.conformance-version }}
93+
name: oci-distribution-spec-conformance-results-${{ steps.build-conformance.outputs.oci-spec-version }}
6894
path: |
95+
results.yaml
6996
report.html
7097
junit.xml

‎conformance/.gitignore‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1-
vendor/
2-
junit.xml
3-
report.html
1+
!go.mod
2+
!go.sum
3+
conformance
44
conformance.test
5-
tags
6-
env.sh
5+
oci-conformance.yaml
6+
results/**
7+
vendor/

0 commit comments

Comments
 (0)