Skip to content

Commit 787664f

Browse files
sudo-bmitchsudo-bmitch
committed
Conformance redesign
This reworks the conformance tests to verify the various APIs with different sets of data. The configuration allows various APIs and types of data to be disabled. The configuration will default to the tests appropriate for a specific version of the spec. However, new types of data may still be pushed when testing older versions of the spec. The tests are also designed to be run with `go build` rather than `go test`. This will allow the conformance tests themselves to be tested in the future. Attempts to run the tests with `go test` and the previous configuration variables should still work, but support for this is deprecated and warning will be included in the output to update. The results now include a new results.yaml including the configuration and the result of each API and type of data tested. Processing this on the oci-conformance repo requires a separate PR there. This redesign was necessary to allow the conformance tests to scale with the different types of data we are seeing. This includes data with a subject, artifacts, and new digest algorithms. Signed-off-by: Brandon Mitchell <git@bmitch.net>
1 parent d59c940 commit 787664f

32 files changed

Lines changed: 6290 additions & 3697 deletions

‎.github/workflows/build-pr.yml‎

Lines changed: 0 additions & 51 deletions
This file was deleted.

‎.github/workflows/build.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,17 @@ on:
44
push:
55
branches:
66
- main
7+
pull_request:
8+
branches:
9+
- main
710

811
jobs:
912
run:
1013
runs-on: ubuntu-latest
1114

1215
strategy:
1316
matrix:
14-
go: ['1.21', '1.22']
17+
go: ['stable', 'oldstable']
1518

1619
steps:
1720
- name: checkout source code
@@ -47,5 +50,6 @@ jobs:
4750
with:
4851
name: oci-conformance-results-${{ matrix.go }}
4952
path: |
53+
./results.yaml
5054
./report.html
5155
./junit.xml

‎.github/workflows/conformance-action-pr.yml‎

Lines changed: 0 additions & 26 deletions
This file was deleted.

‎.github/workflows/conformance-action.yml‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,23 +4,27 @@ on:
44
push:
55
branches:
66
- main
7+
pull_request:
8+
branches:
9+
- main
710

811
jobs:
912
run:
1013
runs-on: ubuntu-latest
1114
steps:
1215
- name: checkout source code
1316
uses: actions/checkout@v4
14-
- name: Start a test registry (zot)
17+
- name: Start a test registry
18+
id: setup
1519
run: |
1620
set -x
1721
make registry-ci
22+
echo "port=$(docker port oci-conformance-olareg 5000| head -1 | cut -f2 -d:)" >>$GITHUB_OUTPUT
1823
- name: Run OCI distribution-spec conformance
1924
env:
20-
OCI_ROOT_URL: http://localhost:5000
21-
OCI_NAMESPACE: myorg/myrepo
22-
OCI_TEST_PULL: 1
23-
OCI_TEST_PUSH: 1
24-
OCI_TEST_CONTENT_DISCOVERY: 1
25-
OCI_TEST_CONTENT_MANAGEMENT: 1
25+
OCI_REGISTRY: "localhost:${{ steps.setup.outputs.port }}"
26+
OCI_TLS: "disabled"
27+
OCI_REPO1: "myorg/myrepo"
28+
OCI_REPO2: "myorg/myrepo2"
29+
OCI_RESULTS_DIR: "."
2630
uses: ./

‎.github/workflows/release.yml‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,6 @@ jobs:
2626
fi
2727
VERSION=$(echo "${VERSION}" | sed -r 's#/+#-#g')
2828
TAGS="${IMAGE}:${VERSION}"
29-
if [[ $VERSION == "${{ github.event.repository.default_branch }}" ]]; then
30-
GITSHA="$(git rev-parse --short HEAD)"
31-
TAGS="${TAGS},${IMAGE}:${GITSHA}"
32-
fi
3329
if [[ $VERSION =~ ^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
3430
MINOR=${VERSION%.*}
3531
MAJOR=${MINOR%.*}

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ tags
66
go.work
77
junit.xml
88
report.html
9+
results.yaml

‎Makefile‎

Lines changed: 53 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,13 @@ ifeq "$(strip $(PANDOC))" ''
2323
endif
2424
endif
2525

26-
GOLANGCILINT_CONTAINER ?= ghcr.io/opencontainers/golangci-lint:v1.52.1@sha256:d3d3d56f9706ebe843c1b06686c385877ba65b33f39507cdbeb22f482adce65a
26+
GOLANGCILINT_CONTAINER ?= docker.io/golangci/golangci-lint:v2.11.3@sha256:e838e8ab68aaefe83e2408691510867ade9329c0e0b895a3fb35eb93d1c2a4ba
2727
ifeq "$(strip $(GOLANGCILINT))" ''
2828
ifneq "$(strip $(DOCKER))" ''
2929
GOLANGCILINT = $(DOCKER) run \
3030
--rm \
3131
-v $(shell pwd)/:/input:ro \
3232
-e GOCACHE=/tmp/.cache \
33-
-e GO111MODULE=on \
3433
-e GOLANGCI_LINT_CACHE=/tmp/.cache \
3534
--entrypoint /bin/bash \
3635
-u $(shell id -u) \
@@ -83,35 +82,70 @@ install.tools: .install.gitvalidation
8382
.install.gitvalidation:
8483
go install github.com/vbatts/git-validation@latest
8584

86-
conformance: conformance-test conformance-binary
85+
conformance: conformance-test conformance-cmd
8786

8887
conformance-test:
8988
$(GOLANGCILINT) -c 'cd conformance && golangci-lint run -v'
9089

9190
conformance-binary: $(OUTPUT_DIRNAME)/conformance.test
9291

93-
TEST_REGISTRY_CONTAINER ?= ghcr.io/project-zot/zot-minimal-linux-amd64:v2.1.7@sha256:2114797f00696011f38cc94c72f5773c84b1036562df5034d05ea19075179ad1
94-
registry-ci:
95-
docker rm -f oci-conformance && \
92+
conformance-cmd: $(OUTPUT_DIRNAME)/conformance
93+
94+
registry-ci: registry-ci-olareg
95+
96+
TEST_REGISTRY_IMAGE_OLAREG ?= ghcr.io/olareg/olareg:edge
97+
registry-ci-olareg:
98+
docker rm -f oci-conformance-olareg && \
99+
docker run --rm -d \
100+
--name=oci-conformance-olareg \
101+
-p 5000 \
102+
$(TEST_REGISTRY_IMAGE_OLAREG) serve --store-type mem --api-delete --api-blob-delete --api-sparse-image --api-sparse-index && \
103+
sleep 2
104+
105+
TEST_REGISTRY_IMAGE_ZOT ?= ghcr.io/project-zot/zot-minimal-linux-amd64:v2.1.7@sha256:2114797f00696011f38cc94c72f5773c84b1036562df5034d05ea19075179ad1
106+
registry-ci-zot:
107+
docker rm -f oci-conformance-zot && \
96108
mkdir -p $(OUTPUT_DIRNAME) && \
97109
echo '{"distSpecVersion":"1.1.0-dev","storage":{"rootDirectory":"/tmp/zot","gc":false,"dedupe":false},"http":{"address":"0.0.0.0","port":"5000"},"log":{"level":"debug"}}' > $(shell pwd)/$(OUTPUT_DIRNAME)/zot-config.json
98-
docker run -d \
110+
docker run --rm -d \
99111
-v $(shell pwd)/$(OUTPUT_DIRNAME)/zot-config.json:/etc/zot/config.json \
100-
--name=oci-conformance \
101-
-p 5000:5000 \
102-
$(TEST_REGISTRY_CONTAINER) && \
112+
--name=oci-conformance-zot \
113+
-p 5000 \
114+
$(TEST_REGISTRY_IMAGE_ZOT) && \
103115
sleep 5
104116

105-
conformance-ci:
106-
export OCI_ROOT_URL="http://localhost:5000" && \
107-
export OCI_NAMESPACE="myorg/myrepo" && \
108-
export OCI_TEST_PULL=1 && \
109-
export OCI_TEST_PUSH=1 && \
110-
export OCI_TEST_CONTENT_DISCOVERY=1 && \
111-
export OCI_TEST_CONTENT_MANAGEMENT=1 && \
112-
$(shell pwd)/$(OUTPUT_DIRNAME)/conformance.test
117+
conformance-ci: conformance-ci-olareg
118+
119+
conformance-ci-olareg: $(OUTPUT_DIRNAME)/conformance
120+
export OCI_VERSION="dev" && \
121+
export OCI_REGISTRY="localhost:$$(docker port oci-conformance-olareg 5000| head -1 | cut -f2 -d:)" && \
122+
export OCI_TLS="disabled" && \
123+
export OCI_REPO1="myorg/myrepo" && \
124+
export OCI_REPO2="myorg/myrepo2" && \
125+
export OCI_RESULTS_DIR="." && \
126+
export OCI_DATA_SPARSE=true && \
127+
$(shell pwd)/$(OUTPUT_DIRNAME)/conformance
128+
129+
conformance-ci-zot: $(OUTPUT_DIRNAME)/conformance
130+
export OCI_REGISTRY="localhost:$$(docker port oci-conformance-zot 5000| head -1 | cut -f2 -d:)" && \
131+
export OCI_TLS="disabled" && \
132+
export OCI_REPO1="myorg/myrepo" && \
133+
export OCI_REPO2="myorg/myrepo2" && \
134+
export OCI_RESULTS_DIR="." && \
135+
$(shell pwd)/$(OUTPUT_DIRNAME)/conformance
136+
137+
clean-ci:
138+
docker rm -f oci-conformance-olareg oci-conformance-zot
139+
140+
$(OUTPUT_DIRNAME)/conformance: conformance/*.go conformance/go.mod
141+
cd conformance && \
142+
CGO_ENABLED=0 go build -o $(shell pwd)/$(OUTPUT_DIRNAME)/conformance \
143+
--ldflags="-X github.com/opencontainers/distribution-spec/conformance.Version=$(CONFORMANCE_VERSION)"
113144

114-
$(OUTPUT_DIRNAME)/conformance.test:
145+
$(OUTPUT_DIRNAME)/conformance.test: conformance/*.go conformance/go.mod
115146
cd conformance && \
116147
CGO_ENABLED=0 go test -c -o $(shell pwd)/$(OUTPUT_DIRNAME)/conformance.test \
117148
--ldflags="-X github.com/opencontainers/distribution-spec/conformance.Version=$(CONFORMANCE_VERSION)"
149+
150+
clean: clean-ci
151+
rm -rf header.html junit.xml report.html results.yaml output conformance/results

‎action.yml‎

Lines changed: 37 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,16 +14,39 @@ runs:
1414
id: check-conformance
1515
run: |
1616
set -x
17+
required_env_vars=(
18+
"OCI_REGISTRY"
19+
"OCI_REPO1"
20+
)
21+
missing=false
22+
for v in ${required_env_vars[@]}; do
23+
if [[ "${!v}" == "" ]]; then
24+
echo "::error title=Missing variable::Missing required variable ${v}"
25+
missing=true
26+
fi
27+
done
28+
if [[ "${missing}" == "false" ]]; then
29+
exit 0
30+
fi
31+
# fallback to previous variables
1732
required_env_vars=(
1833
"OCI_ROOT_URL"
1934
"OCI_NAMESPACE"
2035
)
36+
missing=false
2137
for v in ${required_env_vars[@]}; do
2238
if [[ "${!v}" == "" ]]; then
23-
echo "Error: the following environment variable is required: ${v}"
24-
exit 1
39+
echo "::error title=Missing fallback variable::Missing fallback variable ${v}"
40+
missing=true
41+
else
42+
echo "::warning title=Deprecated fallback variable::Variable ${v} has been deprecated"
2543
fi
2644
done
45+
if [[ "${missing}" == "false" ]]; then
46+
exit 0
47+
else
48+
exit 1
49+
fi
2750
2851
- name: Build OCI distribution-spec conformance binary
2952
shell: bash
@@ -34,14 +57,16 @@ runs:
3457
# Enter the directory containing the checkout of this action which is surpisingly hard to do (but we did it... #OCI)
3558
cd "$(dirname $(find $(find ~/work/_actions -name distribution-spec -print -quit) -name Makefile -print -quit))"
3659
37-
# The .git folder is not present, but the dirname is the requested action ref, so use this as the conformance version
38-
conformance_version="$(basename "${PWD}")"
39-
echo "conformance-version=${conformance_version}" >> $GITHUB_OUTPUT
60+
# The .git folder is not present, but the dirname is the requested action ref, so use this as the commit version
61+
commit_version="$(basename "${PWD}")"
4062
4163
# Build the conformance binary
42-
CONFORMANCE_VERSION="${conformance_version}" OUTPUT_DIRNAME=bin make conformance-binary
64+
CONFORMANCE_VERSION="${commit_version}" OUTPUT_DIRNAME=bin make conformance-cmd
65+
66+
# The spec version is independent of the conformance git commit
67+
echo "oci-spec-version=${OCI_VERSION:-stable}" >> $GITHUB_OUTPUT
4368
44-
# Add bin to the PATH so we can just run "conformance.test"
69+
# Add bin to the PATH so we can just run "conformance"
4570
echo "${PWD}/bin" >> $GITHUB_PATH
4671
4772
- name: Run OCI distribution-spec conformance binary
@@ -50,7 +75,8 @@ runs:
5075
run: |
5176
set -x
5277
set +e
53-
conformance.test
78+
OCI_RESULTS_DIR=${OCI_RESULTS_DIR:-.}
79+
conformance
5480
conformance_rc="$?"
5581
set -e
5682
if [[ -f report.html ]]; then
@@ -62,9 +88,10 @@ runs:
6288
6389
- name: Upload OCI distribution-spec conformance results as build artifact
6490
if: always() && steps.run-conformance.outputs.has-report == 'true'
65-
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
91+
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
6692
with:
67-
name: oci-distribution-spec-conformance-results-${{ steps.build-conformance.outputs.conformance-version }}
93+
name: oci-distribution-spec-conformance-results-${{ steps.build-conformance.outputs.oci-spec-version }}
6894
path: |
95+
results.yaml
6996
report.html
7097
junit.xml

‎conformance/.gitignore‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
vendor/
2-
junit.xml
3-
report.html
1+
conformance
42
conformance.test
5-
tags
6-
env.sh
3+
oci-conformance.yaml
4+
results/**
5+
vendor/

0 commit comments

Comments
 (0)