Validate images can be pulled before pull operation (#39)

Author: jpreese

Dockerfile

@@ -1,33 +1,23 @@
-ARG GOLANG_VERSION=1.15.0
-ARG ALPINE_VERSION=3.12.0
+FROM golang:1.15.0 AS builder
 
-FROM golang:${GOLANG_VERSION} AS builder
-WORKDIR /build
-COPY . /build
-
-# Enable static builds
 ENV CGO_ENABLED=0
 
-RUN go get && \
-    go build
-
-FROM alpine:${ALPINE_VERSION}
+WORKDIR /build
+COPY . /build
 
-# OCI annotations (https://github.com/opencontainers/image-spec/blob/master/annotations.md)
-LABEL org.opencontainers.image.source="https://github.com/plexsystems/sinker" \
-      org.opencontainers.image.title="sinker" \
-      org.opencontainers.image.authors="John Reese <john@reese.dev>" \
-      org.opencontainers.image.description="Application to sync images from one registry to another"
+# SINKER_VERSION is set during the release process
+ARG SINKER_VERSION=0.0.0
+RUN go build -ldflags="-X 'github.com/plexsystems/sinker/internal/commands.sinkerVersion=${SINKER_VERSION}'"
 
-# explicitly set user/group IDs
-# RUN set -eux \
-#     && addgroup -g 1001 -S sinker \
-#     && adduser -S -D -H -u 1001 -s /sbin/nologin -G sinker -g sinker sinker
+FROM alpine:3.12.0
 
 RUN apk update && apk add --no-cache docker-cli
 
 COPY --from=builder /build/sinker /usr/bin/
 
-# USER sinker
+LABEL org.opencontainers.image.source="https://github.com/plexsystems/sinker"
+LABEL org.opencontainers.image.title="sinker"
+LABEL org.opencontainers.image.authors="John Reese <john@reese.dev>"
+LABEL org.opencontainers.image.description="Sync container images from one registry to another"
 
 ENTRYPOINT ["/usr/bin/sinker"]

Makefile

@@ -1,23 +1,24 @@
 .PHONY: build
 build:
-	go build
-
-.PHONY: remove-images
-remove-images:
-	docker rmi `docker images -a -q`
+	@go build
 
 .PHONY: test
 test:
-	go test -v ./... -count=1
+	@go test -v ./... -count=1
 
 .PHONY: acceptance
-acceptance:
-	go build
-	bats acceptance.bats
+acceptance: build
+	@bats acceptance.bats
+
+.PHONY: all
+all: build test acceptance
 
+# When using the release target a version must be specified.
+# e.g. make release version=v0.1.0
 .PHONY: release
 release:
 	@test $(version)
-	GOOS=darwin GOARCH=amd64 go build -o sinker-darwin-amd64 -ldflags="-X 'github.com/plexsystems/sinker/internal/commands.sinkerVersion=$(version)'"
-	GOOS=windows GOARCH=amd64 go build -o sinker-windows-amd64 -ldflags="-X 'github.com/plexsystems/sinker/internal/commands.sinkerVersion=$(version)'"
-	GOOS=linux GOARCH=amd64 go build -o sinker-linux-amd64 -ldflags="-X 'github.com/plexsystems/sinker/internal/commands.sinkerVersion=$(version)'"
+	@docker build --build-arg SINKER_VERSION=$(version) -t plexsystems/sinker:$(version) .
+	@GOOS=darwin GOARCH=amd64 go build -o sinker-darwin-amd64 -ldflags="-X 'github.com/plexsystems/sinker/internal/commands.sinkerVersion=$(version)'"
+	@GOOS=windows GOARCH=amd64 go build -o sinker-windows-amd64 -ldflags="-X 'github.com/plexsystems/sinker/internal/commands.sinkerVersion=$(version)'"
+	@GOOS=linux GOARCH=amd64 go build -o sinker-linux-amd64 -ldflags="-X 'github.com/plexsystems/sinker/internal/commands.sinkerVersion=$(version)'"

internal/commands/check.go

@@ -47,7 +47,7 @@ func runCheckCommand(input string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 
-	client, err := docker.NewClient(log.Infof)
+	client, err := docker.New(log.Infof)
 	if err != nil {
 		return fmt.Errorf("new client: %w", err)
 	}

internal/commands/pull.go

@@ -48,7 +48,7 @@ func runPullCommand(origin string, manifestPath string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
 	defer cancel()
 
-	client, err := docker.NewClient(log.Infof)
+	client, err := docker.New(log.Infof)
 	if err != nil {
 		return fmt.Errorf("new client: %w", err)
 	}
@@ -77,9 +77,18 @@ func runPullCommand(origin string, manifestPath string) error {
 		}
 	}
 
+	// Iterate through each of the images to pull and verify if the client has
+	// the proper authorization to be able to successfully pull the images before
+	// performing the pull operation.
+	for image := range imagesToPull {
+		if _, err := client.ImageExistsAtRemote(ctx, image); err != nil {
+			return fmt.Errorf("validating remote image: %w", err)
+		}
+	}
+
 	for image, auth := range imagesToPull {
 		log.Infof("Pulling %s", image)
-		if err := client.PullImageAndWait(ctx, image, auth); err != nil {
+		if err := client.PullAndWait(ctx, image, auth); err != nil {
 			log.Errorf("pull image and wait: " + err.Error())
 		}
 	}

internal/commands/push.go

@@ -56,7 +56,7 @@ func runPushCommand(manifestPath string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
 	defer cancel()
 
-	client, err := docker.NewClient(log.Infof)
+	client, err := docker.New(log.Infof)
 	if err != nil {
 		return fmt.Errorf("new client: %w", err)
 	}
@@ -112,7 +112,7 @@ func runPushCommand(manifestPath string) error {
 			if err != nil {
 				return fmt.Errorf("get source auth: %w", err)
 			}
-			if err := client.PullImageAndWait(ctx, source.Image(), sourceAuth); err != nil {
+			if err := client.PullAndWait(ctx, source.Image(), sourceAuth); err != nil {
 				return fmt.Errorf("pull image and wait: %w", err)
 			}
 		}
@@ -133,7 +133,7 @@ func runPushCommand(manifestPath string) error {
 		if err != nil {
 			return fmt.Errorf("get target auth: %w", err)
 		}
-		if err := client.PushImageAndWait(ctx, source.TargetImage(), targetAuth); err != nil {
+		if err := client.PushAndWait(ctx, source.TargetImage(), targetAuth); err != nil {
 			return fmt.Errorf("push image and wait: %w", err)
 		}
 	}

internal/docker/docker.go

@@ -23,8 +23,8 @@ type Client struct {
 	logInfo func(format string, args ...interface{})
 }
 
-// NewClient returns a Docker client configured with the given information logger.
-func NewClient(logInfo func(format string, args ...interface{})) (Client, error) {
+// New returns a Docker client configured with the given information logger.
+func New(logInfo func(format string, args ...interface{})) (Client, error) {
 	retry.DefaultDelay = 5 * time.Second
 	retry.DefaultAttempts = 2
 
@@ -41,11 +41,11 @@ func NewClient(logInfo func(format string, args ...interface{})) (Client, error)
 	return client, nil
 }
 
-// PushImageAndWait pushes an image and waits for it to finish pushing.
+// PushAndWait pushes an image and waits for it to finish pushing.
 // If an error occurs when pushing an image, the push will be attempted again before failing.
-func (c Client) PushImageAndWait(ctx context.Context, image string, auth string) error {
+func (c Client) PushAndWait(ctx context.Context, image string, auth string) error {
 	push := func() error {
-		if err := c.tryPushImageAndWait(ctx, image, auth); err != nil {
+		if err := c.tryPushAndWait(ctx, image, auth); err != nil {
 			return fmt.Errorf("try push image: %w", err)
 		}
 
@@ -63,11 +63,11 @@ func (c Client) PushImageAndWait(ctx context.Context, image string, auth string)
 	return nil
 }
 
-// PullImageAndWait pulls an image and waits for it to finish pulling.
+// PullAndWait pulls an image and waits for it to finish pulling.
 // If an error occurs when pulling an image, the pull will be attempted again before failing.
-func (c Client) PullImageAndWait(ctx context.Context, image string, auth string) error {
+func (c Client) PullAndWait(ctx context.Context, image string, auth string) error {
 	pull := func() error {
-		if err := c.tryPullImageAndWait(ctx, image, auth); err != nil {
+		if err := c.tryPullAndWait(ctx, image, auth); err != nil {
 			return fmt.Errorf("try pull image: %w", err)
 		}
 
@@ -170,35 +170,42 @@ func (c Client) Tag(ctx context.Context, sourceImage string, targetImage string)
 
 // ImageExistsAtRemote returns true if the image exists at the remote registry.
 func (c Client) ImageExistsAtRemote(ctx context.Context, image string) (bool, error) {
-	if hasLatestTag(image) {
-		return false, nil
-	}
-
 	reference, err := name.ParseReference(image, name.WeakValidation)
 	if err != nil {
 		return false, fmt.Errorf("parse ref: %w", err)
 	}
 
 	if _, err := remote.Get(reference, remote.WithAuthFromKeychain(authn.DefaultKeychain)); err != nil {
 
-		// If the error is a transport error, check that the error code is of type MANIFEST_UNKNOWN.
-		// This is the expected error if an image does not exist.
+		// If the error is a transport error, check that the error code is of type
+		// MANIFEST_UNKNOWN or NOT_FOUND. These errors are expected if an image does
+		// not exist in the registry.
 		if t, exists := err.(*transport.Error); exists {
 			for _, diagnostic := range t.Errors {
 				if strings.EqualFold("MANIFEST_UNKNOWN", string(diagnostic.Code)) {
 					return false, nil
 				}
+
 				if strings.EqualFold("NOT_FOUND", string(diagnostic.Code)) {
 					return false, nil
 				}
 			}
 		}
 
-		// If the error is not a transport error, some other error occured
-		// that is unrelated to checking if an image exists and it should be returned.
 		return false, fmt.Errorf("get image: %w", err)
 	}
 
+	// Always return false if the image has the latest tag as this method
+	// is used to determine if the image should be pushed or not. The latest
+	// tag is assumed to always need to be pushed, but a better approach
+	// would be to compare digests.
+	//
+	// This check must also be performed after the Get request to the remote
+	// registry to ensure that the client has appropriate access to pull the image.
+	if hasLatestTag(image) {
+		return false, nil
+	}
+
 	return true, nil
 }
 
@@ -245,7 +252,7 @@ func (c Client) waitForScannerComplete(clientScanner *bufio.Scanner, image strin
 	return nil
 }
 
-func (c Client) tryPullImageAndWait(ctx context.Context, image string, auth string) error {
+func (c Client) tryPullAndWait(ctx context.Context, image string, auth string) error {
 	opts := types.ImagePullOptions{
 		RegistryAuth: auth,
 	}
@@ -266,7 +273,7 @@ func (c Client) tryPullImageAndWait(ctx context.Context, image string, auth stri
 	return nil
 }
 
-func (c Client) tryPushImageAndWait(ctx context.Context, image string, auth string) error {
+func (c Client) tryPushAndWait(ctx context.Context, image string, auth string) error {
 	opts := types.ImagePushOptions{
 		RegistryAuth: auth,
 	}