Add secret producer/receiver and docker credential helper

.github/workflows/test-pr.yaml

@@ -46,6 +46,9 @@ jobs:
             vscode-docker:
               filters:
                 - 'features/src/workbench-tools/**'
+            test-app-secrets:
+              filters:
+                - 'src/common/**'
             workbench-jupyter:
               template: custom-workbench-jupyter-template
               maximize_build_space: true

feature-versions/state.json

@@ -2,86 +2,91 @@
   "ghcr.io/devcontainers/features/aws-cli": {
     "tag": "1",
     "installed": "sha256:17cb4a40151f59144b46957b9264683663b0214371a041ecd53dccc015a4b923",
-    "filter": ".*\\/\\.devcontainer\\.json"
+    "filter": "src\\/.*\\/\\.devcontainer\\.json"
   },
   "ghcr.io/devcontainers/features/java": {
     "tag": "1",
     "installed": "sha256:9663ce0219ff85786e87901ce5f0a59f488edd5f99b46015192cda48468b233a",
-    "filter": ".*\\/\\.devcontainer\\.json"
+    "filter": "src\\/.*\\/\\.devcontainer\\.json"
   },
   "ghcr.io/dhoeric/features/google-cloud-cli": {
     "tag": "1",
     "installed": "sha256:fa5d894718825c5ad8009ac8f2c9f0cea3d1661eb108a9d465cba9f3fc48965f",
-    "filter": ".*\\/\\.devcontainer\\.json"
+    "filter": "src\\/.*\\/\\.devcontainer\\.json"
   },
   "ghcr.io/rocker-org/devcontainer-features/r-packages": {
     "tag": "1",
     "installed": "sha256:1a4ec64c4d2060e78e9c812bd3b3622c7e008465d566a2781c0e2b17a82592e5",
-    "filter": ".*\\/\\.devcontainer\\.json"
+    "filter": "src\\/.*\\/\\.devcontainer\\.json"
   },
   "ghcr.io/devcontainers/features/common-utils": {
     "tag": "2",
     "installed": "sha256:dbf431d6b42d55cde50fa1df75c7f7c3999a90cde6d73f7a7071174b3c3d0cc4",
-    "filter": ".*\\/\\.devcontainer\\.json"
+    "filter": "src\\/.*\\/\\.devcontainer\\.json"
   },
   "ghcr.io/devcontainers/features/node": {
     "tag": "1",
     "installed": "sha256:8c0de46939b61958041700ee89e3493f3b2e4131a06dc46b4d9423427d06e5f6",
-    "filter": ".*\\/\\.devcontainer\\.json"
+    "filter": "src\\/.*\\/\\.devcontainer\\.json"
   },
   "ghcr.io/anthropics/devcontainer-features/claude-code": {
     "tag": "1.0",
     "installed": "sha256:cfc2e7d3e9fd3b9b01f8d5cb158508a884c8c0ede2e23ed10f32dea5d4ffe69a",
-    "filter": ".*\\/\\.devcontainer\\.json"
+    "filter": "src\\/.*\\/\\.devcontainer\\.json"
   },
   "us-west2-docker.pkg.dev/shared-pub-buckets-94mvrf/workbench-artifacts/app-wondershaper": {
     "tag": "latest",
     "installed": "sha256:0438761b165f6f8da90383722278be8cf89607f39cd42c386877fe72f26b3b40",
-    "filter": ".*\\/docker-compose.yaml"
+    "filter": "src\\/.*\\/docker-compose.yaml"
   },
   "us-west2-docker.pkg.dev/shared-pub-buckets-94mvrf/workbench-artifacts/app-jupyter-extension-builder": {
     "tag": "latest",
     "installed": "sha256:454db241e887998792175c600b9a04ef78cbf8cede9b75c92f09982cbbd7ae65",
-    "filter": ".*\\/Dockerfile"
+    "filter": "src\\/.*\\/Dockerfile"
   },
   "us-west2-docker.pkg.dev/shared-pub-buckets-94mvrf/workbench-artifacts/app-aou-jupyter": {
     "tag": "latest",
     "installed": "sha256:91bf43412e28a0eeb057ac480f243ce105636be59b16b105cb9e2fcb5851f041",
-    "filter": ".*\\/Dockerfile"
+    "filter": "src\\/.*\\/Dockerfile"
   },
   "us-west2-docker.pkg.dev/shared-pub-buckets-94mvrf/workbench-artifacts/app-workbench-jupyter": {
     "tag": "latest",
     "installed": "sha256:fe512afbcca4d0d112724bcefac8d36af1fd789c74ff912f1de507f2fc0b94ea",
-    "filter": ".*\\/Dockerfile"
+    "filter": "src\\/.*\\/Dockerfile"
   },
   "ghcr.io/rocker-org/devcontainer/tidyverse": {
     "tag": "4",
     "installed": "sha256:289c5d02d8115aa209f4a8a49ee9378dccbf623897eed9cc46c87dfbbca9015b",
-    "filter": ".*\\/(docker-compose\\.yaml|Dockerfile)"
+    "filter": "src\\/.*\\/(docker-compose\\.yaml|Dockerfile)"
   },
   "lscr.io/linuxserver/code-server": {
     "tag": "latest",
     "installed": "sha256:1f384394d473c43ab6a39b2227ba3aa9c95af648ce3a67e1b4da1969c16c7c0d",
-    "filter": ".*\\/(docker-compose\\.yaml|Dockerfile)"
+    "filter": "src\\/.*\\/(docker-compose\\.yaml|Dockerfile)"
   },
   "golang": {
-    "tag": "1.23-bookworm",
-    "installed": "sha256:167053a2bb901972bf2c1611f8f52c44d5fe7e762e5cab213708d82c421614db",
-    "filter": ".*\\/Dockerfile"
+    "tag": "1.26-alpine",
+    "installed": "sha256:f85330846cde1e57ca9ec309382da3b8e6ae3ab943d2739500e08c86393a21b1",
+    "filter": "src\\/.*\\/Dockerfile"
   },
   "nvcr.io/nvidia/nemo": {
     "tag": "25.07.nemotron-nano-v2",
     "installed": "sha256:f96daf8b2f07a4f8fb20e754f91b507e507ceb9119943027a4d43d7ca15e3896",
-    "filter": ".*\\/Dockerfile"
+    "filter": "src\\/.*\\/Dockerfile"
   },
   "nvcr.io/nvidia/clara/clara-parabricks": {
     "tag": "4.6.0-1",
     "installed": "sha256:d0761eb4b9921bc046c53520287316d545eb79feaeb8f22387e9bb5734650447",
-    "filter": ".*\\/Dockerfile"
+    "filter": "src\\/.*\\/Dockerfile"
   },
   "sosedoff/pgweb": {
     "tag": "latest",
     "installed": "sha256:a5256d416e2e8b92d69a4459058e3eca33a9f075d8325491644411d0bc3bd70b",
-    "filter": ".*\\/Dockerfile"
+    "filter": "src\\/.*\\/Dockerfile"
+  },
+  "mikefarah/yq": {
+    "tag": "4",
+    "installed": "sha256:0cb4a78491b6e62ee8a9bf4fbeacbd15b5013d19bc420591b05383a696315e60",
+    "filter": "startupscript\\/butane\\/050-parse-devcontainer\\.sh"
   }
 }

feature-versions/update.sh

@@ -6,7 +6,7 @@ set -o pipefail
 
 
 SCRIPT_DIR="$(dirname "$(realpath "$0")")"
-SRC_DIR="$(realpath "$SCRIPT_DIR/../src")"
+ROOT_DIR="$(realpath "$SCRIPT_DIR/..")"
 STATE_FILE="$SCRIPT_DIR/state.json"
 readonly STATE_FILE
 STATE="$(cat "$STATE_FILE")"
@@ -25,8 +25,8 @@ for IMAGE in $(echo "$STATE" | jq -r 'keys | .[]'); do
     if [ "$INSTALLED" != "$LATEST" ]; then
         echo "Updating $IMAGE from $INSTALLED to $LATEST"
 
-        pushd "$SRC_DIR"
-        find . -regextype posix-extended -regex "$FILTER" -print0 | xargs -0L1 sed -i "s|$INSTALLED|$LATEST|g"
+        pushd "$ROOT_DIR"
+        find . -regextype posix-extended -regex "\.\/$FILTER" -print0 | xargs -0L1 sed -i "s|$INSTALLED|$LATEST|g"
         popd
 
         NEW_STATE="$(jq --arg feat "$IMAGE" --arg latest "$LATEST_DIGEST" '.[$feat].installed = $latest' "$STATE_FILE")"

src/aou-common/load-envs/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang@sha256:167053a2bb901972bf2c1611f8f52c44d5fe7e762e5cab213708d82c421614db
+FROM golang@sha256:f85330846cde1e57ca9ec309382da3b8e6ae3ab943d2739500e08c86393a21b1
 WORKDIR /source
 RUN --mount=type=bind,source=.,target=/source,rw \
     mkdir -p /dist && \

src/common/common-compose.yaml

@@ -0,0 +1,10 @@
+# This docker-compose file is meant to be included to build common image
+# dependencies. Use the .devcontainer.json "runServices" property to prevent it
+# from starting automatically.
+#
+# Paths are relative to the primary docker-compose.yaml file.
+
+services:
+  common-secret-receiver-builder:
+    build:
+      context: ../common/secret-receiver

src/common/secret-receiver/Dockerfile

@@ -0,0 +1,7 @@
+FROM golang@sha256:f85330846cde1e57ca9ec309382da3b8e6ae3ab943d2739500e08c86393a21b1
+WORKDIR /source
+RUN --mount=type=bind,source=.,target=/source,rw \
+    mkdir -p /dist && \
+    go build -v -o /dist/wb-secret-receiver .
+
+ENTRYPOINT ["echo", "This image is only a build layer and should not be run directly."]

src/common/secret-receiver/go.mod

@@ -0,0 +1,5 @@
+module github.com/verily-src/workbench-app-devcontainers/src/common/secret-receiver
+
+go 1.25.0
+
+require golang.org/x/sys v0.43.0

src/common/secret-receiver/go.sum

@@ -0,0 +1,2 @@
+golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
+golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=

src/common/secret-receiver/main.go

@@ -0,0 +1,165 @@
+//go:build linux
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+
+	"golang.org/x/sys/unix"
+)
+
+// Linux pipe buffer capacity; writes up to this size complete without blocking.
+const maxPipeSecretSize = 65536
+
+const (
+	SecretTypePipeVar = "pipeVar"
+	SecretTypePathVar = "pathVar"
+	SecretTypeValueVar = "valueVar"
+)
+
+type Secret struct {
+	Type   string `json:"type"`
+	Value  string `json:"value"`
+	Target string `json:"target"`
+}
+
+func getSecrets() ([]Secret, error) {
+	pipePath := "/tmp/secrets"
+	if err := unix.Mkfifo(pipePath, 0600); err != nil {
+		return nil, err
+	}
+	defer os.Remove(pipePath)
+
+	fmt.Printf("Waiting for secrets to be written to named pipe at %s...\n", pipePath)
+	file, err := os.OpenFile(pipePath, os.O_RDONLY, os.ModeNamedPipe)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	var secrets []Secret
+	decoder := json.NewDecoder(file)
+	if err := decoder.Decode(&secrets); err != nil {
+		return nil, err
+	}
+
+	return secrets, nil
+}
+
+func writeSecretToPipe(secret Secret) (string, error) {
+	if len(secret.Value) > maxPipeSecretSize {
+		return "", fmt.Errorf("secret for %s exceeds pipe buffer size (%d > %d)", secret.Target, len(secret.Value), maxPipeSecretSize)
+	}
+
+	fds := make([]int, 2)
+	if err := unix.Pipe(fds); err != nil {
+		return "", err
+	}
+	readFd, writeFd := fds[0], fds[1]
+
+	if _, err := unix.Write(writeFd, []byte(secret.Value)); err != nil {
+		unix.Close(readFd)
+		unix.Close(writeFd)
+		return "", err
+	}
+	unix.Close(writeFd)
+
+	return fmt.Sprintf("/dev/fd/%d", readFd), nil
+}
+
+func writeSecretToMemfd(secret Secret) (path string, err error) {
+	fd, err := unix.MemfdCreate(secret.Target, 0)
+	if err != nil {
+		return "", err
+	}
+	defer func() {
+		if err != nil {
+			unix.Close(fd)
+		}
+	}()
+
+	if _, err = unix.Write(fd, []byte(secret.Value)); err != nil {
+		return "", err
+	}
+
+	if _, err = unix.Seek(fd, 0, unix.SEEK_SET); err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("/dev/fd/%d", fd), nil
+}
+
+func buildSecretEnvVars(secrets []Secret) ([]string, error) {
+	var envVars []string
+	for _, secret := range secrets {
+		switch secret.Type {
+		case SecretTypePipeVar:
+			secretPath, err := writeSecretToPipe(secret)
+			if err != nil {
+				return nil, fmt.Errorf("writing secret to pipe for %s: %w", secret.Target, err)
+			}
+			envVars = append(envVars, fmt.Sprintf("%s=%s", secret.Target, secretPath))
+		case SecretTypePathVar:
+			secretPath, err := writeSecretToMemfd(secret)
+			if err != nil {
+				return nil, fmt.Errorf("writing secret to memfd for %s: %w", secret.Target, err)
+			}
+			envVars = append(envVars, fmt.Sprintf("%s=%s", secret.Target, secretPath))
+		case SecretTypeValueVar:
+			envVars = append(envVars, fmt.Sprintf("%s=%s", secret.Target, secret.Value))
+		default:
+			return nil, fmt.Errorf("unknown secret type %s for target %s", secret.Type, secret.Target)
+		}
+	}
+
+	return envVars, nil
+}
+
+func usage() {
+	fmt.Fprintf(os.Stderr, "Usage: %s <command> [args...]\n", os.Args[0])
+	os.Exit(1)
+}
+
+func main() {
+	// Retrieve subcommand and arguments
+	args := os.Args[1:]
+	if len(args) < 1 {
+		usage()
+	}
+
+	// Validate command before waiting for secrets
+	if strings.Contains(args[0], " ") {
+		args = append([]string{"sh", "-c"}, args...)
+	}
+
+	cmdPath, err := exec.LookPath(args[0])
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error finding command %s: %v\n", args[0], err)
+		os.Exit(1)
+	}
+	args[0] = cmdPath
+
+	// Get secrets from named pipe
+	secrets, err := getSecrets()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error getting secrets: %v\n", err)
+		os.Exit(1)
+	}
+
+	secretEnvVars, err := buildSecretEnvVars(secrets)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error building secret env vars: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Replace current process with the specified command
+	env := append(os.Environ(), secretEnvVars...)
+	if err := unix.Exec(cmdPath, args, env); err != nil {
+		fmt.Fprintf(os.Stderr, "Error executing command: %v\n", err)
+		os.Exit(1)
+	}
+}

src/test-app-secrets/.devcontainer.json

@@ -0,0 +1,9 @@
+{
+  "name": "test-app-secrets",
+  "dockerComposeFile": "docker-compose.yaml",
+  "service": "app",
+  "runServices": ["app"],
+  "shutdownAction": "none",
+  "workspaceFolder": "/workspace",
+  "remoteUser": "root"
+}

src/test-app-secrets/Dockerfile

@@ -0,0 +1,3 @@
+FROM python:3-slim
+
+COPY --from=wb-secret-receiver /dist/wb-secret-receiver /wb-secret-receiver

src/test-app-secrets/devcontainer-template.json

@@ -0,0 +1,20 @@
+{
+  "id": "test-app-secrets",
+  "version": "1.0.0",
+  "name": "test-app-secrets",
+  "description": "Test app demonstrating secret receiver integration",
+  "options": {
+    "cloud": {
+      "type": "string",
+      "enum": ["gcp", "aws"],
+      "default": "gcp",
+      "description": "Cloud provider (gcp or aws)"
+    },
+    "login": {
+      "type": "string",
+      "description": "Whether to log in to workbench CLI",
+      "proposals": ["true", "false"],
+      "default": "false"
+    }
+  }
+}