Skip to content

Commit 24b18b8

Browse files
Add secret producer/receiver and docker credential helper (#381)
See linked verily1 PR for ordering of when the scripts will be called. - `docker-auth-secrets.sh` registers docker-credential-workbench-secret to be used when pulling a package from one of the "dockerRepository" secrets. This must be called after git-clone-devcontainer since it needs to read the secret from the devcontainer directory - `docker-credential-secrets.sh` is the script behind docker-credential-workbench-secret (the docker-credential scripts are just a thin wrapper providing the resource path) - `provide-secrets.sh` fetches secrets configured in secrets.yml and passes them to the app. This is the "Secret Provider" in the design doc - `secret-receiver` receives the secret values and directs them to environment variables or file descriptors for the app to read - `vscode-secrets` is a sample vscode app that reads a secret named `example-secret` and stores it in environment variable `EXAMPLE_SECRET` PHP-127691
1 parent 0d5d542 commit 24b18b8

24 files changed

Lines changed: 889 additions & 24 deletions

.github/workflows/test-pr.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,9 @@ jobs:
4646
vscode-docker:
4747
filters:
4848
- 'features/src/workbench-tools/**'
49+
test-app-secrets:
50+
filters:
51+
- 'src/common/**'
4952
workbench-jupyter:
5053
template: custom-workbench-jupyter-template
5154
maximize_build_space: true

feature-versions/state.json

Lines changed: 24 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2,86 +2,91 @@
22
"ghcr.io/devcontainers/features/aws-cli": {
33
"tag": "1",
44
"installed": "sha256:17cb4a40151f59144b46957b9264683663b0214371a041ecd53dccc015a4b923",
5-
"filter": ".*\\/\\.devcontainer\\.json"
5+
"filter": "src\\/.*\\/\\.devcontainer\\.json"
66
},
77
"ghcr.io/devcontainers/features/java": {
88
"tag": "1",
99
"installed": "sha256:9663ce0219ff85786e87901ce5f0a59f488edd5f99b46015192cda48468b233a",
10-
"filter": ".*\\/\\.devcontainer\\.json"
10+
"filter": "src\\/.*\\/\\.devcontainer\\.json"
1111
},
1212
"ghcr.io/dhoeric/features/google-cloud-cli": {
1313
"tag": "1",
1414
"installed": "sha256:fa5d894718825c5ad8009ac8f2c9f0cea3d1661eb108a9d465cba9f3fc48965f",
15-
"filter": ".*\\/\\.devcontainer\\.json"
15+
"filter": "src\\/.*\\/\\.devcontainer\\.json"
1616
},
1717
"ghcr.io/rocker-org/devcontainer-features/r-packages": {
1818
"tag": "1",
1919
"installed": "sha256:1a4ec64c4d2060e78e9c812bd3b3622c7e008465d566a2781c0e2b17a82592e5",
20-
"filter": ".*\\/\\.devcontainer\\.json"
20+
"filter": "src\\/.*\\/\\.devcontainer\\.json"
2121
},
2222
"ghcr.io/devcontainers/features/common-utils": {
2323
"tag": "2",
2424
"installed": "sha256:dbf431d6b42d55cde50fa1df75c7f7c3999a90cde6d73f7a7071174b3c3d0cc4",
25-
"filter": ".*\\/\\.devcontainer\\.json"
25+
"filter": "src\\/.*\\/\\.devcontainer\\.json"
2626
},
2727
"ghcr.io/devcontainers/features/node": {
2828
"tag": "1",
2929
"installed": "sha256:8c0de46939b61958041700ee89e3493f3b2e4131a06dc46b4d9423427d06e5f6",
30-
"filter": ".*\\/\\.devcontainer\\.json"
30+
"filter": "src\\/.*\\/\\.devcontainer\\.json"
3131
},
3232
"ghcr.io/anthropics/devcontainer-features/claude-code": {
3333
"tag": "1.0",
3434
"installed": "sha256:cfc2e7d3e9fd3b9b01f8d5cb158508a884c8c0ede2e23ed10f32dea5d4ffe69a",
35-
"filter": ".*\\/\\.devcontainer\\.json"
35+
"filter": "src\\/.*\\/\\.devcontainer\\.json"
3636
},
3737
"us-west2-docker.pkg.dev/shared-pub-buckets-94mvrf/workbench-artifacts/app-wondershaper": {
3838
"tag": "latest",
3939
"installed": "sha256:0438761b165f6f8da90383722278be8cf89607f39cd42c386877fe72f26b3b40",
40-
"filter": ".*\\/docker-compose.yaml"
40+
"filter": "src\\/.*\\/docker-compose.yaml"
4141
},
4242
"us-west2-docker.pkg.dev/shared-pub-buckets-94mvrf/workbench-artifacts/app-jupyter-extension-builder": {
4343
"tag": "latest",
4444
"installed": "sha256:454db241e887998792175c600b9a04ef78cbf8cede9b75c92f09982cbbd7ae65",
45-
"filter": ".*\\/Dockerfile"
45+
"filter": "src\\/.*\\/Dockerfile"
4646
},
4747
"us-west2-docker.pkg.dev/shared-pub-buckets-94mvrf/workbench-artifacts/app-aou-jupyter": {
4848
"tag": "latest",
4949
"installed": "sha256:91bf43412e28a0eeb057ac480f243ce105636be59b16b105cb9e2fcb5851f041",
50-
"filter": ".*\\/Dockerfile"
50+
"filter": "src\\/.*\\/Dockerfile"
5151
},
5252
"us-west2-docker.pkg.dev/shared-pub-buckets-94mvrf/workbench-artifacts/app-workbench-jupyter": {
5353
"tag": "latest",
5454
"installed": "sha256:fe512afbcca4d0d112724bcefac8d36af1fd789c74ff912f1de507f2fc0b94ea",
55-
"filter": ".*\\/Dockerfile"
55+
"filter": "src\\/.*\\/Dockerfile"
5656
},
5757
"ghcr.io/rocker-org/devcontainer/tidyverse": {
5858
"tag": "4",
5959
"installed": "sha256:289c5d02d8115aa209f4a8a49ee9378dccbf623897eed9cc46c87dfbbca9015b",
60-
"filter": ".*\\/(docker-compose\\.yaml|Dockerfile)"
60+
"filter": "src\\/.*\\/(docker-compose\\.yaml|Dockerfile)"
6161
},
6262
"lscr.io/linuxserver/code-server": {
6363
"tag": "latest",
6464
"installed": "sha256:1f384394d473c43ab6a39b2227ba3aa9c95af648ce3a67e1b4da1969c16c7c0d",
65-
"filter": ".*\\/(docker-compose\\.yaml|Dockerfile)"
65+
"filter": "src\\/.*\\/(docker-compose\\.yaml|Dockerfile)"
6666
},
6767
"golang": {
68-
"tag": "1.23-bookworm",
69-
"installed": "sha256:167053a2bb901972bf2c1611f8f52c44d5fe7e762e5cab213708d82c421614db",
70-
"filter": ".*\\/Dockerfile"
68+
"tag": "1.26-alpine",
69+
"installed": "sha256:f85330846cde1e57ca9ec309382da3b8e6ae3ab943d2739500e08c86393a21b1",
70+
"filter": "src\\/.*\\/Dockerfile"
7171
},
7272
"nvcr.io/nvidia/nemo": {
7373
"tag": "25.07.nemotron-nano-v2",
7474
"installed": "sha256:f96daf8b2f07a4f8fb20e754f91b507e507ceb9119943027a4d43d7ca15e3896",
75-
"filter": ".*\\/Dockerfile"
75+
"filter": "src\\/.*\\/Dockerfile"
7676
},
7777
"nvcr.io/nvidia/clara/clara-parabricks": {
7878
"tag": "4.6.0-1",
7979
"installed": "sha256:d0761eb4b9921bc046c53520287316d545eb79feaeb8f22387e9bb5734650447",
80-
"filter": ".*\\/Dockerfile"
80+
"filter": "src\\/.*\\/Dockerfile"
8181
},
8282
"sosedoff/pgweb": {
8383
"tag": "latest",
8484
"installed": "sha256:a5256d416e2e8b92d69a4459058e3eca33a9f075d8325491644411d0bc3bd70b",
85-
"filter": ".*\\/Dockerfile"
85+
"filter": "src\\/.*\\/Dockerfile"
86+
},
87+
"mikefarah/yq": {
88+
"tag": "4",
89+
"installed": "sha256:0cb4a78491b6e62ee8a9bf4fbeacbd15b5013d19bc420591b05383a696315e60",
90+
"filter": "startupscript\\/butane\\/050-parse-devcontainer\\.sh"
8691
}
8792
}

feature-versions/update.sh

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ set -o pipefail
66

77

88
SCRIPT_DIR="$(dirname "$(realpath "$0")")"
9-
SRC_DIR="$(realpath "$SCRIPT_DIR/../src")"
9+
ROOT_DIR="$(realpath "$SCRIPT_DIR/..")"
1010
STATE_FILE="$SCRIPT_DIR/state.json"
1111
readonly STATE_FILE
1212
STATE="$(cat "$STATE_FILE")"
@@ -25,8 +25,8 @@ for IMAGE in $(echo "$STATE" | jq -r 'keys | .[]'); do
2525
if [ "$INSTALLED" != "$LATEST" ]; then
2626
echo "Updating $IMAGE from $INSTALLED to $LATEST"
2727

28-
pushd "$SRC_DIR"
29-
find . -regextype posix-extended -regex "$FILTER" -print0 | xargs -0L1 sed -i "s|$INSTALLED|$LATEST|g"
28+
pushd "$ROOT_DIR"
29+
find . -regextype posix-extended -regex "\.\/$FILTER" -print0 | xargs -0L1 sed -i "s|$INSTALLED|$LATEST|g"
3030
popd
3131

3232
NEW_STATE="$(jq --arg feat "$IMAGE" --arg latest "$LATEST_DIGEST" '.[$feat].installed = $latest' "$STATE_FILE")"

src/aou-common/load-envs/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM golang@sha256:167053a2bb901972bf2c1611f8f52c44d5fe7e762e5cab213708d82c421614db
1+
FROM golang@sha256:f85330846cde1e57ca9ec309382da3b8e6ae3ab943d2739500e08c86393a21b1
22
WORKDIR /source
33
RUN --mount=type=bind,source=.,target=/source,rw \
44
mkdir -p /dist && \

src/common/common-compose.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# This docker-compose file is meant to be included to build common image
2+
# dependencies. Use the .devcontainer.json "runServices" property to prevent it
3+
# from starting automatically.
4+
#
5+
# Paths are relative to the primary docker-compose.yaml file.
6+
7+
services:
8+
common-secret-receiver-builder:
9+
build:
10+
context: ../common/secret-receiver
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
FROM golang@sha256:f85330846cde1e57ca9ec309382da3b8e6ae3ab943d2739500e08c86393a21b1
2+
WORKDIR /source
3+
RUN --mount=type=bind,source=.,target=/source,rw \
4+
mkdir -p /dist && \
5+
go build -v -o /dist/wb-secret-receiver .
6+
7+
ENTRYPOINT ["echo", "This image is only a build layer and should not be run directly."]

src/common/secret-receiver/go.mod

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
module github.com/verily-src/workbench-app-devcontainers/src/common/secret-receiver
2+
3+
go 1.25.0
4+
5+
require golang.org/x/sys v0.43.0

src/common/secret-receiver/go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
2+
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=

src/common/secret-receiver/main.go

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
//go:build linux
2+
3+
package main
4+
5+
import (
6+
"encoding/json"
7+
"fmt"
8+
"os"
9+
"os/exec"
10+
"strings"
11+
12+
"golang.org/x/sys/unix"
13+
)
14+
15+
// Linux pipe buffer capacity; writes up to this size complete without blocking.
16+
const maxPipeSecretSize = 65536
17+
18+
const (
19+
SecretTypePipeVar = "pipeVar"
20+
SecretTypePathVar = "pathVar"
21+
SecretTypeValueVar = "valueVar"
22+
)
23+
24+
type Secret struct {
25+
Type string `json:"type"`
26+
Value string `json:"value"`
27+
Target string `json:"target"`
28+
}
29+
30+
func getSecrets() ([]Secret, error) {
31+
pipePath := "/tmp/secrets"
32+
if err := unix.Mkfifo(pipePath, 0600); err != nil {
33+
return nil, err
34+
}
35+
defer os.Remove(pipePath)
36+
37+
fmt.Printf("Waiting for secrets to be written to named pipe at %s...\n", pipePath)
38+
file, err := os.OpenFile(pipePath, os.O_RDONLY, os.ModeNamedPipe)
39+
if err != nil {
40+
return nil, err
41+
}
42+
defer file.Close()
43+
44+
var secrets []Secret
45+
decoder := json.NewDecoder(file)
46+
if err := decoder.Decode(&secrets); err != nil {
47+
return nil, err
48+
}
49+
50+
return secrets, nil
51+
}
52+
53+
func writeSecretToPipe(secret Secret) (string, error) {
54+
if len(secret.Value) > maxPipeSecretSize {
55+
return "", fmt.Errorf("secret for %s exceeds pipe buffer size (%d > %d)", secret.Target, len(secret.Value), maxPipeSecretSize)
56+
}
57+
58+
fds := make([]int, 2)
59+
if err := unix.Pipe(fds); err != nil {
60+
return "", err
61+
}
62+
readFd, writeFd := fds[0], fds[1]
63+
64+
if _, err := unix.Write(writeFd, []byte(secret.Value)); err != nil {
65+
unix.Close(readFd)
66+
unix.Close(writeFd)
67+
return "", err
68+
}
69+
unix.Close(writeFd)
70+
71+
return fmt.Sprintf("/dev/fd/%d", readFd), nil
72+
}
73+
74+
func writeSecretToMemfd(secret Secret) (path string, err error) {
75+
fd, err := unix.MemfdCreate(secret.Target, 0)
76+
if err != nil {
77+
return "", err
78+
}
79+
defer func() {
80+
if err != nil {
81+
unix.Close(fd)
82+
}
83+
}()
84+
85+
if _, err = unix.Write(fd, []byte(secret.Value)); err != nil {
86+
return "", err
87+
}
88+
89+
if _, err = unix.Seek(fd, 0, unix.SEEK_SET); err != nil {
90+
return "", err
91+
}
92+
93+
return fmt.Sprintf("/dev/fd/%d", fd), nil
94+
}
95+
96+
func buildSecretEnvVars(secrets []Secret) ([]string, error) {
97+
var envVars []string
98+
for _, secret := range secrets {
99+
switch secret.Type {
100+
case SecretTypePipeVar:
101+
secretPath, err := writeSecretToPipe(secret)
102+
if err != nil {
103+
return nil, fmt.Errorf("writing secret to pipe for %s: %w", secret.Target, err)
104+
}
105+
envVars = append(envVars, fmt.Sprintf("%s=%s", secret.Target, secretPath))
106+
case SecretTypePathVar:
107+
secretPath, err := writeSecretToMemfd(secret)
108+
if err != nil {
109+
return nil, fmt.Errorf("writing secret to memfd for %s: %w", secret.Target, err)
110+
}
111+
envVars = append(envVars, fmt.Sprintf("%s=%s", secret.Target, secretPath))
112+
case SecretTypeValueVar:
113+
envVars = append(envVars, fmt.Sprintf("%s=%s", secret.Target, secret.Value))
114+
default:
115+
return nil, fmt.Errorf("unknown secret type %s for target %s", secret.Type, secret.Target)
116+
}
117+
}
118+
119+
return envVars, nil
120+
}
121+
122+
func usage() {
123+
fmt.Fprintf(os.Stderr, "Usage: %s <command> [args...]\n", os.Args[0])
124+
os.Exit(1)
125+
}
126+
127+
func main() {
128+
// Retrieve subcommand and arguments
129+
args := os.Args[1:]
130+
if len(args) < 1 {
131+
usage()
132+
}
133+
134+
// Validate command before waiting for secrets
135+
if strings.Contains(args[0], " ") {
136+
args = append([]string{"sh", "-c"}, args...)
137+
}
138+
139+
cmdPath, err := exec.LookPath(args[0])
140+
if err != nil {
141+
fmt.Fprintf(os.Stderr, "Error finding command %s: %v\n", args[0], err)
142+
os.Exit(1)
143+
}
144+
args[0] = cmdPath
145+
146+
// Get secrets from named pipe
147+
secrets, err := getSecrets()
148+
if err != nil {
149+
fmt.Fprintf(os.Stderr, "Error getting secrets: %v\n", err)
150+
os.Exit(1)
151+
}
152+
153+
secretEnvVars, err := buildSecretEnvVars(secrets)
154+
if err != nil {
155+
fmt.Fprintf(os.Stderr, "Error building secret env vars: %v\n", err)
156+
os.Exit(1)
157+
}
158+
159+
// Replace current process with the specified command
160+
env := append(os.Environ(), secretEnvVars...)
161+
if err := unix.Exec(cmdPath, args, env); err != nil {
162+
fmt.Fprintf(os.Stderr, "Error executing command: %v\n", err)
163+
os.Exit(1)
164+
}
165+
}
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"name": "test-app-secrets",
3+
"dockerComposeFile": "docker-compose.yaml",
4+
"service": "app",
5+
"runServices": ["app"],
6+
"shutdownAction": "none",
7+
"workspaceFolder": "/workspace",
8+
"remoteUser": "root"
9+
}

0 commit comments

Comments
 (0)