Skip to content

Commit c5dd99d

Browse files
authored
Add files property to run configurations (#2848)
Each item in the `files` property maps a local path (a file or a dir) to a container path. Each local path is packed into a tar archive and uploaded to the server similar to the code repo archive/diff. On the server, each archive is stored in the DB as `FileArchiveModel` linked to the user. Archive blobs are optionally uploaded to the storage (again, similar to the code blob). When the job is submitted to the runner, files (if any) are uploaded after `/api/submit` but before `/api/upload_code`. The runner unpacks archives as follows: * If the path already exists, it's removed * If any parent dirs of the path are missing, they are created as owned by the run user. The owner of the existing dirs is not changed. * The owner of the path (and all subpaths in the case of the directory) is set to the run user. The permissions from the archive (and thus from the user's machine) are preserved. Part-of: #2738
1 parent cb9d746 commit c5dd99d

File tree

41 files changed

+1120
-50
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+1120
-50
lines changed

docs/docs/reference/dstack.yml/dev-environment.md

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,4 +90,21 @@ The `dev-environment` configuration type allows running [dev environments](../..
9090
The short syntax for volumes is a colon-separated string in the form of `source:destination`
9191

9292
* `volume-name:/container/path` for network volumes
93-
* `/instance/path:/container/path` for instance volumes
93+
* `/instance/path:/container/path` for instance volumes
94+
95+
### `files[n]` { #_files data-toc-label="files" }
96+
97+
#SCHEMA# dstack._internal.core.models.files.FilePathMapping
98+
overrides:
99+
show_root_heading: false
100+
type:
101+
required: true
102+
103+
??? info "Short syntax"
104+
105+
The short syntax for files is a colon-separated string in the form of `local_path[:path]` where
106+
`path` is optional and can be omitted if it's equal to `local_path`.
107+
108+
* `~/.bashrc`, same as `~/.bashrc:~/.bashrc`
109+
* `/opt/myorg`, same as `/opt/myorg/` and `/opt/myorg:/opt/myorg`
110+
* `libs/patched_libibverbs.so.1:/lib/x86_64-linux-gnu/libibverbs.so.1`

docs/docs/reference/dstack.yml/service.md

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,3 +185,20 @@ The `service` configuration type allows running [services](../../concepts/servic
185185

186186
* `volume-name:/container/path` for network volumes
187187
* `/instance/path:/container/path` for instance volumes
188+
189+
### `files[n]` { #_files data-toc-label="files" }
190+
191+
#SCHEMA# dstack._internal.core.models.files.FilePathMapping
192+
overrides:
193+
show_root_heading: false
194+
type:
195+
required: true
196+
197+
??? info "Short syntax"
198+
199+
The short syntax for files is a colon-separated string in the form of `local_path[:path]` where
200+
`path` is optional and can be omitted if it's equal to `local_path`.
201+
202+
* `~/.bashrc`, same as `~/.bashrc:~/.bashrc`
203+
* `/opt/myorg`, same as `/opt/myorg/` and `/opt/myorg:/opt/myorg`
204+
* `libs/patched_libibverbs.so.1:/lib/x86_64-linux-gnu/libibverbs.so.1`

docs/docs/reference/dstack.yml/task.md

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,3 +91,20 @@ The `task` configuration type allows running [tasks](../../concepts/tasks.md).
9191

9292
* `volume-name:/container/path` for network volumes
9393
* `/instance/path:/container/path` for instance volumes
94+
95+
### `files[n]` { #_files data-toc-label="files" }
96+
97+
#SCHEMA# dstack._internal.core.models.files.FilePathMapping
98+
overrides:
99+
show_root_heading: false
100+
type:
101+
required: true
102+
103+
??? info "Short syntax"
104+
105+
The short syntax for files is a colon-separated string in the form of `local_path[:path]` where
106+
`path` is optional and can be omitted if it's equal to `local_path`.
107+
108+
* `~/.bashrc`, same as `~/.bashrc:~/.bashrc`
109+
* `/opt/myorg`, same as `/opt/myorg/` and `/opt/myorg:/opt/myorg`
110+
* `libs/patched_libibverbs.so.1:/lib/x86_64-linux-gnu/libibverbs.so.1`

runner/go.mod

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,16 @@ go 1.23
55
require (
66
github.com/alexellis/go-execute/v2 v2.2.1
77
github.com/bluekeyes/go-gitdiff v0.7.2
8+
github.com/codeclysm/extract/v4 v4.0.0
89
github.com/creack/pty v1.1.24
910
github.com/docker/docker v26.0.0+incompatible
1011
github.com/docker/go-connections v0.5.0
1112
github.com/docker/go-units v0.5.0
1213
github.com/go-git/go-git/v5 v5.12.0
1314
github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f
15+
github.com/gorilla/websocket v1.5.1
1416
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf
17+
github.com/prometheus/procfs v0.15.1
1518
github.com/shirou/gopsutil/v4 v4.24.11
1619
github.com/sirupsen/logrus v1.9.3
1720
github.com/stretchr/testify v1.10.0
@@ -78,12 +81,6 @@ require (
7881
google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda // indirect
7982
google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect
8083
gopkg.in/warnings.v0 v0.1.2 // indirect
81-
gotest.tools/v3 v3.5.0 // indirect
82-
)
83-
84-
require (
85-
github.com/codeclysm/extract/v3 v3.1.1
86-
github.com/gorilla/websocket v1.5.1
87-
github.com/prometheus/procfs v0.15.1
8884
gopkg.in/yaml.v3 v3.0.1 // indirect
85+
gotest.tools/v3 v3.5.0 // indirect
8986
)

runner/go.sum

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ github.com/alexellis/go-execute/v2 v2.2.1 h1:4Ye3jiCKQarstODOEmqDSRCqxMHLkC92Bhs
1313
github.com/alexellis/go-execute/v2 v2.2.1/go.mod h1:FMdRnUTiFAmYXcv23txrp3VYZfLo24nMpiIneWgKHTQ=
1414
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
1515
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
16-
github.com/arduino/go-paths-helper v1.2.0 h1:qDW93PR5IZUN/jzO4rCtexiwF8P4OIcOmcSgAYLZfY4=
17-
github.com/arduino/go-paths-helper v1.2.0/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3v5YYu35Yb+w31Ck=
16+
github.com/arduino/go-paths-helper v1.12.1 h1:WkxiVUxBjKWlLMiMuYy8DcmVrkxdP7aKxQOAq7r2lVM=
17+
github.com/arduino/go-paths-helper v1.12.1/go.mod h1:jcpW4wr0u69GlXhTYydsdsqAjLaYK5n7oWHfKqOG6LM=
1818
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
1919
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
2020
github.com/bluekeyes/go-gitdiff v0.7.2 h1:42jrcVZdjjxXtVsFNYTo/I6T1ZvIiQL+iDDLiH904hw=
@@ -26,8 +26,8 @@ github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyY
2626
github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
2727
github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
2828
github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
29-
github.com/codeclysm/extract/v3 v3.1.1 h1:iHZtdEAwSTqPrd+1n4jfhr1qBhUWtHlMTjT90+fJVXg=
30-
github.com/codeclysm/extract/v3 v3.1.1/go.mod h1:ZJi80UG2JtfHqJI+lgJSCACttZi++dHxfWuPaMhlOfQ=
29+
github.com/codeclysm/extract/v4 v4.0.0 h1:H87LFsUNaJTu2e/8p/oiuiUsOK/TaPQ5wxsjPnwPEIY=
30+
github.com/codeclysm/extract/v4 v4.0.0/go.mod h1:SFju1lj6as7FvUgalpSct7torJE0zttbJUWtryPRG6s=
3131
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
3232
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
3333
github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=

runner/internal/api/common.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -105,18 +105,18 @@ func DecodeJSONBody(w http.ResponseWriter, r *http.Request, dst interface{}, all
105105
func JSONResponseHandler(handler func(http.ResponseWriter, *http.Request) (interface{}, error)) func(http.ResponseWriter, *http.Request) {
106106
return func(w http.ResponseWriter, r *http.Request) {
107107
status := 200
108-
msg := ""
108+
errMsg := ""
109109
var apiErr *Error
110110

111111
body, err := handler(w, r)
112112
if err != nil {
113113
if errors.As(err, &apiErr) {
114114
status = apiErr.Status
115-
msg = apiErr.Error()
116-
log.Warning(r.Context(), "API error", "err", apiErr.Err)
115+
errMsg = apiErr.Error()
116+
log.Warning(r.Context(), "API error", "err", errMsg, "status", status)
117117
} else {
118118
status = http.StatusInternalServerError
119-
log.Error(r.Context(), "Unexpected API error", "err", err)
119+
log.Error(r.Context(), "Unexpected API error", "err", err, "status", status)
120120
}
121121
}
122122

@@ -125,7 +125,7 @@ func JSONResponseHandler(handler func(http.ResponseWriter, *http.Request) (inter
125125
w.WriteHeader(status)
126126
_ = json.NewEncoder(w).Encode(body)
127127
} else {
128-
http.Error(w, msg, status)
128+
http.Error(w, errMsg, status)
129129
}
130130

131131
log.Debug(r.Context(), "", "method", r.Method, "endpoint", r.URL.Path, "status", status)

runner/internal/executor/base.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package executor
22

33
import (
44
"context"
5+
"io"
56

67
"github.com/dstackai/dstack/runner/internal/schemas"
78
"github.com/dstackai/dstack/runner/internal/types"
@@ -22,6 +23,7 @@ type Executor interface {
2223
termination_message string,
2324
)
2425
SetRunnerState(state string)
26+
AddFileArchive(id string, src io.Reader) error
2527
Lock()
2628
RLock()
2729
RUnlock()

runner/internal/executor/executor.go

Lines changed: 30 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,9 @@ type RunExecutor struct {
3131
tempDir string
3232
homeDir string
3333
workingDir string
34+
archiveDir string
3435
sshPort int
35-
uid uint32
36+
currentUid uint32
3637

3738
run schemas.Run
3839
jobSpec schemas.JobSpec
@@ -77,8 +78,9 @@ func NewRunExecutor(tempDir string, homeDir string, workingDir string, sshPort i
7778
tempDir: tempDir,
7879
homeDir: homeDir,
7980
workingDir: workingDir,
81+
archiveDir: filepath.Join(tempDir, "file_archives"),
8082
sshPort: sshPort,
81-
uid: uid,
83+
currentUid: uid,
8284

8385
mu: mu,
8486
state: WaitSubmit,
@@ -131,6 +133,28 @@ func (ex *RunExecutor) Run(ctx context.Context) (err error) {
131133
ctx = log.WithLogger(ctx, log.NewEntry(logger, int(log.DefaultEntry.Logger.Level))) // todo loglevel
132134
log.Info(ctx, "Run job", "log_level", log.GetLogger(ctx).Logger.Level.String())
133135

136+
if ex.jobSpec.User != nil {
137+
if err := fillUser(ex.jobSpec.User); err != nil {
138+
ex.SetJobStateWithTerminationReason(
139+
ctx,
140+
types.JobStateFailed,
141+
types.TerminationReasonExecutorError,
142+
fmt.Sprintf("Failed to fill in the job user fields (%s)", err),
143+
)
144+
return gerrors.Wrap(err)
145+
}
146+
}
147+
148+
if err := ex.setupFiles(ctx); err != nil {
149+
ex.SetJobStateWithTerminationReason(
150+
ctx,
151+
types.JobStateFailed,
152+
types.TerminationReasonExecutorError,
153+
fmt.Sprintf("Failed to set up files (%s)", err),
154+
)
155+
return gerrors.Wrap(err)
156+
}
157+
134158
if err := ex.setupRepo(ctx); err != nil {
135159
ex.SetJobStateWithTerminationReason(
136160
ctx,
@@ -140,6 +164,7 @@ func (ex *RunExecutor) Run(ctx context.Context) (err error) {
140164
)
141165
return gerrors.Wrap(err)
142166
}
167+
143168
cleanupCredentials, err := ex.setupCredentials(ctx)
144169
if err != nil {
145170
ex.SetJobState(ctx, types.JobStateFailed)
@@ -300,16 +325,13 @@ func (ex *RunExecutor) execJob(ctx context.Context, jobLogFile io.Writer) error
300325

301326
user := ex.jobSpec.User
302327
if user != nil {
303-
if err := fillUser(user); err != nil {
304-
return gerrors.Wrap(err)
305-
}
306328
log.Trace(
307329
ctx, "Using credentials",
308330
"uid", *user.Uid, "gid", *user.Gid, "groups", user.GroupIds,
309331
"username", user.GetUsername(), "groupname", user.GetGroupname(),
310332
"home", user.HomeDir,
311333
)
312-
log.Trace(ctx, "Current user", "uid", ex.uid)
334+
log.Trace(ctx, "Current user", "uid", ex.currentUid)
313335

314336
// 1. Ideally, We should check uid, gid, and supplementary groups mismatches,
315337
// but, for the sake of simplicity, we only check uid. Unprivileged runner
@@ -318,8 +340,8 @@ func (ex *RunExecutor) execJob(ctx context.Context, jobLogFile io.Writer) error
318340
// 2. Strictly speaking, we need CAP_SETUID and CAP_GUID (for Cmd.Start()->
319341
// Cmd.SysProcAttr.Credential) and CAP_CHOWN (for startCommand()->os.Chown()),
320342
// but for the sake of simplicity we instead check if we are root or not
321-
if *user.Uid != ex.uid && ex.uid != 0 {
322-
return gerrors.Newf("cannot start job as %d, current uid is %d", *user.Uid, ex.uid)
343+
if *user.Uid != ex.currentUid && ex.currentUid != 0 {
344+
return gerrors.Newf("cannot start job as %d, current uid is %d", *user.Uid, ex.currentUid)
323345
}
324346

325347
if cmd.SysProcAttr == nil {

runner/internal/executor/files.go

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
package executor
2+
3+
import (
4+
"context"
5+
"errors"
6+
"fmt"
7+
"io"
8+
"os"
9+
"path"
10+
"regexp"
11+
"slices"
12+
"strings"
13+
14+
"github.com/codeclysm/extract/v4"
15+
"github.com/dstackai/dstack/runner/internal/gerrors"
16+
"github.com/dstackai/dstack/runner/internal/log"
17+
)
18+
19+
var renameRegex = regexp.MustCompile(`^([^/]*)(/|$)`)
20+
21+
func (ex *RunExecutor) AddFileArchive(id string, src io.Reader) error {
22+
if err := os.MkdirAll(ex.archiveDir, 0o755); err != nil {
23+
return gerrors.Wrap(err)
24+
}
25+
archivePath := path.Join(ex.archiveDir, id)
26+
archive, err := os.Create(archivePath)
27+
if err != nil {
28+
return gerrors.Wrap(err)
29+
}
30+
defer func() { _ = archive.Close() }()
31+
if _, err = io.Copy(archive, src); err != nil {
32+
return gerrors.Wrap(err)
33+
}
34+
return nil
35+
}
36+
37+
// setupFiles must be called from Run
38+
func (ex *RunExecutor) setupFiles(ctx context.Context) error {
39+
homeDir := ex.workingDir
40+
uid := -1
41+
gid := -1
42+
if ex.jobSpec.User != nil {
43+
if ex.jobSpec.User.HomeDir != "" {
44+
homeDir = ex.jobSpec.User.HomeDir
45+
}
46+
if ex.jobSpec.User.Uid != nil {
47+
uid = int(*ex.jobSpec.User.Uid)
48+
}
49+
if ex.jobSpec.User.Gid != nil {
50+
gid = int(*ex.jobSpec.User.Gid)
51+
}
52+
}
53+
54+
for _, fa := range ex.run.RunSpec.FileArchives {
55+
log.Trace(ctx, "Extracting file archive", "id", fa.Id, "path", fa.Path)
56+
57+
p := path.Clean(fa.Path)
58+
// `~username[/path/to]` is not supported
59+
if p == "~" {
60+
p = homeDir
61+
} else if rest, found := strings.CutPrefix(p, "~/"); found {
62+
p = path.Join(homeDir, rest)
63+
} else if !path.IsAbs(p) {
64+
p = path.Join(ex.workingDir, p)
65+
}
66+
dir, root := path.Split(p)
67+
if err := mkdirAll(ctx, dir, uid, gid); err != nil {
68+
return gerrors.Wrap(err)
69+
}
70+
71+
if err := os.RemoveAll(p); err != nil {
72+
log.Warning(ctx, "Failed to remove", "path", p, "err", err)
73+
}
74+
75+
archivePath := path.Join(ex.archiveDir, fa.Id)
76+
archive, err := os.Open(archivePath)
77+
if err != nil {
78+
return gerrors.Wrap(err)
79+
}
80+
defer func() {
81+
_ = archive.Close()
82+
if err := os.Remove(archivePath); err != nil {
83+
log.Warning(ctx, "Failed to remove archive", "path", archivePath, "err", err)
84+
}
85+
}()
86+
87+
var paths []string
88+
repl := fmt.Sprintf("%s$2", root)
89+
renameAndRemember := func(s string) string {
90+
s = renameRegex.ReplaceAllString(s, repl)
91+
paths = append(paths, s)
92+
return s
93+
}
94+
if err := extract.Tar(ctx, archive, dir, renameAndRemember); err != nil {
95+
return gerrors.Wrap(err)
96+
}
97+
98+
if uid != -1 || gid != -1 {
99+
for _, p := range paths {
100+
if err := os.Chown(path.Join(dir, p), uid, gid); err != nil {
101+
log.Warning(ctx, "Failed to chown", "path", p, "err", err)
102+
}
103+
}
104+
}
105+
}
106+
107+
return nil
108+
}
109+
110+
func mkdirAll(ctx context.Context, p string, uid int, gid int) error {
111+
var paths []string
112+
for {
113+
p = path.Dir(p)
114+
if p == "/" {
115+
break
116+
}
117+
paths = append(paths, p)
118+
}
119+
for _, p := range slices.Backward(paths) {
120+
if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) {
121+
if err := os.Mkdir(p, 0o755); err != nil {
122+
return err
123+
}
124+
if err := os.Chown(p, uid, gid); err != nil {
125+
log.Warning(ctx, "Failed to chown", "path", p, "err", err)
126+
}
127+
} else if err != nil {
128+
return err
129+
}
130+
}
131+
return nil
132+
}

runner/internal/executor/repo.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import (
77
"os/exec"
88
"path/filepath"
99

10-
"github.com/codeclysm/extract/v3"
10+
"github.com/codeclysm/extract/v4"
1111
"github.com/dstackai/dstack/runner/internal/gerrors"
1212
"github.com/dstackai/dstack/runner/internal/log"
1313
"github.com/dstackai/dstack/runner/internal/repo"

0 commit comments

Comments
 (0)