Skip to content

Commit d8745b3

Browse files
Add upper-bound chunk-size validation to put command
Reject chunk sizes below 4MiB, not a multiple of 4MiB, or above 128MiB with distinct error messages. Previously only the modulo check existed, silently allowing values that exceed Dropbox upload-session limits. Update flag descriptions, examples, and generated docs to document the constraints.
1 parent 6b49819 commit d8745b3

4 files changed

Lines changed: 64 additions & 15 deletions

File tree

README.md

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -406,12 +406,21 @@ The `--verbose` option will turn on verbose logging and is useful for debugging.
406406
```sh
407407
$ dbxcli put file.txt /destination/file.txt # upload a single file
408408
$ dbxcli put -r ./project /backup/project # recursively upload a directory
409-
$ dbxcli put -r -w 8 ./large-folder /backup/large # use 8 workers per large file
409+
$ dbxcli put -w 1 -c 134217728 large.zip /backup/large.zip
410410
$ dbxcli put --if-exists skip file.txt /dest.txt # skip if the file already exists
411411
```
412412
413413
By default, `put` overwrites existing destination files. Use `--if-exists overwrite|skip|fail` to choose whether existing files are overwritten, skipped, or treated as an error.
414414
415+
For files larger than 32MiB, `put` uses Dropbox upload sessions. By default, it
416+
uses 4 workers and 16MiB chunks. Each chunk is one upload-session request; chunk
417+
size must be a multiple of 4MiB and no more than 128MiB. On unstable networks,
418+
fewer workers and larger chunks may be more reliable:
419+
420+
```sh
421+
$ dbxcli put -w 1 -c 134217728 large.zip /backup/large.zip
422+
```
423+
415424
### Downloading files and directories
416425
417426
```sh

cmd/put.go

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,11 @@ import (
3838
const singleShotUploadSizeCutoff int64 = 32 * (1 << 20)
3939

4040
const (
41+
putChunkSizeUnit int64 = 1 << 22
42+
// Dropbox upload-session requests should stay below 150 MB. Use a
43+
// conservative 128 MiB max that is also a multiple of 4 MiB.
44+
putMaxChunkSize int64 = 128 * (1 << 20)
45+
4146
putIfExistsOverwrite = "overwrite"
4247
putIfExistsSkip = "skip"
4348
putIfExistsFail = "fail"
@@ -445,8 +450,14 @@ func parsePutOptions(cmd *cobra.Command) (putOptions, error) {
445450
if err != nil {
446451
return putOptions{}, err
447452
}
448-
if chunkSize%(1<<22) != 0 {
449-
return putOptions{}, invalidArgumentsErrorWithDetails("`put` requires chunk size to be multiple of 4MiB", flagErrorDetails("chunksize"))
453+
if chunkSize < putChunkSizeUnit {
454+
return putOptions{}, invalidArgumentsErrorWithDetails("`put` requires chunk size to be at least 4MiB", flagErrorDetails("chunksize"))
455+
}
456+
if chunkSize%putChunkSizeUnit != 0 {
457+
return putOptions{}, invalidArgumentsErrorWithDetails("`put` requires chunk size to be a multiple of 4MiB", flagErrorDetails("chunksize"))
458+
}
459+
if chunkSize > putMaxChunkSize {
460+
return putOptions{}, invalidArgumentsErrorWithDetails("`put` requires chunk size to be no more than 128MiB", flagErrorDetails("chunksize"))
450461
}
451462
workers, err := cmd.Flags().GetInt("workers")
452463
if err != nil {
@@ -889,9 +900,13 @@ var putCmd = &cobra.Command{
889900
- Use - as source to read from stdin (target is required).
890901
Stdin is spooled to a temp file before upload and may use disk
891902
space up to the full input size.
903+
- Files larger than 32MiB use Dropbox upload sessions. Each chunk is one
904+
upload-session request; chunk size must be a multiple of 4MiB and no more
905+
than 128MiB.
892906
`,
893907
Example: ` dbxcli put file.txt /destination/file.txt
894908
dbxcli put -r ./project /backup/project
909+
dbxcli put -w 1 -c 134217728 large.zip /backup/large.zip
895910
printf 'hello' | dbxcli put - /hello.txt
896911
tar cz ./src | dbxcli put - /backups/src.tgz`,
897912
RunE: put,
@@ -901,8 +916,8 @@ func init() {
901916
RootCmd.AddCommand(putCmd)
902917
enableStructuredOutput(putCmd)
903918
putCmd.Flags().BoolP("recursive", "r", false, "Recursively upload directories")
904-
putCmd.Flags().IntP("workers", "w", 4, "Number of concurrent upload workers to use")
905-
putCmd.Flags().Int64P("chunksize", "c", 1<<24, "Chunk size to use (should be multiple of 4MiB)")
919+
putCmd.Flags().IntP("workers", "w", 4, "Number of concurrent upload workers for chunked large-file uploads")
920+
putCmd.Flags().Int64P("chunksize", "c", 1<<24, "Chunk size in bytes for chunked large-file uploads; must be a multiple of 4MiB and no more than 128MiB")
906921
putCmd.Flags().BoolP("debug", "d", false, "Print debug timing")
907922
putCmd.Flags().String("if-exists", putIfExistsOverwrite, "What to do when the destination file exists: overwrite, skip, or fail")
908923
}

cmd/put_test.go

Lines changed: 29 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -667,16 +667,37 @@ func TestPutRecursive_SkipsSymlinks(t *testing.T) {
667667
}
668668

669669
func TestPutChunkSizeValidation(t *testing.T) {
670-
tmpFile := filepath.Join(t.TempDir(), "test.txt")
671-
if err := os.WriteFile(tmpFile, []byte("test"), 0644); err != nil {
672-
t.Fatal(err)
670+
tests := []struct {
671+
name string
672+
chunkSize string
673+
want string
674+
}{
675+
{
676+
name: "below 4MiB",
677+
chunkSize: "100",
678+
want: "`put` requires chunk size to be at least 4MiB",
679+
},
680+
{
681+
name: "not multiple of 4MiB",
682+
chunkSize: "6291456",
683+
want: "`put` requires chunk size to be a multiple of 4MiB",
684+
},
685+
{
686+
name: "above Dropbox request limit",
687+
chunkSize: "268435456",
688+
want: "`put` requires chunk size to be no more than 128MiB",
689+
},
673690
}
674691

675-
cmd := testPutCmd()
676-
_ = cmd.Flags().Set("chunksize", "100")
677-
err := put(cmd, []string{tmpFile, "/test.txt"})
678-
if err == nil || err.Error() != "`put` requires chunk size to be multiple of 4MiB" {
679-
t.Errorf("expected chunk size validation error, got %v", err)
692+
for _, tt := range tests {
693+
t.Run(tt.name, func(t *testing.T) {
694+
cmd := testPutCmd()
695+
_ = cmd.Flags().Set("chunksize", tt.chunkSize)
696+
_, err := parsePutOptions(cmd)
697+
if err == nil || err.Error() != tt.want {
698+
t.Errorf("expected chunk size validation error %q, got %v", tt.want, err)
699+
}
700+
})
680701
}
681702
}
682703

docs/commands/dbxcli_put.md

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@ Upload files or directories to Dropbox.
1212
- Use - as source to read from stdin (target is required).
1313
Stdin is spooled to a temp file before upload and may use disk
1414
space up to the full input size.
15+
- Files larger than 32MiB use Dropbox upload sessions. Each chunk is one
16+
upload-session request; chunk size must be a multiple of 4MiB and no more
17+
than 128MiB.
1518

1619

1720
```
@@ -23,19 +26,20 @@ dbxcli put [flags] <source> [<target>]
2326
```
2427
dbxcli put file.txt /destination/file.txt
2528
dbxcli put -r ./project /backup/project
29+
dbxcli put -w 1 -c 134217728 large.zip /backup/large.zip
2630
printf 'hello' | dbxcli put - /hello.txt
2731
tar cz ./src | dbxcli put - /backups/src.tgz
2832
```
2933

3034
### Options
3135

3236
```
33-
-c, --chunksize int Chunk size to use (should be multiple of 4MiB) (default 16777216)
37+
-c, --chunksize int Chunk size in bytes for chunked large-file uploads; must be a multiple of 4MiB and no more than 128MiB (default 16777216)
3438
-d, --debug Print debug timing
3539
-h, --help help for put
3640
--if-exists string What to do when the destination file exists: overwrite, skip, or fail (default "overwrite")
3741
-r, --recursive Recursively upload directories
38-
-w, --workers int Number of concurrent upload workers to use (default 4)
42+
-w, --workers int Number of concurrent upload workers for chunked large-file uploads (default 4)
3943
```
4044

4145
### Options inherited from parent commands

0 commit comments

Comments
 (0)