Skip to content

Commit 84f9417

Browse files
metonymmetonym
authored
Add LIST for object storage (EA) (#251)
Implement `render ea objects list` to list objects in a region for both cloud and local modes. **Commands:** - `list --region=<region> [--local] [--limit=N]` – list objects (key, content type, size, last modified). Default limit is 100. Cloud mode uses the ListBlobs REST API with cursor-based pagination. Local mode (`--local` or `RENDER_USE_LOCAL_DEV=true`) reads from `.render/objects/{region}/{bucket}/`. Region is required; owner defaults to the workspace. Output is a table by default; use `-o json` or `-o text` for machine-readable formats. Returns one page per run; use `--limit` to control page size.
1 parent 0cdd20d commit 84f9417

8 files changed

Lines changed: 414 additions & 2 deletions

File tree

‎cmd/object.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,17 +10,19 @@ var objectCmd = &cobra.Command{
1010
Long: `Manage object storage for your Render services.
1111
1212
Object storage allows you to store and retrieve arbitrary data. Use these commands
13-
to upload, download, and delete objects.
13+
to list, upload, download, and delete objects.
1414
1515
In local development mode (when running with 'render ea tasks dev' or with the
1616
--local flag), objects are stored in the .render/objects/ directory.
1717
1818
Available commands:
19+
list - List objects in storage
1920
put - Upload a file to object storage
2021
get - Download a file from object storage
2122
delete - Delete an object from storage
2223
2324
Examples:
25+
render ea objects list --region=oregon
2426
render ea objects put my/object/key --file=./data.txt --region=oregon
2527
render ea objects get my/object/key --file=./output.txt --region=oregon
2628
render ea objects delete my/object/key --region=oregon --yes

‎cmd/objectlist.go‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
package cmd
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/spf13/cobra"
7+
8+
"github.com/render-oss/cli/pkg/command"
9+
"github.com/render-oss/cli/pkg/storage"
10+
"github.com/render-oss/cli/pkg/text"
11+
)
12+
13+
type ObjectListInput struct {
14+
Region string `cli:"region"`
15+
Local bool `cli:"local"`
16+
Limit int `cli:"limit"`
17+
}
18+
19+
func (i *ObjectListInput) Validate(interactive bool) error {
20+
if i.Region == "" {
21+
return fmt.Errorf("--region is required")
22+
}
23+
if i.Limit <= 0 {
24+
return fmt.Errorf("--limit must be greater than 0")
25+
}
26+
return nil
27+
}
28+
29+
var objectListCmd = &cobra.Command{
30+
Use: "list",
31+
Short: "List objects in storage",
32+
Long: `List objects in object storage for a specific region.
33+
34+
Displays object keys, content types, sizes, and last modified timestamps.
35+
36+
In local development mode (--local flag or RENDER_USE_LOCAL_DEV=true),
37+
lists files from the .render/objects/ directory.
38+
39+
Examples:
40+
render ea objects list --region=oregon
41+
render ea objects list --region=oregon --limit=50
42+
render ea objects list --region=oregon --local
43+
render ea objects list --region=oregon -o json
44+
render ea objects list --region=oregon -o text
45+
`,
46+
RunE: func(cmd *cobra.Command, args []string) error {
47+
var input ObjectListInput
48+
49+
if err := command.ParseCommand(cmd, args, &input); err != nil {
50+
return fmt.Errorf("failed to parse input: %w", err)
51+
}
52+
53+
if nonInteractive, err := command.NonInteractive(cmd, func() ([]storage.ObjectInfo, error) {
54+
return listObjects(cmd, input)
55+
}, text.ObjectTable); err != nil {
56+
return err
57+
} else if nonInteractive {
58+
return nil
59+
}
60+
61+
// For now, object commands only support non-interactive mode
62+
// Interactive mode (TUI) can be added in the future
63+
result, err := listObjects(cmd, input)
64+
if err != nil {
65+
return err
66+
}
67+
fmt.Print(text.ObjectTable(result))
68+
return nil
69+
},
70+
}
71+
72+
func listObjects(cmd *cobra.Command, input ObjectListInput) ([]storage.ObjectInfo, error) {
73+
ctx := cmd.Context()
74+
75+
cfg := storage.ServiceConfig{
76+
Local: input.Local,
77+
Region: input.Region,
78+
}
79+
80+
svc, err := storage.NewServiceFromContext(ctx, cfg)
81+
if err != nil {
82+
return nil, fmt.Errorf("failed to create storage service: %w", err)
83+
}
84+
85+
result, err := svc.List(ctx, "", input.Limit)
86+
if err != nil {
87+
return nil, fmt.Errorf("failed to list objects: %w", err)
88+
}
89+
90+
return result.Objects, nil
91+
}
92+
93+
func init() {
94+
objectListCmd.Flags().Int("limit", 100, "Maximum number of objects to return")
95+
96+
objectCmd.AddCommand(objectListCmd)
97+
}

‎pkg/storage/local.go‎

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,6 @@ func (s *LocalService) Upload(ctx context.Context, key, filePath string) (*Uploa
8080
func (s *LocalService) Download(ctx context.Context, key string, dest io.Writer) (*DownloadResult, error) {
8181
objectPath := s.objectPath(key)
8282

83-
// Check if object exists
8483
if _, err := os.Stat(objectPath); err != nil {
8584
if os.IsNotExist(err) {
8685
return nil, fmt.Errorf("object not found: %s", key)
@@ -177,6 +176,71 @@ func (s *LocalService) cleanupEmptyParents(path string) {
177176
}
178177
}
179178

179+
// List lists objects in local storage with pagination support
180+
func (s *LocalService) List(ctx context.Context, cursor string, limit int) (*ListResult, error) {
181+
bucketPath := filepath.Join(s.basePath, s.region, s.bucketName)
182+
183+
if _, err := os.Stat(bucketPath); os.IsNotExist(err) {
184+
return &ListResult{Objects: []ObjectInfo{}}, nil
185+
}
186+
187+
entries, err := os.ReadDir(bucketPath)
188+
if err != nil {
189+
return nil, fmt.Errorf("failed to read directory: %w", err)
190+
}
191+
192+
var objects []ObjectInfo
193+
startIndex := 0
194+
195+
if cursor != "" {
196+
encodedCursor := url.PathEscape(cursor)
197+
for i, entry := range entries {
198+
if entry.Name() == encodedCursor {
199+
startIndex = i + 1
200+
break
201+
}
202+
}
203+
}
204+
205+
lastIndex := startIndex
206+
for i := startIndex; i < len(entries) && len(objects) < limit; i++ {
207+
entry := entries[i]
208+
// Skip directories - users may have manually created them in the local storage path
209+
if entry.IsDir() {
210+
continue
211+
}
212+
213+
info, err := entry.Info()
214+
if err != nil {
215+
continue
216+
}
217+
218+
decodedKey, err := url.PathUnescape(entry.Name())
219+
if err != nil {
220+
decodedKey = entry.Name()
221+
}
222+
223+
objects = append(objects, ObjectInfo{
224+
Key: decodedKey,
225+
ContentType: "application/octet-stream",
226+
SizeBytes: info.Size(),
227+
LastModified: info.ModTime(),
228+
})
229+
lastIndex = i + 1
230+
}
231+
232+
// Determine next cursor - only set if there are more entries to process
233+
nextCursor := ""
234+
if len(objects) > 0 && lastIndex < len(entries) {
235+
nextCursor = objects[len(objects)-1].Key
236+
}
237+
238+
return &ListResult{
239+
Objects: objects,
240+
Cursor: nextCursor,
241+
}, nil
242+
}
243+
180244
// IsLocalMode checks if local development mode is enabled
181245
func IsLocalMode() bool {
182246
return os.Getenv("RENDER_USE_LOCAL_DEV") == "true"

‎pkg/storage/local_test.go‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package storage
22

33
import (
44
"context"
5+
"fmt"
56
"net/url"
67
"os"
78
"path/filepath"
@@ -330,3 +331,158 @@ func (b *bufferWriter) Write(p []byte) (n int, err error) {
330331
*b.buf = append(*b.buf, p...)
331332
return len(p), nil
332333
}
334+
335+
func TestLocalService_List_Empty(t *testing.T) {
336+
basePath := t.TempDir()
337+
svc := NewLocalService(basePath, "oregon", "usr-123")
338+
339+
result, err := svc.List(context.Background(), "", 10)
340+
require.NoError(t, err)
341+
require.Empty(t, result.Objects)
342+
require.Empty(t, result.Cursor)
343+
}
344+
345+
func TestLocalService_List_SingleFile(t *testing.T) {
346+
basePath := t.TempDir()
347+
svc := NewLocalService(basePath, "oregon", "usr-123")
348+
349+
// Upload a file
350+
key := "test/file.txt"
351+
srcFile := filepath.Join(t.TempDir(), "source.txt")
352+
require.NoError(t, os.WriteFile(srcFile, []byte("content"), 0644))
353+
_, err := svc.Upload(context.Background(), key, srcFile)
354+
require.NoError(t, err)
355+
356+
// List
357+
result, err := svc.List(context.Background(), "", 10)
358+
require.NoError(t, err)
359+
require.Len(t, result.Objects, 1)
360+
require.Equal(t, key, result.Objects[0].Key)
361+
require.Equal(t, int64(7), result.Objects[0].SizeBytes)
362+
require.Empty(t, result.Cursor) // No more items
363+
}
364+
365+
func TestLocalService_List_MultipleFiles(t *testing.T) {
366+
basePath := t.TempDir()
367+
svc := NewLocalService(basePath, "oregon", "usr-123")
368+
369+
// Upload multiple files
370+
keys := []string{"a.txt", "b.txt", "c.txt"}
371+
for _, key := range keys {
372+
srcFile := filepath.Join(t.TempDir(), "source.txt")
373+
require.NoError(t, os.WriteFile(srcFile, []byte("content"), 0644))
374+
_, err := svc.Upload(context.Background(), key, srcFile)
375+
require.NoError(t, err)
376+
}
377+
378+
// List all
379+
result, err := svc.List(context.Background(), "", 10)
380+
require.NoError(t, err)
381+
require.Len(t, result.Objects, len(keys))
382+
require.Empty(t, result.Cursor) // No more items
383+
384+
// Verify all keys are returned (order depends on filesystem)
385+
returnedKeys := extractKeys(result.Objects)
386+
require.ElementsMatch(t, keys, returnedKeys)
387+
}
388+
389+
// extractKeys returns the keys from a slice of ObjectInfo
390+
func extractKeys(objects []ObjectInfo) []string {
391+
keys := make([]string, len(objects))
392+
for i, obj := range objects {
393+
keys[i] = obj.Key
394+
}
395+
return keys
396+
}
397+
398+
func TestLocalService_List_Pagination(t *testing.T) {
399+
basePath := t.TempDir()
400+
svc := NewLocalService(basePath, "oregon", "usr-123")
401+
402+
// Upload 5 files
403+
for i := 0; i < 5; i++ {
404+
key := fmt.Sprintf("file%d.txt", i)
405+
srcFile := filepath.Join(t.TempDir(), "source.txt")
406+
require.NoError(t, os.WriteFile(srcFile, []byte("content"), 0644))
407+
_, err := svc.Upload(context.Background(), key, srcFile)
408+
require.NoError(t, err)
409+
}
410+
411+
// List with limit 2
412+
result, err := svc.List(context.Background(), "", 2)
413+
require.NoError(t, err)
414+
require.Len(t, result.Objects, 2)
415+
require.NotEmpty(t, result.Cursor) // More items available
416+
417+
// List next page using cursor
418+
result2, err := svc.List(context.Background(), result.Cursor, 2)
419+
require.NoError(t, err)
420+
require.Len(t, result2.Objects, 2)
421+
require.NotEmpty(t, result2.Cursor) // Still more items
422+
423+
// List final page
424+
result3, err := svc.List(context.Background(), result2.Cursor, 2)
425+
require.NoError(t, err)
426+
require.Len(t, result3.Objects, 1) // Only 1 left
427+
require.Empty(t, result3.Cursor) // No more items
428+
429+
// Verify all unique keys returned
430+
allKeys := make(map[string]bool)
431+
for _, b := range result.Objects {
432+
allKeys[b.Key] = true
433+
}
434+
for _, b := range result2.Objects {
435+
allKeys[b.Key] = true
436+
}
437+
for _, b := range result3.Objects {
438+
allKeys[b.Key] = true
439+
}
440+
require.Len(t, allKeys, 5)
441+
}
442+
443+
func TestLocalService_List_PaginationEvenlyDivides(t *testing.T) {
444+
basePath := t.TempDir()
445+
svc := NewLocalService(basePath, "oregon", "usr-123")
446+
447+
// Upload exactly 4 files
448+
for i := 0; i < 4; i++ {
449+
key := fmt.Sprintf("file%d.txt", i)
450+
srcFile := filepath.Join(t.TempDir(), "source.txt")
451+
require.NoError(t, os.WriteFile(srcFile, []byte("content"), 0644))
452+
_, err := svc.Upload(context.Background(), key, srcFile)
453+
require.NoError(t, err)
454+
}
455+
456+
// List with limit 2 (exactly divides total)
457+
result, err := svc.List(context.Background(), "", 2)
458+
require.NoError(t, err)
459+
require.Len(t, result.Objects, 2)
460+
require.NotEmpty(t, result.Cursor)
461+
462+
// Second page - should return remaining 2 with no cursor
463+
result2, err := svc.List(context.Background(), result.Cursor, 2)
464+
require.NoError(t, err)
465+
require.Len(t, result2.Objects, 2)
466+
require.Empty(t, result2.Cursor) // No more items
467+
}
468+
469+
func TestLocalService_List_LimitEqualsTotal(t *testing.T) {
470+
basePath := t.TempDir()
471+
svc := NewLocalService(basePath, "oregon", "usr-123")
472+
473+
// Upload exactly 3 files
474+
numFiles := 3
475+
for i := 0; i < numFiles; i++ {
476+
key := fmt.Sprintf("file%d.txt", i)
477+
srcFile := filepath.Join(t.TempDir(), "source.txt")
478+
require.NoError(t, os.WriteFile(srcFile, []byte("content"), 0644))
479+
_, err := svc.Upload(context.Background(), key, srcFile)
480+
require.NoError(t, err)
481+
}
482+
483+
// List with limit = exact number of files
484+
result, err := svc.List(context.Background(), "", numFiles)
485+
require.NoError(t, err)
486+
require.Len(t, result.Objects, numFiles)
487+
require.Empty(t, result.Cursor) // No cursor when limit equals total
488+
}

‎pkg/storage/model.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,21 @@
11
package storage
22

3+
import "time"
4+
5+
// ObjectInfo represents metadata about a single object
6+
type ObjectInfo struct {
7+
Key string `json:"key"`
8+
ContentType string `json:"contentType"`
9+
SizeBytes int64 `json:"sizeBytes"`
10+
LastModified time.Time `json:"lastModified"`
11+
}
12+
13+
// ListResult represents the result of listing objects
14+
type ListResult struct {
15+
Objects []ObjectInfo `json:"objects"`
16+
Cursor string `json:"cursor,omitempty"`
17+
}
18+
319
// UploadResult represents the result of a successful object upload
420
type UploadResult struct {
521
Key string `json:"key"`

0 commit comments

Comments
 (0)