Skip to content

Commit 9bdae2e

Browse files
somanshreddyclaude
somanshreddy
and
claude
committed
client: remove pagination hard limit, export APIDataField
- Remove 10K hard limit and ErrPaginationTruncated — --all fetches everything, fails loudly on OOM rather than silently truncating - Export APIDataField constant so builder uses it instead of raw "data" - Update --all help text with guidance for large datasets (>10K items) - Remove truncation tests and helpers Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 7f8ad88 commit 9bdae2e

4 files changed

Lines changed: 9 additions & 230 deletions

File tree

‎cmd/heygen/builder.go‎

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package main
22

33
import (
44
"encoding/json"
5-
"errors"
65
"fmt"
76
"io"
87
"os"
@@ -58,15 +57,6 @@ func buildCobraCommand(spec *command.Spec, ctx *cmdContext) *cobra.Command {
5857
cols := defaultColumnsForSpec(spec)
5958
result, err := ctx.client.ExecuteAll(spec, inv)
6059
if err != nil {
61-
var truncErr *client.ErrPaginationTruncated
62-
if errors.As(err, &truncErr) {
63-
// Output partial data, then signal truncation via CLIError.
64-
// The error goes through formatter.Error() — no raw stderr writes.
65-
if fmtErr := ctx.formatter.Data(truncErr.Data, "", cols); fmtErr != nil {
66-
return fmtErr
67-
}
68-
return clierrors.New(truncErr.Error())
69-
}
7060
return err
7161
}
7262

@@ -79,7 +69,7 @@ func buildCobraCommand(spec *command.Spec, ctx *cmdContext) *cobra.Command {
7969
return err
8070
}
8171

82-
return ctx.formatter.Data(result, "data", defaultColumnsForSpec(spec))
72+
return ctx.formatter.Data(result, client.APIDataField, defaultColumnsForSpec(spec))
8373
},
8474
}
8575

@@ -89,7 +79,7 @@ func buildCobraCommand(spec *command.Spec, ctx *cmdContext) *cobra.Command {
8979
}
9080

9181
if spec.Paginated {
92-
cmd.Flags().Bool("all", false, "Fetch all pages (returns flat JSON array instead of API envelope)")
82+
cmd.Flags().Bool("all", false, "Fetch all pages into a single JSON array. For datasets over 10,000 items, consider paginating manually with --token.")
9383
}
9484

9585
// Add -d/--data for commands with JSON request bodies

‎cmd/heygen/builder_test.go‎

Lines changed: 0 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import (
44
"bytes"
55
"encoding/json"
66
"errors"
7-
"fmt"
87
"io"
98
"net/http"
109
"os"
@@ -204,60 +203,6 @@ func TestGenBuilder_VideoList_NoAllFlag_NonPaginated(t *testing.T) {
204203
}
205204
}
206205

207-
func TestGenBuilder_VideoList_AllPages_Truncated(t *testing.T) {
208-
var calls int
209-
srv := setupTestServer(t, map[string]testHandler{
210-
"GET /v3/videos": {
211-
StatusCode: 200,
212-
ValidateRequest: func(t *testing.T, r *http.Request) {
213-
t.Helper()
214-
calls++
215-
},
216-
Body: truncatedPageBody(0, 2500, "cursor_1"),
217-
},
218-
})
219-
defer srv.Close()
220-
221-
originalHandler := srv.Config.Handler
222-
srv.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
223-
token := r.URL.Query().Get("token")
224-
switch token {
225-
case "":
226-
originalHandler.ServeHTTP(w, r)
227-
case "cursor_1":
228-
calls++
229-
w.WriteHeader(http.StatusOK)
230-
_, _ = w.Write([]byte(truncatedPageBody(2500, 2500, "cursor_2")))
231-
case "cursor_2":
232-
calls++
233-
w.WriteHeader(http.StatusOK)
234-
_, _ = w.Write([]byte(truncatedPageBody(5000, 2500, "cursor_3")))
235-
case "cursor_3":
236-
calls++
237-
w.WriteHeader(http.StatusOK)
238-
_, _ = w.Write([]byte(truncatedPageBody(7500, 2500, "cursor_4")))
239-
default:
240-
t.Fatalf("unexpected token %q", token)
241-
}
242-
})
243-
244-
res := runGenCommand(t, srv.URL, "test-key", videoListSpec, "list", "--all")
245-
246-
if res.ExitCode != 1 {
247-
t.Fatalf("ExitCode = %d, want 1\nstderr: %s", res.ExitCode, res.Stderr)
248-
}
249-
if !strings.Contains(res.Stderr, "pagination stopped at 10000 items") {
250-
t.Fatalf("stderr = %s, want truncation message", res.Stderr)
251-
}
252-
253-
var parsed []map[string]any
254-
if err := json.Unmarshal([]byte(res.Stdout), &parsed); err != nil {
255-
t.Fatalf("stdout is not valid JSON array: %v\nstdout: %s", err, res.Stdout)
256-
}
257-
if len(parsed) != 10000 {
258-
t.Fatalf("len(parsed) = %d, want 10000", len(parsed))
259-
}
260-
}
261206

262207
func TestGenBuilder_PostWithBodyFlags(t *testing.T) {
263208
var gotBody map[string]any
@@ -604,19 +549,3 @@ func runGeneratedRootCommand(t *testing.T, serverURL, apiKey string, groups map[
604549
}
605550
}
606551

607-
func truncatedPageBody(start, count int, nextToken string) string {
608-
items := make([]map[string]any, 0, count)
609-
for i := 0; i < count; i++ {
610-
items = append(items, map[string]any{"id": fmt.Sprintf("v%d", start+i)})
611-
}
612-
body := map[string]any{
613-
"data": items,
614-
}
615-
if nextToken == "" {
616-
body["next_token"] = nil
617-
} else {
618-
body["next_token"] = nextToken
619-
}
620-
raw, _ := json.Marshal(body)
621-
return string(raw)
622-
}

‎internal/client/executor.go‎

Lines changed: 7 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -17,25 +17,15 @@ import (
1717
)
1818

1919
const (
20-
paginationHardLimit = 10000
20+
// APIDataField is the response envelope field containing the payload.
21+
// Exported so the builder can pass it to the formatter for --human rendering.
22+
APIDataField = "data"
2123

22-
// HeyGen API conventions — consistent across all v3 endpoints.
23-
// Hardcoded rather than per-Spec because no endpoint deviates.
24-
apiDataField = "data" // response envelope field containing the payload
25-
apiCursorField = "next_token" // response field with the next page cursor
26-
apiCursorParam = "token" // request query parameter for the cursor
24+
// HeyGen API pagination conventions — consistent across all v3 endpoints.
25+
apiCursorField = "next_token" // response field with the next page cursor
26+
apiCursorParam = "token" // request query parameter for the cursor
2727
)
2828

29-
// ErrPaginationTruncated is returned when ExecuteAll stops early at the hard
30-
// item limit. It carries the partial data so callers can still render it.
31-
type ErrPaginationTruncated struct {
32-
Data json.RawMessage
33-
Count int
34-
}
35-
36-
func (e *ErrPaginationTruncated) Error() string {
37-
return fmt.Sprintf("pagination stopped at %d items (hard limit); results may be incomplete", e.Count)
38-
}
3929

4030
// Execute sends an HTTP request described by the Spec (static metadata)
4131
// and Invocation (resolved user values). Returns the raw JSON response.
@@ -104,35 +94,16 @@ func (c *Client) ExecuteAll(spec *command.Spec, inv *command.Invocation) (json.R
10494
return nil, err
10595
}
10696

107-
items, nextToken, err := extractPage(page, apiDataField, apiCursorField)
97+
items, nextToken, err := extractPage(page, APIDataField, apiCursorField)
10898
if err != nil {
10999
return nil, err
110100
}
111101

112-
remaining := paginationHardLimit - len(accumulated)
113-
if remaining <= 0 {
114-
data, err := marshalItems(accumulated)
115-
if err != nil {
116-
return nil, err
117-
}
118-
return nil, &ErrPaginationTruncated{Data: data, Count: len(accumulated)}
119-
}
120-
121-
if len(items) > remaining {
122-
items = items[:remaining]
123-
}
124102
accumulated = append(accumulated, items...)
125103

126104
if nextToken == "" {
127105
return marshalItems(accumulated)
128106
}
129-
if len(accumulated) >= paginationHardLimit {
130-
data, err := marshalItems(accumulated)
131-
if err != nil {
132-
return nil, err
133-
}
134-
return nil, &ErrPaginationTruncated{Data: data, Count: len(accumulated)}
135-
}
136107

137108
workingInv.QueryParams.Set(apiCursorParam, nextToken)
138109
}

‎internal/client/executor_test.go‎

Lines changed: 0 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package client
33
import (
44
"encoding/json"
55
"errors"
6-
"fmt"
76
"io"
87
"net/http"
98
"net/http/httptest"
@@ -471,98 +470,6 @@ func TestExecuteAll_MissingDataField(t *testing.T) {
471470
}
472471
}
473472

474-
func TestExecuteAll_Truncated(t *testing.T) {
475-
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
476-
token := r.URL.Query().Get("token")
477-
switch token {
478-
case "":
479-
w.WriteHeader(http.StatusOK)
480-
_, _ = w.Write([]byte(paginationPageBody(0, 2500, "cursor_1")))
481-
case "cursor_1":
482-
w.WriteHeader(http.StatusOK)
483-
_, _ = w.Write([]byte(paginationPageBody(2500, 2500, "cursor_2")))
484-
case "cursor_2":
485-
w.WriteHeader(http.StatusOK)
486-
_, _ = w.Write([]byte(paginationPageBody(5000, 2500, "cursor_3")))
487-
case "cursor_3":
488-
w.WriteHeader(http.StatusOK)
489-
_, _ = w.Write([]byte(paginationPageBody(7500, 2500, "cursor_4")))
490-
default:
491-
t.Fatalf("unexpected token %q", token)
492-
}
493-
}))
494-
defer srv.Close()
495-
496-
c := New("key", WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
497-
spec := &command.Spec{
498-
Endpoint: "/v3/videos",
499-
Method: "GET",
500-
Paginated: true,
501-
}
502-
inv := &command.Invocation{PathParams: make(map[string]string), QueryParams: make(url.Values)}
503-
504-
_, err := c.ExecuteAll(spec, inv)
505-
var truncErr *ErrPaginationTruncated
506-
if !errors.As(err, &truncErr) {
507-
t.Fatalf("err = %T, want *ErrPaginationTruncated", err)
508-
}
509-
if truncErr.Count != 10000 {
510-
t.Fatalf("Count = %d, want 10000", truncErr.Count)
511-
}
512-
var parsed []map[string]any
513-
if err := json.Unmarshal(truncErr.Data, &parsed); err != nil {
514-
t.Fatalf("truncErr.Data is not valid JSON array: %v", err)
515-
}
516-
if len(parsed) != 10000 {
517-
t.Fatalf("len(parsed) = %d, want 10000", len(parsed))
518-
}
519-
}
520-
521-
func TestExecuteAll_ExactlyAtLimit(t *testing.T) {
522-
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
523-
token := r.URL.Query().Get("token")
524-
switch token {
525-
case "":
526-
w.WriteHeader(http.StatusOK)
527-
_, _ = w.Write([]byte(paginationPageBody(0, 2000, "cursor_1")))
528-
case "cursor_1":
529-
w.WriteHeader(http.StatusOK)
530-
_, _ = w.Write([]byte(paginationPageBody(2000, 2000, "cursor_2")))
531-
case "cursor_2":
532-
w.WriteHeader(http.StatusOK)
533-
_, _ = w.Write([]byte(paginationPageBody(4000, 2000, "cursor_3")))
534-
case "cursor_3":
535-
w.WriteHeader(http.StatusOK)
536-
_, _ = w.Write([]byte(paginationPageBody(6000, 2000, "cursor_4")))
537-
case "cursor_4":
538-
w.WriteHeader(http.StatusOK)
539-
_, _ = w.Write([]byte(paginationPageBody(8000, 2000, "")))
540-
default:
541-
t.Fatalf("unexpected token %q", token)
542-
}
543-
}))
544-
defer srv.Close()
545-
546-
c := New("key", WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
547-
spec := &command.Spec{
548-
Endpoint: "/v3/videos",
549-
Method: "GET",
550-
Paginated: true,
551-
}
552-
inv := &command.Invocation{PathParams: make(map[string]string), QueryParams: make(url.Values)}
553-
554-
result, err := c.ExecuteAll(spec, inv)
555-
if err != nil {
556-
t.Fatalf("unexpected error: %v", err)
557-
}
558-
var parsed []map[string]any
559-
if err := json.Unmarshal(result, &parsed); err != nil {
560-
t.Fatalf("result is not valid JSON array: %v", err)
561-
}
562-
if len(parsed) != 10000 {
563-
t.Fatalf("len(parsed) = %d, want 10000", len(parsed))
564-
}
565-
}
566473

567474
func TestExecute_MultipartUpload(t *testing.T) {
568475
var gotContentType string
@@ -631,24 +538,6 @@ func TestExecute_MultipartUpload(t *testing.T) {
631538
}
632539
}
633540

634-
func paginationPageBody(start, count int, nextToken string) string {
635-
items := make([]map[string]any, 0, count)
636-
for i := 0; i < count; i++ {
637-
items = append(items, map[string]any{"id": fmt.Sprintf("v%d", start+i)})
638-
}
639-
640-
body := map[string]any{
641-
"data": items,
642-
}
643-
if nextToken == "" {
644-
body["next_token"] = nil
645-
} else {
646-
body["next_token"] = nextToken
647-
}
648-
raw, _ := json.Marshal(body)
649-
return string(raw)
650-
}
651-
652541
func TestExecute_MultipartMissingFilePath(t *testing.T) {
653542
c := New("key")
654543

0 commit comments

Comments
 (0)