client: add pagination with --all flag (PRINFRA-125)

Author: somanshreddy

cmd/heygen/builder.go

@@ -2,12 +2,14 @@ package main
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"os"
 	"strconv"
 	"strings"
 
+	"github.com/heygen-com/heygen-cli/internal/client"
 	"github.com/heygen-com/heygen-cli/internal/command"
 	clierrors "github.com/heygen-com/heygen-cli/internal/errors"
 	"github.com/spf13/cobra"
@@ -41,6 +43,33 @@ func buildCobraCommand(spec *command.Spec, ctx *cmdContext) *cobra.Command {
 				return err
 			}
 
+			if spec.Paginated {
+				allPages, _ := cmd.Flags().GetBool("all")
+				cursorFlagName := cursorFlagForSpec(spec)
+				cursorSet := cursorFlagName != "" && cmd.Flags().Changed(cursorFlagName)
+
+				if allPages && cursorSet {
+					return clierrors.NewUsage(fmt.Sprintf("--all and --%s are mutually exclusive", cursorFlagName))
+				}
+
+				if allPages {
+					result, err := ctx.client.ExecuteAll(spec, inv)
+					if err != nil {
+						var truncErr *client.ErrPaginationTruncated
+						if errors.As(err, &truncErr) {
+							fmt.Fprintf(cmd.ErrOrStderr(), "Warning: %s\n", truncErr.Error())
+							if fmtErr := ctx.formatter.Data(truncErr.Data); fmtErr != nil {
+								return fmtErr
+							}
+							return clierrors.New(truncErr.Error())
+						}
+						return err
+					}
+
+					return ctx.formatter.Data(result)
+				}
+			}
+
 			result, err := ctx.client.Execute(spec, inv)
 			if err != nil {
 				return err
@@ -55,6 +84,10 @@ func buildCobraCommand(spec *command.Spec, ctx *cmdContext) *cobra.Command {
 		registerFlag(cmd, flag)
 	}
 
+	if spec.Paginated {
+		cmd.Flags().Bool("all", false, "Fetch all pages (returns flat JSON array instead of API envelope)")
+	}
+
 	// Add -d/--data for commands with JSON request bodies
 	if spec.BodyEncoding == "json" {
 		cmd.Flags().StringVarP(&rawData, "data", "d", "",
@@ -86,6 +119,15 @@ func buildUseLine(spec *command.Spec) string {
 	return strings.Join(parts, " ")
 }
 
+func cursorFlagForSpec(spec *command.Spec) string {
+	for _, flag := range spec.Flags {
+		if flag.JSONName == spec.TokenParam {
+			return flag.Name
+		}
+	}
+	return ""
+}
+
 // registerFlag adds a typed flag to the Cobra command based on the FlagSpec.
 func registerFlag(cmd *cobra.Command, flag command.FlagSpec) {
 	helpText := flag.Help

cmd/heygen/builder_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"net/http"
 	"os"
@@ -17,13 +18,14 @@ import (
 // videoListSpec mirrors the hand-written video list command as a Spec,
 // proving the generic builder produces identical behavior.
 var videoListSpec = &command.Spec{
-	Group:    "video",
-	Name:     "list",
-	Summary:  "List videos",
-	Endpoint: "/v3/videos",
-	Method:   "GET",
-	// TokenField/DataField for pagination (used by paginator, not tested here)
+	Group:      "video",
+	Name:       "list",
+	Summary:    "List videos",
+	Endpoint:   "/v3/videos",
+	Method:     "GET",
+	Paginated:  true,
 	TokenField: "next_token",
+	TokenParam: "token",
 	DataField:  "data",
 	Flags: []command.FlagSpec{
 		{Name: "limit", Type: "int", Source: "query", JSONName: "limit"},
@@ -92,6 +94,174 @@ func TestGenBuilder_VideoList_Flags(t *testing.T) {
 	}
 }
 
+func TestGenBuilder_VideoList_AllPages(t *testing.T) {
+	var calls int
+	srv := setupTestServer(t, map[string]testHandler{
+		"GET /v3/videos": {
+			StatusCode: 200,
+			ValidateRequest: func(t *testing.T, r *http.Request) {
+				t.Helper()
+				calls++
+				switch calls {
+				case 1:
+					if got := r.URL.Query().Get("token"); got != "" {
+						t.Fatalf("first page token = %q, want empty", got)
+					}
+				case 2:
+					if got := r.URL.Query().Get("token"); got != "cursor_2" {
+						t.Fatalf("second page token = %q, want %q", got, "cursor_2")
+					}
+				default:
+					t.Fatalf("unexpected request count %d", calls)
+				}
+			},
+			Body: `{"data":[{"id":"v1"},{"id":"v2"}],"next_token":"cursor_2"}`,
+		},
+	})
+	defer srv.Close()
+
+	originalHandler := srv.Config.Handler
+	srv.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Query().Get("token") == "cursor_2" {
+			w.WriteHeader(http.StatusOK)
+			_, _ = w.Write([]byte(`{"data":[{"id":"v3"}],"next_token":null}`))
+			return
+		}
+		originalHandler.ServeHTTP(w, r)
+	})
+
+	res := runGenCommand(t, srv.URL, "test-key", videoListSpec, "list", "--all")
+
+	if res.ExitCode != 0 {
+		t.Fatalf("ExitCode = %d, want 0\nstderr: %s", res.ExitCode, res.Stderr)
+	}
+	var parsed []map[string]any
+	if err := json.Unmarshal([]byte(res.Stdout), &parsed); err != nil {
+		t.Fatalf("stdout is not valid JSON array: %v\nstdout: %s", err, res.Stdout)
+	}
+	if len(parsed) != 3 {
+		t.Fatalf("len(parsed) = %d, want 3", len(parsed))
+	}
+}
+
+func TestGenBuilder_VideoList_AllPages_SinglePage(t *testing.T) {
+	srv := setupTestServer(t, map[string]testHandler{
+		"GET /v3/videos": {
+			StatusCode: 200,
+			Body:       `{"data":[{"id":"v1"}],"next_token":null}`,
+		},
+	})
+	defer srv.Close()
+
+	res := runGenCommand(t, srv.URL, "test-key", videoListSpec, "list", "--all")
+
+	if res.ExitCode != 0 {
+		t.Fatalf("ExitCode = %d, want 0\nstderr: %s", res.ExitCode, res.Stderr)
+	}
+	var parsed []map[string]any
+	if err := json.Unmarshal([]byte(res.Stdout), &parsed); err != nil {
+		t.Fatalf("stdout is not valid JSON array: %v\nstdout: %s", err, res.Stdout)
+	}
+	if len(parsed) != 1 {
+		t.Fatalf("len(parsed) = %d, want 1", len(parsed))
+	}
+}
+
+func TestGenBuilder_VideoList_AllAndTokenConflict(t *testing.T) {
+	srv := setupTestServer(t, map[string]testHandler{})
+	defer srv.Close()
+
+	res := runGenCommand(t, srv.URL, "test-key", videoListSpec, "list", "--all", "--token", "cursor_abc")
+
+	if res.ExitCode != 2 {
+		t.Fatalf("ExitCode = %d, want 2\nstderr: %s", res.ExitCode, res.Stderr)
+	}
+	if !strings.Contains(res.Stderr, "--all and --token are mutually exclusive") {
+		t.Fatalf("stderr = %s, want conflict message", res.Stderr)
+	}
+}
+
+func TestGenBuilder_VideoList_NoAllFlag_NonPaginated(t *testing.T) {
+	nonPaginated := &command.Spec{
+		Group:    "video",
+		Name:     "get",
+		Summary:  "Get video",
+		Endpoint: "/v3/videos/{video_id}",
+		Method:   "GET",
+		Args: []command.ArgSpec{
+			{Name: "video-id", Param: "video_id"},
+		},
+		Examples: []string{"heygen video get <video-id>"},
+	}
+
+	srv := setupTestServer(t, map[string]testHandler{})
+	defer srv.Close()
+
+	res := runGenCommand(t, srv.URL, "test-key", nonPaginated, "get", "vid_123", "--all")
+
+	if res.ExitCode != 2 {
+		t.Fatalf("ExitCode = %d, want 2\nstderr: %s", res.ExitCode, res.Stderr)
+	}
+	if !strings.Contains(res.Stderr, "unknown flag: --all") {
+		t.Fatalf("stderr = %s, want unknown flag error", res.Stderr)
+	}
+}
+
+func TestGenBuilder_VideoList_AllPages_Truncated(t *testing.T) {
+	var calls int
+	srv := setupTestServer(t, map[string]testHandler{
+		"GET /v3/videos": {
+			StatusCode: 200,
+			ValidateRequest: func(t *testing.T, r *http.Request) {
+				t.Helper()
+				calls++
+			},
+			Body: truncatedPageBody(0, 2500, "cursor_1"),
+		},
+	})
+	defer srv.Close()
+
+	originalHandler := srv.Config.Handler
+	srv.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		token := r.URL.Query().Get("token")
+		switch token {
+		case "":
+			originalHandler.ServeHTTP(w, r)
+		case "cursor_1":
+			calls++
+			w.WriteHeader(http.StatusOK)
+			_, _ = w.Write([]byte(truncatedPageBody(2500, 2500, "cursor_2")))
+		case "cursor_2":
+			calls++
+			w.WriteHeader(http.StatusOK)
+			_, _ = w.Write([]byte(truncatedPageBody(5000, 2500, "cursor_3")))
+		case "cursor_3":
+			calls++
+			w.WriteHeader(http.StatusOK)
+			_, _ = w.Write([]byte(truncatedPageBody(7500, 2500, "cursor_4")))
+		default:
+			t.Fatalf("unexpected token %q", token)
+		}
+	})
+
+	res := runGenCommand(t, srv.URL, "test-key", videoListSpec, "list", "--all")
+
+	if res.ExitCode != 1 {
+		t.Fatalf("ExitCode = %d, want 1\nstderr: %s", res.ExitCode, res.Stderr)
+	}
+	if !strings.Contains(res.Stderr, "Warning: pagination stopped at 10000 items") {
+		t.Fatalf("stderr = %s, want truncation warning", res.Stderr)
+	}
+
+	var parsed []map[string]any
+	if err := json.Unmarshal([]byte(res.Stdout), &parsed); err != nil {
+		t.Fatalf("stdout is not valid JSON array: %v\nstdout: %s", err, res.Stdout)
+	}
+	if len(parsed) != 10000 {
+		t.Fatalf("len(parsed) = %d, want 10000", len(parsed))
+	}
+}
+
 func TestGenBuilder_PostWithBodyFlags(t *testing.T) {
 	var gotBody map[string]any
 
@@ -436,3 +606,20 @@ func runGeneratedRootCommand(t *testing.T, serverURL, apiKey string, groups map[
 		ExitCode: exitCode,
 	}
 }
+
+func truncatedPageBody(start, count int, nextToken string) string {
+	items := make([]map[string]any, 0, count)
+	for i := 0; i < count; i++ {
+		items = append(items, map[string]any{"id": fmt.Sprintf("v%d", start+i)})
+	}
+	body := map[string]any{
+		"data": items,
+	}
+	if nextToken == "" {
+		body["next_token"] = nil
+	} else {
+		body["next_token"] = nextToken
+	}
+	raw, _ := json.Marshal(body)
+	return string(raw)
+}

codegen/grouper.go

@@ -53,6 +53,7 @@ import (
 //	  → sessions → sub-group, {session_id} → arg, stop → sub-group
 //	  → x-cli-action: true → no terminal verb appended
 //	  → result: heygen video-agent sessions stop <session-id>
+//
 // GroupDescriptions maps group name → description from the OpenAPI tag.
 // Used by the builder for group command help text.
 type GroupDescriptions map[string]string
@@ -157,7 +158,7 @@ func buildSpec(
 	}
 
 	// Pagination
-	_, spec.TokenField, spec.DataField = detectPagination(op)
+	spec.Paginated, spec.TokenField, spec.TokenParam, spec.DataField = detectPagination(op, pathItem)
 
 	// Flags from query params
 	for _, paramRef := range collectParams(pathItem, op) {
@@ -439,7 +440,7 @@ func isComplexField(s *openapi3.Schema) bool {
 
 // --- Response analysis ---
 
-func detectPagination(op *openapi3.Operation) (hasMore bool, tokenField, dataField string) {
+func detectPagination(op *openapi3.Operation, pathItem *openapi3.PathItem) (paginated bool, tokenField, tokenParam, dataField string) {
 	respSchema := successResponseSchema(op)
 	if respSchema == nil {
 		return
@@ -455,11 +456,39 @@ func detectPagination(op *openapi3.Operation) (hasMore bool, tokenField, dataFie
 	for _, schema := range schemasToCheck {
 		for _, candidate := range []string{"next_token", "token", "cursor"} {
 			if _, ok := schema.Properties[candidate]; ok {
-				return true, candidate, dataField
+				tokenField = candidate
+				break
 			}
 		}
+		if tokenField != "" {
+			break
+		}
+	}
+	if tokenField == "" {
+		return false, "", "", dataField
+	}
+
+	tokenParam = detectCursorParam(pathItem, op)
+	if tokenParam == "" {
+		return false, tokenField, "", dataField
 	}
-	return
+
+	return true, tokenField, tokenParam, dataField
+}
+
+func detectCursorParam(pathItem *openapi3.PathItem, op *openapi3.Operation) string {
+	params := collectParams(pathItem, op)
+	for _, paramRef := range params {
+		param := paramRef.Value
+		if param == nil || param.In != "query" {
+			continue
+		}
+		switch param.Name {
+		case "token", "cursor", "page_token":
+			return param.Name
+		}
+	}
+	return ""
 }
 
 func successResponseSchema(op *openapi3.Operation) *openapi3.Schema {

codegen/grouper_test.go

@@ -156,9 +156,15 @@ func TestGroupEndpoints_Pagination(t *testing.T) {
 		if s.Name != "list" {
 			continue
 		}
+		if !s.Paginated {
+			t.Error("Paginated = false, want true")
+		}
 		if s.TokenField != "token" {
 			t.Errorf("TokenField = %q, want 'token'", s.TokenField)
 		}
+		if s.TokenParam != "token" {
+			t.Errorf("TokenParam = %q, want 'token'", s.TokenParam)
+		}
 		if s.DataField != "data" {
 			t.Errorf("DataField = %q, want 'data'", s.DataField)
 		}