client: keep pagination metadata without --all (#45)

## Description

Replaces PR #31 (which added `--all` auto-pagination) with a minimal version that keeps only what downstream PRs depend on. James flagged `--all` as an anti-pattern ripe for abuse — agents that need multiple pages should paginate explicitly with `--token`.

**What changed:**

The `Spec` struct previously carried `TokenField` and `DataField` — per-spec strings that named the cursor field and data envelope field. In practice, every HeyGen v3 endpoint uses the same values (`"next_token"` and `"data"`), so these were redundant. This PR:

1. **Replaces `TokenField`/`DataField` with `Paginated bool`** on `command.Spec`. Codegen sets `Paginated: true` for list endpoints (detected by the presence of a `next_token` response field + `token` query param). The bool is metadata only — it marks which commands support cursor pagination but doesn't drive any runtime behavior yet.

2. **Exports `client.APIDataField` constant** (`"data"`) — a single source of truth for the response envelope field name. The builder passes this to `formatter.Data()` for `--human` table rendering. Downstream PRs (polling, auth status) also use it.

3. **Updates codegen** — `detectPagination()` now returns `bool` instead of `(bool, string, string)`. Checks both response schema (has `next_token`) and request params (has `token` query param) before marking as paginated. Golden files updated.

4. **Documents the pagination contract** in CLAUDE.md and AGENTS.md: manual pagination via `--limit` and `--token`, no `--all`.

**What was removed vs PR #31:** `ExecuteAll()`, `extractPage()`, `marshalItems()`, `cloneInvocation()`, `--all` flag registration, `--all`/`--token` mutual exclusion, and all associated tests.

**What was kept:** `Paginated bool` (used by downstream PRs for flag presence detection), `APIDataField` (used by builder, polling, and auth status), and `detectPagination()` in codegen.

## Testing

- All existing tests pass — `make test` across linux/macos/windows
- `TestGroupEndpoints_Pagination` updated to assert `Paginated: true` instead of `TokenField`/`DataField` strings
- `videoListSpec` in builder tests updated to use `Paginated: true`
- Golden files regenerated and verified
- `./bin/heygen video list --all` → unknown flag error (confirmed removed)
- `./bin/heygen video list --limit 5` → works (manual pagination unchanged)

Author: somanshreddy

AGENTS.md

@@ -57,11 +57,15 @@ Files in `gen/` are deleted and regenerated by `make generate`. Never put hand-w
 ```go
 Data(v json.RawMessage, dataField string, columns []command.Column) error
 ```
-- **Generated commands** pass `spec.DataField` and `defaultColumnsForSpec(spec)`.
+- **Generated commands** pass `client.APIDataField` and `defaultColumnsForSpec(spec)`.
 - **Hand-written commands** (auth, config) pass `"", nil`.
 - `JSONFormatter` ignores `dataField` and `columns` — always renders the full response. No contract change from the agent's perspective.
 - `HumanFormatter` uses `dataField` to unwrap the envelope, then renders a table (array) or key-value (object).
 
+### Pagination
+- List commands paginate manually via API query flags like `--limit` and `--token`.
+- Do not add a generic `--all` mode. If an agent needs multiple pages, it should read `next_token` from the JSON response and request the next page explicitly.
+
 ### Auth and `skipAuth`
 `initContext()` resolves credentials via `ChainCredentialResolver` (env → file). Commands that don't need auth (e.g., `auth login`, `config set`) annotate with `skipAuth: true` in Cobra `Annotations`. Check `skipAuth` before adding new non-auth commands.
 

CLAUDE.md

@@ -55,11 +55,15 @@ Files in `gen/` are deleted and regenerated by `make generate`. Never put hand-w
 ```go
 Data(v json.RawMessage, dataField string, columns []command.Column) error
 ```
-- **Generated commands** pass `spec.DataField` and `defaultColumnsForSpec(spec)`.
+- **Generated commands** pass `client.APIDataField` and `defaultColumnsForSpec(spec)`.
 - **Hand-written commands** (auth, config) pass `"", nil`.
 - `JSONFormatter` ignores `dataField` and `columns` — always renders the full response. No contract change from the agent's perspective.
 - `HumanFormatter` uses `dataField` to unwrap the envelope, then renders a table (array) or key-value (object).
 
+### Pagination
+- List commands paginate manually via API query flags like `--limit` and `--token`.
+- Do not add a generic `--all` mode. If an agent needs multiple pages, it should read `next_token` from the JSON response and request the next page explicitly.
+
 ### Auth and `skipAuth`
 `initContext()` resolves credentials via `ChainCredentialResolver` (env → file). Commands that don't need auth (e.g., `auth login`, `config set`) annotate with `skipAuth: true` in Cobra `Annotations`. Check `skipAuth` before adding new non-auth commands.
 

cmd/heygen/builder.go

@@ -8,6 +8,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/heygen-com/heygen-cli/internal/client"
 	"github.com/heygen-com/heygen-cli/internal/command"
 	clierrors "github.com/heygen-com/heygen-cli/internal/errors"
 	"github.com/spf13/cobra"
@@ -46,7 +47,7 @@ func buildCobraCommand(spec *command.Spec, ctx *cmdContext) *cobra.Command {
 				return err
 			}
 
-			return ctx.formatter.Data(result, spec.DataField, defaultColumnsForSpec(spec))
+			return ctx.formatter.Data(result, client.APIDataField, defaultColumnsForSpec(spec))
 		},
 	}
 

cmd/heygen/builder_test.go

@@ -17,14 +17,12 @@ import (
 // videoListSpec mirrors the hand-written video list command as a Spec,
 // proving the generic builder produces identical behavior.
 var videoListSpec = &command.Spec{
-	Group:    "video",
-	Name:     "list",
-	Summary:  "List videos",
-	Endpoint: "/v3/videos",
-	Method:   "GET",
-	// TokenField/DataField for pagination (used by paginator, not tested here)
-	TokenField: "next_token",
-	DataField:  "data",
+	Group:     "video",
+	Name:      "list",
+	Summary:   "List videos",
+	Endpoint:  "/v3/videos",
+	Method:    "GET",
+	Paginated: true,
 	Flags: []command.FlagSpec{
 		{Name: "limit", Type: "int", Source: "query", JSONName: "limit"},
 		{Name: "token", Type: "string", Source: "query", JSONName: "token"},

codegen/grouper.go

@@ -53,6 +53,7 @@ import (
 //	  → sessions → sub-group, {session_id} → arg, stop → sub-group
 //	  → x-cli-action: true → no terminal verb appended
 //	  → result: heygen video-agent sessions stop <session-id>
+//
 // GroupDescriptions maps group name → description from the OpenAPI tag.
 // Used by the builder for group command help text.
 type GroupDescriptions map[string]string
@@ -156,8 +157,9 @@ func buildSpec(
 		spec.BodyEncoding = "multipart"
 	}
 
-	// Pagination
-	_, spec.TokenField, spec.DataField = detectPagination(op)
+	// Pagination — only sets Paginated bool. The cursor field names and data
+	// field are API conventions hardcoded in the client package.
+	spec.Paginated = detectPagination(op, pathItem)
 
 	// Flags from query params
 	for _, paramRef := range collectParams(pathItem, op) {
@@ -439,27 +441,46 @@ func isComplexField(s *openapi3.Schema) bool {
 
 // --- Response analysis ---
 
-func detectPagination(op *openapi3.Operation) (hasMore bool, tokenField, dataField string) {
+// detectPagination returns true if the endpoint supports cursor-based pagination.
+// An endpoint is paginated when both: (1) the response has a cursor field
+// (next_token) and (2) the request has a cursor query param (token).
+func detectPagination(op *openapi3.Operation, pathItem *openapi3.PathItem) bool {
 	respSchema := successResponseSchema(op)
 	if respSchema == nil {
-		return
-	}
-	if dataProp := respSchema.Properties["data"]; dataProp != nil && dataProp.Value != nil {
-		dataField = "data"
+		return false
 	}
-	// Check for token at root level and inside data wrapper
+
+	// Check for cursor field in response (at root or inside data wrapper).
+	hasCursorField := false
 	schemasToCheck := []*openapi3.Schema{respSchema}
-	if dataField != "" {
-		schemasToCheck = append(schemasToCheck, respSchema.Properties["data"].Value)
+	if dataProp := respSchema.Properties["data"]; dataProp != nil && dataProp.Value != nil {
+		schemasToCheck = append(schemasToCheck, dataProp.Value)
 	}
 	for _, schema := range schemasToCheck {
-		for _, candidate := range []string{"next_token", "token", "cursor"} {
-			if _, ok := schema.Properties[candidate]; ok {
-				return true, candidate, dataField
-			}
+		if _, ok := schema.Properties["next_token"]; ok {
+			hasCursorField = true
+			break
 		}
 	}
-	return
+	if !hasCursorField {
+		return false
+	}
+
+	return detectCursorParam(pathItem, op) != ""
+}
+
+func detectCursorParam(pathItem *openapi3.PathItem, op *openapi3.Operation) string {
+	params := collectParams(pathItem, op)
+	for _, paramRef := range params {
+		param := paramRef.Value
+		if param == nil || param.In != "query" {
+			continue
+		}
+		if param.Name == "token" {
+			return param.Name
+		}
+	}
+	return ""
 }
 
 func successResponseSchema(op *openapi3.Operation) *openapi3.Schema {

codegen/grouper_test.go

@@ -156,11 +156,8 @@ func TestGroupEndpoints_Pagination(t *testing.T) {
 		if s.Name != "list" {
 			continue
 		}
-		if s.TokenField != "token" {
-			t.Errorf("TokenField = %q, want 'token'", s.TokenField)
-		}
-		if s.DataField != "data" {
-			t.Errorf("DataField = %q, want 'data'", s.DataField)
+		if !s.Paginated {
+			t.Error("Paginated = false, want true")
 		}
 		return
 	}

codegen/templates/command.go.tmpl

@@ -14,11 +14,8 @@ var {{pascalCase .Group}}{{pascalCase .Name}} = &command.Spec{
 	Endpoint:    {{quote .Endpoint}},
 	Method:      {{quote .Method}},
 	BodyEncoding: {{quote .BodyEncoding}},
-{{- if .TokenField}}
-	TokenField:  {{quote .TokenField}},
-{{- end}}
-{{- if .DataField}}
-	DataField:   {{quote .DataField}},
+{{- if .Paginated}}
+	Paginated:   true,
 {{- end}}
 {{- if .Examples}}
 	Examples: []string{

codegen/testdata/golden/upload.go

@@ -37,7 +37,7 @@ paths:
                         type: array
                         items:
                           type: object
-                      token:
+                      next_token:
                         type: string
     post:
       tags: [Videos]