Fix processing enormous json (#100)

* wip

* wiop

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

Author: jamierpond

.gitignore

@@ -57,3 +57,4 @@ cli/bin/yapi
 specs
 .specify
 .claude/commands/specify.*
+.idea

cli/cmd/yapi/run.go

@@ -27,6 +27,7 @@ type runContext struct {
 	envName      string // Target environment from yapi.config.yml
 	jsonOutput   bool   // If true, output structured JSON instead of formatted output
 	strictEnv    bool   // If true, error on missing env files and disable OS env fallback
+	verbose      bool   // If true, show verbose output (request details, timing, headers)
 }
 
 func (app *rootCommand) runInteractiveE(cmd *cobra.Command, args []string) error {
@@ -47,7 +48,8 @@ func (app *rootCommand) runE(cmd *cobra.Command, args []string) error {
 	envName, _ := cmd.Flags().GetString("env")
 	jsonOutput, _ := cmd.Flags().GetBool("json")
 	strictEnv, _ := cmd.Flags().GetBool("strict-env")
-	return app.runConfigPathWithOptionsE(path, envName, jsonOutput, strictEnv)
+	verbose, _ := cmd.Flags().GetBool("verbose")
+	return app.runConfigPathWithOptionsE(path, envName, jsonOutput, strictEnv, verbose)
 }
 
 func (app *rootCommand) watchE(cmd *cobra.Command, args []string) error {
@@ -128,37 +130,77 @@ func printWatchHeader(path string) {
 	fmt.Printf("%s\n\n", color.Dim("["+time.Now().Format("15:04:05")+"]"))
 }
 
+// OutputSavedError is returned when output is too large for terminal and was saved to file.
+type OutputSavedError struct {
+	Path string
+}
+
+func (e *OutputSavedError) Error() string {
+	return fmt.Sprintf("output saved to %s (too large for terminal)\nview with: cat %s | jq", e.Path, e.Path)
+}
+
+// printResultOptions configures printResult behavior.
+type printResultOptions struct {
+	skipMeta bool // Don't print URL/Time/Size (already shown in verbose mode)
+}
+
 // printResult outputs a single result with optional expectation.
-func (app *rootCommand) printResult(result *runner.Result, expectRes *runner.ExpectationResult) {
+// configPath is used for generating auto-save filenames when output is too large.
+// Returns OutputSavedError if output was saved to file instead of printed.
+func (app *rootCommand) printResult(result *runner.Result, expectRes *runner.ExpectationResult, configPath string, opts printResultOptions) error {
+	var savedPath string
 	if result != nil {
 		// Check if stdout is a TTY (terminal)
 		isTTY := isTerminal(os.Stdout)
 
 		// Check if content is binary
 		isBinary := utils.IsBinaryContent(result.Body)
 
-		// Skip dumping binary output unless explicitly requested with --binary-output
-		if isBinary && !app.binaryOutput {
+		switch {
+		case isBinary && !app.binaryOutput:
+			// Skip dumping binary output unless explicitly requested with --binary-output
 			if isTTY {
 				fmt.Fprintf(os.Stderr, "\n%s\n", color.Yellow("Binary content detected. Output hidden to prevent terminal corruption."))
 				fmt.Fprintf(os.Stderr, "%s\n", color.Dim("To display binary output, use --binary-output flag or pipe to a file."))
 			}
 			// In non-TTY (CI/piped), silently skip binary output
-		} else {
-			body := strings.TrimRight(output.Highlight(result.Body, result.ContentType, app.noColor), "\n\r")
-			fmt.Println(body)
+		case result.OutputFile != "":
+			// Output was already saved via output_file config - don't write again
+			if len(result.Body) > maxOutputSize {
+				savedPath = result.OutputFile
+			} else {
+				// Small enough to print, but also saved to file
+				body := output.Highlight(result.Body, result.ContentType, app.noColor)
+				fmt.Println(strings.TrimRight(body, "\n\r"))
+			}
+		default:
+			// No output_file specified - render normally (may auto-save if large)
+			body := result.Body
+			if len(body) <= maxOutputSize {
+				body = output.Highlight(body, result.ContentType, app.noColor)
+			}
+			outputResult := renderOutput(body, configPath)
+			savedPath = outputResult.SavedPath
 		}
 
-		printResultMeta(result)
+		if !opts.skipMeta {
+			printResultMeta(result)
+		}
 	}
 	if expectRes != nil {
 		printExpectationResult(expectRes)
 	}
+	if savedPath != "" {
+		return &OutputSavedError{Path: savedPath}
+	}
+	return nil
 }
 
 // executeRunE is the unified execution pipeline for both Run and Watch modes.
 // Returns error for middleware to capture.
 func (app *rootCommand) executeRunE(ctx runContext) error {
+	log := NewLogger(ctx.verbose)
+
 	opts := runner.Options{
 		URLOverride:    app.urlOverride,
 		NoColor:        app.noColor,
@@ -168,6 +210,8 @@ func (app *rootCommand) executeRunE(ctx runContext) error {
 		StrictEnv:      ctx.strictEnv,
 	}
 
+	log.Verbose("Loading config: %s", ctx.path)
+
 	// Load project and environment configuration
 	projEnv, err := loadProjectAndEnv(ctx.path, ctx.envName, true)
 	if err != nil {
@@ -180,15 +224,25 @@ func (app *rootCommand) executeRunE(ctx runContext) error {
 
 	// Apply project settings if found
 	if projEnv != nil {
+		log.Verbose("Project: %s", projEnv.projectRoot)
 		opts.ProjectRoot = projEnv.projectRoot
 		if projEnv.envVars != nil {
+			log.Verbose("Environment: %s (%d vars)", projEnv.envName, len(projEnv.envVars))
 			opts.EnvOverrides = projEnv.envVars
 			opts.ProjectEnv = projEnv.envName
 		}
 	}
 
+	log.Verbose("Sending request...")
 	runRes := app.engine.RunConfig(context.Background(), ctx.path, opts)
 
+	// Log response details if available
+	if runRes.Result != nil {
+		log.Response(runRes.Result.StatusCode, runRes.Result.Headers, runRes.Result.Duration, runRes.Result.BodyBytes)
+	} else if runRes.Error != nil {
+		log.Verbose("Request failed: %v", runRes.Error)
+	}
+
 	// Handle validation/parse errors first
 	if runRes.Error != nil && runRes.Analysis == nil {
 		if ctx.strict || ctx.returnErrors {
@@ -236,7 +290,9 @@ func (app *rootCommand) executeRunE(ctx runContext) error {
 		})
 	}
 
-	app.printResult(runRes.Result, runRes.ExpectRes)
+	if err := app.printResult(runRes.Result, runRes.ExpectRes, ctx.path, printResultOptions{skipMeta: ctx.verbose}); err != nil {
+		return err
+	}
 
 	if runRes.Error != nil {
 		if ctx.strict || ctx.returnErrors {
@@ -261,14 +317,17 @@ func (app *rootCommand) executeChain(ctx runContext, runRes *core.RunConfigResul
 	}
 
 	// Print results from all completed steps (even if chain failed)
+	var outputSavedErr error
 	if chainResult != nil {
 		for i, stepResult := range chainResult.Results {
 			fmt.Fprintf(os.Stderr, "\n--- Step %d: %s ---\n", i+1, chainResult.StepNames[i])
 			var expectRes *runner.ExpectationResult
 			if i < len(chainResult.ExpectationResults) {
 				expectRes = chainResult.ExpectationResults[i]
 			}
-			app.printResult(stepResult, expectRes)
+			if err := app.printResult(stepResult, expectRes, ctx.path, printResultOptions{}); err != nil {
+				outputSavedErr = err
+			}
 		}
 	}
 
@@ -283,7 +342,7 @@ func (app *rootCommand) executeChain(ctx runContext, runRes *core.RunConfigResul
 	fmt.Fprintln(os.Stderr, "\nChain completed successfully.")
 	out, noColor := app.io(ctx.strict)
 	validation.PrintWarnings(runRes.Analysis, out, noColor)
-	return nil
+	return outputSavedErr
 }
 
 // runConfigPathE runs a config file in strict mode (returns error)
@@ -299,8 +358,8 @@ func (app *rootCommand) runConfigPathWithEnvAndJSONE(path string, envName string
 }
 
 // runConfigPathWithOptionsE runs a config file with all options
-func (app *rootCommand) runConfigPathWithOptionsE(path string, envName string, jsonOutput bool, strictEnv bool) error {
-	return app.executeRunE(runContext{path: path, strict: true, envName: envName, jsonOutput: jsonOutput, strictEnv: strictEnv})
+func (app *rootCommand) runConfigPathWithOptionsE(path string, envName string, jsonOutput bool, strictEnv bool, verbose bool) error {
+	return app.executeRunE(runContext{path: path, strict: true, envName: envName, jsonOutput: jsonOutput, strictEnv: strictEnv, verbose: verbose})
 }
 
 // printExpectationResult prints expectation results to stderr

cli/cmd/yapi/utils.go

@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+	"time"
 
 	"yapi.run/cli/internal/config"
 	"yapi.run/cli/internal/tui"
@@ -157,3 +158,116 @@ func isTerminal(f *os.File) bool {
 	}
 	return (stat.Mode() & os.ModeCharDevice) != 0
 }
+
+// maxOutputSize is the threshold above which output is auto-saved to a file instead of printed.
+// 1MB is a reasonable limit - terminals struggle with larger outputs.
+const maxOutputSize = 1024 * 1024
+
+// OutputResult holds the result of rendering output.
+type OutputResult struct {
+	Printed   bool   // Output was printed to terminal
+	SavedPath string // If not empty, output was saved to this file
+	SaveErr   error  // If not nil, save failed with this error
+}
+
+// renderOutput prints body to stdout, or saves to a file if too large.
+// configPath is used to generate a meaningful filename for auto-saved files.
+// When output is saved to file, no message is printed - caller handles that via OutputSavedError.
+func renderOutput(body string, configPath string) OutputResult {
+	if len(body) <= maxOutputSize {
+		fmt.Println(strings.TrimRight(body, "\n\r"))
+		return OutputResult{Printed: true}
+	}
+
+	// Output too large - save to file
+	outputPath := generateOutputPath(configPath)
+	if err := os.WriteFile(outputPath, []byte(body), 0600); err != nil {
+		fmt.Fprintf(os.Stderr, "failed to save large output: %v\n", err)
+		return OutputResult{SaveErr: err}
+	}
+
+	return OutputResult{SavedPath: outputPath}
+}
+
+// Logger provides leveled logging for verbose output.
+type Logger struct {
+	verbose bool
+}
+
+// NewLogger creates a logger. If verbose is false, Debug calls are no-ops.
+func NewLogger(verbose bool) *Logger {
+	return &Logger{verbose: verbose}
+}
+
+// Verbose prints a message only if verbose mode is enabled.
+func (l *Logger) Verbose(format string, args ...any) {
+	if !l.verbose {
+		return
+	}
+	fmt.Fprintf(os.Stderr, "[VERBOSE] "+format+"\n", args...)
+}
+
+// Section prints a section header only if verbose mode is enabled.
+func (l *Logger) Section(name string) {
+	if !l.verbose {
+		return
+	}
+	fmt.Fprintf(os.Stderr, "\n=== %s ===\n", name)
+}
+
+// Request logs request details in verbose mode.
+func (l *Logger) Request(method, url string, headers map[string]string, body string) {
+	if !l.verbose {
+		return
+	}
+	l.Section("REQUEST")
+	fmt.Fprintf(os.Stderr, "%s %s\n", method, url)
+
+	if len(headers) > 0 {
+		fmt.Fprintln(os.Stderr, "\nHeaders:")
+		for k, v := range headers {
+			fmt.Fprintf(os.Stderr, "  %s: %s\n", k, v)
+		}
+	}
+
+	if body != "" {
+		fmt.Fprintln(os.Stderr, "\nBody:")
+		if len(body) > 1000 {
+			fmt.Fprintf(os.Stderr, "  %s... (%d bytes total)\n", body[:1000], len(body))
+		} else {
+			fmt.Fprintf(os.Stderr, "  %s\n", body)
+		}
+	}
+	fmt.Fprintln(os.Stderr, "")
+}
+
+// Response logs response details in verbose mode.
+func (l *Logger) Response(statusCode int, headers map[string]string, duration time.Duration, bodySize int) {
+	if !l.verbose {
+		return
+	}
+	l.Section("RESPONSE")
+	fmt.Fprintf(os.Stderr, "Status: %d\n", statusCode)
+	fmt.Fprintf(os.Stderr, "Time: %s\n", duration)
+	fmt.Fprintf(os.Stderr, "Size: %s\n", formatBytes(bodySize))
+
+	if len(headers) > 0 {
+		fmt.Fprintln(os.Stderr, "\nHeaders:")
+		for k, v := range headers {
+			fmt.Fprintf(os.Stderr, "  %s: %s\n", k, v)
+		}
+	}
+	fmt.Fprintln(os.Stderr, "")
+}
+
+// generateOutputPath creates a filename like: config-name-output-20060102-150405.json
+func generateOutputPath(configPath string) string {
+	base := filepath.Base(configPath)
+	name := strings.TrimSuffix(base, ".yapi.yml")
+	if name == base {
+		name = strings.TrimSuffix(base, filepath.Ext(base))
+	}
+
+	timestamp := time.Now().Format("20060102-150405")
+	return fmt.Sprintf("%s-output-%s.json", name, timestamp)
+}

cli/go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/graphql-go/graphql v0.8.1
 	github.com/itchyny/gojq v0.12.17
 	github.com/jhump/protoreflect v1.17.0
+	github.com/joho/godotenv v1.5.1
 	github.com/mattn/go-isatty v0.0.20
 	github.com/sahilm/fuzzy v0.1.1
 	github.com/spf13/cobra v1.10.1
@@ -54,7 +55,6 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/itchyny/timefmt-go v0.1.6 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
-	github.com/joho/godotenv v1.5.1 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mattn/go-localereader v0.0.1 // indirect

cli/internal/cli/commands/commands.go

@@ -74,6 +74,7 @@ var cmdManifest = []CommandSpec{
 			{Name: "env", Shorthand: "e", Type: "string", Default: "", Usage: "Target environment from yapi.config.yml"},
 			{Name: "json", Type: "bool", Default: false, Usage: "Output result as JSON with full metadata"},
 			{Name: "strict-env", Type: "bool", Default: false, Usage: "Strict env mode: error on missing env files, no OS env fallback"},
+			{Name: "verbose", Shorthand: "v", Type: "bool", Default: false, Usage: "Show verbose output (request details, timing, headers)"},
 		},
 	},
 	{

cli/internal/runner/runner.go

@@ -29,6 +29,7 @@ type Result struct {
 	BodyChars   int
 	BodyBytes   int
 	Headers     map[string]string // Response headers
+	OutputFile  string            // Path where output was saved (if output_file was specified)
 }
 
 // Options for execution
@@ -81,6 +82,7 @@ func Run(ctx context.Context, exec executor.TransportFunc, req *domain.Request,
 	}
 
 	// Write to output file if specified
+	var savedOutputFile string
 	if outputFile, ok := req.Metadata["output_file"]; ok && outputFile != "" {
 		// Resolve relative paths against the config file directory
 		if !filepath.IsAbs(outputFile) && opts.ConfigFilePath != "" {
@@ -95,6 +97,7 @@ func Run(ctx context.Context, exec executor.TransportFunc, req *domain.Request,
 		if err := os.WriteFile(outputFile, []byte(body), 0600); err != nil {
 			return nil, fmt.Errorf("failed to write output file '%s': %w", outputFile, err)
 		}
+		savedOutputFile = outputFile
 	}
 
 	bodyLines := strings.Count(body, "\n") + 1
@@ -112,6 +115,7 @@ func Run(ctx context.Context, exec executor.TransportFunc, req *domain.Request,
 		BodyChars:   bodyChars,
 		BodyBytes:   bodyBytesLen,
 		Headers:     resp.Headers,
+		OutputFile:  savedOutputFile,
 	}, nil
 }