refactor(interpreter): model stdin as a single shared StdinStream

.changeset/stdin-stream.md

@@ -0,0 +1,39 @@
+---
+"just-bash": minor
+---
+
+interpreter: model stdin as a single shared stream (`StdinStream`), fixing loops and compound commands losing stdin position
+
+Stdin is now represented exactly one way everywhere: a `StdinStream` object
+that behaves like a bash file descriptor — it holds the content bytes and a
+read offset, and is shared by reference. Reading is consuming: a command
+that drains stdin (`cat`, `grep`, `sed`, ...) advances the offset for every
+other holder of the stream, including across subshell and pipeline-stage
+boundaries. This replaces the old `groupStdin?: string` snapshot plus
+per-call-site fallback rules, which made it easy for a command to read the
+loop's stdin without advancing it.
+
+Fixes, all verified against real bash via recorded comparison fixtures:
+
+- Commands inside `while read` loop bodies (pipelines, `cat`, `grep`, `tr`,
+  and ~30 other stdin consumers) now advance the loop's stdin instead of
+  re-reading the same input each iteration.
+- Stdin redirections (`<`, `<<`, `<<-`, `<<<`, `<&`) now work uniformly on
+  all compound commands (`if`, `for`, `while`, `until`, `case`, subshells,
+  groups) through one shared resolution path.
+- Subshells share the stdin offset with their parent (matching bash fd
+  semantics): `{ (read x); read y; } < f` gives `y` the second line.
+- A pipeline stage that produces empty output no longer lets the next
+  command fall back to the enclosing scope's stdin.
+- Heredoc/here-string content on compound commands and functions is now
+  UTF-8 encoded into the byte pipeline consistently (previously only simple
+  commands did this).
+- Function definition redirects (`f() { ...; } < file`) now apply on every
+  call and win over piped stdin; a missing redirect file is an error and
+  the body does not run (both match bash).
+
+BREAKING (TypeScript API): `CommandContext.stdin` is now a `StdinStream`
+instead of a `ByteString`. Custom commands call `ctx.stdin.readAll()` to
+consume input (or `ctx.stdin.peek()` to inspect without consuming) and
+convert with `latin1FromBytes` / `decodeBytesToUtf8` as before. The
+`StdinStream` class is exported from the package root.

packages/just-bash/README.md

@@ -37,9 +37,10 @@ const hello = defineCommand("hello", async (args, ctx) => {
 });
 
 const upper = defineCommand("upper", async (args, ctx) => {
-  // ctx.stdin is a ByteString — decode to text before string ops.
+  // ctx.stdin is a stream: readAll() consumes it and returns a ByteString —
+  // decode to text before string ops.
   return {
-    stdout: decodeBytesToUtf8(ctx.stdin).toUpperCase(),
+    stdout: decodeBytesToUtf8(ctx.stdin.readAll()).toUpperCase(),
     stderr: "",
     exitCode: 0,
   };
@@ -51,7 +52,7 @@ await bash.exec("hello Alice"); // "Hello, Alice!\n"
 await bash.exec("echo 'test' | upper"); // "TEST\n"
 ```
 
-Custom commands receive a `CommandContext` with `fs`, `cwd`, `env`, `stdin`, and `exec` (for subcommands), and work with pipes, redirections, and all shell features.
+Custom commands receive a `CommandContext` with `fs`, `cwd`, `env`, `stdin` (a consumable stream: `readAll()` to drain it, `peek()` to inspect without consuming), and `exec` (for subcommands), and work with pipes, redirections, and all shell features.
 
 <details>
 <summary><h2>Supported Commands</h2></summary>

packages/just-bash/src/Bash.ts

@@ -62,6 +62,7 @@ import {
   SecurityViolationError,
 } from "./security/defense-in-depth-box.js";
 import type { DefenseInDepthConfig } from "./security/types.js";
+import { StdinStream } from "./stdin-stream.js";
 import { serialize } from "./transform/serialize.js";
 import type {
   BashTransformResult,
@@ -374,6 +375,7 @@ export class Bash {
     this.state = {
       env,
       cwd,
+      stdin: new StdinStream(),
       previousDir: "/home/user",
       functions: new Map<string, FunctionDefNode>(),
       localScopes: [],
@@ -648,7 +650,9 @@ export class Bash {
       // UTF-8 bytes. Callers that already prepared a byte buffer (e.g.
       // `Buffer.from(buf).toString("latin1")`) opt into raw passthrough
       // via `stdinKind: "bytes"`.
-      groupStdin: encodeStdinForPipeline(options?.stdin, options?.stdinKind),
+      stdin: new StdinStream(
+        encodeStdinForPipeline(options?.stdin, options?.stdinKind) ?? "",
+      ),
       // Cooperative cancellation signal (used by timeout command)
       signal: options?.signal,
       // Extra arguments injected directly into first command's arg list

packages/just-bash/src/commands/awk/awk2.ts

@@ -239,7 +239,7 @@ export const awkCommand2: Command = {
       } else {
         // awk parses fields with regex / FS — decode bytes to UTF-8 so
         // non-ASCII data isn't split mid-codepoint.
-        const lines = decodeBytesToUtf8(ctx.stdin).split("\n");
+        const lines = decodeBytesToUtf8(ctx.stdin.readAll()).split("\n");
         if (lines.length > 0 && lines[lines.length - 1] === "") {
           lines.pop();
         }

packages/just-bash/src/commands/base64/base64.ts

@@ -34,7 +34,9 @@ async function readBinary(
     // Convert binary string directly to bytes without UTF-8 re-encoding
     return {
       ok: true,
-      data: Uint8Array.from(latin1FromBytes(ctx.stdin), (c) => c.charCodeAt(0)),
+      data: Uint8Array.from(latin1FromBytes(ctx.stdin.readAll()), (c) =>
+        c.charCodeAt(0),
+      ),
     };
   }
 
@@ -44,7 +46,9 @@ async function readBinary(
     if (file === "-") {
       // Convert binary string directly to bytes without UTF-8 re-encoding
       chunks.push(
-        Uint8Array.from(latin1FromBytes(ctx.stdin), (c) => c.charCodeAt(0)),
+        Uint8Array.from(latin1FromBytes(ctx.stdin.readAll()), (c) =>
+          c.charCodeAt(0),
+        ),
       );
       continue;
     }

packages/just-bash/src/commands/bash/bash.ts

@@ -39,7 +39,7 @@ export const bashCommand: Command = {
     // No arguments - read script from stdin if available. Decode bytes — a
     // bash script's UTF-8 string literals must reach the parser as text.
     if (args.length === 0) {
-      const stdinText = decodeBytesToUtf8(ctx.stdin);
+      const stdinText = decodeBytesToUtf8(ctx.stdin.readAll());
       if (stdinText.trim()) {
         return executeScript(stdinText, "bash", [], ctx);
       }
@@ -91,7 +91,7 @@ export const shCommand: Command = {
     // No arguments - read script from stdin if available. Decode bytes — a
     // shell script's UTF-8 string literals must reach the parser as text.
     if (args.length === 0) {
-      const stdinText = decodeBytesToUtf8(ctx.stdin);
+      const stdinText = decodeBytesToUtf8(ctx.stdin.readAll());
       if (stdinText.trim()) {
         return executeScript(stdinText, "sh", [], ctx);
       }
@@ -162,7 +162,9 @@ async function executeScript(
   const result = await ctx.exec(scriptToRun, {
     env: positionalEnv,
     cwd: ctx.cwd,
-    stdin: latin1FromBytes(ctx.stdin),
+    // Forward stdin without consuming the outer stream — the nested
+    // exec seeds its own stream from this copy.
+    stdin: latin1FromBytes(ctx.stdin.peek()),
     stdinKind: "bytes",
     signal: ctx.signal,
   });

packages/just-bash/src/commands/column/column.ts

@@ -172,12 +172,12 @@ export const column: Command = {
     // see follow-up.)
     let content: string;
     if (files.length === 0) {
-      content = decodeBytesToUtf8(ctx.stdin) ?? "";
+      content = decodeBytesToUtf8(ctx.stdin.readAll()) ?? "";
     } else {
       const parts: string[] = [];
       for (const file of files) {
         if (file === "-") {
-          parts.push(decodeBytesToUtf8(ctx.stdin) ?? "");
+          parts.push(decodeBytesToUtf8(ctx.stdin.readAll()) ?? "");
         } else {
           const filePath = ctx.fs.resolvePath(ctx.cwd, file);
           const fileContent = await ctx.fs.readFile(filePath);

packages/just-bash/src/commands/comm/comm.ts

@@ -80,7 +80,7 @@ export const commCommand: Command = {
     // compares equal regardless of which leg it came from.
     const readFile = async (file: string): Promise<string | null> => {
       if (file === "-") {
-        return decodeBytesToUtf8(ctx.stdin);
+        return decodeBytesToUtf8(ctx.stdin.readAll());
       }
       try {
         const path = ctx.fs.resolvePath(ctx.cwd, file);

packages/just-bash/src/commands/diff/diff.ts

@@ -58,10 +58,14 @@ export const diffCommand: Command = {
 
     // diff compares lines as strings. Normalize stdin (byte buffer) to
     // UTF-8 so it compares correctly against file content (utf8 by default).
+    // Read stdin once so `diff - -` compares stdin against itself (GNU
+    // behavior) instead of draining it on the first operand.
+    const stdinContent =
+      f1 === "-" || f2 === "-" ? decodeBytesToUtf8(ctx.stdin.readAll()) : "";
     try {
       c1 =
         f1 === "-"
-          ? decodeBytesToUtf8(ctx.stdin)
+          ? stdinContent
           : await ctx.fs.readFile(ctx.fs.resolvePath(ctx.cwd, f1));
     } catch {
       return {
@@ -74,7 +78,7 @@ export const diffCommand: Command = {
     try {
       c2 =
         f2 === "-"
-          ? decodeBytesToUtf8(ctx.stdin)
+          ? stdinContent
           : await ctx.fs.readFile(ctx.fs.resolvePath(ctx.cwd, f2));
     } catch {
       return {

packages/just-bash/src/commands/env/env.ts

@@ -111,8 +111,10 @@ export const envCommand: Command = {
       cwd: ctx.cwd,
       env: mapToRecord(newEnv),
       replaceEnv: true,
-      stdin: latin1FromBytes(ctx.stdin),
-      // ctx.stdin is already byte-shaped — forward verbatim.
+      // Forward stdin to the subcommand without consuming the outer
+      // stream — the subcommand's exec gets its own copy. Bytes are
+      // already latin1-shaped; pass verbatim.
+      stdin: latin1FromBytes(ctx.stdin.peek()),
       stdinKind: "bytes",
       signal: ctx.signal,
       args: cmdArgs,

packages/just-bash/src/commands/expand/expand.ts

@@ -218,7 +218,7 @@ export const expand: Command = {
     if (files.length === 0) {
       // expand counts column positions for tab-stop math; decode bytes so a
       // multibyte char counts as one column rather than 2–4.
-      const input = decodeBytesToUtf8(ctx.stdin) ?? "";
+      const input = decodeBytesToUtf8(ctx.stdin.readAll()) ?? "";
       output = processContent(input, options);
     } else {
       for (const file of files) {

packages/just-bash/src/commands/expand/unexpand.ts

@@ -260,7 +260,7 @@ export const unexpand: Command = {
     if (files.length === 0) {
       // unexpand counts column positions for tab-stop math; decode bytes so
       // a multibyte char doesn't pretend to be several columns wide.
-      const input = decodeBytesToUtf8(ctx.stdin) ?? "";
+      const input = decodeBytesToUtf8(ctx.stdin.readAll()) ?? "";
       output = processContent(input, options);
     } else {
       for (const file of files) {

packages/just-bash/src/commands/fold/fold.ts

@@ -244,7 +244,7 @@ export const fold: Command = {
     if (files.length === 0) {
       // Read from stdin. fold counts width in codepoints (and tab-stops),
       // so decode bytes — wrapping mid-multibyte would corrupt the data.
-      const input = decodeBytesToUtf8(ctx.stdin) ?? "";
+      const input = decodeBytesToUtf8(ctx.stdin.readAll()) ?? "";
       output = processContent(input, options);
     } else {
       // Process each file

packages/just-bash/src/commands/grep/grep.ts

@@ -226,22 +226,26 @@ export const grepCommand: Command = {
       };
     }
 
-    // If no files and stdin is provided (including empty string), read from
-    // stdin. grep runs regex over text — decode bytes to UTF-8 so multibyte
-    // codepoints match `.` / character classes correctly.
-    if (files.length === 0 && ctx.stdin !== undefined) {
-      const result = searchContent(decodeBytesToUtf8(ctx.stdin), regex, {
-        invertMatch,
-        showLineNumbers,
-        countOnly,
-        filename: "",
-        onlyMatching,
-        beforeContext,
-        afterContext,
-        maxCount,
-        kResetGroup,
-        preFilter,
-      });
+    // If no files, read from stdin (even when empty). grep runs regex over
+    // text — decode bytes to UTF-8 so multibyte codepoints match `.` /
+    // character classes correctly.
+    if (files.length === 0) {
+      const result = searchContent(
+        decodeBytesToUtf8(ctx.stdin.readAll()),
+        regex,
+        {
+          invertMatch,
+          showLineNumbers,
+          countOnly,
+          filename: "",
+          onlyMatching,
+          beforeContext,
+          afterContext,
+          maxCount,
+          kResetGroup,
+          preFilter,
+        },
+      );
       if (quietMode) {
         return { stdout: "", stderr: "", exitCode: result.matched ? 0 : 1 };
       }

packages/just-bash/src/commands/gzip/gzip.ts

@@ -288,7 +288,7 @@ async function processFile(
 
   // Handle stdin
   if (file === "-" || file === "") {
-    inputData = Uint8Array.from(latin1FromBytes(ctx.stdin), (c) =>
+    inputData = Uint8Array.from(latin1FromBytes(ctx.stdin.readAll()), (c) =>
       c.charCodeAt(0),
     );
     if (decompress) {
@@ -611,7 +611,7 @@ async function listFile(
   let inputData: Uint8Array;
 
   if (file === "-" || file === "") {
-    inputData = Uint8Array.from(latin1FromBytes(ctx.stdin), (c) =>
+    inputData = Uint8Array.from(latin1FromBytes(ctx.stdin.readAll()), (c) =>
       c.charCodeAt(0),
     );
   } else {
@@ -664,7 +664,7 @@ async function testFile(
   let inputData: Uint8Array;
 
   if (file === "-" || file === "") {
-    inputData = Uint8Array.from(latin1FromBytes(ctx.stdin), (c) =>
+    inputData = Uint8Array.from(latin1FromBytes(ctx.stdin.readAll()), (c) =>
       c.charCodeAt(0),
     );
   } else {

packages/just-bash/src/commands/head/head-tail-shared.ts

@@ -126,7 +126,7 @@ export async function processHeadTailFiles(
   // marked binary so the pipeline glue / redirects don't UTF-8 re-encode it.
   if (files.length === 0) {
     return {
-      stdout: contentProcessor(latin1FromBytes(ctx.stdin)),
+      stdout: contentProcessor(latin1FromBytes(ctx.stdin.readAll())),
       stderr: "",
       exitCode: 0,
       stdoutEncoding: "binary",

packages/just-bash/src/commands/html-to-markdown/html-to-markdown.ts

@@ -98,7 +98,7 @@ export const htmlToMarkdownCommand: Command = {
     // default already.
     let input: string;
     if (files.length === 0 || (files.length === 1 && files[0] === "-")) {
-      input = decodeBytesToUtf8(ctx.stdin);
+      input = decodeBytesToUtf8(ctx.stdin.readAll());
     } else {
       try {
         const filePath = ctx.fs.resolvePath(ctx.cwd, files[0]);

packages/just-bash/src/commands/join/join.ts

@@ -294,7 +294,7 @@ export const join: Command = {
     const contents: string[] = [];
     for (const file of files) {
       if (file === "-") {
-        contents.push(decodeBytesToUtf8(ctx.stdin) ?? "");
+        contents.push(decodeBytesToUtf8(ctx.stdin.readAll()) ?? "");
       } else {
         const filePath = ctx.fs.resolvePath(ctx.cwd, file);
         const content = await ctx.fs.readFile(filePath);

packages/just-bash/src/commands/jq/jq.ts

@@ -332,7 +332,10 @@ export const jqCommand: Command = {
     if (nullInput) {
       // No input
     } else if (files.length === 0 || (files.length === 1 && files[0] === "-")) {
-      inputs.push({ source: "stdin", content: decodeBytesToUtf8(ctx.stdin) });
+      inputs.push({
+        source: "stdin",
+        content: decodeBytesToUtf8(ctx.stdin.readAll()),
+      });
     } else {
       // Read all files in parallel using shared utility
       const result = await withDefenseContext("file read", () =>

packages/just-bash/src/commands/js-exec/js-exec.ts

@@ -598,18 +598,21 @@ export const jsExecCommand: Command = {
           exitCode: 2,
         };
       }
-    } else if (decodeBytesToUtf8(ctx.stdin).trim()) {
+    } else {
+      // readAll() consumes stdin, so read once and branch on the result.
       // Decode bytes — JS source can contain unicode identifiers and string
       // literals; running latin1 bytes as code corrupts them.
-      jsCode = decodeBytesToUtf8(ctx.stdin);
+      const stdinCode = decodeBytesToUtf8(ctx.stdin.readAll());
+      if (!stdinCode.trim()) {
+        return {
+          stdout: "",
+          stderr:
+            "js-exec: no input provided (use -c CODE or provide a script file)\n",
+          exitCode: 2,
+        };
+      }
+      jsCode = stdinCode;
       scriptPath = "<stdin>";
-    } else {
-      return {
-        stdout: "",
-        stderr:
-          "js-exec: no input provided (use -c CODE or provide a script file)\n",
-        exitCode: 2,
-      };
     }
 
     // Auto-detect module mode and type stripping from file extension

packages/just-bash/src/commands/md5sum/checksum.ts

@@ -170,7 +170,7 @@ export function createChecksumCommand(
       // through without decoding.
       const readBinary = async (file: string): Promise<Uint8Array | null> => {
         if (file === "-") {
-          return Uint8Array.from(latin1FromBytes(ctx.stdin), (c) =>
+          return Uint8Array.from(latin1FromBytes(ctx.stdin.readAll()), (c) =>
             c.charCodeAt(0),
           );
         }
@@ -190,7 +190,7 @@ export function createChecksumCommand(
           // Decode UTF-8 so non-ASCII filenames in the list survive.
           const content =
             file === "-"
-              ? decodeBytesToUtf8(ctx.stdin)
+              ? decodeBytesToUtf8(ctx.stdin.readAll())
               : await ctx.fs
                   .readFile(ctx.fs.resolvePath(ctx.cwd, file))
                   .catch(() => null);