fix(interpreter): preserve leading whitespace in multi-line quoted strings (#259) (#276)

normalizeScript() trimStart()s leading indentation from script lines so
indented template literals parse, but it ran line-by-line and also stripped
leading whitespace inside multi-line single/double-quoted strings (e.g.
python3 -c with an indented body -> IndentationError). Make it quote-aware,
mirroring the earlier heredoc-aware fix: track open quote state across lines
and only strip indentation from lines that begin outside any quote; lines
that begin inside an unterminated quote are preserved verbatim. Un-skips four
sed spec tests whose indented stdin was being corrupted.

Author: mutewinter

.changeset/multiline-quoted-whitespace.md

@@ -0,0 +1,18 @@
+---
+"just-bash": patch
+---
+
+interpreter: preserve leading whitespace in multi-line quoted strings (fixes #259)
+
+`exec()` runs each script through `normalizeScript()`, which `trimStart()`s
+leading indentation from lines so indented template-literal scripts parse. It
+was applied line-by-line and stripped the leading whitespace inside multi-line
+single- and double-quoted strings too. The visible symptom was `python3 -c
+'...'` (and `node -e`, `awk`, etc.) with an indented body failing with
+`IndentationError`, while the same code via heredoc or pipe worked.
+
+`normalizeScript()` is now quote-aware (mirroring the earlier heredoc-aware
+fix): it only strips indentation from lines that begin outside any quote, and
+preserves lines that begin inside an unterminated single- or double-quoted
+string verbatim. This also un-skips four sed spec tests whose indented stdin
+was previously being corrupted.

packages/just-bash/src/Bash.ts

@@ -861,9 +861,51 @@ export class Bash {
   }
 }
 
+type QuoteScanState = "none" | "single" | "double";
+
+/**
+ * Track open single/double-quote state across one physical line, starting from
+ * `start` (the state carried over from the previous line). Used by
+ * normalizeScript to know whether a line begins inside a multi-line quoted
+ * string, where leading whitespace is literal and must not be trimmed.
+ *
+ * Only single and double quotes matter: those are the contexts where leading
+ * whitespace is significant. Backslash escapes and `#` comments are honored so
+ * a quote inside a comment (e.g. `# don't`) doesn't desync the state.
+ */
+function scanLineQuoteState(
+  line: string,
+  start: QuoteScanState,
+): QuoteScanState {
+  let state = start;
+  for (let i = 0; i < line.length; i++) {
+    const ch = line[i];
+    if (state === "single") {
+      // Inside single quotes only a closing quote is special (no escapes).
+      if (ch === "'") state = "none";
+    } else if (state === "double") {
+      if (ch === "\\") {
+        i++; // backslash escapes the next char inside double quotes
+      } else if (ch === '"') {
+        state = "none";
+      }
+    } else if (ch === "'") {
+      state = "single";
+    } else if (ch === '"') {
+      state = "double";
+    } else if (ch === "\\") {
+      i++; // backslash escapes the next char (e.g. \" or \')
+    } else if (ch === "#" && (i === 0 || /\s/.test(line[i - 1]))) {
+      break; // start of a comment: the rest of the line is not shell-significant
+    }
+  }
+  return state;
+}
+
 /**
- * Normalize a script by stripping leading whitespace from lines,
- * while preserving whitespace inside heredoc content.
+ * Normalize a script by stripping leading whitespace from lines, while
+ * preserving whitespace inside heredoc content and inside multi-line quoted
+ * strings.
  *
  * This allows writing indented bash scripts in template literals:
  * ```
@@ -874,6 +916,11 @@ export class Bash {
  * `);
  * ```
  *
+ * Leading whitespace is only stripped from lines that begin outside any quote.
+ * A line that begins inside an unterminated single- or double-quoted string is
+ * literal quoted content (e.g. the body of `python3 -c "..."`), so it is kept
+ * verbatim.
+ *
  * Heredocs are detected by looking for << or <<- operators and their delimiters.
  */
 function normalizeScript(script: string): string {
@@ -883,6 +930,10 @@ function normalizeScript(script: string): string {
   // Stack of pending heredoc delimiters (for nested heredocs)
   const pendingDelimiters: { delimiter: string; stripTabs: boolean }[] = [];
 
+  // Open single/double-quote state carried across lines. When a line begins
+  // inside an open quote, its leading whitespace is literal and preserved.
+  let quoteState: QuoteScanState = "none";
+
   for (let i = 0; i < lines.length; i++) {
     const line = lines[i];
 
@@ -903,9 +954,19 @@ function normalizeScript(script: string): string {
       continue;
     }
 
-    // Not inside a heredoc - normalize the line and check for heredoc starts
-    const normalizedLine = line.trimStart();
+    // Not inside a heredoc. Lines that begin inside an open quote are literal
+    // quoted content (leading whitespace is significant), so preserve them
+    // verbatim; otherwise strip leading indentation.
+    const startState = quoteState;
+    const normalizedLine = startState === "none" ? line.trimStart() : line;
     result.push(normalizedLine);
+    quoteState = scanLineQuoteState(line, startState);
+
+    // Only detect heredoc operators on lines that begin outside any quote;
+    // a `<<WORD` inside quoted content is not a heredoc.
+    if (startState !== "none") {
+      continue;
+    }
 
     // Check for heredoc operators in this line
     // Match: <<DELIM, <<-DELIM, << 'DELIM', <<- "DELIM", etc.

packages/just-bash/src/spec-tests/sed/skips.ts

@@ -126,14 +126,6 @@ const SKIP_TESTS: Map<string, string> = new Map<string, string>([
   // ============================================================
   // PythonSed chang.suite - complex N/D/P scripts
   // ============================================================
-  [
-    "pythonsed-chang.suite:Delete two consecutive lines if the first one contains PAT1 and the second one contains PAT2.",
-    "N/P/D commands",
-  ],
-  [
-    "pythonsed-chang.suite:Get the line following a line containing PAT - Case 1 - 1.",
-    "N/D commands",
-  ],
   [
     "pythonsed-chang.suite:Remove comments (/* ... */, maybe multi-line) of a C program. - 1",
     "N command behavior",
@@ -171,10 +163,6 @@ const SKIP_TESTS: Map<string, string> = new Map<string, string>([
     "pythonsed-chang.suite:Join every N lines to one - 1.",
     "complex N/D branching",
   ],
-  [
-    'pythonsed-chang.suite:Extract "Received:" header(s) from a mailbox.',
-    "complex N/D branching",
-  ],
   [
     "pythonsed-chang.suite:Extract every IMG elements from an HTML file.",
     "complex branching",
@@ -183,10 +171,6 @@ const SKIP_TESTS: Map<string, string> = new Map<string, string>([
     "pythonsed-chang.suite:Find failed instances without latter successful ones.",
     "complex N/D branching",
   ],
-  [
-    "pythonsed-chang.suite:Change the first quote of every single-quoted string to backquote(`). - 1",
-    "complex pattern manipulation",
-  ],
   ["pythonsed-chang.suite:1 cat chicken", "test name parsing issue"],
   [
     "pythonsed-chang.suite:First number 1111 Second <2222>",

packages/just-bash/src/syntax/quoted-multiline-whitespace.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it } from "vitest";
+import { Bash } from "../Bash.js";
+
+// Regression tests: script normalization (which strips leading indentation so
+// indented template literals parse) must NOT trim lines that begin inside a
+// multi-line single- or double-quoted string. There the leading whitespace is
+// literal (POSIX) and must be preserved verbatim, e.g. the body of
+// `python3 -c '...'`.
+describe("multi-line quoted string whitespace", () => {
+  it("preserves leading indentation inside a single-quoted string", async () => {
+    const env = new Bash();
+    const result = await env.exec(
+      "printf '%s' 'import sys\nfor p in [1]:\n    print(p)\n'",
+    );
+    expect(result.stdout).toBe("import sys\nfor p in [1]:\n    print(p)\n");
+    expect(result.exitCode).toBe(0);
+  });
+
+  it("preserves leading indentation inside a double-quoted string", async () => {
+    const env = new Bash();
+    const result = await env.exec(
+      'printf "%s" "first\n    second\n        third\n"',
+    );
+    expect(result.stdout).toBe("first\n    second\n        third\n");
+    expect(result.exitCode).toBe(0);
+  });
+
+  it("preserves indentation through a variable assignment and expansion", async () => {
+    const env = new Bash();
+    const result = await env.exec(
+      "v='a\n    b\n        c'\nprintf '%s' \"$v\"",
+    );
+    expect(result.stdout).toBe("a\n    b\n        c");
+    expect(result.exitCode).toBe(0);
+  });
+
+  it("still strips indentation from the surrounding (unquoted) script", async () => {
+    const env = new Bash();
+    const result = await env.exec(
+      "    if true; then\n        printf '%s' '    keep'\n    fi",
+    );
+    expect(result.stdout).toBe("    keep");
+    expect(result.exitCode).toBe(0);
+  });
+
+  it("is not confused by an apostrophe inside a comment", async () => {
+    const env = new Bash();
+    // The `'` in the comment must not open a quote that swallows the next
+    // line's indentation handling.
+    const result = await env.exec(
+      "echo start # don't trip on this\n    printf '%s' 'x\n    y'",
+    );
+    expect(result.stdout).toBe("start\nx\n    y");
+    expect(result.exitCode).toBe(0);
+  });
+});