fix: save binary fetch responses to a file instead of corrupting them as text

package.json

@@ -2,7 +2,7 @@
   "name": "@getalby/cli",
   "description": "CLI for Nostr Wallet Connect (NIP-47) with a few additional useful lightning tools",
   "repository": "https://github.com/getAlby/cli.git",
-  "version": "0.10.0",
+  "version": "0.10.1",
   "type": "module",
   "main": "build/index.js",
   "bin": {

src/commands/fetch.ts

@@ -101,6 +101,13 @@ export function registerFetch402Command(program: Command) {
     .option("-X, --method <method>", "HTTP method (GET, POST, etc.)")
     .option("-b, --body <json>", "Request body (JSON string)")
     .option("-H, --headers <json>", "Additional headers (JSON string)")
+    .option(
+      "-o, --output <path>",
+      "Save the response body to this file and return the path instead of " +
+        "inline content. Binary responses (audio, images, ...) are saved to a " +
+        "temp file automatically; use this to pick the location, or to keep a " +
+        "large text response out of the JSON output.",
+    )
     .option(
       "--dry-run",
       "Preview the price without paying: sends the request unpaid and reports " +
@@ -146,6 +153,10 @@ export function registerFetch402Command(program: Command) {
       "after",
       "\nExample:\n" +
         '  $ npx @getalby/cli fetch "https://example.com/api" --max-amount 1000 --currency BTC --unit sats --network lightning\n' +
+        "\nText responses are returned inline as `content`. Binary responses (audio,\n" +
+        "images, ...) are saved to a temp file - the output then carries `outputPath`,\n" +
+        "`contentType` and `sizeBytes` instead. Pass -o <path> to choose the file\n" +
+        "location, or to force any response (e.g. a large text body) to a file.\n" +
         "\nA successful response includes a `payment` object with the amount paid\n" +
         "(amountSat), routing fees (feesPaidMsat, in millisatoshis) and a reusable\n" +
         "`credentials` value. Reuse it to avoid paying again:\n" +
@@ -213,6 +224,7 @@ export function registerFetch402Command(program: Command) {
             ? parseCredentials(options.credentials)
             : undefined,
           resume: options.resume ? parseResume(options.resume) : undefined,
+          outputPath: options.output,
         });
         output(result);
       });

src/index.ts

@@ -45,7 +45,7 @@ program
       "    $ npx @getalby/cli pay alice@getalby.com --amount 5 --currency USD --network lightning\n" +
       '    $ npx @getalby/cli receive --amount 2100 --currency BTC --unit sats --network lightning --description "Coffee"',
   )
-  .version("0.10.0")
+  .version("0.10.1")
   .configureHelp({ showGlobalOptions: true })
   .option(
     "-w, --wallet-name <name>",

src/test/fetch-binary.test.ts

@@ -0,0 +1,123 @@
+import { describe, test, expect, vi, afterEach } from "vitest";
+import { readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { NWCClient } from "@getalby/sdk";
+import { fetch402 } from "../tools/lightning/fetch.js";
+
+// A minimal WAV header: contains bytes (0xAC, 0x88, ...) that are invalid
+// UTF-8, so a text decode would corrupt them into U+FFFD.
+const WAV_BYTES = new Uint8Array([
+  0x52, 0x49, 0x46, 0x46, 0x24, 0x00, 0x00, 0x00, 0x57, 0x41, 0x56, 0x45,
+  0x66, 0x6d, 0x74, 0x20, 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00,
+  0x44, 0xac, 0x00, 0x00, 0x88, 0x58, 0x01, 0x00,
+]);
+
+const URL = "https://example.com/protected";
+
+const client = {} as unknown as NWCClient;
+const savedFiles: string[] = [];
+
+function stubResponse(
+  body: BodyInit,
+  headers: Record<string, string> = {},
+): void {
+  vi.stubGlobal(
+    "fetch",
+    vi.fn().mockResolvedValue(new Response(body, { status: 200, headers })),
+  );
+}
+
+afterEach(() => {
+  vi.unstubAllGlobals();
+  for (const file of savedFiles.splice(0)) {
+    rmSync(file, { force: true });
+  }
+});
+
+describe("fetch binary responses", () => {
+  test("saves a binary body to a temp file instead of returning mangled text", async () => {
+    stubResponse(WAV_BYTES, { "content-type": "audio/wav" });
+
+    const result = await fetch402(client, { url: URL });
+    savedFiles.push(result.outputPath!);
+
+    expect(result.content).toBeUndefined();
+    expect(result.outputPath).toContain(tmpdir());
+    expect(result.outputPath).toMatch(/\.wav$/);
+    expect(result.contentType).toBe("audio/wav");
+    expect(result.sizeBytes).toBe(WAV_BYTES.length);
+    // The saved bytes are the original response, byte for byte.
+    expect(new Uint8Array(readFileSync(result.outputPath!))).toEqual(WAV_BYTES);
+  });
+
+  test("saves an unlabeled binary body with a .bin extension", async () => {
+    stubResponse(WAV_BYTES);
+
+    const result = await fetch402(client, { url: URL });
+    savedFiles.push(result.outputPath!);
+
+    expect(result.outputPath).toMatch(/\.bin$/);
+    expect(new Uint8Array(readFileSync(result.outputPath!))).toEqual(WAV_BYTES);
+  });
+
+  test("returns a declared-text body inline even when not valid UTF-8", async () => {
+    stubResponse(new Uint8Array([0x68, 0x69, 0xff]), {
+      "content-type": "text/plain; charset=utf-8",
+    });
+
+    const result = await fetch402(client, { url: URL });
+
+    expect(result.outputPath).toBeUndefined();
+    expect(result.content).toBe("hi�");
+  });
+
+  test("returns an unlabeled body inline when it decodes cleanly as UTF-8", async () => {
+    // Bytes without a content-type header: a string body would get an
+    // automatic text/plain and never reach the sniffing path.
+    stubResponse(new TextEncoder().encode('{"ok":true}'));
+
+    const result = await fetch402(client, { url: URL });
+
+    expect(result.outputPath).toBeUndefined();
+    expect(result.content).toBe('{"ok":true}');
+  });
+
+  test("--output saves any body, text included, to the given path", async () => {
+    stubResponse('{"ok":true}', { "content-type": "application/json" });
+    const outputPath = join(tmpdir(), `fetch-binary-test-${process.pid}.json`);
+
+    const result = await fetch402(client, { url: URL, outputPath });
+    savedFiles.push(outputPath);
+
+    expect(result.content).toBeUndefined();
+    expect(result.outputPath).toBe(outputPath);
+    expect(readFileSync(outputPath, "utf-8")).toBe('{"ok":true}');
+  });
+
+  test("falls back to base64 inline when a binary body cannot be saved", async () => {
+    // A failed write must not throw away the (possibly paid) response.
+    stubResponse(WAV_BYTES, { "content-type": "audio/wav" });
+    const outputPath = join(tmpdir(), "no-such-dir", "out.wav");
+
+    const result = await fetch402(client, { url: URL, outputPath });
+
+    expect(result.outputPath).toBeUndefined();
+    expect(result.writeError).toContain("ENOENT");
+    expect(result.contentType).toBe("audio/wav");
+    expect(
+      new Uint8Array(Buffer.from(result.contentBase64!, "base64")),
+    ).toEqual(WAV_BYTES);
+  });
+
+  test("falls back to inline text when a text body cannot be saved", async () => {
+    stubResponse('{"ok":true}', { "content-type": "application/json" });
+    const outputPath = join(tmpdir(), "no-such-dir", "out.json");
+
+    const result = await fetch402(client, { url: URL, outputPath });
+
+    expect(result.outputPath).toBeUndefined();
+    expect(result.writeError).toContain("ENOENT");
+    expect(result.content).toBe('{"ok":true}');
+  });
+});

src/tools/lightning/fetch.ts

@@ -7,6 +7,10 @@ import {
 } from "@getalby/lightning-tools/402";
 import { Invoice } from "@getalby/lightning-tools";
 import { NWCClient } from "@getalby/sdk";
+import { randomUUID } from "node:crypto";
+import { writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { DetailedError } from "../../utils.js";
 
 const DEFAULT_MAX_AMOUNT_SATS = 5000;
@@ -35,6 +39,29 @@ export interface Fetch402Params {
    * and NEVER pays again.
    */
   resume?: Fetch402Resume;
+  /**
+   * File path to save the response body to instead of returning it inline.
+   * Binary responses are saved to a temp file even without it; setting it
+   * forces a file (and chooses its location) for any response, so a large body
+   * never has to round-trip through the JSON output.
+   */
+  outputPath?: string;
+}
+
+export interface Fetch402Result {
+  /** The response body, when it is text and no --output path was given. */
+  content?: string;
+  /** The body base64-encoded, when it is binary and saving it failed. */
+  contentBase64?: string;
+  /** Path the body was saved to (binary responses, or --output). */
+  outputPath?: string;
+  /** The response Content-Type, reported alongside `outputPath`. */
+  contentType?: string | null;
+  /** Size of the body in bytes, reported when it is not inline text. */
+  sizeBytes?: number;
+  /** Why the body could not be saved to a file, when that fell back. */
+  writeError?: string;
+  payment?: PaymentInfo;
 }
 
 function buildRequestOptions(params: Fetch402Params): RequestInit {
@@ -63,7 +90,10 @@ function buildRequestOptions(params: Fetch402Params): RequestInit {
   return requestOptions;
 }
 
-export async function fetch402(client: NWCClient, params: Fetch402Params) {
+export async function fetch402(
+  client: NWCClient,
+  params: Fetch402Params,
+): Promise<Fetch402Result> {
   const requestOptions = buildRequestOptions(params);
   const maxAmountSats = params.maxAmountSats ?? DEFAULT_MAX_AMOUNT_SATS;
 
@@ -82,28 +112,91 @@ export async function fetch402(client: NWCClient, params: Fetch402Params) {
     throw error;
   }
 
-  const responseContent = await result.text();
   if (!result.ok) {
     // A non-OK response after a payment must not lose the payment metadata -
     // with the credential the request can be retried without paying the same
     // invoice again.
     throw new DetailedError(
-      `fetch returned non-OK status: ${result.status} ${responseContent}`,
+      `fetch returned non-OK status: ${result.status} ${await result.text()}`,
       result.payment?.credentials
         ? paidRecoveryDetails(result.payment)
         : undefined,
     );
   }
 
-  return {
-    content: responseContent,
-    // Payment metadata attached by the 402 helper: whether a payment was made,
-    // the amount (amountSat), routing fees (feesPaidMsat, in millisatoshis),
-    // and the reusable credential. Pass `credentials` back via --credentials
-    // on a follow-up request to authorize it without paying again. Absent when
-    // no 402 payment was involved (e.g. an already-open resource).
-    payment: result.payment,
-  };
+  const bytes = new Uint8Array(await result.arrayBuffer());
+  const contentType = result.headers.get("content-type");
+  // Payment metadata attached by the 402 helper: whether a payment was made,
+  // the amount (amountSat), routing fees (feesPaidMsat, in millisatoshis),
+  // and the reusable credential. Pass `credentials` back via --credentials
+  // on a follow-up request to authorize it without paying again. Absent when
+  // no 402 payment was involved (e.g. an already-open resource).
+  const payment = result.payment;
+
+  if (!params.outputPath) {
+    const text = decodeText(contentType, bytes);
+    if (text !== null) {
+      return { content: text, payment };
+    }
+  }
+
+  // Reading a binary body as text destroys it (every invalid UTF-8 sequence
+  // becomes U+FFFD), so it goes to a file and the output carries the path.
+  const outputPath =
+    params.outputPath ??
+    join(tmpdir(), `alby-cli-fetch-${randomUUID()}${extensionFor(contentType)}`);
+  try {
+    writeFileSync(outputPath, bytes);
+  } catch (error) {
+    // A failed write (ENOSPC, an unwritable --output path) after a payment
+    // must not throw away the paid response and its reusable credential -
+    // fall back to returning the body inline, base64-encoded when binary.
+    const writeError = errorMessage(error);
+    const text = decodeText(contentType, bytes);
+    if (text !== null) {
+      return { content: text, writeError, payment };
+    }
+    return {
+      contentBase64: Buffer.from(bytes).toString("base64"),
+      contentType,
+      sizeBytes: bytes.length,
+      writeError,
+      payment,
+    };
+  }
+  return { outputPath, contentType, sizeBytes: bytes.length, payment };
+}
+
+/**
+ * Decode the body for inline `content`, or return null when it is binary:
+ * a declared text content type is decoded as such, and without one the bytes
+ * qualify only when they are strictly valid UTF-8 (e.g. JSON from an API that
+ * mislabels or omits the content type).
+ */
+function decodeText(
+  contentType: string | null,
+  bytes: Uint8Array,
+): string | null {
+  const type = contentType?.split(";")[0].trim().toLowerCase() ?? "";
+  if (
+    type.startsWith("text/") ||
+    /[/+](json|xml)$/.test(type) ||
+    type === "application/x-www-form-urlencoded"
+  ) {
+    return new TextDecoder().decode(bytes);
+  }
+  try {
+    return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+  } catch {
+    return null;
+  }
+}
+
+// Filename extension for a saved binary, from the content type's subtype
+// (audio/wav -> .wav, application/epub+zip -> .epub). Unknown types get .bin.
+function extensionFor(contentType: string | null): string {
+  const subtype = contentType?.split(";")[0].split("/")[1]?.trim().toLowerCase();
+  return subtype ? `.${subtype.split("+")[0]}` : ".bin";
 }
 
 export interface DryRun402Result {