fix(node): split CLI streaming from library API to prevent stdout double-write (#473)

Objective: CLI users without --quiet see every output line printed twice.
The earlier streaming-flag-on-the-Promise approach (d6cded2 / 7e750d2)
fixed CLI behavior at the cost of a library API regression — convert(),
when called with quiet undefined, returned an empty string instead of
the real stdout.

Approach: Separate concerns at the layer boundary.
  - convert() (library API) is pinned to streamOutput: false. It always
    returns the full stdout string and never touches the parent process's
    stdio. No quiet-dependent return shape, no surprise side-effects.
  - _runForCli() (internal helper, used only by the bundled CLI) is
    pinned to streamOutput: true. It forwards Java's stdout and stderr
    to the parent process in real time and resolves to void — so the
    CLI cannot re-print what was already streamed.
  - cli.ts now calls _runForCli() and never touches the resolved value,
    closing the double-write loop at the source.

This keeps Java's progress logs flowing in real time on stderr — the
property that makes long hybrid runs observable — while the result text
streams once on stdout.

Evidence: Added a deterministic mock-based unit suite
(executeJar.unit.test.ts, 9 tests) and a real-jar integration suite
(streaming.integration.test.ts, 3 tests). Manual verification on the
bundled CLI:

| Scenario                                | Expected                       | Actual                                          |
|-----------------------------------------|--------------------------------|-------------------------------------------------|
| CLI, no --quiet, --to-stdout (1 page)   | 459 bytes (single copy)        | 459 bytes                                       |
| CLI, no --quiet, --to-stdout (14 pages) | 63006 bytes (single copy)      | 63006 bytes                                     |
| 14-page CLI: stderr progress logs       | streamed before stdout         | confirmed (28 lines, "Number of pages" present) |
| Library: convert(quiet:true) returns    | full stdout string             | 459 bytes                                       |
| Library: convert(quiet:undef) returns   | full stdout string             | 459 bytes                                       |
| Library: process.stdout side-effect     | none                           | none                                            |
| Library: process.stderr side-effect     | none                           | none                                            |

vitest: 30/30 pass.

Fixes #398

Co-authored-by: kuishou68 <54054995+kuishou68@users.noreply.github.com>

Author: bundolee

node/opendataloader-pdf/src/cli.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import { Command, CommanderError } from 'commander';
-import { convert } from './index.js';
+import { _runForCli } from './index.js';
 import { CliOptions, buildConvertOptions } from './convert-options.generated.js';
 import { registerCliOptions } from './cli-options.generated.js';
 
@@ -56,13 +56,9 @@ async function main(): Promise<number> {
   const convertOptions = buildConvertOptions(cliOptions);
 
   try {
-    const output = await convert(inputPaths, convertOptions);
-    if (output && !convertOptions.quiet) {
-      process.stdout.write(output);
-      if (!output.endsWith('\n')) {
-        process.stdout.write('\n');
-      }
-    }
+    // _runForCli streams stdout/stderr to the parent process as they arrive;
+    // we deliberately do not re-print anything here. (Issue #398.)
+    await _runForCli(inputPaths, convertOptions);
     return 0;
   } catch (err) {
     const message = err instanceof Error ? err.message : String(err);

node/opendataloader-pdf/src/index.ts

@@ -1,6 +1,7 @@
 import { spawn } from 'child_process';
 import * as path from 'path';
 import * as fs from 'fs';
+import { StringDecoder } from 'string_decoder';
 import { fileURLToPath } from 'url';
 
 // Re-export types and utilities from auto-generated file
@@ -15,6 +16,11 @@ const __dirname = path.dirname(__filename);
 const JAR_NAME = 'opendataloader-pdf-cli.jar';
 
 interface JarExecutionOptions {
+  /**
+   * When true, forwards Java's stdout and stderr chunks to the parent
+   * process in real time as well as accumulating them. Used by the bundled
+   * CLI so long-running conversions show progress as it happens.
+   */
   streamOutput?: boolean;
 }
 
@@ -37,24 +43,55 @@ function executeJar(args: string[], executionOptions: JarExecutionOptions = {}):
 
     let stdout = '';
     let stderr = '';
-
-    javaProcess.stdout.on('data', (data) => {
-      const chunk = data.toString();
+    // StringDecoder buffers incomplete multi-byte UTF-8 sequences across
+    // chunk boundaries — Buffer.toString() alone would emit replacement
+    // characters when, e.g., a 3-byte Korean codepoint splits across two
+    // 'data' events. One decoder per stream so they don't share state.
+    const stdoutDecoder = new StringDecoder('utf8');
+    const stderrDecoder = new StringDecoder('utf8');
+
+    javaProcess.stdout.on('data', (data: Buffer) => {
+      const chunk = stdoutDecoder.write(data);
+      if (chunk.length === 0) return;
       if (streamOutput) {
+        // Stream-only: don't also accumulate, or a multi-hour conversion
+        // would buffer its entire (potentially gigabyte) stdout in memory
+        // for no consumer.
         process.stdout.write(chunk);
+      } else {
+        stdout += chunk;
       }
-      stdout += chunk;
     });
 
-    javaProcess.stderr.on('data', (data) => {
-      const chunk = data.toString();
+    javaProcess.stderr.on('data', (data: Buffer) => {
+      const chunk = stderrDecoder.write(data);
+      if (chunk.length === 0) return;
       if (streamOutput) {
         process.stderr.write(chunk);
       }
+      // stderr is always accumulated (progress logs are small and we need
+      // them for error messages on non-zero exit).
       stderr += chunk;
     });
 
     javaProcess.on('close', (code) => {
+      // Flush any trailing bytes the decoder is still holding (always emit
+      // them — if we drop them on error paths, error messages with non-ASCII
+      // characters lose their tail).
+      const stdoutTail = stdoutDecoder.end();
+      const stderrTail = stderrDecoder.end();
+      if (stdoutTail.length > 0) {
+        if (streamOutput) {
+          process.stdout.write(stdoutTail);
+        } else {
+          stdout += stdoutTail;
+        }
+      }
+      if (stderrTail.length > 0) {
+        if (streamOutput) process.stderr.write(stderrTail);
+        stderr += stderrTail;
+      }
+
       if (code === 0) {
         resolve(stdout);
       } else {
@@ -80,26 +117,58 @@ function executeJar(args: string[], executionOptions: JarExecutionOptions = {}):
   });
 }
 
-export function convert(
+function buildJarArgs(
   inputPaths: string | string[],
-  options: ConvertOptions = {},
-): Promise<string> {
+  options: ConvertOptions,
+): string[] | Error {
   const inputList = Array.isArray(inputPaths) ? inputPaths : [inputPaths];
   if (inputList.length === 0) {
-    return Promise.reject(new Error('At least one input path must be provided.'));
+    return new Error('At least one input path must be provided.');
   }
 
   for (const input of inputList) {
     if (!fs.existsSync(input)) {
-      return Promise.reject(new Error(`Input file or folder not found: ${input}`));
+      return new Error(`Input file or folder not found: ${input}`);
     }
   }
 
-  const args: string[] = [...inputList, ...buildArgs(options)];
+  return [...inputList, ...buildArgs(options)];
+}
 
-  return executeJar(args, {
-    streamOutput: !options.quiet,
-  });
+export function convert(
+  inputPaths: string | string[],
+  options: ConvertOptions = {},
+): Promise<string> {
+  const argsOrError = buildJarArgs(inputPaths, options);
+  if (argsOrError instanceof Error) {
+    return Promise.reject(argsOrError);
+  }
+  // Library API: never streams to the parent process. Returns the full stdout
+  // string so callers can do `const out = await convert(...)` without surprise
+  // side-effects on process.stdout / process.stderr.
+  return executeJar(argsOrError, { streamOutput: false });
+}
+
+/**
+ * Internal entry point used by the bundled CLI. Streams Java's stdout and
+ * stderr to the parent process in real time (so long-running conversions like
+ * hybrid mode show progress as it happens) and resolves without a stdout
+ * payload — preventing the caller from re-printing what was already streamed.
+ *
+ * Not part of the public API: do not import this from application code. Use
+ * {@link convert} instead.
+ *
+ * @internal
+ */
+export async function _runForCli(
+  inputPaths: string | string[],
+  options: ConvertOptions = {},
+): Promise<void> {
+  const argsOrError = buildJarArgs(inputPaths, options);
+  if (argsOrError instanceof Error) {
+    throw argsOrError;
+  }
+  await executeJar(argsOrError, { streamOutput: true });
 }
 
 /**