Optionally store source maps as VLQ encoded (2/2): Transformer output, unstable_compactSourceMaps

Summary:
## This stack
Decoded tuple arrays are the single largest contributor to Metro's dev-server heap on large bundles (~10 million retained small arrays on FBiOS entry bundle, for example). Storing the same data as a compact VLQ string instead removes most of that footprint.

This reduces source map memory by ~51% on the heap and ~48% RSS for that ~16K module bundle.

The emitted whole-bundle source map is unchanged. When a module's map is stored as VLQ, `fromRawMappings` decodes it back to tuples just-in-time, with request-scoped caching. The trade-off is therefore decode + re-encode CPU when a `.map` is actually requested or `/symbolicate` request is made.

A plain `string` is used for `mappings` for now, since VLQ is ASCII by design. A `UInt8Array` would be marginally more efficient and potentially transferrable to/from worker threads, but would require more invasive changes to cache (de)serialisation. I did some benchmarking with this and it doesn't justify the complexity right now.

## This diff

Adds `unstable_compactSourceMaps` (default `false`). When enabled, the transform
worker stores each module's source map as a compact VLQ string (`VlqMap`)
instead of a decoded `Array<MetroSourceMapSegmentTuple>`.

Each module's map originates from one of three sources, so we encode the VLQ the
cheapest way available in each case (all byte-identical to the decoded-tuple
output):

- transformJS, not minifying (the dominant path — Hermes targets don't minify):
  encode the `VlqMap` straight from `result.decodedMap`, which `babel/generator`
  computes eagerly while generating, via `vlqMapFromBabelDecodedMap` — never
  materialising tuples.
- transformJS, minifying: the minifier returns its own map (not Babel's), so we
  re-encode the resulting tuples with `vlqMapFromTuples`.
- transformJSON: builds tuples directly (no Babel generate), so it likewise
  re-encodes with `vlqMapFromTuples`.

`countLines` is split out of `countLinesAndTerminateMap` so the decoded-map fast
path can compute the terminating mapping without building and terminating a
tuple array first.

## Benchmarks

*Cold cache (n=3, means)*

| Metric | base | compact |
|---|---|---|---|
| **Heap used** | 1653.7 MB | **809.7 MB (−51.0%)** |
| **RSS** | 1854.2 MB | 955.2 MB (−48.5%) |
| Heap growth (build) | 1606.5 MB | 761.2 MB (−52.6%) |
| Build CPU (`.bundle`) | 23.05 s | 22.42 s (n.s.) |
| **Serialize CPU (`.map`)** | 11.99 s | **14.19 s (+18.4%)** |

*Warm cache (n=3, means)*

| Metric | base | compact |
|---|---|---|---|
| **Heap used** | 1552 MB | **731 MB (−52.9%)** |
| **RSS** | 1775 MB | 923 MB (−48.0%) |
| Build CPU (`.bundle`) | 10.92 s | 8.86 s (−18.9%) |
| **Serialize CPU (`.map`)** | 11.87 s | **13.89 s (+17.0%)** |

## Why behind a flag?

1) The `map` structure is exposed to custom serialisers, so changing it is semver-breaking. Landing this as experimental opt-in in a non-breaking release allows integrators to experiment with it.
2) This is a trade-off of retained memory vs CPU required to emit a flat source map or symbolicate errors. The trade-off largely goes away with indexed maps (coming next) - but that is a semver-breaking change to output.

Changelog:
```
 - **[Experimental]**: Add `unstable_compactSourceMaps` to use a more memory-efficient source map format.
```

Differential Revision: D109216060

Author: robhogan

packages/metro-config/src/defaults/index.js

@@ -135,6 +135,7 @@ const getDefaultValues = (projectRoot: ?string): ConfigT => ({
     unstable_disableNormalizePseudoGlobals: false,
     unstable_renameRequire: true,
     unstable_compactOutput: false,
+    unstable_compactSourceMaps: false,
     unstable_memoizeInlineRequires: false,
     unstable_workerThreads: false,
   },

packages/metro-source-map/src/__tests__/source-map-test.js

@@ -9,14 +9,15 @@
  * @oncall react_native
  */
 
-import type {MetroSourceMapSegmentTuple} from '../source-map';
+import type {BabelDecodedMap, MetroSourceMapSegmentTuple} from '../source-map';
 
 import Generator from '../Generator';
 import {
   fromRawMappings,
   isVlqMap,
   toBabelSegments,
   toSegmentTuple,
+  vlqMapFromBabelDecodedMap,
   vlqMapFromTuples,
 } from '../source-map';
 
@@ -344,3 +345,47 @@ describe('vlqMapFromTuples', () => {
     expect(decode(vlqMap)).toEqual([]);
   });
 });
+
+describe('vlqMapFromBabelDecodedMap', () => {
+  test('matches vlqMapFromTuples, appending a terminator when needed', () => {
+    // Decoded format: grouped by generated line (0-based), source lines 0-based.
+    const decodedMap: BabelDecodedMap = {
+      names: ['foo'],
+      mappings: [
+        [[0, 0, 0, 0]], // gen 1:0 -> src 1:0
+        [[2, 0, 0, 4, 0]], // gen 2:2 -> src 1:4 name 'foo'
+        [[0]], // gen 3:0 generated-only
+      ],
+    };
+    // Equivalent Metro tuples (source lines 1-based) + terminator at gen 3:5.
+    const tuples: Array<MetroSourceMapSegmentTuple> = [
+      [1, 0, 1, 0],
+      [2, 2, 1, 4, 'foo'],
+      [3, 0],
+      [3, 5],
+    ];
+    expect(vlqMapFromBabelDecodedMap(decodedMap, [3, 5])).toEqual(
+      vlqMapFromTuples(tuples),
+    );
+  });
+
+  test('does not append a terminator already present', () => {
+    const decodedMap: BabelDecodedMap = {
+      names: [],
+      mappings: [[[0], [5]]],
+    };
+    expect(vlqMapFromBabelDecodedMap(decodedMap, [1, 5])).toEqual(
+      vlqMapFromTuples([
+        [1, 0],
+        [1, 5],
+      ]),
+    );
+  });
+
+  test('handles an empty decoded map (terminator only)', () => {
+    const decodedMap: BabelDecodedMap = {names: [], mappings: []};
+    expect(vlqMapFromBabelDecodedMap(decodedMap, [1, 0])).toEqual(
+      vlqMapFromTuples([[1, 0]]),
+    );
+  });
+});

packages/metro-source-map/src/source-map.js

@@ -425,6 +425,76 @@ function vlqMapFromTuples(
   return {mappings: map.mappings, names: map.names};
 }
 
+/**
+ * Encodes a `VlqMap` directly from a Babel/gen-mapping "decoded" source map
+ * (`result.decodedMap` from `@babel/generator`), without ever materialising the
+ * intermediate `Array<MetroSourceMapSegmentTuple>`.
+ *
+ * `@babel/generator` computes `decodedMap` eagerly while generating, so reusing
+ * it avoids the separate, more expensive `result.rawMappings` decode (which
+ * allocates a flat array of segment objects) plus the per-segment tuple
+ * allocation that `vlqMapFromTuples` would otherwise consume. The result is
+ * byte-identical to `vlqMapFromTuples(decoded -> tuples)`.
+ *
+ * `terminatingMapping` is a `[generatedLine1Based, generatedColumn0Based]`
+ * generated-only mapping appended at the end (matching the transform worker's
+ * `countLinesAndTerminateMap`) unless the last real mapping already sits there.
+ */
+function vlqMapFromBabelDecodedMap(
+  decodedMap: BabelDecodedMap,
+  terminatingMapping: [number, number],
+): VlqMap {
+  const generator = new Generator();
+  generator.startFile('', '', null);
+  const {mappings, names} = decodedMap;
+  let lastGeneratedLine = -1;
+  let lastGeneratedColumn = -1;
+  for (let line = 0, n = mappings.length; line < n; ++line) {
+    // Decoded mappings are grouped by generated line (0-based); Generator
+    // expects 1-based generated lines.
+    const generatedLine = line + 1;
+    const segments = mappings[line];
+    for (let i = 0, m = segments.length; i < m; ++i) {
+      const segment = segments[i];
+      const generatedColumn = segment[0];
+      switch (segment.length) {
+        case 1:
+          generator.addSimpleMapping(generatedLine, generatedColumn);
+          break;
+        case 4:
+          // Decoded source lines are 0-based; Generator expects 1-based.
+          generator.addSourceMapping(
+            generatedLine,
+            generatedColumn,
+            segment[2] + 1,
+            segment[3],
+          );
+          break;
+        case 5:
+          generator.addNamedSourceMapping(
+            generatedLine,
+            generatedColumn,
+            segment[2] + 1,
+            segment[3],
+            names[segment[4]],
+          );
+          break;
+      }
+      lastGeneratedLine = generatedLine;
+      lastGeneratedColumn = generatedColumn;
+    }
+  }
+  if (
+    lastGeneratedLine !== terminatingMapping[0] ||
+    lastGeneratedColumn !== terminatingMapping[1]
+  ) {
+    generator.addSimpleMapping(terminatingMapping[0], terminatingMapping[1]);
+  }
+  generator.endFile();
+  const map = generator.toMap();
+  return {mappings: map.mappings, names: map.names};
+}
+
 export {
   BundleBuilder,
   composeSourceMaps,
@@ -439,6 +509,7 @@ export {
   toBabelSegments,
   toSegmentTuple,
   tuplesFromBabelDecodedMap,
+  vlqMapFromBabelDecodedMap,
   vlqMapFromTuples,
 };
 

packages/metro-transform-worker/src/__tests__/index-test.js

@@ -32,6 +32,8 @@ import type {JsTransformerConfig, JsTransformOptions} from '../index';
 import typeof * as TransformerType from '../index';
 import typeof FSType from 'fs';
 
+import {vlqMapFromTuples} from 'metro-source-map';
+
 const {Buffer} = require('buffer');
 const path = require('path');
 
@@ -409,6 +411,56 @@ test('uses a reserved dependency map name and prevents it from being minified',
   `);
 });
 
+test('unstable_compactSourceMaps emits a VlqMap byte-identical to the tuple path', async () => {
+  const source = Buffer.from(
+    [
+      'function foo(aaa, bbb) {',
+      '  const ccc = aaa + bbb;',
+      '  return ccc * 2;',
+      '}',
+      'export default function entry(items) {',
+      '  return items.map(x => x.value).filter(Boolean);',
+      '}',
+      '',
+    ].join('\n'),
+    'utf8',
+  );
+
+  // Default path stores decoded tuples (line-counted + terminated).
+  const tupleResult = await Transformer.transform(
+    {...baseConfig, unstable_compactSourceMaps: false},
+    '/root',
+    'local/file.js',
+    source,
+    {...baseTransformOptions, experimentalImportSupport: true},
+  );
+  // Compact path encodes VLQ straight from Babel's decoded map (no tuples).
+  const vlqResult = await Transformer.transform(
+    {...baseConfig, unstable_compactSourceMaps: true},
+    '/root',
+    'local/file.js',
+    source,
+    {...baseTransformOptions, experimentalImportSupport: true},
+  );
+
+  const tupleMap = tupleResult.output[0].data.map;
+  const vlqMap = vlqResult.output[0].data.map;
+
+  // Generated code and line count are unaffected by map storage.
+  expect(vlqResult.output[0].data.code).toBe(tupleResult.output[0].data.code);
+  expect(vlqResult.output[0].data.lineCount).toBe(
+    tupleResult.output[0].data.lineCount,
+  );
+
+  if (Array.isArray(vlqMap) || !Array.isArray(tupleMap)) {
+    throw new Error('Expected a VlqMap (compact) and a tuple array (default)');
+  }
+  // The compact fast path is byte-identical to re-encoding the tuple output.
+  expect(vlqMap).toEqual(vlqMapFromTuples(tupleMap));
+  expect(typeof vlqMap.mappings).toBe('string');
+  expect(vlqMap.mappings.length).toBeGreaterThan(0);
+});
+
 test('throws if the reserved dependency map name appears in the input', async () => {
   await expect(
     Transformer.transform(

packages/metro-transform-worker/src/index.js

@@ -20,6 +20,7 @@ import type {
   BasicSourceMap,
   FBSourceFunctionMap,
   MetroSourceMapSegmentTuple,
+  VlqMap,
 } from 'metro-source-map';
 import type {
   ImportExportPluginOptions,
@@ -47,6 +48,8 @@ import {
   toBabelSegments,
   toSegmentTuple,
   tuplesFromBabelDecodedMap,
+  vlqMapFromBabelDecodedMap,
+  vlqMapFromTuples,
 } from 'metro-source-map';
 import metroTransformPlugins from 'metro-transform-plugins';
 import collectDependencies from 'metro/private/ModuleGraph/worker/collectDependencies';
@@ -111,6 +114,8 @@ export type JsTransformerConfig = Readonly<{
   unstable_nonMemoizedInlineRequires?: ReadonlyArray<string>,
   /** Whether to rename scoped `require` functions to `_$$_REQUIRE`, usually an extraneous operation when serializing to iife (default). */
   unstable_renameRequire?: boolean,
+  /** Store source maps as compact VLQ-encoded strings (`VlqMap`) instead of decoded tuple arrays. Reduces source-map memory ~51% on the heap. Opt-in; changes `JsOutput.data.map` for consumers. */
+  unstable_compactSourceMaps?: boolean,
 }>;
 
 export type {CustomTransformOptions} from 'metro-babel-transformer';
@@ -169,7 +174,7 @@ export type JsOutput = Readonly<{
   data: Readonly<{
     code: string,
     lineCount: number,
-    map: Array<MetroSourceMapSegmentTuple>,
+    map: Array<MetroSourceMapSegmentTuple> | VlqMap,
     functionMap: ?FBSourceFunctionMap,
   }>,
   type: JSFileType,
@@ -472,29 +477,49 @@ async function transformJS(
     file.code,
   );
 
-  // Derive tuples from Babel's eagerly-computed decoded map rather than
-  // `result.rawMappings`, which would trigger a second, more expensive decode
-  // (`allMappings`). Byte-identical to `result.rawMappings.map(toSegmentTuple)`.
-  let map = result.decodedMap
-    ? tuplesFromBabelDecodedMap(result.decodedMap)
-    : [];
   let code = result.code;
+  let map: Array<MetroSourceMapSegmentTuple> | VlqMap;
+  let lineCount: number;
+
+  if (config.unstable_compactSourceMaps === true && !minify) {
+    // Dominant path (e.g. Hermes, which doesn't minify): encode the compact VLQ
+    // map straight from Babel's eagerly-computed decoded map, never
+    // materialising tuples. Byte-identical to the tuple path below.
+    const {lineCount: lines, lastLineColumn} = countLines(code);
+    lineCount = lines;
+    map = vlqMapFromBabelDecodedMap(nullthrows(result.decodedMap), [
+      lines,
+      lastLineColumn,
+    ]);
+  } else {
+    // Derive tuples from Babel's eagerly-computed decoded map rather than
+    // `result.rawMappings`, which would trigger a second, more expensive decode
+    // (`allMappings`). Byte-identical to `result.rawMappings.map(toSegmentTuple)`.
+    let tuples = result.decodedMap
+      ? tuplesFromBabelDecodedMap(result.decodedMap)
+      : [];
+
+    if (minify) {
+      // The minifier returns its own map (not Babel's `decodedMap`), so the
+      // fast path above can't apply; re-encode the resulting tuples if compact.
+      ({map: tuples, code} = await minifyCode(
+        config,
+        projectRoot,
+        file.filename,
+        result.code,
+        file.code,
+        tuples,
+        reserved,
+      ));
+    }
 
-  if (minify) {
-    ({map, code} = await minifyCode(
-      config,
-      projectRoot,
-      file.filename,
-      result.code,
-      file.code,
-      map,
-      reserved,
-    ));
+    ({lineCount, map: tuples} = countLinesAndTerminateMap(code, tuples));
+    map =
+      config.unstable_compactSourceMaps === true
+        ? vlqMapFromTuples(tuples)
+        : tuples;
   }
 
-  let lineCount;
-  ({lineCount, map} = countLinesAndTerminateMap(code, map));
-
   const output: Array<JsOutput> = [
     {
       data: {
@@ -622,9 +647,13 @@ async function transformJSON(
 
   let lineCount;
   ({lineCount, map} = countLinesAndTerminateMap(code, map));
+  // The JSON path builds tuples directly (no Babel `decodedMap`), so when
+  // compact we re-encode the finished tuples to VLQ.
+  const outputMap =
+    config.unstable_compactSourceMaps === true ? vlqMapFromTuples(map) : map;
   const output: Array<JsOutput> = [
     {
-      data: {code, functionMap: null, lineCount, map},
+      data: {code, functionMap: null, lineCount, map: outputMap},
       type: jsType,
     },
   ];
@@ -761,12 +790,9 @@ export const getCacheKey = (
   ].join('$');
 };
 
-function countLinesAndTerminateMap(
-  code: string,
-  map: ReadonlyArray<MetroSourceMapSegmentTuple>,
-): {
+function countLines(code: string): {
   lineCount: number,
-  map: Array<MetroSourceMapSegmentTuple>,
+  lastLineColumn: number,
 } {
   const NEWLINE = /\r\n?|\n|\u2028|\u2029/g;
   let lineCount = 1;
@@ -777,9 +803,19 @@ function countLinesAndTerminateMap(
     lineCount++;
     lastLineStart = match.index + match[0].length;
   }
-  const lastLineLength = code.length - lastLineStart;
+  return {lineCount, lastLineColumn: code.length - lastLineStart};
+}
+
+function countLinesAndTerminateMap(
+  code: string,
+  map: ReadonlyArray<MetroSourceMapSegmentTuple>,
+): {
+  lineCount: number,
+  map: Array<MetroSourceMapSegmentTuple>,
+} {
+  const {lineCount, lastLineColumn} = countLines(code);
   const lastLineIndex1Based = lineCount;
-  const lastLineNextColumn0Based = lastLineLength;
+  const lastLineNextColumn0Based = lastLineColumn;
 
   // If there isn't a mapping at one-past-the-last column of the last line,
   // add one that maps to nothing. This ensures out-of-bounds lookups hit the