Optionally store source maps as VLQ encoded (2/2): Transformer output, unstable_compactSourceMaps (#1743)

Summary:
Pull Request resolved: #1743

## This stack
Decoded tuple arrays are the single largest contributor to Metro's dev-server heap on large bundles (~10 million retained small arrays on FBiOS entry bundle, for example). Storing the same data as a compact VLQ string instead removes most of that footprint.

This reduces source map memory by ~51% on the heap and ~48% RSS for that ~16K module bundle.

The emitted whole-bundle source map is unchanged. When a module's map is stored as VLQ, `fromRawMappings` decodes it back to tuples just-in-time, with request-scoped caching. The trade-off is therefore decode + re-encode CPU when a `.map` is actually requested or `/symbolicate` request is made.

A plain `string` is used for `mappings` for now, since VLQ is ASCII by design. A `UInt8Array` would be marginally more efficient and potentially transferrable to/from worker threads, but would require more invasive changes to cache (de)serialisation. I did some benchmarking with this and it doesn't justify the complexity right now.

## This diff

Adds `unstable_compactSourceMaps` (default `false`). When enabled, the transform
worker stores each module's source map as a compact VLQ string (`VlqMap`)
instead of a decoded `Array<MetroSourceMapSegmentTuple>`.

Each module's map originates from one of three sources, so we encode the VLQ the
cheapest way available in each case (all byte-identical to the decoded-tuple
output):

- transformJS, not minifying (the dominant path — Hermes targets don't minify):
  encode the `VlqMap` straight from `result.decodedMap`, which `babel/generator`
  computes eagerly while generating, via `vlqMapFromBabelDecodedMap` — never
  materialising tuples.
- transformJS, minifying: the minifier returns its own map (not Babel's), so we
  re-encode the resulting tuples with `vlqMapFromTuples`.
- transformJSON: builds tuples directly (no Babel generate), so it likewise
  re-encodes with `vlqMapFromTuples`.

`countLines` is split out of `countLinesAndTerminateMap` so the decoded-map fast
path can compute the terminating mapping without building and terminating a
tuple array first.

## Benchmarks

*Cold cache (n=3, means)*

| Metric | base | compact |
|---|---|---|---|
| **Heap used** | 1653.7 MB | **809.7 MB (−51.0%)** |
| **RSS** | 1854.2 MB | 955.2 MB (−48.5%) |
| Heap growth (build) | 1606.5 MB | 761.2 MB (−52.6%) |
| Build CPU (`.bundle`) | 23.05 s | 22.42 s (n.s.) |
| **Serialize CPU (`.map`)** | 11.99 s | **14.19 s (+18.4%)** |

*Warm cache (n=3, means)*

| Metric | base | compact |
|---|---|---|---|
| **Heap used** | 1552 MB | **731 MB (−52.9%)** |
| **RSS** | 1775 MB | 923 MB (−48.0%) |
| Build CPU (`.bundle`) | 10.92 s | 8.86 s (−18.9%) |
| **Serialize CPU (`.map`)** | 11.87 s | **13.89 s (+17.0%)** |

## Why behind a flag?

1) The `map` structure is exposed to custom serialisers, so changing it is semver-breaking. Landing this as experimental opt-in in a non-breaking release allows integrators to experiment with it.
2) This is a trade-off of retained memory vs CPU required to emit a flat source map or symbolicate errors. The trade-off largely goes away with indexed maps (coming next) - but that is a semver-breaking change to output.

Changelog:
```
 - **[Experimental]**: Add `unstable_compactSourceMaps` to use a more memory-efficient source map format.
```

Reviewed By: huntie

Differential Revision: D109216060

fbshipit-source-id: c3cfbc97e5e46e8af86aefb2116d8d1365260406

Author: robhogan

packages/metro-config/src/defaults/index.js

@@ -135,6 +135,7 @@ const getDefaultValues = (projectRoot: ?string): ConfigT => ({
     unstable_disableNormalizePseudoGlobals: false,
     unstable_renameRequire: true,
     unstable_compactOutput: false,
+    unstable_compactSourceMaps: false,
     unstable_memoizeInlineRequires: false,
     unstable_workerThreads: false,
   },

packages/metro-source-map/src/__tests__/source-map-test.js

@@ -9,14 +9,15 @@
  * @oncall react_native
  */
 
-import type {MetroSourceMapSegmentTuple} from '../source-map';
+import type {BabelDecodedMap, MetroSourceMapSegmentTuple} from '../source-map';
 
 import Generator from '../Generator';
 import {
   fromRawMappings,
   isVlqMap,
   toBabelSegments,
   toSegmentTuple,
+  vlqMapFromBabelDecodedMap,
   vlqMapFromTuples,
 } from '../source-map';
 
@@ -344,3 +345,47 @@ describe('vlqMapFromTuples', () => {
     expect(decode(vlqMap)).toEqual([]);
   });
 });
+
+describe('vlqMapFromBabelDecodedMap', () => {
+  test('matches vlqMapFromTuples, appending a terminator when needed', () => {
+    // Decoded format: grouped by generated line (0-based), source lines 0-based.
+    const decodedMap: BabelDecodedMap = {
+      names: ['foo'],
+      mappings: [
+        [[0, 0, 0, 0]], // gen 1:0 -> src 1:0
+        [[2, 0, 0, 4, 0]], // gen 2:2 -> src 1:4 name 'foo'
+        [[0]], // gen 3:0 generated-only
+      ],
+    };
+    // Equivalent Metro tuples (source lines 1-based) + terminator at gen 3:5.
+    const tuples: Array<MetroSourceMapSegmentTuple> = [
+      [1, 0, 1, 0],
+      [2, 2, 1, 4, 'foo'],
+      [3, 0],
+      [3, 5],
+    ];
+    expect(vlqMapFromBabelDecodedMap(decodedMap, [3, 5])).toEqual(
+      vlqMapFromTuples(tuples),
+    );
+  });
+
+  test('does not append a terminator already present', () => {
+    const decodedMap: BabelDecodedMap = {
+      names: [],
+      mappings: [[[0], [5]]],
+    };
+    expect(vlqMapFromBabelDecodedMap(decodedMap, [1, 5])).toEqual(
+      vlqMapFromTuples([
+        [1, 0],
+        [1, 5],
+      ]),
+    );
+  });
+
+  test('handles an empty decoded map (terminator only)', () => {
+    const decodedMap: BabelDecodedMap = {names: [], mappings: []};
+    expect(vlqMapFromBabelDecodedMap(decodedMap, [1, 0])).toEqual(
+      vlqMapFromTuples([[1, 0]]),
+    );
+  });
+});

packages/metro-source-map/src/source-map.js

@@ -425,6 +425,78 @@ function vlqMapFromTuples(
   return {mappings: map.mappings, names: map.names};
 }
 
+/**
+ * Encodes a `VlqMap` directly from a Babel/gen-mapping "decoded" source map
+ * (`result.decodedMap` from `@babel/generator`), without ever materialising the
+ * intermediate `Array<MetroSourceMapSegmentTuple>`.
+ *
+ * `@babel/generator` computes `decodedMap` eagerly while generating, so reusing
+ * it avoids the separate, more expensive `result.rawMappings` decode (which
+ * allocates a flat array of segment objects) plus the per-segment tuple
+ * allocation that `vlqMapFromTuples` would otherwise consume. The result is
+ * byte-identical to `vlqMapFromTuples(decoded -> tuples)`.
+ *
+ * `terminatingMapping` is a `[generatedLine1Based, generatedColumn0Based]`
+ * generated-only mapping appended at the end (matching the transform worker's
+ * `countLinesAndTerminateMap`) unless the last real mapping already sits there.
+ */
+function vlqMapFromBabelDecodedMap(
+  decodedMap: BabelDecodedMap,
+  terminatingMapping: [number, number],
+): VlqMap {
+  const generator = new Generator();
+  generator.startFile('', '', null);
+  const {mappings, names} = decodedMap;
+  let lastGeneratedLine = -1;
+  let lastGeneratedColumn = -1;
+  for (let line = 0, n = mappings.length; line < n; ++line) {
+    // Decoded mappings are grouped by generated line (0-based); Generator
+    // expects 1-based generated lines.
+    const generatedLine = line + 1;
+    const segments = mappings[line];
+    for (let i = 0, m = segments.length; i < m; ++i) {
+      const segment = segments[i];
+      const generatedColumn = segment[0];
+      switch (segment.length) {
+        case 1:
+          generator.addSimpleMapping(generatedLine, generatedColumn);
+          break;
+        case 4:
+          // Decoded source lines are 0-based; Generator expects 1-based.
+          generator.addSourceMapping(
+            generatedLine,
+            generatedColumn,
+            segment[2] + 1,
+            segment[3],
+          );
+          break;
+        case 5:
+          generator.addNamedSourceMapping(
+            generatedLine,
+            generatedColumn,
+            segment[2] + 1,
+            segment[3],
+            names[segment[4]],
+          );
+          break;
+        default:
+          throw new Error(`Invalid mapping: [${segment.join(', ')}]`);
+      }
+      lastGeneratedLine = generatedLine;
+      lastGeneratedColumn = generatedColumn;
+    }
+  }
+  if (
+    lastGeneratedLine !== terminatingMapping[0] ||
+    lastGeneratedColumn !== terminatingMapping[1]
+  ) {
+    generator.addSimpleMapping(terminatingMapping[0], terminatingMapping[1]);
+  }
+  generator.endFile();
+  const map = generator.toMap();
+  return {mappings: map.mappings, names: map.names};
+}
+
 export {
   BundleBuilder,
   composeSourceMaps,
@@ -439,6 +511,7 @@ export {
   toBabelSegments,
   toSegmentTuple,
   tuplesFromBabelDecodedMap,
+  vlqMapFromBabelDecodedMap,
   vlqMapFromTuples,
 };
 

packages/metro-source-map/types/source-map.d.ts

@@ -6,7 +6,7 @@
  *
  * @noformat
  * @oncall react_native
- * @generated SignedSource<<313a3bbbf29c3ac69821b3124678d4e0>>
+ * @generated SignedSource<<9ec89353742743678e422f0bf81e488d>>
  *
  * This file was translated from Flow by scripts/generateTypeScriptDefinitions.js
  * Original file: packages/metro-source-map/src/source-map.js
@@ -157,6 +157,25 @@ declare function tuplesFromBabelDecodedMap(
 declare function vlqMapFromTuples(
   mappings: ReadonlyArray<MetroSourceMapSegmentTuple>,
 ): VlqMap;
+/**
+ * Encodes a `VlqMap` directly from a Babel/gen-mapping "decoded" source map
+ * (`result.decodedMap` from `@babel/generator`), without ever materialising the
+ * intermediate `Array<MetroSourceMapSegmentTuple>`.
+ *
+ * `@babel/generator` computes `decodedMap` eagerly while generating, so reusing
+ * it avoids the separate, more expensive `result.rawMappings` decode (which
+ * allocates a flat array of segment objects) plus the per-segment tuple
+ * allocation that `vlqMapFromTuples` would otherwise consume. The result is
+ * byte-identical to `vlqMapFromTuples(decoded -> tuples)`.
+ *
+ * `terminatingMapping` is a `[generatedLine1Based, generatedColumn0Based]`
+ * generated-only mapping appended at the end (matching the transform worker's
+ * `countLinesAndTerminateMap`) unless the last real mapping already sits there.
+ */
+declare function vlqMapFromBabelDecodedMap(
+  decodedMap: BabelDecodedMap,
+  terminatingMapping: [number, number],
+): VlqMap;
 export {
   BundleBuilder,
   composeSourceMaps,
@@ -171,6 +190,7 @@ export {
   toBabelSegments,
   toSegmentTuple,
   tuplesFromBabelDecodedMap,
+  vlqMapFromBabelDecodedMap,
   vlqMapFromTuples,
 };
 /**

packages/metro-transform-worker/src/__tests__/index-test.js

@@ -32,6 +32,8 @@ import type {JsTransformerConfig, JsTransformOptions} from '../index';
 import typeof * as TransformerType from '../index';
 import typeof FSType from 'fs';
 
+import {vlqMapFromTuples} from 'metro-source-map';
+
 const {Buffer} = require('buffer');
 const path = require('path');
 
@@ -409,6 +411,56 @@ test('uses a reserved dependency map name and prevents it from being minified',
   `);
 });
 
+test('unstable_compactSourceMaps emits a VlqMap byte-identical to the tuple path', async () => {
+  const source = Buffer.from(
+    [
+      'function foo(aaa, bbb) {',
+      '  const ccc = aaa + bbb;',
+      '  return ccc * 2;',
+      '}',
+      'export default function entry(items) {',
+      '  return items.map(x => x.value).filter(Boolean);',
+      '}',
+      '',
+    ].join('\n'),
+    'utf8',
+  );
+
+  // Default path stores decoded tuples (line-counted + terminated).
+  const tupleResult = await Transformer.transform(
+    {...baseConfig, unstable_compactSourceMaps: false},
+    '/root',
+    'local/file.js',
+    source,
+    {...baseTransformOptions, experimentalImportSupport: true},
+  );
+  // Compact path encodes VLQ straight from Babel's decoded map (no tuples).
+  const vlqResult = await Transformer.transform(
+    {...baseConfig, unstable_compactSourceMaps: true},
+    '/root',
+    'local/file.js',
+    source,
+    {...baseTransformOptions, experimentalImportSupport: true},
+  );
+
+  const tupleMap = tupleResult.output[0].data.map;
+  const vlqMap = vlqResult.output[0].data.map;
+
+  // Generated code and line count are unaffected by map storage.
+  expect(vlqResult.output[0].data.code).toBe(tupleResult.output[0].data.code);
+  expect(vlqResult.output[0].data.lineCount).toBe(
+    tupleResult.output[0].data.lineCount,
+  );
+
+  if (Array.isArray(vlqMap) || !Array.isArray(tupleMap)) {
+    throw new Error('Expected a VlqMap (compact) and a tuple array (default)');
+  }
+  // The compact fast path is byte-identical to re-encoding the tuple output.
+  expect(vlqMap).toEqual(vlqMapFromTuples(tupleMap));
+  expect(typeof vlqMap.mappings).toBe('string');
+  expect(vlqMap.mappings.length).toBeGreaterThan(0);
+});
+
 test('throws if the reserved dependency map name appears in the input', async () => {
   await expect(
     Transformer.transform(