Optionally store source maps as VLQ-encoded (1/2): Type widening, consumer support (#1742)

Summary:
Pull Request resolved: #1742

## This stack
Decoded tuple arrays are the single largest contributor to Metro's dev-server heap on large bundles (~10 million retained small arrays on FBiOS entry bundle, for example). Storing the same data as a compact VLQ string instead removes most of that footprint.

This reduces source map memory by ~51% on the heap and ~48% RSS for that ~16K module bundle.

The emitted whole-bundle source map is unchanged. When a module's map is stored as VLQ, `fromRawMappings` decodes it back to tuples just-in-time, with request-scoped caching. The trade-off is therefore decode + re-encode CPU when a `.map` is actually requested or `/symbolicate` request is made.

A plain `string` is used for `mappings` for now, since VLQ is ASCII by design. A `UInt8Array` would be marginally more efficient and potentially transferrable to/from worker threads, but would require more invasive changes to cache (de)serialisation. I did some benchmarking with this and it doesn't justify the complexity right now.

## This diff
Adds a `VlqMap` type (`{mappings: string, names: ReadonlyArray<string>}`) as an
alternative to the current `Array<MetroSourceMapSegmentTuple>` for storing
per-module source maps in `Module` graph nodes (and transform results, and cache artifacts).

Adds the ability to store, thread, decode and (flat-)emit VLQ
maps - **nothing actually produces them yet**, so these code paths are unused except by tests. The opt-in producer flag lands in the next diff.

## Follow up
After this mini-stack, we'll add an opt-in for emitting index source maps, directly re-using per-module VLQ and eliminating the trade-off mentioned above.

Reviewed By: huntie, javache

Differential Revision: D107973884

fbshipit-source-id: 138b18c940e4598f78678daccd57eeab0fcd9a0f

Author: robhogan

packages/metro-source-map/src/__tests__/source-map-test.js

@@ -9,8 +9,16 @@
  * @oncall react_native
  */
 
+import type {MetroSourceMapSegmentTuple} from '../source-map';
+
 import Generator from '../Generator';
-import {fromRawMappings, toBabelSegments, toSegmentTuple} from '../source-map';
+import {
+  fromRawMappings,
+  isVlqMap,
+  toBabelSegments,
+  toSegmentTuple,
+  vlqMapFromTuples,
+} from '../source-map';
 
 describe('flattening mappings / compacting', () => {
   test('flattens simple mappings', () => {
@@ -167,3 +175,172 @@ describe('build map from raw mappings', () => {
 });
 
 const lines = (n: number) => Array(n).join('\n');
+
+function makeVlqMap(
+  mappings: string,
+  names: ReadonlyArray<string>,
+): {readonly mappings: string, readonly names: ReadonlyArray<string>} {
+  return {
+    mappings,
+    names,
+  };
+}
+
+describe('isVlqMap', () => {
+  test('returns false for null', () => {
+    expect(isVlqMap(null)).toBe(false);
+  });
+
+  test('returns false for tuple array', () => {
+    expect(isVlqMap([[1, 2, 3, 4]])).toBe(false);
+  });
+
+  test('returns true for VlqMap', () => {
+    expect(isVlqMap(makeVlqMap('AAAA', []))).toBe(true);
+  });
+
+  test('returns false for plain object without string mappings', () => {
+    // $FlowFixMe[incompatible-type] Testing runtime behavior with invalid type
+    expect(isVlqMap({mappings: 123, names: []})).toBe(false);
+  });
+});
+
+describe('fromRawMappings with VlqMap', () => {
+  // Shared tuple definitions. We build two parallel module lists from these —
+  // one storing decoded tuples, one storing the equivalent VLQ — and assert the
+  // serialized flat map is byte-identical, i.e. VLQ storage is transparent.
+  const tuples0: Array<MetroSourceMapSegmentTuple> = [
+    [1, 2],
+    [3, 4, 5, 6, 'apples'],
+    [7, 8, 9, 10],
+    [11, 12, 13, 14, 'pears'],
+  ];
+  const tuples1: Array<MetroSourceMapSegmentTuple> = [
+    [1, 2],
+    [3, 4, 15, 16, 'bananas'],
+  ];
+
+  const tupleModules = [
+    {
+      code: lines(11),
+      functionMap: {names: ['<global>'], mappings: 'AAA'},
+      map: tuples0,
+      source: 'code1',
+      path: 'path1',
+      isIgnored: false,
+    },
+    {
+      code: lines(3),
+      functionMap: null,
+      map: tuples1,
+      source: 'code2',
+      path: 'path2',
+      isIgnored: true,
+    },
+  ];
+
+  const vlqModules = [
+    {...tupleModules[0], map: vlqMapFromTuples(tuples0)},
+    {...tupleModules[1], map: vlqMapFromTuples(tuples1)},
+  ];
+
+  test('produces a flat (non-indexed) map for VlqMap inputs', () => {
+    const map = fromRawMappings(vlqModules).toMap();
+    expect(typeof map.mappings).toBe('string');
+    expect(map.sources).toEqual(['path1', 'path2']);
+    expect(map.version).toBe(3);
+  });
+
+  test('VlqMap input serializes byte-identically to tuple input', () => {
+    expect(fromRawMappings(vlqModules).toString()).toBe(
+      fromRawMappings(tupleModules).toString(),
+    );
+    expect(fromRawMappings(vlqModules).toMap()).toEqual(
+      fromRawMappings(tupleModules).toMap(),
+    );
+  });
+
+  test('preserves functionMap and ignoreList from VlqMap modules', () => {
+    const map = fromRawMappings(vlqModules).toMap();
+    expect(map.x_facebook_sources).toEqual([
+      [{names: ['<global>'], mappings: 'AAA'}],
+      null,
+    ]);
+    expect(map.x_google_ignoreList).toEqual([1]);
+  });
+
+  test('handles mixed tuple and VlqMap modules identically to all-tuple', () => {
+    const mixed = [tupleModules[0], vlqModules[1]];
+    expect(fromRawMappings(mixed).toString()).toBe(
+      fromRawMappings(tupleModules).toString(),
+    );
+  });
+
+  test('applies offsetLines identically for VlqMap and tuple inputs', () => {
+    expect(fromRawMappings(vlqModules, 8).toString()).toBe(
+      fromRawMappings(tupleModules, 8).toString(),
+    );
+  });
+
+  test('excludeSource option omits sourcesContent', () => {
+    const map = fromRawMappings(vlqModules).toMap(undefined, {
+      excludeSource: true,
+    });
+    expect(map.sourcesContent).toBeUndefined();
+  });
+});
+
+describe('vlqMapFromTuples', () => {
+  // Decode via Metro's existing string->tuples path, the inverse of
+  // vlqMapFromTuples.
+  const decode = (vlqMap: {
+    readonly mappings: string,
+    readonly names: ReadonlyArray<string>,
+  }) =>
+    toBabelSegments({
+      version: 3,
+      sources: [''],
+      names: [...vlqMap.names],
+      mappings: vlqMap.mappings,
+    }).map(toSegmentTuple);
+
+  test('encodes tuples into a VlqMap', () => {
+    const vlqMap = vlqMapFromTuples([
+      [1, 2],
+      [3, 4, 5, 6, 'apples'],
+      [7, 8, 9, 10],
+      [11, 12, 13, 14, 'pears'],
+    ]);
+    expect(isVlqMap(vlqMap)).toBe(true);
+    expect(typeof vlqMap.mappings).toBe('string');
+    expect(vlqMap.names).toEqual(['apples', 'pears']);
+  });
+
+  test('round-trips via toBabelSegments + toSegmentTuple', () => {
+    const tuples = [
+      [1, 2],
+      [3, 4, 5, 6, 'apples'],
+      [7, 8, 9, 10],
+      [11, 12, 13, 14, 'pears'],
+      [11, 20, 30, 40],
+    ];
+    expect(decode(vlqMapFromTuples(tuples))).toEqual(tuples);
+  });
+
+  test('round-trips multi-line, multi-segment maps', () => {
+    const tuples = [
+      [1, 0, 1, 0],
+      [1, 8, 1, 4, 'foo'],
+      [2, 0, 2, 0],
+      [3, 4, 3, 2, 'bar'],
+      [5, 0],
+    ];
+    expect(decode(vlqMapFromTuples(tuples))).toEqual(tuples);
+  });
+
+  test('encodes an empty map', () => {
+    const vlqMap = vlqMapFromTuples([]);
+    expect(vlqMap.mappings).toBe('');
+    expect(decode(vlqMap)).toEqual([]);
+  });
+});

packages/metro-source-map/src/source-map.js

@@ -21,6 +21,7 @@ import {
   generateFunctionMap,
 } from './generateFunctionMap';
 import Generator from './Generator';
+import nullthrows from 'nullthrows';
 // $FlowFixMe[untyped-import] - source-map
 import SourceMap from 'source-map';
 
@@ -53,6 +54,11 @@ export type BabelDecodedMap = {
   ...
 };
 
+export type VlqMap = {
+  readonly mappings: string,
+  readonly names: ReadonlyArray<string>,
+};
+
 export type HermesFunctionOffsets = {[number]: ReadonlyArray<number>, ...};
 
 export type FBSourcesArray = ReadonlyArray<?FBSourceMetadata>;
@@ -123,18 +129,26 @@ type SourceMapConsumerMapping = {
   name: ?string,
 };
 
+export type RawMappingsModule = {
+  readonly map: ?ReadonlyArray<MetroSourceMapSegmentTuple> | VlqMap,
+  readonly functionMap: ?FBSourceFunctionMap,
+  readonly path: string,
+  readonly source: string,
+  readonly code: string,
+  readonly isIgnored: boolean,
+  readonly lineCount?: number,
+};
+
+function isVlqMap(
+  map: ?ReadonlyArray<MetroSourceMapSegmentTuple> | VlqMap,
+): implies map is VlqMap {
+  return map != null && !Array.isArray(map) && typeof map.mappings === 'string';
+}
+
 function fromRawMappingsImpl(
   isBlocking: boolean,
   onDone: Generator => void,
-  modules: ReadonlyArray<{
-    readonly map: ?ReadonlyArray<MetroSourceMapSegmentTuple>,
-    readonly functionMap: ?FBSourceFunctionMap,
-    readonly path: string,
-    readonly source: string,
-    readonly code: string,
-    readonly isIgnored: boolean,
-    readonly lineCount?: number,
-  }>,
+  modules: ReadonlyArray<RawMappingsModule>,
   offsetLines: number,
 ): void {
   const modulesToProcess = modules.slice();
@@ -146,15 +160,18 @@ function fromRawMappingsImpl(
       return true;
     }
 
-    const mod = modulesToProcess.shift();
-    // $FlowFixMe[incompatible-use]
+    const mod = nullthrows(modulesToProcess.shift());
     const {code, map} = mod;
-    if (Array.isArray(map)) {
-      // $FlowFixMe[incompatible-type]
+    if (isVlqMap(map)) {
+      // Modules may store their map compactly as VLQ. Decode it back to tuples
+      // just-in-time so it can be folded into the flat Generator like any other
+      // module. Decoding one module at a time keeps the transient tuple arrays
+      // short-lived, preserving the memory win of VLQ storage.
+      addMappingsForFile(generator, decodeVlqMap(map), mod, carryOver);
+    } else if (Array.isArray(map)) {
       addMappingsForFile(generator, map, mod, carryOver);
     } else if (map != null) {
       throw new Error(
-        // $FlowFixMe[incompatible-use]
         `Unexpected module with full source map found: ${mod.path}`,
       );
     }
@@ -197,15 +214,7 @@ function fromRawMappingsImpl(
  * the resulting bundle, e.g. by some prefix code.
  */
 function fromRawMappings(
-  modules: ReadonlyArray<{
-    readonly map: ?ReadonlyArray<MetroSourceMapSegmentTuple>,
-    readonly functionMap: ?FBSourceFunctionMap,
-    readonly path: string,
-    readonly source: string,
-    readonly code: string,
-    readonly isIgnored: boolean,
-    readonly lineCount?: number,
-  }>,
+  modules: ReadonlyArray<RawMappingsModule>,
   offsetLines: number = 0,
 ): Generator {
   let generator: void | Generator;
@@ -224,15 +233,7 @@ function fromRawMappings(
 }
 
 async function fromRawMappingsNonBlocking(
-  modules: ReadonlyArray<{
-    readonly map: ?ReadonlyArray<MetroSourceMapSegmentTuple>,
-    readonly functionMap: ?FBSourceFunctionMap,
-    readonly path: string,
-    readonly source: string,
-    readonly code: string,
-    readonly isIgnored: boolean,
-    readonly lineCount?: number,
-  }>,
+  modules: ReadonlyArray<RawMappingsModule>,
   offsetLines: number = 0,
 ): Promise<Generator> {
   return new Promise(resolve => {
@@ -344,16 +345,8 @@ function tuplesFromBabelDecodedMap(
 
 function addMappingsForFile(
   generator: Generator,
-  mappings: Array<MetroSourceMapSegmentTuple>,
-  module: {
-    readonly code: string,
-    readonly functionMap: ?FBSourceFunctionMap,
-    readonly map: ?Array<MetroSourceMapSegmentTuple>,
-    readonly path: string,
-    readonly source: string,
-    readonly isIgnored: boolean,
-    readonly lineCount?: number,
-  },
+  mappings: ReadonlyArray<MetroSourceMapSegmentTuple>,
+  module: RawMappingsModule,
   carryOver: number,
 ) {
   generator.startFile(module.path, module.source, module.functionMap, {
@@ -400,6 +393,38 @@ const newline = /\r\n?|\n|\u2028|\u2029/g;
 const countLines = (string: string): number =>
   (string.match(newline) || []).length + 1;
 
+/**
+ * Decodes a compact VLQ map back into raw mapping tuples — the inverse of
+ * `vlqMapFromTuples`, reusing Metro's existing source-map consumer.
+ */
+function decodeVlqMap(vlqMap: VlqMap): Array<MetroSourceMapSegmentTuple> {
+  return toBabelSegments({
+    version: 3,
+    sources: [''],
+    names: [...vlqMap.names],
+    mappings: vlqMap.mappings,
+  }).map(toSegmentTuple);
+}
+
+/**
+ * Encodes raw mapping tuples into a compact VLQ `mappings` string + `names`
+ * table. Decode the inverse via `decodeVlqMap` (or `toBabelSegments` +
+ * `toSegmentTuple`). Storing maps in this form uses far less memory than the
+ * equivalent decoded tuple arrays.
+ */
+function vlqMapFromTuples(
+  mappings: ReadonlyArray<MetroSourceMapSegmentTuple>,
+): VlqMap {
+  const generator = new Generator();
+  generator.startFile('', '', null);
+  for (const mapping of mappings) {
+    addMapping(generator, mapping, 0);
+  }
+  generator.endFile();
+  const map = generator.toMap();
+  return {mappings: map.mappings, names: map.names};
+}
+
 export {
   BundleBuilder,
   composeSourceMaps,
@@ -409,10 +434,12 @@ export {
   fromRawMappings,
   fromRawMappingsNonBlocking,
   functionMapBabelPlugin,
+  isVlqMap,
   normalizeSourcePath,
   toBabelSegments,
   toSegmentTuple,
   tuplesFromBabelDecodedMap,
+  vlqMapFromTuples,
 };
 
 /**