Change the raw stack table to store the prefix as an index offset.

Replace `RawStackTable.prefix` with a `prefixOffset` column. For each
stack i, `prefixOffset[i] === 0` means i is a root, and `prefixOffset[i]
=== k` (with k > 0) means i's parent is at index i - k. All values are
non-negative because the stack table is stored in topological order.

The motivation is profile size: relative offsets are typically small numbers
(stack depth) instead of large absolute indexes, so they compress better in
the on-wire JSON. The derived `StackTable.prefix` keeps its absolute Int32Array
representation for consumers that prefer that form.

Author: mstange

docs-developer/CHANGELOG-formats.md

@@ -6,6 +6,16 @@ Note that this is not an exhaustive list. Processed profile format upgraders can
 
 ## Processed profile format
 
+### Version 67
+
+The `prefix` column of `profile.shared.stackTable` was replaced with a
+`prefixOffset` column. For each stack `i`, `prefixOffset[i] === 0` means
+`i` is a root, and `prefixOffset[i] === k` (with `k > 0`) means `i`'s parent
+is at index `i - k`. All values are non-negative because the stack table is
+stored in topological order. In-memory the column is an `Int32Array`. The
+motivation is to shrink the encoded prefix values for large profiles, so that
+they compress better in the on-wire JSON.
+
 ### Version 66
 
 The `prefix` column of `profile.shared.stackTable` now uses `-1` instead of `null`

src/app-logic/constants.ts

@@ -12,7 +12,7 @@ export const GECKO_PROFILE_VERSION = 34;
 // The current version of the "processed" profile format.
 // Please don't forget to update the processed profile format changelog in
 // `docs-developer/CHANGELOG-formats.md`.
-export const PROCESSED_PROFILE_VERSION = 66;
+export const PROCESSED_PROFILE_VERSION = 67;
 
 // The following are the margin sizes for the left and right of the timeline. Independent
 // components need to share these values.

src/app-logic/url-handling.ts

@@ -1654,7 +1654,8 @@ function getStackIndexFromVersion3JSCallNodePath(
   stackIndexDepth.set(-1, -1);
 
   for (let stackIndex = 0; stackIndex < stackTable.length; stackIndex++) {
-    const prefix = stackTable.prefix[stackIndex];
+    const offset = stackTable.prefixOffset[stackIndex];
+    const prefix = offset === 0 ? -1 : stackIndex - offset;
     const frameIndex = stackTable.frame[stackIndex];
     const funcIndex = frameTable.func[frameIndex];
     const isJS = funcTable.isJS[funcIndex];
@@ -1703,7 +1704,8 @@ function getVersion4JSCallNodePathFromStackIndex(
     if (funcTable.isJS[funcIndex] || funcTable.relevantForJS[funcIndex]) {
       callNodePath.unshift(funcIndex);
     }
-    nextStackIndex = stackTable.prefix[nextStackIndex];
+    const offset = stackTable.prefixOffset[nextStackIndex];
+    nextStackIndex = offset === 0 ? -1 : nextStackIndex - offset;
   }
   return callNodePath;
 }

src/profile-logic/data-structures.ts

@@ -72,8 +72,8 @@ export function getRawStackTableBuilderWithExistingContents(
 ): RawStackTableBuilder {
   const prefix = new Array<IndexIntoStackTable | null>(existing.length);
   for (let i = 0; i < existing.length; i++) {
-    const p = existing.prefix[i];
-    prefix[i] = p === -1 ? null : p;
+    const offset = existing.prefixOffset[i];
+    prefix[i] = offset === 0 ? null : i - offset;
   }
   return {
     frame: [...existing.frame],
@@ -86,14 +86,14 @@ export function finishRawStackTableBuilder(
   builder: RawStackTableBuilder
 ): RawStackTable {
   const { frame, prefix, length } = builder;
-  const prefixCol = new Int32Array(length);
+  const prefixOffset = new Int32Array(length);
   for (let i = 0; i < length; i++) {
     const p = prefix[i];
-    prefixCol[i] = p === null ? -1 : p;
+    prefixOffset[i] = p === null ? 0 : i - p;
   }
   return {
     frame: new Int32Array(frame),
-    prefix: prefixCol,
+    prefixOffset,
     length,
   };
 }

src/profile-logic/insert-stack-labels.ts

@@ -196,10 +196,10 @@ export function insertStackLabels(
   );
   let stacksToInsertCount = 0;
   for (let stackIndex = 0; stackIndex < oldStackTable.length; stackIndex++) {
-    const parentStackIndex = oldStackTable.prefix[stackIndex];
+    const prefixOffset = oldStackTable.prefixOffset[stackIndex];
     const inheritedLabelFrameIndex =
-      parentStackIndex !== -1
-        ? inheritedLabelFrameIndexAtStack[parentStackIndex]
+      prefixOffset !== 0
+        ? inheritedLabelFrameIndexAtStack[stackIndex - prefixOffset]
         : null;
     const frameIndex = oldStackTable.frame[stackIndex];
     const funcIndex = oldFrameTable.func[frameIndex];
@@ -217,7 +217,7 @@ export function insertStackLabels(
     ) {
       labelFrameIndexToInsertAtStack[stackIndex] = null;
       inheritedLabelFrameIndexAtStack[stackIndex] = null;
-    } else if (parentStackIndex === -1) {
+    } else if (prefixOffset === 0) {
       labelFrameIndexToInsertAtStack[stackIndex] = rootLabelFrameIndex;
       inheritedLabelFrameIndexAtStack[stackIndex] = rootLabelFrameIndex;
       stacksToInsertCount++;
@@ -229,7 +229,7 @@ export function insertStackLabels(
 
   // Now compute the new stack table.
   const newStackCount = oldStackTable.length + stacksToInsertCount;
-  const newPrefixCol = new Int32Array(newStackCount);
+  const newPrefixOffsetCol = new Int32Array(newStackCount);
   const newFrameCol = new Array<IndexIntoFrameTable>(newStackCount);
   const oldStackToNewStackPlusOne = new Int32Array(oldStackTable.length);
   let nextNewStackIndex = 0;
@@ -240,18 +240,22 @@ export function insertStackLabels(
   ) {
     const labelFrameIndexToInsert =
       labelFrameIndexToInsertAtStack[oldStackIndex];
-    const oldPrefix = oldStackTable.prefix[oldStackIndex];
+    const oldPrefixOffset = oldStackTable.prefixOffset[oldStackIndex];
+    const oldPrefix =
+      oldPrefixOffset !== 0 ? oldStackIndex - oldPrefixOffset : -1;
     let newPrefix =
       oldPrefix !== -1 ? oldStackToNewStackPlusOne[oldPrefix] - 1 : -1;
     const frameIndex = oldStackTable.frame[oldStackIndex];
     if (labelFrameIndexToInsert !== null) {
       const insertedStackIndex = nextNewStackIndex++;
-      newPrefixCol[insertedStackIndex] = newPrefix;
+      newPrefixOffsetCol[insertedStackIndex] =
+        newPrefix === -1 ? 0 : insertedStackIndex - newPrefix;
       newFrameCol[insertedStackIndex] = labelFrameIndexToInsert;
       newPrefix = insertedStackIndex;
     }
     const newStackIndex = nextNewStackIndex++;
-    newPrefixCol[newStackIndex] = newPrefix;
+    newPrefixOffsetCol[newStackIndex] =
+      newPrefix === -1 ? 0 : newStackIndex - newPrefix;
     newFrameCol[newStackIndex] = frameIndex;
     oldStackToNewStackPlusOne[oldStackIndex] = newStackIndex + 1;
   }
@@ -265,7 +269,7 @@ export function insertStackLabels(
   }
 
   const stackTable: RawStackTable = {
-    prefix: newPrefixCol,
+    prefixOffset: newPrefixOffsetCol,
     frame: newFrameCol,
     length: newStackCount,
   };

src/profile-logic/merge-compare.ts

@@ -1130,7 +1130,8 @@ function mergeStackTables(
         stackTable.frame[i],
         oldFrameToNewFramePlusOne
       );
-      const prefix = stackTable.prefix[i];
+      const offset = stackTable.prefixOffset[i];
+      const prefix = offset === 0 ? -1 : i - offset;
       const newPrefix =
         prefix === -1 ? null : oldStackToNewStackPlusOne[prefix] - 1;
 

src/profile-logic/processed-profile-versioning.ts

@@ -112,8 +112,8 @@ export function attemptToUpgradeProcessedProfileThroughMutation(
 // arrays. Idempotent; safe to run on an already-normalized profile.
 function _normalizeAfterUpgrade(profile: any): void {
   const stackTable = profile.shared?.stackTable ?? null;
-  if (stackTable && !(stackTable.prefix instanceof Int32Array)) {
-    stackTable.prefix = new Int32Array(stackTable.prefix);
+  if (stackTable && !(stackTable.prefixOffset instanceof Int32Array)) {
+    stackTable.prefixOffset = new Int32Array(stackTable.prefixOffset);
   }
 }
 
@@ -3284,6 +3284,23 @@ const _upgraders: {
       stackTable.prefix = newPrefix;
     }
   },
+  [67]: (profile: any) => {
+    // The stackTable.prefix column was replaced with a stackTable.prefixOffset
+    // column. For each stack i, prefixOffset[i] is 0 if i is a root, otherwise
+    // it is i's offset from its parent (parent index = i - prefixOffset[i]).
+    const stackTable = profile.shared?.stackTable ?? null;
+    if (stackTable && stackTable.prefix !== undefined) {
+      const oldPrefix = stackTable.prefix;
+      const length = oldPrefix.length;
+      const prefixOffset = new Int32Array(length);
+      for (let i = 0; i < length; i++) {
+        const p = oldPrefix[i];
+        prefixOffset[i] = p === -1 || p === null ? 0 : i - p;
+      }
+      stackTable.prefixOffset = prefixOffset;
+      delete stackTable.prefix;
+    }
+  },
   // If you add a new upgrader here, please document the change in
   // `docs-developer/CHANGELOG-formats.md`.
 };

src/profile-logic/profile-compacting.ts

@@ -52,6 +52,7 @@ type ColumnDescription<TCol> = null extends (
     ?
         | { type: 'INDEX_REF_INT32'; referencedTable: TableCompactionState }
         | { type: 'SELF_INDEX_REF_OR_NEG_ONE_INT32' }
+        | { type: 'SELF_RELATIVE_PARENT' }
         | { type: 'NO_REF' }
     :
         | { type: 'INDEX_REF'; referencedTable: TableCompactionState }
@@ -88,6 +89,7 @@ const ColDesc = {
   selfIndexRefOrNegOneInt32: () => ({
     type: 'SELF_INDEX_REF_OR_NEG_ONE_INT32' as const,
   }),
+  selfPrefixOffset: () => ({ type: 'SELF_RELATIVE_PARENT' as const }),
   noRef: () => ({ type: 'NO_REF' as const }),
 };
 
@@ -161,7 +163,7 @@ export function computeCompactedProfile(
 
   const stackTableDesc: TableDescription<RawStackTable> = {
     frame: ColDesc.indexRefInt32(tcs.frameTable),
-    prefix: ColDesc.selfIndexRefOrNegOneInt32(),
+    prefixOffset: ColDesc.selfPrefixOffset(),
   };
   const frameTableDesc: TableDescription<FrameTable> = {
     address: ColDesc.noRef(),
@@ -334,6 +336,8 @@ function _markTableAndComputeTranslation<T>(
       markSelfColumnWithNullableFields((table as any)[key], markBuffer);
     } else if (desc.type === 'SELF_INDEX_REF_OR_NEG_ONE_INT32') {
       markSelfColumnWithNegOneFieldsInt32((table as any)[key], markBuffer);
+    } else if (desc.type === 'SELF_RELATIVE_PARENT') {
+      markSelfColumnPrefixOffset((table as any)[key], markBuffer);
     }
   }
 
@@ -355,6 +359,7 @@ function _markTableAndComputeTranslation<T>(
         break;
       case 'SELF_INDEX_REF_OR_NULL':
       case 'SELF_INDEX_REF_OR_NEG_ONE_INT32':
+      case 'SELF_RELATIVE_PARENT':
         break; // already handled in the first pass
       case 'INDEX_REF_OR_NEG_ONE':
         markColumnWithNegOneableFields(
@@ -431,6 +436,20 @@ function markSelfColumnWithNegOneFieldsInt32(col: Int32Array, markBuf: BitSet) {
   }
 }
 
+function markSelfColumnPrefixOffset(
+  col: Int32Array<ArrayBuffer>,
+  markBuf: BitSet
+) {
+  for (let i = col.length - 1; i >= 0; i--) {
+    if (checkBit(markBuf, i)) {
+      const offset = col[i];
+      if (offset !== 0) {
+        setBit(markBuf, i - offset);
+      }
+    }
+  }
+}
+
 function markColumnWithNegOneableFields(
   col: Array<number | -1>,
   shouldMark: BitSet,
@@ -567,6 +586,14 @@ function _compactTable<T extends { length: number }>(
           newLength
         );
         break;
+      case 'SELF_RELATIVE_PARENT':
+        result[key] = _compactColSelfPrefixOffset(
+          oldCol,
+          markBuffer,
+          thisTableCompactionState.oldIndexToNewIndexPlusOne,
+          newLength
+        );
+        break;
       case 'INDEX_REF_OR_NEG_ONE':
         result[key] = _compactColIndexOrNegOne(
           oldCol,
@@ -684,6 +711,29 @@ function _compactColIndexOrNegOneInt32(
   return newCol;
 }
 
+function _compactColSelfPrefixOffset(
+  oldCol: Int32Array<ArrayBuffer>,
+  markBuffer: BitSet,
+  oldIndexToNewIndexPlusOne: Int32Array,
+  newLength: number
+): Int32Array<ArrayBuffer> {
+  const newCol = new Int32Array(newLength);
+  let newIndex = 0;
+  for (let i = 0; i < oldCol.length; i++) {
+    if (checkBit(markBuffer, i)) {
+      const offset = oldCol[i];
+      if (offset === 0) {
+        newCol[newIndex] = 0;
+      } else {
+        const newParent = oldIndexToNewIndexPlusOne[i - offset] - 1;
+        newCol[newIndex] = newIndex - newParent;
+      }
+      newIndex++;
+    }
+  }
+  return newCol;
+}
+
 function _createCompactedThread(
   thread: RawThread,
   tcs: TableCompactionStates,

src/profile-logic/profile-data.ts

@@ -1993,7 +1993,7 @@ export function hasUsefulSamples(
     // All samples were null.
     return false;
   }
-  if (stackTable.prefix[stackIndex] === -1) {
+  if (stackTable.prefixOffset[stackIndex] === 0) {
     // There's only a single stack frame, check if it's '(root)'.
     const frameIndex = stackTable.frame[stackIndex];
     const funcIndex = frameTable.func[frameIndex];
@@ -4081,11 +4081,7 @@ export function collectSourceIndicesFromThreads(
 
   const addStackChain = (stackIndex: IndexIntoStackTable | null) => {
     let current = stackIndex;
-    while (
-      current !== null &&
-      current !== -1 &&
-      !checkBit(visitedStacks, current)
-    ) {
+    while (current !== null && !checkBit(visitedStacks, current)) {
       setBit(visitedStacks, current);
       const frameIndex = stackTable.frame[current];
       const funcIndex = frameTable.func[frameIndex];
@@ -4105,7 +4101,8 @@ export function collectSourceIndicesFromThreads(
         sourceIndices.add(sourceLocationTable.source[funcOriginalLocationIdx]);
       }
 
-      current = stackTable.prefix[current];
+      const prefixOffset = stackTable.prefixOffset[current];
+      current = prefixOffset !== 0 ? current - prefixOffset : null;
     }
   };
 
@@ -4289,8 +4286,12 @@ export function nudgeReturnAddresses(profile: Profile): Profile {
     }
   }
   for (let stack = 0; stack < stackTable.length; stack++) {
-    const prefix = stackTable.prefix[stack];
-    if (prefix === -1 || prefixStacks.has(prefix)) {
+    const offset = stackTable.prefixOffset[stack];
+    if (offset === 0) {
+      continue;
+    }
+    const prefix = stack - offset;
+    if (prefixStacks.has(prefix)) {
       continue;
     }
     prefixStacks.add(prefix);
@@ -4363,7 +4364,8 @@ export function nudgeReturnAddresses(profile: Profile): Profile {
   const prefixMap = new Uint32Array(stackTable.length);
   for (let stack = 0; stack < stackTable.length; stack++) {
     const frame = stackTable.frame[stack];
-    const prefix = stackTable.prefix[stack];
+    const offset = stackTable.prefixOffset[stack];
+    const prefix = offset === 0 ? -1 : stack - offset;
 
     const newPrefix = prefix === -1 ? null : prefixMap[prefix];
 
@@ -4747,7 +4749,12 @@ export function computeStackTableFromRawStackTable(
     maxSubcategoryCount < 256
       ? new Uint8Array(rawStackTable.length)
       : new Uint16Array(rawStackTable.length);
+  const prefix = new Int32Array(rawStackTable.length);
   for (let stackIndex = 0; stackIndex < rawStackTable.length; stackIndex++) {
+    const offset = rawStackTable.prefixOffset[stackIndex];
+    const prefixStack = offset === 0 ? -1 : stackIndex - offset;
+    prefix[stackIndex] = prefixStack;
+
     const frameIndex = rawStackTable.frame[stackIndex];
     const frameCategory = frameTable.category[frameIndex];
     const frameSubcategory = frameTable.subcategory[frameIndex];
@@ -4756,17 +4763,14 @@ export function computeStackTableFromRawStackTable(
     if (frameCategory !== null) {
       stackCategory = frameCategory;
       stackSubcategory = frameSubcategory || 0;
+    } else if (prefixStack !== -1) {
+      // Because of the structure of the stack table, prefix < stackIndex.
+      // So we've already computed the category for the prefix.
+      stackCategory = categoryColumn[prefixStack];
+      stackSubcategory = subcategoryColumn[prefixStack];
     } else {
-      const prefix = rawStackTable.prefix[stackIndex];
-      if (prefix !== -1) {
-        // Because of the structure of the stack table, prefix < stackIndex.
-        // So we've already computed the category for the prefix.
-        stackCategory = categoryColumn[prefix];
-        stackSubcategory = subcategoryColumn[prefix];
-      } else {
-        stackCategory = defaultCategory;
-        stackSubcategory = 0;
-      }
+      stackCategory = defaultCategory;
+      stackSubcategory = 0;
     }
     categoryColumn[stackIndex] = stackCategory;
     subcategoryColumn[stackIndex] = stackSubcategory;
@@ -4782,7 +4786,7 @@ export function computeStackTableFromRawStackTable(
     frame,
     category: categoryColumn,
     subcategory: subcategoryColumn,
-    prefix: rawStackTable.prefix,
+    prefix,
     length: rawStackTable.length,
   };
 }