Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 41 additions & 37 deletions packages/producer/src/services/hdrCompositor.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
// fallow-ignore-file complexity code-duplication
/**
* HDR Compositor — pixel-level compositing primitives for the HDR
* layered render path.
Expand Down Expand Up @@ -66,6 +65,22 @@ function countNonZeroRgb48(buf: Uint8Array): number {
return n;
}

// ─── Constants ────────────────────────────────────────────────────────────

const TRANSFORM_IDENTITY_EPSILON = 0.001;
const OPAQUE_ALPHA_THRESHOLD = 0.999;
const RGB48_BYTES_PER_PIXEL = 6;

type AffineMatrix = [number, number, number, number, number, number];

function isAffineMatrix(m: number[]): m is AffineMatrix {
return m.length === 6;
}

function resolveBlitOpacity(opacity: number): number | undefined {
return opacity < OPAQUE_ALPHA_THRESHOLD ? opacity : undefined;
}

// ─── Types ─────────────────────────────────────────────────────────────────

/**
Expand Down Expand Up @@ -96,14 +111,13 @@ function cropRgb48le(
cropW: number,
cropH: number,
): Buffer {
const BPP = 6;
const dst = Buffer.alloc(cropW * cropH * BPP);
const dst = Buffer.alloc(cropW * cropH * RGB48_BYTES_PER_PIXEL);
for (let row = 0; row < cropH; row++) {
const srcRow = cropY + row;
if (srcRow < 0 || srcRow >= srcH) continue;
const srcOff = (srcRow * srcW + cropX) * BPP;
const dstOff = row * cropW * BPP;
const copyLen = Math.min(cropW, srcW - cropX) * BPP;
const srcOff = (srcRow * srcW + cropX) * RGB48_BYTES_PER_PIXEL;
const dstOff = row * cropW * RGB48_BYTES_PER_PIXEL;
const copyLen = Math.min(cropW, srcW - cropX) * RGB48_BYTES_PER_PIXEL;
if (copyLen > 0) src.copy(dst, dstOff, srcOff, srcOff + copyLen);
}
return dst;
Expand Down Expand Up @@ -138,6 +152,7 @@ export function closeHdrVideoFrameSource(source: HdrVideoFrameSource, log?: Prod
}
}

// fallow-ignore-next-line complexity
export function blitHdrVideoLayer(
canvas: Buffer,
el: ElementStackingInfo,
Expand Down Expand Up @@ -184,18 +199,13 @@ export function blitHdrVideoLayer(
);
}

const viewportMatrix = parseTransformMatrix(el.transform);
const rawMatrix = parseTransformMatrix(el.transform);
const matrix = rawMatrix && isAffineMatrix(rawMatrix) ? rawMatrix : null;

// Pass border-radius for rounded-corner masking (only when non-zero)
const br = el.borderRadius;
const hasBorderRadius = br[0] > 0 || br[1] > 0 || br[2] > 0 || br[3] > 0;
const borderRadiusParam = hasBorderRadius ? br : undefined;

// Apply ancestor overflow:hidden clip rect by constraining the blit
// bounds. For the no-transform (region) path, we crop the source
// image and adjust the destination position. For the affine path,
// clip rect support is not yet implemented (would require per-pixel
// scissor in the affine blit); log a warning and skip clipping.
let blitX = el.x;
let blitY = el.y;
let blitSrcX = 0;
Expand All @@ -210,7 +220,7 @@ export function blitHdrVideoLayer(
const cy1 = Math.max(blitY, cr.y);
const cx2 = Math.min(blitX + blitW, cr.x + cr.width);
const cy2 = Math.min(blitY + blitH, cr.y + cr.height);
if (cx2 <= cx1 || cy2 <= cy1) return; // fully clipped
if (cx2 <= cx1 || cy2 <= cy1) return;
blitSrcX = cx1 - blitX;
blitSrcY = cy1 - blitY;
blitW = cx2 - cx1;
Expand All @@ -220,23 +230,16 @@ export function blitHdrVideoLayer(
clipped = true;
}

// Detect translation-only matrix (no scale/rotation) — route through the
// region path which supports clip rects. Chrome reports a viewport matrix
// for all HDR elements, even untransformed ones or those with only layout
// translation (e.g. `left: 960px` → `matrix(1,0,0,1,960,0)`). The region
// blit handles translation via el.x/el.y, so we only need the affine path
// for actual scale/rotation transforms.
// parseTransformMatrix returns a 6-element array or null — length check unnecessary.
const isTranslationOnly = !!(
viewportMatrix &&
Math.abs(viewportMatrix[0]! - 1) < 0.001 &&
Math.abs(viewportMatrix[1]!) < 0.001 &&
Math.abs(viewportMatrix[2]!) < 0.001 &&
Math.abs(viewportMatrix[3]! - 1) < 0.001
matrix &&
Math.abs(matrix[0] - 1) < TRANSFORM_IDENTITY_EPSILON &&
Math.abs(matrix[1]) < TRANSFORM_IDENTITY_EPSILON &&
Math.abs(matrix[2]) < TRANSFORM_IDENTITY_EPSILON &&
Math.abs(matrix[3] - 1) < TRANSFORM_IDENTITY_EPSILON
);

timeHdrPhase(hdrPerf, "hdrVideoBlitMs", () => {
if (viewportMatrix && !isTranslationOnly) {
if (matrix && !isTranslationOnly) {
if (clipped && log) {
log.debug(
`HDR clip rect on affine-transformed element ${el.id} — clip not applied (affine scissor not yet supported)`,
Expand All @@ -245,16 +248,15 @@ export function blitHdrVideoLayer(
blitRgb48leAffine(
canvas,
hdrRgb,
viewportMatrix,
matrix,
srcW,
srcH,
width,
height,
el.opacity < 0.999 ? el.opacity : undefined,
resolveBlitOpacity(el.opacity),
borderRadiusParam,
);
} else if (clipped) {
// Crop the source buffer to the clipped region before blitting
const croppedBuf = cropRgb48le(hdrRgb, srcW, srcH, blitSrcX, blitSrcY, blitW, blitH);
blitRgb48leRegion(
canvas,
Expand All @@ -265,7 +267,7 @@ export function blitHdrVideoLayer(
blitH,
width,
height,
el.opacity < 0.999 ? el.opacity : undefined,
resolveBlitOpacity(el.opacity),
borderRadiusParam,
);
} else {
Expand All @@ -278,7 +280,7 @@ export function blitHdrVideoLayer(
srcH,
width,
height,
el.opacity < 0.999 ? el.opacity : undefined,
resolveBlitOpacity(el.opacity),
borderRadiusParam,
);
}
Expand Down Expand Up @@ -343,23 +345,24 @@ export function blitHdrImageLayer(
: buf.data,
);

const viewportMatrix = parseTransformMatrix(el.transform);
const rawMatrix = parseTransformMatrix(el.transform);
const matrix = rawMatrix && isAffineMatrix(rawMatrix) ? rawMatrix : null;

const br = el.borderRadius;
const hasBorderRadius = br[0] > 0 || br[1] > 0 || br[2] > 0 || br[3] > 0;
const borderRadiusParam = hasBorderRadius ? br : undefined;

timeHdrPhase(hdrPerf, "hdrImageBlitMs", () => {
if (viewportMatrix) {
if (matrix) {
blitRgb48leAffine(
canvas,
hdrRgb,
viewportMatrix,
matrix,
buf.width,
buf.height,
width,
height,
el.opacity < 0.999 ? el.opacity : undefined,
resolveBlitOpacity(el.opacity),
borderRadiusParam,
);
} else {
Expand All @@ -372,7 +375,7 @@ export function blitHdrImageLayer(
buf.height,
width,
height,
el.opacity < 0.999 ? el.opacity : undefined,
resolveBlitOpacity(el.opacity),
borderRadiusParam,
);
}
Expand Down Expand Up @@ -462,6 +465,7 @@ export interface HdrCompositeContext {
* dumps. Pass `-1` to disable per-layer dumps even
* when `KEEP_TEMP=1` (e.g. for warmup frames).
*/
// fallow-ignore-next-line complexity
export async function compositeHdrFrame(
ctx: HdrCompositeContext,
canvas: Buffer,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

import { rmSync } from "node:fs";
import {
type BeforeCaptureHook,
type CaptureSession,
type ElementStackingInfo,
applyDomLayerMask,
Expand All @@ -29,7 +30,12 @@ import {
blitHdrVideoLayer,
closeHdrVideoFrameSource,
} from "../../hdrCompositor.js";
import { type HdrPerfCollector, timeHdrPhase, timeHdrPhaseAsync } from "../hdrPerf.js";
import {
type HdrPerfCollector,
type HdrPerfTimingKey,
timeHdrPhase,
timeHdrPhaseAsync,
} from "../hdrPerf.js";

// ─── Hybrid path gating + partitioning ─────────────────────────────────────

Expand Down Expand Up @@ -101,6 +107,53 @@ export interface LayeredTransitionBuffers {
output: Buffer;
}

// ─── Seek + inject + stacking query (shared across all three loop paths) ──

/**
* Seek the page to `time` and run the optional before-capture hook.
* Used by `captureSceneIntoBuffer` which receives stacking info from
* its caller, and by `seekInjectAndQueryStacking` which appends a
* stacking query.
*/
async function seekAndInject(
page: CaptureSession["page"],
time: number,
beforeCaptureHook: BeforeCaptureHook | null,
hdrPerf: HdrPerfCollector | undefined,
seekKey: HdrPerfTimingKey,
injectKey: HdrPerfTimingKey,
): Promise<void> {
await timeHdrPhaseAsync(hdrPerf, seekKey, () =>
page.evaluate((t: number) => {
if (window.__hf && typeof window.__hf.seek === "function") window.__hf.seek(t);
}, time),
);
if (beforeCaptureHook) {
await timeHdrPhaseAsync(hdrPerf, injectKey, () => beforeCaptureHook(page, time));
}
}

/**
* Seek the page to `time`, run the optional before-capture hook, then
* query element stacking order. Each phase is individually timed via the
* caller-provided perf keys so the sequential loop, hybrid worker, and
* per-scene transition capture each emit the correct telemetry label
* (`frameSeekMs` vs. `domLayerSeekMs`, etc.).
*/
export async function seekInjectAndQueryStacking(
page: CaptureSession["page"],
time: number,
beforeCaptureHook: BeforeCaptureHook | null,
nativeHdrIds: Set<string>,
hdrPerf: HdrPerfCollector | undefined,
seekKey: HdrPerfTimingKey,
injectKey: HdrPerfTimingKey,
stackingKey: HdrPerfTimingKey,
): Promise<ElementStackingInfo[]> {
await seekAndInject(page, time, beforeCaptureHook, hdrPerf, seekKey, injectKey);
return timeHdrPhaseAsync(hdrPerf, stackingKey, () => queryElementStacking(page, nativeHdrIds));
}

// ─── Per-scene capture (shared by sequential transition + hybrid worker) ──

export interface CaptureSceneArgs {
Expand Down Expand Up @@ -141,16 +194,14 @@ export async function captureSceneIntoBuffer(a: CaptureSceneArgs): Promise<void>
log,
frameIdx,
} = a;
await timeHdrPhaseAsync(hdrPerf, "domLayerSeekMs", () =>
session.page.evaluate((t: number) => {
if (window.__hf && typeof window.__hf.seek === "function") window.__hf.seek(t);
}, time),
await seekAndInject(
session.page,
time,
beforeCaptureHook,
hdrPerf,
"domLayerSeekMs",
"domLayerInjectMs",
);
if (beforeCaptureHook) {
await timeHdrPhaseAsync(hdrPerf, "domLayerInjectMs", () =>
beforeCaptureHook(session.page, time),
);
}
for (const el of stackingInfo) {
if (!el.isHdr || !sceneIds.has(el.id)) continue;
if (nativeHdrImageIds.has(el.id)) {
Expand Down Expand Up @@ -256,28 +307,28 @@ export async function captureTransitionFrameOnWorker(
hdrPerf.frames += 1;
hdrPerf.transitionFrames += 1;
}
await timeHdrPhaseAsync(hdrPerf, "frameSeekMs", () =>
session.page.evaluate((t: number) => {
if (window.__hf && typeof window.__hf.seek === "function") window.__hf.seek(t);
}, time),
);
if (beforeCaptureHook) {
await timeHdrPhaseAsync(hdrPerf, "frameInjectMs", () => beforeCaptureHook(session.page, time));
}
const stackingInfo = await timeHdrPhaseAsync(hdrPerf, "stackingQueryMs", () =>
queryElementStacking(session.page, nativeHdrIds),
const stackingInfo = await seekInjectAndQueryStacking(
session.page,
time,
beforeCaptureHook,
nativeHdrIds,
hdrPerf,
"frameSeekMs",
"frameInjectMs",
"stackingQueryMs",
);
const sceneAIds = new Set(sceneElements[transition.fromScene] ?? []);
const sceneBIds = new Set(sceneElements[transition.toScene] ?? []);
buffers.bufferA.fill(0);
buffers.bufferB.fill(0);
for (const [sceneBuf, sceneIds] of [
const sceneCaptures: [Buffer, Set<string>][] = [
[buffers.bufferA, sceneAIds],
[buffers.bufferB, sceneBIds],
] as const) {
];
for (const [sceneBuf, sceneIds] of sceneCaptures) {
await captureSceneIntoBuffer({
session,
sceneBuf: sceneBuf as Buffer,
sceneBuf,
sceneIds,
stackingInfo,
time,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ import {
crossfade,
initTransparentBackground,
initializeSession,
queryElementStacking,
} from "@hyperframes/engine";
import type { FileServerHandle } from "../../fileServer.js";
import type { ProducerLogger } from "../../../logger.js";
Expand All @@ -54,6 +53,7 @@ import {
distributeLayeredHybridFrameRanges,
ensureFrameWritten,
partitionTransitionFrames,
seekInjectAndQueryStacking,
} from "./captureHdrFrameShared.js";
import { updateJobStatus } from "../shared.js";

Expand Down Expand Up @@ -305,19 +305,15 @@ export async function runHybridLayeredFrameLoop(input: HybridLoopInput): Promise
throw err instanceof Error ? err : new Error(String(err));
});
} else {
const beforeCaptureHook = session.onBeforeCapture;
await timeHdrPhaseAsync(hdrPerf, "frameSeekMs", () =>
session.page.evaluate((t: number) => {
if (window.__hf && typeof window.__hf.seek === "function") window.__hf.seek(t);
}, time),
);
if (beforeCaptureHook) {
await timeHdrPhaseAsync(hdrPerf, "frameInjectMs", () =>
beforeCaptureHook(session.page, time),
);
}
const stackingInfo = await timeHdrPhaseAsync(hdrPerf, "stackingQueryMs", () =>
queryElementStacking(session.page, nativeHdrIds),
const stackingInfo = await seekInjectAndQueryStacking(
session.page,
time,
session.onBeforeCapture,
nativeHdrIds,
hdrPerf,
"frameSeekMs",
"frameInjectMs",
"stackingQueryMs",
);
canvas.fill(0);
// Rebind ctx to this worker's session for per-layer captures
Expand Down
Loading
Loading