Add air text (#410)

* Add air text

* Fix install

* Update example/vite.config.js

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: pearmini

example/air-text/content.js

@@ -0,0 +1,47 @@
+/** Short headline for the title block (layout measures this string). */
+export const TITLE_TEXT = "What if air were text.";
+
+/**
+ * Body copy for the filled region: laid along paths by layoutTextInPath.
+ */
+export const TEXT = `
+What if air were text — not metaphorically, but on the page: a field of language so dense it becomes atmosphere.
+We breathe letters we never read; we move through sentences that hang like humidity. Now the screen is black; the type is a
+pale film on the glass, like breath on a window. The room is still filled with phrases about breath, wind, oxygen, and the
+thin film that wraps the planet. To stand in front of this page is to displace the words: your outline becomes a quiet
+island where no characters land, as if your body were the only place the air refuses to be written. Step aside and the
+text floods back; step forward and you carve a human-shaped silence from the prose. This is a small experiment in negative
+space: machine vision finds the pose, geometry cuts the path, and type fills what remains — the page always full, except
+where you are.
+
+Every font is a weather system: serifs like ridges of pressure, commas like droplets, paragraphs like fronts that never
+quite arrive. You are not reading so much as standing in a storm of meaning that has forgotten how to land. The camera
+guesses where you end and the room begins; the mask is a stencil cut from probability. What remains is not emptiness but
+refusal — a pocket where the sentence cannot stick, a small sovereignty of skin and heat.
+
+If language is a medium, then we are always swimming. Sometimes the water is news; sometimes it is a novel; sometimes it
+is the same paragraph copied until it becomes texture. Repetition is not boredom here: it is thickness, a way to make the
+margin feel infinite. Scroll, resize, lean closer: the words do not care. They only know how to obey the shape they are
+given — and the shape, today, is you.
+
+So call it atmosphere, call it interface, call it a joke about breath and bytes. The joke still holds: you are the one
+place the poem cannot go without becoming something else. When you leave, the letters close over the wake, seamless as
+water, and the page pretends it was never broken at all.
+
+Dark mode is not decoration: it is a kind of night inside the machine. Stars are pixels; constellations are kerning;
+the Milky Way is a long line that broke and kept going. On black, the eye stops hunting for a margin and starts hunting
+for contrast — and contrast, here, is you: warmer than the glass, slower than refresh.
+
+Latency is a form of patience. The model loads; the stream stutters; the mask catches up to your shoulder a frame late,
+and for a moment you are two people — one made of light, one made of guesswork. That doubled self is also text: a caption
+nobody wrote, a subtitle to being alive in front of a lens.
+
+We used to think the page was a rectangle of paper. Now it is a rectangle of policy: permissions, codecs, cooling fans,
+the soft politics of who gets to be foreground. The prose does not judge; it only fills. But you can feel, in the empty
+channels where your silhouette passes, that someone once decided what “foreground” means — and that decision is older
+than this paragraph, and will outlive it.
+
+Listen: if you hold still long enough, the text forgets you are there and treats you like architecture — a column, an
+arch, a doorway. Move, and the language remembers you are weather. That oscillation between object and storm is the real
+subject of the work. Everything else is atmosphere — which is to say, everything else is air, pretending to be words.
+`.trim();

example/air-text/helpers.js

@@ -0,0 +1,242 @@
+import * as cm from "charmingjs";
+import * as d3 from "d3";
+import ClipperLib from "clipper-lib";
+
+export const CLIPPER_SCALE = 256;
+export const SCREEN_PATH_PADDING = 20;
+export const SCREEN_PATH_PADDING_TOP = 30;
+export const MASK_CONTOUR_MAX_DIM = 140;
+export const BODYPIX_NUM_PARTS = 24;
+export const VIDEO_MAX_LONG_EDGE = 480;
+
+export function videoBufferDimensions(layerW, layerH) {
+  const lw = Math.max(2, Math.floor(layerW));
+  const lh = Math.max(2, Math.floor(layerH));
+  const ar = lw / lh;
+  let vw;
+  let vh;
+  if (lw >= lh) {
+    vw = Math.min(VIDEO_MAX_LONG_EDGE, lw);
+    vh = Math.max(2, Math.round(vw / ar));
+  } else {
+    vh = Math.min(VIDEO_MAX_LONG_EDGE, lh);
+    vw = Math.max(2, Math.round(vh * ar));
+  }
+  return {vw, vh};
+}
+
+export function rectToRing(x, y, w, h) {
+  return [
+    [x, y],
+    [x + w, y],
+    [x + w, y + h],
+    [x, y + h],
+  ];
+}
+
+export function unionRingsToHolePathStrings(rings) {
+  const valid = rings.filter((ring) => ring.length >= 3);
+  if (valid.length === 0) {
+    return [];
+  }
+  const s = CLIPPER_SCALE;
+  const paths = valid.map((ring) =>
+    ring.map(([x, y]) => ({
+      X: Math.round(x * s),
+      Y: Math.round(y * s),
+    })),
+  );
+  const c = new ClipperLib.Clipper();
+  c.AddPaths(paths, ClipperLib.PolyType.ptSubject, true);
+  const solution = new ClipperLib.Paths();
+  const ok = c.Execute(
+    ClipperLib.ClipType.ctUnion,
+    solution,
+    ClipperLib.PolyFillType.pftNonZero,
+    ClipperLib.PolyFillType.pftNonZero,
+  );
+  const inv = 1 / s;
+  if (!ok || !solution.length) {
+    return [cm.pathPolygon(valid[0])];
+  }
+  const out = solution
+    .filter((path) => path.length >= 3)
+    .map((path) => cm.pathPolygon(path.map(({X, Y}) => [X * inv, Y * inv])));
+  return out.length ? out : [cm.pathPolygon(valid[0])];
+}
+
+export function getScreenInsetBounds(width, height) {
+  const maxPad = Math.max(0, (Math.min(width, height) - 8) / 2);
+  const p = Math.min(SCREEN_PATH_PADDING, maxPad);
+  const pt = Math.min(SCREEN_PATH_PADDING_TOP, maxPad);
+  const innerW = Math.max(1, width - 2 * p);
+  const innerH = Math.max(1, height - pt - p);
+  return {
+    p,
+    pt,
+    innerW,
+    innerH,
+    left: p,
+    top: pt,
+    right: p + innerW,
+    bottom: pt + innerH,
+  };
+}
+
+function clipPolygonRingToInset(ring, left, top, right, bottom) {
+  const s = CLIPPER_SCALE;
+  if (ring.length < 3) {
+    return [];
+  }
+  const subj = [
+    ring.map(([x, y]) => ({
+      X: Math.round(x * s),
+      Y: Math.round(y * s),
+    })),
+  ];
+  const clip = [
+    [
+      {X: Math.round(left * s), Y: Math.round(top * s)},
+      {X: Math.round(right * s), Y: Math.round(top * s)},
+      {X: Math.round(right * s), Y: Math.round(bottom * s)},
+      {X: Math.round(left * s), Y: Math.round(bottom * s)},
+    ],
+  ];
+  const c = new ClipperLib.Clipper();
+  c.AddPaths(subj, ClipperLib.PolyType.ptSubject, true);
+  c.AddPaths(clip, ClipperLib.PolyType.ptClip, true);
+  const solution = new ClipperLib.Paths();
+  const ok = c.Execute(
+    ClipperLib.ClipType.ctIntersection,
+    solution,
+    ClipperLib.PolyFillType.pftNonZero,
+    ClipperLib.PolyFillType.pftNonZero,
+  );
+  if (!ok || !solution.length) {
+    return [];
+  }
+  const inv = 1 / s;
+  const out = [];
+  for (const path of solution) {
+    if (path.length < 3) {
+      continue;
+    }
+    out.push(path.map(({X, Y}) => [X * inv, Y * inv]));
+  }
+  return out;
+}
+
+function samplePartIdGrid(imageData, mw, mh) {
+  const iw = imageData.width;
+  const ih = imageData.height;
+  const ids = new Int32Array(mw * mh);
+  const alpha = new Uint8Array(mw * mh);
+  for (let j = 0; j < mh; j++) {
+    for (let i = 0; i < mw; i++) {
+      const sx = Math.min(iw - 1, Math.floor((i + 0.5) * (iw / mw)));
+      const sy = Math.min(ih - 1, Math.floor((j + 0.5) * (ih / mh)));
+      const o = (sy * iw + sx) * 4;
+      const ix = j * mw + i;
+      ids[ix] = imageData.data[o];
+      alpha[ix] = imageData.data[o + 3];
+    }
+  }
+  return {ids, alpha};
+}
+
+function contourBinaryMaskToRings(values, mw, mh, layerW, layerH, minAreaGrid, inset) {
+  let sum = 0;
+  for (let i = 0; i < values.length; i++) {
+    sum += values[i];
+  }
+  if (sum < minAreaGrid) {
+    return [];
+  }
+
+  const contourGen = d3.contours().size([mw, mh]).smooth(true).thresholds([0.5]);
+  const layers = contourGen(values);
+  if (!layers.length) {
+    return [];
+  }
+
+  const multi = layers[0];
+  if (!multi.coordinates?.length) {
+    return [];
+  }
+
+  const sx = layerW / mw;
+  const sy = layerH / mh;
+  const maxHoleArea = layerW * layerH * 0.45;
+  const out = [];
+
+  for (const poly of multi.coordinates) {
+    if (!poly?.length) {
+      continue;
+    }
+    const outer = poly[0];
+    const aGrid = Math.abs(d3.polygonArea(outer));
+    if (aGrid < minAreaGrid || outer.length < 4) {
+      continue;
+    }
+    const aLayer = aGrid * sx * sy;
+    if (aLayer > maxHoleArea) {
+      continue;
+    }
+    const scaled = outer.map(([x, y]) => [x * sx, y * sy]);
+    const clipped = clipPolygonRingToInset(scaled, inset.left, inset.top, inset.right, inset.bottom);
+    for (const ring of clipped) {
+      out.push(ring);
+    }
+  }
+
+  return out;
+}
+
+export function maskImageDataToPartHoleRings(imageData, layerW, layerH) {
+  if (!imageData?.data || imageData.width < 2 || imageData.height < 2) {
+    return [];
+  }
+
+  const inset = getScreenInsetBounds(layerW, layerH);
+  const iw = imageData.width;
+  const ih = imageData.height;
+  const scale = Math.min(MASK_CONTOUR_MAX_DIM / iw, MASK_CONTOUR_MAX_DIM / ih, 1);
+  const mw = Math.max(8, Math.floor(iw * scale));
+  const mh = Math.max(8, Math.floor(ih * scale));
+
+  const {ids, alpha} = samplePartIdGrid(imageData, mw, mh);
+  const seen = new Set();
+  for (let i = 0; i < ids.length; i++) {
+    if (alpha[i] <= 40) {
+      continue;
+    }
+    const pid = ids[i];
+    if (pid >= 0 && pid < BODYPIX_NUM_PARTS) {
+      seen.add(pid);
+    }
+  }
+
+  const sortedParts = Array.from(seen).sort((a, b) => a - b);
+  const values = new Float64Array(mw * mh);
+  const minAreaGrid = 6;
+  const holeRings = [];
+
+  for (const pid of sortedParts) {
+    for (let i = 0; i < values.length; i++) {
+      values[i] = ids[i] === pid && alpha[i] > 40 ? 1 : 0;
+    }
+    holeRings.push(...contourBinaryMaskToRings(values, mw, mh, layerW, layerH, minAreaGrid, inset));
+  }
+
+  return holeRings;
+}
+
+export function buildTextPath(width, height, holes) {
+  const {p, pt, innerW, innerH} = getScreenInsetBounds(width, height);
+  const outer = cm.pathRect(p, pt, innerW, innerH);
+  const list = Array.isArray(holes) ? holes.filter(Boolean) : holes ? [holes] : [];
+  if (list.length === 0) {
+    return outer;
+  }
+  return `${outer} ${list.join(" ")}`;
+}

example/air-text/index.css

@@ -0,0 +1,79 @@
+* {
+  margin: 0;
+  padding: 0;
+  box-sizing: border-box;
+}
+
+:root {
+  --air-bg: #0c0c0e;
+  --air-text: #b8b8c2;
+  --air-title: #eaeaf0;
+}
+
+html,
+body {
+  width: 100%;
+  height: 100%;
+  overflow: hidden;
+  background: var(--air-bg);
+  color-scheme: dark;
+}
+
+.stage {
+  position: relative;
+  width: 100vw;
+  width: 100dvw;
+  height: 100vh;
+  height: 100dvh;
+  min-height: 100%;
+}
+
+.stage__inner {
+  position: relative;
+  width: 100%;
+  height: 100%;
+  background: var(--air-bg);
+  overflow: hidden;
+}
+
+/* Fills the stage; object-fit fill matches ml5 scaling to the element box. */
+.stage__webcam {
+  position: absolute;
+  inset: 0;
+  z-index: 0;
+  width: 100%;
+  height: 100%;
+  object-fit: fill;
+  opacity: 0;
+  pointer-events: none;
+}
+
+.stage__inner .line {
+  z-index: 1;
+}
+
+.stage__inner .title-line {
+  z-index: 2;
+}
+
+.title-line {
+  position: absolute;
+  transform-origin: left center;
+  white-space: pre;
+  line-height: 1;
+  font: inherit;
+  user-select: text;
+  color: var(--air-title);
+  cursor: text;
+}
+
+.line {
+  position: absolute;
+  transform-origin: center;
+  white-space: pre;
+  line-height: 1;
+  font: inherit;
+  user-select: text;
+  color: var(--air-text);
+  cursor: text;
+}

example/air-text/index.html

@@ -0,0 +1,17 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Air text</title>
+    <link rel="stylesheet" href="index.css" />
+  </head>
+  <body>
+    <div id="stage" class="stage">
+      <div class="stage__inner" id="layer">
+        <video id="webcam" class="stage__webcam" playsinline muted autoplay></video>
+      </div>
+    </div>
+    <script src="index.js" type="module"></script>
+  </body>
+</html>