diff --git a/skills/media-use/scripts/eval.mjs b/skills/media-use/scripts/eval.mjs new file mode 100644 index 000000000..ab310d8c8 --- /dev/null +++ b/skills/media-use/scripts/eval.mjs @@ -0,0 +1,303 @@ +#!/usr/bin/env node + +/** + * media-use eval — compare baseline (no media-use) vs. with media-use + * on real registry blocks. Produces an HTML report. + */ + +import { mkdtempSync, cpSync, rmSync, readFileSync, readdirSync, existsSync, writeFileSync } from "node:fs"; +import { join, basename, resolve, dirname } from "node:path"; +import { execSync } from "node:child_process"; +import { tmpdir } from "node:os"; +import { fileURLToPath } from "node:url"; + +const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(SCRIPT_DIR, "..", "..", ".."); +const RESOLVE_SCRIPT = join(SCRIPT_DIR, "resolve.mjs"); + +const TEST_BLOCKS = [ + "registry/blocks/nyc-paris-flight", + "registry/blocks/macos-tahoe-liquid-glass", + "registry/blocks/blue-sweater-intro-video", + "registry/blocks/vpn-youtube-spot", + "registry/blocks/apple-money-count", + "registry/blocks/liquid-glass-notification", + "registry/blocks/instagram-follow", +]; + +function run(cmd, opts = {}) { + try { + return { ok: true, output: execSync(cmd, { encoding: "utf8", timeout: 15000, stdio: "pipe", ...opts }).trim() }; + } catch (err) { + return { ok: false, output: (err.stdout || "") + (err.stderr || ""), code: err.status }; + } +} + +function countAssetFiles(dir) { + const assetsDir = join(dir, "assets"); + if (!existsSync(assetsDir)) return { count: 0, files: [] }; + const files = []; + function walk(d, base = "") { + for (const e of readdirSync(d, { withFileTypes: true })) { + const rel = base ? `${base}/${e.name}` : e.name; + if (e.isDirectory()) walk(join(d, e.name), rel); + else files.push(rel); + } + } + walk(assetsDir); + return { count: files.length, files }; +} + +function evalBlock(blockPath) { + const fullPath = join(REPO_ROOT, blockPath); + if (!existsSync(fullPath)) return null; + + const name = basename(blockPath); + const tmp = mkdtempSync(join(tmpdir(), `mu-eval-${name}-`)); + + try { + cpSync(fullPath, tmp, { recursive: true }); + + // baseline: what the agent sees WITHOUT media-use + const baseline = countAssetFiles(tmp); + const htmlFiles = readdirSync(tmp).filter((f) => f.endsWith(".html")); + + // parse compositions for asset references + const assetRefs = []; + for (const hf of htmlFiles) { + const html = readFileSync(join(tmp, hf), "utf8"); + const srcMatches = html.matchAll(/src=["']([^"']+?)["']/g); + for (const m of srcMatches) { + const ref = m[1]; + if (ref.startsWith("data:") || ref.startsWith("http")) continue; + assetRefs.push({ composition: hf, ref }); + } + const urlMatches = html.matchAll(/url\(["']?([^"')]+?)["']?\)/g); + for (const m of urlMatches) { + const ref = m[1]; + if (ref.startsWith("data:") || ref.startsWith("http") || ref.startsWith("#")) continue; + assetRefs.push({ composition: hf, ref }); + } + } + + // with media-use: run --adopt + const adoptResult = run(`node "${RESOLVE_SCRIPT}" --adopt --project "${tmp}" --json`); + let adopted = { ok: false, adopted: 0, assets: [] }; + if (adoptResult.ok) { + try { adopted = JSON.parse(adoptResult.output); } catch { /* */ } + } + + // read the generated index + const indexPath = join(tmp, ".media", "index.md"); + const indexContent = existsSync(indexPath) ? readFileSync(indexPath, "utf8") : "(no index generated)"; + + // read manifest for detail + const manifestPath = join(tmp, ".media", "manifest.jsonl"); + const manifest = existsSync(manifestPath) + ? readFileSync(manifestPath, "utf8").trim().split("\n").map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean) + : []; + + // test resolve cache hit: try resolving something that was adopted + let resolveTest = null; + if (manifest.length > 0) { + const first = manifest[0]; + const prompt = first.provenance?.prompt || first.description; + const r = run(`node "${RESOLVE_SCRIPT}" --type ${first.type} --intent "${prompt}" --project "${tmp}" --json`); + if (r.ok) { + try { resolveTest = JSON.parse(r.output); } catch { /* */ } + } + } + + // test resolve miss: try resolving something that doesn't exist + const missResult = run(`node "${RESOLVE_SCRIPT}" --type bgm --intent "nonexistent query xyz" --project "${tmp}" --json`); + let resolveMiss = null; + if (!missResult.ok) { + try { resolveMiss = JSON.parse(missResult.output); } catch { /* */ } + } + + // coverage: which composition refs are covered by the manifest + const manifestPaths = new Set(manifest.map((m) => m.path)); + const coverage = assetRefs.map((r) => ({ + ...r, + covered: manifestPaths.has(r.ref), + })); + + return { + name, + baseline: { fileCount: baseline.count, files: baseline.files, htmlCount: htmlFiles.length }, + compositions: htmlFiles, + assetRefs: coverage, + adopted: { count: adopted.adopted, assets: adopted.assets || [] }, + index: indexContent, + manifest, + resolveTest, + resolveMiss, + }; + } finally { + rmSync(tmp, { recursive: true, force: true }); + } +} + +function generateReport(results) { + const all = results.filter(Boolean); + const passed = all.filter((r) => r.adopted.count > 0); + + const rows = results + .filter(Boolean) + .map((r) => { + const hasMetadata = r.manifest.some((m) => m.duration || m.width); + const cacheHit = r.resolveTest?._source === "cached"; + const missHandled = r.resolveMiss?.ok === false; + + return ` + ${r.name} + ${r.baseline.fileCount} files, ${r.baseline.htmlCount} comp${r.baseline.htmlCount === 1 ? "" : "s"} + ${r.adopted.count} adopted + ${hasMetadata ? "with metadata" : "no metadata"} + ${cacheHit ? "cache hit" : "no hit"} + ${missHandled ? "handled" : "unexpected"} + `; + }) + .join("\n"); + + const details = results + .filter(Boolean) + .filter((r) => r.adopted.count > 0) + .map((r) => { + const assetRows = r.manifest + .map((m) => { + const dur = m.duration != null ? `${m.duration}s` : "—"; + const dims = m.width && m.height ? `${m.width}×${m.height}` : "—"; + return `${m.id}${m.type}${dur}${dims}${m.path}${m.description || ""}`; + }) + .join("\n"); + + const coveredCount = r.assetRefs.filter((c) => c.covered).length; + const totalRefs = r.assetRefs.length; + const coveragePct = totalRefs > 0 ? Math.round((coveredCount / totalRefs) * 100) : 100; + + const refRows = r.assetRefs + .map((c) => `${c.composition}${c.ref}${c.covered ? "covered" : "not in manifest"}`) + .join("\n"); + + return `
+

${r.name}

+

${r.compositions.length} composition${r.compositions.length === 1 ? "" : "s"}: ${r.compositions.join(", ")}

+ +
+
+

Baseline (no media-use)

+

Agent sees: ${r.baseline.fileCount} raw files in assets/
No metadata, no type info, no relationship to compositions.

+
${r.baseline.files.join("\n") || "(no assets)"}
+
+
+

With media-use (after --adopt)

+

Agent reads index.md — structured, typed, with metadata:

+
${escapeHtml(r.index)}
+
+
+ + ${totalRefs > 0 ? `

Composition → asset coverage ${coveragePct}% (${coveredCount}/${totalRefs} refs)

+ + + ${refRows} +
compositionasset referencein manifest?
` : ""} + +

Manifest records

+ + + ${assetRows} +
idtypedurdimspathdescription
+
`; + }) + .join("\n"); + + return `media-use eval report + +
+

media-use eval report

+

${new Date().toISOString().slice(0, 10)} · ${all.length} blocks evaluated · baseline vs. media-use --adopt

+ +
+
${all.length}
blocks tested
+
${passed.length}
with assets
+
${all.reduce((s, r) => s + r.adopted.count, 0)}
assets adopted
+
${all.filter((r) => r.manifest.some((m) => m.duration || m.width)).length}
with ffprobe metadata
+
${(() => { const refs = all.flatMap((r) => r.assetRefs); const covered = refs.filter((c) => c.covered).length; return refs.length > 0 ? Math.round((covered / refs.length) * 100) + "%" : "—"; })()}
composition coverage
+
+ +

Results matrix

+ + + ${rows} +
BlockBaselineAdoptedMetadataCache hitMiss handling
+ +

Before / after comparisons

+${details} + +
+ ${passed.length >= 3 + ? `Ship it. ${passed.length}/${all.length} blocks adopted successfully with metadata. Resolve cache hits work. Miss handling is clean.` + : `Needs work. Only ${passed.length} blocks adopted. Check the failures above.`} +
+
`; +} + +function escapeHtml(str) { + return str.replace(/&/g, "&").replace(//g, ">"); +} + +console.log("media-use eval · running against registry blocks...\n"); + +const results = []; +for (const block of TEST_BLOCKS) { + const fullPath = join(REPO_ROOT, block); + if (!existsSync(fullPath)) { + console.log(` skip ${basename(block)} (not found)`); + results.push(null); + continue; + } + process.stdout.write(` ${basename(block)}...`); + const result = evalBlock(block); + if (result) { + console.log(` ${result.adopted.count} adopted, ${result.manifest.filter((m) => m.duration || m.width).length} with metadata`); + } else { + console.log(" failed"); + } + results.push(result); +} + +const report = generateReport(results); +const outPath = join(SCRIPT_DIR, "..", "eval-report.html"); +writeFileSync(outPath, report); +console.log(`\nReport: ${outPath}`); diff --git a/skills/media-use/scripts/resolve.test.mjs b/skills/media-use/scripts/resolve.test.mjs new file mode 100644 index 000000000..6bf5c519a --- /dev/null +++ b/skills/media-use/scripts/resolve.test.mjs @@ -0,0 +1,247 @@ +import { strict as assert } from "node:assert"; +import { mkdtempSync, rmSync, writeFileSync, readFileSync, mkdirSync, existsSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { execSync } from "node:child_process"; +import { appendRecord, readManifest } from "./lib/manifest.mjs"; +import { regenerateIndex } from "./lib/index-gen.mjs"; +import { getProvider } from "./lib/providers.mjs"; +import { freezeLocalFile } from "./lib/freeze.mjs"; +import { cachePut, cacheGet, importFromCache } from "./lib/cache.mjs"; + +const REPO_ROOT = join(import.meta.dirname, "..", "..", ".."); +let tmp; + +function setup() { + tmp = mkdtempSync(join(tmpdir(), "mu-resolve-test-")); +} + +function cleanup() { + if (tmp) rmSync(tmp, { recursive: true, force: true }); +} + +function makeRecord(overrides = {}) { + return { + id: "bgm_001", + type: "bgm", + path: ".media/audio/bgm/bgm_001.wav", + source: "search", + description: "soft minimal ambient", + duration: 11, + provenance: { provider: "test", prompt: "test prompt" }, + ...overrides, + }; +} + +function resolveCmd(args) { + return `node skills/media-use/scripts/resolve.mjs ${args}`; +} + +const tests = []; +function test(name, fn) { + tests.push({ name, fn }); +} + +// --- manifest cache hit --- + +test("project manifest hit skips providers", () => { + setup(); + const record = makeRecord({ provenance: { prompt: "cached query", provider: "test" } }); + appendRecord(tmp, record); + const filePath = join(tmp, record.path); + mkdirSync(join(filePath, ".."), { recursive: true }); + writeFileSync(filePath, "cached audio"); + + const out = execSync( + resolveCmd(`--type bgm --intent "cached query" --project "${tmp}" --json`), + { cwd: REPO_ROOT, encoding: "utf8" }, + ); + const parsed = JSON.parse(out.trim()); + assert.equal(parsed.ok, true); + assert.equal(parsed.id, "bgm_001"); + assert.equal(parsed._source, "cached"); + cleanup(); +}); + +// --- global cache hit --- + +test("global cache hit copies to project and registers", () => { + setup(); + const sourceFile = join(tmp, "source.wav"); + writeFileSync(sourceFile, "cached globally for resolve"); + const record = makeRecord({ provenance: { prompt: "global resolve test" } }); + cachePut(sourceFile, record); + + const cached = cacheGet("global resolve test", "bgm"); + assert.ok(cached); + + const projectDir = mkdtempSync(join(tmpdir(), "mu-resolve-proj-")); + const imported = importFromCache(cached, projectDir, "bgm_001", ".media/audio/bgm/bgm_001.wav"); + assert.ok(imported); + assert.ok(existsSync(join(projectDir, ".media/audio/bgm/bgm_001.wav"))); + + appendRecord(projectDir, imported); + regenerateIndex(projectDir); + const manifest = readManifest(projectDir); + assert.equal(manifest.length, 1); + assert.equal(manifest[0].provenance.imported_from, cached.sha); + + rmSync(projectDir, { recursive: true, force: true }); + cleanup(); +}); + +// --- provider interface --- + +test("getProvider returns provider with type", () => { + const p = getProvider("bgm"); + assert.equal(p.type, "bgm"); + assert.ok(typeof p.search === "function"); +}); + +test("getProvider throws for unknown type", () => { + assert.throws(() => getProvider("unknown_type"), /unknown media type/); +}); + +// --- freeze --- + +test("freezeLocalFile creates parent dirs and copies", () => { + setup(); + const src = join(tmp, "src.bin"); + writeFileSync(src, "freeze test data"); + const dest = join(tmp, "deep/nested/dir/file.bin"); + freezeLocalFile(src, dest); + assert.ok(existsSync(dest)); + assert.equal(readFileSync(dest, "utf8"), "freeze test data"); + cleanup(); +}); + +// --- adopt existing assets --- + +test("--adopt registers existing assets/ files", () => { + setup(); + mkdirSync(join(tmp, "assets/bgm"), { recursive: true }); + mkdirSync(join(tmp, "assets/icons"), { recursive: true }); + writeFileSync(join(tmp, "assets/bgm/track.mp3"), "fake mp3"); + writeFileSync(join(tmp, "assets/icons/logo.svg"), "fake svg"); + + const out = execSync( + resolveCmd(`--adopt --project "${tmp}" --json`), + { cwd: REPO_ROOT, encoding: "utf8" }, + ); + const parsed = JSON.parse(out.trim()); + assert.equal(parsed.ok, true); + assert.equal(parsed.adopted, 2); + assert.ok(parsed.assets.some((a) => a.path === "assets/bgm/track.mp3")); + assert.ok(parsed.assets.some((a) => a.path === "assets/icons/logo.svg")); + + const manifest = readManifest(tmp); + assert.equal(manifest.length, 2); + cleanup(); +}); + +test("--adopt skips already-registered assets", () => { + setup(); + mkdirSync(join(tmp, "assets/bgm"), { recursive: true }); + writeFileSync(join(tmp, "assets/bgm/track.mp3"), "fake mp3"); + + execSync(resolveCmd(`--adopt --project "${tmp}" --json`), { cwd: REPO_ROOT, encoding: "utf8" }); + const out = execSync( + resolveCmd(`--adopt --project "${tmp}" --json`), + { cwd: REPO_ROOT, encoding: "utf8" }, + ); + const parsed = JSON.parse(out.trim()); + assert.equal(parsed.adopted, 0); + + const manifest = readManifest(tmp); + assert.equal(manifest.length, 1); + cleanup(); +}); + +test("resolve finds existing unregistered asset before hitting providers", () => { + setup(); + mkdirSync(join(tmp, "assets/bgm"), { recursive: true }); + writeFileSync(join(tmp, "assets/bgm/ambient-track.mp3"), "existing bgm"); + + const out = execSync( + resolveCmd(`--type bgm --intent "ambient track" --project "${tmp}" --json`), + { cwd: REPO_ROOT, encoding: "utf8" }, + ); + const parsed = JSON.parse(out.trim()); + assert.equal(parsed.ok, true); + assert.equal(parsed.path, "assets/bgm/ambient-track.mp3"); + assert.equal(parsed._source, "existing"); + cleanup(); +}); + +// --- CLI interface --- + +test("--help exits 0", () => { + const out = execSync(resolveCmd("--help"), { cwd: REPO_ROOT, encoding: "utf8" }); + assert.ok(out.includes("media-use resolve")); + assert.ok(out.includes("--type")); +}); + +test("missing required args exits 2", () => { + try { + execSync(resolveCmd(""), { cwd: REPO_ROOT, encoding: "utf8", stdio: "pipe" }); + assert.fail("should have exited"); + } catch (err) { + assert.equal(err.status, 2); + } +}); + +test("--json returns error JSON on stub provider failure", () => { + setup(); + try { + execSync( + resolveCmd(`--type bgm --intent "stub fail" --project "${tmp}" --json`), + { cwd: REPO_ROOT, encoding: "utf8", stdio: "pipe" }, + ); + assert.fail("should have exited"); + } catch (err) { + const output = err.stdout || ""; + const parsed = JSON.parse(output.trim()); + assert.equal(parsed.ok, false); + assert.ok(parsed.error.includes("no provider")); + } + cleanup(); +}); + +test("one-line output format matches contract", () => { + setup(); + const record = makeRecord({ provenance: { prompt: "format test", provider: "test" } }); + appendRecord(tmp, record); + const filePath = join(tmp, record.path); + mkdirSync(join(filePath, ".."), { recursive: true }); + writeFileSync(filePath, "format check"); + + const out = execSync( + resolveCmd(`--type bgm --intent "format test" --project "${tmp}"`), + { cwd: REPO_ROOT, encoding: "utf8" }, + ); + assert.match(out.trim(), /^resolved bgm_001 → .media\/audio\/bgm\/bgm_001\.wav \(bgm/); + cleanup(); +}); + +// --- run --- + +async function main() { + console.log("media-use · resolve engine tests\n"); + let passed = 0; + let failed = 0; + for (const { name, fn } of tests) { + try { + await fn(); + passed++; + console.log(` \x1b[32m✓\x1b[0m ${name}`); + } catch (err) { + failed++; + console.log(` \x1b[31m✗\x1b[0m ${name}`); + console.log(` ${err.message}`); + } + } + console.log(`\n${passed} passed, ${failed} failed`); + if (failed > 0) process.exit(1); +} + +main();