Preserve Composer overlay source revisions (#1849)

* Preserve Composer overlay source revisions

* Align Composer overlay provenance datasets

Author: chubes4

package.json

@@ -201,6 +201,8 @@
     "test:preview-options": "tsx tests/preview-options.test.ts",
     "test:evidence-bundle-digest-and-recipe-artifact": "tsx tests/evidence-bundle-digest-and-recipe-artifact.test.ts",
     "test:runtime-overlay-descriptors": "tsx tests/runtime-overlay-descriptors.test.ts",
+    "test:composer-package-overlay-revision": "tsx scripts/composer-backed-source-hydration-smoke.ts",
+    "test:composer-installed-versions-loader-order": "tsx scripts/composer-installed-versions-loader-order-smoke.ts",
     "test:runtime-preset-registry": "tsx tests/runtime-preset-registry.test.ts",
     "test:generic-ability-runtime-run": "tsx tests/generic-ability-runtime-run.test.ts",
     "test:provider-runtime-contracts": "tsx tests/provider-runtime-contracts.test.ts",

packages/cli/src/commands/agent-task-run.ts

@@ -793,7 +793,7 @@ function compactPreparedPaths(context: Record<string, unknown>, recipeInputs: Re
     component_contracts: arrayRecords(context.preparedComponentContracts).map((contract) => compactRecord(contract, ["slug", "requestedPath", "preparedPath", "pluginFile", "loadAs", "activate", "status"])),
     workspaces: arrayRecords(context.preparedWorkspaces).map((workspace) => compactRecord(workspace, ["target", "mode", "metadata"])),
     staged_files: arrayRecords(context.preparedStagedFiles).map((file) => compactRecord(file, ["sourceRef", "target", "type", "provenance", "metadata"])),
-    dependency_overlays: arrayRecords(context.preparedDependencyOverlays).map((overlay) => compactRecord(overlay, ["package", "target", "type", "mode", "metadata"])),
+    dependency_overlays: arrayRecords(context.preparedDependencyOverlays).map((overlay) => compactRecord(overlay, ["package", "reference", "target", "type", "mode", "metadata"])),
     runtime_overlays: arrayRecords(context.preparedRuntimeOverlays).map((overlay) => compactRecord(overlay, ["target", "type", "mode", "metadata"])),
     requested_component_contracts: arrayRecords(recipeInputs.component_contracts ?? taskInputs.component_contracts).map((contract) => compactRecord(contract, ["slug", "path", "loadAs", "activate"])),
   })

packages/cli/src/commands/recipe-runtime-setup.ts

@@ -249,6 +249,7 @@ export function recipeRunDependencyOverlay(overlay: PreparedDependencyOverlay):
   return {
     source: overlay.source,
     sourceRef: overlay.sourceRef,
+    ...(overlay.reference ? { reference: overlay.reference } : {}),
     target: overlay.target,
     package: overlay.package,
     consumer: overlay.consumer,

packages/cli/src/recipe-sources.ts

@@ -108,6 +108,7 @@ export interface PreparedRuntimeOverlay {
 export interface PreparedDependencyOverlay {
   source: string
   sourceRef: string
+  reference?: string
   target: string
   package: string
   consumer: string
@@ -428,14 +429,15 @@ function composerInstalledPackageClassmapPaths(pkg: ComposerInstalledPackage): s
 
 export async function prepareRecipeDependencyOverlays(recipe: WorkspaceRecipe, recipeDirectory: string, extraPlugins: PreparedExtraPlugin[]): Promise<PreparedDependencyOverlay[]> {
   const overlays: PreparedDependencyOverlay[] = []
+  const stagedConsumers = new Set<PreparedExtraPlugin>()
   for (const [index, overlay] of (recipe.inputs?.dependency_overlays ?? []).entries()) {
-    overlays.push(await prepareRecipeDependencyOverlay(overlay, recipeDirectory, extraPlugins, index))
+    overlays.push(await prepareRecipeDependencyOverlay(overlay, recipeDirectory, extraPlugins, stagedConsumers, index))
   }
 
   return overlays
 }
 
-async function prepareRecipeDependencyOverlay(overlay: WorkspaceRecipeDependencyOverlay, recipeDirectory: string, extraPlugins: PreparedExtraPlugin[], index: number): Promise<PreparedDependencyOverlay> {
+async function prepareRecipeDependencyOverlay(overlay: WorkspaceRecipeDependencyOverlay, recipeDirectory: string, extraPlugins: PreparedExtraPlugin[], stagedConsumers: Set<PreparedExtraPlugin>, index: number): Promise<PreparedDependencyOverlay> {
   if (overlay.kind !== "composer-package") {
     throw new Error(`Unsupported dependency overlay kind: ${overlay.kind}`)
   }
@@ -450,14 +452,20 @@ async function prepareRecipeDependencyOverlay(overlay: WorkspaceRecipeDependency
 
   const source = resolve(recipeDirectory, overlay.source)
   await validateExistingDirectoryForOverlay(source, overlay.source)
+  const reference = await resolvedGitSourceReference(source)
   const stagingRoot = await mkdtemp(join(tmpdir(), "wp-codebox-dependency-overlay-"))
   const preparedSource = await prepareComposerBackedSource(source, stagingRoot, `dependency overlay ${overlay.package}`)
+  if (reference) {
+    await preserveComposerDependencyReference(consumer, overlay.package, reference, stagedConsumers)
+    await preserveComposerPackageReference(preparedSource, overlay.package, reference)
+  }
   const target = `${consumer.target}/vendor/${composerPackageVendorPath(overlay.package)}`
   const digest = await directoryContentDigest(preparedSource)
 
   return {
     source: preparedSource,
     sourceRef: overlay.source,
+    ...(reference ? { reference } : {}),
     target,
     package: overlay.package,
     consumer: overlay.consumer,
@@ -470,6 +478,7 @@ async function prepareRecipeDependencyOverlay(overlay: WorkspaceRecipeDependency
       overlayKind: overlay.kind,
       package: overlay.package,
       source: overlay.source,
+      ...(reference ? { reference } : {}),
       consumer: overlay.consumer,
       target,
       digest: { sha256: digest },
@@ -478,6 +487,158 @@ async function prepareRecipeDependencyOverlay(overlay: WorkspaceRecipeDependency
   }
 }
 
+/**
+ * Capture the clean checkout commit before Composer staging so runtime
+ * provenance remains tied to the source revision rather than its staged path.
+ */
+async function resolvedGitSourceReference(source: string): Promise<string | undefined> {
+  try {
+    const status = await executeManagedHostCommand({
+      command: "git",
+      args: ["status", "--porcelain", "--untracked-files=all"],
+      cwd: source,
+      allowedCwdRoots: [source],
+      timeoutMs: 10_000,
+      maxOutputBytes: 64 * 1024,
+      label: "inspect Composer overlay Git source",
+    })
+    if (status.stdout.trim()) {
+      return undefined
+    }
+
+    const revision = await executeManagedHostCommand({
+      command: "git",
+      args: ["rev-parse", "--verify", "HEAD"],
+      cwd: source,
+      allowedCwdRoots: [source],
+      timeoutMs: 10_000,
+      maxOutputBytes: 64 * 1024,
+      label: "resolve Composer overlay Git revision",
+    })
+    const reference = revision.stdout.trim()
+    return /^[a-f0-9]{40,64}$/i.test(reference) ? reference : undefined
+  } catch {
+    return undefined
+  }
+}
+
+async function preserveComposerDependencyReference(consumer: PreparedExtraPlugin, packageName: string, reference: string, stagedConsumers: Set<PreparedExtraPlugin>): Promise<void> {
+  const installedJson = join(consumer.source, "vendor", "composer", "installed.json")
+  if (!await composerInstalledJsonHasPackage(installedJson, packageName)) {
+    return
+  }
+
+  if (!stagedConsumers.has(consumer)) {
+    const stagingRoot = await mkdtemp(join(tmpdir(), `wp-codebox-dependency-consumer-${consumer.slug}-`))
+    const stagedSource = join(stagingRoot, basename(consumer.source))
+    await cp(consumer.source, stagedSource, { recursive: true })
+    consumer.source = stagedSource
+    consumer.cleanupPaths.push(stagingRoot)
+    stagedConsumers.add(consumer)
+  }
+
+  await writeComposerDependencyReference(join(consumer.source, "vendor", "composer", "installed.json"), packageName, reference)
+  await writeComposerInstalledPhpReference(join(consumer.source, "vendor", "composer", "installed.php"), packageName, reference)
+}
+
+async function preserveComposerPackageReference(source: string, packageName: string, reference: string): Promise<void> {
+  await writeComposerDependencyReference(join(source, "vendor", "composer", "installed.json"), packageName, reference)
+  await writeComposerInstalledPhpReference(join(source, "vendor", "composer", "installed.php"), packageName, reference)
+}
+
+async function composerInstalledJsonHasPackage(path: string, packageName: string): Promise<boolean> {
+  try {
+    return composerInstalledPackageRecords(JSON.parse(await readFile(path, "utf8"))).some((pkg) => pkg.name === packageName)
+  } catch {
+    return false
+  }
+}
+
+async function writeComposerDependencyReference(path: string, packageName: string, reference: string): Promise<void> {
+  const installed = JSON.parse(await readFile(path, "utf8")) as unknown
+  const pkg = composerInstalledPackageRecords(installed).find((candidate) => candidate.name === packageName)
+  if (!pkg) {
+    return
+  }
+  pkg.source = { ...(isRecord(pkg.source) ? pkg.source : {}), reference }
+  await writeFile(path, `${JSON.stringify(installed, null, 2)}\n`)
+}
+
+async function writeComposerInstalledPhpReference(path: string, packageName: string, reference: string): Promise<void> {
+  if (!await pathIsFile(path)) {
+    return
+  }
+  const contents = await readFile(path, "utf8")
+  const packageStart = contents.search(new RegExp(`['\"]${escapeRegExp(packageName)}['\"]\\s*=>\\s*array\\s*\\(`))
+  if (packageStart < 0) {
+    return
+  }
+  const before = contents.slice(0, packageStart)
+  const entry = contents.slice(packageStart)
+  const packageEntryEnd = composerInstalledPhpPackageEntryEnd(entry, packageName)
+  if (packageEntryEnd === undefined) {
+    return
+  }
+  const packageEntry = entry.slice(0, packageEntryEnd)
+  const remainingEntries = entry.slice(packageEntryEnd)
+  const updatedEntry = packageEntry.replace(/(['"]reference['"]\s*=>\s*)(?:NULL|null|'[^']*'|"[^"]*")/, `$1'${reference}'`)
+  if (updatedEntry !== packageEntry) {
+    await writeFile(path, before + updatedEntry + remainingEntries)
+    return
+  }
+
+  // Composer may omit an unavailable reference entirely. Add it to this
+  // package record so InstalledVersions exposes the clean overlay revision.
+  const entryWithReference = packageEntry.replace(
+    new RegExp(`((?:['"]${escapeRegExp(packageName)}['"]\\s*=>\\s*array\\s*\\(\\s*))`),
+    `$1'reference' => '${reference}',\n`,
+  )
+  if (entryWithReference !== packageEntry) {
+    await writeFile(path, before + entryWithReference + remainingEntries)
+  }
+}
+
+function composerInstalledPhpPackageEntryEnd(entry: string, packageName: string): number | undefined {
+  const header = new RegExp(`['"]${escapeRegExp(packageName)}['"]\\s*=>\\s*array\\s*\\(`).exec(entry)
+  if (!header || header.index === undefined) {
+    return undefined
+  }
+
+  const openingParenthesis = header.index + header[0].lastIndexOf("(")
+  let depth = 0
+  let quote = ""
+  for (let index = openingParenthesis; index < entry.length; index++) {
+    const character = entry[index]
+    if (quote) {
+      if (character === "\\") {
+        index++
+      } else if (character === quote) {
+        quote = ""
+      }
+      continue
+    }
+    if (character === "'" || character === '"') {
+      quote = character
+    } else if (character === "(") {
+      depth++
+    } else if (character === ")" && --depth === 0) {
+      return index + 1
+    }
+  }
+  return undefined
+}
+
+function composerInstalledPackageRecords(installed: unknown): Array<Record<string, unknown> & { name?: string }> {
+  if (Array.isArray(installed)) {
+    return installed.filter(isRecord)
+  }
+  return isRecord(installed) && Array.isArray(installed.packages) ? installed.packages.filter(isRecord) : []
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return !!value && typeof value === "object" && !Array.isArray(value)
+}
+
 export async function prepareRecipeStagedFiles(recipe: WorkspaceRecipe, recipeDirectory: string): Promise<PreparedStagedFile[]> {
   const stagedFiles: PreparedStagedFile[] = []
   for (const [index, stagedFile] of (recipe.inputs?.stagedFiles ?? []).entries()) {

scripts/composer-backed-source-hydration-smoke.ts

@@ -1,18 +1,23 @@
 import assert from "node:assert/strict"
+import { execFile as execFileCallback } from "node:child_process"
 import { chmod, mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"
 import { tmpdir } from "node:os"
 import { join } from "node:path"
+import { promisify } from "node:util"
+import { recipeRunDependencyOverlay } from "../packages/cli/src/commands/recipe-runtime-setup.js"
 import { prepareRecipeDependencyOverlays, prepareRecipeRuntimeOverlays } from "../packages/cli/src/recipe-sources.js"
 import type { PreparedExtraPlugin } from "../packages/cli/src/recipe-sources.js"
 import type { WorkspaceRecipe } from "../packages/runtime-core/src/runtime-contracts.js"
 
 const root = await mkdtemp(join(tmpdir(), "wp-codebox-runtime-overlay-hydration-"))
 const overlaySource = join(root, "php-ai-client")
 const dependencySource = join(root, "generic-composer-package")
+const nonGitDependencySource = join(root, "non-git-composer-package")
 const binDir = join(root, "bin")
 const scoperPath = join(root, "php-scoper.phar")
 const originalPath = process.env.PATH
 const originalScoper = process.env.WP_CODEBOX_PHP_SCOPER_PHAR
+const execFile = promisify(execFileCallback)
 
 async function exists(path: string): Promise<boolean> {
   try {
@@ -51,6 +56,35 @@ await writeFile(join(dependencySource, "composer.json"), JSON.stringify({
   autoload: { "psr-4": { "Acme\\Package\\": "src/" } },
   require: { "psr/log": "^3.0" },
 }, null, 2))
+await execFile("git", ["init", "--quiet"], { cwd: dependencySource })
+await execFile("git", ["add", "."], { cwd: dependencySource })
+await execFile("git", ["-c", "user.name=WP Codebox", "-c", "user.email=wp-codebox@example.test", "commit", "--quiet", "-m", "fixture"], { cwd: dependencySource })
+const { stdout: dependencyReference } = await execFile("git", ["rev-parse", "HEAD"], { cwd: dependencySource })
+
+await mkdir(join(nonGitDependencySource, "src"), { recursive: true })
+await writeFile(join(nonGitDependencySource, "src", "Package.php"), "<?php\n")
+await writeFile(join(nonGitDependencySource, "composer.json"), JSON.stringify({ name: "acme/non-git-package" }, null, 2))
+
+const consumerSource = join(root, "consumer-plugin")
+await mkdir(join(consumerSource, "vendor", "composer"), { recursive: true })
+await writeFile(join(consumerSource, "vendor", "composer", "installed.json"), JSON.stringify({ packages: [
+  { name: "acme/package", version: "1.0.0+no-version-set" },
+  { name: "acme/non-git-package", version: "1.0.0+no-version-set" },
+] }, null, 2))
+await writeFile(join(consumerSource, "vendor", "composer", "installed.php"), `<?php
+
+return array(
+  'versions' => array(
+    'acme/package' => array(
+      'pretty_version' => '1.0.0+no-version-set',
+    ),
+    'acme/non-git-package' => array(
+      'pretty_version' => '1.0.0+no-version-set',
+      'reference' => NULL,
+    ),
+  ),
+);
+`)
 
 await writeFile(scoperPath, `<?php
 $workingDir = '';
@@ -116,38 +150,66 @@ const recipe: WorkspaceRecipe = {
 }
 
 const overlays = await prepareRecipeRuntimeOverlays(recipe, root)
+const consumers: PreparedExtraPlugin[] = [{
+  source: consumerSource,
+  slug: "consumer-plugin",
+  target: "/wordpress/wp-content/plugins/consumer-plugin",
+  pluginFile: "consumer-plugin.php",
+  activate: true,
+  loadAs: "plugin",
+  cleanupPaths: [],
+  provenance: { kind: "local", original: consumerSource },
+}]
 const dependencyOverlays = await prepareRecipeDependencyOverlays({
   inputs: {
     dependency_overlays: [{
       kind: "composer-package",
       package: "acme/package",
       source: dependencySource,
       consumer: "consumer-plugin",
+    }, {
+      kind: "composer-package",
+      package: "acme/non-git-package",
+      source: nonGitDependencySource,
+      consumer: "consumer-plugin",
     }],
   },
-}, root, [{
-  source: join(root, "consumer-plugin"),
-  slug: "consumer-plugin",
-  target: "/wordpress/wp-content/plugins/consumer-plugin",
-  pluginFile: "consumer-plugin.php",
-  activate: true,
-  loadAs: "plugin",
-  cleanupPaths: [],
-  provenance: { kind: "local", original: join(root, "consumer-plugin") },
-}] satisfies PreparedExtraPlugin[])
+}, root, consumers)
 try {
   assert.equal(overlays.length, 1)
-  assert.equal(dependencyOverlays.length, 1)
+  assert.equal(dependencyOverlays.length, 2)
   assert.equal(await exists(join(overlaySource, "vendor")), false, "overlay source checkout must not be mutated")
   assert.equal(await exists(join(dependencySource, "vendor")), false, "dependency overlay source checkout must not be mutated")
   assert.equal(await exists(join(overlays[0].source, "autoload.php")), true)
   assert.equal(await exists(join(overlays[0].source, "src", "Client.php")), true)
   assert.equal(await exists(join(overlays[0].source, "third-party", "Psr", "Log", "LoggerInterface.php")), true)
   assert.equal(await exists(join(dependencyOverlays[0].source, "vendor", "composer", "installed.json")), true)
   assert.equal(dependencyOverlays[0].target, "/wordpress/wp-content/plugins/consumer-plugin/vendor/acme/package")
+  assert.equal(dependencyOverlays[0].reference, dependencyReference.trim(), "clean Git source revision survives Composer staging")
+  assert.equal(dependencyOverlays[0].metadata.reference, dependencyReference.trim(), "mounted dependency metadata preserves the source revision")
+  assert.equal(recipeRunDependencyOverlay(dependencyOverlays[0]).reference, dependencyReference.trim(), "runtime dependency provenance exposes the source revision")
+  assert.equal(dependencyOverlays[1].reference, undefined, "non-Git source has no fabricated revision")
+  assert.equal(dependencyOverlays[1].metadata.reference, undefined, "non-Git source metadata omits the revision")
+  assert.equal((JSON.parse(await readFile(join(consumerSource, "vendor", "composer", "installed.json"), "utf8")) as { packages: Array<{ source?: unknown }> }).packages[0].source, undefined, "original consumer Composer provenance remains unchanged")
+  const runtimeInstalled = JSON.parse(await readFile(join(consumers[0].source, "vendor", "composer", "installed.json"), "utf8")) as { packages: Array<{ name: string, version: string, source?: { reference?: string } }> }
+  assert.deepEqual(runtimeInstalled.packages[0], { name: "acme/package", version: "1.0.0+no-version-set", source: { reference: dependencyReference.trim() } }, "runtime Composer dependency provenance includes the immutable source reference")
+  assert.deepEqual(runtimeInstalled.packages[1], { name: "acme/non-git-package", version: "1.0.0+no-version-set" }, "runtime Composer provenance omits unresolved source references")
+  const { stdout: runtimeInstalledPhp } = await execFile("php", ["-r", "echo json_encode(require $argv[1]);", join(consumers[0].source, "vendor", "composer", "installed.php")])
+  const runtimePhpVersions = JSON.parse(runtimeInstalledPhp) as { versions: Record<string, { reference?: string | null }> }
+  assert.equal(runtimePhpVersions.versions["acme/package"]?.reference, dependencyReference.trim(), "Composer runtime metadata includes the immutable source reference")
+  assert.equal(runtimePhpVersions.versions["acme/non-git-package"]?.reference, null, "Composer runtime metadata leaves unresolved references unchanged")
+  const runtimePackageRow = runtimePhpVersions.versions["acme/package"]
+  assert.deepEqual(runtimePackageRow, {
+    pretty_version: "1.0.0+no-version-set",
+    reference: dependencyReference.trim(),
+  }, "the final PHP-visible package row exposes the clean Git reference")
+  assert.deepEqual(runtimePhpVersions.versions["acme/non-git-package"], {
+    pretty_version: "1.0.0+no-version-set",
+    reference: null,
+  }, "the final PHP-visible package row leaves an unavailable reference unchanged")
   assert.match(await readFile(join(overlays[0].source, "src", "Client.php"), "utf8"), /WordPress\\AiClientDependencies\\Psr\\Log\\LoggerInterface/)
 } finally {
-  await Promise.all([...overlays, ...dependencyOverlays].flatMap((overlay) => overlay.cleanupPaths).map((path) => rm(path, { recursive: true, force: true })))
+  await Promise.all([...overlays, ...dependencyOverlays, ...consumers].flatMap((overlay) => overlay.cleanupPaths).map((path) => rm(path, { recursive: true, force: true })))
   process.env.PATH = originalPath
   if (originalScoper === undefined) {
     delete process.env.WP_CODEBOX_PHP_SCOPER_PHAR