infra: align embed-llamacpp integration model helpers with diffusion-cpp

Make the integration-test model tooling uniform with diffusion-cpp:
- ensureModel now takes an object ({ modelName, modelDir, manifest, download }),
  matching diffusion-cpp/llm-llamacpp; update all callsites
- log the "no sha256/bytes pinned — integrity check SKIPPED" hint pointing at
  scripts/generate-model-manifest.mjs, same as diffusion
- rename the pinning script to scripts/generate-model-manifest.mjs (verbatim
  diffusion copy) and the unit suite to test/unit/ensure-model-integrity.test.js
- manifest integrity stays null until pinned in CI (HF_TOKEN + network); the
  suite tolerates null now and enforces shape once pinned

Ref: QVAC-21937
Co-authored-by: Cursor <cursoragent@cursor.com>

Author: tobi-legan

…-llamacpp/scripts/pin-model-manifest.mjs …acpp/scripts/generate-model-manifest.mjspackages/embed-llamacpp/scripts/pin-model-manifest.mjs renamed to packages/embed-llamacpp/scripts/generate-model-manifest.mjs packages/embed-llamacpp/scripts/pin-model-manifest.mjs renamed to packages/embed-llamacpp/scripts/generate-model-manifest.mjs

@@ -4,16 +4,13 @@
 // Populates sha256 + bytes in test/integration/models.manifest.json by
 // downloading each pinned URL FRESH into a temp directory and hashing it.
 //
-// This is the integration-test model cache manifest (single source of truth for
-// URLs + integrity, and the .github/actions/cache-models cache key).
-//
 // IMPORTANT (integrity provenance): shas are computed from a clean download of
 // the pinned URL, never from packages/embed-llamacpp/test/model (which may be a
 // stale or poisoned restored cache). Run this on a machine/CI with network and
 // an HF_TOKEN for gated repos.
 //
 // Usage:
-//   HF_TOKEN=hf_xxx node scripts/pin-model-manifest.mjs [--only <modelName>] [--force]
+//   HF_TOKEN=hf_xxx node scripts/generate-model-manifest.mjs [--only <modelName>] [--force]
 //
 // Flags:
 //   --only <name>   Only (re)generate the named model entry.
@@ -31,7 +28,7 @@ import https from 'node:https'
 const __dirname = dirname(fileURLToPath(import.meta.url))
 const MANIFEST_PATH = resolve(__dirname, '../test/integration/models.manifest.json')
 
-function parseArgs (argv) {
+function parseArgs(argv) {
   const args = { only: null, force: false }
   for (let i = 0; i < argv.length; i++) {
     if (argv[i] === '--only') args.only = argv[++i]
@@ -40,15 +37,15 @@ function parseArgs (argv) {
   return args
 }
 
-function authHeaders (url) {
+function authHeaders(url) {
   const headers = { 'user-agent': 'qvac-manifest-generator' }
   if (url.includes('huggingface.co') && process.env.HF_TOKEN) {
     headers.authorization = `Bearer ${process.env.HF_TOKEN}`
   }
   return headers
 }
 
-function download (url, dest, redirectsLeft = 10) {
+function download(url, dest, redirectsLeft = 10) {
   return new Promise((resolve, reject) => {
     const req = https.get(url, { headers: authHeaders(url) }, (res) => {
       if ([301, 302, 307, 308].includes(res.statusCode)) {
@@ -69,7 +66,7 @@ function download (url, dest, redirectsLeft = 10) {
 
 // Hash is streamed because fs.readFile is hard-capped at 2 GiB
 // (kIoMaxLength) and most of these model files are larger than that.
-function sha256Stream (filePath) {
+function sha256Stream(filePath) {
   return new Promise((resolve, reject) => {
     const hash = createHash('sha256')
     const stream = createReadStream(filePath)
@@ -79,13 +76,13 @@ function sha256Stream (filePath) {
   })
 }
 
-async function sha256AndSize (filePath) {
+async function sha256AndSize(filePath) {
   const sha256 = await sha256Stream(filePath)
   const { size } = await stat(filePath)
   return { sha256, bytes: size }
 }
 
-async function main () {
+async function main() {
   const args = parseArgs(process.argv.slice(2))
   const manifest = JSON.parse(await readFile(MANIFEST_PATH, 'utf8'))
   const tmp = await mkdtemp(join(tmpdir(), 'qvac-manifest-'))

packages/embed-llamacpp/test/integration/addon.test.js

@@ -747,7 +747,7 @@ const DEVICE_API = isDarwinX64 || isLinuxArm64 ? 'cpu' : 'gpu'
 const MODEL_NAME_API = getModelConfigs()[0]?.modelName ?? 'embeddinggemma-300M-Q8_0.gguf'
 
 async function setupModelApiBehavior(t) {
-  await ensureModel(MODEL_NAME_API)
+  await ensureModel({ modelName: MODEL_NAME_API })
   const { inference } = await createEmbeddingsTestInstance(
     t,
     MODEL_NAME_API,

packages/embed-llamacpp/test/integration/multi-gpu.test.js

@@ -37,7 +37,7 @@ async function runMultiGpuTest(t, extraConfig, assertDevices) {
     return
   }
 
-  const [modelName, dirPath] = await ensureModel(MODEL.name)
+  const [modelName, dirPath] = await ensureModel({ modelName: MODEL.name })
   const modelPath = path.join(dirPath, modelName)
   const specLogger = attachSpecLogger({ forwardToConsole: true })
 

packages/embed-llamacpp/test/integration/utils.js

@@ -8,7 +8,7 @@ const os = require('bare-os')
 // Lazily loaded: requiring index.js pulls in the native binding, which isn't
 // needed for the model download/integrity helpers and lets those run in unit
 // tests without a compiled addon.
-function getGGMLBert () {
+function getGGMLBert() {
   return require('../../index.js')
 }
 
@@ -217,7 +217,7 @@ let _manifestCache
 // Loads and caches the model manifest (single source of truth for model URLs +
 // sha256/bytes integrity). Returns null when absent so callers fall back to the
 // URL declared in MODEL_CONFIGS.
-function loadManifest (manifestPath = DEFAULT_MANIFEST_PATH) {
+function loadManifest(manifestPath = DEFAULT_MANIFEST_PATH) {
   if (manifestPath === DEFAULT_MANIFEST_PATH && _manifestCache !== undefined) {
     return _manifestCache
   }
@@ -231,13 +231,13 @@ function loadManifest (manifestPath = DEFAULT_MANIFEST_PATH) {
   return parsed
 }
 
-function resolveModelEntry (modelName, { manifest } = {}) {
+function resolveModelEntry(modelName, { manifest } = {}) {
   const m = manifest !== undefined ? manifest : loadManifest()
   if (!m || !m.models) return null
   return m.models[modelName] || null
 }
 
-function entryHasIntegrity (entry) {
+function entryHasIntegrity(entry) {
   if (!entry) return false
   const hasSha = typeof entry.sha256 === 'string' && entry.sha256.length === 64
   const hasBytes = Number.isInteger(entry.bytes)
@@ -246,12 +246,20 @@ function entryHasIntegrity (entry) {
 
 // Streaming sha256 via bare-crypto. Returns null when bare-crypto is
 // unavailable so callers can degrade rather than fail.
-async function sha256File (filePath) {
+async function sha256File(filePath) {
   let crypto
-  try { crypto = require('bare-crypto') } catch (_) { return null }
+  try {
+    crypto = require('bare-crypto')
+  } catch (_) {
+    return null
+  }
   return await new Promise((resolve, reject) => {
     let hash
-    try { hash = crypto.createHash('sha256') } catch (_) { return resolve(null) }
+    try {
+      hash = crypto.createHash('sha256')
+    } catch (_) {
+      return resolve(null)
+    }
     const stream = fs.createReadStream(filePath)
     stream.on('data', (chunk) => hash.update(chunk))
     stream.on('error', reject)
@@ -262,7 +270,7 @@ async function sha256File (filePath) {
 // Verifies a model file against a manifest entry. Byte-length is checked first
 // (cheap) so a size mismatch fails fast before hashing a multi-GB file.
 // { ok: true } when it passes (or when no integrity value is pinned yet).
-async function verifyModelFile (filePath, entry) {
+async function verifyModelFile(filePath, entry) {
   let stats
   try {
     stats = fs.statSync(filePath)
@@ -279,7 +287,9 @@ async function verifyModelFile (filePath, entry) {
   if (hasSha) {
     const got = await sha256File(filePath)
     if (got === null) {
-      console.log('[download] WARNING: sha256 pinned for this model but bare-crypto is unavailable — integrity NOT verified')
+      console.log(
+        '[download] WARNING: sha256 pinned for this model but bare-crypto is unavailable — integrity NOT verified'
+      )
       return { ok: true, unverified: true }
     }
     if (got !== entry.sha256.toLowerCase()) {
@@ -293,8 +303,12 @@ async function verifyModelFile (filePath, entry) {
 
 // Counts real download attempts so warm-vs-cold behaviour is unit-testable.
 let _downloadCount = 0
-function getDownloadCount () { return _downloadCount }
-function resetDownloadCount () { _downloadCount = 0 }
+function getDownloadCount() {
+  return _downloadCount
+}
+function resetDownloadCount() {
+  _downloadCount = 0
+}
 
 /**
  * Model configurations for testing
@@ -356,10 +370,12 @@ function prestagedModelDir(modelName) {
 
 /**
  * Ensures the model file exists, downloading it if necessary
- * @param {string} modelName - The model name to ensure
+ * @param {Object} opts
+ * @param {string} opts.modelName - The model name to ensure
  * @returns {Promise<[string, string]>} Returns [modelName, modelDir]
  */
-async function ensureModel (modelName, { modelDir = path.resolve(__dirname, '../model'), manifest, download } = {}) {
+async function ensureModel({ modelName, modelDir, manifest, download } = {}) {
+  const dir = modelDir || path.resolve(__dirname, '../model')
   const modelConfig = getModelConfig(modelName)
 
   // Model URL + sha256/bytes come from models.manifest.json (by modelName); the
@@ -373,7 +389,7 @@ async function ensureModel (modelName, { modelDir = path.resolve(__dirname, '../
     throw new Error(`Unknown model: ${modelName}`)
   }
 
-  const modelPath = path.join(modelDir, modelName)
+  const modelPath = path.join(dir, modelName)
   const doDownload = download || downloadFileWithRetries
   const urls = hasManifestUrls ? entry.urls : [modelConfig.downloadUrl]
   const hasIntegrity = entryHasIntegrity(entry)
@@ -383,17 +399,27 @@ async function ensureModel (modelName, { modelDir = path.resolve(__dirname, '../
       const res = await verifyModelFile(modelPath, entry)
       if (res.ok) {
         console.log(`[download] ${modelName}: cached copy verified, skipping download`)
-        return [modelName, modelDir]
+        return [modelName, dir]
       }
-      console.log(`[download] ${modelName}: cached copy failed integrity (${res.reason}); deleting and re-downloading`)
-      try { fs.unlinkSync(modelPath) } catch (_) {}
+      console.log(
+        `[download] ${modelName}: cached copy failed integrity (${res.reason}); deleting and re-downloading`
+      )
+      try {
+        fs.unlinkSync(modelPath)
+      } catch (_) {}
     } else {
       const stat = fs.statSync(modelPath)
-      if (stat.size > 0) {
-        return [modelName, modelDir]
+      if (stat.size === 0) {
+        console.log(`[download] Removing zero-byte cached file: ${modelName}`)
+        fs.unlinkSync(modelPath)
+      } else {
+        if (entry) {
+          console.log(
+            `[download] ${modelName}: no sha256/bytes pinned in manifest — integrity check SKIPPED (run scripts/generate-model-manifest.mjs to pin)`
+          )
+        }
+        return [modelName, dir]
       }
-      console.log(`[download] Removing zero-byte cached file: ${modelName}`)
-      fs.unlinkSync(modelPath)
     }
   }
 
@@ -403,18 +429,18 @@ async function ensureModel (modelName, { modelDir = path.resolve(__dirname, '../
   // load() writes sibling files next to the model (e.g. openclCacheDir).
   const staged = prestagedModelDir(modelName)
   if (staged) {
-    fs.mkdirSync(modelDir, { recursive: true })
+    fs.mkdirSync(dir, { recursive: true })
     console.log(`[prestage] Using pre-staged model ${modelName} (copying into writable modelDir)`)
     fs.copyFileSync(path.join(staged, modelName), modelPath)
     const stat = fs.statSync(modelPath)
     if (stat.size === 0) {
       fs.unlinkSync(modelPath)
       throw new Error(`[prestage] copied model ${modelName} is empty`)
     }
-    return [modelName, modelDir]
+    return [modelName, dir]
   }
 
-  fs.mkdirSync(modelDir, { recursive: true })
+  fs.mkdirSync(dir, { recursive: true })
   console.log(`[download] Downloading test model: ${modelName}...`)
   _downloadCount++
 
@@ -423,15 +449,19 @@ async function ensureModel (modelName, { modelDir = path.resolve(__dirname, '../
   if (hasIntegrity) {
     const res = await verifyModelFile(modelPath, entry)
     if (!res.ok) {
-      try { fs.unlinkSync(modelPath) } catch (_) {}
-      throw new Error(`[download] ${modelName}: freshly downloaded file failed integrity: ${res.reason}`)
+      try {
+        fs.unlinkSync(modelPath)
+      } catch (_) {}
+      throw new Error(
+        `[download] ${modelName}: freshly downloaded file failed integrity: ${res.reason}`
+      )
     }
   }
 
   const stat = fs.statSync(modelPath)
   console.log(`[download] Model ready: ${(stat.size / 1024 / 1024).toFixed(1)}MB`)
 
-  return [modelName, modelDir]
+  return [modelName, dir]
 }
 
 /**
@@ -471,7 +501,7 @@ async function createEmbeddingsTestInstance(
   gpuLayers = null,
   batchSize = '1024'
 ) {
-  const [, modelDir] = await ensureModel(modelName)
+  const [, modelDir] = await ensureModel({ modelName })
   const modelPath = path.join(modelDir, modelName)
 
   t.ok(fs.existsSync(modelPath), 'Model file should exist')