|
3 | 3 | */ |
4 | 4 |
|
5 | 5 | import { createHash } from 'node:crypto'; |
6 | | -import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync, unlinkSync, writeFileSync } from 'node:fs'; |
7 | | -import { dirname, resolve } from 'node:path'; |
| 6 | +import { |
| 7 | + copyFileSync, |
| 8 | + existsSync, |
| 9 | + mkdirSync, |
| 10 | + readdirSync, |
| 11 | + readFileSync, |
| 12 | + rmSync, |
| 13 | + unlinkSync, |
| 14 | + writeFileSync, |
| 15 | +} from 'node:fs'; |
| 16 | +import { dirname, relative, resolve, sep } from 'node:path'; |
8 | 17 | import { fileURLToPath } from 'node:url'; |
9 | 18 |
|
10 | 19 | const __dirname = dirname(fileURLToPath(import.meta.url)); |
@@ -65,12 +74,97 @@ export function cleanArgs(args) { |
65 | 74 | return rest; |
66 | 75 | } |
67 | 76 |
|
| 77 | +// --- SDK fingerprint (for cache invalidation) --- |
| 78 | + |
| 79 | +const SDK_TOOLS_DIR = resolve(EVALS_ROOT, '..', 'packages/sdk/tools'); |
| 80 | +const SDK_DIST_DIR = resolve(EVALS_ROOT, '..', 'packages/sdk/langs/node/dist'); |
| 81 | +const SDK_FINGERPRINT_FILES = [ |
| 82 | + resolve(SDK_TOOLS_DIR, 'tools.vercel.json'), |
| 83 | + resolve(SDK_TOOLS_DIR, 'tools.openai.json'), |
| 84 | + PATHS.prompt, |
| 85 | + PATHS.cliBin, |
| 86 | +]; |
| 87 | +const SDK_FINGERPRINT_DIRECTORIES = [SDK_DIST_DIR]; |
| 88 | + |
| 89 | +function normalizeFingerprintPath(path) { |
| 90 | + return (path || '.').split(sep).join('/'); |
| 91 | +} |
| 92 | + |
| 93 | +function updateHashWithFile(hash, filePath, rootPath = dirname(filePath)) { |
| 94 | + const fingerprintPath = normalizeFingerprintPath(relative(rootPath, filePath)); |
| 95 | + hash.update(`file:${fingerprintPath}\n`); |
| 96 | + hash.update(readFileSync(filePath)); |
| 97 | +} |
| 98 | + |
| 99 | +function updateHashWithDirectory(hash, dirPath, rootPath = dirPath) { |
| 100 | + const fingerprintPath = normalizeFingerprintPath(relative(rootPath, dirPath)); |
| 101 | + hash.update(`dir:${fingerprintPath}\n`); |
| 102 | + |
| 103 | + let entries; |
| 104 | + try { |
| 105 | + entries = readdirSync(dirPath, { withFileTypes: true }) |
| 106 | + .sort((a, b) => a.name.localeCompare(b.name)); |
| 107 | + } catch { |
| 108 | + hash.update(`missing-dir:${dirPath}\n`); |
| 109 | + return; |
| 110 | + } |
| 111 | + |
| 112 | + for (const entry of entries) { |
| 113 | + const entryPath = resolve(dirPath, entry.name); |
| 114 | + const entryFingerprintPath = normalizeFingerprintPath(relative(rootPath, entryPath)); |
| 115 | + |
| 116 | + if (entry.isDirectory()) { |
| 117 | + updateHashWithDirectory(hash, entryPath, rootPath); |
| 118 | + continue; |
| 119 | + } |
| 120 | + |
| 121 | + if (entry.isFile()) { |
| 122 | + updateHashWithFile(hash, entryPath, rootPath); |
| 123 | + continue; |
| 124 | + } |
| 125 | + |
| 126 | + hash.update(`other:${entryFingerprintPath}\n`); |
| 127 | + } |
| 128 | +} |
| 129 | + |
| 130 | +/** |
| 131 | + * Compute the artifact fingerprint used to invalidate cached eval results when |
| 132 | + * the local tool surface or runtime artifacts change. |
| 133 | + * |
| 134 | + * @param {{files?: string[], directories?: string[]}} [options] |
| 135 | + * @returns {string} |
| 136 | + */ |
| 137 | +export function computeSdkFingerprint({ |
| 138 | + files = SDK_FINGERPRINT_FILES, |
| 139 | + directories = SDK_FINGERPRINT_DIRECTORIES, |
| 140 | +} = {}) { |
| 141 | + const hash = createHash('sha256'); |
| 142 | + for (const file of [...files].sort()) { |
| 143 | + try { |
| 144 | + updateHashWithFile(hash, file); |
| 145 | + } catch { |
| 146 | + hash.update(`missing:${file}`); |
| 147 | + } |
| 148 | + } |
| 149 | + |
| 150 | + for (const directory of [...directories].sort()) { |
| 151 | + updateHashWithDirectory(hash, directory); |
| 152 | + } |
| 153 | + |
| 154 | + return hash.digest('hex').slice(0, 12); |
| 155 | +} |
| 156 | + |
| 157 | +const SDK_FINGERPRINT = computeSdkFingerprint(); |
| 158 | + |
68 | 159 | // --- Cache --- |
69 | 160 |
|
70 | | -/** Generate a cache key from model + fixture + task + prompt hash. */ |
| 161 | +/** Generate a cache key from model + fixture + task + prompt hash + SDK fingerprint. */ |
71 | 162 | export function cacheKey(model, fixture, task, prompt) { |
72 | 163 | const promptSig = prompt ? createHash('sha256').update(prompt).digest('hex').slice(0, 8) : ''; |
73 | | - const hash = createHash('sha256').update(`${model}|${fixture}|${task}|${promptSig}`).digest('hex').slice(0, 16); |
| 164 | + const hash = createHash('sha256') |
| 165 | + .update(`${model}|${fixture}|${task}|${promptSig}|${SDK_FINGERPRINT}`) |
| 166 | + .digest('hex') |
| 167 | + .slice(0, 16); |
74 | 168 | return hash; |
75 | 169 | } |
76 | 170 |
|
|
0 commit comments