Skip to content

Commit 03b7889

Browse files
feat: add persistent repository metadata cache
- introduce a reusable persistent cache helper plus unit tests - wire the cache into updateRepositoryApiData to reuse successful API responses - persist cache files between CI runs via GitHub Actions and ignore them in Git
1 parent 038ad6f commit 03b7889

7 files changed

Lines changed: 562 additions & 172 deletions

File tree

.github/workflows/container-build.yaml

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,17 @@ jobs:
2323
steps:
2424
- name: Checkout code
2525
uses: actions/checkout@v6
26+
- name: Restore repository API cache
27+
id: repository-api-cache-restore
28+
uses: actions/cache/restore@v4
29+
with:
30+
path: website/data/cache
31+
key: repository-api-cache-${{ github.ref_name }}-${{ github.run_id }}
32+
restore-keys: |
33+
repository-api-cache-${{ github.ref_name }}-
34+
repository-api-cache-
35+
- name: Ensure cache directory exists
36+
run: mkdir -p website/data/cache
2637
- name: Build container
2738
run: |
2839
RUN_KIND="${{ github.event_name }}"
@@ -48,3 +59,9 @@ jobs:
4859
--local dockerfile=container \
4960
--opt build-arg:GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \
5061
$PARAMS
62+
- name: Save repository API cache
63+
if: always()
64+
uses: actions/cache/save@v4
65+
with:
66+
path: website/data/cache
67+
key: repository-api-cache-${{ github.ref_name }}-${{ github.run_id }}

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,4 @@
77
.idea
88
.vscode/*
99
ownModuleList.json
10+
website/data/cache/

docs/pipeline-refactor-roadmap.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ This document captures the long-term improvements we want to implement in the mo
3737

3838
| Task | Description | Dependencies | Effort |
3939
| ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ | ------ |
40-
| P3.1 | Add persistent caches for API responses and HEAD validations with expiration metadata | none | M |
40+
| P3.1 | Add persistent caches for API responses and HEAD validations with expiration metadata ✅ Completed Oct 2025 | none | M |
4141
| P3.1.5 | Implement smart incremental checking: skip modules when (A) module has no new commits since last check AND (B) this repository has no new commits since last check; reuse cached results from modules.magicmirror.builders for unchanged modules to dramatically reduce check stage runtime | P3.1 | M |
4242
| P3.2 | Introduce a central rate limiter + retry strategy for GitHub/GitLab requests | P3.1 | M |
4343
| P3.3 | Capture structured logs (JSON) and aggregate per-stage timing metrics | P1.2 | M |
@@ -100,7 +100,7 @@ Routine reminders for keeping the written guidance in sync with the code:
100100

101101
Immediate action items:
102102

103-
1. **P3.1**Add persistent caches for API responses and HEAD validations with expiration metadata (foundation for incremental checking and rate-limit reduction).
103+
1. **P3.1.5**Implement smart incremental checking: reuse cached results when modules and their repositories haven’t changed since the last run.
104104
2. Recommend `npm ci --omit=dev` when modules list devDependencies in instructions (P4.7).
105105
3. Flag modules with multi-year inactivity that are not marked `outdated` and nudge maintainers to review status (P4.8).
106106
4. Inspect Dependabot configs for schedule scope (quarterly cadence, production-only) and suggest adjustments (P4.9).
Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
import {equal, ok} from "node:assert/strict";
2+
import {mkdtemp, readFile} from "node:fs/promises";
3+
import {createPersistentCache} from "../../shared/persistent-cache.js";
4+
import {join} from "node:path";
5+
import {test} from "node:test";
6+
import {tmpdir} from "node:os";
7+
8+
async function createTempFilePath (prefix = "cache-test-") {
9+
const dir = await mkdtemp(join(tmpdir(), prefix));
10+
return join(dir, "cache.json");
11+
}
12+
13+
function createClock () {
14+
let now = Date.now();
15+
return {
16+
now () {
17+
return now;
18+
},
19+
advance (ms) {
20+
now += ms;
21+
return now;
22+
}
23+
};
24+
}
25+
26+
test("persistent cache stores and retrieves values with ttl", async () => {
27+
const clock = createClock();
28+
const filePath = await createTempFilePath();
29+
const cache = createPersistentCache({filePath, now: () => clock.now(), defaultTtlMs: 1000});
30+
31+
await cache.load();
32+
const inserted = cache.set("foo", {answer: 42});
33+
equal(inserted.value.answer, 42);
34+
35+
const hit = cache.get("foo");
36+
ok(hit, "expected cache hit");
37+
equal(hit.value.answer, 42);
38+
ok(hit.expiresAt, "should set expiration");
39+
40+
clock.advance(1500);
41+
const expired = cache.get("foo");
42+
equal(expired, null, "entry should expire after ttl");
43+
});
44+
45+
test("persistent cache persists entries to disk", async () => {
46+
const clock = createClock();
47+
const filePath = await createTempFilePath();
48+
const cache = createPersistentCache({filePath, now: () => clock.now(), defaultTtlMs: 0});
49+
50+
await cache.load();
51+
cache.set("alpha", {value: "one"});
52+
await cache.flush();
53+
54+
const contents = await readFile(filePath, "utf8");
55+
ok(contents.includes("alpha"));
56+
57+
const second = createPersistentCache({filePath, now: () => clock.now(), defaultTtlMs: 0});
58+
await second.load();
59+
const hit = second.get("alpha");
60+
ok(hit, "expected persisted hit");
61+
equal(hit.value.value, "one");
62+
});
63+
64+
test("persistent cache supports per-entry ttl overrides", async () => {
65+
const clock = createClock();
66+
const filePath = await createTempFilePath();
67+
const cache = createPersistentCache({filePath, now: () => clock.now(), defaultTtlMs: 5000});
68+
69+
await cache.load();
70+
cache.set("short", {value: 1}, {ttlMs: 1000});
71+
cache.set("long", {value: 2});
72+
73+
clock.advance(1500);
74+
75+
const short = cache.get("short");
76+
const longer = cache.get("long");
77+
78+
equal(short, null, "short ttl should expire");
79+
ok(longer, "long ttl should persist");
80+
equal(longer.value.value, 2);
81+
});

scripts/shared/persistent-cache.js

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
import {ensureDirectory, fileExists, readJson, writeJson} from "./fs-utils.js";
2+
import path from "node:path";
3+
4+
const DEFAULT_VERSION = 1;
5+
6+
function normalizeKey (key) {
7+
if (typeof key !== "string" || key.length === 0) {
8+
throw new TypeError("Persistent cache keys must be non-empty strings");
9+
}
10+
return key;
11+
}
12+
13+
function cloneCacheEntry (entry) {
14+
const cloned = {
15+
value: structuredClone(entry.value),
16+
updatedAt: entry.updatedAt,
17+
expiresAt: entry.expiresAt,
18+
ttlMs: entry.ttlMs
19+
};
20+
if (entry.metadata) {
21+
cloned.metadata = structuredClone(entry.metadata);
22+
}
23+
return cloned;
24+
}
25+
26+
function buildStoredEntry ({value, metadata, expiresAt, timestamp, ttlMs}) {
27+
const stored = {
28+
value: structuredClone(value),
29+
updatedAt: timestamp,
30+
expiresAt,
31+
ttlMs: ttlMs ?? null
32+
};
33+
if (metadata) {
34+
stored.metadata = structuredClone(metadata);
35+
}
36+
return stored;
37+
}
38+
39+
function toIsoString (timestamp) {
40+
return new Date(timestamp).toISOString();
41+
}
42+
43+
function isExpired ({expiresAt}, now) {
44+
if (!expiresAt) {
45+
return false;
46+
}
47+
48+
return new Date(expiresAt).getTime() <= now;
49+
}
50+
51+
function createEmptyState ({version}) {
52+
return {
53+
version,
54+
generatedAt: toIsoString(Date.now()),
55+
entries: {}
56+
};
57+
}
58+
59+
export function createPersistentCache ({filePath, version = DEFAULT_VERSION, defaultTtlMs = 0, now = () => Date.now()} = {}) {
60+
if (typeof filePath !== "string" || filePath.length === 0) {
61+
throw new TypeError("createPersistentCache requires a non-empty filePath");
62+
}
63+
64+
const resolvedPath = path.resolve(filePath);
65+
const state = createEmptyState({version});
66+
let loaded = false;
67+
let dirty = false;
68+
69+
async function load () {
70+
if (loaded) {
71+
return;
72+
}
73+
74+
if (await fileExists(resolvedPath)) {
75+
try {
76+
const stored = await readJson(resolvedPath);
77+
if (stored && typeof stored === "object" && stored.entries) {
78+
state.version = stored.version ?? version;
79+
state.generatedAt = stored.generatedAt ?? toIsoString(now());
80+
state.entries = stored.entries ?? {};
81+
}
82+
} catch (error) {
83+
const message = error instanceof Error ? error.message : String(error);
84+
throw new Error(`Failed to load persistent cache at ${resolvedPath}: ${message}`, {cause: error});
85+
}
86+
}
87+
88+
loaded = true;
89+
await pruneExpired();
90+
}
91+
92+
function snapshot () {
93+
return {
94+
version: state.version,
95+
generatedAt: state.generatedAt,
96+
entries: state.entries
97+
};
98+
}
99+
100+
function pruneExpired () {
101+
const current = now();
102+
let removed = false;
103+
104+
for (const [key, entry] of Object.entries(state.entries)) {
105+
if (isExpired(entry, current)) {
106+
delete state.entries[key];
107+
removed = true;
108+
}
109+
}
110+
111+
if (removed) {
112+
dirty = true;
113+
}
114+
}
115+
116+
function get (key) {
117+
const normalizedKey = normalizeKey(key);
118+
const entry = state.entries[normalizedKey];
119+
120+
if (!entry) {
121+
return null;
122+
}
123+
124+
if (isExpired(entry, now())) {
125+
delete state.entries[normalizedKey];
126+
dirty = true;
127+
return null;
128+
}
129+
130+
return cloneCacheEntry(entry);
131+
}
132+
133+
function set (key, value, {ttlMs = defaultTtlMs, metadata} = {}) {
134+
const normalizedKey = normalizeKey(key);
135+
const current = now();
136+
const expiresAt = ttlMs && ttlMs > 0 ? toIsoString(current + ttlMs) : null;
137+
138+
state.entries[normalizedKey] = buildStoredEntry({
139+
value,
140+
metadata,
141+
expiresAt,
142+
timestamp: toIsoString(current),
143+
ttlMs
144+
});
145+
dirty = true;
146+
return cloneCacheEntry(state.entries[normalizedKey]);
147+
}
148+
149+
function deleteKey (key) {
150+
const normalizedKey = normalizeKey(key);
151+
if (Object.hasOwn(state.entries, normalizedKey)) {
152+
delete state.entries[normalizedKey];
153+
dirty = true;
154+
}
155+
}
156+
157+
function entries () {
158+
return Object.entries(state.entries).map(([key, entry]) => ({key, entry: cloneCacheEntry(entry)}));
159+
}
160+
161+
async function flush () {
162+
if (!loaded || !dirty) {
163+
return;
164+
}
165+
166+
const dirPath = path.dirname(resolvedPath);
167+
await ensureDirectory(dirPath);
168+
169+
const snapshotData = snapshot();
170+
snapshotData.generatedAt = toIsoString(now());
171+
await writeJson(resolvedPath, snapshotData, {pretty: 2, ensureDir: true});
172+
dirty = false;
173+
}
174+
175+
return {
176+
load,
177+
flush,
178+
get,
179+
set,
180+
delete: deleteKey,
181+
entries,
182+
pruneExpired,
183+
snapshot
184+
};
185+
}

0 commit comments

Comments
 (0)