Skip to content

Commit b872124

Browse files
cigolplcigolpl
authored
Add snapshot serialization and benchmarks (#147)
* Add snapshot serialization and benchmarks * Fix lint warnings in benchmarks and serve script
1 parent 5cbc656 commit b872124

10 files changed

Lines changed: 489 additions & 10 deletions

File tree

‎README.md‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,8 @@ Responsible for defining global configuration. Look for full example here - [con
206206

207207
- **`removeStopWordFilter`** set to `true` if you want to remove the stopWordFilter. See https://github.com/itemsapi/itemsjs/issues/46.
208208

209+
- **`fulltextSnapshot`** / **`facetsSnapshot`** optional prebuilt snapshots (from `serializeAll` or `serializeFulltext`/`serializeFacets`) to skip rebuilding indexes on cold start.
210+
209211
### `itemsjs.search(options)`
210212

211213
#### `options`
@@ -259,3 +261,39 @@ It's used in case you need to reindex the whole data
259261
#### `data`
260262

261263
An array of objects.
264+
265+
## Snapshots (optional)
266+
267+
Fast cold starts without reindexing. Snapshots are plain JSON, so you can store them wherever you like (localStorage, IndexedDB, file, CDN).
268+
269+
**Generating a snapshot**
270+
```js
271+
const engine = itemsjs(data, config);
272+
const snapshot = engine.serializeAll(); // { version, fulltext, facets }
273+
// persist snapshot (e.g., localStorage / IndexedDB / file)
274+
```
275+
276+
**Using a snapshot**
277+
```js
278+
const snapshot = loadSnapshot(); // e.g., JSON.parse(...)
279+
const engine = itemsjs(data, {
280+
...config,
281+
fulltextSnapshot: snapshot.fulltext,
282+
facetsSnapshot: snapshot.facets,
283+
});
284+
```
285+
286+
APIs:
287+
- `itemsjs.serializeFulltext()` → `{ index, store }`
288+
- `itemsjs.serializeFacets()` → `{ bitsData, ids, idsMap }`
289+
- `itemsjs.serializeAll()` → `{ version: 'itemsjs-snapshot-v1', fulltext, facets }`
290+
291+
Snapshots are optional; if you don’t provide them, itemsjs rebuilds indexes as before.
292+
293+
Benchmark (Node):
294+
- Run `npm run benchmark:snapshot` to compare fresh build vs snapshot load (defaults to 1k, 10k and 30k items). Override sizes with `SIZES=5000,20000 npm run benchmark:snapshot`.
295+
- Output includes cold-start speedup ratio (build/load). Note: real-world cost in browser also includes `fetch` + `JSON.parse` time if you download the snapshot.
296+
297+
Browser smoke test (manual/optional):
298+
- Build the bundle: `npm run build`.
299+
- EITHER open `benchmarks/browser-snapshot.html` directly in a browser, OR run `npm run serve:benchmark` and open `http://localhost:4173/` (auto-loads the snapshot page). It builds once, saves a snapshot to `localStorage`, and on refresh loads from it and logs a sample search.

‎benchmarks/browser-snapshot.html‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
<!DOCTYPE html>
2+
<html lang="en">
3+
<head>
4+
<meta charset="UTF-8" />
5+
<title>ItemsJS Snapshot Smoke Test</title>
6+
<style>
7+
body { font-family: sans-serif; padding: 16px; }
8+
pre { background: #f6f8fa; padding: 12px; overflow: auto; }
9+
</style>
10+
</head>
11+
<body>
12+
<h1>ItemsJS Snapshot Smoke Test</h1>
13+
<p style="max-width: 640px;">
14+
A green message below means the snapshot was created and stored in localStorage (first load)
15+
or successfully loaded from it (subsequent refreshes). If it stays on “Loading…” or you see a
16+
red message, check the browser console for errors.
17+
</p>
18+
<p id="status">Loading…</p>
19+
<div id="msg" style="margin: 8px 0; font-weight: bold;"></div>
20+
<pre id="output"></pre>
21+
22+
<script type="module">
23+
import itemsjs from '../dist/index.module.js';
24+
25+
const config = {
26+
searchableFields: ['name', 'category', 'actors', 'name'],
27+
aggregations: {
28+
tags: { title: 'Tags', conjunction: true },
29+
actors: { title: 'Actors', conjunction: true },
30+
year: { title: 'Year', conjunction: true },
31+
in_cinema: { title: 'Is played in Cinema', conjunction: true },
32+
category: { title: 'Category', conjunction: true },
33+
},
34+
};
35+
36+
const log = (obj) => {
37+
document.getElementById('output').textContent += `${obj}\n`;
38+
};
39+
40+
const setStatus = (msg) => {
41+
document.getElementById('status').textContent = msg;
42+
};
43+
44+
const setMsg = (msg, ok = true) => {
45+
const el = document.getElementById('msg');
46+
el.style.color = ok ? 'green' : 'red';
47+
el.textContent = msg;
48+
};
49+
50+
async function loadData() {
51+
const res = await fetch('../tests/fixtures/items.json');
52+
if (!res.ok) throw new Error(`Failed to load data: ${res.status}`);
53+
return res.json();
54+
}
55+
56+
async function run() {
57+
try {
58+
const data = await loadData();
59+
60+
// Try to load snapshot from localStorage if present
61+
const stored = localStorage.getItem('itemsjs-snapshot');
62+
const snap = stored ? JSON.parse(stored) : null;
63+
64+
const engine = itemsjs(data, {
65+
...config,
66+
fulltextSnapshot: snap?.fulltext,
67+
facetsSnapshot: snap?.facets,
68+
});
69+
70+
if (!snap) {
71+
setStatus('Built fresh, creating snapshot…');
72+
const newSnap = engine.serializeAll();
73+
localStorage.setItem('itemsjs-snapshot', JSON.stringify(newSnap));
74+
setMsg('Snapshot created and stored in localStorage.', true);
75+
} else {
76+
setStatus('Loaded from snapshot');
77+
setMsg('Loaded from snapshot in localStorage.', true);
78+
}
79+
80+
const result = engine.search({
81+
query: 'comedy',
82+
filters: { tags: ['a'] },
83+
});
84+
85+
log(JSON.stringify({
86+
status: document.getElementById('status').textContent,
87+
message: document.getElementById('msg').textContent,
88+
total: result.pagination.total,
89+
items: result.data.items.map((v) => v.name),
90+
aggregations: result.data.aggregations.tags.buckets.slice(0, 3),
91+
}, null, 2));
92+
} catch (e) {
93+
setStatus('Error');
94+
setMsg(e.message || 'Error', false);
95+
log(e);
96+
console.error(e);
97+
}
98+
}
99+
100+
run();
101+
</script>
102+
</body>
103+
</html>

‎benchmarks/snapshot.js‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
import itemsjs from '../src/index.js';
2+
import { performance } from 'node:perf_hooks';
3+
4+
const defaultSizes = [1000, 10000, 30000];
5+
const sizes = process.env.SIZES
6+
? process.env.SIZES.split(',').map((v) => parseInt(v, 10)).filter(Boolean)
7+
: defaultSizes;
8+
9+
const tagsPool = Array.from({ length: 40 }, (_, i) => `tag${i}`);
10+
11+
function makeItems(count) {
12+
return Array.from({ length: count }, (_, i) => {
13+
const t1 = tagsPool[i % tagsPool.length];
14+
const t2 = tagsPool[(i * 7) % tagsPool.length];
15+
const t3 = tagsPool[(i * 13) % tagsPool.length];
16+
return {
17+
id: `id-${i}`,
18+
name: `Item ${i} ${t1}`,
19+
tags: [t1, t2, t3],
20+
};
21+
});
22+
}
23+
24+
function runBenchmark(count) {
25+
const data = makeItems(count);
26+
const config = {
27+
searchableFields: ['name', 'tags'],
28+
aggregations: {
29+
tags: { title: 'Tags', size: tagsPool.length },
30+
},
31+
};
32+
33+
const t0 = performance.now();
34+
const engine = itemsjs(data, config);
35+
const t1 = performance.now();
36+
37+
const snapshot = engine.serializeAll();
38+
const t2 = performance.now();
39+
40+
const snapshotJson = JSON.stringify(snapshot);
41+
const t3 = performance.now();
42+
43+
itemsjs(data, {
44+
...config,
45+
fulltextSnapshot: snapshot.fulltext,
46+
facetsSnapshot: snapshot.facets,
47+
});
48+
const t4 = performance.now();
49+
50+
return {
51+
build: t1 - t0,
52+
serialize: t2 - t1,
53+
stringify: t3 - t2,
54+
load: t4 - t3,
55+
snapshotSizeMb: snapshotJson.length / (1024 * 1024),
56+
counts: data.length,
57+
speedup: (t1 - t0) / (t4 - t3),
58+
};
59+
}
60+
61+
function formatMs(value) {
62+
return value.toFixed(1);
63+
}
64+
65+
console.log('Snapshot benchmark (Node) – sizes:', sizes.join(', '));
66+
console.log('Fields: name, tags (3 tags per item), 1 facet (tags)');
67+
console.log('');
68+
69+
sizes.forEach((size) => {
70+
const result = runBenchmark(size);
71+
console.log(`items: ${result.counts}`);
72+
console.log(` build fresh (ms): ${formatMs(result.build)}`);
73+
console.log(` serialize (ms): ${formatMs(result.serialize)}`);
74+
console.log(` stringify (ms): ${formatMs(result.stringify)}`);
75+
console.log(` load snapshot (ms):${formatMs(result.load)}`);
76+
console.log(` snapshot size (MB):${result.snapshotSizeMb.toFixed(2)}`);
77+
console.log(` cold-start speedup (build/load): ${result.speedup.toFixed(2)}x`);
78+
console.log('');
79+
});

‎package-lock.json‎

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
11
{
22
"name": "itemsjs",
3-
"version": "2.2.0",
3+
"version": "2.3.0",
44
"description": "Created to perform fast search on small json dataset (up to 1000 elements).",
55
"type": "module",
66
"scripts": {
77
"test": "mocha tests/*",
88
"lint": "eslint \"**/*.js\" --ext js",
99
"lint:fix": "eslint \"**/*.js\" --ext js --fix",
10+
"benchmark:snapshot": "node benchmarks/snapshot.js",
11+
"serve:benchmark": "node scripts/serve-benchmark.js",
1012
"prepublishOnly": "npm run build",
1113
"build": "microbundle",
1214
"dev": "microbundle watch",

‎scripts/serve-benchmark.js‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import http from 'node:http';
2+
import fs from 'node:fs';
3+
import path from 'node:path';
4+
5+
const port = process.env.PORT || 4173;
6+
const root = process.cwd();
7+
8+
const mime = {
9+
'.html': 'text/html; charset=utf-8',
10+
'.js': 'application/javascript; charset=utf-8',
11+
'.mjs': 'application/javascript; charset=utf-8',
12+
'.css': 'text/css; charset=utf-8',
13+
'.json': 'application/json; charset=utf-8',
14+
'.map': 'application/json; charset=utf-8',
15+
'.txt': 'text/plain; charset=utf-8',
16+
'.svg': 'image/svg+xml',
17+
'.png': 'image/png',
18+
'.jpg': 'image/jpeg',
19+
'.jpeg': 'image/jpeg',
20+
'.gif': 'image/gif',
21+
'.ico': 'image/x-icon',
22+
};
23+
24+
const server = http.createServer((req, res) => {
25+
const url = new URL(req.url, 'http://localhost');
26+
let pathname = url.pathname;
27+
28+
if (pathname === '/') {
29+
pathname = '/benchmarks/browser-snapshot.html';
30+
}
31+
32+
const filePath = path.join(root, pathname);
33+
const normalized = path.normalize(filePath);
34+
35+
if (!normalized.startsWith(root)) {
36+
res.writeHead(403);
37+
return res.end('Forbidden');
38+
}
39+
40+
fs.stat(normalized, (err, stats) => {
41+
if (err || !stats.isFile()) {
42+
res.writeHead(404);
43+
return res.end('Not found');
44+
}
45+
46+
const ext = path.extname(normalized).toLowerCase();
47+
const type = mime[ext] || 'application/octet-stream';
48+
res.writeHead(200, { 'Content-Type': type });
49+
fs.createReadStream(normalized).pipe(res);
50+
});
51+
});
52+
53+
server.listen(port, () => {
54+
const indexPath = path.join(root, 'benchmarks', 'browser-snapshot.html');
55+
console.log(`Static server running on http://localhost:${port}`);
56+
console.log(`Open http://localhost:${port}/benchmarks/browser-snapshot.html`);
57+
console.log(`(or file://${indexPath})`);
58+
});

0 commit comments

Comments
 (0)