Skip to content

Commit b80a24d

Browse files
ryan-williamsclaude
andcommitted
Add ZarrVolumeStore + wire ?zarr=1 toggle
`zarr-volume.ts` opens a multi-resolution Zarr store via zarrita.js, reads metadata + per-level shapes, and converts the chosen level into the existing `VolumeData` shape. Currently transposes C-order zarr bytes to F-order to match VASP/marching-cubes convention; will revisit if perf matters. `resolveLoadUrl(record, role, format)` gains an optional `format` arg; electrai-205 has both `chgcar` and `zarr` URL templates. URL routing in `handleUrlSubmit` detects `.zarr/` and uses `fetchZarrVolume`. Toggle via `?zarr=1` or Shift+Z; passes `format='zarr'` down to MaterialsSearch + BrowseMaterials so Omnibar/table selections route to the right URL. Untested in browser (S3 bucket needs CORS for cross-origin reads). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent 492ddfe commit b80a24d

8 files changed

Lines changed: 218 additions & 18 deletions

File tree

pkgs/core/package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@
1313
"@react-three/fiber": "^9.1.0",
1414
"react": "^19.1.1",
1515
"react-dom": "^19.1.1",
16-
"three": "^0.175.0"
16+
"three": "^0.175.0",
17+
"zarrita": "^0.7.2"
1718
},
1819
"devDependencies": {
1920
"@eslint/js": "^9.36.0",

pkgs/core/src/index.ts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,13 @@ export type {
99

1010
// Storage
1111
export type { StoredVolume, VolumeStore } from './storage/types.ts'
12+
export {
13+
openZarrVolume,
14+
readZarrLevel,
15+
zarrToVolumeData,
16+
fetchZarrVolume,
17+
} from './storage/zarr-volume.ts'
18+
export type { OpenedZarrVolume, ZarrVolumeMetadata } from './storage/zarr-volume.ts'
1219

1320
// Parsers
1421
export { parseCHGCAR, parseCHGCARHeader } from './parsers/chgcar.ts'
Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
import * as zarr from 'zarrita'
2+
import type { LatticeMatrix, VolumeData } from '../types.ts'
3+
4+
/** OME-NGFF + ELvis-custom metadata captured by `convert-to-zarr.py`. */
5+
export interface ZarrVolumeMetadata {
6+
multiscales: {
7+
name: string
8+
version: string
9+
axes: { name: string; type: string }[]
10+
datasets: { path: string; coordinateTransformations: { type: string; scale: number[] }[] }[]
11+
}[]
12+
elvis: {
13+
material_id: string
14+
role: 'input' | 'label'
15+
/** 3x3 lattice matrix, rows = a/b/c vectors */
16+
lattice: number[][]
17+
atoms: { element: string; frac: [number, number, number] }[]
18+
stats: { min: number; max: number; mean: number }
19+
/** 201 equally-spaced quantiles, sampled from the full grid; matches client convention. */
20+
quantiles: number[]
21+
}
22+
}
23+
24+
export interface OpenedZarrVolume {
25+
url: string
26+
meta: ZarrVolumeMetadata
27+
/** Number of pyramid levels; level 0 is full resolution. */
28+
levels: number
29+
/** Per-level shape from each level's .zarray, ordered [level0, level1, ...]. */
30+
levelShapes: [number, number, number][]
31+
}
32+
33+
/** Open a multi-resolution Zarr store and read its metadata + per-level shapes. */
34+
export async function openZarrVolume(url: string): Promise<OpenedZarrVolume> {
35+
const store = new zarr.FetchStore(url)
36+
const root = await zarr.open(store, { kind: 'group' })
37+
const meta = root.attrs as unknown as ZarrVolumeMetadata
38+
const levels = meta.multiscales[0].datasets.length
39+
const levelShapes: [number, number, number][] = []
40+
for (let i = 0; i < levels; i++) {
41+
const arr = await zarr.open(root.resolve(String(i)), { kind: 'array' })
42+
levelShapes.push(arr.shape as [number, number, number])
43+
}
44+
return { url, meta, levels, levelShapes }
45+
}
46+
47+
/** Read one pyramid level into a flat Float32Array. */
48+
export async function readZarrLevel(
49+
opened: OpenedZarrVolume,
50+
level: number,
51+
): Promise<{ data: Float32Array; dims: [number, number, number] }> {
52+
const store = new zarr.FetchStore(opened.url)
53+
const root = await zarr.open(store, { kind: 'group' })
54+
const arr = await zarr.open(root.resolve(String(level)), { kind: 'array' })
55+
const result = await zarr.get(arr)
56+
// zarrita returns C-ordered raw bytes; pymatgen->Zarr writes in C-order too.
57+
// VASP/marching-cubes expects F-order (flat[i + j*Nx + k*Nx*Ny] = data[i,j,k]).
58+
const dims = arr.shape as [number, number, number]
59+
const data = transposeToFortran(result.data as Float32Array, dims)
60+
return { data, dims }
61+
}
62+
63+
/** Convert a C-ordered (Nx,Ny,Nz) flat array to F-ordered: flat[i + j*Nx + k*Nx*Ny] = a[i,j,k]. */
64+
function transposeToFortran(c: Float32Array, dims: [number, number, number]): Float32Array {
65+
const [nx, ny, nz] = dims
66+
const f = new Float32Array(c.length)
67+
for (let i = 0; i < nx; i++) {
68+
for (let j = 0; j < ny; j++) {
69+
const cBase = (i * ny + j) * nz
70+
for (let k = 0; k < nz; k++) {
71+
f[i + j * nx + k * nx * ny] = c[cBase + k]
72+
}
73+
}
74+
}
75+
return f
76+
}
77+
78+
/** Convert ELvis Zarr metadata + level data into the existing VolumeData shape. */
79+
export function zarrToVolumeData(
80+
opened: OpenedZarrVolume,
81+
level: { data: Float32Array; dims: [number, number, number] },
82+
): VolumeData {
83+
const { meta } = opened
84+
const m = meta.elvis.lattice
85+
const lattice: LatticeMatrix = [
86+
m[0][0], m[0][1], m[0][2],
87+
m[1][0], m[1][1], m[1][2],
88+
m[2][0], m[2][1], m[2][2],
89+
]
90+
const counts = new Map<string, number>()
91+
for (const a of meta.elvis.atoms) counts.set(a.element, (counts.get(a.element) ?? 0) + 1)
92+
const elements = Array.from(counts.keys())
93+
return {
94+
title: `${meta.elvis.material_id}/${meta.elvis.role}`,
95+
scaleFactor: 1.0,
96+
lattice,
97+
structure: {
98+
elements,
99+
counts: elements.map((e) => counts.get(e)!),
100+
atoms: meta.elvis.atoms.map((a) => ({ element: a.element, fracCoords: a.frac })),
101+
},
102+
grid: { dims: level.dims, data: level.data },
103+
}
104+
}
105+
106+
/** Convenience: open + read level 0 + assemble VolumeData in one call. */
107+
export async function fetchZarrVolume(url: string, level = 0): Promise<VolumeData> {
108+
const opened = await openZarrVolume(url)
109+
const lvl = await readZarrLevel(opened, level)
110+
return zarrToVolumeData(opened, lvl)
111+
}

pkgs/corpora/src/resolve-url.ts

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,25 @@
11
import type { CorpusId, MaterialRecord } from './types.ts'
22

3-
/** S3 URI patterns per corpus+role. `{task}` is replaced with the task ID. */
4-
const CORPUS_URLS: Record<string, { label: string; input?: string }> = {
3+
/** S3 URI patterns per corpus+role+format. `{task}` is replaced with the task ID. */
4+
const CORPUS_URLS: Record<string, {
5+
chgcar: { label: string; input?: string }
6+
zarr?: { label: string; input?: string }
7+
}> = {
58
'electrai-205': {
6-
label: 's3://openathena/electrai/label/{task}.CHGCAR',
7-
input: 's3://openathena/electrai/input/{task}.CHGCAR',
9+
chgcar: {
10+
label: 's3://openathena/electrai/label/{task}.CHGCAR',
11+
input: 's3://openathena/electrai/input/{task}.CHGCAR',
12+
},
13+
zarr: {
14+
label: 's3://openathena/electrai/zarr/{task}-label.zarr/',
15+
input: 's3://openathena/electrai/zarr/{task}-input.zarr/',
16+
},
817
},
918
'dataset_4': {
10-
label: 's3://openathena/electrai/mp/chg_datasets/dataset_4/label/{task}.CHGCAR',
11-
input: 's3://openathena/electrai/mp/chg_datasets/dataset_4/data/{task}.CHGCAR',
19+
chgcar: {
20+
label: 's3://openathena/electrai/mp/chg_datasets/dataset_4/label/{task}.CHGCAR',
21+
input: 's3://openathena/electrai/mp/chg_datasets/dataset_4/data/{task}.CHGCAR',
22+
},
1223
},
1324
}
1425

@@ -23,19 +34,26 @@ const CORPUS_PRIORITY: CorpusId[] = ['electrai-205', 'dataset_4', 'mp-public', '
2334
* Falls back to materialsproject-parsed for materials only in mp-public.
2435
*
2536
* @param role - 'label' (DFT ground truth, default) or 'input' (SAD guess)
37+
* @param format - 'chgcar' (default) or 'zarr' (multi-resolution chunked)
2638
*/
27-
export function resolveLoadUrl(record: MaterialRecord, role: 'label' | 'input' = 'label'): string | undefined {
39+
export function resolveLoadUrl(
40+
record: MaterialRecord,
41+
role: 'label' | 'input' = 'label',
42+
format: 'chgcar' | 'zarr' = 'chgcar',
43+
): string | undefined {
2844
for (const corpus of CORPUS_PRIORITY) {
2945
const m = record.datasets[corpus]
3046
if (!m || !m.task_ids.length) continue
3147
const taskId = m.task_ids[0]
3248
const urls = CORPUS_URLS[corpus]
3349
if (urls) {
34-
const template = role === 'input' && urls.input ? urls.input : urls.label
50+
const formatUrls = format === 'zarr' ? urls.zarr : urls.chgcar
51+
if (!formatUrls) continue // try next corpus if this one lacks the format
52+
const template = role === 'input' && formatUrls.input ? formatUrls.input : formatUrls.label
3553
return template.replace('{task}', taskId)
3654
}
37-
// mp-public fallback
38-
return MP_PARSED_URL.replace('{task}', taskId)
55+
// mp-public fallback (CHGCAR-equivalent only)
56+
if (format === 'chgcar') return MP_PARSED_URL.replace('{task}', taskId)
3957
}
4058
return undefined
4159
}

pkgs/static/src/App.tsx

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import {
1919
sampleRamp,
2020
densityToQuantile,
2121
DEFAULT_RAMP,
22+
fetchZarrVolume,
2223
} from '@elvis/core'
2324
import { ShortcutsModal, Omnibar, SequenceModal, LookupModal, SpeedDial, ModeIndicator, useAction, useActionPair, useActionTriplet, useArrowGroup, useMode } from 'use-kbd'
2425
import type { SpeedDialAction } from 'use-kbd'
@@ -196,6 +197,7 @@ export default function App() {
196197
const [opacity, setOpacity] = useUrlState('op', floatParam({ default: 0.6, encoding: 'string', decimals: 2 }), { debounce: 300 })
197198
const [useGpuVolume, setUseGpuVolume] = useUrlState('gpu', boolParam)
198199
const [useGlbPreview, setUseGlbPreview] = useUrlState('glb', boolParam)
200+
const [useZarr, setUseZarr] = useUrlState('zarr', boolParam)
199201
const [colorByDensity, setColorByDensity] = useUrlState('cd', boolParam)
200202
const [showAtoms, setShowAtoms] = useUrlState('ha', boolTrueParam)
201203
const [showAbcCell, setShowAbcCell] = useUrlState('hc', boolTrueParam)
@@ -491,6 +493,14 @@ export default function App() {
491493
defaultBindings: ['shift+g'],
492494
handler: () => setUseGlbPreview(!useGlbPreview),
493495
})
496+
useAction('data:toggle-zarr', {
497+
label: 'Toggle Zarr loader',
498+
description: 'Fetch density from multi-resolution Zarr instead of full CHGCAR',
499+
keywords: ['zarr', 'progressive', 'pyramid', 'chunks'],
500+
group: 'Data',
501+
defaultBindings: ['shift+z'],
502+
handler: () => setUseZarr(!useZarr),
503+
})
494504
useAction('view:toggle-color-by-density', {
495505
label: 'Toggle iso color by density',
496506
description: 'Color/opacity of isosurface varies with iso-level quantile',
@@ -1070,6 +1080,17 @@ export default function App() {
10701080
setFiles([])
10711081
try {
10721082
const isJsonGz = url.toLowerCase().endsWith('.json.gz')
1083+
const isZarr = /\.zarr\/?$/i.test(url)
1084+
1085+
if (isZarr) {
1086+
const httpsUrl = url.startsWith('s3://') ? s3UriToHttps(url) : url
1087+
setFetchStatus('Loading Zarr...')
1088+
const data = await fetchZarrVolume(httpsUrl)
1089+
const filename = data.title
1090+
handleLoad(data, filename)
1091+
setFetchStatus(null)
1092+
return
1093+
}
10731094

10741095
const onProgress = (p: FetchProgress) => {
10751096
if (p.phase === 'head') setFetchStatus('Checking file...')
@@ -1481,8 +1502,8 @@ export default function App() {
14811502
/>
14821503

14831504
<ShortcutsModal editable arrowIcon="move" TooltipComponent={MuiTooltip} />
1484-
<MaterialsSearch onSelect={handleUrlSubmit} />
1485-
<BrowseMaterials open={browseOpen} onClose={() => setBrowseOpen(false)} onSelect={handleUrlSubmit} />
1505+
<MaterialsSearch onSelect={handleUrlSubmit} format={useZarr ? 'zarr' : 'chgcar'} />
1506+
<BrowseMaterials open={browseOpen} onClose={() => setBrowseOpen(false)} onSelect={handleUrlSubmit} format={useZarr ? 'zarr' : 'chgcar'} />
14861507
<Omnibar />
14871508
<SequenceModal />
14881509
<LookupModal />

pkgs/static/src/BrowseMaterials.tsx

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,11 @@ interface BrowseMaterialsProps {
2525
open: boolean
2626
onClose: () => void
2727
onSelect: (url: string) => void
28+
role?: 'label' | 'input'
29+
format?: 'chgcar' | 'zarr'
2830
}
2931

30-
export function BrowseMaterials({ open, onClose, onSelect }: BrowseMaterialsProps) {
32+
export function BrowseMaterials({ open, onClose, onSelect, role = 'label', format = 'chgcar' }: BrowseMaterialsProps) {
3133
const [query, setQuery] = useState('')
3234
const [crystalSystem, setCrystalSystem] = useState<string>('')
3335
const [requireElements, setRequireElements] = useState('')
@@ -67,7 +69,7 @@ export function BrowseMaterials({ open, onClose, onSelect }: BrowseMaterialsProp
6769
}
6870

6971
const handleRowClick = (r: MaterialRecord) => {
70-
const url = resolveLoadUrl(r)
72+
const url = resolveLoadUrl(r, role, format)
7173
if (!url) return
7274
onSelect(url)
7375
onClose()

pkgs/static/src/MaterialsSearch.tsx

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,11 @@ function formatDescription(record: MaterialRecord): string {
2323
interface MaterialsSearchProps {
2424
/** Called with the resolved S3 URL for loading the selected material. */
2525
onSelect: (url: string) => void
26+
role?: 'label' | 'input'
27+
format?: 'chgcar' | 'zarr'
2628
}
2729

28-
export function MaterialsSearch({ onSelect }: MaterialsSearchProps) {
30+
export function MaterialsSearch({ onSelect, role = 'label', format = 'chgcar' }: MaterialsSearchProps) {
2931
const config = useMemo(() => ({
3032
group: 'Materials',
3133
minQueryLength: 1,
@@ -36,7 +38,7 @@ export function MaterialsSearch({ onSelect }: MaterialsSearchProps) {
3638
const page = hits.slice(pagination.offset, pagination.offset + pagination.limit)
3739
return {
3840
entries: page.map(r => {
39-
const url = resolveLoadUrl(r)
41+
const url = resolveLoadUrl(r, role, format)
4042
return {
4143
id: r.id,
4244
label: `${r.id} ${r.formula}`,
@@ -49,7 +51,7 @@ export function MaterialsSearch({ onSelect }: MaterialsSearchProps) {
4951
hasMore: pagination.offset + pagination.limit < hits.length,
5052
}
5153
},
52-
}), [onSelect])
54+
}), [onSelect, role, format])
5355

5456
useOmnibarEndpoint('materials', config)
5557

0 commit comments

Comments
 (0)