Skip to content

icechunk fixes #634

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lazarusA merged 4 commits into from
Apr 16, 2026
Merged

icechunk fixes #634

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2629aae
ice
lazarusA Apr 15, 2026
1bbb9ef
format
lazarusA Apr 15, 2026
857b32d
merge
lazarusA Apr 15, 2026
85223cd
suggestions
lazarusA Apr 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
226 changes: 119 additions & 107 deletions src/components/zarr/ZarrLoaderLRU.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,140 +1,152 @@
import * as zarr from "zarrita";
import { IcechunkStore } from "icechunk-js";
import { ZarrMetadata, ZarrTitleDescription, FetchStoreOptions } from "./Interfaces";
import {
ZarrMetadata,
ZarrTitleDescription,
FetchStoreOptions,
} from "./Interfaces";
import { IcechunkStoreOptions } from "./Interfaces";
import { useGlobalStore } from "@/GlobalStates/GlobalStore";
import { useCacheStore } from '@/GlobalStates/CacheStore';
import { useCacheStore } from "@/GlobalStates/CacheStore";
import { useZarrStore } from "@/GlobalStates/ZarrStore";
import {
getFetchStore,
getFetchStoreMetadata,
getFetchStoreTitleDescription,
getFetchStoreAttributes,
getFetchStoreDims} from "./zarrita-store";
import {
getIcechunkStore,
getIcechunkMetadata,
getIcechunkTitleDescription,
getIcechunkAttributes,
getIcechunkDims} from "./icechunk-store";
import {
getFetchStore,
getFetchStoreMetadata,
getFetchStoreTitleDescription,
getFetchStoreAttributes,
getFetchStoreDims,
} from "./zarrita-store";
import {
getIcechunkStore,
getIcechunkMetadata,
getIcechunkTitleDescription,
getIcechunkAttributes,
getIcechunkDims,
} from "./icechunk-store";
import { GetNCMetadata } from "./NCGetters";

type GroupType = zarr.Group<zarr.FetchStore | zarr.Listable<zarr.FetchStore> | IcechunkStore>;
type GroupType = zarr.Group<
zarr.FetchStore | zarr.Listable<zarr.FetchStore> | IcechunkStore
>;

export async function GetStore(
storePath: string,
fetchOptions?: FetchStoreOptions,
icechunkOptions?: IcechunkStoreOptions
storePath: string,
fetchOptions?: FetchStoreOptions,
icechunkOptions?: IcechunkStoreOptions,
): Promise<GroupType | undefined> {
if (icechunkOptions) {
return getIcechunkStore(storePath, icechunkOptions);
}
return getFetchStore(storePath, fetchOptions);
if (icechunkOptions) {
return getIcechunkStore(storePath, icechunkOptions);
}
return getFetchStore(storePath, fetchOptions);
}

export async function GetZarrMetadata(
groupStore: Promise<GroupType>
groupStore: Promise<GroupType>,
): Promise<ZarrMetadata[]> {
const group = await groupStore;
if (group.store instanceof IcechunkStore) {
console.log('Fetching metadata from icechunk store');
return getIcechunkMetadata(group as zarr.Group<IcechunkStore>);
}
console.log('Fetching metadata from fetch store');
return getFetchStoreMetadata(
group as zarr.Group<zarr.FetchStore | zarr.Listable<zarr.FetchStore>>
);
const group = await groupStore;
if (group.store instanceof IcechunkStore) {
return getIcechunkMetadata(group as zarr.Group<IcechunkStore>);
}
return getFetchStoreMetadata(
group as zarr.Group<zarr.FetchStore | zarr.Listable<zarr.FetchStore>>,
);
}

export async function GetTitleDescription(
groupStore: Promise<GroupType>
groupStore: Promise<GroupType>,
): Promise<ZarrTitleDescription> {
const group = await groupStore;
if (group.store instanceof IcechunkStore) {
console.log('Fetching title and description from icechunk store');
return getIcechunkTitleDescription(group as zarr.Group<IcechunkStore>);
}
console.log('Fetching title and description from fetch store');
return getFetchStoreTitleDescription(
group as zarr.Group<zarr.FetchStore | zarr.Listable<zarr.FetchStore>>
);
const group = await groupStore;
if (group.store instanceof IcechunkStore) {
return getIcechunkTitleDescription(group as zarr.Group<IcechunkStore>);
}
return getFetchStoreTitleDescription(
group as zarr.Group<zarr.FetchStore | zarr.Listable<zarr.FetchStore>>,
);
}

export async function GetZarrAttributes(thisVariable?: string) {
const { initStore, variable } = useGlobalStore.getState();
const { cache } = useCacheStore.getState();
const { currentStore } = useZarrStore.getState();
const resolvedVariable = thisVariable ?? variable;
const cacheName = `${initStore}_${resolvedVariable}_meta`;
const { initStore, variable } = useGlobalStore.getState();
const { cache } = useCacheStore.getState();
const { currentStore } = useZarrStore.getState();
const resolvedVariable = thisVariable ?? variable;
const cacheName = `${initStore}_${resolvedVariable}_meta`;

if (cache.has(cacheName)) return cache.get(cacheName);
if (cache.has(cacheName)) return cache.get(cacheName);

const group = await currentStore;
if (!group) throw new Error(`Failed to open store: ${initStore}`);
const group = await currentStore;
if (!group) throw new Error(`Failed to open store: ${initStore}`);

if (group.store instanceof IcechunkStore) {
return getIcechunkAttributes(
group as zarr.Group<IcechunkStore>,
resolvedVariable,
cacheName
);
}
return getFetchStoreAttributes(group, resolvedVariable, cacheName);
if (group.store instanceof IcechunkStore) {
return getIcechunkAttributes(
group as zarr.Group<IcechunkStore>,
resolvedVariable,
cacheName,
);
}
return getFetchStoreAttributes(group, resolvedVariable, cacheName);
}

export async function GetZarrDims(variable: string) {
const { cache } = useCacheStore.getState();
const { initStore } = useGlobalStore.getState();
const cacheName = `${initStore}_${variable}_meta`;
console.log(`Getting dimensions for variable "${variable}" with cache key "${cacheName}"`);
// Cache hit — same for both store types
if (cache.has(cacheName)) {
const meta = cache.get(cacheName);
const dimNames = (meta._ARRAY_DIMENSIONS ?? meta.dimensionNames) as string[] | undefined;
console.log('Cache hit for dimensions:', { meta, dimNames });
const dimArrays: unknown[] = [];
const dimUnits: unknown[] = [];
const { cache } = useCacheStore.getState();
const { initStore } = useGlobalStore.getState();
const cacheName = `${initStore}_${variable}_meta`;
// Cache hit — same for both store types
if (cache.has(cacheName)) {
const meta = cache.get(cacheName);
const dimNames = (meta._ARRAY_DIMENSIONS ?? meta.dimensionNames) as
| string[]
| undefined;
const dimArrays: unknown[] = [];
const dimUnits: unknown[] = [];

if (dimNames) {
for (const dim of dimNames) {
dimArrays.push(cache.get(`${initStore}_${dim}`) ?? [0]);
dimUnits.push(cache.get(`${initStore}_${dim}_meta`)?.units ?? null);
}
} else {
for (const dimLength of meta.shape) {
dimArrays.push(Array(dimLength).fill(0));
dimUnits.push('Default');
}
}
return { dimNames: dimNames ?? Array(meta.shape.length).fill('Default'), dimArrays, dimUnits };
}
if (dimNames) {
for (const dim of dimNames) {
dimArrays.push(cache.get(`${initStore}_${dim}`) ?? [0]);
dimUnits.push(
cache.get(`${initStore}_${dim}_meta`)?.units ?? null,
);
}
} else {
for (const dimLength of meta.shape) {
dimArrays.push(Array(dimLength).fill(0));
dimUnits.push("Default");
}
}
return {
dimNames: dimNames ?? Array(meta.shape.length).fill("Default"),
dimArrays,
dimUnits,
};
}

const group = await useZarrStore.getState().currentStore;
if (!group) throw new Error(`Failed to open store: ${initStore}`);
const group = await useZarrStore.getState().currentStore;
if (!group) throw new Error(`Failed to open store: ${initStore}`);

if (group.store instanceof IcechunkStore) {
console.log('Fetching dimensions from icechunk store for variable:', variable);
return getIcechunkDims(group as zarr.Group<IcechunkStore>, variable, initStore);
}
return getFetchStoreDims(group, variable, initStore);
if (group.store instanceof IcechunkStore) {
return getIcechunkDims(
group as zarr.Group<IcechunkStore>,
variable,
initStore,
);
}
return getFetchStoreDims(group, variable, initStore);
}

export async function GetAttributes(thisVariable? : string){
const {initStore, variable } = useGlobalStore.getState();
const {cache} = useCacheStore.getState();
const {useNC} = useZarrStore.getState();
const cacheName = `${initStore}_${thisVariable?? variable}_meta`
if (cache.has(cacheName)){
const meta = cache.get(cacheName)
return useNC ? meta.attributes : meta;
}
else {
if (useNC){
const meta = await GetNCMetadata(thisVariable)
return meta.attributes
} else {
const meta = await GetZarrAttributes(thisVariable)
return meta
}
}
export async function GetAttributes(thisVariable?: string) {
const { initStore, variable } = useGlobalStore.getState();
const { cache } = useCacheStore.getState();
const { useNC } = useZarrStore.getState();
const cacheName = `${initStore}_${thisVariable ?? variable}_meta`;
if (cache.has(cacheName)) {
const meta = cache.get(cacheName);
return useNC ? meta.attributes : meta;
} else {
if (useNC) {
const meta = await GetNCMetadata(thisVariable);
return meta.attributes;
} else {
const meta = await GetZarrAttributes(thisVariable);
return meta;
}
}
}
Loading
Loading