Revamp file-based data access

- New datasource URL syntax based on ZEP 8
proposal (zarr-developers/zeps#48)
- Support for ZIP archives

Author: jbms

.github/workflows/build.yml

@@ -31,11 +31,16 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-node@v4
         with:
-          node-version: 20.x
+          node-version: 22.x
           cache: "npm"
           cache-dependency-path: |
             package-lock.json
             examples/**/package-lock.json
+      # go needed for fake_gcs_server used by the javascript tests
+      - name: Setup go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "stable"
       - run: npm install
       - run: npm run format:fix
       - name: Check for dirty working directory
@@ -99,7 +104,7 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-node@v4
         with:
-          node-version: 20.x
+          node-version: 22.x
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
@@ -146,7 +151,7 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-node@v4
         with:
-          node-version: 20.x
+          node-version: 22.x
           cache: "npm"
       - name: Set up Python
         uses: actions/setup-python@v5
@@ -221,7 +226,7 @@ jobs:
       - name: Use Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: 20.x
+          node-version: 22.x
           registry-url: "https://registry.npmjs.org"
       - uses: actions/download-artifact@v4
         with:

.github/workflows/build_preview.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       matrix:
         node-version:
-          - "20.x"
+          - "22.x"
     runs-on: ubuntu-latest
 
     steps:

.prettierignore

@@ -1,9 +1,12 @@
 /templates/
 /python/
 /third_party/jpgjs/jpg.js
-/testdata/*.json
+/testdata/**/*.json
+zarr.json
 .parcel-cache
 dist
 /lib
 /docs/_build/
 /.ruff_cache
+package.json
+package-lock.json

build_tools/build-package.ts

@@ -71,6 +71,7 @@ async function buildPackage(options: {
     outbase: srcDir,
     bundle: false,
     outdir: libDir,
+    target: "es2022",
   });
 
   let compilerOptionsFromConfigFile: ts.CompilerOptions = {};

build_tools/update-conditions.ts

@@ -17,70 +17,100 @@ const imports: Record<string, any> = {};
 imports["#src/third_party/jpgjs/jpg.js"] = "./src/third_party/jpgjs/jpg.js";
 imports["#src/*.js"] = "./src/*.ts";
 imports["#src/*"] = "./src/*";
+imports["#tests/fixtures/msw"] = {
+  node: "./tests/fixtures/msw_node.ts",
+  default: "./tests/fixtures/msw_browser.ts",
+};
+imports["#tests/fixtures/gl"] = {
+  node: "./tests/fixtures/gl_node.ts",
+  default: "./tests/fixtures/gl_browser.ts",
+};
+imports["#tests/*.js"] = "./tests/*.ts";
 imports["#testdata/*"] = "./testdata/*";
 
-const datasourceDir = path.resolve(rootDir, "src", "datasource");
-const layerDir = path.resolve(rootDir, "src", "layer");
-
-const datasources = (
-  await fs.promises.readdir(datasourceDir, { withFileTypes: true })
-)
-  .filter((e) => e.isDirectory())
-  .map((e) => e.name);
-
-const layers = (await fs.promises.readdir(layerDir, { withFileTypes: true }))
-  .filter((e) => e.isDirectory())
-  .map((e) => e.name);
-
-const datasourceKeys = {
-  backend: "backend",
-  async_computation: "async_computation",
-  register_default: "frontend",
-  register_credentials_provider: "frontend",
-} as const;
+async function listSubdirs(dir: string): Promise<string[]> {
+  return (await fs.promises.readdir(dir, { withFileTypes: true }))
+    .filter((e) => e.isDirectory())
+    .map((e) => e.name);
+}
 
-const datasourceModules = Object.fromEntries(
-  Object.values(datasourceKeys).map((key) => [key, new Array<string>()]),
-);
+async function writeModule(modulePath: string, imports: string[]) {
+  await fs.promises.writeFile(
+    modulePath,
+    "// DO NOT EDIT: Generated by config/update_conditions.ts\n" +
+      imports.map((name) => `import ${JSON.stringify(name)};\n`).join(""),
+    { encoding: "utf-8" },
+  );
+}
 
-for (const datasource of datasources) {
-  for (const [filePrefix, moduleKind] of Object.entries(datasourceKeys)) {
-    const sourcePrefix = `./src/datasource/${datasource}/${filePrefix}`;
-    if (
-      await fs.promises
-        .stat(path.resolve(rootDir, `${sourcePrefix}.ts`))
-        .catch(() => undefined)
-    ) {
-      const source = sourcePrefix + JS_EXT;
-      const conditions: Record<string, string> = {};
-      if (datasource === "python") {
-        conditions["neuroglancer/python"] = source;
-        conditions.default = NOOP;
-      } else {
-        if (filePrefix === "register_credentials_provider") {
-          conditions["neuroglancer/python"] = NOOP;
+async function handleDrivers(
+  kind: string,
+  moduleMap: Record<string, string[]>,
+) {
+  const driverDir = path.resolve(rootDir, "src", kind);
+  const drivers = await listSubdirs(driverDir);
+  const modules: Record<string, string[]> = {};
+  for (const driver of drivers) {
+    for (const [filePrefix, moduleKinds] of Object.entries(moduleMap)) {
+      const sourcePrefix = `./src/${kind}/${driver}/${filePrefix}`;
+      if (
+        await fs.promises
+          .stat(path.resolve(rootDir, `${sourcePrefix}.ts`))
+          .catch(() => undefined)
+      ) {
+        const source = sourcePrefix + JS_EXT;
+        const conditions: Record<string, string> = {};
+        if (driver === "python") {
+          conditions["neuroglancer/python"] = source;
+          conditions.default = NOOP;
+        } else {
+          if (filePrefix === "register_credentials_provider") {
+            conditions["neuroglancer/python"] = NOOP;
+          }
+          conditions[`neuroglancer/${kind}/${driver}:enabled`] = source;
+          conditions[`neuroglancer/${kind}:none_by_default`] = NOOP;
+          conditions[`neuroglancer/${kind}/${driver}:disabled`] = NOOP;
+          conditions.default = source;
+        }
+        let moduleId = `#${kind}/${driver}`;
+        if (filePrefix !== "index") {
+          moduleId += `/${filePrefix}`;
+        }
+        imports[moduleId] = conditions;
+        for (const moduleKind of moduleKinds) {
+          if (modules[moduleKind] === undefined) {
+            modules[moduleKind] = [];
+          }
+          modules[moduleKind].push(moduleId);
         }
-        conditions[`neuroglancer/datasource/${datasource}:enabled`] = source;
-        conditions["neuroglancer/datasource:none_by_default"] = NOOP;
-        conditions[`neuroglancer/datasource/${datasource}:disabled`] = source;
-        conditions.default = source;
       }
-      const moduleId = `#datasource/${datasource}/${filePrefix}`;
-      imports[moduleId] = conditions;
-      datasourceModules[moduleKind].push(moduleId);
     }
   }
+  for (const [moduleKind, moduleIds] of Object.entries(modules)) {
+    await writeModule(
+      path.resolve(driverDir, `enabled_${moduleKind}_modules.ts`),
+      moduleIds,
+    );
+  }
 }
 
-for (const layer of layers) {
-  const source = `./src/layer/${layer}/index` + JS_EXT;
-  imports[`#layer/${layer}`] = {
-    [`neuroglancer/layer/${layer}:enabled`]: source,
-    "neuroglancer/layer:none_by_default": NOOP,
-    [`neuroglancer/layer/${layer}:enabled`]: source,
-    default: source,
-  };
-}
+await handleDrivers("datasource", {
+  backend: ["backend"],
+  async_computation: ["async_computation"],
+  register_default: ["frontend"],
+  register_credentials_provider: ["frontend"],
+});
+
+await handleDrivers("kvstore", {
+  register: ["frontend", "backend"],
+  register_frontend: ["frontend"],
+  register_backend: ["backend"],
+  register_credentials_provider: ["frontend"],
+});
+
+await handleDrivers("layer", {
+  index: ["frontend"],
+});
 
 // main entrypoint.
 imports["#main"] = {
@@ -94,32 +124,6 @@ imports["#python_integration_build"] = {
   default: NOOP,
 };
 
-async function writeModule(modulePath: string, imports: string[]) {
-  await fs.promises.writeFile(
-    modulePath,
-    "// DO NOT EDIT: Generated by config/update_conditions.ts\n" +
-      imports.map((name) => `import ${JSON.stringify(name)};\n`).join(""),
-    { encoding: "utf-8" },
-  );
-}
-
-for (const [moduleKind, moduleIds] of Object.entries(datasourceModules)) {
-  await writeModule(
-    path.resolve(
-      rootDir,
-      "src",
-      "datasource",
-      `enabled_${moduleKind}_modules.ts`,
-    ),
-    moduleIds,
-  );
-}
-
-await writeModule(
-  path.resolve(rootDir, "src", "layer", "enabled_frontend_modules.ts"),
-  layers.map((name) => `#layer/${name}`),
-);
-
 packageJson.imports = imports;
 
 packageJson.exports = {

build_tools/vitest/fake_ngauth_server.ts

@@ -0,0 +1,96 @@
+/**
+ * @license
+ * Copyright 2024 Google Inc.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Fake ngauth server used for tests.
+
+import type { AddressInfo } from "node:net";
+import cookie from "cookie";
+import express from "express";
+
+const COOKIE_NAME = "ngauth_login";
+const COOKIE_VALUE = "fake_login";
+const TOKEN_VALUE = "fake_token";
+
+export interface FakeNgauthServer extends AsyncDisposable {
+  url: string;
+}
+
+export function startFakeNgauthServer(): Promise<FakeNgauthServer> {
+  const app = express();
+  app.use(express.json({ inflate: false, type: "*/*" }));
+  app.get("/login", (req, res) => {
+    const origin = (req.query.origin ?? "").toString();
+    res.contentType("text/html");
+    // Note: The real ngauth marks this cookie as http-only, but that prevents
+    // JavaScript from clearing it.
+    res.cookie(COOKIE_NAME, COOKIE_VALUE, { sameSite: "lax" });
+    const jsonToken = JSON.stringify({ token: TOKEN_VALUE });
+    const jsonOrigin = JSON.stringify(origin);
+    res.send(`
+<html>
+<body>
+<script>
+window.opener.postMessage(${jsonToken}, ${jsonOrigin});
+</script>
+</body>
+</html>
+`);
+  });
+  app.post("/token", (req, res) => {
+    const cookies = cookie.parse(req.headers.cookie ?? "");
+    const origin = req.headers.origin ?? "";
+    res.set("x-frame-options", "deny");
+    res.set("access-control-allow-origin", origin);
+    res.set("access-control-allow-credentials", "true");
+    res.set("vary", "origin");
+
+    if (cookies[COOKIE_NAME] === COOKIE_VALUE) {
+      res.contentType("text/plain");
+      res.send(TOKEN_VALUE);
+    } else {
+      res.status(401);
+      res.send();
+    }
+  });
+  app.post("/gcs_token", (req, res) => {
+    const origin = req.headers.origin ?? "";
+    res.set("access-control-allow-origin", origin);
+    res.set("vary", "origin");
+    const { body } = req;
+    if (
+      typeof body !== "object" ||
+      Array.isArray(body) ||
+      body.token !== TOKEN_VALUE
+    ) {
+      res.status(400);
+      res.send();
+    } else {
+      res.json({ token: "fake_gcs_token:" + body.bucket });
+    }
+  });
+  const server = app.listen(0, "localhost");
+  return new Promise((resolve, reject) => {
+    server.on("error", reject);
+    server.on("listening", () => {
+      const port = (server.address() as AddressInfo).port;
+      resolve({
+        url: `http://localhost:${port}`,
+        [Symbol.asyncDispose]: () =>
+          new Promise<void>((resolve) => server.close(() => resolve())),
+      });
+    });
+  });
+}

build_tools/vitest/polyfill-browser-globals-in-node.ts

@@ -0,0 +1,16 @@
+import { webcrypto } from "node:crypto";
+import type { JSDOM } from "jsdom";
+
+declare let jsdom: JSDOM;
+
+Object.defineProperty(globalThis, "crypto", {
+  value: webcrypto,
+});
+
+for (const name of [
+  /*"DOMParser", "XPathResult", "navigator"*/
+] as const) {
+  Object.defineProperty(globalThis, name, {
+    value: jsdom.window[name],
+  });
+}