fix: Defer initialization of redis

Author: effigies

packages/openneuro-server/src/app.ts

@@ -22,7 +22,7 @@ import * as jwt from "./libs/authentication/jwt"
 import * as auth from "./libs/authentication/states"
 import { sitemapHandler } from "./handlers/sitemap"
 import { setupPassportAuth } from "./libs/authentication/passport"
-import { redis } from "./libs/redis"
+import { getRedis } from "./libs/redis"
 import { version } from "./lerna.json"
 export { Express } from "express-serve-static-core"
 
@@ -67,7 +67,7 @@ export async function expressApolloSetup() {
     // Always allow introspection - our schema is public
     introspection: true,
     // @ts-expect-error Type mismatch for keyv and ioredis recent releases
-    cache: new KeyvAdapter(new Keyv({ store: new KeyvRedis(redis) })),
+    cache: new KeyvAdapter(new Keyv({ store: new KeyvRedis(getRedis()) })),
     plugins: [
       ApolloServerPluginLandingPageLocalDefault(),
       ApolloServerPluginDrainHttpServer({

packages/openneuro-server/src/datalad/__tests__/contributors.spec.ts

@@ -27,7 +27,7 @@ vi.mock("../../cache/item")
 vi.mock("../files")
 vi.mock("../../utils/datasetOrSnapshot")
 vi.mock("../libs/redis", () => ({
-  redis: vi.fn(),
+  getRedis: vi.fn(),
 }))
 
 const mockYamlLoad = vi.mocked(yaml.load)

packages/openneuro-server/src/datalad/__tests__/files.spec.ts

@@ -13,7 +13,7 @@ import type { TreeEntry } from "../../cache/tree"
 
 vi.mock("ioredis")
 vi.mock("../../config.ts")
-vi.mock("../../libs/redis", () => ({ redis: {} }))
+vi.mock("../../libs/redis", () => ({ getRedis: () => ({}) }))
 vi.mock("../../libs/presign", () => ({
   getPresignedUrl: vi.fn().mockResolvedValue(
     "https://s3.amazonaws.com/bucket/key?presigned=true",

packages/openneuro-server/src/datalad/__tests__/snapshots.spec.ts

@@ -10,12 +10,12 @@ import { connect } from "mongoose"
 // Mock requests to Datalad service
 vi.mock("superagent")
 vi.mock("../../libs/redis.js", () => ({
-  redis: {
+  getRedis: () => ({
     del: vi.fn(),
-  },
-  redlock: {
+  }),
+  getRedlock: () => ({
     lock: vi.fn().mockImplementation(() => ({ unlock: vi.fn() })),
-  },
+  }),
 }))
 // Mock draft files calls
 vi.mock("../draft.ts", () => ({

packages/openneuro-server/src/datalad/contributors.ts

@@ -1,6 +1,6 @@
 import * as Sentry from "@sentry/node"
 import CacheItem, { CacheType } from "../cache/item"
-import { redis } from "../libs/redis"
+import { getRedis } from "../libs/redis"
 import {
   type DatasetOrSnapshot,
   datasetOrSnapshot,
@@ -26,7 +26,7 @@ export const contributors = async (
   if (!datasetId) return []
 
   const revisionShort = revision ? revision.substring(0, 7) : "HEAD"
-  const dataciteCache = new CacheItem(redis, CacheType.dataciteYml, [
+  const dataciteCache = new CacheItem(getRedis(), CacheType.dataciteYml, [
     datasetId,
     revisionShort,
   ])

packages/openneuro-server/src/datalad/dataset.ts

@@ -12,7 +12,7 @@ import type * as Mongoose from "mongoose"
 import config from "../config"
 import * as subscriptions from "../handlers/subscriptions"
 import { generateDataladCookie } from "../libs/authentication/jwt"
-import { redis } from "../libs/redis"
+import { getRedis } from "../libs/redis"
 import CacheItem, { CacheType } from "../cache/item"
 import { getDraftRevision, updateDatasetRevision } from "./draft"
 import { encodeFilePath, filesUrl, fileUrl, getFileName } from "./files"
@@ -114,7 +114,7 @@ export const deleteDataset = async (datasetId, user) => {
 export const cacheDatasetConnection = (options) => (connectionArguments) => {
   const connection = datasetsConnection(options)
   const cache = new CacheItem(
-    redis,
+    getRedis(),
     CacheType.datasetsConnection,
     [objectHash(options)],
     60,

packages/openneuro-server/src/datalad/description.ts

@@ -3,7 +3,7 @@
  */
 import config from "../config"
 import request from "superagent"
-import { redis } from "../libs/redis"
+import { getRedis } from "../libs/redis"
 import { commitFiles } from "./dataset"
 import { fileUrl } from "./files"
 import { generateDataladCookie } from "../libs/authentication/jwt"
@@ -135,7 +135,7 @@ export const description = async (obj) => {
     Name: datasetId,
     BIDSVersion: "1.8.0",
   }
-  const cache = new CacheItem(redis, CacheType.datasetDescription, [
+  const cache = new CacheItem(getRedis(), CacheType.datasetDescription, [
     datasetId,
     revision.substring(0, 7),
   ])

packages/openneuro-server/src/datalad/draft.ts

@@ -5,7 +5,7 @@ import request from "superagent"
 import Dataset from "../models/dataset"
 import { getDatasetWorker } from "../libs/datalad-service"
 import CacheItem, { CacheType } from "../cache/item"
-import { redis } from "../libs/redis"
+import { getRedis } from "../libs/redis"
 
 // Draft info resolver
 type DraftInfo = {
@@ -22,7 +22,12 @@ export const getDraftRevision = async (datasetId): Promise<string> => {
 }
 
 export const getDraftInfo = async (datasetId) => {
-  const cache = new CacheItem(redis, CacheType.draftRevision, [datasetId], 10)
+  const cache = new CacheItem(
+    getRedis(),
+    CacheType.draftRevision,
+    [datasetId],
+    10,
+  )
   return cache.get(async (_doNotCache): Promise<DraftInfo> => {
     const draftUrl = `http://${
       getDatasetWorker(

packages/openneuro-server/src/datalad/files.ts

@@ -1,4 +1,4 @@
-import { redis } from "../libs/redis"
+import { getRedis } from "../libs/redis"
 import { getDatasetWorker } from "../libs/datalad-service"
 import {
   getPresignedUrl,
@@ -184,7 +184,7 @@ export async function entryToDatasetFile(
   }
   let url: string
   if (entry.p && entry.k && entry.v) {
-    url = await getPresignedUrl(redis, entry.b, entry.k, entry.v)
+    url = await getPresignedUrl(getRedis(), entry.b, entry.k, entry.v)
   } else if (entry.k && entry.v) {
     url = publicS3Url(entry.b, entry.k, entry.v)
   } else {
@@ -260,15 +260,15 @@ async function cacheWorkerTrees(
         (f) => f.directory || f.urls[0]?.includes("s3.amazonaws.com"),
       )
       if (allExported) {
-        void setTree(redis, hash, entries)
+        void setTree(getRedis(), hash, entries)
         permanentHashes.push(hash)
       } else {
-        void setTree(redis, hash, entries, 600)
+        void setTree(getRedis(), hash, entries, 600)
       }
     }
   }
   if (permanentHashes.length > 0) {
-    void addDatasetTrees(redis, datasetId, permanentHashes)
+    void addDatasetTrees(getRedis(), datasetId, permanentHashes)
   }
   return result
 }
@@ -283,7 +283,10 @@ export const resolveGitRef = async (
   datasetId: string,
   treeish: string,
 ): Promise<string> => {
-  const cache = new CacheItem(redis, CacheType.gitRef, [datasetId, treeish])
+  const cache = new CacheItem(getRedis(), CacheType.gitRef, [
+    datasetId,
+    treeish,
+  ])
   return cache.get(async () => {
     const url = `http://${
       getDatasetWorker(datasetId)
@@ -324,7 +327,7 @@ export const getFiles = async (
     }
   }
   // Try cache first
-  const cached = await getTree(redis, treeish)
+  const cached = await getTree(getRedis(), treeish)
   if (cached) {
     return entriesToDatasetFiles(cached, datasetId)
   }
@@ -354,10 +357,10 @@ export async function getFilesRecursive(
 ): Promise<DatasetFile[]> {
   const needsPresign = await datasetNeedsPresign(datasetId)
   // Check for cached commit-to-trees mapping
-  const cachedTreeHashes = await getCommitTrees(redis, tree)
+  const cachedTreeHashes = await getCommitTrees(getRedis(), tree)
   if (cachedTreeHashes) {
     // Bulk-fetch all trees in one pipeline
-    const treesMap = await getTreesBulk(redis, cachedTreeHashes)
+    const treesMap = await getTreesBulk(getRedis(), cachedTreeHashes)
     if (treesMap.size < cachedTreeHashes.length) {
       // Batch-fetch all missing trees from the worker in one request
       const missingHashes = cachedTreeHashes.filter((h) => !treesMap.has(h))
@@ -381,7 +384,7 @@ export async function getFilesRecursive(
 
   while (pendingHashes.length > 0) {
     // Check cache for all pending hashes
-    const cached = await getTreesBulk(redis, pendingHashes)
+    const cached = await getTreesBulk(getRedis(), pendingHashes)
     const uncached = pendingHashes.filter((h) => !cached.has(h))
 
     // Fetch all uncached trees in one worker request
@@ -417,8 +420,8 @@ export async function getFilesRecursive(
   // Cache the commit-to-trees mapping for next time
   if (collectedHashes.size > 0) {
     const hashArray = [...collectedHashes]
-    void setCommitTrees(redis, tree, hashArray)
-    void addDatasetTrees(redis, datasetId, hashArray)
+    void setCommitTrees(getRedis(), tree, hashArray)
+    void addDatasetTrees(getRedis(), datasetId, hashArray)
   }
 
   return reconstructFromTrees(treesMap, tree, path, datasetId)
@@ -485,7 +488,7 @@ async function reconstructFromTrees(
   // Bulk-resolve presigned URLs in minimal Redis requests
   if (presignIndices.length > 0) {
     const urls = await getPresignedUrlsBulk(
-      redis,
+      getRedis(),
       presignIndices.map((i) => ({
         bucket: fileEntries[i].entry.b,
         s3Key: fileEntries[i].entry.k,

packages/openneuro-server/src/datalad/readme.ts

@@ -1,5 +1,5 @@
 import { addFileString, commitFiles } from "./dataset"
-import { redis } from "../libs/redis"
+import { getRedis } from "../libs/redis"
 import CacheItem, { CacheType } from "../cache/item"
 import { getDatasetWorker } from "../libs/datalad-service"
 import { datasetOrSnapshot } from "../utils/datasetOrSnapshot"
@@ -14,7 +14,7 @@ export const readmeUrl = (datasetId, revision) => {
 
 export const readme = (obj) => {
   const { datasetId, revision } = datasetOrSnapshot(obj)
-  const cache = new CacheItem(redis, CacheType.readme, [
+  const cache = new CacheItem(getRedis(), CacheType.readme, [
     datasetId,
     revision.substring(0, 7),
   ])