Merge pull request #5 from cuappdev/fanhao/caching2

more caching and data retrieval

Author: yufanhao

src/eateries/eateries.schema.ts

@@ -7,4 +7,5 @@ export const getAllEateriesSchema = z.object({
 });
 
 // Schema for validating cached eatery data
-export const EaterySchema = z.array(z.any()); // This will be validated by the scraper that sends the data
+// This will be validated by the scraper that sends the data, max 500 eateries for safety
+export const EaterySchema = z.array(z.any()).max(500);

src/eateries/eateryService.ts

@@ -2,7 +2,8 @@ import type { Event } from '@prisma/client';
 
 import { prisma } from '../prisma.js';
 import { NotFoundError } from '../utils/AppError.js';
-import { getAllEateriesData } from '../utils/cache.js';
+import { getAllEateriesData, refreshCacheFromDB } from '../utils/cache.js';
+import type { EateryWithEvents } from '../utils/cache.js';
 
 export const getAllEateries = async (days: number = 0) => {
   // Calculate date range for filtering events
@@ -22,36 +23,26 @@ export const getAllEateries = async (days: number = 0) => {
     const cachedEateries = getAllEateriesData();
 
     // Filter events to only include those on the specified day
-    const filteredEateries = cachedEateries.map((eatery) => ({
-      ...eatery,
-      events: eatery.events.filter((event: Event) => {
-        const eventStart = new Date(event.startTimestamp);
-        const eventEnd = new Date(event.endTimestamp);
-
-        // Include event if it overlaps with the target day
-        return eventStart <= endOfDay && eventEnd >= startOfDay;
-      }),
-    }));
+    const filteredEateries = filterEateries(
+      cachedEateries,
+      startOfDay,
+      endOfDay,
+    );
 
     return filteredEateries;
   } catch {
     // If cache is cold (should never happen), fall back to database
-    const eateries = await prisma.eatery.findMany({
-      include: {
-        events: {
-          where: {
-            startTimestamp: {
-              lte: endOfDay,
-            },
-            endTimestamp: {
-              gte: startOfDay,
-            },
-          },
-        },
-      },
-    });
+    refreshCacheFromDB();
+    const cachedEateries = getAllEateriesData();
+
+    // Filter events to only include those on the specified day
+    const filteredEateries = filterEateries(
+      cachedEateries,
+      startOfDay,
+      endOfDay,
+    );
 
-    return eateries;
+    return filteredEateries;
   }
 };
 
@@ -71,8 +62,16 @@ export const getEateryById = async (eateryId: number) => {
     if (error instanceof NotFoundError) {
       throw error;
     }
+    refreshCacheFromDB();
+    const cachedEateries = getAllEateriesData();
+    const eatery = cachedEateries.find((e) => e.id === eateryId);
+
+    if (eatery) {
+      return eatery;
+    }
 
-    const eatery = await prisma.eatery.findUnique({
+    // As a last resort, fetch directly from the database
+    const fromDB = await prisma.eatery.findUnique({
       where: { id: eateryId },
       include: {
         events: {
@@ -96,10 +95,28 @@ export const getEateryById = async (eateryId: number) => {
       },
     });
 
-    if (!eatery) {
-      throw new NotFoundError('Eatery not found');
+    if (!fromDB) {
+      throw new NotFoundError('Eatery not in DB');
     }
 
-    return eatery;
+    return fromDB;
   }
 };
+
+function filterEateries(
+  cachedEateries: EateryWithEvents[],
+  startOfDay: Date,
+  endOfDay: Date,
+) {
+  const filteredEateries = cachedEateries.map((eatery) => ({
+    ...eatery,
+    events: eatery.events.filter((event: Event) => {
+      const eventStart = new Date(event.startTimestamp);
+      const eventEnd = new Date(event.endTimestamp);
+
+      // Include event if it overlaps with the target day
+      return eventStart <= endOfDay && eventEnd >= startOfDay;
+    }),
+  }));
+  return filteredEateries;
+}

src/server.ts

@@ -10,6 +10,7 @@ import { globalErrorHandler } from './middleware/errorHandler.js';
 import { requestLogger } from './middleware/logger.js';
 import { ipRateLimiter } from './middleware/rateLimit.js';
 import { prisma } from './prisma.js';
+import { refreshCacheFromDB } from './utils/cache.js';
 
 const app = express();
 
@@ -65,6 +66,8 @@ const server = app.listen(port, async () => {
   try {
     await prisma.$connect();
     console.log('Database connected successfully');
+    await refreshCacheFromDB();
+    console.log('Cache initialized from database');
   } catch (error) {
     console.error('Failed to connect to database:', error);
     process.exit(1);

src/utils/cache.ts

@@ -2,15 +2,21 @@ import NodeCache from 'node-cache';
 
 import type { Eatery, Event } from '@prisma/client';
 
-import { type NextFunction, type Request, type Response, Router } from 'express';
+import {
+  type NextFunction,
+  type Request,
+  type Response,
+  Router,
+} from 'express';
 
 import { EaterySchema } from '../eateries/eateries.schema.js';
+import { prisma } from '../prisma.js';
 import { UnauthorizedError } from './AppError.js';
 
 export const cacheRouter = Router();
 export const appCache = new NodeCache({ stdTTL: 0 }); // never expire
 
-type EateryWithEvents = Eatery & { events: Event[] };
+export type EateryWithEvents = Eatery & { events: Event[] };
 
 function requireSecret(req: Request, _res: Response, next: NextFunction): void {
   const provided = req.header(process.env.CACHE_REFRESH_HEADER!);
@@ -50,6 +56,36 @@ cacheRouter.post('/internal/refresh-cache', requireSecret, (req, res) => {
   res.status(200).json({ ok: true });
 });
 
+export async function refreshCacheFromDB() {
+  appCache.del('allEateriesData');
+  const eateries = await prisma.eatery.findMany({
+    include: {
+      events: {
+        include: {
+          menu: {
+            include: {
+              items: {
+                include: {
+                  dietaryPreferences: true,
+                  allergens: true,
+                },
+              },
+            },
+          },
+          userEventVotes: true,
+        },
+        orderBy: {
+          startTimestamp: 'asc',
+        },
+      },
+    },
+  });
+  appCache.set('allEateriesData', eateries);
+  const etag = `"eateries-${Date.now()}"`;
+  appCache.set('allEateriesEtag', etag);
+  console.log('Cache updated from DB, allEateriesData refreshed.');
+}
+
 export function clearAppCache(): void {
   appCache.flushAll();
 }