small changes

Author: Aayush-Agnihotri

.env.example

@@ -1,5 +1,6 @@
 # Scraper
 CORNELL_DINING_API_URL=
+WORKERS=
 
 # Server Configuration
 PORT=8000

prisma/scraper.ts

@@ -4,7 +4,7 @@ import { readFileSync } from 'fs';
 import { join } from 'path';
 import type { RawScrapedData, RawEatery, RawStaticEatery } from './scraperTypes.js';
 import { mapCampusArea, mapEateryType, mapPaymentMethod, mapEventType, mapImageUrl, createWeeklyDate } from './mappers.js';
-import { CampusArea, PaymentMethod, EateryType, EventType } from '@prisma/client';
+import type { CampusArea, PaymentMethod, EateryType } from '@prisma/client';
 
 dotenv.config();
 
@@ -376,11 +376,11 @@ async function testProcessEateries(numEateries: number) {
 
 export async function main() {
   const startTime = Date.now();
-  console.log('Starting scraper...\n');
+  console.log('Starting scraper at', new Date(startTime).toString(), '\n');
 
   // Load static eateries
   const staticStartTime = Date.now();
-  console.log('Loading static eateries...');
+  console.log('Loading static eateries at', new Date(staticStartTime).toString());
   const staticEateries = loadStaticEateries();
   const transformedStaticEateries: ReturnType<typeof transformStaticEatery>[] = [];
   const skippedStaticEateries: Array<{ name: string; error: string }> = [];
@@ -411,8 +411,8 @@ export async function main() {
   console.log(`Found ${diningData.data.eateries.length} eateries from API (${apiFetchDuration}s)`);
 
   const transformStartTime = Date.now();
-  console.log('Transforming API eatery data with 5 concurrent workers...');
-  const transformResults = await transformEateriesConcurrently(diningData.data.eateries, 5);
+  console.log(`Transforming API eatery data with ${process.env.WORKERS} concurrent workers...`);
+  const transformResults = await transformEateriesConcurrently(diningData.data.eateries, parseInt(process.env.WORKERS || '4', 10));
   const transformedApiEateries = transformResults.map((r) => r.result);
   const transformDuration = ((Date.now() - transformStartTime) / 1000).toFixed(2);
   console.log(`✓ Successfully transformed ${transformedApiEateries.length} API eateries (${transformDuration}s)\n`);

prisma/scraperTypes.ts

@@ -117,14 +117,8 @@ export type RawStaticEatery = {
   latitude: number;
   longitude: number;
   location: string;
-  campusArea: {
-    descr: string;
-    descrshort: string;
-  };
-  eateryTypes: Array<{
-    descr: string;
-    descrshort: string;
-  }>;
+  campusArea: RawCampusArea;
+  eateryTypes: RawEateryType[];
   onlineOrdering?: boolean;
   onlineOrderUrl?: string | null;
   operatingHours: Array<{

prisma/staticEateries.json

@@ -990,4 +990,4 @@
       ]
     }
   ]
-}
+}

src/scheduler/scraperScheduler.ts

@@ -6,13 +6,17 @@ let isRunning = false;
 
 export async function runScraperSafely() {
   if (isRunning) {
-    console.log('[Scheduler] Scraper is already running, skipping this execution');
+    console.log(
+      '[Scheduler] Scraper is already running, skipping this execution',
+    );
     return;
   }
 
   isRunning = true;
   const startTime = Date.now();
-  console.log(`[Scheduler] Starting scheduled scraper run at ${new Date().toISOString()}`);
+  console.log(
+    `[Scheduler] Starting scheduled scraper run at ${new Date().toISOString()}`,
+  );
 
   try {
     await runScraper();
@@ -45,4 +49,3 @@ export function startScraperScheduler() {
 
   return task;
 }
-