lots of classifier fixes

Author: ndbroadbent

biome.json

@@ -1,6 +1,6 @@
 {
   "root": false,
-  "$schema": "https://biomejs.dev/schemas/2.3.10/schema.json",
+  "$schema": "https://biomejs.dev/schemas/2.3.11/schema.json",
   "vcs": {
     "enabled": true,
     "clientKind": "git",

bun.lock

@@ -74,7 +74,7 @@
     "prepublishOnly": "bun run build && bun run test"
   },
   "devDependencies": {
-    "@biomejs/biome": "^2.0.0",
+    "@biomejs/biome": "^2.3.11",
     "@types/bun": "^1.2.9",
     "@types/pdfkit": "^0.17.4",
     "@vitest/coverage-v8": "^3.0.0",
@@ -86,6 +86,7 @@
     "vitest": "^3.0.0"
   },
   "dependencies": {
+    "biome": "^0.3.3",
     "commander": "^14.0.2",
     "csv-parse": "^6.1.0",
     "exceljs": "^4.4.0",

package.json

@@ -339,7 +339,7 @@ describe('Cache Integration', () => {
 
       const result = await lookupActivityPlaces(suggestions, config, cache)
 
-      expect(result).toHaveLength(1)
+      expect(result.activities).toHaveLength(1)
       expect(mockFetch).toHaveBeenCalledTimes(1)
     })
 

src/caching/integration.test.ts

@@ -0,0 +1,138 @@
+/**
+ * Shared Prompt Sections
+ *
+ * Reusable prompt sections used by both suggestion and agreement prompts.
+ */
+
+import { VALID_CATEGORIES } from '../categories'
+import { VALID_LINK_TYPES } from '../search/types'
+
+export const SHARED_INCLUDE_RULES = `INCLUDE (output these):
+- Named places: restaurants, cafes, food trucks, bars, venues, parks, trails
+- Specific activities: hiking, kayaking, concerts, movies, shows
+- Travel plans: trips, destinations, hotels, Airbnb
+- Events: festivals, markets, concerts, exhibitions
+- Things to do: hobbies, experiences, skills, sports, games
+- Generic but actionable: "Let's go to a cafe" (specific type of place)`
+
+export const SHARED_TENSE_RULES = `CRITICAL - ONLY FUTURE SUGGESTIONS:
+We want IDEAS for things to do in the future - NOT things already happening or already done.
+
+✅ SUGGESTIONS (include): "We should go to X", "Let's try X", "Want to visit X?", "X looks cool"
+❌ PRESENT (skip): "I'm going to X", "I'm at X now", "Heading to X", "Going to get Y"
+❌ PAST (skip): "I went to X", "I was at X yesterday", "We did X last week"
+
+Even if a message contains a Google Maps link or place name, SKIP IT if the person is describing what they're doing RIGHT NOW or what they already did. We only want future plans and suggestions.
+
+Examples:
+- "I'm going to the shops" → SKIP (present action, not a suggestion)
+- "I'm going here to get some boxes [maps link]" → SKIP (current errand, even with link)
+- "Let's go to the department store sometime" → INCLUDE (future suggestion)
+- "We should check out this place [maps link]" → INCLUDE (suggestion with link)`
+
+export const SHARED_SKIP_RULES = `SKIP (don't output):
+- Vague: "wanna go out?", "do something fun", "go somewhere"
+- Logistics: "leave at 3:50pm", "skip the nachos"
+- Questions: "where should we go?"
+- Links without clear discussion about visiting/attending
+- Errands: groceries, vet, mechanic, cleaning, picking up items
+- Work/appointments/chores
+- Romantic/intimate, adult content
+- Sad or stressful: funerals, hospitals, work deadlines, financial worries
+- Sensitive: potential secrets, embarrassing messages, offensive content, or illegal activities
+- Unclear references: "go there again" (where?), "check it out" (what?)`
+
+export const SHARED_IMAGE_SECTION = `IMAGE HINTS:
+image.stock: ALWAYS REQUIRED - specific stock photo query with location context when relevant.
+image.mediaKey: Media library key (e.g., "hot air balloon", "restaurant").
+image.preferStock: true if stock is more specific than mediaKey (e.g., "balloon in Cappadocia" vs generic balloon).`
+
+export const SHARED_LINK_SECTION = `LINK HINTS (specific media titles only): Types: ${VALID_LINK_TYPES.join(', ')}
+- "watch Oppenheimer" → link:{type:"movie", query:"Oppenheimer"}
+- "watch The Bear" → link:{type:"tv_show", query:"The Bear"}
+- "play Wingspan" → link:{type:"physical_game", query:"Wingspan"}
+- "play Baldur's Gate 3" → link:{type:"video_game", query:"Baldur's Gate 3"}
+Use "media" when UNSURE if movie or TV show. Use "game" when UNSURE if video game or board game.
+DON'T use for: generic ("go to movies"), places (use placeName), bands (use wikiName).`
+
+export const SHARED_CATEGORIES_SECTION = `CATEGORIES: ${VALID_CATEGORIES.join(', ')}
+("other" should be used only as a last resort. Only use it if no other category applies.)`
+
+export const SHARED_NORMALIZATION = `NORMALIZATION:
+- Distinct categories: cafe≠restaurant, bar≠restaurant
+- KEEP mediaKey specificity: "glow worm cave" not "cave", "hot air balloon" not "balloon"
+- Disambiguation: "play pool"→"billiards" (cue game), "swim in pool"→"swimming pool"`
+
+export const SHARED_COMPOUND_SECTION = `COMPOUND vs MULTIPLE: For complex activities that one JSON object can't fully represent (e.g., "Go to Iceland and see the aurora"), emit ONE object. For truly separate activities, emit multiple objects.`
+
+// Examples from IMAGES.md - used by both suggestion and agreement prompts
+export const SHARED_EXAMPLES = `EXAMPLES:
+1. "let's go to Paris" → city:"Paris", country:"France", cat:"travel", image:{stock:"paris france eiffel tower", mediaKey:"city", preferStock:true}
+2. "trip to Waiheke" → placeName:"Waiheke Island", region:"Auckland", country:"New Zealand", image:{stock:"waiheke island beach vineyard", mediaKey:"island", preferStock:true}
+3. "board games at Dice Goblin" → placeQuery:"Dice Goblin Auckland", cat:"gaming", image:{stock:"board game cafe meetup", mediaKey:"board game", preferStock:true}
+4. "see Infected Mushroom in Auckland" → wikiName:"Infected Mushroom", city:"Auckland", cat:"music", image:{stock:"psytrance rave edm concert", mediaKey:"concert", preferStock:true}
+5. "visit geothermal park in Rotorua" → city:"Rotorua", cat:"nature", image:{stock:"rotorua mud pools geyser geothermal", mediaKey:"geothermal park", preferStock:true}
+6. "watch The Matrix" → cat:"entertainment", link:{type:"movie", query:"The Matrix"}, image:{stock:"movie night popcorn", mediaKey:"movie night"}
+7. "watch Severance" (unsure if movie/TV) → cat:"entertainment", link:{type:"media", query:"Severance"}, image:{stock:"tv show streaming", mediaKey:"movie night"}
+8. "play Exploding Kittens" (unsure if video/board game) → cat:"gaming", link:{type:"game", query:"Exploding Kittens"}, image:{stock:"card game friends", mediaKey:"card game"}
+9. "go to the theatre" → cat:"entertainment", image:{stock:"theatre stage performance", mediaKey:"theatre"}
+10. "hot air balloon ride" (generic) → cat:"experiences", image:{stock:"hot air balloon sunrise", mediaKey:"hot air balloon"}
+11. "hot air balloon in Turkey" → country:"Turkey", cat:"experiences", image:{stock:"cappadocia hot air balloon sunrise", mediaKey:"hot air balloon", preferStock:true}`
+
+export function buildUserContextSection(homeCountry: string, timezone?: string): string {
+  const timezoneInfo = timezone ? `\nTimezone: ${timezone}` : ''
+  return `USER CONTEXT:
+Home country: ${homeCountry}${timezoneInfo}`
+}
+
+export function buildJsonSchemaSection(includeOffset: boolean): string {
+  const offsetField = includeOffset
+    ? `    "off": <message_offset: 0 if activity is in >>> message, -1 for immediately before, -2 for two before, etc.>,\n`
+    : ''
+
+  return `OUTPUT FORMAT:
+Return JSON array with ONLY activities worth saving. Skip non-activities entirely. Return [] if none found.
+
+\`\`\`json
+[
+  {
+    "msg": <message_id>,
+${offsetField}    "title": "<activity description, under 100 chars, fix any typos (e.g., 'ballon'→'balloon')>",
+    "fun": <0.0-5.0 how fun/enjoyable>,
+    "int": <0.0-5.0 how interesting/unique>,
+    "cat": "<category>",
+
+    // Location fields (top-level, for geocoding + images)
+    "wikiName": "<Wikipedia topic for things like bands, board games, concepts>",
+    "placeName": "<canonical named place - valid Wikipedia title (e.g., 'Waiheke Island', 'Mount Fuji')>",
+    "placeQuery": "<specific named business for Google Places (e.g., 'Dice Goblin Auckland') - NOT generic searches>",
+    "city": "<city name>",
+    "region": "<state/province>",
+    "country": "<country>",
+
+    // Image hints (REQUIRED - stock is always required, mediaKey is optional)
+    "image": {
+      "stock": "<stock photo query - ALWAYS REQUIRED (e.g., 'hot air balloon cappadocia sunrise')>",
+      "mediaKey": "<media library key (e.g., 'hot air balloon', 'restaurant')>",
+      "preferStock": <true if stock query is more specific than generic mediaKey>
+    },
+
+    // Link hints (for resolving media entities to canonical URLs) - use for movies, books, games, music, etc.
+    "link": {
+      "type": "<${VALID_LINK_TYPES.join('|')}>",
+      "query": "<canonical title (e.g., 'The Matrix', 'Project Hail Mary', 'Wingspan')>"
+    }
+  }
+]
+\`\`\`
+
+(OMIT fields that would be null - don't include them. placeName and placeQuery are mutually exclusive - prefer placeName for canonical places.)`
+}
+
+export function buildLocationSection(homeCountry: string): string {
+  return `LOCATION FIELDS (only if explicitly mentioned):
+wikiName: Wikipedia topic for bands/games/concepts (NOT movies/books - use link).
+placeName: Canonical place with Wikipedia article (e.g., "Waiheke Island"). Mutually exclusive with placeQuery.
+placeQuery: SPECIFIC named business for Google Places (e.g., "Dice Goblin Auckland"). NOT generic searches.
+city/region/country: For ambiguous names, assume ${homeCountry}.`
+}

src/classifier/prompt-sections.ts

@@ -7,10 +7,22 @@
  * - Agreement prompt: For [AGREE] candidates pointing to earlier messages
  */
 
-import { VALID_CATEGORIES } from '../categories'
 import type { ScrapedMetadata } from '../scraper/types'
-import { VALID_LINK_TYPES } from '../search/types'
 import type { CandidateMessage, ContextMessage } from '../types'
+import {
+  buildJsonSchemaSection,
+  buildLocationSection,
+  buildUserContextSection,
+  SHARED_CATEGORIES_SECTION,
+  SHARED_COMPOUND_SECTION,
+  SHARED_EXAMPLES,
+  SHARED_IMAGE_SECTION,
+  SHARED_INCLUDE_RULES,
+  SHARED_LINK_SECTION,
+  SHARED_NORMALIZATION,
+  SHARED_SKIP_RULES,
+  SHARED_TENSE_RULES
+} from './prompt-sections'
 
 // Re-export parsing function (types re-exported from ./index.ts)
 export { parseClassificationResponse } from './response-parser'
@@ -135,125 +147,6 @@ export interface ClassificationContext {
   readonly urlMetadata?: Map<string, ScrapedMetadata> | undefined
 }
 
-// ============================================================================
-// SHARED PROMPT SECTIONS
-// ============================================================================
-
-function buildUserContextSection(context: ClassificationContext): string {
-  const timezoneInfo = context.timezone ? `\nTimezone: ${context.timezone}` : ''
-  return `USER CONTEXT:
-Home country: ${context.homeCountry}${timezoneInfo}`
-}
-
-const SHARED_INCLUDE_RULES = `INCLUDE (output these):
-- Named places: restaurants, cafes, food trucks, bars, venues, parks, trails
-- Specific activities: hiking, kayaking, concerts, movies, shows
-- Travel plans: trips, destinations, hotels, Airbnb
-- Events: festivals, markets, concerts, exhibitions
-- Things to do: hobbies, experiences, skills, sports, games
-- Generic but actionable: "Let's go to a cafe" (specific type of place)`
-
-const SHARED_SKIP_RULES = `SKIP (don't output):
-- Vague: "wanna go out?", "do something fun", "go somewhere"
-- Logistics: "leave at 3:50pm", "skip the nachos"
-- Questions: "where should we go?"
-- Links without clear discussion about visiting/attending
-- Errands: groceries, vet, mechanic, cleaning
-- Work/appointments/chores
-- Romantic/intimate, adult content
-- Sad or stressful: funerals, hospitals, work deadlines, financial worries
-- Sensitive: potential secrets, embarrassing messages, offensive content, or illegal activities
-- Unclear references: "go there again" (where?), "check it out" (what?)`
-
-function buildJsonSchemaSection(includeOffset: boolean): string {
-  const offsetField = includeOffset
-    ? `    "off": <message_offset: 0 if activity is in >>> message, -1 for immediately before, -2 for two before, etc.>,\n`
-    : ''
-
-  return `OUTPUT FORMAT:
-Return JSON array with ONLY activities worth saving. Skip non-activities entirely. Return [] if none found.
-
-\`\`\`json
-[
-  {
-    "msg": <message_id>,
-${offsetField}    "title": "<activity description, under 100 chars, fix any typos (e.g., 'ballon'→'balloon')>",
-    "fun": <0.0-5.0 how fun/enjoyable>,
-    "int": <0.0-5.0 how interesting/unique>,
-    "cat": "<category>",
-
-    // Location fields (top-level, for geocoding + images)
-    "wikiName": "<Wikipedia topic for things like bands, board games, concepts>",
-    "placeName": "<canonical named place - valid Wikipedia title (e.g., 'Waiheke Island', 'Mount Fuji')>",
-    "placeQuery": "<specific named business for Google Places (e.g., 'Dice Goblin Auckland') - NOT generic searches>",
-    "city": "<city name>",
-    "region": "<state/province>",
-    "country": "<country>",
-
-    // Image hints (REQUIRED - stock is always required, mediaKey is optional)
-    "image": {
-      "stock": "<stock photo query - ALWAYS REQUIRED (e.g., 'hot air balloon cappadocia sunrise')>",
-      "mediaKey": "<media library key (e.g., 'hot air balloon', 'restaurant')>",
-      "preferStock": <true if stock query is more specific than generic mediaKey>
-    },
-
-    // Link hints (for resolving media entities to canonical URLs) - use for movies, books, games, music, etc.
-    "link": {
-      "type": "<${VALID_LINK_TYPES.join('|')}>",
-      "query": "<canonical title (e.g., 'The Matrix', 'Project Hail Mary', 'Wingspan')>"
-    }
-  }
-]
-\`\`\`
-
-(OMIT fields that would be null - don't include them. placeName and placeQuery are mutually exclusive - prefer placeName for canonical places.)`
-}
-
-const SHARED_IMAGE_SECTION = `IMAGE HINTS:
-image.stock: ALWAYS REQUIRED - specific stock photo query with location context when relevant.
-image.mediaKey: Media library key (e.g., "hot air balloon", "restaurant").
-image.preferStock: true if stock is more specific than mediaKey (e.g., "balloon in Cappadocia" vs generic balloon).`
-
-const SHARED_LINK_SECTION = `LINK HINTS (specific media titles only): Types: ${VALID_LINK_TYPES.join(', ')}
-- "watch Oppenheimer" → link:{type:"movie", query:"Oppenheimer"}
-- "watch The Bear" → link:{type:"tv_show", query:"The Bear"}
-- "play Wingspan" → link:{type:"physical_game", query:"Wingspan"}
-- "play Baldur's Gate 3" → link:{type:"video_game", query:"Baldur's Gate 3"}
-Use "media" when UNSURE if movie or TV show. Use "game" when UNSURE if video game or board game.
-DON'T use for: generic ("go to movies"), places (use placeName), bands (use wikiName).`
-
-function buildLocationSection(homeCountry: string): string {
-  return `LOCATION FIELDS (only if explicitly mentioned):
-wikiName: Wikipedia topic for bands/games/concepts (NOT movies/books - use link).
-placeName: Canonical place with Wikipedia article (e.g., "Waiheke Island"). Mutually exclusive with placeQuery.
-placeQuery: SPECIFIC named business for Google Places (e.g., "Dice Goblin Auckland"). NOT generic searches.
-city/region/country: For ambiguous names, assume ${homeCountry}.`
-}
-
-const SHARED_CATEGORIES_SECTION = `CATEGORIES: ${VALID_CATEGORIES.join(', ')}
-("other" should be used only as a last resort. Only use it if no other category applies.)`
-
-const SHARED_NORMALIZATION = `NORMALIZATION:
-- Distinct categories: cafe≠restaurant, bar≠restaurant
-- KEEP mediaKey specificity: "glow worm cave" not "cave", "hot air balloon" not "balloon"
-- Disambiguation: "play pool"→"billiards" (cue game), "swim in pool"→"swimming pool"`
-
-const SHARED_COMPOUND_SECTION = `COMPOUND vs MULTIPLE: For complex activities that one JSON object can't fully represent (e.g., "Go to Iceland and see the aurora"), emit ONE object. For truly separate activities, emit multiple objects.`
-
-// Examples from IMAGES.md - used by both suggestion and agreement prompts
-const SHARED_EXAMPLES = `EXAMPLES:
-1. "let's go to Paris" → city:"Paris", country:"France", cat:"travel", image:{stock:"paris france eiffel tower", mediaKey:"city", preferStock:true}
-2. "trip to Waiheke" → placeName:"Waiheke Island", region:"Auckland", country:"New Zealand", image:{stock:"waiheke island beach vineyard", mediaKey:"island", preferStock:true}
-3. "board games at Dice Goblin" → placeQuery:"Dice Goblin Auckland", cat:"gaming", image:{stock:"board game cafe meetup", mediaKey:"board game", preferStock:true}
-4. "see Infected Mushroom in Auckland" → wikiName:"Infected Mushroom", city:"Auckland", cat:"music", image:{stock:"psytrance rave edm concert", mediaKey:"concert", preferStock:true}
-5. "visit geothermal park in Rotorua" → city:"Rotorua", cat:"nature", image:{stock:"rotorua mud pools geyser geothermal", mediaKey:"geothermal park", preferStock:true}
-6. "watch The Matrix" → cat:"entertainment", link:{type:"movie", query:"The Matrix"}, image:{stock:"movie night popcorn", mediaKey:"movie night"}
-7. "watch Severance" (unsure if movie/TV) → cat:"entertainment", link:{type:"media", query:"Severance"}, image:{stock:"tv show streaming", mediaKey:"movie night"}
-8. "play Exploding Kittens" (unsure if video/board game) → cat:"gaming", link:{type:"game", query:"Exploding Kittens"}, image:{stock:"card game friends", mediaKey:"card game"}
-9. "go to the theatre" → cat:"entertainment", image:{stock:"theatre stage performance", mediaKey:"theatre"}
-10. "hot air balloon ride" (generic) → cat:"experiences", image:{stock:"hot air balloon sunrise", mediaKey:"hot air balloon"}
-11. "hot air balloon in Turkey" → country:"Turkey", cat:"experiences", image:{stock:"cappadocia hot air balloon sunrise", mediaKey:"hot air balloon", preferStock:true}`
-
 // ============================================================================
 // SUGGESTION PROMPT (regular candidates)
 // ============================================================================
@@ -276,7 +169,7 @@ ${formatted}
 
   return `GOAL: Extract "things to do" from chat history - activities, places, and plans worth putting on a map or list.
 
-${buildUserContextSection(context)}
+${buildUserContextSection(context.homeCountry, context.timezone)}
 
 WHY THESE MESSAGES:
 You're seeing messages pre-filtered by heuristics (regex patterns like "let's go", "we should try") and semantic search (embeddings). We intentionally cast a wide net - you'll see some false positives. Your job is to identify the real activities worth saving.
@@ -290,6 +183,8 @@ CRITICAL: Only extract activities that are IN the >>> message itself. Context is
 
 URLs may have [URL_META: {...}] with scraped metadata - use this to understand what links are about.
 
+${SHARED_TENSE_RULES}
+
 ${SHARED_INCLUDE_RULES}
 
 ${SHARED_SKIP_RULES}
@@ -338,7 +233,7 @@ ${formatted}
 
   return `GOAL: Extract activities that the user is agreeing to or expressing enthusiasm about.
 
-${buildUserContextSection(context)}
+${buildUserContextSection(context.homeCountry, context.timezone)}
 
 These are AGREEMENT messages - phrases like "sounds great!", "I'm keen!", "let's do it!". Your job is to find WHAT they are agreeing to by looking at the messages BEFORE the >>> candidate.
 
@@ -352,6 +247,8 @@ If you can't find a clear activity in the context before the agreement, skip it
 
 URLs may have [URL_META: {...}] with scraped metadata - use this to understand what links are about.
 
+${SHARED_TENSE_RULES}
+
 ${SHARED_INCLUDE_RULES}
 
 ${SHARED_SKIP_RULES}

src/classifier/prompt.ts

@@ -78,7 +78,7 @@ describe('Classifier Pronoun Resolution', () => {
     if (!result.ok) throw new Error(result.error.message)
 
     // Verify the structure without checking exact timestamp (timezone-dependent)
-    expect(result.value).toHaveLength(1)
+    expect(result.value.activities).toHaveLength(1)
     const activity = result.value.activities[0]
     expect(activity).toBeDefined()
     if (!activity) throw new Error('No activity found')

src/classifier/pronoun-resolution.integration.test.ts

@@ -13,7 +13,7 @@ import { EMPTY_LLM_USAGE } from '../types'
 import { DEFAULT_MODELS } from './models'
 
 /** Result from a provider call including usage data */
-export interface ProviderResult {
+interface ProviderResult {
   readonly text: string
   readonly usage: LlmUsage
 }