more aggregation test cases to figure out what the AI can reliably deduplicate

Author: ndbroadbent

bun.lock

@@ -72,6 +72,7 @@
   },
   "dependencies": {
     "commander": "^14.0.2",
+    "csv-parse": "^6.1.0",
     "exceljs": "^4.4.0",
     "html-entities": "^2.6.0",
     "i18n-iso-countries": "^7.14.0",

package.json

@@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest'
 import { createActivity as createTestActivity } from '../test-support'
 import type { ClassifiedActivity } from '../types'
 import {
-  deduplicateActivities,
+  aggregateActivities,
   filterByMentionCount,
   getFirstMentionedAt,
   getLastMentionedAt,
@@ -29,9 +29,9 @@ function createActivity(
 }
 
 describe('Aggregation Module', () => {
-  describe('deduplicateActivities', () => {
+  describe('aggregateActivities', () => {
     it('returns empty array for empty input', () => {
-      const result = deduplicateActivities([])
+      const result = aggregateActivities([])
       expect(result).toEqual([])
     })
 
@@ -48,7 +48,7 @@ describe('Aggregation Module', () => {
         })
       ]
 
-      const result = deduplicateActivities(activities)
+      const result = aggregateActivities(activities)
 
       expect(result).toHaveLength(1)
       const first = result[0]
@@ -82,7 +82,7 @@ describe('Aggregation Module', () => {
         })
       ]
 
-      const result = deduplicateActivities(activities)
+      const result = aggregateActivities(activities)
 
       expect(result).toHaveLength(1)
       const first = result[0]
@@ -110,7 +110,7 @@ describe('Aggregation Module', () => {
         })
       ]
 
-      const result = deduplicateActivities(activities)
+      const result = aggregateActivities(activities)
 
       expect(result).toHaveLength(1)
       const first = result[0]
@@ -138,7 +138,7 @@ describe('Aggregation Module', () => {
         })
       ]
 
-      const result = deduplicateActivities(activities)
+      const result = aggregateActivities(activities)
 
       // Compound activities only match on exact title, not fields
       expect(result).toHaveLength(2)
@@ -169,7 +169,7 @@ describe('Aggregation Module', () => {
         })
       ]
 
-      const result = deduplicateActivities(activities)
+      const result = aggregateActivities(activities)
 
       expect(result).toHaveLength(3)
       expect(result.every((r) => getMentionCount(r) === 1)).toBe(true)
@@ -195,7 +195,7 @@ describe('Aggregation Module', () => {
         })
       ]
 
-      const result = deduplicateActivities(activities)
+      const result = aggregateActivities(activities)
 
       // "Kazuya" vs "Kazuya Restaurant" - not 95% similar, should NOT match
       expect(result).toHaveLength(2)
@@ -232,7 +232,7 @@ describe('Aggregation Module', () => {
         })
       ]
 
-      const result = deduplicateActivities(activities)
+      const result = aggregateActivities(activities)
 
       expect(result).toHaveLength(1)
       const first = result[0]
@@ -280,7 +280,7 @@ describe('Aggregation Module', () => {
         })
       ]
 
-      const result = deduplicateActivities(activities)
+      const result = aggregateActivities(activities)
 
       expect(result).toHaveLength(1)
       expect(result[0]?.messages).toHaveLength(3)
@@ -317,7 +317,7 @@ describe('Aggregation Module', () => {
         interestingScore: 0.5
       })
 
-      const result = deduplicateActivities([act1, act2, act3])
+      const result = aggregateActivities([act1, act2, act3])
 
       expect(result).toHaveLength(1)
       expect(result[0]?.funScore).toBe(0.7) // (0.8 + 0.6 + 0.7) / 3 = 0.7
@@ -351,7 +351,7 @@ describe('Aggregation Module', () => {
         })
       ]
 
-      const result = deduplicateActivities(activities)
+      const result = aggregateActivities(activities)
 
       expect(result).toHaveLength(1)
       // First occurrence is the primary
@@ -403,7 +403,7 @@ describe('Aggregation Module', () => {
         createActivity({ id: 6, activity: 'Thrice Mentioned', action: 'explore', object: 'trail' })
       ]
 
-      const deduped = deduplicateActivities(raw)
+      const deduped = aggregateActivities(raw)
       const result = getMostWanted(deduped)
 
       expect(result).toHaveLength(2)
@@ -421,7 +421,7 @@ describe('Aggregation Module', () => {
         createActivity({ id: 6, activity: 'activity c', action: 'try', object: 'thing-c' })
       ]
 
-      const deduped = deduplicateActivities(raw)
+      const deduped = aggregateActivities(raw)
       const result = getMostWanted(deduped, 2)
 
       expect(result).toHaveLength(2)

src/cli/aggregation.test.ts

@@ -134,7 +134,7 @@ function round(value: number, decimals: number): number {
  * @param activities Classified activities to deduplicate
  * @returns Deduplicated activities with merged messages
  */
-export function deduplicateActivities(
+export function aggregateActivities(
   activities: readonly ClassifiedActivity[]
 ): ClassifiedActivity[] {
   if (activities.length === 0) return []

src/cli/aggregation.ts

@@ -50,7 +50,7 @@ describe('scan command', () => {
   })
 
   it('finds specific suggestions in output', () => {
-    const { stdout } = runCli(`scan ${FIXTURE_INPUT} --cache-dir ${testState.tempCacheDir}`)
+    const { stdout } = runCli(`scan ${FIXTURE_INPUT} -n 20 --cache-dir ${testState.tempCacheDir}`)
 
     expect(stdout).toContain('Karangahake Gorge')
     expect(stdout).toContain('Prinzhorn collection')

src/cli/e2e/02-scan.test.ts

@@ -56,8 +56,8 @@ describe('classify command', () => {
     expect(hotAirBalloon).toBeDefined()
     expect(hotAirBalloon?.category).toBeOneOf(['experiences', 'travel'])
     expect(hotAirBalloon?.messages[0]?.sender).toBe('Alice Smith')
-    expect(hotAirBalloon?.funScore).toBe(0.9)
-    expect(hotAirBalloon?.interestingScore).toBe(0.9)
+    expect(hotAirBalloon?.funScore).toBeGreaterThanOrEqual(0.8)
+    expect(hotAirBalloon?.interestingScore).toBeGreaterThanOrEqual(0.8)
     expect(hotAirBalloon?.country).toBe('Turkey')
     expect(hotAirBalloon?.messages[0]?.message).toMatch(/hot air ballon/i)
 
@@ -93,12 +93,82 @@ describe('classify command', () => {
     expect(yellowstone?.messages[0]?.sender).toBe('John Smith')
     expect(yellowstone?.venue).toMatch(/Yellowstone/i)
 
-    // Check Karangahake Gorge
+    // Check Karangahake Gorge - should be aggregated from 2 mentions
     const karangahake = activities.find((a) => a.activity.includes('Karangahake'))
     expect(karangahake).toBeDefined()
     expect(karangahake?.category).toBe('nature')
     expect(karangahake?.messages[0]?.sender).toBe('John Smith')
     expect(karangahake?.country).toBe('New Zealand')
+    // Should have 2 messages from aggregation
+    expect(karangahake?.messages.length).toBe(2)
+  })
+
+  it('aggregates duplicate activities by merging messages', () => {
+    const activities = readCacheJson<ClassifiedActivity[]>(
+      testState.tempCacheDir,
+      'classifications.json'
+    )
+
+    // Karangahake Gorge is mentioned twice in the chat - should be aggregated
+    const karangahake = activities.find((a) => a.activity.toLowerCase().includes('karangahake'))
+    expect(karangahake).toBeDefined()
+    expect(karangahake?.messages.length).toBe(2)
+
+    // Check both messages are preserved with correct senders
+    const senders = karangahake?.messages.map((m) => m.sender) ?? []
+    expect(senders).toContain('John Smith')
+
+    // Check date range spans both mentions (Oct 11 and Nov 15)
+    const dates = karangahake?.messages.map((m) => new Date(m.timestamp)) ?? []
+    const sortedDates = dates.sort((a, b) => a.getTime() - b.getTime())
+    if (sortedDates.length >= 2) {
+      const firstDate = sortedDates[0]
+      const lastDate = sortedDates[sortedDates.length - 1]
+      if (firstDate && lastDate) {
+        // First mention is Oct 11, second is Nov 15 - at least a month apart
+        const daysDiff = (lastDate.getTime() - firstDate.getTime()) / (1000 * 60 * 60 * 24)
+        expect(daysDiff).toBeGreaterThanOrEqual(30)
+      }
+    }
+
+    // Paintball is mentioned twice - should be aggregated
+    const paintball = activities.find((a) => a.activity.toLowerCase().includes('paintball'))
+    expect(paintball).toBeDefined()
+    expect(paintball?.messages.length).toBe(2)
+  })
+
+  it('does not aggregate compound activities with non-compound', () => {
+    const activities = readCacheJson<ClassifiedActivity[]>(
+      testState.tempCacheDir,
+      'classifications.json'
+    )
+
+    // "Go to a play or a concert" is compound, "Go to a play" is not
+    // They should NOT be aggregated together
+    const playActivities = activities.filter((a) =>
+      a.activity.toLowerCase().includes('go to a play')
+    )
+    // sometimes one is compound, sometimes the AI adds both as separate activities and they get aggregated
+    expect(playActivities.length).toBeOneOf([1, 2])
+  })
+
+  it('does not create duplicate entries for aggregated activities', () => {
+    const activities = readCacheJson<ClassifiedActivity[]>(
+      testState.tempCacheDir,
+      'classifications.json'
+    )
+
+    // Count activities mentioning Karangahake - should be exactly 1 (aggregated)
+    const karangahakeCount = activities.filter((a) =>
+      a.activity.toLowerCase().includes('karangahake')
+    ).length
+    expect(karangahakeCount).toBe(1)
+
+    // Count paintball activities - should be exactly 1 (aggregated)
+    const paintballCount = activities.filter((a) =>
+      a.activity.toLowerCase().includes('paintball')
+    ).length
+    expect(paintballCount).toBe(1)
   })
 
   it('sorts activities by score (interesting * 2 + fun)', () => {

src/cli/e2e/07-classify.test.ts

@@ -189,7 +189,7 @@ describe('geocode command', () => {
     // Original classification fields should be preserved (AI may classify as experiences or nature)
     expect(whaleSafari?.category).toBeOneOf(['experiences', 'nature'])
     expect(whaleSafari?.messages[0]?.sender).toBe('John Smith')
-    expect(whaleSafari?.venue).toBe('Auckland Whale & Dolphin Safari')
+    expect(whaleSafari?.venue).toMatch(/Whale/i)
     expect(whaleSafari?.city).toBe('Auckland')
     expect(whaleSafari?.country).toBe('New Zealand')
     expect(whaleSafari?.funScore).toBeGreaterThanOrEqual(0.8)

src/cli/e2e/08-geocode.test.ts

@@ -7,6 +7,7 @@
 import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
+import { parse } from 'csv-parse/sync'
 import { afterAll, beforeAll, describe, expect, it } from 'vitest'
 import { FIXTURE_INPUT, runCli, testState } from './helpers'
 
@@ -68,6 +69,7 @@ describe('analyze command', () => {
       expect(header).toContain('location')
       expect(header).toContain('latitude')
       expect(header).toContain('longitude')
+      expect(header).toContain('mention_count')
     })
 
     it('contains activity data rows', () => {
@@ -81,6 +83,21 @@ describe('analyze command', () => {
       expect(csv.toLowerCase()).toContain('hot air balloon')
       expect(csv.toLowerCase()).toContain('whale')
     })
+
+    it('shows mention count for aggregated activities', () => {
+      const csv = readFileSync(join(outputDir, 'activities.csv'), 'utf-8')
+      const records = parse(csv, { columns: true }) as Array<Record<string, string>>
+
+      // Find Karangahake row - should have mention_count of 2
+      const karangahake = records.find((r) => r.activity?.toLowerCase().includes('karangahake'))
+      expect(karangahake).toBeDefined()
+      expect(karangahake?.mention_count).toBe('2')
+
+      // Find paintball row - should have mention_count of 2
+      const paintball = records.find((r) => r.activity?.toLowerCase().includes('paintball'))
+      expect(paintball).toBeDefined()
+      expect(paintball?.mention_count).toBe('2')
+    })
   })
 
   describe('JSON export', () => {
@@ -114,6 +131,25 @@ describe('analyze command', () => {
       expect(activity).toHaveProperty('funScore')
       expect(activity).toHaveProperty('interestingScore')
     })
+
+    it('aggregated activities have multiple messages', () => {
+      const json = readFileSync(join(outputDir, 'activities.json'), 'utf-8')
+      const data = JSON.parse(json)
+
+      // Find Karangahake Gorge - mentioned twice, should have 2 messages
+      const karangahake = data.activities.find((a: { activity: string }) =>
+        a.activity.toLowerCase().includes('karangahake')
+      )
+      expect(karangahake).toBeDefined()
+      expect(karangahake.messages.length).toBe(2)
+
+      // Find paintball activity - mentioned twice, should have 2 messages
+      const paintball = data.activities.find((a: { activity: string }) =>
+        a.activity.toLowerCase().includes('paintball')
+      )
+      expect(paintball).toBeDefined()
+      expect(paintball.messages.length).toBe(2)
+    })
   })
 
   describe('Map HTML export', () => {

src/cli/e2e/10-analyze.test.ts

@@ -314,6 +314,10 @@ export function setupE2ETests(): E2ETestState {
     console.log('⚠️  No cache fixture found - API calls will be made')
   }
 
+  if (process.env.DEBUG_E2E) {
+    console.log(`🔍 E2E temp cache dir: ${tempCacheDir}`)
+  }
+
   extractCacheFixture(tempCacheDir)
   const initialCacheHash = hashCacheDirectories(tempCacheDir)
   const initialCacheFiles = listCacheFiles(tempCacheDir)
@@ -384,5 +388,9 @@ export function teardownE2ETests(state: E2ETestState): void {
     console.error('')
   }
 
-  rmSync(state.tempCacheDir, { recursive: true, force: true })
+  if (process.env.DEBUG_E2E) {
+    console.log(`🔍 Preserving temp cache dir for debugging: ${state.tempCacheDir}`)
+  } else {
+    rmSync(state.tempCacheDir, { recursive: true, force: true })
+  }
 }

src/cli/e2e/helpers.ts

@@ -9,6 +9,7 @@
 import { classifyBatch, filterActivities, sortActivitiesByScore } from '../../index'
 import type { ScrapedMetadata } from '../../scraper/types'
 import type { CandidateMessage, ClassifiedActivity, ClassifierConfig } from '../../types'
+import { aggregateActivities } from '../aggregation'
 import { resolveModelConfig, resolveUserContext } from '../model'
 import { runWorkerPool } from '../worker-pool'
 import type { PipelineContext } from './context'
@@ -173,8 +174,9 @@ export async function stepClassify(
     allActivities.push(...batchResult)
   }
 
-  // Filter and sort activities by score (interesting prioritized over fun)
-  const activities = sortActivitiesByScore(filterActivities(allActivities))
+  // Deduplicate, filter, and sort activities by score (interesting prioritized over fun)
+  const deduplicated = aggregateActivities(allActivities)
+  const activities = sortActivitiesByScore(filterActivities(deduplicated))
 
   const stats: ClassifyStats = {
     candidatesClassified: candidates.length,