Handle bare URL activity suggestions

Author: ndbroadbent

src/classifier/index.test.ts

@@ -1,4 +1,5 @@
 import { beforeEach, describe, expect, it, vi } from 'vitest'
+import type { ScrapedMetadata } from '../scraper/types'
 import { createActivity as createTestActivity } from '../test-support'
 import type { CandidateMessage } from '../types'
 
@@ -271,6 +272,67 @@ describe('Classifier Module', () => {
       }
     })
 
+    it('classifies a bare website URL using scraped OG metadata', async () => {
+      const url = 'https://www.omata.co.nz/kitchen'
+      const metadata: ScrapedMetadata = {
+        canonicalUrl: url,
+        contentId: null,
+        title: 'Kitchen — Omata Estate',
+        description: 'A relaxed style eatery with views overlooking the ocean.',
+        hashtags: [],
+        creator: null,
+        imageUrl: null,
+        categories: ['omata.co.nz'],
+        suggestedKeywords: []
+      }
+      mockFetch.mockResolvedValue({
+        ok: true,
+        json: async () => ({
+          content: [
+            {
+              type: 'text',
+              text: createMockClassifierResponse([
+                {
+                  message_id: 1,
+                  activity: 'Go to Omata Estate',
+                  category: 'food',
+                  confidence: 0.95,
+                  city: 'Russell',
+                  country: 'New Zealand',
+                  action: 'go'
+                }
+              ])
+            }
+          ]
+        })
+      })
+
+      const candidates: CandidateMessage[] = [
+        {
+          ...createCandidate(1, url),
+          source: { type: 'url', urlType: 'website' },
+          confidence: 0.55,
+          urls: [url]
+        }
+      ]
+
+      const result = await classifyMessages(candidates, {
+        ...BASE_CONFIG,
+        provider: 'anthropic',
+        apiKey: 'test-key',
+        urlMetadata: new Map([[url, metadata]])
+      })
+
+      expect(result.ok).toBe(true)
+      if (!result.ok) throw new Error(result.error.message)
+      expect(result.value.activities[0]?.activity).toBe('Go to Omata Estate')
+
+      const call = mockFetch.mock.calls[0] as [string, { body: string }]
+      const body = JSON.parse(call[1].body) as { messages: Array<{ content: string }> }
+      expect(body.messages[0]?.content).toContain('[URL_META:')
+      expect(body.messages[0]?.content).toContain('Kitchen — Omata Estate')
+    })
+
     it('handles HTTP errors', async () => {
       mockFetch.mockResolvedValue({
         ok: false,

src/extraction/heuristics/index.test.ts

@@ -250,6 +250,33 @@ Sun, 26/04/2026
 
         expect(result.candidates).toHaveLength(1)
       })
+
+      it('matches a bare generic website URL so OG metadata can be scraped', () => {
+        const url = 'https://www.omata.co.nz/kitchen'
+        const messages = [createMessage(0, url, 'User', [url])]
+
+        const result = extractCandidatesByHeuristics(messages)
+
+        expect(result.urlMatches).toBe(1)
+        expect(result.candidates).toEqual([
+          expect.objectContaining({
+            content: url,
+            confidence: 0.55,
+            source: { type: 'url', urlType: 'website' },
+            urls: [url]
+          })
+        ])
+      })
+
+      it('does not match bare reference URLs without activity context', () => {
+        const url = 'https://en.wikipedia.org/wiki/The_Matrix'
+        const messages = [createMessage(0, url, 'User', [url])]
+
+        const result = extractCandidatesByHeuristics(messages)
+
+        expect(result.candidates).toHaveLength(0)
+        expect(result.urlMatches).toBe(0)
+      })
     })
 
     describe('exclusion patterns', () => {

src/extraction/heuristics/index.ts

@@ -49,6 +49,8 @@ export {
 const DEFAULT_MIN_CONFIDENCE = 0.5
 const ACTIVITY_KEYWORD_BOOST = 0.15
 const URL_SUGGESTION_BOOST = 0.25
+const BARE_WEBSITE_URL_CONFIDENCE = 0.55
+const REFERENCE_WEBSITE_DOMAINS = ['wikipedia.org', 'wikimedia.org']
 
 /**
  * Check if content contains activity-related keywords.
@@ -225,15 +227,49 @@ function findBestUrl(urls: readonly string[]): BestUrl {
   return { type: bestType, confidence: bestConfidence }
 }
 
-function shouldIncludeUrl(firstUrl: string, content: string): boolean {
+function isBareUrlMessage(content: string, urls: readonly string[]): boolean {
+  let remaining = content
+  for (const url of urls) {
+    remaining = remaining.replaceAll(url, ' ')
+  }
+
+  return remaining.replace(/[\s.,;:!?()[\]{}'"“”‘’<>-]/g, '').length === 0
+}
+
+function isReferenceWebsiteUrl(url: string): boolean {
+  try {
+    const hostname = new URL(url).hostname.toLowerCase()
+    return REFERENCE_WEBSITE_DOMAINS.some(
+      (domain) => hostname === domain || hostname.endsWith(`.${domain}`)
+    )
+  } catch {
+    return false
+  }
+}
+
+function shouldIncludeUrl(firstUrl: string, content: string, urls: readonly string[]): boolean {
   // Skip social media URLs - they could be anything (memes, random videos)
   if (isSocialUrl(firstUrl)) return false
   // Include activity URLs or messages with activity phrases
-  return isActivityUrl(firstUrl) || hasActivityPhrase(content)
+  return (
+    isActivityUrl(firstUrl) ||
+    hasActivityPhrase(content) ||
+    (classifyUrl(firstUrl) === 'website' &&
+      !isReferenceWebsiteUrl(firstUrl) &&
+      isBareUrlMessage(content, urls))
+  )
 }
 
-function applyUrlBoosts(confidence: number, content: string): number {
+function applyUrlBoosts(
+  confidence: number,
+  content: string,
+  urlType: string,
+  urls: readonly string[]
+): number {
   let result = confidence
+  if (urlType === 'website' && isBareUrlMessage(content, urls)) {
+    result = Math.max(result, BARE_WEBSITE_URL_CONFIDENCE)
+  }
   if (hasActivityPhrase(content)) {
     result = Math.min(1.0, result + URL_SUGGESTION_BOOST)
   }
@@ -262,10 +298,10 @@ function findUrlMatches(
     if (!msg || !msg.urls || msg.urls.length === 0) continue
 
     const firstUrl = msg.urls[0] ?? ''
-    if (!shouldIncludeUrl(firstUrl, msg.content)) continue
+    if (!shouldIncludeUrl(firstUrl, msg.content, msg.urls)) continue
 
     const best = findBestUrl(msg.urls)
-    const confidence = applyUrlBoosts(best.confidence, msg.content)
+    const confidence = applyUrlBoosts(best.confidence, msg.content, best.type, msg.urls)
 
     if (confidence >= minConfidence) {
       const ctx = getMessageContext(messages, i)
@@ -276,7 +312,7 @@ function findUrlMatches(
         timestamp: msg.timestamp,
         confidence,
         urlType: best.type,
-        candidateType: 'suggestion', // Sharing a URL = suggesting
+        candidateType: 'suggestion',
         urls: msg.urls,
         contextBefore: ctx.before,
         contextAfter: ctx.after

src/scraper/metadata.test.ts

@@ -4,7 +4,8 @@
 
 import { describe, expect, it } from 'vitest'
 import type { CandidateMessage } from '../types'
-import { extractUrlsFromCandidates, extractUrlsFromText } from './metadata'
+import { extractUrlsFromCandidates, extractUrlsFromText, fetchMetadataForUrls } from './metadata'
+import type { FetchFn } from './types'
 
 describe('extractUrlsFromText', () => {
   it('extracts http URLs', () => {
@@ -93,3 +94,38 @@ describe('extractUrlsFromCandidates', () => {
     expect(extractUrlsFromCandidates(candidates)).toEqual(['https://a.com', 'https://b.com'])
   })
 })
+
+describe('fetchMetadataForUrls', () => {
+  it('fetches OG metadata for a bare Omata Kitchen URL', async () => {
+    const url = 'https://www.omata.co.nz/kitchen'
+    const fetch: FetchFn = async () => ({
+      ok: true,
+      status: 200,
+      headers: { get: () => null },
+      text: async () => `
+        <html>
+          <head>
+            <meta property="og:title" content="Kitchen &mdash; Omata Estate">
+            <meta
+              property="og:description"
+              content="A relaxed style eatery with views overlooking the ocean."
+            >
+            <meta property="og:url" content="${url}">
+          </head>
+        </html>
+      `,
+      json: async () => ({}),
+      arrayBuffer: async () => new ArrayBuffer(0)
+    })
+
+    const metadata = await fetchMetadataForUrls([url], { fetch, concurrency: 1 })
+
+    expect(metadata.get(url)).toEqual(
+      expect.objectContaining({
+        canonicalUrl: url,
+        title: 'Kitchen — Omata Estate',
+        description: 'A relaxed style eatery with views overlooking the ocean.'
+      })
+    )
+  })
+})

tests/fixtures/cli/cache-fixture.tar.gz

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2847fa37aded18d0661d9c2b3fe913cb21416544e2cdce074974fd032026bb07
-size 16840445
+oid sha256:10a5b46602cbb4835b4b4ba895c55c22112c689014607d5b016d9700030051ac
+size 18459094