Merge pull request #153 from shloksooch/main

updates

Author: shloksooch

.dev/rebuild-cache.js

@@ -19,6 +19,9 @@
 import { createClient } from '@supabase/supabase-js'
 import { config } from 'dotenv'
 import { execSync } from 'child_process'
+import { readFileSync, existsSync } from 'fs'
+import { resolve, dirname } from 'path'
+import { fileURLToPath } from 'url'
 import area from '@turf/area'
 
 // Load .env file for local development (GitHub Actions sets env vars directly)
@@ -162,6 +165,47 @@ async function rebuildCache() {
     console.log('⚙️ Processing service areas...')
     const serviceAreas = buildServiceAreasFromEvents(events, geometryMap)
 
+    // STEP 3.5: Load news.csv if it exists
+    console.log('📰 Loading news data...')
+    let newsItems = []
+    try {
+      const __dirname = dirname(fileURLToPath(import.meta.url))
+      const newsPath = resolve(__dirname, '..', 'news.csv')
+      if (existsSync(newsPath)) {
+        const newsCSV = readFileSync(newsPath, 'utf-8')
+        const lines = newsCSV.trim().split('\n')
+        if (lines.length > 1) {
+          const header = lines[0].split(',')
+          for (let i = 1; i < lines.length; i++) {
+            const line = lines[i].trim()
+            if (!line) continue
+            // Simple CSV parse (handles quoted fields)
+            const fields = []
+            let current = '', inQuotes = false
+            for (let j = 0; j < line.length; j++) {
+              const ch = line[j]
+              if (inQuotes) {
+                if (ch === '"' && line[j + 1] === '"') { current += '"'; j++ }
+                else if (ch === '"') { inQuotes = false }
+                else { current += ch }
+              } else {
+                if (ch === '"') { inQuotes = true }
+                else if (ch === ',') { fields.push(current); current = '' }
+                else { current += ch }
+              }
+            }
+            fields.push(current)
+            const newsItem = {}
+            header.forEach((col, idx) => { newsItem[col.trim()] = (fields[idx] || '').trim() })
+            if (newsItem.headline) newsItems.push(newsItem)
+          }
+        }
+      }
+      console.log(`   Found ${newsItems.length} news items`)
+    } catch (err) {
+      console.warn('⚠️ Failed to load news.csv:', err.message)
+    }
+
     // STEP 4: Create final data structure
     // Transform geometries to camelCase for frontend
     const geometriesForFrontend = geometriesWithData.map(geo => ({
@@ -188,6 +232,7 @@ async function rebuildCache() {
         }
       },
       events: events,
+      news: newsItems,
       geometries: geometriesForFrontend,
       service_areas: serviceAreas,
       date_range: {

.dev/schema.json

@@ -31,7 +31,8 @@
         "access_policy_changed",
         "service_model_updated",
         "platform_updated",
-        "fleet_partner_changed"
+        "fleet_partner_changed",
+        "direct_booking_updated"
       ],
       "description": "Type of event"
     },
@@ -210,6 +211,10 @@
     "fleet_partner_changed": {
       "requiredFields": ["company", "city", "fleet_partner"],
       "description": "Fleet management partner change"
+    },
+    "direct_booking_updated": {
+      "requiredFields": ["company", "city", "direct_booking"],
+      "description": "Direct booking availability change"
     }
   },
 

.github/workflows/deploy-data.yml

@@ -5,12 +5,14 @@ on:
     branches: [main, staging]
     paths:
       - 'events.csv'
+      - 'news.csv'
       - 'geometries/**'
 
   pull_request:
     branches: [main, staging]
     paths:
       - 'events.csv'
+      - 'news.csv'
       - 'geometries/**'
 
   workflow_dispatch:

LIVE_SURFACE.md

@@ -0,0 +1,41 @@
+# Live Surface: av-map-data
+
+This file documents the currently active production surface area for the `av-map-data` repo.
+
+## Production Entrypoints
+
+- Canonical data files:
+  - `events.csv`
+  - `geometries/*.geojson`
+- CI pipeline: `.github/workflows/deploy-data.yml`
+
+## Active Runtime Flow
+
+1. `validate` job runs `pytest tests/ -v`
+2. `update-cache` job runs on non-PR events after validation
+3. CSV import: `.dev/import-csv.py`
+4. Geometry upload: `.dev/upload-geometries.js`
+5. Geometry metadata sync: `.dev/sync-geometries-table.js`
+6. Cache rebuild and upload: `.dev/rebuild-cache.js`
+
+## Critical Dependencies
+
+- GitHub Actions for validation and publish automation
+- Supabase database tables (`av_events*`, geometry metadata tables)
+- Supabase storage buckets (cache + geometry buckets)
+- Python 3.11 + Node 20 in CI
+
+## Active Environment Variables
+
+- `SUPABASE_URL`
+- `SUPABASE_SERVICE_KEY`
+- `STAGING` (controls staging vs production target)
+- `GITHUB_ACTIONS` (CI behavior toggle)
+
+## Source of Truth for Data Contract
+
+- Schema contract: `.dev/schema.json`
+- Validation tests: `tests/test_validation.py`
+- Contributor guidance: `CONTRIBUTING.md`
+
+These three must remain aligned for event types, required fields, and geometry naming rules.

news.csv

@@ -0,0 +1 @@
+date,company,city,headline,summary,source_url,image_url,category

tests/test_validation.py

@@ -85,12 +85,7 @@ def test_date_format(csv_file):
 
 def test_event_types(csv_file):
     """Test that event types are valid."""
-    valid_event_types = [
-        'service_created', 'service_ended', 'geometry_updated',
-        'vehicle_types_updated', 'supervision_updated', 'fares_policy_changed',
-        'access_policy_changed', 'service_model_updated', 'platform_updated', 'fleet_partner_changed',
-        'direct_booking_updated', 'service_testing', 'service_announced'
-    ]
+    valid_event_types = get_event_types()
     errors = []
 
     with open(csv_file, 'r') as f: