Skip to content

Commit 3bb7b8f

Browse files
lspassos1lspassos1
authored
feat(unrest): surface protest source links (koala73#3492)
* feat(unrest): surface protest source links * fix(unrest): cap protest source links
1 parent 898ecd8 commit 3bb7b8f

15 files changed

Lines changed: 215 additions & 3 deletions

File tree

‎docs/api/UnrestService.openapi.json‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎docs/api/UnrestService.openapi.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,11 @@ components:
281281
description: |-
282282
ConfidenceLevel represents the confidence in event data accuracy.
283283
Used across multiple domains.
284+
sourceUrls:
285+
type: array
286+
items:
287+
type: string
288+
description: Source article URLs, when provided by the upstream feed.
284289
required:
285290
- id
286291
description: |-

‎docs/api/worldmonitor.openapi.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21244,6 +21244,11 @@ components:
2124421244
description: |-
2124521245
ConfidenceLevel represents the confidence in event data accuracy.
2124621246
Used across multiple domains.
21247+
sourceUrls:
21248+
type: array
21249+
items:
21250+
type: string
21251+
description: Source article URLs, when provided by the upstream feed.
2124721252
required:
2124821253
- id
2124921254
description: |-

‎proto/worldmonitor/unrest/v1/unrest_event.proto‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ message UnrestEvent {
4545
repeated string actors = 15;
4646
// Confidence in the event data.
4747
ConfidenceLevel confidence = 16;
48+
// Source article URLs, when provided by the upstream feed.
49+
repeated string source_urls = 17;
4850
}
4951

5052
// UnrestCluster represents a geographic cluster of related unrest events.

‎scripts/seed-unrest-events.mjs‎

Lines changed: 59 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ const GDELT_GKG_URL = 'https://api.gdeltproject.org/api/v1/gkg_geojson';
99
const ACLED_API_URL = 'https://acleddata.com/api/acled/read';
1010
const CANONICAL_KEY = 'unrest:events:v1';
1111
const CACHE_TTL = 16200; // 4.5h — 6x the 45 min cron interval (was 1.3x)
12+
const MAX_SOURCE_URLS = 5;
1213

1314
// ---------- ACLED Event Type Mapping (from _shared.ts) ----------
1415

@@ -44,6 +45,50 @@ function classifyGdeltEventType(name) {
4445
return 'UNREST_EVENT_TYPE_PROTEST';
4546
}
4647

48+
function normalizeSourceUrl(value) {
49+
if (typeof value !== 'string') return '';
50+
const trimmed = value.trim();
51+
if (!trimmed) return '';
52+
try {
53+
const parsed = new URL(trimmed);
54+
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return '';
55+
if (parsed.username || parsed.password) return '';
56+
parsed.hash = '';
57+
return parsed.toString();
58+
} catch {
59+
return '';
60+
}
61+
}
62+
63+
function uniqueSourceUrls(values) {
64+
return [...new Set(values.map(normalizeSourceUrl).filter(Boolean))];
65+
}
66+
67+
function extractAcledSourceUrls(event) {
68+
return mergeSourceUrls([
69+
event.source_url,
70+
event.sourceUrl,
71+
event.url,
72+
event.link,
73+
]);
74+
}
75+
76+
function extractGdeltSourceUrls(properties = {}) {
77+
return mergeSourceUrls([
78+
properties.url,
79+
properties.source_url,
80+
properties.sourceUrl,
81+
properties.document_url,
82+
properties.documentUrl,
83+
properties.article_url,
84+
properties.articleUrl,
85+
]);
86+
}
87+
88+
function mergeSourceUrls(...groups) {
89+
return uniqueSourceUrls(groups.flatMap((group) => Array.isArray(group) ? group : [])).slice(0, MAX_SOURCE_URLS);
90+
}
91+
4792
// ---------- Deduplication (from _shared.ts) ----------
4893

4994
function deduplicateEvents(events) {
@@ -61,11 +106,14 @@ function deduplicateEvents(events) {
61106
unique.set(key, event);
62107
} else if (event.sourceType === 'UNREST_SOURCE_TYPE_ACLED' && existing.sourceType !== 'UNREST_SOURCE_TYPE_ACLED') {
63108
event.sources = [...new Set([...event.sources, ...existing.sources])];
109+
event.sourceUrls = mergeSourceUrls(event.sourceUrls, existing.sourceUrls);
64110
unique.set(key, event);
65111
} else if (existing.sourceType === 'UNREST_SOURCE_TYPE_ACLED') {
66112
existing.sources = [...new Set([...existing.sources, ...event.sources])];
113+
existing.sourceUrls = mergeSourceUrls(existing.sourceUrls, event.sourceUrls);
67114
} else {
68115
existing.sources = [...new Set([...existing.sources, ...event.sources])];
116+
existing.sourceUrls = mergeSourceUrls(existing.sourceUrls, event.sourceUrls);
69117
if (existing.sources.length >= 2) existing.confidence = 'CONFIDENCE_LEVEL_HIGH';
70118
}
71119
}
@@ -153,6 +201,7 @@ async function fetchAcledProtests() {
153201
tags: e.tags?.split(';').map((t) => t.trim()).filter(Boolean) ?? [],
154202
actors: [e.actor1, e.actor2].filter(Boolean),
155203
confidence: 'CONFIDENCE_LEVEL_HIGH',
204+
sourceUrls: extractAcledSourceUrls(e),
156205
};
157206
});
158207
}
@@ -246,8 +295,16 @@ export async function fetchGdeltEvents(opts = {}) {
246295
if (feature.properties?.urltone < existing.worstTone) {
247296
existing.worstTone = feature.properties.urltone;
248297
}
298+
existing.sourceUrls = mergeSourceUrls(existing.sourceUrls, extractGdeltSourceUrls(feature.properties));
249299
} else {
250-
locationMap.set(key, { name, lat, lon, count: 1, worstTone: feature.properties?.urltone ?? 0 });
300+
locationMap.set(key, {
301+
name,
302+
lat,
303+
lon,
304+
count: 1,
305+
worstTone: feature.properties?.urltone ?? 0,
306+
sourceUrls: mergeSourceUrls(extractGdeltSourceUrls(feature.properties)),
307+
});
251308
}
252309
}
253310

@@ -273,6 +330,7 @@ export async function fetchGdeltEvents(opts = {}) {
273330
tags: [],
274331
actors: [],
275332
confidence: loc.count > 20 ? 'CONFIDENCE_LEVEL_HIGH' : 'CONFIDENCE_LEVEL_MEDIUM',
333+
sourceUrls: loc.sourceUrls,
276334
});
277335
}
278336

‎server/worldmonitor/unrest/v1/_shared.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,14 @@ export function classifyGdeltEventType(name: string): UnrestEventType {
6464
// Deduplication (ported from src/services/protests.ts lines 226-258)
6565
// ========================================================================
6666

67+
const MAX_SOURCE_URLS = 5;
68+
69+
export function mergeSourceUrls(...groups: Array<string[] | undefined>): string[] {
70+
return [
71+
...new Set(groups.flatMap((group) => group ?? []).filter((url): url is string => typeof url === 'string' && url.length > 0)),
72+
].slice(0, MAX_SOURCE_URLS);
73+
}
74+
6775
export function deduplicateEvents(events: UnrestEvent[]): UnrestEvent[] {
6876
const unique = new Map<string, UnrestEvent>();
6977

@@ -77,6 +85,7 @@ export function deduplicateEvents(events: UnrestEvent[]): UnrestEvent[] {
7785

7886
const existing = unique.get(key);
7987
if (!existing) {
88+
event.sourceUrls = mergeSourceUrls(event.sourceUrls);
8089
unique.set(key, event);
8190
} else {
8291
// Merge: prefer ACLED (higher confidence), combine sources
@@ -85,12 +94,15 @@ export function deduplicateEvents(events: UnrestEvent[]): UnrestEvent[] {
8594
existing.sourceType !== 'UNREST_SOURCE_TYPE_ACLED'
8695
) {
8796
event.sources = [...new Set([...event.sources, ...existing.sources])];
97+
event.sourceUrls = mergeSourceUrls(event.sourceUrls, existing.sourceUrls);
8898
unique.set(key, event);
8999
} else if (existing.sourceType === 'UNREST_SOURCE_TYPE_ACLED') {
90100
existing.sources = [...new Set([...existing.sources, ...event.sources])];
101+
existing.sourceUrls = mergeSourceUrls(existing.sourceUrls, event.sourceUrls);
91102
} else {
92103
// Both GDELT: combine sources, upgrade confidence if 2+ sources
93104
existing.sources = [...new Set([...existing.sources, ...event.sources])];
105+
existing.sourceUrls = mergeSourceUrls(existing.sourceUrls, event.sourceUrls);
94106
if (existing.sources.length >= 2) {
95107
existing.confidence = 'CONFIDENCE_LEVEL_HIGH';
96108
}

‎src/components/MapPopup.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import { sparkline } from '@/utils/sparkline';
2626
import { getAuthState } from '@/services/auth-state';
2727
import { hasPremiumAccess } from '@/services/panel-gating';
2828
import { trackGateHit } from '@/services/analytics';
29+
import { renderPopupSourceLinks } from './map-popup-source-links';
2930

3031
// ── Static HS2 sector breakdown per chokepoint ────────────────────────────────
3132
// Based on IEA/UNCTAD estimated trade composition. Updated periodically.
@@ -1461,6 +1462,7 @@ export class MapPopup {
14611462
const actorsSection = event.actors?.length
14621463
? `<div class="popup-stat"><span class="stat-label">${t('popups.actors')}</span><span class="stat-value">${event.actors.map(a => escapeHtml(a)).join(', ')}</span></div>`
14631464
: '';
1465+
const sourceLinks = renderPopupSourceLinks(event.sourceUrls, { label: t('popups.source') });
14641466
const tagsSection = event.tags?.length
14651467
? `<div class="popup-tags">${event.tags.map(t => `<span class="popup-tag">${escapeHtml(t)}</span>`).join('')}</div>`
14661468
: '';
@@ -1491,6 +1493,7 @@ export class MapPopup {
14911493
${actorsSection}
14921494
</div>
14931495
${event.title ? `<p class="popup-description">${escapeHtml(event.title)}</p>` : ''}
1496+
${sourceLinks}
14941497
${tagsSection}
14951498
${relatedHotspots}
14961499
</div>
@@ -1518,7 +1521,11 @@ export class MapPopup {
15181521
const dateStr = event.time.toLocaleDateString(undefined, { month: 'short', day: 'numeric' });
15191522
const city = event.city ? escapeHtml(event.city) : '';
15201523
const title = event.title ? `: ${escapeHtml(event.title.slice(0, 40))}${event.title.length > 40 ? '...' : ''}` : '';
1521-
return `<li class="cluster-item ${sevClass}">${icon} ${dateStr}${city ? ` • ${city}` : ''}${title}</li>`;
1524+
const sourceUrl = event.sourceUrls?.find(url => sanitizeUrl(url));
1525+
const sourceLink = sourceUrl
1526+
? ` <a class="popup-link cluster-source-link" href="${sanitizeUrl(sourceUrl)}" target="_blank" rel="noopener noreferrer nofollow">${t('popups.source')} →</a>`
1527+
: '';
1528+
return `<li class="cluster-item ${sevClass}">${icon} ${dateStr}${city ? ` • ${city}` : ''}${title}${sourceLink}</li>`;
15221529
}).join('');
15231530

15241531
const renderedCount = Math.min(10, data.items.length);

‎src/components/map-popup-source-links.ts‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
import { escapeHtml, sanitizeUrl } from '@/utils/sanitize';
2+
3+
interface RenderSourceLinksOptions {
4+
limit?: number;
5+
label?: string;
6+
containerClass?: string;
7+
linkClass?: string;
8+
}
9+
10+
function extractSourceDomain(url: string): string {
11+
try {
12+
return new URL(url).hostname.replace('www.', '');
13+
} catch {
14+
return '';
15+
}
16+
}
17+
18+
export function renderPopupSourceLinks(sourceUrls: readonly string[] | undefined, options: RenderSourceLinksOptions = {}): string {
19+
const limit = options.limit ?? 3;
20+
const label = options.label ?? 'Source';
21+
const containerClass = options.containerClass ?? 'popup-source-links';
22+
const linkClass = options.linkClass ?? 'popup-link';
23+
const links: string[] = [];
24+
25+
for (const url of sourceUrls ?? []) {
26+
if (links.length >= limit) break;
27+
const safeUrl = sanitizeUrl(url);
28+
if (!safeUrl) continue;
29+
const domain = extractSourceDomain(url) || `${label} ${links.length + 1}`;
30+
links.push(
31+
`<a class="${escapeHtml(linkClass)}" href="${safeUrl}" target="_blank" rel="noopener noreferrer nofollow">${escapeHtml(domain)} →</a>`,
32+
);
33+
}
34+
35+
return links.length
36+
? `<div class="${escapeHtml(containerClass)}">${links.join('')}</div>`
37+
: '';
38+
}

‎src/generated/client/worldmonitor/unrest/v1/service_client.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ export interface UnrestEvent {
3737
tags: string[];
3838
actors: string[];
3939
confidence: ConfidenceLevel;
40+
sourceUrls: string[];
4041
}
4142

4243
export interface GeoCoordinates {

‎src/generated/server/worldmonitor/unrest/v1/service_server.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ export interface UnrestEvent {
3737
tags: string[];
3838
actors: string[];
3939
confidence: ConfidenceLevel;
40+
sourceUrls: string[];
4041
}
4142

4243
export interface GeoCoordinates {

0 commit comments

Comments
 (0)