diff --git a/.env.example b/.env.example index 445ed41..5ace46c 100644 --- a/.env.example +++ b/.env.example @@ -44,6 +44,11 @@ TAP_ADMIN_PASSWORD= # When set, URL enrichment is deferred until each actor's PDS is known and skipped for matching test PDS hosts. TEST_PDS_HOSTS= +# Comma-separated PDS hosts whose actors receive a score bonus after DID resolution. +# This checks the actor PDS, not the profile website or organization URLs. +TRUSTED_PDS_HOSTS=certified.one,gainforest.id +TRUSTED_PDS_BONUS=10 + # --- Railway deployment --- # Mount a persistent volume at /app/data and set: # ACTIVITY_DB_PATH=/app/data/activity-log.db diff --git a/README.md b/README.md index 7bcb2ea..a37980e 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ The labeler auto-detects the PDS for non-bsky.social accounts via DID document r ## Scoring -Scores `app.certified.actor.organization` records on 13 completeness signals (100 points total): +Scores `app.certified.actor.organization` records on 13 completeness signals (100 points total), plus a configurable actor-PDS trust bonus: | Signal | Max Points | What it checks | |--------|-----------|----------------| @@ -89,8 +89,11 @@ Scores `app.certified.actor.organization` records on 13 completeness signals (10 | Founded Date Age | 5 | Founded date is at least one year old | | Avatar | 10 | Has an avatar image | | Banner | 10 | Has a banner image | +| Trusted PDS Bonus | configurable, default 10 | Actor DID resolves to a PDS host in `TRUSTED_PDS_HOSTS` (`certified.one` and `gainforest.id` by default) | -Test detection: authenticity checks catch common placeholder strings (`test`, `asdf`, `lorem ipsum`, etc.) and override the score to force ⚠ Likely Test. Operators can also set `TEST_PDS_HOSTS` to force actors from known development PDS hosts into ⚠ Likely Test. Actor PDS lookup runs through the durable recompute queue; the first record from an uncached actor may be labeled by content score first, then corrected once the actor DID document is resolved. +Trusted PDS scoring uses the actor's resolved PDS host, not the profile website or organization URLs. Actor PDS lookup runs through the durable recompute queue; the first record from an uncached actor may be labeled by content score first, then corrected once the actor DID document is resolved. + +Test detection: authenticity checks catch common placeholder strings (`test`, `asdf`, `lorem ipsum`, etc.) and override the score to force ⚠ Likely Test. Operators can also set `TEST_PDS_HOSTS` to force actors from known development PDS hosts into ⚠ Likely Test. ### URL enrichment @@ -129,6 +132,8 @@ When `TEST_PDS_HOSTS` is configured, URL enrichment is PDS-aware: it defers URL | `TAP_URL` | required | URL of the separate Tap service (no localhost default) | | `TAP_ADMIN_PASSWORD` | empty | App-side password for Tap admin auth; must match the Tap service when auth is enabled | | `TEST_PDS_HOSTS` | empty | Comma-separated PDS hosts whose actors should always be labeled `likely-test`; when set, URL enrichment waits for actor PDS resolution and skips matching test PDS hosts | +| `TRUSTED_PDS_HOSTS` | `certified.one,gainforest.id` | Comma-separated PDS hosts whose actors receive the trusted-PDS score bonus | +| `TRUSTED_PDS_BONUS` | `10` | Score points added when an actor's resolved PDS host matches `TRUSTED_PDS_HOSTS`; set to `0` to disable the bonus | | `ACTIVITY_DB_PATH` | `activity-log.db` | Activity log database path | | `URL_ENRICHMENT_ENABLED` | `true` | Enables async URL checks through the detachable `url_checks` cache | | `URL_CHECK_TIMEOUT_MS` | `4000` | Timeout for one URL resolution attempt | diff --git a/src/app/docs/page.tsx b/src/app/docs/page.tsx index 26d74d4..8c63cb2 100644 --- a/src/app/docs/page.tsx +++ b/src/app/docs/page.tsx @@ -1,5 +1,5 @@ import { ScoreBadge } from '@/components/ScoreBadge' -import { COMPLETENESS_WEIGHTS } from '@/lib/constants' +import { COMPLETENESS_WEIGHTS, DEFAULT_TRUSTED_PDS_BONUS, DEFAULT_TRUSTED_PDS_HOSTS } from '@/lib/constants' const getLabelerBaseUrl = () => (process.env.NEXT_PUBLIC_LABELER_ENDPOINT ?? '').replace(/\/$/, '') @@ -74,6 +74,11 @@ const SCORING_CRITERIA = [ points: COMPLETENESS_WEIGHTS.banner, description: 'Awards meaningful points when the profile has a banner.', }, + { + label: 'Trusted PDS bonus', + points: DEFAULT_TRUSTED_PDS_BONUS, + description: `Awards the configured trusted-PDS bonus when the actor DID resolves to ${DEFAULT_TRUSTED_PDS_HOSTS.join(' or ')} by default.`, + }, ] const API_ENDPOINTS = [ @@ -181,10 +186,11 @@ export default function DocsPage() { Scoring criteria

- Each passing organization record is evaluated on 13 completeness criteria for a maximum of 100 points. The - rubric now gives less credit to basic identity fields and more weight to harder-to-fake credibility signals - like website resolution, location, and visual/profile completeness. The authenticity gate runs first, - validation notes are informational only, and labels stay attached to the organization record URI. + Each passing organization record is evaluated on 13 completeness criteria for a maximum of 100 completeness + points, plus a configurable actor-PDS trust bonus. The rubric now gives less credit to basic identity fields + and more weight to harder-to-fake credibility signals like website resolution, location, and visual/profile + completeness. The authenticity gate runs first, validation notes are informational only, and labels stay + attached to the organization record URI.

@@ -218,7 +224,7 @@ export default function DocsPage() { @@ -263,7 +269,7 @@ export default function DocsPage() { {[ { tier: 'high-quality' as const, - range: '70 – 100', + range: '70+', detail: 'Strong merged profile + organization record with broad completeness across the rubric.', }, { diff --git a/src/components/ScoreBreakdown.tsx b/src/components/ScoreBreakdown.tsx index 764b43e..605a061 100644 --- a/src/components/ScoreBreakdown.tsx +++ b/src/components/ScoreBreakdown.tsx @@ -1,6 +1,6 @@ 'use client' -import { COMPLETENESS_WEIGHTS } from '@/lib/constants' +import { COMPLETENESS_WEIGHTS, DEFAULT_TRUSTED_PDS_BONUS } from '@/lib/constants' import type { ScoreBreakdown as ScoreBreakdownType } from '@/lib/types' interface ScoreBreakdownProps { @@ -9,7 +9,13 @@ interface ScoreBreakdownProps { testSignals: string[] } -const CRITERIA: Array<{ label: string; field: keyof ScoreBreakdownType; max: number }> = [ +type Criterion = { + label: string + field: keyof ScoreBreakdownType + max: number | ((breakdown: ScoreBreakdownType) => number) +} + +const CRITERIA: Criterion[] = [ { label: 'Display name', field: 'displayName', max: COMPLETENESS_WEIGHTS.displayName }, { label: 'Description', field: 'description', max: COMPLETENESS_WEIGHTS.description }, { label: 'Organization type', field: 'organizationType', max: COMPLETENESS_WEIGHTS.organizationType }, @@ -23,6 +29,11 @@ const CRITERIA: Array<{ label: string; field: keyof ScoreBreakdownType; max: num { label: 'Founded date age bonus', field: 'foundedDateAge', max: COMPLETENESS_WEIGHTS.foundedDateAge }, { label: 'Avatar', field: 'avatar', max: COMPLETENESS_WEIGHTS.avatar }, { label: 'Banner', field: 'banner', max: COMPLETENESS_WEIGHTS.banner }, + { + label: 'Trusted PDS bonus', + field: 'trustedPds', + max: (breakdown) => (breakdown.trustedPds ?? 0) > 0 ? breakdown.trustedPds : DEFAULT_TRUSTED_PDS_BONUS, + }, ] function getBarColor(ratio: number): string { @@ -41,8 +52,9 @@ export function ScoreBreakdown({ breakdown, validationNotes = [], testSignals }: {CRITERIA.map(({ label, field, max }) => { const value = breakdown[field] ?? 0 - const ratio = max > 0 ? value / max : 0 - const widthPct = Math.round(ratio * 100) + const maxValue = typeof max === 'function' ? max(breakdown) : max + const ratio = maxValue > 0 ? value / maxValue : 0 + const widthPct = Math.max(0, Math.min(Math.round(ratio * 100), 100)) const barColor = getBarColor(ratio) return ( @@ -55,7 +67,7 @@ export function ScoreBreakdown({ breakdown, validationNotes = [], testSignals }: /> - {value}/{max} + {value}/{maxValue} ) diff --git a/src/components/ScoreCard.tsx b/src/components/ScoreCard.tsx index 643d73f..d31c6c1 100644 --- a/src/components/ScoreCard.tsx +++ b/src/components/ScoreCard.tsx @@ -52,6 +52,7 @@ export function ScoreCard({ entry, defaultExpanded = false }: ScoreCardProps) { const shortDid = entry.did.replace('did:plc:', '').slice(0, 12) + '…' const recordUrl = buildRecordUrl(entry.did, ORGANIZATION_COLLECTION, entry.rkey) const barColor = TIER_BAR_COLORS[entry.tier] + const scoreWidth = Math.max(0, Math.min(entry.score, 100)) const relativeTime = getRelativeTime(entry.labeledAt) return ( @@ -99,7 +100,7 @@ export function ScoreCard({ entry, defaultExpanded = false }: ScoreCardProps) { ) : (
)}
diff --git a/src/labeler/tap-consumer.ts b/src/labeler/tap-consumer.ts index f292f00..2c9b6be 100644 --- a/src/labeler/tap-consumer.ts +++ b/src/labeler/tap-consumer.ts @@ -1,6 +1,6 @@ import { Tap, SimpleIndexer } from '@atproto/tap' import type { TapChannel } from '@atproto/tap' -import { TAP_URL, TAP_ADMIN_PASSWORD, ACTIVITY_COLLECTION } from '../lib/config' +import { TAP_URL, TAP_ADMIN_PASSWORD, ACTIVITY_COLLECTION, TRUSTED_PDS_BONUS, TRUSTED_PDS_HOSTS } from '../lib/config' import { cachedActorPdsHostForScoring, ACTOR_PDS_CACHE_TTL_MS } from '../lib/actor-pds-policy' import { applyPdsTestOverride } from '../lib/pds-test-override' import { normalizePdsHostFromUrl } from '../lib/pds-utils' @@ -213,6 +213,9 @@ export async function recomputeLabeledOrganizationRow(did: string): Promise 0 } -/** Enqueues actor PDS resolution when TEST_PDS_HOSTS is configured. */ +/** Returns true when trusted-PDS scoring can affect the final score. */ +export function trustedPdsBonusEnabled(): boolean { + return TRUSTED_PDS_HOSTS.length > 0 && TRUSTED_PDS_BONUS > 0 +} + +/** Returns true when actor PDS lookup is needed for any scoring policy. */ +export function actorPdsResolutionEnabled(): boolean { + return testPdsDetectionEnabled() || trustedPdsBonusEnabled() +} + +/** Enqueues actor PDS resolution when any actor-PDS scoring policy is configured. */ export function enqueueActorPdsResolution(did: string, reason: string, delayMs = 0): void { - if (!testPdsDetectionEnabled()) return + if (!actorPdsResolutionEnabled()) return recordActorPdsPending(did, ACTOR_PDS_PENDING_TTL_MS) enqueueRecomputeJob('resolve-actor-pds', did, { @@ -31,10 +41,11 @@ export function enqueueActorPdsResolution(did: string, reason: string, delayMs = /** * Returns the best cached actor PDS host for scoring. Stale hosts are still used - * for the current score, but a refresh is queued so later recomputes can correct labels. + * for the current score, but a refresh is queued so later recomputes can correct + * trusted-PDS bonuses or test-PDS labels. */ export function cachedActorPdsHostForScoring(did: string): string | null { - if (!testPdsDetectionEnabled()) return null + if (!actorPdsResolutionEnabled()) return null const cache = getActorPdsCache(did) if (!cache || isActorPdsCacheStale(cache)) { diff --git a/src/lib/config.ts b/src/lib/config.ts index 32c25a0..bec6751 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -1,4 +1,5 @@ -import { parseTestPdsHosts } from './pds-utils' +import { DEFAULT_TRUSTED_PDS_BONUS, DEFAULT_TRUSTED_PDS_HOSTS } from './constants' +import { parsePdsHosts, parseTestPdsHosts } from './pds-utils' export const DID = process.env.DID ?? '' export const SIGNING_KEY = process.env.SIGNING_KEY ?? '' @@ -22,6 +23,10 @@ export const HF_MODEL = 'facebook/bart-large-mnli' export const HYPERSCAN_RECORD_URL_BASE = process.env.HYPERSCAN_RECORD_URL_BASE ?? 'https://hyperscan.dev/data' /** Comma-separated PDS hosts whose actors should always be labeled likely-test. */ export const TEST_PDS_HOSTS = parseTestPdsHosts(process.env.TEST_PDS_HOSTS ?? '') +/** PDS hosts whose actors receive the configured trusted-operator score bonus. */ +export const TRUSTED_PDS_HOSTS = parsePdsHosts(process.env.TRUSTED_PDS_HOSTS ?? DEFAULT_TRUSTED_PDS_HOSTS.join(',')) +/** Score points added when an actor's resolved PDS host matches TRUSTED_PDS_HOSTS. */ +export const TRUSTED_PDS_BONUS = nonNegativeIntegerEnv('TRUSTED_PDS_BONUS', DEFAULT_TRUSTED_PDS_BONUS) function integerEnv(name: string, fallback: number): number { const value = process.env[name] @@ -31,6 +36,14 @@ function integerEnv(name: string, fallback: number): number { return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback } +function nonNegativeIntegerEnv(name: string, fallback: number): number { + const value = process.env[name] + if (value === undefined || value.trim() === '') return fallback + + const parsed = Number(value) + return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback +} + /** Enables the detachable URL enrichment worker; set to false to keep scoring fully provisional. */ export const URL_ENRICHMENT_ENABLED = process.env.URL_ENRICHMENT_ENABLED !== 'false' /** Poll interval for checking one due URL cache row. */ diff --git a/src/lib/constants.ts b/src/lib/constants.ts index e348ad0..5a25243 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -39,11 +39,12 @@ export const COMPLETENESS_WEIGHTS = { banner: 10, } as const -// Final score bands are intentionally coarse and should stay readable here. +// Final score bands are intentionally coarse. High-quality is open-ended +// because configured bonuses can raise the final score above 100 completeness points. export const SCORE_THRESHOLDS: Record = { 'likely-test': { min: 0, max: 39 }, standard: { min: 40, max: 69 }, - 'high-quality': { min: 70, max: 100 }, + 'high-quality': { min: 70, max: Number.POSITIVE_INFINITY }, } // Founded-date age scoring is binary: the 5-point bonus applies once the @@ -55,9 +56,15 @@ export const FOUNDED_DATE_AGE_BUCKETS = { }, } as const -// Declared raw total for the completeness model. +// Declared raw total for the completeness model before any configured bonuses. export const COMPLETENESS_WEIGHT_TOTAL = 100 +/** Default PDS hosts that receive the trusted-operator score bonus. */ +export const DEFAULT_TRUSTED_PDS_HOSTS = ['certified.one', 'gainforest.id'] as const + +/** Default score points added when an actor is hosted on a trusted PDS. */ +export const DEFAULT_TRUSTED_PDS_BONUS = 10 + export const TEST_PATTERNS: RegExp[] = [ // Word-boundary "test" — catches "Another Test", "Test Contributors", "This is testing", "test 123" /\btest(ing|ed|er|s)?\b/i, diff --git a/src/lib/pds-utils.ts b/src/lib/pds-utils.ts index d1b15a3..852a3fb 100644 --- a/src/lib/pds-utils.ts +++ b/src/lib/pds-utils.ts @@ -21,8 +21,8 @@ export function normalizePdsHostFromUrl(pdsUrl: string): string | null { return normalizePdsHost(pdsUrl) } -/** Parses a comma-separated TEST_PDS_HOSTS value into unique normalized hosts. */ -export function parseTestPdsHosts(value: string): string[] { +/** Parses comma-separated PDS hosts or endpoint URLs into unique normalized hostnames. */ +export function parsePdsHosts(value: string): string[] { const hosts = value .split(',') .map(host => normalizePdsHost(host)) @@ -31,11 +31,24 @@ export function parseTestPdsHosts(value: string): string[] { return [...new Set(hosts)] } +/** Parses TEST_PDS_HOSTS into normalized exact-match hostnames. */ +export function parseTestPdsHosts(value: string): string[] { + return parsePdsHosts(value) +} + +/** Checks whether a PDS host exactly matches one of the configured hosts. */ +export function isConfiguredPdsHost( + pdsHost: string | null | undefined, + configuredHosts: readonly string[], +): boolean { + const normalized = pdsHost ? normalizePdsHost(pdsHost) : null + return normalized !== null && configuredHosts.includes(normalized) +} + /** Checks whether a PDS host exactly matches one of the configured test hosts. */ export function isConfiguredTestPdsHost( pdsHost: string | null | undefined, testPdsHosts: readonly string[], ): boolean { - const normalized = pdsHost ? normalizePdsHost(pdsHost) : null - return normalized !== null && testPdsHosts.includes(normalized) + return isConfiguredPdsHost(pdsHost, testPdsHosts) } diff --git a/src/lib/scorer.ts b/src/lib/scorer.ts index 9806922..0dac333 100644 --- a/src/lib/scorer.ts +++ b/src/lib/scorer.ts @@ -3,6 +3,7 @@ import type { MergedScoringInput, UrlResolutionMap, UrlResolutionState } from '. import { AUTHENTICITY_FAILURE_TIER, COMPLETENESS_WEIGHTS, FOUNDED_DATE_AGE_BUCKETS, SCORE_THRESHOLDS } from './constants' import { evaluateMergedActorAuthenticity } from './scoring-authenticity' import { validateOrganizationLocationRef } from './location-utils' +import { isConfiguredPdsHost } from './pds-utils' import { displayNameMatchesWebsiteDomain, normalizePublicWebsiteUrl } from './website-utils' export type ScoreResultWithValidationNotes = ScoreResult & { @@ -23,6 +24,7 @@ const ZERO_BREAKDOWN: ScoreBreakdown = { foundedDateAge: 0, avatar: 0, banner: 0, + trustedPds: 0, } // Organization records do not currently cap the number of URL refs in the @@ -128,6 +130,19 @@ function scoreBanner(hasBanner: boolean): number { return hasBanner ? COMPLETENESS_WEIGHTS.banner : 0 } +/** + * Returns the trusted-PDS score bonus for an actor's resolved PDS host. + * This must be called with the actor PDS host from DID resolution, not website domains. + */ +export function scoreTrustedPdsBonus( + actorPdsHost: string | null | undefined, + trustedPdsHosts: readonly string[] = [], + trustedPdsBonus = 0, +): number { + if (trustedPdsBonus <= 0) return 0 + return isConfiguredPdsHost(actorPdsHost, trustedPdsHosts) ? trustedPdsBonus : 0 +} + export async function scoreActivity(record: MergedScoringInput): Promise { const authenticity = evaluateMergedActorAuthenticity(record) const validationNotes = record.validationNotes ?? [] @@ -157,6 +172,7 @@ export async function scoreActivity(record: MergedScoringInput): Promise sum + value, 0) diff --git a/src/lib/scoring-input.ts b/src/lib/scoring-input.ts index c6eeaff..44ff572 100644 --- a/src/lib/scoring-input.ts +++ b/src/lib/scoring-input.ts @@ -27,6 +27,12 @@ export interface MergedScoringInput { organizationType: string[] urls: MergedOrganizationUrlItem[] urlResolution?: UrlResolutionMap + /** Normalized actor PDS host from DID resolution, when the durable cache knows it. */ + actorPdsHost?: string | null + /** Exact-match PDS hosts that should receive the trusted-PDS score bonus. */ + trustedPdsHosts?: readonly string[] + /** Number of points to add when actorPdsHost matches trustedPdsHosts. */ + trustedPdsBonus?: number location: OrganizationRecord['location'] | null foundedDate: string | null } diff --git a/src/lib/types.ts b/src/lib/types.ts index 26fdc5e..33287c6 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -89,7 +89,7 @@ export type LegacyLabelTier = 'pending' | 'draft' export type LabelTier = RuntimeLabelTier | LegacyLabelTier export interface ScoreResult { - totalScore: number // normalized 0-100 completeness score + totalScore: number // final score: 0-100 completeness plus configured bonuses tier: LabelTier breakdown: ScoreBreakdown testSignals: string[] // reasons flagged as test data @@ -109,6 +109,7 @@ export interface ScoreBreakdown { foundedDateAge: number avatar: number banner: number + trustedPds: number // bonus for actors hosted on configured trusted PDS hosts } export interface LabelDefinition { diff --git a/tests/pds-test-override.test.ts b/tests/pds-test-override.test.ts index 56f75e2..ec6a672 100644 --- a/tests/pds-test-override.test.ts +++ b/tests/pds-test-override.test.ts @@ -21,6 +21,7 @@ function scoreResult(): ScoreResult { foundedDateAge: 0, avatar: 5, banner: 0, + trustedPds: 0, }, testSignals: [], } diff --git a/tests/trusted-pds-bonus.test.ts b/tests/trusted-pds-bonus.test.ts new file mode 100644 index 0000000..832cbf8 --- /dev/null +++ b/tests/trusted-pds-bonus.test.ts @@ -0,0 +1,49 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import type { MergedScoringInput } from '../src/lib/scoring-input' + +const { scoreActivity, scoreTrustedPdsBonus } = await import('../src/lib/scorer') + +function record(overrides: Partial = {}): MergedScoringInput { + return { + did: 'did:plc:trustedpdsactor', + displayName: 'Forest Recovery Collective', + displayNameSource: 'profile', + profileDisplayName: 'Forest Recovery Collective', + profileDescription: 'A community organization restoring native forest habitats.', + profileWebsite: null, + validationNotes: [], + hasAvatar: true, + hasBanner: false, + organizationType: ['nonprofit'], + urls: [{ url: 'https://forest-recovery.example.coop', label: 'Website' }], + trustedPdsHosts: ['certified.one', 'gainforest.id'], + trustedPdsBonus: 10, + location: null, + foundedDate: null, + ...overrides, + } +} + +test('trusted actor PDS hosts receive the configured score bonus', async () => { + const baseline = await scoreActivity(record({ actorPdsHost: 'bsky.social' })) + const trusted = await scoreActivity(record({ actorPdsHost: 'https://CERTIFIED.ONE/' })) + + assert.equal(trusted.breakdown.trustedPds, 10) + assert.equal(trusted.totalScore, baseline.totalScore + 10) +}) + +test('trusted PDS bonus checks actor PDS host rather than website domain', async () => { + const result = await scoreActivity(record({ + actorPdsHost: null, + profileWebsite: 'https://certified.one', + })) + + assert.equal(result.breakdown.trustedPds, 0) +}) + +test('trusted PDS host matching is exact after normalization', () => { + assert.equal(scoreTrustedPdsBonus('https://GAINforest.ID/', ['certified.one', 'gainforest.id'], 10), 10) + assert.equal(scoreTrustedPdsBonus('not-gainforest.id', ['certified.one', 'gainforest.id'], 10), 0) + assert.equal(scoreTrustedPdsBonus('gainforest.id.evil.example', ['certified.one', 'gainforest.id'], 10), 0) +})
Total - 100 + 100 + trusted PDS bonus