Merge pull request #3 from hypercerts-org/fix/scorer-hard-signals-likely-test

fix(scorer): require hard signals for likely-test

Author: Kzoeps

README.md

@@ -8,9 +8,9 @@ This fork monitors both `app.certified.actor.profile` and `app.certified.actor.o
 
 | Label | Score | Meaning |
 |-------|-------|---------|
-| ⚠ Likely Test | 0-39 | Placeholder, junk, or obvious test data |
-| ● Standard | 40-69 | A valid organization record with the basics filled in |
-| ✦ High Quality | 70-100 | A complete organization record with several strong details |
+| ⚠ Likely Test | signal-based | Explicit test evidence, such as a configured test PDS or obvious placeholder identity data |
+| ● Standard | 0-69 without test signals | A non-test organization record that does not meet the high-quality threshold |
+| ✦ High Quality | 70+ without test signals | A complete organization record with several strong details |
 
 ## Quick Start
 
@@ -93,7 +93,7 @@ Scores `app.certified.actor.organization` records on 13 completeness signals (10
 
 Trusted PDS scoring uses the actor's resolved PDS host, not the profile website or organization URLs. Actor PDS lookup runs through the durable recompute queue; the first record from an uncached actor may be labeled by content score first, then corrected once the actor DID document is resolved.
 
-Test detection: authenticity checks catch common placeholder strings (`test`, `asdf`, `lorem ipsum`, etc.) and override the score to force ⚠ Likely Test. Operators can also set `TEST_PDS_HOSTS` to force actors from known development PDS hosts into ⚠ Likely Test.
+Test detection is intentionally conservative: hard `testSignals` such as configured `TEST_PDS_HOSTS`, obvious placeholder display names, placeholder domains, or `lorem ipsum` descriptions force ⚠ Likely Test. Softer data-quality issues become `validationNotes` for the dashboard but do not change the tier. Generic words like `test`, `testing`, or `tested` are allowed in profile descriptions, but still show as validation notes in short metadata fields such as organization type or URL labels.
 
 ### URL enrichment
 

src/app/docs/page.tsx

@@ -17,7 +17,7 @@ const SCORING_CRITERIA = [
   {
     label: 'Description',
     points: COMPLETENESS_WEIGHTS.description,
-    description: 'Awards points when the profile description is present and not obvious test data.',
+    description: 'Awards points when the profile description is present; generic words like test, testing, or tested are allowed in descriptions.',
   },
   {
     label: 'Organization type',
@@ -166,9 +166,9 @@ export default function DocsPage() {
           <li className='flex gap-3'>
             <span className='font-[family-name:var(--font-syne)] font-bold text-foreground shrink-0'>3.</span>
             <span>
-              The scoring engine checks an authenticity gate first. Records with authenticity failures are labeled
-              <ScoreBadge tier='likely-test' /> before completeness scoring begins. Validation notes are shown
-              separately and do not affect tiering. Passing records then receive the 100-point completeness score.
+              The scoring engine separates hard test signals from softer validation notes. Records with test signals
+              are labeled <ScoreBadge tier='likely-test' />. Records without test signals receive the 100-point
+              completeness score and are labeled standard or high-quality from that score.
             </span>
           </li>
           <li className='flex gap-3'>
@@ -186,11 +186,11 @@ export default function DocsPage() {
           Scoring criteria
         </h2>
         <p className='text-sm text-muted-foreground'>
-          Each passing organization record is evaluated on 13 completeness criteria for a maximum of 100 completeness
-          points, plus a configurable actor-PDS trust bonus. The rubric now gives less credit to basic identity fields
+          Each organization record is evaluated on 13 completeness criteria for a maximum of 100 completeness
+          points, plus a configurable actor-PDS trust bonus. The rubric gives less credit to basic identity fields
           and more weight to harder-to-fake credibility signals like website resolution, location, and visual/profile
-          completeness. The authenticity gate runs first, validation notes are informational only, and labels stay
-          attached to the organization record URI.
+          completeness. Hard test signals override the score to likely-test; validation notes are informational only,
+          and labels stay attached to the organization record URI.
         </p>
         <div className='border border-border rounded-lg bg-card overflow-hidden'>
           <table className='w-full text-sm'>
@@ -232,27 +232,29 @@ export default function DocsPage() {
         </div>
 
         <h3 className='font-[family-name:var(--font-syne)] text-sm font-bold mt-4'>
-          Authenticity gate
+          Test signals and validation notes
         </h3>
         <p className='text-sm text-muted-foreground'>
-          Matching any gate failure forces <ScoreBadge tier='likely-test' />. These are authenticity failures, not
-          validation notes, and there are no separate numeric deductions.
+          Matching a hard test signal forces <ScoreBadge tier='likely-test' />. Validation notes are softer data-quality
+          issues shown to operators, but they do not change the tier.
         </p>
         <div className='border border-border rounded-lg bg-card p-4 text-xs text-muted-foreground space-y-1'>
-          <p className='font-medium text-foreground text-xs'>Authenticity gate failure patterns</p>
+          <p className='font-medium text-foreground text-xs'>Hard test signal patterns</p>
           <ul className='space-y-1 pl-3'>
-            <li>• Common junk values: <span className='font-mono bg-secondary rounded px-1'>test</span>, <span className='font-mono bg-secondary rounded px-1'>asdf</span>, <span className='font-mono bg-secondary rounded px-1'>lorem ipsum</span>, <span className='font-mono bg-secondary rounded px-1'>placeholder</span>, <span className='font-mono bg-secondary rounded px-1'>delete me</span>, <span className='font-mono bg-secondary rounded px-1'>ignore</span>, <span className='font-mono bg-secondary rounded px-1'>todo</span>, <span className='font-mono bg-secondary rounded px-1'>foo</span>, <span className='font-mono bg-secondary rounded px-1'>bar</span>, <span className='font-mono bg-secondary rounded px-1'>abc</span>, <span className='font-mono bg-secondary rounded px-1'>wip</span>, <span className='font-mono bg-secondary rounded px-1'>sample</span>, and <span className='font-mono bg-secondary rounded px-1'>example</span>.</li>
-            <li>• Empty-style values: <span className='font-mono bg-secondary rounded px-1'>n/a</span>, <span className='font-mono bg-secondary rounded px-1'>none</span>, <span className='font-mono bg-secondary rounded px-1'>null</span>, <span className='font-mono bg-secondary rounded px-1'>undefined</span>, <span className='font-mono bg-secondary rounded px-1'>blank</span>, <span className='font-mono bg-secondary rounded px-1'>draft</span>, <span className='font-mono bg-secondary rounded px-1'>temp</span>, and <span className='font-mono bg-secondary rounded px-1'>tmp</span>.</li>
-            <li>• Display-name workflow/test terms such as <span className='font-mono bg-secondary rounded px-1'>demo</span>, <span className='font-mono bg-secondary rounded px-1'>dev</span>, <span className='font-mono bg-secondary rounded px-1'>staging</span>, <span className='font-mono bg-secondary rounded px-1'>qa</span>, <span className='font-mono bg-secondary rounded px-1'>e2e</span>, <span className='font-mono bg-secondary rounded px-1'>sandbox</span>, <span className='font-mono bg-secondary rounded px-1'>fixture</span>, <span className='font-mono bg-secondary rounded px-1'>seed data</span>, obvious glued test fixtures like <span className='font-mono bg-secondary rounded px-1'>tobytest</span>, generic names like <span className='font-mono bg-secondary rounded px-1'>org</span>/<span className='font-mono bg-secondary rounded px-1'>org 1</span>/<span className='font-mono bg-secondary rounded px-1'>first org</span>/<span className='font-mono bg-secondary rounded px-1'>new org</span>, <span className='font-mono bg-secondary rounded px-1'>new db</span>, <span className='font-mono bg-secondary rounded px-1'>published</span>, <span className='font-mono bg-secondary rounded px-1'>unpublished</span>, and <span className='font-mono bg-secondary rounded px-1'>changes requested</span>.</li>
-            <li>• Placeholder domains such as <span className='font-mono bg-secondary rounded px-1'>example.com</span>, <span className='font-mono bg-secondary rounded px-1'>example.org</span>, <span className='font-mono bg-secondary rounded px-1'>example.net</span>, and reserved <span className='font-mono bg-secondary rounded px-1'>.test</span>/<span className='font-mono bg-secondary rounded px-1'>.example</span>/<span className='font-mono bg-secondary rounded px-1'>.invalid</span> hostnames fail URL authenticity checks.</li>
-            <li>• Single-word greetings, repeated characters, repeated-character runs in display names, and numeric-only values are also treated as test data.</li>
+            <li>• Actors hosted on configured <span className='font-mono bg-secondary rounded px-1'>TEST_PDS_HOSTS</span>.</li>
+            <li>• Placeholder display names such as <span className='font-mono bg-secondary rounded px-1'>test org</span>, <span className='font-mono bg-secondary rounded px-1'>demo</span>, <span className='font-mono bg-secondary rounded px-1'>dev</span>, <span className='font-mono bg-secondary rounded px-1'>staging</span>, <span className='font-mono bg-secondary rounded px-1'>qa</span>, <span className='font-mono bg-secondary rounded px-1'>e2e</span>, <span className='font-mono bg-secondary rounded px-1'>sandbox</span>, <span className='font-mono bg-secondary rounded px-1'>fixture</span>, obvious glued test fixtures like <span className='font-mono bg-secondary rounded px-1'>tobytest</span>, generic names like <span className='font-mono bg-secondary rounded px-1'>org</span>/<span className='font-mono bg-secondary rounded px-1'>org 1</span>/<span className='font-mono bg-secondary rounded px-1'>first org</span>/<span className='font-mono bg-secondary rounded px-1'>new org</span>, and workflow states like <span className='font-mono bg-secondary rounded px-1'>published</span>, <span className='font-mono bg-secondary rounded px-1'>unpublished</span>, or <span className='font-mono bg-secondary rounded px-1'>changes requested</span>.</li>
+            <li>• Obvious junk descriptions such as <span className='font-mono bg-secondary rounded px-1'>lorem ipsum</span> or a description that is only placeholder text. Generic words like <span className='font-mono bg-secondary rounded px-1'>test</span>, <span className='font-mono bg-secondary rounded px-1'>testing</span>, or <span className='font-mono bg-secondary rounded px-1'>tested</span> are allowed in descriptions.</li>
+            <li>• Placeholder domains such as <span className='font-mono bg-secondary rounded px-1'>example.com</span>, <span className='font-mono bg-secondary rounded px-1'>example.org</span>, <span className='font-mono bg-secondary rounded px-1'>example.net</span>, and reserved <span className='font-mono bg-secondary rounded px-1'>.test</span>/<span className='font-mono bg-secondary rounded px-1'>.example</span>/<span className='font-mono bg-secondary rounded px-1'>.invalid</span> hostnames.</li>
+            <li>• Repeated-character runs in display names.</li>
           </ul>
         </div>
         <div className='border border-border rounded-lg bg-card p-4 text-xs text-muted-foreground space-y-1'>
           <p className='font-medium text-foreground text-xs'>Validation notes</p>
           <p>
-            These are informational only. They can appear when fallback handling keeps usable profile fields and
-            drops malformed optional fields, and they do not imply suspicious activity or affect tiering.
+            These are informational only. They can appear when fallback handling keeps usable profile fields,
+            optional fields are malformed, short metadata fields such as organization type or URL label contain
+            placeholder/test words, or the record has very little metadata. They do not imply suspicious activity
+            or affect tiering.
           </p>
         </div>
       </section>
@@ -262,8 +264,8 @@ export default function DocsPage() {
           Quality tiers
         </h2>
         <p className='text-sm text-muted-foreground'>
-          Scores map to three tiers. Authenticity gate failures override the numeric score and always produce a
-          <ScoreBadge tier='likely-test' /> label. Validation notes do not change the tier.
+          Scores map to standard or high-quality unless a hard test signal is present. Test signals override the
+          numeric score and produce a <ScoreBadge tier='likely-test' /> label. Validation notes do not change the tier.
         </p>
         <div className='grid gap-3 sm:grid-cols-2'>
           {[
@@ -274,13 +276,13 @@ export default function DocsPage() {
             },
             {
               tier: 'standard' as const,
-              range: '40 – 69',
-              detail: 'Decent merged record with some useful metadata, but not full rubric coverage.',
+              range: '0 – 69 without test signals',
+              detail: 'Non-test merged record with anything below the high-quality threshold.',
             },
             {
               tier: 'likely-test' as const,
-              range: '0 – 39 or gate failures',
-              detail: 'Contains authenticity gate failures, or falls below the standard threshold.',
+              range: 'test signals only',
+              detail: 'Contains hard test evidence, such as a configured test PDS, placeholder display name, or placeholder domain.',
             },
           ].map(({ tier, range, detail }) => (
             <div key={tier} className='border border-border rounded-lg bg-card p-4 space-y-2'>

src/components/ScoreBreakdown.tsx

@@ -47,7 +47,7 @@ export function ScoreBreakdown({ breakdown, validationNotes = [], testSignals }:
   return (
     <div>
       <p className='mb-2 text-[11px] text-muted-foreground'>
-        Authenticity gate failures are checked before completeness scoring.
+        Hard test signals override tiering; validation notes are informational only.
       </p>
 
       {CRITERIA.map(({ label, field, max }) => {
@@ -137,7 +137,7 @@ export function ScoreBreakdown({ breakdown, validationNotes = [], testSignals }:
               <circle cx='6' cy='8.5' r='0.5' fill='currentColor' className='text-rose-700 dark:text-rose-400' />
             </svg>
             <span className='text-[11px] font-medium text-rose-700 dark:text-rose-400'>
-              Authenticity gate failed
+              Hard test signal
             </span>
           </div>
           {testSignals.map((signal, i) => (

src/lib/constants.ts

@@ -1,8 +1,7 @@
 import type { LabelDefinition } from './types'
 
 type RuntimeQualityTier = 'likely-test' | 'standard' | 'high-quality'
-
-export const AUTHENTICITY_FAILURE_TIER: RuntimeQualityTier = 'likely-test'
+type ScoreBasedQualityTier = Exclude<RuntimeQualityTier, 'likely-test'>
 
 // Shared time unit for score boundaries.
 export const MS_PER_DAY = 24 * 60 * 60 * 1000
@@ -39,11 +38,12 @@ export const COMPLETENESS_WEIGHTS = {
   banner: 10,
 } as const
 
-// Final score bands are intentionally coarse. High-quality is open-ended
-// because configured bonuses can raise the final score above 100 completeness points.
-export const SCORE_THRESHOLDS: Record<RuntimeQualityTier, { min: number; max: number }> = {
-  'likely-test': { min: 0, max: 39 },
-  standard: { min: 40, max: 69 },
+// Final score bands are intentionally coarse. Likely-test is signal-based:
+// records without hard test evidence default to standard until they meet the
+// high-quality threshold. High-quality is open-ended because configured bonuses
+// can raise the final score above 100 completeness points.
+export const SCORE_THRESHOLDS: Record<ScoreBasedQualityTier, { min: number; max: number }> = {
+  standard: { min: 0, max: 69 },
   'high-quality': { min: 70, max: Number.POSITIVE_INFINITY },
 }
 
@@ -65,10 +65,16 @@ export const DEFAULT_TRUSTED_PDS_HOSTS = ['certified.one', 'gainforest.id'] as c
 /** Default score points added when an actor is hosted on a trusted PDS. */
 export const DEFAULT_TRUSTED_PDS_BONUS = 10
 
-export const TEST_PATTERNS: RegExp[] = [
-  // Word-boundary "test" — catches "Another Test", "Test Contributors", "This is testing", "test 123"
+/**
+ * Test-word patterns are hard evidence in short identity fields like display
+ * names, but too common for free-form descriptions.
+ */
+export const TEST_WORD_PATTERNS: RegExp[] = [
   /\btest(ing|ed|er|s)?\b/i,
+]
 
+/** Obvious placeholder strings that can be checked outside descriptions. */
+export const TEST_PATTERNS: RegExp[] = [
   // Common junk prefixes and phrases.
   /^asdf/i, /\blorem ipsum\b/i, /^placeholder/i, /^delete me/i, /^ignore/i, /^zzz/i,
 
@@ -102,11 +108,37 @@ export const AUTHENTICITY_TEXT_PATTERNS: RegExp[] = [
   /^unlisted$/i,
 ]
 
+/** Placeholder patterns for short metadata fields, where test words are meaningful signals. */
+export const SHORT_FIELD_AUTHENTICITY_TEXT_PATTERNS: RegExp[] = [
+  ...TEST_WORD_PATTERNS,
+  ...AUTHENTICITY_TEXT_PATTERNS,
+]
+
+/**
+ * Placeholder patterns for descriptions. Generic test words are intentionally
+ * excluded because real organization descriptions often mention tests, testing,
+ * or tested methods.
+ */
+export const DESCRIPTION_AUTHENTICITY_TEXT_PATTERNS: RegExp[] = [
+  /\blorem ipsum\b/i,
+  /^asdf/i,
+  /^placeholder$/i,
+  /^delete me$/i,
+  /^ignore$/i,
+  /^todo$/i,
+  /^n\/a$/i,
+  /^none$/i,
+  /^null$/i,
+  /^undefined$/i,
+  /^blank$/i,
+]
+
 /**
  * Extra authenticity patterns for display names, where workflow/test labels are
  * much stronger evidence than the same words appearing in longer descriptions.
  */
 export const DISPLAY_NAME_AUTHENTICITY_TEXT_PATTERNS: RegExp[] = [
+  ...TEST_WORD_PATTERNS,
   ...AUTHENTICITY_TEXT_PATTERNS,
   /(?:^|[^\p{Letter}\p{Number}])(?:demo|dev|staging|qa|e2e|sandbox|fixture)(?:$|[^\p{Letter}\p{Number}])/iu,
   /^tobytest\d*$/i,
@@ -122,7 +154,7 @@ export const DISPLAY_NAME_AUTHENTICITY_TEXT_PATTERNS: RegExp[] = [
 /** Reserved example domains that should never count as organization evidence. */
 export const PLACEHOLDER_DOMAINS = ['example.com', 'example.net', 'example.org'] as const
 
-/** Reserved non-production TLDs that should fail the authenticity gate. */
+/** Reserved non-production TLDs that should produce a hard test signal. */
 export const PLACEHOLDER_TLDS = ['example', 'invalid', 'test'] as const
 
 export const LABEL_LIMIT = 1

src/lib/hf-classifier.ts

@@ -2,6 +2,7 @@ import { HfInference } from '@huggingface/inference'
 import * as config from './config'
 import { HF_POSITIVE_LABEL, updateActivityHfFields, getActivityByDidRkey, getHfClassifiedNonFlagged } from './db'
 import { updateActivity } from './db'
+import { tierForScore } from './scorer'
 
 export interface ContentClassification {
   label: string
@@ -94,12 +95,20 @@ function reclassifyWithHfSignal(did: string, rkey: string, classification: Conte
     ? [...existingSignals, signal]
     : existingSignals
 
+  const tier = tierForScore(row.score, updatedSignals)
+
   updateActivity(did, rkey, {
     score: row.score,
-    tier: row.tier,
+    tier,
     breakdown: row.breakdown,
     testSignals: JSON.stringify(updatedSignals),
   })
+
+  if (tier !== row.tier && _onReclassify) {
+    void _onReclassify(row.uri, tier).catch(err => {
+      console.warn('[hf-classifier] label update failed:', err instanceof Error ? err.message : err)
+    })
+  }
 }
 
 let hf: HfInference | null = null