diff --git a/.env.example b/.env.example index 5ace46c..2917a0f 100644 --- a/.env.example +++ b/.env.example @@ -1,12 +1,12 @@ # DID of your labeler account (e.g. did:plc:xxxx) DID=did:plc:your-labeler-did -# Signing key for the labeler (hex-encoded private key) +# Private signing key for the labeler SIGNING_KEY=your-signing-key -# Bluesky account credentials for the labeler bot -BSKY_IDENTIFIER=your-handle.bsky.social -BSKY_PASSWORD=your-app-password +# Labeler account credentials +LABELER_IDENTIFIER=orglabeler.certified.one +LABELER_PASSWORD=your-app-password # Public HTTP port for Caddy. Hosted platforms usually set PORT automatically. PORT=8080 @@ -29,12 +29,12 @@ ACTIVITY_DB_PATH=activity-log.db # Path to the labels database (used by @skyware/labeler LabelerServer) LABELS_DB_PATH=labels.db -# Public URL of the Caddy-fronted app/labeler endpoint (e.g. https://labeler.yourdomain.com) +# Public URL of the Caddy-fronted app/labeler endpoint (e.g. https://orglabeler.hypercerts.dev) # /xrpc/* is routed to the labeler and dashboard/API routes are routed to Next.js. -NEXT_PUBLIC_LABELER_ENDPOINT=https://labeler.yourdomain.com +NEXT_PUBLIC_LABELER_ENDPOINT=https://orglabeler.hypercerts.dev # PDS URL for the labeler account (auto-detected during setup, override if needed) -PDS_URL=https://climateai.org +PDS_URL=https://certified.one # External Tap service URL. Tap runs as a separate service, not inside this app container. TAP_URL=https://your-tap-service.example.com diff --git a/AGENTS.md b/AGENTS.md index 378a9e6..985bea0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,14 +7,14 @@ This is the **Certified Organization Labeler** fork — an AT Protocol labeler f ``` src/ ├── app/ # Next.js App Router (dashboard) -│ ├── api/ # API routes (read from activity-log.db) +│ ├── api/ # API routes (read from configured ACTIVITY_DB_PATH) │ ├── feed/ # Feed page with tier filtering │ ├── globals.css # Tailwind v4 design system (OKLCH) │ ├── layout.tsx # Root layout with fonts + theme │ └── page.tsx # Dashboard page -├── components/ # React components (all "use client") +├── components/ # React UI components (client components only where interactive) ├── labeler/ # Standalone labeler process (NO React/Next.js imports) -│ ├── start.ts # Entry point: spawns Tap sidecar + LabelerServer +│ ├── start.ts # Entry point: LabelerServer + workers + external Tap connection │ ├── tap-consumer.ts # Tap client, SimpleIndexer record handler │ ├── server.ts # Label application logic │ ├── setup.ts # Account setup (PDS-aware) @@ -34,26 +34,31 @@ src/ 1. **src/labeler/ must NEVER import from React or Next.js** — it runs as a standalone Node.js process 2. **src/lib/ is shared** — importable by both labeler and Next.js, but must not import from src/labeler/ 3. **src/components/ and src/app/ are Next.js only** — they can import from src/lib/ but never from src/labeler/ -4. **Two databases**: labels.db (AT Proto, managed by @skyware/labeler) and activity-log.db (dashboard, managed by src/lib/db.ts) +4. **Two app databases**: `LABELS_DB_PATH` (AT Proto, managed by @skyware/labeler) and `ACTIVITY_DB_PATH` (dashboard, managed by src/lib/db.ts) 5. **Design system**: Tailwind CSS v4, OKLCH colors (hue 260), Syne display font, no component libraries 6. **Code style**: no semicolons, single quotes, 2-space indent ## Development ```bash -npm run dev:all # Start dashboard + labeler together -npm run build # Verify Next.js build passes -npm run reset # Clear databases for fresh start +npm run dev:service # Start dashboard + labeler together +npm run build # Verify Next.js build passes +npm run test # Run Node test suite +npm run reset # Clear local databases for fresh start ``` -## Issue Tracking +## Documentation Maintenance -This project uses **hb** (heartbeads) for issue tracking. +When behavior, configuration, deployment, scripts, or operator workflow changes, check whether the relevant docs need updates in the same change. -```bash -hb ready # Find available work -hb show # View issue details -hb update --status in_progress # Claim work -hb close # Complete work -hb sync # Sync with git -``` +Current docs to consider: + +- `README.md` — project overview, setup, scripts, env vars, production shape +- `DEPLOYMENT.md` — generic hosted deployment guidance +- `RAILWAY.md` — Railway-specific deployment guidance +- `AGENTS.md` — agent-facing architecture, workflow, and maintenance instructions +- `src/app/docs/page.tsx` — dashboard documentation shown in the app +- `.agents/skills/orglabeler/SKILL.md` — downstream consumer integration guidance +- `.env.example` — documented environment variable defaults/examples + +If a new documentation file is introduced, add it to this list and include when it should be checked. diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index ce6e2a2..bc9ec3e 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -40,59 +40,67 @@ The direct Caddy route is required for `subscribeLabels`, because it is a WebSoc ## Environment variables -### Required for the running service +### Required for the running app service -- `DID` — labeler DID -- `SIGNING_KEY` — labeler private key used by `@skyware/labeler` - -These are validated at startup by `src/lib/config.ts`. +- `DID` — labeler account DID +- `SIGNING_KEY` — private key material used by `@skyware/labeler` to sign labels +- `TAP_URL` — URL of the separate Tap service; there is no localhost fallback +- `NEXT_PUBLIC_LABELER_ENDPOINT` — public HTTPS base URL for the dashboard and labeler XRPC endpoint ### Required for first-time labeler setup / label sync -- `BSKY_IDENTIFIER` -- `BSKY_PASSWORD` -- `NEXT_PUBLIC_LABELER_ENDPOINT` - -Use these with `npm run setup` and `npm run set-labels` when registering the labeler account and label definitions. - -### Required for the app service to reach Tap +- `LABELER_IDENTIFIER` — labeler account identifier, for example `orglabeler.certified.one` +- `LABELER_PASSWORD` — labeler account password or app password +- `PDS_URL` — optional override for the labeler account PDS; setup normally resolves this from the DID document -- `TAP_URL` — required URL of the separate Tap service +Use these with `npm run setup` and `npm run set-labels` when registering the labeler account and label definitions. The labeler account identifier is the signing source; `NEXT_PUBLIC_LABELER_ENDPOINT` is the hosted app endpoint and may be a different domain. ### SQLite paths Set these to persistent storage in hosted environments: -- `ACTIVITY_DB_PATH` (default `activity-log.db`) -- `LABELS_DB_PATH` (default `labels.db`) +- `ACTIVITY_DB_PATH` (code default `/data/activity-log.db`) +- `LABELS_DB_PATH` (code default `/data/labels.db`) -### Tap service storage +### Tap service settings The Tap service manages its own persistent SQLite file and any Tap-specific settings outside this repo. -If your Tap deployment uses admin auth, configure `TAP_ADMIN_PASSWORD` on the Tap service itself. +If your Tap deployment uses admin auth, configure `TAP_ADMIN_PASSWORD` on the Tap service itself and set the same `TAP_ADMIN_PASSWORD` on the app service so health checks and the Tap WebSocket can authenticate. -### Optional +### Optional app settings -- `HF_TOKEN` — enables HuggingFace classification; if unset, HF scoring stays disabled -- `PDS_URL` — overrides PDS discovery from the DID document +- `HF_TOKEN` — enables Hugging Face classification; if unset, HF scoring stays disabled +- `HYPERSCAN_RECORD_URL_BASE` — base URL used when the dashboard links to source AT Protocol records; default `https://hyperscan.dev/data` - `TEST_PDS_HOSTS` — comma-separated PDS hosts whose actors should always be labeled `likely-test`; URL enrichment waits for actor PDS resolution and skips matching test PDS hosts -- `PORT` — public Caddy HTTP port (hosted platforms usually set this; default fallback `8080`) +- `TRUSTED_PDS_HOSTS` — comma-separated PDS hosts whose actors receive the trusted-PDS score bonus; default `certified.one,gainforest.id` +- `TRUSTED_PDS_BONUS` — score points added for trusted actor PDS hosts; default `10` +- `PORT` — public Caddy HTTP port (hosted platforms usually set this; Caddy fallback `8080`) - `NEXT_PORT` — internal Next.js HTTP port (`3000` by default) -- `HOST` — bind host for the labeler server (`127.0.0.1` is recommended when Caddy runs in the same container) +- `HOST` — bind host for the labeler server (`0.0.0.0` by default; `127.0.0.1` is recommended when only same-container Caddy should reach it) - `LABELER_PORT` — internal labeler HTTP port (`4100` by default) - `METRICS_PORT` — metrics port (`4101` by default) -- `RESET_DB=true` — deletes the SQLite files before startup +- `RESET_DB=true` — deletes configured app SQLite files plus WAL/SHM companions before startup; remove it after one reset +- `URL_ENRICHMENT_ENABLED` — enables async URL checks through `url_checks`; default `true` +- `URL_CHECK_INTERVAL_MS` — poll interval for processing due URL checks; default `1000` +- `URL_CHECK_DISCOVERY_INTERVAL_MS` — how often the URL worker scans local snapshots for newly referenced URLs; default `30000` +- `URL_CHECK_TIMEOUT_MS` — timeout for one URL resolution attempt; default `4000` +- `URL_CHECK_OK_TTL_MS` — freshness window for successful URL checks; default `604800000` +- `URL_CHECK_FAILED_TTL_MS` — downgrade window for hard failed URL checks before another attempt; default `86400000` +- `URL_CHECK_RETRY_BASE_MS` — initial retry delay for temporary URL check failures; default `300000` +- `URL_CHECK_MAX_RETRY_MS` — maximum retry delay for temporary URL check failures; default `3600000` +- `URL_CHECK_HARD_FAILURE_ATTEMPTS` — number of hard failures required before URL scoring removes resolve points; default `2` +- `URL_CHECK_MAX_URLS_PER_DID` — maximum profile/organization URLs cached and checked per DID; default `5` ## Persistence requirements This fork uses two SQLite databases in the app service and one in the Tap service: -- `activity-log.db` — dashboard data and scoring history -- `labels.db` — AT Protocol label records -- `tap.db` — Tap cursor / replay state (Tap service) +- `ACTIVITY_DB_PATH` — dashboard data and scoring history +- `LABELS_DB_PATH` — AT Protocol label records +- Tap service database — Tap cursor / replay state -The app service should mount a volume for `activity-log.db` and `labels.db`. The Tap service should mount its own volume for `tap.db`. If the files are not persisted, redeploys will lose dashboard history and Tap will replay from an old or empty cursor. +The app service should mount a volume for the configured `ACTIVITY_DB_PATH` and `LABELS_DB_PATH`. The Tap service should mount its own volume for its database. If the files are not persisted, redeploys will lose dashboard history and Tap will replay from old or empty state. Also persist the `-wal` and `-shm` companions that SQLite may create beside each DB file. diff --git a/FORK_MIGRATION_PLAN.md b/FORK_MIGRATION_PLAN.md deleted file mode 100644 index f3ef1b9..0000000 --- a/FORK_MIGRATION_PLAN.md +++ /dev/null @@ -1,389 +0,0 @@ -# Fork Migration Plan - -## Goal - -Turn this fork into a labeler for `app.certified.actor.organization` while upgrading the main dependencies safely. - -## Guiding principle - -Do this in two tracks: - -1. **Infra upgrades** - - package bumps - - Tap handling improvements - - generic cleanup -2. **Domain migration** - - switch collection - - switch lexicon/types/parser - - adjust scoring - - rename branding/docs/UI - -Keep those separate as long as possible. - -## Phase 1: Stabilize the fork structure - -### Branches - -Create or use branches like: - -- `main` → stable fork baseline -- `upgrade-infra` → dependency upgrades only -- `organization-labeler` → collection/domain migration -- optional: `generic-refactor` → reusable abstractions if needed - -### Outcome - -Keep upstreamable work separate from app-specific work. - -## Phase 2: Baseline audit before changing behavior - -### Confirm current behavior - -Document the current working behavior in the fork: - -- Tap receives backfill and live events -- collection filtered to the current source collection -- records validated against the current lexicon -- score written to SQLite -- AT Protocol label applied to record URI -- dashboard shows recent items and stats - -### Why - -You want a known-good baseline before upgrades. - -## Phase 3: Infra upgrades only - -### Upgrade targets - -In `upgrade-infra`: - -1. `@atproto/tap` → `0.2.13` -2. `@atproto/api` → `0.19.9` -3. `next` → `16.x` -4. align: - - `react` - - `react-dom` -5. refresh the lockfile and test - -### Validate after each step - -After each package family bump, verify: - -- install works -- typecheck/build works -- labeler starts -- Tap consumer still receives events -- dashboard still builds and loads - -### Important - -Do Tap/API upgrades before Next 16 if possible, because those affect the labeler core more directly. - -## Phase 4: Identify reusable vs app-specific code - -Split the code mentally into: - -### Reusable infrastructure - -These likely stay: - -- labeler server wrapper -- Tap startup/shutdown flow -- DB logging pattern -- label sync / negation logic -- metrics/logging -- dashboard API pattern - -### App-specific pieces - -These must change: - -- collection constant -- lexicon parser/type imports -- scorer -- UI copy -- docs -- metadata/OpenGraph -- any hardcoded links or labels - -## Phase 5: Collection migration - -### Replace collection target - -Move from the legacy collection to: - -- `app.certified.actor.organization` - -### Update all collection-bound logic - -That includes: - -- config constant -- Tap filter -- record URI construction -- lexicon validation import -- record type imports -- any collection-specific links in UI/docs - -### Outcome - -The pipeline listens to and validates organization records instead of the old source collection. - -## Phase 6: Adjust the scoring model - -Current scoring is tuned for the old domain, so do not try to force it. - -### New scoring rubric for organizations - -Define what you actually want to label. Likely one of: - -- organization completeness -- organization quality -- likely spam/test -- trust/readiness - -### Example signals - -Potential organization criteria: - -- display name quality -- description quality -- avatar present -- banner present -- website present -- placeholder/test text detection -- repetition/gibberish detection - -### Decide labels - -Ask whether the labels should remain: - -- `high-quality` -- `standard` -- `draft` -- `likely-test` - -or become organization-specific. - -That decision affects both label definitions and UI wording. - -## Phase 7: Rename product/UI/docs - -The fork should read as an organization-labeling product, not a renamed activity scorer. - -### Update - -- metadata descriptions -- dashboard copy -- docs page -- feed text -- loading/empty states -- label descriptions -- OpenGraph assets/text -- any hardcoded external URLs pointing at old viewers - -### Outcome - -The product matches the organization migration path. - -## Phase 8: Make it configurable where it helps - -Not everything needs to be generic, but a few things should be. - -### Good candidates for config/abstraction - -- target collection NSID -- parser/validator -- score function -- UI copy strings -- label definitions -- external record URL builder - -### Avoid overengineering - -Do not fully genericize everything on day one unless you know you need multi-collection support. - -## Phase 9: End-to-end validation - -Before merging back to the fork `main`, test: - -### Labeler runtime - -- starts cleanly -- Tap healthy -- receives backfill/live events -- skips invalid/deletes correctly -- stores rows in DB -- applies labels correctly -- re-labels correctly if classification changes - -### Dashboard - -- builds on Next 16 -- API routes work -- stats load -- recent feed renders -- empty state and search/filter still make sense - -### Data correctness - -- record URI correct for `app.certified.actor.organization` -- parser matches the actual organization schema -- scorer does not rely on missing fields from the old domain - -## Phase 10: Upstream strategy - -After things are stable, split changes into buckets: - -### Potentially upstreamable - -- package upgrades -- Tap improvements -- startup/shutdown robustness -- generic label sync fixes -- config cleanup - -### Probably not upstreamable - -- organization-specific collection migration -- scoring rubric changes -- new branding/docs/product direction - -## Key risks to watch - -### 1. Major-version package breakage - -Especially: - -- Tap API/event shape -- AT Protocol client types -- Next 16 compatibility - -### 2. Hidden old-domain assumptions - -There are likely many across code, docs, labels, and UI. - -### 3. Scoring mismatch - -Organization records may be simpler than the previous domain, so the old scoring model cannot just be reused. - -### 4. Mixed commits - -Do not mix: - -- infra upgrades -- collection migration -- scoring adjustment -- branding cleanup - -That makes debugging much harder. - -## Suggested execution order - -1. Fork baseline confirmed -2. Create `upgrade-infra` -3. Upgrade Tap/API -4. Upgrade Next/React -5. Merge infra branch into fork `main` -6. Create `organization-labeler` -7. Switch collection and parser/types -8. Adjust scorer -9. Rename UI/docs/branding -10. End-to-end test -11. Decide what to upstream - -## Recommendation in one line - -Upgrade the reusable infrastructure first, then migrate the domain to certified actor organizations in a separate branch. - -## Execution checklist / TODO - -Use this as the working checklist while implementing the migration. - -### Fork baseline - -- [ ] Confirm fork is cloned locally and original repo is added as `upstream` -- [ ] Confirm current `main` builds and starts before changes -- [ ] Confirm current labeler still ingests the current source collection -- [ ] Capture baseline notes on current behavior and any existing breakage - -### Branching - -- [ ] Create `upgrade-infra` branch -- [ ] Create `organization-labeler` branch -- [ ] Keep infra commits separate from domain/product commits - -### Infra upgrades - -- [ ] Upgrade `@atproto/tap` to `0.2.13` -- [ ] Verify Tap startup, health checks, and event handling still work -- [ ] Upgrade `@atproto/api` to `0.19.9` -- [ ] Fix any AT Protocol client/type breakage -- [ ] Upgrade `next` to `16.x` -- [ ] Align `react` and `react-dom` with the Next 16-compatible versions -- [ ] Refresh lockfile -- [ ] Run install/build/typecheck after each package family change - -### Reusable infra review - -- [ ] Identify which improvements are generic enough to upstream later -- [ ] Isolate generic Tap/runtime fixes from organization-specific logic -- [ ] Isolate generic config cleanup from branding/scoring changes - -### Collection migration - -- [ ] Replace the current collection with `app.certified.actor.organization` where appropriate -- [ ] Update config constant(s) -- [ ] Update Tap filter configuration -- [ ] Update URI construction logic -- [ ] Switch lexicon validation to `app.certified.actor.organization` -- [ ] Switch record types/imports to the organization lexicon - -### Scoring adjustment - -- [ ] Decide what the new labels actually represent -- [ ] Define scoring criteria for organization records -- [ ] Remove old-domain assumptions from the scorer -- [ ] Add organization-specific quality/test/spam signals -- [ ] Decide whether to keep current tier names or rename them -- [ ] Update label definitions and descriptions accordingly - -### UI / product cleanup - -- [ ] Replace stale copy in the dashboard -- [ ] Update docs page text and examples -- [ ] Update metadata and OpenGraph text -- [ ] Remove or replace old-domain links/viewers -- [ ] Update empty states and feed descriptions for organization labeling - -### Validation - -- [ ] Confirm labeler starts cleanly after migration -- [ ] Confirm Tap receives backfill and live organization events -- [ ] Confirm invalid/deleted records are handled safely -- [ ] Confirm DB rows are written correctly for organization records -- [ ] Confirm labels are applied to the correct record URIs -- [ ] Confirm reclassification flows still work if the result changes -- [ ] Confirm dashboard pages and API routes still work - -### Review before merge - -- [ ] Review commit history for mixed concerns and split if needed -- [ ] Identify which commits could be upstreamed later -- [ ] Merge `upgrade-infra` into fork `main` only when stable -- [ ] Merge `organization-labeler` after end-to-end validation passes -- [ ] Document any intentional divergence from upstream - -## Post-migration review - -After the migration is complete, review whether the fork now does the following: - -- [ ] Upgraded core dependencies successfully -- [ ] Preserved the reusable Tap → score → label pipeline -- [ ] Switched ingestion to certified actor organizations -- [ ] Uses the correct organization lexicon and record shape throughout -- [ ] Applies a scoring model that makes sense for organization data -- [ ] No longer relies on stale copy, docs, or links -- [ ] Still has a clean separation between upstreamable infra changes and app-specific changes -- [ ] Is in a state where future upstream PRs are still practical diff --git a/RAILWAY.md b/RAILWAY.md index 76478e0..2d8757b 100644 --- a/RAILWAY.md +++ b/RAILWAY.md @@ -13,7 +13,7 @@ The app service connects to Tap over `TAP_URL`. | Service | Responsibility | Storage | |---------|----------------|---------| -| App service | Dashboard, API routes, labeler process | `activity-log.db`, `labels.db` | +| App service | Dashboard, API routes, labeler process | configured `ACTIVITY_DB_PATH`, `LABELS_DB_PATH` | | Tap service | Firehose replay / indexing | `tap.db` | Do not share the Tap database between services. Each service should have its own volume. @@ -33,39 +33,60 @@ Required: | Variable | Description | |----------|-------------| -| `DID` | Labeler DID | -| `SIGNING_KEY` | Labeler private key | -| `NEXT_PUBLIC_LABELER_ENDPOINT` | Public HTTPS URL of the app service | +| `DID` | Labeler account DID | +| `SIGNING_KEY` | Private key material used by `@skyware/labeler` to sign labels | +| `NEXT_PUBLIC_LABELER_ENDPOINT` | Public HTTPS URL of the app service, for example `https://orglabeler.hypercerts.dev` | | `TAP_URL` | Required URL of the separate Tap service; no localhost default | -| `TAP_ADMIN_PASSWORD` | App-side Tap admin password; required when the Tap service enables admin auth | + +Required when Tap admin auth is enabled: + +| Variable | Description | +|----------|-------------| +| `TAP_ADMIN_PASSWORD` | App-side Tap admin password; must match the Tap service | Usually needed for setup/sync: | Variable | Description | |----------|-------------| -| `BSKY_IDENTIFIER` | Labeler account handle or email | -| `BSKY_PASSWORD` | Labeler account password or app password | +| `LABELER_IDENTIFIER` | Labeler account identifier, for example `orglabeler.certified.one` | +| `LABELER_PASSWORD` | Labeler account password or app password | +| `PDS_URL` | Optional labeler account PDS override; setup normally resolves it from the DID document | + +The labeler account identifier is the signing source. `NEXT_PUBLIC_LABELER_ENDPOINT` is the hosted app endpoint and may be a different domain. Storage: -| Variable | Description | Example | -|----------|-------------|---------| -| `ACTIVITY_DB_PATH` | Activity log SQLite path | `/app/data/activity-log.db` | -| `LABELS_DB_PATH` | Label records SQLite path | `/app/data/labels.db` | +| Variable | Description | Code default | Railway example | +|----------|-------------|--------------|-----------------| +| `ACTIVITY_DB_PATH` | Activity log SQLite path | `/data/activity-log.db` | `/app/data/activity-log.db` | +| `LABELS_DB_PATH` | Label records SQLite path | `/data/labels.db` | `/app/data/labels.db` | Optional: | Variable | Description | Default | |----------|-------------|---------| -| `HF_TOKEN` | Enables HuggingFace scoring | _(empty)_ | +| `HF_TOKEN` | Enables Hugging Face scoring | _(empty)_ | +| `HYPERSCAN_RECORD_URL_BASE` | Base URL for dashboard links to source AT Protocol records | `https://hyperscan.dev/data` | +| `TEST_PDS_HOSTS` | PDS hosts whose actors should always be labeled `likely-test` | _(empty)_ | +| `TRUSTED_PDS_HOSTS` | PDS hosts whose actors receive the trusted-PDS score bonus | `certified.one,gainforest.id` | +| `TRUSTED_PDS_BONUS` | Score points added for trusted actor PDS hosts | `10` | +| `NEXT_PUBLIC_SITE_URL` | Dashboard metadata base URL | `VERCEL_URL` or `http://localhost:3000` | +| `NEXT_PUBLIC_COMMIT_SHA` | Optional deployment SHA shown in the footer | `RAILWAY_GIT_COMMIT_SHA` when available | +| `NEXT_PUBLIC_DEPLOY_TIME` | Optional deployment timestamp shown in the footer | image build time via `scripts/build.sh` | | `NEXT_PORT` | Internal Next.js port behind Caddy | `3000` | -| `HOST` | Labeler bind host | `127.0.0.1` | +| `HOST` | Labeler bind host | `0.0.0.0` | | `LABELER_PORT` | Internal labeler HTTP port behind Caddy | `4100` | | `METRICS_PORT` | Metrics port | `4101` | -| `RESET_DB` | Clear app DB files on startup | _(empty)_ | +| `RESET_DB` | Clear configured app DB files on startup; remove after one reset | _(empty)_ | | `URL_ENRICHMENT_ENABLED` | Enable detachable async URL checks through `url_checks` | `true` | -| `URL_CHECK_TIMEOUT_MS` | Timeout for one URL resolution attempt | `4000` | | `URL_CHECK_INTERVAL_MS` | URL worker polling interval | `1000` | +| `URL_CHECK_DISCOVERY_INTERVAL_MS` | New URL discovery scan interval | `30000` | +| `URL_CHECK_TIMEOUT_MS` | Timeout for one URL resolution attempt | `4000` | +| `URL_CHECK_OK_TTL_MS` | Freshness window for successful URL checks | `604800000` | +| `URL_CHECK_FAILED_TTL_MS` | Downgrade window for hard failed URL checks | `86400000` | +| `URL_CHECK_RETRY_BASE_MS` | Initial retry delay for temporary failures | `300000` | +| `URL_CHECK_MAX_RETRY_MS` | Maximum retry delay for temporary failures | `3600000` | +| `URL_CHECK_HARD_FAILURE_ATTEMPTS` | Hard failures needed before URL scoring removes resolve points | `2` | | `URL_CHECK_MAX_URLS_PER_DID` | Maximum profile/organization URLs cached and checked per DID | `5` | Start the app with: diff --git a/README.md b/README.md index 2522c39..4d2f0d6 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ This fork monitors both `app.certified.actor.profile` and `app.certified.actor.o ### Prerequisites - Node.js 22+ -- A Bluesky/AT Protocol account dedicated to the labeler +- An AT Protocol account dedicated to the labeler ### Setup ```bash @@ -24,11 +24,8 @@ git clone cd orglabeler npm install -# Works with any AT Protocol PDS (not just bsky.social) -npm run setup -- your-handle.bsky.social your-password https://labeler.yourdomain.com - -# Example with custom PDS: -npm run setup -- satyam2.climateai.org yourpassword https://labeler.climateai.org +# The first argument is the labeler account identifier; the final argument is the public app/labeler endpoint. +npm run setup -- orglabeler.certified.one your-password https://orglabeler.hypercerts.dev ``` The setup script automatically resolves your account's PDS endpoint from the DID document, so it works with any AT Protocol PDS. @@ -62,13 +59,13 @@ The runtime is split into three pieces: - AT Protocol relay → separate Tap service - Tap service → labeler process over `TAP_URL` -- Labeler process → `labels.db` + `activity-log.db` +- Labeler process → configured `LABELS_DB_PATH` + `ACTIVITY_DB_PATH` SQLite databases -The Tap sidecar listens to both `app.certified.actor.profile` and `app.certified.actor.organization`, merges them by DID for actor context, and applies quality labels to the actor DID. Fresh deployments should wipe the app and Tap volumes together so old local labels, activity rows, and Tap cursor state are not reused. +The Tap service listens to both `app.certified.actor.profile` and `app.certified.actor.organization`, merges them by DID for actor context, and applies quality labels to the actor DID. Fresh deployments should wipe the app and Tap volumes together so old local labels, activity rows, and Tap cursor state are not reused. -The Next.js dashboard reads from `activity-log.db`. +The Next.js dashboard reads from the configured `ACTIVITY_DB_PATH`. -The labeler auto-detects the PDS for non-bsky.social accounts via DID document resolution, so it works across any AT Protocol PDS. +The labeler auto-detects the account PDS via DID document resolution, so it works across any AT Protocol PDS. ## Scoring @@ -80,10 +77,10 @@ Scores `app.certified.actor.organization` records on 13 completeness signals (10 | Description | 10 | Has a profile description | | Organization Type | 5 | Includes at least one organization type value | | Website Present | 10 | Has a public website URL | -| Website Resolves | 15 | Public website resolves successfully | +| Website Resolves | 15 | Valid-looking public website URL, with points removed after confirmed async URL failures | | Website Matches Name | 5 | Website domain matches the display name | | Organization URLs Present | 5 | Includes at least one organization URL | -| Organization URLs Resolve | 5 | At least one organization URL resolves successfully | +| Organization URLs Resolve | 5 | At least one valid-looking organization URL, with points removed after confirmed async URL failures | | Location | 10 | Has a valid organization location reference | | Founded Date | 5 | Has a valid founded date | | Founded Date Age | 5 | Founded date is at least one year old | @@ -105,39 +102,59 @@ When `TEST_PDS_HOSTS` is configured, URL enrichment is PDS-aware: it defers URL | Script | Description | |--------|-------------| -| `npm run dev` | Start Next.js dashboard | +| `npm run dev` | Start Next.js dashboard in development | +| `npm run build` | Build the production Next.js app | +| `npm run start` | Start the built Next.js dashboard only | +| `npm run start:next` | Start the built Next.js dashboard on `NEXT_PORT` | +| `npm run start:proxy` | Start Caddy from `Caddyfile` | | `npm run labeler` | Start labeler backend | | `npm run dev:service` | Start dashboard + labeler concurrently | | `npm run start:service` | Start Caddy reverse proxy + production dashboard + labeler process | | `npm run setup` | Initialize labeler account | | `npm run set-labels` | Push/update label definitions | -| `npm run build` | Production build | -| `npm run reset` | Clear databases (fresh start) | +| `npm run reset` | Clear local databases (fresh start) | +| `npm run test` | Run Node test suite | +| `npm run gen:lexicons` | Rebuild generated TypeScript lexicon files | ## Environment Variables +The code defaults are shown below. `.env.example` uses local SQLite paths for development; deployments should set database paths to the mounted persistent volume. + | Variable | Default | Description | |----------|---------|-------------| -| `DID` | (set by setup) | Labeler account DID | -| `SIGNING_KEY` | (set by setup) | secp256k1 private key hex | -| `BSKY_IDENTIFIER` | (set by setup) | Account handle | -| `BSKY_PASSWORD` | (set by setup) | Account password or app password | -| `PDS_URL` | (auto-detected) | PDS endpoint URL | -| `NEXT_PUBLIC_LABELER_ENDPOINT` | https://labeler. | Public HTTPS URL for the labeler | -| `PORT` | 8080 | Public HTTP port listened to by Caddy; hosted platforms usually set this | -| `NEXT_PORT` | 3000 | Internal Next.js port behind Caddy | -| `HOST` | 127.0.0.1 | Labeler server bind address; keep local when Caddy is in the same container | -| `LABELER_PORT` | 4100 | Internal labeler server port behind Caddy | -| `METRICS_PORT` | 4101 | Prometheus metrics port | -| `TAP_URL` | required | URL of the separate Tap service (no localhost default) | +| `DID` | (set by setup; required) | Labeler account DID | +| `SIGNING_KEY` | (set by setup; required) | Private key material used by `@skyware/labeler` to sign labels | +| `LABELER_IDENTIFIER` | (set by setup) | Labeler account identifier for setup and label definition updates | +| `LABELER_PASSWORD` | (set by setup) | Labeler account password or app password | +| `PDS_URL` | (auto-detected by setup) | PDS endpoint URL for the labeler account | +| `NEXT_PUBLIC_LABELER_ENDPOINT` | empty | Public HTTPS base URL for the dashboard and labeler XRPC endpoint, for example `https://orglabeler.hypercerts.dev` | +| `NEXT_PUBLIC_SITE_URL` | `VERCEL_URL` or `http://localhost:3000` | Dashboard metadata base URL | +| `NEXT_PUBLIC_COMMIT_SHA` | `RAILWAY_GIT_COMMIT_SHA` when available | Optional deployment SHA shown in the footer | +| `NEXT_PUBLIC_DEPLOY_TIME` | startup time | Optional deployment timestamp shown in the footer | +| `PORT` | `8080` | Public HTTP port listened to by Caddy; hosted platforms usually set this | +| `NEXT_PORT` | `3000` | Internal Next.js port behind Caddy | +| `HOST` | `0.0.0.0` | Labeler server bind address; set `127.0.0.1` when only same-container Caddy should reach it | +| `LABELER_PORT` | `4100` | Internal labeler server port behind Caddy | +| `METRICS_PORT` | `4101` | Prometheus metrics port | +| `TAP_URL` | empty; required | URL of the separate Tap service; there is no localhost fallback | | `TAP_ADMIN_PASSWORD` | empty | App-side password for Tap admin auth; must match the Tap service when auth is enabled | +| `ACTIVITY_DB_PATH` | `/data/activity-log.db` | Dashboard activity log SQLite database path | +| `LABELS_DB_PATH` | `/data/labels.db` | AT Protocol label SQLite database path used by `@skyware/labeler` | +| `RESET_DB` | unset | When set to `true`, deletes configured app database files plus WAL/SHM files on startup; remove it after the reset | | `TEST_PDS_HOSTS` | empty | Comma-separated PDS hosts whose actors should always be labeled `likely-test`; when set, URL enrichment waits for actor PDS resolution and skips matching test PDS hosts | | `TRUSTED_PDS_HOSTS` | `certified.one,gainforest.id` | Comma-separated PDS hosts whose actors receive the trusted-PDS score bonus | | `TRUSTED_PDS_BONUS` | `10` | Score points added when an actor's resolved PDS host matches `TRUSTED_PDS_HOSTS`; set to `0` to disable the bonus | -| `ACTIVITY_DB_PATH` | `activity-log.db` | Activity log database path | +| `HYPERSCAN_RECORD_URL_BASE` | `https://hyperscan.dev/data` | Base URL used when the dashboard links to source AT Protocol records | +| `HF_TOKEN` | empty | Optional Hugging Face token; when set, enables the zero-shot authenticity classifier | | `URL_ENRICHMENT_ENABLED` | `true` | Enables async URL checks through the detachable `url_checks` cache | +| `URL_CHECK_INTERVAL_MS` | `1000` | Poll interval for processing due URL checks | +| `URL_CHECK_DISCOVERY_INTERVAL_MS` | `30000` | How often the URL worker scans local snapshots for newly referenced URLs | | `URL_CHECK_TIMEOUT_MS` | `4000` | Timeout for one URL resolution attempt | -| `URL_CHECK_INTERVAL_MS` | `1000` | Poll interval for the URL enrichment worker | +| `URL_CHECK_OK_TTL_MS` | `604800000` | Freshness window for successful URL checks | +| `URL_CHECK_FAILED_TTL_MS` | `86400000` | Downgrade window for hard failed URL checks before another attempt | +| `URL_CHECK_RETRY_BASE_MS` | `300000` | Initial retry delay for temporary URL check failures | +| `URL_CHECK_MAX_RETRY_MS` | `3600000` | Maximum retry delay for temporary URL check failures | +| `URL_CHECK_HARD_FAILURE_ATTEMPTS` | `2` | Number of hard failures required before URL scoring removes resolve points | | `URL_CHECK_MAX_URLS_PER_DID` | `5` | Maximum profile/organization URLs cached and checked per DID | Tap runtime settings belong on the Tap service. If the Tap service sets `TAP_ADMIN_PASSWORD`, set the same value on the app service so health checks and the Tap WebSocket can authenticate. @@ -146,6 +163,8 @@ Tap runtime settings belong on the Tap service. If the Tap service sets `TAP_ADM Deploy the app service and Tap as separate services. The app service runs the dashboard plus labeler backend and connects to Tap over `TAP_URL`; Tap owns its own database, volume, lifecycle, and any Tap-specific auth settings. +Set `NEXT_PUBLIC_LABELER_ENDPOINT` to the public app URL, for example `https://orglabeler.hypercerts.dev`. The labeler account identifier can still be `orglabeler.certified.one`; that account is the signing source, not necessarily the app endpoint. + The production app uses Caddy as the front door. Caddy routes public AT Protocol XRPC label methods directly to the labeler process and everything else to Next.js: ```txt diff --git a/src/app/docs/page.tsx b/src/app/docs/page.tsx index 731ade2..0b146b8 100644 --- a/src/app/docs/page.tsx +++ b/src/app/docs/page.tsx @@ -32,7 +32,7 @@ const SCORING_CRITERIA = [ { label: 'Profile website resolves', points: COMPLETENESS_WEIGHTS.websiteResolves, - description: 'Awards stronger points when profile.website is a valid public HTTP(S) URL.', + description: 'Awards stronger provisional points for a valid-looking public HTTP(S) URL; async URL enrichment removes them after confirmed hard failures.', }, { label: 'Profile website matches name', @@ -47,7 +47,7 @@ const SCORING_CRITERIA = [ { label: 'Organization URLs resolve', points: COMPLETENESS_WEIGHTS.organizationUrlsResolve, - description: 'Awards a small amount when at least one organization URL is a valid public HTTP(S) URL.', + description: 'Awards a small provisional bonus when at least one organization URL is a valid-looking public HTTP(S) URL; async URL enrichment removes it after confirmed hard failures.', }, { label: 'Location valid', @@ -151,7 +151,7 @@ export default function DocsPage() { and{' '} app.certified.actor.organization{' '} records. Tap automatically discovers repos, backfills historical records from each PDS, - and streams live events with cryptographic verification. The sidecar merges those records by DID + and streams live events with cryptographic verification. The labeler merges those records by DID so the dashboard can show the profile alongside the organization context. When a profile record needs fallback handling, the usable core fields are kept and malformed optional fields are ignored. @@ -360,8 +360,8 @@ export default function DocsPage() {
  • — - Each label is signed with ed25519 and includes: source DID, subject DID, label value, - timestamp, and a cryptographic signature. + Each signed label includes: source DID, subject DID, label value, timestamp, + and a cryptographic signature.
  • diff --git a/src/labeler/set-labels.ts b/src/labeler/set-labels.ts index 28b5c72..2e5eb38 100644 --- a/src/labeler/set-labels.ts +++ b/src/labeler/set-labels.ts @@ -1,17 +1,17 @@ -// Run with: npx tsx src/labeler/set-labels.ts [handle] [password] +// Run with: npx tsx src/labeler/set-labels.ts [labeler-identifier] [password] -import { BSKY_IDENTIFIER, BSKY_PASSWORD } from '../lib/config' +import { LABELER_IDENTIFIER, LABELER_PASSWORD } from '../lib/config' import { LABELS } from '../lib/constants' import { resolvePds } from '../lib/resolve-pds' import { setLabelerLabelDefinitions } from './declare-labeler' async function main() { - const identifier = process.argv[2] || BSKY_IDENTIFIER - const password = process.argv[3] || BSKY_PASSWORD + const identifier = process.argv[2] || LABELER_IDENTIFIER + const password = process.argv[3] || LABELER_PASSWORD if (!identifier || !password) { - console.error('Error: BSKY_IDENTIFIER and BSKY_PASSWORD must be set in .env or passed as CLI args') - console.error('Usage: npx tsx src/labeler/set-labels.ts ') + console.error('Error: LABELER_IDENTIFIER and LABELER_PASSWORD must be set in .env or passed as CLI args') + console.error('Usage: npx tsx src/labeler/set-labels.ts ') process.exit(1) } diff --git a/src/labeler/setup.ts b/src/labeler/setup.ts index 9485cea..d1d6a42 100644 --- a/src/labeler/setup.ts +++ b/src/labeler/setup.ts @@ -1,6 +1,6 @@ #!/usr/bin/env tsx -// Run with: npx tsx src/labeler/setup.ts [labeler-endpoint] [--token PLC_TOKEN] -// Or: npm run setup -- satyam2.climateai.org mypassword https://labeler.example.com +// Run with: npx tsx src/labeler/setup.ts [labeler-endpoint] [--token PLC_TOKEN] +// Or: npm run setup -- orglabeler.certified.one mypassword https://orglabeler.hypercerts.dev import { plcRequestToken, plcSetupLabeler } from '@skyware/labeler/scripts' import 'dotenv/config' @@ -13,19 +13,19 @@ import { resolvePds } from '../lib/resolve-pds' // --- Arg / env parsing --- -const handle = process.argv[2] || process.env.BSKY_IDENTIFIER || "" -const password = process.argv[3] || process.env.BSKY_PASSWORD || "" +const identifier = process.argv[2] || process.env.LABELER_IDENTIFIER || "" +const password = process.argv[3] || process.env.LABELER_PASSWORD || "" -if (!handle || !password) { - console.error("Usage: npx tsx src/labeler/setup.ts [labeler-endpoint] [--token PLC_TOKEN]") - console.error(" Or set BSKY_IDENTIFIER and BSKY_PASSWORD env vars.") +if (!identifier || !password) { + console.error("Usage: npx tsx src/labeler/setup.ts [labeler-endpoint] [--token PLC_TOKEN]") + console.error(" Or set LABELER_IDENTIFIER and LABELER_PASSWORD env vars.") process.exit(1) } const labelerEndpoint = process.argv[4] || process.env.NEXT_PUBLIC_LABELER_ENDPOINT || - `https://labeler.${handle}` + `https://labeler.${identifier}` // Find --token flag or use 5th positional arg or PLC_TOKEN env var const existingToken = (() => { @@ -35,7 +35,7 @@ const existingToken = (() => { })() console.log(`\nLabeler endpoint: ${labelerEndpoint}`) -console.log(`Account: ${handle}\n`) +console.log(`Account: ${identifier}\n`) // --- Helpers --- @@ -111,23 +111,23 @@ function writeEnvFile(path: string, updates: Record): void { async function main() { // Step 1: Generate signing key - console.log("Generating secp256k1 signing key...") + console.log("Generating label signing key...") const privateKeyBytes = secp256k1.utils.randomPrivateKey() const signingKeyHex = Buffer.from(privateKeyBytes).toString("hex") console.log(` Key generated (${signingKeyHex.slice(0, 8)}...)\n`) - // Step 1.5: Resolve PDS from handle + // Step 1.5: Resolve PDS from the labeler account identifier console.log("Resolving account PDS...") let did = "" let pdsUrl = "" try { - const resolved = await resolvePds(handle) + const resolved = await resolvePds(identifier) did = resolved.did pdsUrl = resolved.pds console.log(` DID: ${did}`) console.log(` PDS: ${pdsUrl}\n`) } catch (err) { - console.error(" ✗ Failed to resolve handle. Check that the handle exists.") + console.error(" ✗ Failed to resolve labeler account identifier. Check that the account exists.") console.error(" Error:", err) process.exit(1) } @@ -138,10 +138,10 @@ async function main() { // Step 2: Request PLC token (sends email) console.log("Requesting PLC operation token (this sends a confirmation email)...") try { - await plcRequestToken({ identifier: handle, password, pds: pdsUrl }) + await plcRequestToken({ identifier, password, pds: pdsUrl }) console.log(" ✓ Token requested.\n") } catch (err) { - console.error(" ✗ Failed to request PLC token. Check your handle and password.") + console.error(" ✗ Failed to request PLC token. Check your labeler account identifier and password.") console.error(" Error:", err) process.exit(1) } @@ -162,7 +162,7 @@ async function main() { console.log("\nSetting up labeler on PLC directory...") try { await plcSetupLabeler({ - identifier: handle, + identifier, password, pds: pdsUrl, plcToken, @@ -190,7 +190,7 @@ async function main() { })) try { - await declareLabeler({ identifier: handle, password, pds: pdsUrl }, labelDefs, true) + await declareLabeler({ identifier, password, pds: pdsUrl }, labelDefs, true) for (const label of LABELS) { console.log(` ✓ ${label.identifier}: ${label.locales[0]?.name}`) } @@ -205,8 +205,8 @@ async function main() { // Step 6: Write .env (preserve existing values, update/add our keys) const envPath = ".env" const updates: Record = { - BSKY_IDENTIFIER: handle, - BSKY_PASSWORD: password, + LABELER_IDENTIFIER: identifier, + LABELER_PASSWORD: password, SIGNING_KEY: signingKeyHex, NEXT_PUBLIC_LABELER_ENDPOINT: labelerEndpoint, PDS_URL: pdsUrl, diff --git a/src/lib/config.ts b/src/lib/config.ts index bec6751..10f4423 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -6,8 +6,8 @@ export const SIGNING_KEY = process.env.SIGNING_KEY ?? '' export const HOST = process.env.HOST ?? '0.0.0.0' export const LABELER_PORT = process.env.LABELER_PORT ? Number(process.env.LABELER_PORT) : 4100 export const METRICS_PORT = process.env.METRICS_PORT ? Number(process.env.METRICS_PORT) : 4101 -export const BSKY_IDENTIFIER = process.env.BSKY_IDENTIFIER ?? '' -export const BSKY_PASSWORD = process.env.BSKY_PASSWORD ?? '' +export const LABELER_IDENTIFIER = process.env.LABELER_IDENTIFIER ?? '' +export const LABELER_PASSWORD = process.env.LABELER_PASSWORD ?? '' export const PROFILE_COLLECTION = 'app.certified.actor.profile' export const ORGANIZATION_COLLECTION = 'app.certified.actor.organization' export const ACTIVITY_COLLECTION = ORGANIZATION_COLLECTION diff --git a/tests/config-env.test.ts b/tests/config-env.test.ts new file mode 100644 index 0000000..a64638c --- /dev/null +++ b/tests/config-env.test.ts @@ -0,0 +1,34 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' + +function readConfigWithEnv(env: Partial): Record { + const result = spawnSync(process.execPath, [ + '--import', + 'tsx', + '--input-type=module', + '-e', + `import { LABELER_IDENTIFIER, LABELER_PASSWORD } from './src/lib/config.ts' +console.log(JSON.stringify({ LABELER_IDENTIFIER, LABELER_PASSWORD }))`, + ], { + cwd: process.cwd(), + env: { + ...process.env, + ...env, + }, + encoding: 'utf8', + }) + + assert.equal(result.status, 0, result.stderr) + return JSON.parse(result.stdout) as Record +} + +test('labeler account credentials use LABELER_* environment variables', () => { + const config = readConfigWithEnv({ + LABELER_IDENTIFIER: 'orglabeler.certified.one', + LABELER_PASSWORD: 'app-password', + }) + + assert.equal(config.LABELER_IDENTIFIER, 'orglabeler.certified.one') + assert.equal(config.LABELER_PASSWORD, 'app-password') +})