Skip to content

Commit 0d84bc3

Browse files
joanreyeroclaude
andauthored
feat: dockerhub-sync worker for repo_docker pull counts (CM-1213) (#4163)
Signed-off-by: Joan Reyero <joan@reyero.io> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 90c86b2 commit 0d84bc3

16 files changed

Lines changed: 1038 additions & 1 deletion

File tree

backend/.env.dist.local

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -209,4 +209,11 @@ MAVEN_FETCHER_NON_CRITICAL_CONCURRENCY=20
209209
MAVEN_FETCHER_REFRESH_DAYS=1
210210
MAVEN_FETCHER_GROUP_DELAY_MS=100
211211
MAVEN_FETCHER_BASE_URL_BACKFILL=https://maven-central.storage-download.googleapis.com/maven2
212-
MAVEN_FETCHER_BASE_URL_INCREMENTAL=https://repo1.maven.org/maven2
212+
MAVEN_FETCHER_BASE_URL_INCREMENTAL=https://repo1.maven.org/maven2
213+
214+
# dockerhub-sync (see services/apps/packages_worker/src/dockerhub/)
215+
DOCKERHUB_API_BASE_URL=https://hub.docker.com/v2
216+
DOCKERHUB_BATCH_SIZE=100
217+
DOCKERHUB_REFRESH_INTERVAL_HOURS=24
218+
DOCKERHUB_DISCOVERY_INTERVAL_DAYS=14
219+
DOCKERHUB_IDLE_SLEEP_SEC=60
Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
-- dockerhub-sync (CM-1213)
2+
--
3+
-- Adds discovery/refresh bookkeeping for the dockerhub-sync worker
4+
-- (services/apps/packages_worker/src/dockerhub) and a daily snapshot table
5+
-- for Docker Hub lifetime pull counts.
6+
7+
-- Last time dockerhub-sync probed this repo for a published Docker image
8+
-- (Dockerfile detection + Hub candidate lookup). NULL = never checked.
9+
-- Separate from repos.last_synced_at because discovery cadence (weeks)
10+
-- differs from light-metadata refresh cadence (daily).
11+
ALTER TABLE repos
12+
ADD COLUMN IF NOT EXISTS docker_checked_at timestamptz;
13+
14+
-- Partial index for the discovery backlog query: pages repos that have never
15+
-- been probed for a Docker image. Once docker_checked_at is set the row drops
16+
-- out of the index, so this stays small even as the repos table grows.
17+
CREATE INDEX IF NOT EXISTS repos_docker_pending_idx ON repos (id)
18+
WHERE
19+
host = 'github' AND docker_checked_at IS NULL;
20+
21+
-- Supports the refresh query (WHERE last_synced_at < NOW() - interval).
22+
CREATE INDEX IF NOT EXISTS repo_docker_stale_idx ON repo_docker (last_synced_at);
23+
24+
-- ============================================================
25+
-- REPO DOCKER PULLS DAILY
26+
-- One row per image per day storing the *lifetime* pull_count as returned
27+
-- by hub.docker.com/v2/repositories/<image>. Docker Hub does not expose
28+
-- per-day download counts, so daily deltas are derived at query time:
29+
-- pulls_total - LAG(pulls_total) OVER (PARTITION BY image_name ORDER BY date)
30+
-- Keyed by image_name (matches repo_docker UNIQUE) so rows survive a
31+
-- repo_docker re-discovery without an FK cascade.
32+
--
33+
-- Partitioned monthly via pg_partman (extension + schema already created in
34+
-- V1780231200__npm_worker.sql).
35+
-- ============================================================
36+
CREATE TABLE IF NOT EXISTS repo_docker_pulls_daily (
37+
image_name text NOT NULL,
38+
date date NOT NULL,
39+
pulls_total bigint NOT NULL,
40+
PRIMARY KEY (image_name, date)
41+
)
42+
PARTITION BY RANGE (date);
43+
44+
-- Guard so this migration is idempotent against environments where the
45+
-- table was already registered manually (e.g. local dev that applied the
46+
-- earlier in-place schema edit).
47+
DO $$
48+
BEGIN
49+
IF NOT EXISTS (
50+
SELECT 1 FROM partman.part_config
51+
WHERE parent_table = 'public.repo_docker_pulls_daily'
52+
) THEN
53+
PERFORM partman.create_parent(
54+
p_parent_table => 'public.repo_docker_pulls_daily',
55+
p_control => 'date',
56+
p_interval => '1 month',
57+
p_premake => 3
58+
);
59+
END IF;
60+
END
61+
$$;
Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
version: '3.1'
2+
3+
x-env-args: &env-args
4+
DOCKER_BUILDKIT: 1
5+
NODE_ENV: docker
6+
SERVICE: dockerhub-sync
7+
SHELL: /bin/sh
8+
SUPPRESS_NO_CONFIG_WARNING: 'true'
9+
DOCKERHUB_API_BASE_URL: 'https://hub.docker.com/v2'
10+
DOCKERHUB_BATCH_SIZE: '100'
11+
DOCKERHUB_REFRESH_INTERVAL_HOURS: '24'
12+
DOCKERHUB_DISCOVERY_INTERVAL_DAYS: '14'
13+
DOCKERHUB_IDLE_SLEEP_SEC: '60'
14+
15+
services:
16+
dockerhub-sync:
17+
build:
18+
context: ../../
19+
dockerfile: ./scripts/services/docker/Dockerfile.packages-worker
20+
command: 'pnpm run start:dockerhub-sync'
21+
working_dir: /usr/crowd/app/services/apps/packages_worker
22+
env_file:
23+
- ../../backend/.env.dist.local
24+
- ../../backend/.env.dist.composed
25+
- ../../backend/.env.override.local
26+
- ../../backend/.env.override.composed
27+
environment:
28+
<<: *env-args
29+
restart: always
30+
networks:
31+
- crowd-bridge
32+
33+
dockerhub-sync-dev:
34+
build:
35+
context: ../../
36+
dockerfile: ./scripts/services/docker/Dockerfile.packages-worker
37+
command: 'pnpm run dev:dockerhub-sync'
38+
working_dir: /usr/crowd/app/services/apps/packages_worker
39+
# user: '${USER_ID}:${GROUP_ID}'
40+
env_file:
41+
- ../../backend/.env.dist.local
42+
- ../../backend/.env.dist.composed
43+
- ../../backend/.env.override.local
44+
- ../../backend/.env.override.composed
45+
environment:
46+
<<: *env-args
47+
hostname: dockerhub-sync
48+
networks:
49+
- crowd-bridge
50+
volumes:
51+
- ../../services/libs/audit-logs/src:/usr/crowd/app/services/libs/audit-logs/src
52+
- ../../services/libs/common/src:/usr/crowd/app/services/libs/common/src
53+
- ../../services/libs/common_services/src:/usr/crowd/app/services/libs/common_services/src
54+
- ../../services/libs/data-access-layer/src:/usr/crowd/app/services/libs/data-access-layer/src
55+
- ../../services/libs/database/src:/usr/crowd/app/services/libs/database/src
56+
- ../../services/libs/integrations/src:/usr/crowd/app/services/libs/integrations/src
57+
- ../../services/libs/logging/src:/usr/crowd/app/services/libs/logging/src
58+
- ../../services/libs/nango/src:/usr/crowd/app/services/libs/nango/src
59+
- ../../services/libs/opensearch/src:/usr/crowd/app/services/libs/opensearch/src
60+
- ../../services/libs/queue/src:/usr/crowd/app/services/libs/queue/src
61+
- ../../services/libs/redis/src:/usr/crowd/app/services/libs/redis/src
62+
- ../../services/libs/snowflake/src:/usr/crowd/app/services/libs/snowflake/src
63+
- ../../services/libs/telemetry/src:/usr/crowd/app/services/libs/telemetry/src
64+
- ../../services/libs/temporal/src:/usr/crowd/app/services/libs/temporal/src
65+
- ../../services/libs/types/src:/usr/crowd/app/services/libs/types/src
66+
- ../../services/apps/packages_worker/src:/usr/crowd/app/services/apps/packages_worker/src
67+
68+
networks:
69+
crowd-bridge:
70+
external: true

services/apps/packages_worker/package.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@
1111
"start:github-repos-enricher": "SERVICE=github-repos-enricher tsx src/bin/github-repos-enricher.ts",
1212
"dev:github-repos-enricher": "SERVICE=github-repos-enricher LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9234 src/bin/github-repos-enricher.ts",
1313
"dev:github-repos-enricher:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=github-repos-enricher LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9234 src/bin/github-repos-enricher.ts",
14+
"start:dockerhub-sync": "SERVICE=dockerhub-sync tsx src/bin/dockerhub-sync.ts",
15+
"dev:dockerhub-sync": "SERVICE=dockerhub-sync LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9239 src/bin/dockerhub-sync.ts",
16+
"dev:dockerhub-sync:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=dockerhub-sync LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9239 src/bin/dockerhub-sync.ts",
1417
"start:bq-dataset-ingest": "CROWD_TEMPORAL_TASKQUEUE=bq-dataset-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=bq-dataset-ingest tsx src/bin/bq-dataset-ingest.ts",
1518
"dev:bq-dataset-ingest": "CROWD_TEMPORAL_TASKQUEUE=bq-dataset-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=bq-dataset-ingest nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/bq-dataset-ingest.ts",
1619
"dev:bq-dataset-ingest:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=bq-dataset-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=bq-dataset-ingest nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/bq-dataset-ingest.ts",
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import { getServiceLogger } from '@crowd/logging'
2+
3+
import { getDockerhubConfig, getGithubAppConfig } from '../config'
4+
import { getPackagesDb } from '../db'
5+
import { runDockerhubLoop } from '../dockerhub'
6+
import { fetchRateLimitDiagnostics, resolveInstallations } from '../enricher/githubAppAuth'
7+
8+
const log = getServiceLogger()
9+
10+
let shuttingDown = false
11+
12+
const shutdown = async () => {
13+
if (shuttingDown) return
14+
shuttingDown = true
15+
log.info('Shutting down dockerhub-sync...')
16+
}
17+
18+
process.on('SIGINT', shutdown)
19+
process.on('SIGTERM', shutdown)
20+
21+
const main = async () => {
22+
log.info('dockerhub-sync starting...')
23+
24+
const config = getDockerhubConfig()
25+
const appConfig = getGithubAppConfig()
26+
27+
const installationIds = await resolveInstallations(appConfig)
28+
29+
if (installationIds.length === 0) {
30+
log.error('No GitHub App installations found — cannot build token pool')
31+
process.exit(1)
32+
}
33+
34+
await fetchRateLimitDiagnostics(appConfig.appId, appConfig.privateKeyPem, installationIds)
35+
36+
const qx = await getPackagesDb()
37+
await qx.selectOne('SELECT 1')
38+
log.info('Connected to packages-db.')
39+
40+
log.info(
41+
{
42+
installations: installationIds.length,
43+
batchSize: config.batchSize,
44+
hubBaseUrl: config.hubBaseUrl,
45+
},
46+
'Starting dockerhub loop',
47+
)
48+
49+
await runDockerhubLoop(qx, installationIds, appConfig, config, () => shuttingDown)
50+
51+
log.info('dockerhub-sync stopped.')
52+
process.exit(0)
53+
}
54+
55+
main().catch((err) => {
56+
log.error({ err }, 'dockerhub-sync fatal error')
57+
process.exit(1)
58+
})

services/apps/packages_worker/src/config.ts

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,3 +56,13 @@ export function getMavenConfig() {
5656
groupDelayMs: requireEnvInt('MAVEN_FETCHER_GROUP_DELAY_MS'),
5757
}
5858
}
59+
60+
export function getDockerhubConfig() {
61+
return {
62+
hubBaseUrl: requireEnv('DOCKERHUB_API_BASE_URL'),
63+
batchSize: requireEnvInt('DOCKERHUB_BATCH_SIZE'),
64+
refreshIntervalHours: requireEnvInt('DOCKERHUB_REFRESH_INTERVAL_HOURS'),
65+
discoveryIntervalDays: requireEnvInt('DOCKERHUB_DISCOVERY_INTERVAL_DAYS'),
66+
idleSleepSec: requireEnvInt('DOCKERHUB_IDLE_SLEEP_SEC'),
67+
}
68+
}
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
import { describe, expect, it } from 'vitest'
2+
3+
import { buildCandidates } from '../candidates'
4+
5+
describe('buildCandidates', () => {
6+
it('lowercases owner and repo into a single <owner>/<repo> candidate', () => {
7+
expect(buildCandidates('Grafana', 'Grafana')).toEqual(['grafana/grafana'])
8+
})
9+
10+
it('passes through already-valid lowercase slugs', () => {
11+
expect(buildCandidates('prometheus', 'node_exporter')).toEqual(['prometheus/node_exporter'])
12+
})
13+
14+
it('rejects components with characters Docker Hub does not accept', () => {
15+
// GitHub allows '+' in org names via renames; Hub would 400.
16+
expect(buildCandidates('foo+bar', 'baz')).toEqual([])
17+
})
18+
19+
it('rejects components that start or end with a separator', () => {
20+
expect(buildCandidates('-leading', 'repo')).toEqual([])
21+
expect(buildCandidates('owner', 'trailing.')).toEqual([])
22+
})
23+
24+
it('does not emit a library/<repo> candidate', () => {
25+
// Guard against accidental reintroduction — see comment in candidates.ts.
26+
expect(buildCandidates('nodejs', 'node')).toEqual(['nodejs/node'])
27+
})
28+
})
Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
import { afterEach, describe, expect, it, vi } from 'vitest'
2+
3+
import { fetchDockerhub } from '../fetchDockerhub'
4+
import { FetchError } from '../types'
5+
6+
const BASE = 'https://hub.docker.com/v2'
7+
8+
function mockFetch(status: number, body: unknown, headers: Record<string, string> = {}) {
9+
return vi.spyOn(globalThis, 'fetch').mockResolvedValue(
10+
new Response(typeof body === 'string' ? body : JSON.stringify(body), {
11+
status,
12+
headers: { 'Content-Type': 'application/json', ...headers },
13+
}),
14+
)
15+
}
16+
17+
afterEach(() => {
18+
vi.restoreAllMocks()
19+
})
20+
21+
describe('fetchDockerhub', () => {
22+
it('returns pull/star counts on 200', async () => {
23+
mockFetch(200, {
24+
name: 'grafana',
25+
namespace: 'grafana',
26+
pull_count: 12345,
27+
star_count: 678,
28+
last_updated: '2026-05-01T00:00:00Z',
29+
})
30+
31+
const r = await fetchDockerhub(BASE, 'grafana/grafana')
32+
expect(r).toEqual({
33+
imageName: 'grafana/grafana',
34+
pulls: 12345,
35+
stars: 678,
36+
lastUpdated: '2026-05-01T00:00:00Z',
37+
})
38+
})
39+
40+
it('appends trailing slash to the request URL', async () => {
41+
const spy = mockFetch(200, { pull_count: 1, star_count: 0 })
42+
await fetchDockerhub(BASE, 'a/b')
43+
expect(spy).toHaveBeenCalledWith(`${BASE}/repositories/a/b/`, expect.anything())
44+
})
45+
46+
it('normalizes a trailing slash on the configured base URL', async () => {
47+
const spy = mockFetch(200, { pull_count: 1, star_count: 0 })
48+
await fetchDockerhub(`${BASE}/`, 'a/b')
49+
expect(spy).toHaveBeenCalledWith(`${BASE}/repositories/a/b/`, expect.anything())
50+
})
51+
52+
it('classifies 404 as NOT_FOUND', async () => {
53+
mockFetch(404, { message: 'object not found' })
54+
await expect(fetchDockerhub(BASE, 'a/b')).rejects.toMatchObject({ kind: 'NOT_FOUND' })
55+
})
56+
57+
it('classifies 400 as NOT_FOUND (Hub 400s on malformed slugs)', async () => {
58+
mockFetch(400, { message: 'bad request' })
59+
await expect(fetchDockerhub(BASE, 'a/b')).rejects.toMatchObject({ kind: 'NOT_FOUND' })
60+
})
61+
62+
it('classifies 429 as RATE_LIMIT with resetAt from header', async () => {
63+
const resetSec = Math.floor(Date.now() / 1000) + 120
64+
mockFetch(429, { message: 'too many' }, { 'x-ratelimit-reset': String(resetSec) })
65+
66+
expect.assertions(3)
67+
try {
68+
await fetchDockerhub(BASE, 'a/b')
69+
} catch (err) {
70+
expect(err).toBeInstanceOf(FetchError)
71+
const fe = err as FetchError
72+
expect(fe.kind).toBe('RATE_LIMIT')
73+
expect(fe.resetAt).toBeGreaterThan(Date.now())
74+
}
75+
})
76+
77+
it('does NOT discard a 200 response when x-ratelimit-remaining is 0', async () => {
78+
// remaining=0 means this request consumed the last slot; the response itself
79+
// is valid. The next call will 429 and park then.
80+
mockFetch(200, { pull_count: 1, star_count: 0 }, { 'x-ratelimit-remaining': '0' })
81+
const r = await fetchDockerhub(BASE, 'a/b')
82+
expect(r.pulls).toBe(1)
83+
})
84+
85+
it('classifies 401/403 as AUTH so misconfig surfaces instead of looking like a miss', async () => {
86+
mockFetch(401, { message: 'unauthorized' })
87+
await expect(fetchDockerhub(BASE, 'a/b')).rejects.toMatchObject({ kind: 'AUTH' })
88+
mockFetch(403, { message: 'forbidden' })
89+
await expect(fetchDockerhub(BASE, 'a/b')).rejects.toMatchObject({ kind: 'AUTH' })
90+
})
91+
92+
it('classifies 5xx as TRANSIENT', async () => {
93+
mockFetch(503, 'Service Unavailable')
94+
await expect(fetchDockerhub(BASE, 'a/b')).rejects.toMatchObject({ kind: 'TRANSIENT' })
95+
})
96+
97+
it('classifies network failure as TRANSIENT', async () => {
98+
vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('ECONNRESET'))
99+
await expect(fetchDockerhub(BASE, 'a/b')).rejects.toMatchObject({ kind: 'TRANSIENT' })
100+
})
101+
102+
it('classifies non-JSON 200 body as MALFORMED', async () => {
103+
mockFetch(200, '<html>not json</html>')
104+
await expect(fetchDockerhub(BASE, 'a/b')).rejects.toMatchObject({ kind: 'MALFORMED' })
105+
})
106+
107+
it('classifies missing pull_count as MALFORMED', async () => {
108+
mockFetch(200, { name: 'x', star_count: 0 })
109+
await expect(fetchDockerhub(BASE, 'a/b')).rejects.toMatchObject({ kind: 'MALFORMED' })
110+
})
111+
})
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
// Docker Hub repository slugs are lowercase and limited to [a-z0-9._-].
2+
// GitHub allows uppercase and a few characters Hub rejects, so we lowercase
3+
// and validate before probing — anything that fails the regex would 400 on
4+
// Hub anyway and isn't worth an HTTP round-trip.
5+
const HUB_COMPONENT = /^[a-z0-9](?:[a-z0-9._-]*[a-z0-9])?$/
6+
7+
export function buildCandidates(owner: string, name: string): string[] {
8+
const ns = owner.toLowerCase()
9+
const repo = name.toLowerCase()
10+
11+
if (!HUB_COMPONENT.test(ns) || !HUB_COMPONENT.test(repo)) {
12+
return []
13+
}
14+
15+
// v1 deliberately omits `library/<repo>`: a random github.com/foo/node with a
16+
// dev Dockerfile would false-positive onto the official library/node image.
17+
// Official images (~150) to be seeded via allowlist in a follow-up.
18+
return [`${ns}/${repo}`]
19+
}

0 commit comments

Comments
 (0)