Skip to content

Commit 9f6f068

Browse files
authored
feat: add rubygems and Go support to osv worker [CM-1241] (#4333)
Signed-off-by: Mouad BANI <mouad-mb@outlook.com>
1 parent e07f84c commit 9f6f068

4 files changed

Lines changed: 154 additions & 20 deletions

File tree

backend/.env.dist.local

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,7 @@ OSSPCKGS_GCP_CREDENTIALS_B64=e30=
199199
# maven/all.zip 404s). The allowlist check and DB storage normalize to lowercase
200200
# internally per ADR-0001 §OSV "Ecosystem normalization", so downstream stays lowercase.
201201
OSV_BULK_BASE_URL=https://osv-vulnerabilities.storage.googleapis.com
202-
OSV_ECOSYSTEMS=npm,Maven,cargo,NuGet
202+
OSV_ECOSYSTEMS=npm,Maven,cargo,NuGet,RubyGems,Go
203203
OSV_TMP_DIR=/tmp/osv
204204
OSV_BATCH_SIZE=500
205205
OSV_DERIVE_BATCH_SIZE=1000

services/apps/packages_worker/src/osv/__tests__/versionCompare.test.ts

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,56 @@ describe('compareVersion — nuget (semver)', () => {
111111
})
112112
})
113113

114+
describe('compareVersion — go (semver)', () => {
115+
it.each([
116+
['v1.2.3', 'v1.2.4', -1],
117+
['v1.2.4', 'v1.2.3', 1],
118+
['v1.2.3', 'v1.2.3', 0],
119+
['v1.10.0', 'v1.9.0', 1], // numeric, not lex
120+
['v1.0.0-alpha', 'v1.0.0', -1], // prerelease < release
121+
['v0.0.0-20220314234659-1baeb1ce4c0b', 'v0.0.0-20220315000000-2caec2d5d1c1', -1], // pseudo-versions
122+
])('compareVersion("go", %s, %s) sign = %s', (a, b, expected) => {
123+
expect(sign(compareVersion('go', a, b))).toBe(expected)
124+
})
125+
126+
it('returns null for unparseable go versions', () => {
127+
expect(compareVersion('go', 'not-a-version', 'v1.0.0')).toBeNull()
128+
})
129+
130+
it('rejects titlecase "Go" — production storage is always lowercase', () => {
131+
expect(compareVersion('Go', 'v1.0.0', 'v2.0.0')).toBeNull()
132+
})
133+
})
134+
135+
describe('compareVersion — rubygems (Gem::Version-style)', () => {
136+
it.each([
137+
['1.0.0', '1.0.1', -1],
138+
['1.0.1', '1.0.0', 1],
139+
['1.0.0', '1.0.0', 0],
140+
['1.10.0', '1.9.0', 1], // numeric, not lex
141+
['1.0', '1.0.0', 0], // missing trailing segment pads as 0
142+
['1.0.pre', '1.0', -1], // prerelease sorts below the corresponding release
143+
['1.0.0.rc1', '1.0.0.rc2', -1],
144+
// rack CVE-2022-30123 boundary
145+
['2.2.3', '2.2.3.1', -1],
146+
['1.0-1', '1.0.pre.1', 0], // hyphen is a prerelease marker, same as ".pre."
147+
['1.0.a10', '1.0.a9', 1], // letter+digit runs split: a10 -> a.10, a9 -> a.9
148+
['1.99999999999999999999999999999999', '1.99999999999999999999999999999998', 1], // beyond Number.MAX_SAFE_INTEGER — must not collapse to equal
149+
])('compareVersion("rubygems", %s, %s) sign = %s', (a, b, expected) => {
150+
expect(sign(compareVersion('rubygems', a, b))).toBe(expected)
151+
})
152+
153+
it('returns null for unparseable rubygems versions', () => {
154+
expect(compareVersion('rubygems', '', '1.0.0')).toBeNull()
155+
expect(compareVersion('rubygems', '---', '1.0.0')).toBeNull()
156+
expect(compareVersion('rubygems', '1..2', '1.0.0')).toBeNull()
157+
})
158+
159+
it('rejects titlecase "RubyGems" — production storage is always lowercase', () => {
160+
expect(compareVersion('RubyGems', '1.0.0', '2.0.0')).toBeNull()
161+
})
162+
})
163+
114164
describe('compareVersion — unsupported ecosystems', () => {
115165
it('returns null for ecosystems we have no comparator for', () => {
116166
expect(compareVersion('PyPI', '1.0.0', '2.0.0')).toBeNull()

services/apps/packages_worker/src/osv/schedule.ts

Lines changed: 32 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ const SCHEDULE_ID = 'osv-advisories-sync'
1111
// validate the env input against this list and refuse to register the
1212
// schedule on a mismatch — better a loud startup error than a silent miss.
1313
// Add new entries here when v1 expands beyond npm + Maven.
14-
const VALID_ECOSYSTEMS = ['npm', 'Maven', 'cargo', 'NuGet'] as const
14+
const VALID_ECOSYSTEMS = ['npm', 'Maven', 'cargo', 'NuGet', 'RubyGems', 'Go'] as const
1515

1616
function getEcosystems(): string[] {
1717
const raw = process.env.OSV_ECOSYSTEMS
@@ -35,8 +35,27 @@ function getEcosystems(): string[] {
3535
return deduped
3636
}
3737

38+
function scheduleAction() {
39+
return {
40+
type: 'startWorkflow' as const,
41+
workflowType: osvSync,
42+
taskQueue: 'osv-worker',
43+
// Headroom for npm (~1 hour today) + Maven (~5 minutes) + derive
44+
// (~5 minutes for 600-700k packages); 4 hours leaves space for the
45+
// upsertOne N+1 deferred fix being slower than expected.
46+
workflowExecutionTimeout: '4 hours',
47+
retry: {
48+
initialInterval: '30 seconds',
49+
backoffCoefficient: 2,
50+
maximumAttempts: 3,
51+
},
52+
args: [{ ecosystems: getEcosystems() }] as [{ ecosystems: string[] }],
53+
}
54+
}
55+
3856
// Registers the daily OSV advisory sync schedule if it doesn't already exist.
39-
// If the schedule already exists in Temporal, we log and leave it unchanged (no update).
57+
// If the schedule already exists in Temporal, we reconcile its action so a
58+
// changed OSV_ECOSYSTEMS env var reaches an already-running schedule on restart.
4059
// Cron is offset from npm-registry-ingest (`15 3 * * *`) so the two large daily ingest jobs don't fight for the same DB at the same minute.
4160
export async function scheduleOsvSync(): Promise<void> {
4261
const { temporal } = svc
@@ -55,25 +74,20 @@ export async function scheduleOsvSync(): Promise<void> {
5574
overlap: ScheduleOverlapPolicy.SKIP,
5675
catchupWindow: '1 hour',
5776
},
58-
action: {
59-
type: 'startWorkflow',
60-
workflowType: osvSync,
61-
taskQueue: 'osv-worker',
62-
// Headroom for npm (~1 hour today) + Maven (~5 minutes) + derive
63-
// (~5 minutes for 600-700k packages); 4 hours leaves space for the
64-
// upsertOne N+1 deferred fix being slower than expected.
65-
workflowExecutionTimeout: '4 hours',
66-
retry: {
67-
initialInterval: '30 seconds',
68-
backoffCoefficient: 2,
69-
maximumAttempts: 3,
70-
},
71-
args: [{ ecosystems: getEcosystems() }],
72-
},
77+
action: scheduleAction(),
7378
})
7479
} catch (err) {
7580
if (err instanceof ScheduleAlreadyRunning) {
76-
svc.log.info(`Schedule ${SCHEDULE_ID} already registered.`)
81+
svc.log.info(`Schedule ${SCHEDULE_ID} already exists, reconciling action.`)
82+
const handle = temporal.schedule.getHandle(SCHEDULE_ID)
83+
await handle.update((prev) => ({
84+
...prev,
85+
policies: {
86+
...prev.policies,
87+
overlap: ScheduleOverlapPolicy.SKIP,
88+
},
89+
action: scheduleAction(),
90+
}))
7791
} else {
7892
throw err
7993
}

services/apps/packages_worker/src/osv/versionCompare.ts

Lines changed: 71 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,13 +126,83 @@ function compareMaven(a: string, b: string): number | null {
126126
return 0
127127
}
128128

129-
const SEMVER_ECOSYSTEMS = new Set(['npm', 'cargo', 'nuget'])
129+
// Mirrors Gem::Version::VERSION_PATTERN / ANCHORED_VERSION_PATTERN — a hyphen
130+
// only ever introduces a single trailing prerelease group.
131+
const RUBYGEMS_VERSION_PATTERN =
132+
/^\s*[0-9]+(\.[0-9a-zA-Z]+)*(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?\s*$/
133+
134+
// Splits a version into alternating digit/letter runs, per Gem::Version#segments.
135+
// A hyphen is a prerelease marker (Gem::Version#initialize rewrites "-" to
136+
// ".pre." before tokenizing), not a plain separator. Numeric runs use bigint
137+
// so segments beyond Number.MAX_SAFE_INTEGER still compare exactly.
138+
function toRubyGemsSegments(version: string): (bigint | string)[] {
139+
if (!RUBYGEMS_VERSION_PATTERN.test(version)) return []
140+
const normalized = version.replace(/-/g, '.pre.')
141+
const runs = normalized.match(/[0-9]+|[a-zA-Z]+/g) ?? []
142+
return runs.map((run) => (/^[0-9]+$/.test(run) ? BigInt(run) : run))
143+
}
144+
145+
// Mirrors Gem::Version#canonical_segments: drop trailing zero segments, then
146+
// drop any zero segments that sit directly before the first letter segment.
147+
function canonicalRubyGemsSegments(segments: (bigint | string)[]): (bigint | string)[] {
148+
const trimmed = [...segments]
149+
while (trimmed.length > 1 && trimmed[trimmed.length - 1] === BigInt(0)) trimmed.pop()
150+
151+
const firstLetterIndex = trimmed.findIndex((segment) => typeof segment === 'string')
152+
if (firstLetterIndex === -1) return trimmed
153+
154+
let zeroRunStart = firstLetterIndex
155+
while (zeroRunStart > 0 && trimmed[zeroRunStart - 1] === BigInt(0)) zeroRunStart--
156+
trimmed.splice(zeroRunStart, firstLetterIndex - zeroRunStart)
157+
158+
return trimmed
159+
}
160+
161+
// Compares one pair of segments the way Gem::Version#<=> does: a letter
162+
// segment always sorts below a numeric segment, regardless of its value.
163+
function compareRubyGemsSegment(lhs: bigint | string, rhs: bigint | string): number {
164+
if (typeof lhs === 'string' && typeof rhs !== 'string') return -1
165+
if (typeof lhs !== 'string' && typeof rhs === 'string') return 1
166+
return lhs < rhs ? -1 : lhs > rhs ? 1 : 0
167+
}
168+
169+
// Once the shorter side is exhausted, Gem::Version#<=> walks the longer
170+
// side's remaining segments: a letter segment means the longer side is a
171+
// prerelease of the shorter one (sorts lower); a nonzero number means the
172+
// longer side sorts higher; zeros (already mostly trimmed) are skipped.
173+
function rubyGemsTailSign(segments: (bigint | string)[], startIndex: number): number {
174+
for (let i = startIndex; i < segments.length; i++) {
175+
const segment = segments[i]
176+
if (typeof segment === 'string') return -1
177+
if (segment !== BigInt(0)) return 1
178+
}
179+
return 0
180+
}
181+
182+
function compareRubyGems(a: string, b: string): number | null {
183+
const aSegments = canonicalRubyGemsSegments(toRubyGemsSegments(a))
184+
const bSegments = canonicalRubyGemsSegments(toRubyGemsSegments(b))
185+
if (aSegments.length === 0 || bSegments.length === 0) return null
186+
187+
const limit = Math.min(aSegments.length, bSegments.length)
188+
for (let i = 0; i < limit; i++) {
189+
if (aSegments[i] === bSegments[i]) continue
190+
return compareRubyGemsSegment(aSegments[i], bSegments[i])
191+
}
192+
193+
if (aSegments.length > bSegments.length) return rubyGemsTailSign(aSegments, limit)
194+
if (bSegments.length > aSegments.length) return -rubyGemsTailSign(bSegments, limit)
195+
return 0
196+
}
197+
198+
const SEMVER_ECOSYSTEMS = new Set(['npm', 'cargo', 'nuget', 'go'])
130199

131200
// Ecosystem names are stored lowercase in packages-db per ADR-0001 §OSV
132201
// "Ecosystem normalization" — 'npm', 'maven', 'cargo'. Callers (deriveCriticalFlag)
133202
// pull the value straight from the DB so the literals here must match.
134203
export function compareVersion(ecosystem: string, a: string, b: string): number | null {
135204
if (SEMVER_ECOSYSTEMS.has(ecosystem)) return compareSemver(a, b)
136205
if (ecosystem === 'maven') return compareMaven(a, b)
206+
if (ecosystem === 'rubygems') return compareRubyGems(a, b)
137207
return null
138208
}

0 commit comments

Comments
 (0)