feat: add rubygems dependent-counts closure variant (CM-1241) (#4329)

Signed-off-by: Uroš Marolt <uros@marolt.me>

Author: themarolt

docs/adr/0004-go-nuget-transitive-dependent-counts.md

@@ -97,4 +97,14 @@ methodology. We chose exact over an HLL-sketch approximation, accepting the high
 - **Weekly trigger.** No new schedule — the `go`/`nuget` variants are already child workflows of
   `bootstrapOsspckgs`, which the existing `osspckgs-bootstrap-weekly` schedule runs incrementally.
 
+## Addendum: RUBYGEMS (2026-07-13)
+RubyGems is also absent from deps.dev's `Dependents` reverse index (verified via BQ 2026-07-13:
+`Dependents` contains only NPM/MAVEN/PYPI/CARGO) and has no resolved/transitive graph table, only
+the raw manifest `RubyGemsRequirementsLatest` — same shape as GO/NUGET. It was added as a third
+`isScript` closure variant (`dependent_counts_rubygems`), reusing this ADR's design unchanged:
+same semi-naive fixpoint, same `isScript` pipeline integration, same guard/monitor pattern. Corpus
+is NUGET-scale (1.7M pkgs / 4.5M runtime edges), so cost/runtime are expected to track NUGET, not
+GO. This ADR's title and body still say "GO/NUGET" for the original decision; treat this addendum
+as extending scope to a third ecosystem rather than superseding the analysis above.
+
 Related: ADR-0003 (deps BQ table selection — same root cause: GO/NUGET absent from the resolved-graph tables).

services/apps/packages_worker/src/deps-dev/README.md

@@ -7,11 +7,33 @@ import { getPackagesDb } from '../../db'
 export interface CheckDependentCountsGuardInput {
   currentRowCount: number
   snapshotDate: string
-  // Which dependent-counts job kind to baseline against. The three ways each guard against their
-  // own history: 'dependent_counts' (edges) | 'dependent_counts_go' | 'dependent_counts_nuget'.
+  // Which dependent-counts job kind to baseline against. The four ways each guard against their
+  // own history: 'dependent_counts' (edges) | 'dependent_counts_go' | 'dependent_counts_nuget'
+  // | 'dependent_counts_rubygems'.
   jobKind?: OsspckgsJobKind
 }
 
+// Source table to point operators at in the Slack alert, keyed by job kind — the edge variant reads
+// deps.dev's `Dependents` reverse index, while GO/NUGET/RUBYGEMS invert their own manifest table
+// (they're absent from `Dependents`, see ADR-0004). A hardcoded "check Dependents" message would send
+// on-call to the wrong table for those three.
+const GUARD_SOURCE_TABLE: Partial<Record<OsspckgsJobKind, string>> = {
+  dependent_counts: 'Dependents',
+  dependent_counts_go: 'GoRequirementsLatest',
+  dependent_counts_nuget: 'NuGetRequirementsLatest',
+  dependent_counts_rubygems: 'RubyGemsRequirementsLatest',
+}
+
+// `Dependents` is snapshotted per-date, so "for this snapshot date" is actionable there. The
+// GO/NUGET/RubyGems `*RequirementsLatest` views have no snapshot history (see bootstrapOsspckgs.ts,
+// RETENTION_DAYS_BY_KIND) — telling an operator to check "this snapshot date" on those is misleading.
+const GUARD_HAS_SNAPSHOT_HISTORY: Partial<Record<OsspckgsJobKind, boolean>> = {
+  dependent_counts: true,
+  dependent_counts_go: false,
+  dependent_counts_nuget: false,
+  dependent_counts_rubygems: false,
+}
+
 export interface CheckDependentCountsGuardOutput {
   ok: boolean
   prevRowCount: number | null
@@ -49,7 +71,7 @@ export async function checkDependentCountsGuard(
         },
         {
           title: 'Action',
-          text: 'Ingest aborted — existing `dependent_count` values preserved. Check deps.dev `Dependents` table for this snapshot date.',
+          text: `Ingest aborted — existing \`dependent_count\` values preserved. Check deps.dev \`${GUARD_SOURCE_TABLE[jobKind] ?? 'Dependents'}\` table${(GUARD_HAS_SNAPSHOT_HISTORY[jobKind] ?? true) ? ' for this snapshot date' : ''}.`,
         },
       ],
     )

services/apps/packages_worker/src/deps-dev/activities/checkDependentCountsGuard.ts

@@ -47,11 +47,12 @@ GROUP BY pm.purl
 `
 }
 
-// GO/NUGET reverse-dependent counts. deps.dev publishes NO reverse-dependent or transitive graph for
-// these two ecosystems (absent from Dependents/Dependencies/DependencyGraphEdges — see ADR-0004), so
-// we compute the EXACT reverse transitive closure ourselves from the direct manifests it DOES publish
-// (GoRequirementsLatest / NuGetRequirementsLatest). The "Latest" tables hold the highest release per
-// package, matching the DependentIsHighestReleaseWithResolution semantics of the edge query.
+// GO/NUGET/RUBYGEMS reverse-dependent counts. deps.dev publishes NO reverse-dependent or transitive
+// graph for these three ecosystems (absent from Dependents/Dependencies/DependencyGraphEdges — see
+// ADR-0004), so we compute the EXACT reverse transitive closure ourselves from the direct manifests
+// it DOES publish (GoRequirementsLatest / NuGetRequirementsLatest / RubyGemsRequirementsLatest). The
+// "Latest" tables hold the highest release per package, matching the
+// DependentIsHighestReleaseWithResolution semantics of the edge query.
 //
 // Unlike the edge query (a single SELECT), this is a multi-statement BQ SCRIPT: a semi-naive fixpoint
 // over session-scoped TEMP tables, run via bqExportToGcs's isScript mode. It ends by creating
@@ -68,7 +69,7 @@ const MAX_CLOSURE_ITERATIONS = 60
 // Builds the full closure script for one ecosystem. `edgesSql` must SELECT DISTINCT a `dep` (requirer)
 // and `subj` (depended-upon) name column from that ecosystem's manifest table.
 function buildClosureScript(
-  system: 'GO' | 'NUGET',
+  system: 'GO' | 'NUGET' | 'RUBYGEMS',
   edgesSql: string,
   snapshotDate: string,
 ): string {
@@ -194,3 +195,16 @@ UNNEST(grp.Dependencies) AS dp
 WHERE dp.Name NOT LIKE '%>%' AND n.Name NOT LIKE '%>%' AND n.Name != dp.Name`
   return buildClosureScript('NUGET', edgesSql, snapshotDate)
 }
+
+// RubyGems has no reverse-dependent/transitive graph in deps.dev either (Dependents contains only
+// {NPM, MAVEN, PYPI, CARGO} — verified via BQ 2026-07-13), so — like GO/NUGET — the exact reverse
+// transitive closure is computed from the runtime-dependency manifest (RubyGemsRequirementsLatest),
+// matching the RuntimeDependencies-only scope used for the forward package_dependencies ingest.
+export function buildRubygemsDependentCountsSql(snapshotDate: string): string {
+  const edgesSql = `
+SELECT DISTINCT r.Name AS dep, d.Name AS subj
+FROM \`bigquery-public-data.deps_dev_v1.RubyGemsRequirementsLatest\` r,
+UNNEST(r.RuntimeDependencies) AS d
+WHERE d.Name NOT LIKE '%>%' AND r.Name NOT LIKE '%>%' AND r.Name != d.Name`
+  return buildClosureScript('RUBYGEMS', edgesSql, snapshotDate)
+}

services/apps/packages_worker/src/deps-dev/queries/dependentCountsSql.ts

@@ -35,10 +35,12 @@ type JobKind =
   | 'dependent_counts'
   | 'dependent_counts_go'
   | 'dependent_counts_nuget'
+  | 'dependent_counts_rubygems'
 
 // deps.dev retains weekly snapshots for ~3 years; 1095 days (3 years) gives comfortable headroom.
 // advisories/advisory_packages use AdvisoriesLatest (no partition history) → effectively unlimited.
-// dependent_counts_go/_nuget read *RequirementsLatest (no partition history) → effectively unlimited.
+// dependent_counts_go/_nuget/_rubygems read *RequirementsLatest (no partition history) → effectively
+// unlimited.
 const RETENTION_DAYS_BY_KIND: Record<JobKind, number> = {
   packages: 1095,
   repos: 1095,
@@ -50,6 +52,7 @@ const RETENTION_DAYS_BY_KIND: Record<JobKind, number> = {
   dependent_counts: 1095,
   dependent_counts_go: 999_999,
   dependent_counts_nuget: 999_999,
+  dependent_counts_rubygems: 999_999,
 }
 
 // Kinds whose incremental diff is driven by a BQ partition snapshot date.
@@ -204,13 +207,14 @@ export async function bootstrapOsspckgs(opts: {
       }
     }
   }
-  // GO/NUGET reverse-dependent counts: separate kinds, manifest-sourced (GoRequirementsLatest /
-  // NuGetRequirementsLatest), computed via the exact reverse transitive closure script. The manifests
+  // GO/NUGET/RUBYGEMS reverse-dependent counts: separate kinds, manifest-sourced (GoRequirementsLatest /
+  // NuGetRequirementsLatest / RubyGemsRequirementsLatest), computed via the exact reverse transitive
+  // closure script. The manifests
   // are *Latest views (no resolution needed); `today` is the snapshot_at stamp AND the anchor for the
   // dependent_repos partition window (latest PackageVersionToProject snapshot within 60 days). Each
   // guards against its own history and merges a disjoint purl space, so an edge-snapshot corruption
   // that aborts `dependent_counts` never blocks these.
-  for (const variant of ['go', 'nuget'] as const) {
+  for (const variant of ['go', 'nuget', 'rubygems'] as const) {
     const kind = `dependent_counts_${variant}` as const
     if (!runs(kind)) continue
     try {

services/apps/packages_worker/src/deps-dev/workflows/bootstrapOsspckgs.ts

@@ -7,6 +7,7 @@ import {
   buildDependentCountsSql,
   buildGoDependentCountsSql,
   buildNugetDependentCountsSql,
+  buildRubygemsDependentCountsSql,
 } from '../queries/dependentCountsSql'
 
 const { bqExportToGcs } = proxyActivities<typeof depsDevActivities>({
@@ -40,18 +41,19 @@ const { setJobStep } = proxyActivities<typeof depsDevActivities>({
   retry: { maximumAttempts: 3 },
 })
 
-// The dependent-counts ingest runs three independent ways, one per source of reverse-dependent data:
-//   - 'edges' : NPM/MAVEN/PYPI/CARGO from the deps.dev `Dependents` reverse index (single SELECT)
-//   - 'go'    : GO from the GoRequirementsLatest exact reverse transitive closure (BQ script)
-//   - 'nuget' : NUGET from the NuGetRequirementsLatest exact reverse transitive closure (BQ script)
-// All three produce the same three columns (dependent_count, transitive_dependent_count,
+// The dependent-counts ingest runs four independent ways, one per source of reverse-dependent data:
+//   - 'edges'    : NPM/MAVEN/PYPI/CARGO from the deps.dev `Dependents` reverse index (single SELECT)
+//   - 'go'       : GO from the GoRequirementsLatest exact reverse transitive closure (BQ script)
+//   - 'nuget'    : NUGET from the NuGetRequirementsLatest exact reverse transitive closure (BQ script)
+//   - 'rubygems' : RUBYGEMS from the RubyGemsRequirementsLatest exact reverse transitive closure (BQ script)
+// All four produce the same three columns (dependent_count, transitive_dependent_count,
 // dependent_repos_count) — deps.dev gives the edge systems their transitive graph directly, while
-// GO/NUGET compute it via the semi-naive closure script (isScript, see ADR-0004 + dependentCountsSql).
-// Each way is its own job kind, staging table, guard baseline, and purl-disjoint merge — so a corrupt
-// edge snapshot aborts only the edge way and the GO/NUGET counts (manifest-sourced, unaffected) still
-// update. 'edges' keeps the original `dependent_counts` kind so existing job history / monitor / guard
-// baseline stay continuous.
-export type DependentCountsVariant = 'edges' | 'go' | 'nuget'
+// GO/NUGET/RUBYGEMS compute it via the semi-naive closure script (isScript, see ADR-0004 +
+// dependentCountsSql). Each way is its own job kind, staging table, guard baseline, and purl-disjoint
+// merge — so a corrupt edge snapshot aborts only the edge way and the manifest-sourced counts
+// (unaffected) still update. 'edges' keeps the original `dependent_counts` kind so existing job
+// history / monitor / guard baseline stay continuous.
+export type DependentCountsVariant = 'edges' | 'go' | 'nuget' | 'rubygems'
 
 interface VariantConfig {
   jobKind: OsspckgsJobKind
@@ -61,10 +63,11 @@ interface VariantConfig {
   pgColumns: string[]
   maxBytesGb: number
   buildSql: (snapshotDate: string) => string
-  // When true, buildSql returns a multi-statement BQ script (the GO/NUGET exact reverse transitive
-  // closure) that ends by creating TEMP TABLE _export_data, rather than a single SELECT. The export
-  // activity then appends only EXPORT DATA and enforces the byte ceiling via maximumBytesBilled
-  // instead of a dry-run. See ADR-0004. Unset (single-SELECT) for the edges variant; set for GO/NUGET.
+  // When true, buildSql returns a multi-statement BQ script (the GO/NUGET/RUBYGEMS exact reverse
+  // transitive closure) that ends by creating TEMP TABLE _export_data, rather than a single SELECT.
+  // The export activity then appends only EXPORT DATA and enforces the byte ceiling via
+  // maximumBytesBilled instead of a dry-run. See ADR-0004. Unset (single-SELECT) for the edges
+  // variant; set for GO/NUGET/RUBYGEMS.
   isScript?: boolean
 }
 
@@ -73,8 +76,9 @@ interface VariantConfig {
 const EDGES_STAGING = 'staging.osspckgs_dependent_counts_raw'
 const GO_STAGING = 'staging.osspckgs_dependent_counts_go_raw'
 const NUGET_STAGING = 'staging.osspckgs_dependent_counts_nuget_raw'
+const RUBYGEMS_STAGING = 'staging.osspckgs_dependent_counts_rubygems_raw'
 
-// All three ways now produce the same shape: purl + the three count columns. Each variant has its own
+// All four ways now produce the same shape: purl + the three count columns. Each variant has its own
 // staging table (parallel-safe) but identical DDL/merge; the merges touch disjoint purl spaces (one
 // per ecosystem), so they never collide.
 const PG_COLUMNS = [
@@ -116,11 +120,11 @@ const VARIANTS: Record<DependentCountsVariant, VariantConfig> = {
     maxBytesGb: 2000,
     buildSql: (snapshotDate) => buildDependentCountsSql(snapshotDate),
   },
-  // GO/NUGET run the exact reverse transitive closure script (isScript). maxBytesGb is the
-  // server-side maximumBytesBilled runaway cap, set well above the validated full-pipeline spend
-  // (GO 2.31 TB incl. the all-depth repos aggregation; NUGET ~32 GB) so a normal week never trips
-  // it — the iteration cap inside the script is the deterministic guard. Env-overridable via
-  // BQ_DATASET_INGEST_DEPENDENT_COUNTS_GO_MAX_BQ_GB / _NUGET_.
+  // GO/NUGET/RUBYGEMS run the exact reverse transitive closure script (isScript). maxBytesGb is
+  // the server-side maximumBytesBilled runaway cap, set well above the validated full-pipeline
+  // spend (GO 2.31 TB incl. the all-depth repos aggregation; NUGET ~32 GB) so a normal week never
+  // trips it — the iteration cap inside the script is the deterministic guard. Env-overridable via
+  // BQ_DATASET_INGEST_DEPENDENT_COUNTS_GO_MAX_BQ_GB / _NUGET_ / _RUBYGEMS_.
   go: {
     jobKind: 'dependent_counts_go',
     stagingTable: GO_STAGING,
@@ -141,6 +145,18 @@ const VARIANTS: Record<DependentCountsVariant, VariantConfig> = {
     buildSql: (snapshotDate) => buildNugetDependentCountsSql(snapshotDate),
     isScript: true,
   },
+  // RubyGems: same shape as NUGET (manifest-sourced closure), similarly small corpus
+  // (~1.7M pkgs / ~4.5M runtime edges — verified via BQ 2026-07-13).
+  rubygems: {
+    jobKind: 'dependent_counts_rubygems',
+    stagingTable: RUBYGEMS_STAGING,
+    stagingDdl: stagingDdl(RUBYGEMS_STAGING),
+    mergeSql: dependentCountsMerge(RUBYGEMS_STAGING),
+    pgColumns: PG_COLUMNS,
+    maxBytesGb: 200,
+    buildSql: (snapshotDate) => buildRubygemsDependentCountsSql(snapshotDate),
+    isScript: true,
+  },
 }
 
 const ROWS_PER_CHUNK = 1_000_000

services/apps/packages_worker/src/deps-dev/workflows/ingestDependentCounts.ts

@@ -248,6 +248,7 @@ const KIND_TABLES: Record<string, string[]> = {
   dependent_counts: ['packages'],
   dependent_counts_go: ['packages'],
   dependent_counts_nuget: ['packages'],
+  dependent_counts_rubygems: ['packages'],
   ranking: ['packages'],
   pypi_downloads_30d: ['downloads_last_30d', 'packages'],
   pypi_downloads_daily: ['downloads_daily'],

services/apps/packages_worker/src/scripts/monitorOsspckgs.ts

@@ -13,6 +13,7 @@ const VALID_KINDS = [
   'dependent_counts',
   'dependent_counts_go',
   'dependent_counts_nuget',
+  'dependent_counts_rubygems',
   'scorecard',
 ] as const
 
@@ -29,8 +30,9 @@ Options:
                            ${VALID_KINDS.join(', ')}
                          "repos" and "package_repos" always run together (one workflow).
                          "advisories" and "advisory_packages" always run together (one workflow).
-                         "dependent_counts" = edges (NPM/MAVEN/PYPI/CARGO) only; GO and NUGET
-                         reverse counts run as "dependent_counts_go" / "dependent_counts_nuget".
+                         "dependent_counts" = edges (NPM/MAVEN/PYPI/CARGO) only; GO, NUGET and
+                         RUBYGEMS reverse counts run as "dependent_counts_go" /
+                         "dependent_counts_nuget" / "dependent_counts_rubygems".
   --snapshot-date DATE   Override BQ snapshot resolution for all partition-filtered kinds.
                          Use YYYY-MM-DD. Skips resolveSnapshotDate BQ call and uses this date
                          directly. Useful for re-running with a known-good older snapshot.

services/apps/packages_worker/src/scripts/triggerBootstrap.ts

@@ -11,6 +11,7 @@ export type OsspckgsJobKind =
   | 'dependent_counts'
   | 'dependent_counts_go'
   | 'dependent_counts_nuget'
+  | 'dependent_counts_rubygems'
   | 'scorecard_repos'
   | 'scorecard_checks'
   | 'ranking'