From f6249a2137576f26a6edaf959b3e6c6f663e9bf2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Wed, 17 Jun 2026 12:06:59 +0200
Subject: [PATCH 01/29] fix: trying to prevent a deadlock
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 docs/adr/0003-deps-bq-table-selection.md      | 17 ++++++----
 docs/adr/README.md                            |  2 +-
 .../scorecard/workflows/ingestScorecard.ts    | 32 +++++++++++++------
 3 files changed, 34 insertions(+), 17 deletions(-)

diff --git a/docs/adr/0003-deps-bq-table-selection.md b/docs/adr/0003-deps-bq-table-selection.md
index 84a222105d..697f1eb2fb 100644
--- a/docs/adr/0003-deps-bq-table-selection.md
+++ b/docs/adr/0003-deps-bq-table-selection.md
@@ -1,4 +1,4 @@
-# ADR-0003: Use DependencyGraphEdgesLatest for deps ingestion; defer DependenciesLatest until NUGET needed
+# ADR-0003: Use DependencyGraphEdgesLatest for deps ingestion; defer DependenciesLatest until NUGET or GO needed
 
 **Date**: 2026-05-29
 **Status**: accepted
@@ -14,9 +14,9 @@ NPM + MAVEN only.
 ## Decision
 Use `DependencyGraphEdgesLatest` (Option A) with `From.Name = Name AND
 From.Version = Version` as the depth-1 filter. Switch to `DependenciesLatest`
-(Option B) only when NUGET ingestion is required, since Option A does not
-support NUGET. At that point evaluate whether to migrate all ecosystems or
-add NUGET via Option B only.
+(Option B) only when NUGET or GO ingestion is required, since Option A does not
+support either ecosystem. At that point evaluate whether to migrate all ecosystems
+or add GO/NUGET via Option B only.
 
 ## Alternatives Considered
 
@@ -27,7 +27,7 @@ add NUGET via Option B only.
   e.g. `^1.2.3`); only exposes resolved version
 - **Why not**: NPM and MAVEN are the only ecosystems needed now. Option A
   retains `version_constraint` which has security feature value. Cost delta is
-  acceptable for the current scope. Re-evaluate when NUGET is required.
+  acceptable for the current scope. Re-evaluate when NUGET or GO is required.
 
 ## Consequences
 
@@ -36,11 +36,14 @@ add NUGET via Option B only.
 - No migration risk — simpler to stay on the table already coded
 
 ### Negative
-- NUGET not supported; must revisit when NUGET ingestion is added
+- NUGET and GO not supported; must revisit when either ecosystem is added.
+  Confirmed empirically: full bootstrap of GO against `DependencyGraphEdgesLatest`
+  returns 0 rows — deps.dev does not resolve GO dependency graphs in that table
+  (GO uses Minimal Version Selection, a different resolution model from NPM/MAVEN/PYPI/CARGO).
 - Full bootstrap ~$1,291 vs ~$494 for NPM+MAVEN (Option B cheaper)
 - Weekly incremental ~$26 vs ~$7 (Option B cheaper)
 
 ### Risks
-- If NUGET is added before a re-evaluation, the team may miss that Option B
+- If NUGET or GO is added before a re-evaluation, the team may miss that Option B
   is required. Mitigation: this ADR and the note in
   `personal/osspckgs-bq-cost-report.txt` flag the trigger condition.
diff --git a/docs/adr/README.md b/docs/adr/README.md
index 7a4c84f09d..9bb2045dca 100644
--- a/docs/adr/README.md
+++ b/docs/adr/README.md
@@ -9,7 +9,7 @@ Use the `/adr` skill in Claude Code to record new ADRs or query past decisions.
 | ADR                                                 | Title                                    | Status | Date       |
 | --------------------------------------------------- | ---------------------------------------- | ------ | ---------- |
 | [ADR-0001](./0001-oss-packages-design-decisions.md) | OSS packages — design decisions (living) | living | 2026-05-27 |
-| [ADR-0003](./0003-deps-bq-table-selection.md) | Use DependencyGraphEdgesLatest for deps ingestion | accepted | 2026-05-29 |
+| [ADR-0003](./0003-deps-bq-table-selection.md) | Use DependencyGraphEdgesLatest for deps ingestion; defer DependenciesLatest until NUGET or GO needed | accepted | 2026-05-29 |
 
 ## Why ADRs?
 
diff --git a/services/apps/packages_worker/src/scorecard/workflows/ingestScorecard.ts b/services/apps/packages_worker/src/scorecard/workflows/ingestScorecard.ts
index de4b2773a2..8c56fc96eb 100644
--- a/services/apps/packages_worker/src/scorecard/workflows/ingestScorecard.ts
+++ b/services/apps/packages_worker/src/scorecard/workflows/ingestScorecard.ts
@@ -38,21 +38,26 @@ CREATE UNLOGGED TABLE IF NOT EXISTS staging.osspckgs_scorecard_repos_raw (
 // Cast to timestamptz happens in merge SQL.
 const SCORECARD_REPOS_PG_COLUMNS = ['repo_url', 'score', 'scanned_at']
 
+// Two-CTE pattern prevents deadlocks with concurrent repos UPDATE transactions
+// (e.g. github-repos-enricher). `locked` materialises candidates then acquires
+// row-level locks in ascending repos.id order — consistent ordering eliminates
+// the circular-wait condition. The outer UPDATE touches already-locked rows only.
 const SCORECARD_REPOS_MERGE_SQL = `
-UPDATE repos r
-SET scorecard_score = CASE
+WITH candidates AS (
+  SELECT
+    r.id,
+    CASE
       WHEN s.score IS NULL
         OR s.score = 'NaN'::float8
         OR s.score = 'Infinity'::float8
         OR s.score = '-Infinity'::float8
       THEN NULL
       ELSE s.score::numeric(3,1)
-    END,
-    scorecard_last_run_at = s.scanned_at::timestamptz,
-    updated_at = NOW()
-FROM (SELECT * FROM staging.osspckgs_scorecard_repos_raw ORDER BY repo_url) s
-WHERE r.url = s.repo_url
-  AND (
+    END AS new_score,
+    s.scanned_at::timestamptz AS new_scanned_at
+  FROM staging.osspckgs_scorecard_repos_raw s
+  JOIN repos r ON r.url = s.repo_url
+  WHERE
     r.scorecard_score IS DISTINCT FROM CASE
       WHEN s.score IS NULL
         OR s.score = 'NaN'::float8
@@ -62,7 +67,16 @@ WHERE r.url = s.repo_url
       ELSE s.score::numeric(3,1)
     END
     OR r.scorecard_last_run_at IS DISTINCT FROM s.scanned_at::timestamptz
-  )
+),
+locked AS (
+  SELECT * FROM candidates ORDER BY id FOR UPDATE
+)
+UPDATE repos r
+SET scorecard_score       = locked.new_score,
+    scorecard_last_run_at = locked.new_scanned_at,
+    updated_at            = NOW()
+FROM locked
+WHERE r.id = locked.id
 `
 
 const SCORECARD_CHECKS_STAGING_TABLE = 'staging.osspckgs_scorecard_checks_raw'

From eafcb534777c041ea76babb48805d8164d099d3c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Wed, 17 Jun 2026 14:27:51 +0200
Subject: [PATCH 02/29] fix: using proper tables for nuget and go
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 docs/adr/0003-deps-bq-table-selection.md      |   9 +-
 .../src/deps-dev/queries/depsSql.ts           | 331 +++++++++++++++---
 .../deps-dev/workflows/ingestDependencies.ts  |  13 +-
 .../src/scripts/exportToBucket.ts             |   4 +-
 4 files changed, 294 insertions(+), 63 deletions(-)

diff --git a/docs/adr/0003-deps-bq-table-selection.md b/docs/adr/0003-deps-bq-table-selection.md
index 697f1eb2fb..78247575dd 100644
--- a/docs/adr/0003-deps-bq-table-selection.md
+++ b/docs/adr/0003-deps-bq-table-selection.md
@@ -36,10 +36,11 @@ or add GO/NUGET via Option B only.
 - No migration risk — simpler to stay on the table already coded
 
 ### Negative
-- NUGET and GO not supported; must revisit when either ecosystem is added.
-  Confirmed empirically: full bootstrap of GO against `DependencyGraphEdgesLatest`
-  returns 0 rows — deps.dev does not resolve GO dependency graphs in that table
-  (GO uses Minimal Version Selection, a different resolution model from NPM/MAVEN/PYPI/CARGO).
+- Only NPM, MAVEN, PYPI, CARGO are present in `DependencyGraphEdgesLatest`.
+  GO and NUGET are absent entirely — confirmed via BQ query on the live table
+  (`SELECT System, COUNT(*) … GROUP BY System` returns exactly those 4).
+  GO uses Minimal Version Selection (no graph resolution by deps.dev);
+  NUGET is simply not covered. Must revisit when either ecosystem is added.
 - Full bootstrap ~$1,291 vs ~$494 for NPM+MAVEN (Option B cheaper)
 - Weekly incremental ~$26 vs ~$7 (Option B cheaper)
 
diff --git a/services/apps/packages_worker/src/deps-dev/queries/depsSql.ts b/services/apps/packages_worker/src/deps-dev/queries/depsSql.ts
index 1ef88bf138..37aaa3efcd 100644
--- a/services/apps/packages_worker/src/deps-dev/queries/depsSql.ts
+++ b/services/apps/packages_worker/src/deps-dev/queries/depsSql.ts
@@ -1,6 +1,49 @@
-// ADR-0003 Option A: DependencyGraphEdgesLatest — has version_constraint; no NUGET support.
-export function buildDepsFullSqlA(systems: string): string {
-  return `
+// NPM, MAVEN, PYPI, CARGO are in DependencyGraphEdgesLatest / DependenciesLatest.
+// GO uses GoRequirementsLatest (DirectDependencies, no resolved to_version).
+// NUGET uses NuGetRequirementsLatest (DependencyGroups → Dependencies, no resolved to_version).
+// Confirmed via BQ query 2026-06-17: DependencyGraphEdgesLatest and DependenciesLatest
+// both contain exactly {NPM, MAVEN, PYPI, CARGO} — GO and NUGET absent from both.
+
+const EDGE_SYSTEMS = new Set(['NPM', 'MAVEN', 'PYPI', 'CARGO'])
+
+export const DEPS_DEFAULT_ECOSYSTEMS = ['NPM', 'GO', 'MAVEN', 'PYPI', 'NUGET', 'CARGO']
+
+// --- Full SQL helpers ---
+
+const GO_FULL_PART = `
+SELECT
+  'go'                   AS ecosystem,
+  g.Name                 AS root_name,
+  g.Version              AS root_version,
+  d.Name                 AS to_name,
+  CAST(NULL AS STRING)   AS to_version,
+  d.Requirement          AS version_constraint
+FROM \`bigquery-public-data.deps_dev_v1.GoRequirementsLatest\` g,
+UNNEST(g.DirectDependencies) AS d`
+
+// NuGet groups deps by TargetFramework — flatten all groups, dedup handled downstream
+// by DISTINCT ON in MERGE_SQL_FULL and ON CONFLICT in MERGE_SQL.
+const NUGET_FULL_PART = `
+SELECT
+  'nuget'                AS ecosystem,
+  n.Name                 AS root_name,
+  n.Version              AS root_version,
+  dep.Name               AS to_name,
+  CAST(NULL AS STRING)   AS to_version,
+  dep.Requirement        AS version_constraint
+FROM \`bigquery-public-data.deps_dev_v1.NuGetRequirementsLatest\` n,
+UNNEST(n.DependencyGroups) AS grp,
+UNNEST(grp.Dependencies) AS dep`
+
+// ADR-0003 Option A: DependencyGraphEdgesLatest for NPM/MAVEN/PYPI/CARGO — has version_constraint.
+// GO + NUGET always come from their ecosystem-specific tables regardless of option.
+export function buildDepsFullSqlA(ecosystems: string[]): string {
+  const parts: string[] = []
+
+  const edgeSystems = ecosystems.filter((s) => EDGE_SYSTEMS.has(s))
+  if (edgeSystems.length > 0) {
+    const filter = edgeSystems.map((s) => `'${s}'`).join(', ')
+    parts.push(`
 SELECT
   LOWER(e.System) AS ecosystem,
   e.Name          AS root_name,
@@ -9,19 +52,64 @@ SELECT
   e.To.Version    AS to_version,
   e.Requirement   AS version_constraint
 FROM \`bigquery-public-data.deps_dev_v1.DependencyGraphEdgesLatest\` e
-WHERE e.System IN (${systems})
+WHERE e.System IN (${filter})
   AND e.From.Name = e.Name
-  AND e.From.Version = e.Version
-`
+  AND e.From.Version = e.Version`)
+  }
+
+  if (ecosystems.includes('GO')) parts.push(GO_FULL_PART)
+  if (ecosystems.includes('NUGET')) parts.push(NUGET_FULL_PART)
+
+  return parts.join('\nUNION ALL\n')
+}
+
+// ADR-0003 Option B: DependenciesLatest for NPM/MAVEN/PYPI/CARGO — cheaper, no version_constraint.
+// GO + NUGET same as Option A (ecosystem-specific tables, version_constraint available).
+export function buildDepsFullSqlB(ecosystems: string[]): string {
+  const parts: string[] = []
+
+  const depsSystems = ecosystems.filter((s) => EDGE_SYSTEMS.has(s))
+  if (depsSystems.length > 0) {
+    const filter = depsSystems.map((s) => `'${s}'`).join(', ')
+    parts.push(`
+SELECT
+  LOWER(d.System)           AS ecosystem,
+  d.Name                    AS root_name,
+  d.Version                 AS root_version,
+  d.Dependency.Name         AS to_name,
+  d.Dependency.Version      AS to_version,
+  CAST(NULL AS STRING)      AS version_constraint
+FROM \`bigquery-public-data.deps_dev_v1.DependenciesLatest\` d
+WHERE d.System IN (${filter})
+  AND d.MinimumDepth = 1`)
+  }
+
+  if (ecosystems.includes('GO')) parts.push(GO_FULL_PART)
+  if (ecosystems.includes('NUGET')) parts.push(NUGET_FULL_PART)
+
+  return parts.join('\nUNION ALL\n')
 }
 
+// --- Incremental SQL helpers ---
+// Uses base tables (GoRequirements, NuGetRequirements, DependencyGraphEdges) with SnapshotAt filter.
+// All CTEs combined in one WITH clause so UNION ALL can reference them freely.
+
 export function buildDepsIncrementalSqlA(
   today: string,
   watermark: string,
-  systems: string,
+  ecosystems: string[],
 ): string {
-  return `
-WITH today AS (
+  const edgeSystems = ecosystems.filter((s) => EDGE_SYSTEMS.has(s))
+  const includeGo = ecosystems.includes('GO')
+  const includeNuget = ecosystems.includes('NUGET')
+
+  const ctes: string[] = []
+  const selects: string[] = []
+
+  if (edgeSystems.length > 0) {
+    const filter = edgeSystems.map((s) => `'${s}'`).join(', ')
+    ctes.push(
+      `today_edges AS (
   SELECT
     LOWER(e.System) AS ecosystem,
     e.Name          AS root_name,
@@ -32,50 +120,116 @@ WITH today AS (
   FROM \`bigquery-public-data.deps_dev_v1.DependencyGraphEdges\` e
   WHERE e.SnapshotAt >= TIMESTAMP('${today}')
     AND e.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${today}', INTERVAL 1 DAY))
-    AND e.System IN (${systems})
+    AND e.System IN (${filter})
     AND e.From.Name = e.Name AND e.From.Version = e.Version
-),
-last_watermark AS (
+)`,
+      `watermark_edges AS (
   SELECT e.System, e.Name, e.Version, e.To.Name AS to_name, e.To.Version AS to_version
   FROM \`bigquery-public-data.deps_dev_v1.DependencyGraphEdges\` e
   WHERE e.SnapshotAt >= TIMESTAMP('${watermark}')
     AND e.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${watermark}', INTERVAL 1 DAY))
-    AND e.System IN (${systems})
+    AND e.System IN (${filter})
     AND e.From.Name = e.Name AND e.From.Version = e.Version
   GROUP BY e.System, e.Name, e.Version, e.To.Name, e.To.Version
-)
-SELECT t.*
-FROM today t
-LEFT JOIN last_watermark l
+)`,
+    )
+    selects.push(
+      `SELECT t.*
+FROM today_edges t
+LEFT JOIN watermark_edges l
   ON LOWER(l.System) = t.ecosystem AND l.Name = t.root_name AND l.Version = t.root_version
  AND l.to_name = t.to_name AND l.to_version = t.to_version
-WHERE l.to_name IS NULL
-`
-}
+WHERE l.to_name IS NULL`,
+    )
+  }
 
-// ADR-0003 Option B: DependenciesLatest — cheaper (~65%), covers NUGET, but no version_constraint.
-export function buildDepsFullSqlB(systems: string): string {
-  return `
-SELECT
-  LOWER(d.System)           AS ecosystem,
-  d.Name                    AS root_name,
-  d.Version                 AS root_version,
-  d.Dependency.Name         AS to_name,
-  d.Dependency.Version      AS to_version,
-  CAST(NULL AS STRING)      AS version_constraint
-FROM \`bigquery-public-data.deps_dev_v1.DependenciesLatest\` d
-WHERE d.System IN (${systems})
-  AND d.MinimumDepth = 1
-`
+  if (includeGo) {
+    ctes.push(
+      `today_go AS (
+  SELECT
+    'go'                   AS ecosystem,
+    g.Name                 AS root_name,
+    g.Version              AS root_version,
+    d.Name                 AS to_name,
+    CAST(NULL AS STRING)   AS to_version,
+    d.Requirement          AS version_constraint
+  FROM \`bigquery-public-data.deps_dev_v1.GoRequirements\` g,
+  UNNEST(g.DirectDependencies) AS d
+  WHERE g.SnapshotAt >= TIMESTAMP('${today}')
+    AND g.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${today}', INTERVAL 1 DAY))
+)`,
+      `watermark_go AS (
+  SELECT g.Name, g.Version, d.Name AS to_name
+  FROM \`bigquery-public-data.deps_dev_v1.GoRequirements\` g,
+  UNNEST(g.DirectDependencies) AS d
+  WHERE g.SnapshotAt >= TIMESTAMP('${watermark}')
+    AND g.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${watermark}', INTERVAL 1 DAY))
+  GROUP BY g.Name, g.Version, d.Name
+)`,
+    )
+    selects.push(
+      `SELECT t.*
+FROM today_go t
+LEFT JOIN watermark_go l
+  ON l.Name = t.root_name AND l.Version = t.root_version AND l.to_name = t.to_name
+WHERE l.to_name IS NULL`,
+    )
+  }
+
+  if (includeNuget) {
+    ctes.push(
+      `today_nuget AS (
+  SELECT
+    'nuget'                AS ecosystem,
+    n.Name                 AS root_name,
+    n.Version              AS root_version,
+    dep.Name               AS to_name,
+    CAST(NULL AS STRING)   AS to_version,
+    dep.Requirement        AS version_constraint
+  FROM \`bigquery-public-data.deps_dev_v1.NuGetRequirements\` n,
+  UNNEST(n.DependencyGroups) AS grp,
+  UNNEST(grp.Dependencies) AS dep
+  WHERE n.SnapshotAt >= TIMESTAMP('${today}')
+    AND n.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${today}', INTERVAL 1 DAY))
+)`,
+      `watermark_nuget AS (
+  SELECT n.Name, n.Version, dep.Name AS to_name
+  FROM \`bigquery-public-data.deps_dev_v1.NuGetRequirements\` n,
+  UNNEST(n.DependencyGroups) AS grp,
+  UNNEST(grp.Dependencies) AS dep
+  WHERE n.SnapshotAt >= TIMESTAMP('${watermark}')
+    AND n.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${watermark}', INTERVAL 1 DAY))
+  GROUP BY n.Name, n.Version, dep.Name
+)`,
+    )
+    selects.push(
+      `SELECT t.*
+FROM today_nuget t
+LEFT JOIN watermark_nuget l
+  ON l.Name = t.root_name AND l.Version = t.root_version AND l.to_name = t.to_name
+WHERE l.to_name IS NULL`,
+    )
+  }
+
+  return `WITH\n${ctes.join(',\n')}\n${selects.join('\nUNION ALL\n')}`
 }
 
 export function buildDepsIncrementalSqlB(
   today: string,
   watermark: string,
-  systems: string,
+  ecosystems: string[],
 ): string {
-  return `
-WITH today AS (
+  const depsSystems = ecosystems.filter((s) => EDGE_SYSTEMS.has(s))
+  const includeGo = ecosystems.includes('GO')
+  const includeNuget = ecosystems.includes('NUGET')
+
+  const ctes: string[] = []
+  const selects: string[] = []
+
+  if (depsSystems.length > 0) {
+    const filter = depsSystems.map((s) => `'${s}'`).join(', ')
+    ctes.push(
+      `today_deps AS (
   SELECT
     LOWER(d.System)           AS ecosystem,
     d.Name                    AS root_name,
@@ -86,38 +240,111 @@ WITH today AS (
   FROM \`bigquery-public-data.deps_dev_v1.Dependencies\` d
   WHERE d.SnapshotAt >= TIMESTAMP('${today}')
     AND d.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${today}', INTERVAL 1 DAY))
-    AND d.System IN (${systems})
+    AND d.System IN (${filter})
     AND d.MinimumDepth = 1
-),
-last_watermark AS (
+)`,
+      `watermark_deps AS (
   SELECT d.System, d.Name, d.Version, d.Dependency.Name AS to_name, d.Dependency.Version AS to_version
   FROM \`bigquery-public-data.deps_dev_v1.Dependencies\` d
   WHERE d.SnapshotAt >= TIMESTAMP('${watermark}')
     AND d.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${watermark}', INTERVAL 1 DAY))
-    AND d.System IN (${systems})
+    AND d.System IN (${filter})
     AND d.MinimumDepth = 1
   GROUP BY d.System, d.Name, d.Version, d.Dependency.Name, d.Dependency.Version
-)
-SELECT t.*
-FROM today t
-LEFT JOIN last_watermark l
+)`,
+    )
+    selects.push(
+      `SELECT t.*
+FROM today_deps t
+LEFT JOIN watermark_deps l
   ON LOWER(l.System) = t.ecosystem AND l.Name = t.root_name AND l.Version = t.root_version
  AND l.to_name = t.to_name AND l.to_version = t.to_version
-WHERE l.to_name IS NULL
-`
+WHERE l.to_name IS NULL`,
+    )
+  }
+
+  if (includeGo) {
+    ctes.push(
+      `today_go AS (
+  SELECT
+    'go'                   AS ecosystem,
+    g.Name                 AS root_name,
+    g.Version              AS root_version,
+    d.Name                 AS to_name,
+    CAST(NULL AS STRING)   AS to_version,
+    d.Requirement          AS version_constraint
+  FROM \`bigquery-public-data.deps_dev_v1.GoRequirements\` g,
+  UNNEST(g.DirectDependencies) AS d
+  WHERE g.SnapshotAt >= TIMESTAMP('${today}')
+    AND g.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${today}', INTERVAL 1 DAY))
+)`,
+      `watermark_go AS (
+  SELECT g.Name, g.Version, d.Name AS to_name
+  FROM \`bigquery-public-data.deps_dev_v1.GoRequirements\` g,
+  UNNEST(g.DirectDependencies) AS d
+  WHERE g.SnapshotAt >= TIMESTAMP('${watermark}')
+    AND g.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${watermark}', INTERVAL 1 DAY))
+  GROUP BY g.Name, g.Version, d.Name
+)`,
+    )
+    selects.push(
+      `SELECT t.*
+FROM today_go t
+LEFT JOIN watermark_go l
+  ON l.Name = t.root_name AND l.Version = t.root_version AND l.to_name = t.to_name
+WHERE l.to_name IS NULL`,
+    )
+  }
+
+  if (includeNuget) {
+    ctes.push(
+      `today_nuget AS (
+  SELECT
+    'nuget'                AS ecosystem,
+    n.Name                 AS root_name,
+    n.Version              AS root_version,
+    dep.Name               AS to_name,
+    CAST(NULL AS STRING)   AS to_version,
+    dep.Requirement        AS version_constraint
+  FROM \`bigquery-public-data.deps_dev_v1.NuGetRequirements\` n,
+  UNNEST(n.DependencyGroups) AS grp,
+  UNNEST(grp.Dependencies) AS dep
+  WHERE n.SnapshotAt >= TIMESTAMP('${today}')
+    AND n.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${today}', INTERVAL 1 DAY))
+)`,
+      `watermark_nuget AS (
+  SELECT n.Name, n.Version, dep.Name AS to_name
+  FROM \`bigquery-public-data.deps_dev_v1.NuGetRequirements\` n,
+  UNNEST(n.DependencyGroups) AS grp,
+  UNNEST(grp.Dependencies) AS dep
+  WHERE n.SnapshotAt >= TIMESTAMP('${watermark}')
+    AND n.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${watermark}', INTERVAL 1 DAY))
+  GROUP BY n.Name, n.Version, dep.Name
+)`,
+    )
+    selects.push(
+      `SELECT t.*
+FROM today_nuget t
+LEFT JOIN watermark_nuget l
+  ON l.Name = t.root_name AND l.Version = t.root_version AND l.to_name = t.to_name
+WHERE l.to_name IS NULL`,
+    )
+  }
+
+  return `WITH\n${ctes.join(',\n')}\n${selects.join('\nUNION ALL\n')}`
 }
 
-export function buildDepsFullSql(systems: string, tableOption: 'A' | 'B' = 'A'): string {
-  return tableOption === 'B' ? buildDepsFullSqlB(systems) : buildDepsFullSqlA(systems)
+export function buildDepsFullSql(ecosystems: string[], tableOption: 'A' | 'B' = 'A'): string {
+  return tableOption === 'B' ? buildDepsFullSqlB(ecosystems) : buildDepsFullSqlA(ecosystems)
 }
 
 export function buildDepsIncrementalSql(
   today: string,
   watermark: string,
-  systems: string,
+  ecosystems: string[],
   tableOption: 'A' | 'B' = 'A',
 ): string {
   return tableOption === 'B'
-    ? buildDepsIncrementalSqlB(today, watermark, systems)
-    : buildDepsIncrementalSqlA(today, watermark, systems)
+    ? buildDepsIncrementalSqlB(today, watermark, ecosystems)
+    : buildDepsIncrementalSqlA(today, watermark, ecosystems)
 }
diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
index cb19d55485..7d56a94b17 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
@@ -1,8 +1,11 @@
 import { proxyActivities } from '@temporalio/workflow'
 
 import type * as depsDevActivities from '../activities'
-import { buildDepsFullSql, buildDepsIncrementalSql } from '../queries/depsSql'
-import { toSystemsFilter } from '../queries/systems'
+import {
+  DEPS_DEFAULT_ECOSYSTEMS,
+  buildDepsFullSql,
+  buildDepsIncrementalSql,
+} from '../queries/depsSql'
 
 const { bqExportToGcs } = proxyActivities<typeof depsDevActivities>({
   startToCloseTimeout: '2 hours',
@@ -167,12 +170,12 @@ export async function ingestDependencies(opts: {
   depsTableOption?: 'A' | 'B'
   exportName?: string
 }): Promise<{ rowCountBq: number }> {
-  const systems = toSystemsFilter(opts.ecosystems)
+  const ecosystems = opts.ecosystems ?? DEPS_DEFAULT_ECOSYSTEMS
   const tableOption = opts.depsTableOption ?? 'A'
   const sql =
     opts.syncMode === 'full'
-      ? buildDepsFullSql(systems, tableOption)
-      : buildDepsIncrementalSql(opts.today, opts.watermark ?? '', systems, tableOption)
+      ? buildDepsFullSql(ecosystems, tableOption)
+      : buildDepsIncrementalSql(opts.today, opts.watermark ?? '', ecosystems, tableOption)
 
   const exportResult = await bqExportToGcs({
     jobKind: 'package_dependencies',
diff --git a/services/apps/packages_worker/src/scripts/exportToBucket.ts b/services/apps/packages_worker/src/scripts/exportToBucket.ts
index 62d60b987d..60c4054f97 100644
--- a/services/apps/packages_worker/src/scripts/exportToBucket.ts
+++ b/services/apps/packages_worker/src/scripts/exportToBucket.ts
@@ -10,7 +10,7 @@ import { extractBqStats } from '../deps-dev/bqStats'
 import { GCS_BUCKET, bigquery, bucket } from '../deps-dev/config'
 import { ADVISORIES_SQL, buildAdvisoryPackagesSql } from '../deps-dev/queries/advisoriesSql'
 import { buildDependentCountsSql } from '../deps-dev/queries/dependentCountsSql'
-import { buildDepsFullSql } from '../deps-dev/queries/depsSql'
+import { DEPS_DEFAULT_ECOSYSTEMS, buildDepsFullSql } from '../deps-dev/queries/depsSql'
 import { buildPackageReposSql } from '../deps-dev/queries/packageReposSql'
 import { buildPackagesFullSql } from '../deps-dev/queries/packagesSql'
 import { buildReposSql } from '../deps-dev/queries/reposSql'
@@ -292,7 +292,7 @@ async function main(): Promise<void> {
   const SQL_BY_PART: Record<ExportPart, string> = {
     packages: buildPackagesFullSql(systems),
     versions: buildVersionsFullSql(systems),
-    deps: buildDepsFullSql(systems, depsTableOption),
+    deps: buildDepsFullSql(ecosystems ?? DEPS_DEFAULT_ECOSYSTEMS, depsTableOption),
     repos: buildReposSql(reposSnapshotDate ?? today, systems),
     package_repos: buildPackageReposSql(reposSnapshotDate ?? today, systems),
     counts: buildDependentCountsSql(countsSnapshotDate ?? today),

From 9b776400373b893feae8fa0d33b0a4f48eb02986 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Wed, 17 Jun 2026 16:11:29 +0200
Subject: [PATCH 03/29] fix: flag to update constraints
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../deps-dev/workflows/bootstrapOsspckgs.ts   |  2 +
 .../deps-dev/workflows/ingestDependencies.ts  | 56 +++++++++++++++++--
 .../src/scripts/triggerBootstrap.ts           | 19 ++++++-
 3 files changed, 71 insertions(+), 6 deletions(-)

diff --git a/services/apps/packages_worker/src/deps-dev/workflows/bootstrapOsspckgs.ts b/services/apps/packages_worker/src/deps-dev/workflows/bootstrapOsspckgs.ts
index d185b4fc8d..0258fa40e7 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/bootstrapOsspckgs.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/bootstrapOsspckgs.ts
@@ -63,6 +63,7 @@ export async function bootstrapOsspckgs(opts: {
   depsTableOption?: 'A' | 'B'
   exportName?: string
   snapshotDate?: string // YYYY-MM-DD — override BQ snapshot resolution for all partition-filtered kinds
+  fillConstraints?: boolean // re-export full deps BQ data, upsert version_constraint where NULL
 }): Promise<void> {
   // B3: deterministic timestamps — workflowInfo().startTime is replay-stable; new Date() is not.
   const start = workflowInfo().startTime
@@ -221,6 +222,7 @@ export async function bootstrapOsspckgs(opts: {
           reuseExports: opts.reuseExports,
           depsTableOption: opts.depsTableOption,
           exportName: opts.exportName,
+          fillConstraints: opts.fillConstraints,
         },
       ],
     })
diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
index 7d56a94b17..33d1f03db8 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
@@ -138,6 +138,45 @@ LEFT JOIN versions dv ON dv.package_id = pd.id AND dv.number = sp.to_version
 ORDER BY pv.id, pd.id, sp.to_version DESC NULLS LAST
 `
 
+// Fill-constraints variant: UNIQUE constraint stays in place (not dropped), so ON CONFLICT is valid.
+// Upserts version_constraint only for rows where it is currently NULL — safe to run against a table
+// already populated by --deps-table-b (which sets version_constraint = NULL for all rows).
+// DISTINCT ON matches MERGE_SQL_FULL to resolve duplicate (root, dep) pairs from BQ before the upsert.
+const MERGE_SQL_FILL_CONSTRAINTS = `
+INSERT INTO package_dependencies (
+  package_id, version_id, depends_on_id, depends_on_version_id,
+  version_constraint, dependency_kind, is_optional, created_at, updated_at
+)
+SELECT DISTINCT ON (pv.id, pd.id)
+  pv.package_id, pv.id, pd.id, dv.id,
+  sp.version_constraint, 'direct', FALSE, NOW(), NOW()
+FROM staging.osspckgs_deps_raw sp
+JOIN staging.osspckgs_versions_lookup pv ON pv.ecosystem = sp.ecosystem
+  AND pv.ns = CASE
+      WHEN sp.ecosystem = 'maven'   THEN SPLIT_PART(sp.root_name, ':', 1)
+      WHEN sp.root_name LIKE '@%/%' THEN SPLIT_PART(sp.root_name, '/', 1)
+      ELSE '' END
+  AND pv.name = CASE
+      WHEN sp.ecosystem = 'maven'   THEN SPLIT_PART(sp.root_name, ':', 2)
+      WHEN sp.root_name LIKE '@%/%' THEN SPLIT_PART(sp.root_name, '/', 2)
+      ELSE sp.root_name END
+  AND pv.number = sp.root_version
+JOIN packages pd ON pd.ecosystem = sp.ecosystem
+  AND COALESCE(pd.namespace, '') = CASE
+      WHEN sp.ecosystem = 'maven'  THEN SPLIT_PART(sp.to_name, ':', 1)
+      WHEN sp.to_name LIKE '@%/%'  THEN SPLIT_PART(sp.to_name, '/', 1)
+      ELSE '' END
+  AND pd.name = CASE
+      WHEN sp.ecosystem = 'maven'  THEN SPLIT_PART(sp.to_name, ':', 2)
+      WHEN sp.to_name LIKE '@%/%'  THEN SPLIT_PART(sp.to_name, '/', 2)
+      ELSE sp.to_name END
+LEFT JOIN versions dv ON dv.package_id = pd.id AND dv.number = sp.to_version
+ORDER BY pv.id, pd.id, sp.to_version DESC NULLS LAST
+ON CONFLICT (version_id, depends_on_id, dependency_kind) DO UPDATE
+  SET version_constraint = EXCLUDED.version_constraint
+  WHERE package_dependencies.version_constraint IS NULL
+`
+
 // SET LOCAL scopes settings to this transaction only.
 // synchronous_commit=off skips WAL flush wait — safe for plain INSERT on full loads.
 // max_parallel_workers_per_gather parallelises the SELECT side of INSERT...SELECT.
@@ -169,11 +208,14 @@ export async function ingestDependencies(opts: {
   reuseExports?: boolean
   depsTableOption?: 'A' | 'B'
   exportName?: string
+  fillConstraints?: boolean // re-export full BQ data, upsert version_constraint only where NULL
 }): Promise<{ rowCountBq: number }> {
   const ecosystems = opts.ecosystems ?? DEPS_DEFAULT_ECOSYSTEMS
   const tableOption = opts.depsTableOption ?? 'A'
+  const isFill = opts.fillConstraints === true
+  // Fill mode always uses full SQL — needs all rows to find which have NULL version_constraint in DB.
   const sql =
-    opts.syncMode === 'full'
+    opts.syncMode === 'full' || isFill
       ? buildDepsFullSql(ecosystems, tableOption)
       : buildDepsIncrementalSql(opts.today, opts.watermark ?? '', ecosystems, tableOption)
 
@@ -204,7 +246,7 @@ export async function ingestDependencies(opts: {
 
   await createVersionsLookup({ ecosystems: opts.ecosystems })
 
-  if (opts.syncMode === 'full') {
+  if (opts.syncMode === 'full' && !isFill) {
     await dropPackageDepsConstraints()
     await dropPackageDepsIndexes()
   }
@@ -239,7 +281,11 @@ export async function ingestDependencies(opts: {
       const { rowsAffected, tableRowCounts } = await mergeStagingToTable({
         jobId: exportResult.jobId,
         prepareSql: MERGE_PREPARE_SQL,
-        mergeSql: opts.syncMode === 'full' ? MERGE_SQL_FULL : MERGE_SQL,
+        mergeSql: isFill
+          ? MERGE_SQL_FILL_CONSTRAINTS
+          : opts.syncMode === 'full'
+            ? MERGE_SQL_FULL
+            : MERGE_SQL,
         tableNames: 'package_dependencies',
         isFinal,
         priorRowsAffected,
@@ -255,12 +301,12 @@ export async function ingestDependencies(opts: {
       }
     }
 
-    if (opts.syncMode === 'full') {
+    if (opts.syncMode === 'full' && !isFill) {
       await rebuildPackageDepsIndexes()
       await rebuildPackageDepsConstraints()
     }
   } catch (err) {
-    if (opts.syncMode === 'full') {
+    if (opts.syncMode === 'full' && !isFill) {
       try {
         await rebuildPackageDepsIndexes()
       } catch (_) {
diff --git a/services/apps/packages_worker/src/scripts/triggerBootstrap.ts b/services/apps/packages_worker/src/scripts/triggerBootstrap.ts
index 1732862d35..e688cb42a5 100644
--- a/services/apps/packages_worker/src/scripts/triggerBootstrap.ts
+++ b/services/apps/packages_worker/src/scripts/triggerBootstrap.ts
@@ -37,6 +37,10 @@ Options:
   --deps-table-b         Use DependenciesLatest instead of DependencyGraphEdgesLatest
                          for deps. Cheaper (~$4.69 vs $12.67 for CARGO) but loses
                          version_constraint. ADR-0003 Option B. Good for local testing.
+  --fill-constraints     Re-export full BQ deps data and upsert version_constraint for
+                         rows where it is currently NULL. Use after a --deps-table-b run
+                         to backfill missing version_constraint values without re-inserting
+                         rows. Only affects package_dependencies.
   --help                 Show this help
 
 Examples:
@@ -63,6 +67,7 @@ async function main(): Promise<void> {
 
   const reuseExports = args.includes('--reuse-exports')
   const depsTableOption: 'A' | 'B' = args.includes('--deps-table-b') ? 'B' : 'A'
+  const fillConstraints = args.includes('--fill-constraints')
 
   const exportNameIdx = args.indexOf('--export-name')
   if (
@@ -148,7 +153,18 @@ async function main(): Promise<void> {
   const handle = await client.workflow.start(bootstrapOsspckgs, {
     taskQueue: 'bq-dataset-ingest',
     workflowId,
-    args: [{ mode, ecosystems, kinds, reuseExports, depsTableOption, exportName, snapshotDate }],
+    args: [
+      {
+        mode,
+        ecosystems,
+        kinds,
+        reuseExports,
+        depsTableOption,
+        exportName,
+        snapshotDate,
+        fillConstraints,
+      },
+    ],
   })
 
   const flags = [
@@ -157,6 +173,7 @@ async function main(): Promise<void> {
     reuseExports ? '--reuse-exports' : '',
     depsTableOption === 'B' ? '--deps-table-b' : '',
     exportName ? `--export-name ${exportName}` : '',
+    fillConstraints ? '--fill-constraints' : '',
   ].filter(Boolean)
   console.log(
     `Started workflow ${handle.workflowId}${ecosystems ? ` (ecosystems: ${ecosystems.join(', ')})` : ''}${flags.length ? ` [${flags.join(' ')}]` : ''}`,

From 9fd28f754d89b22998a56a72668291802b8a6224 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Wed, 17 Jun 2026 18:44:51 +0200
Subject: [PATCH 04/29] chore: improvements to monitoring
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../src/deps-dev/activities/bqExportToGcs.ts  | 24 ++++-
 .../src/deps-dev/activities/index.ts          |  1 +
 .../src/deps-dev/activities/setJobStep.ts     |  8 ++
 .../deps-dev/workflows/ingestAdvisories.ts    |  2 +
 .../deps-dev/workflows/ingestDependencies.ts  | 13 +++
 .../src/deps-dev/workflows/ingestPackages.ts  |  1 +
 .../src/deps-dev/workflows/ingestRepos.ts     |  2 +
 .../src/deps-dev/workflows/ingestVersions.ts  | 12 +++
 .../src/scripts/monitorOsspckgs.ts            | 87 ++++++++++++++++---
 .../src/osspckgs/ingestJobs.ts                | 17 +++-
 10 files changed, 150 insertions(+), 17 deletions(-)
 create mode 100644 services/apps/packages_worker/src/deps-dev/activities/setJobStep.ts

diff --git a/services/apps/packages_worker/src/deps-dev/activities/bqExportToGcs.ts b/services/apps/packages_worker/src/deps-dev/activities/bqExportToGcs.ts
index da5c89c2cc..a594295af0 100644
--- a/services/apps/packages_worker/src/deps-dev/activities/bqExportToGcs.ts
+++ b/services/apps/packages_worker/src/deps-dev/activities/bqExportToGcs.ts
@@ -24,6 +24,7 @@ export interface BqExportToGcsInput {
   maxBytesGb: number
   reuseExports?: boolean
   exportName?: string
+  ecosystems?: string[]
 }
 
 export interface BqExportToGcsOutput {
@@ -34,7 +35,17 @@ export interface BqExportToGcsOutput {
 }
 
 export async function bqExportToGcs(input: BqExportToGcsInput): Promise<BqExportToGcsOutput> {
-  const { jobKind, sql, runId, syncMode, snapshotAt, maxBytesGb, reuseExports, exportName } = input
+  const {
+    jobKind,
+    sql,
+    runId,
+    syncMode,
+    snapshotAt,
+    maxBytesGb,
+    reuseExports,
+    exportName,
+    ecosystems,
+  } = input
 
   // Named exports use a stable GCS path independent of runId so they survive across bootstrap runs.
   const namedGcsPrefix = exportName
@@ -93,7 +104,10 @@ export async function bqExportToGcs(input: BqExportToGcsInput): Promise<BqExport
           gcsPrefix: namedPrefix,
           rowCountBq: 0,
           bqBytesBilled: 0,
-          tableRowCounts: { 'bq:export': 0 },
+          tableRowCounts: {
+            'bq:export': 0,
+            ...(ecosystems ? { 'meta:ecosystems': ecosystems } : {}),
+          },
         })
         return { gcsPrefix: namedPrefix, rowCount: 0, bqBytesBilled: 0, jobId }
       }
@@ -177,8 +191,10 @@ export async function bqExportToGcs(input: BqExportToGcsInput): Promise<BqExport
   const provisionalDate = snapshotAt ? new Date(snapshotAt) : null
   const jobId = await createIngestJob(qx, jobKind, syncMode, provisionalDate, exportName)
 
-  // H7: mark exporting before we start the BQ job
-  await markJobStatus(qx, jobId, 'exporting')
+  // H7: mark exporting before we start the BQ job; store ecosystems filter in table_row_counts JSONB.
+  await markJobStatus(qx, jobId, 'exporting', {
+    ...(ecosystems ? { tableRowCounts: { 'meta:ecosystems': ecosystems } } : {}),
+  })
 
   // B9: wrap in SELECT * FROM (...) so QUALIFY / top-level set ops don't break EXPORT DATA syntax.
   // CREATE TEMP TABLE first so BQ materializes the result before exporting — direct EXPORT DATA
diff --git a/services/apps/packages_worker/src/deps-dev/activities/index.ts b/services/apps/packages_worker/src/deps-dev/activities/index.ts
index 94085de19a..11e1ed49c8 100644
--- a/services/apps/packages_worker/src/deps-dev/activities/index.ts
+++ b/services/apps/packages_worker/src/deps-dev/activities/index.ts
@@ -1,4 +1,5 @@
 export * from './bqExportToGcs'
+export * from './setJobStep'
 export * from './createVersionsLookup'
 export * from './managePackageDepsConstraints'
 export * from './managePackageDepsIndexes'
diff --git a/services/apps/packages_worker/src/deps-dev/activities/setJobStep.ts b/services/apps/packages_worker/src/deps-dev/activities/setJobStep.ts
new file mode 100644
index 0000000000..e54ad673ce
--- /dev/null
+++ b/services/apps/packages_worker/src/deps-dev/activities/setJobStep.ts
@@ -0,0 +1,8 @@
+import { mergeJobTableRowCounts } from '@crowd/data-access-layer'
+
+import { getPackagesDb } from '../../db'
+
+export async function setJobStep(input: { jobId: number; step: string }): Promise<void> {
+  const qx = await getPackagesDb()
+  await mergeJobTableRowCounts(qx, input.jobId, { 'meta:step': input.step })
+}
diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestAdvisories.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestAdvisories.ts
index 4669eba8ef..4fc55de34d 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestAdvisories.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestAdvisories.ts
@@ -138,6 +138,7 @@ export async function ingestAdvisories(opts: {
     maxBytesGb: 10,
     reuseExports: opts.reuseExports,
     exportName: opts.exportName,
+    ecosystems: opts.ecosystems,
   })
 
   const { fileNames: advFileNames, rowCounts: advRowCounts } = await listParquetFiles({
@@ -211,6 +212,7 @@ export async function ingestAdvisories(opts: {
     maxBytesGb: 1500,
     reuseExports: opts.reuseExports,
     exportName: opts.exportName,
+    ecosystems: opts.ecosystems,
   })
 
   const { fileNames: pkgFileNames, rowCounts: pkgRowCounts } = await listParquetFiles({
diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
index 33d1f03db8..1243d9bf28 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
@@ -49,6 +49,11 @@ const { dropPackageDepsConstraints, rebuildPackageDepsConstraints } = proxyActiv
   retry: { maximumAttempts: 2, initialInterval: '1 minute' },
 })
 
+const { setJobStep } = proxyActivities<typeof depsDevActivities>({
+  startToCloseTimeout: '30 seconds',
+  retry: { maximumAttempts: 3 },
+})
+
 const STAGING_TABLE = 'staging.osspckgs_deps_raw'
 
 const STAGING_DDL = `
@@ -228,6 +233,7 @@ export async function ingestDependencies(opts: {
     maxBytesGb: 10000,
     reuseExports: opts.reuseExports,
     exportName: opts.exportName,
+    ecosystems: opts.ecosystems,
   })
 
   const { fileNames, rowCounts } = await listParquetFiles({ gcsPrefix: exportResult.gcsPrefix })
@@ -244,10 +250,13 @@ export async function ingestDependencies(opts: {
     return { rowCountBq: exportResult.rowCount }
   }
 
+  await setJobStep({ jobId: exportResult.jobId, step: 'creating_lookup' })
   await createVersionsLookup({ ecosystems: opts.ecosystems })
 
   if (opts.syncMode === 'full' && !isFill) {
+    await setJobStep({ jobId: exportResult.jobId, step: 'drop_constraints' })
     await dropPackageDepsConstraints()
+    await setJobStep({ jobId: exportResult.jobId, step: 'drop_indexes' })
     await dropPackageDepsIndexes()
   }
 
@@ -302,17 +311,21 @@ export async function ingestDependencies(opts: {
     }
 
     if (opts.syncMode === 'full' && !isFill) {
+      await setJobStep({ jobId: exportResult.jobId, step: 'rebuild_indexes' })
       await rebuildPackageDepsIndexes()
+      await setJobStep({ jobId: exportResult.jobId, step: 'rebuild_constraints' })
       await rebuildPackageDepsConstraints()
     }
   } catch (err) {
     if (opts.syncMode === 'full' && !isFill) {
       try {
+        await setJobStep({ jobId: exportResult.jobId, step: 'rebuild_indexes' })
         await rebuildPackageDepsIndexes()
       } catch (_) {
         /* best-effort */
       }
       try {
+        await setJobStep({ jobId: exportResult.jobId, step: 'rebuild_constraints' })
         await rebuildPackageDepsConstraints()
       } catch (_) {
         /* best-effort */
diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestPackages.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestPackages.ts
index e87a0674bf..083ec0c44c 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestPackages.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestPackages.ts
@@ -110,6 +110,7 @@ export async function ingestPackages(opts: {
     maxBytesGb: opts.syncMode === 'full' ? 6000 : 400,
     reuseExports: opts.reuseExports,
     exportName: opts.exportName,
+    ecosystems: opts.ecosystems,
   })
 
   const { fileNames, rowCounts } = await listParquetFiles({ gcsPrefix: exportResult.gcsPrefix })
diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestRepos.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestRepos.ts
index c5a8cb7a6e..f09f6ff3ba 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestRepos.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestRepos.ts
@@ -117,6 +117,7 @@ export async function ingestRepos(opts: {
     maxBytesGb: 2000,
     reuseExports: opts.reuseExports,
     exportName: opts.exportName,
+    ecosystems: opts.ecosystems,
   })
 
   const { fileNames: repoFileNames, rowCounts: repoRowCounts } = await listParquetFiles({
@@ -187,6 +188,7 @@ export async function ingestRepos(opts: {
     maxBytesGb: 2000,
     reuseExports: opts.reuseExports,
     exportName: opts.exportName,
+    ecosystems: opts.ecosystems,
   })
 
   const { fileNames: pkgRepoFileNames, rowCounts: pkgRepoRowCounts } = await listParquetFiles({
diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestVersions.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestVersions.ts
index 57debeb1f8..5da1467121 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestVersions.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestVersions.ts
@@ -39,6 +39,11 @@ const { dropVersionsConstraints, rebuildVersionsConstraints } = proxyActivities<
   retry: { maximumAttempts: 2, initialInterval: '1 minute' },
 })
 
+const { setJobStep } = proxyActivities<typeof depsDevActivities>({
+  startToCloseTimeout: '30 seconds',
+  retry: { maximumAttempts: 3 },
+})
+
 const STAGING_TABLE = 'staging.osspckgs_versions_raw'
 
 const STAGING_DDL = `
@@ -129,6 +134,7 @@ export async function ingestVersions(opts: {
     maxBytesGb: 400,
     reuseExports: opts.reuseExports,
     exportName: opts.exportName,
+    ecosystems: opts.ecosystems,
   })
 
   const { fileNames, rowCounts } = await listParquetFiles({ gcsPrefix: exportResult.gcsPrefix })
@@ -145,7 +151,9 @@ export async function ingestVersions(opts: {
   }
 
   if (opts.syncMode === 'full') {
+    await setJobStep({ jobId: exportResult.jobId, step: 'drop_constraints' })
     await dropVersionsConstraints()
+    await setJobStep({ jobId: exportResult.jobId, step: 'drop_indexes' })
     await dropVersionsIndexes()
   }
 
@@ -198,17 +206,21 @@ export async function ingestVersions(opts: {
     }
 
     if (opts.syncMode === 'full') {
+      await setJobStep({ jobId: exportResult.jobId, step: 'rebuild_indexes' })
       await rebuildVersionsIndexes()
+      await setJobStep({ jobId: exportResult.jobId, step: 'rebuild_constraints' })
       await rebuildVersionsConstraints()
     }
   } catch (err) {
     if (opts.syncMode === 'full') {
       try {
+        await setJobStep({ jobId: exportResult.jobId, step: 'rebuild_indexes' })
         await rebuildVersionsIndexes()
       } catch (_) {
         /* best-effort */
       }
       try {
+        await setJobStep({ jobId: exportResult.jobId, step: 'rebuild_constraints' })
         await rebuildVersionsConstraints()
       } catch (_) {
         /* best-effort */
diff --git a/services/apps/packages_worker/src/scripts/monitorOsspckgs.ts b/services/apps/packages_worker/src/scripts/monitorOsspckgs.ts
index 8df57a4622..13488ac652 100644
--- a/services/apps/packages_worker/src/scripts/monitorOsspckgs.ts
+++ b/services/apps/packages_worker/src/scripts/monitorOsspckgs.ts
@@ -59,7 +59,7 @@ async function fetchJobs() {
     SELECT
       id, job_kind, status, sync_mode,
       snapshot_at, provisional_snapshot_at,
-      gcs_prefix, export_name,
+      gcs_prefix, export_name, ecosystems,
       row_count_bq, row_count_staging, row_count_pg,
       bq_bytes_billed,
       table_row_counts,
@@ -110,10 +110,12 @@ const STATUS_ICON = {
   cleaned: '–',
 }
 
-function statusStr(status: string) {
-  const c = STATUS_COLOR[status as keyof typeof STATUS_COLOR] ?? ''
-  const i = STATUS_ICON[status as keyof typeof STATUS_ICON] ?? '?'
-  return `${c}${i} ${status}${A.reset}`
+function statusStr(status: string, step?: string | null, stuck?: boolean) {
+  const c = stuck ? A.yellow : (STATUS_COLOR[status as keyof typeof STATUS_COLOR] ?? '')
+  const i = stuck ? '⚠' : (STATUS_ICON[status as keyof typeof STATUS_ICON] ?? '?')
+  const label =
+    step && !['done', 'failed', 'cleaned'].includes(status) ? `${status}·${step}` : status
+  return `${c}${i} ${label}${A.reset}`
 }
 
 function fmtNum(n: unknown) {
@@ -265,10 +267,16 @@ async function fetchTableCounts(): Promise<Record<string, number>> {
 
 const KNOWN_ECOSYSTEMS = ['npm', 'go', 'maven', 'pypi', 'nuget', 'cargo']
 
-// Parses ecosystem names from gcs_prefix or export_name.
-// Looks for -<eco>- or -<eco>/ patterns so "go" doesn't match inside "cargo".
+// Reads ecosystems from table_row_counts['meta:ecosystems'] (new jobs) or falls back to
+// parsing gcs_prefix/export_name for jobs created before the meta key was added.
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 function extractEcosystem(job: any) {
+  const trc = job.table_row_counts ?? {}
+  const meta = trc['meta:ecosystems']
+  if (Array.isArray(meta) && meta.length > 0) {
+    return meta.map((e: string) => e.toLowerCase()).join(',')
+  }
+  // Fallback: parse GCS path for old rows
   const src = [job.gcs_prefix ?? '', job.export_name ?? ''].join(' ').toLowerCase()
   const found = KNOWN_ECOSYSTEMS.filter((e) => new RegExp(`(^|[-/,])${e}([-/,]|$)`).test(src))
   return found.length > 0 ? found.join(',') : null
@@ -277,7 +285,7 @@ function extractEcosystem(job: any) {
 // ── Layout ────────────────────────────────────────────────────────────────────
 
 const COL = {
-  id: 12,
+  id: 18,
   kind: 20,
   eco: 10,
   status: 18,
@@ -393,7 +401,17 @@ function tableRow(job: any, selected: boolean) {
   const startedTime = job.started_at
     ? new Date(job.started_at as string).toISOString().slice(11, 16)
     : '—'
-  const idCell = `${String(job.id)} ${A.dim}${startedTime}${A.reset}${bg}`
+  const isFinished = ['done', 'failed', 'cleaned'].includes(job.status)
+  const jobDuration =
+    isFinished && job.started_at && job.finished_at
+      ? fmtEtaStr(
+          new Date(job.finished_at as string).getTime() -
+            new Date(job.started_at as string).getTime(),
+        )
+      : null
+  const idCell = jobDuration
+    ? `${String(job.id)} ${A.dim}${startedTime} ${jobDuration}${A.reset}${bg}`
+    : `${String(job.id)} ${A.dim}${startedTime}${A.reset}${bg}`
 
   return (
     bg +
@@ -408,7 +426,17 @@ function tableRow(job: any, selected: boolean) {
       COL.eco,
       bg,
     ) +
-    padCell(statusStr(job.status), COL.status, bg) +
+    padCell(
+      statusStr(
+        job.status,
+        typeof (job.table_row_counts ?? {})['meta:step'] === 'string'
+          ? (job.table_row_counts['meta:step'] as string)
+          : null,
+        stuckIds.has(job.id as number),
+      ),
+      COL.status,
+      bg,
+    ) +
     padCell(fmtMode(job.sync_mode), COL.mode, bg) +
     padCell(fmtCompact(job.row_count_bq), COL.bq, bg) +
     padCell(fmtUsd(job.bq_bytes_billed), COL.cost, bg) +
@@ -436,8 +464,13 @@ function renderDetail(job: any, cols: number) {
       ? A.dim + String(job.provisional_snapshot_at).slice(0, 10) + ' (provisional)' + A.reset
       : A.dim + '—' + A.reset
 
+  const trcStep =
+    typeof (job.table_row_counts ?? {})['meta:step'] === 'string'
+      ? (job.table_row_counts['meta:step'] as string)
+      : null
+  const isStuckJob = stuckIds.has(job.id as number)
   lines.push(
-    ` ${A.bold}Job #${job.id} — ${job.job_kind}${A.reset}   ${statusStr(job.status)}  ${A.dim}${job.sync_mode}${A.reset}`,
+    ` ${A.bold}Job #${job.id} — ${job.job_kind}${A.reset}   ${statusStr(job.status, trcStep, isStuckJob)}  ${A.dim}${job.sync_mode}${A.reset}`,
   )
   lines.push(` ${sep}`)
   lines.push(` ${A.dim}snapshot:${A.reset}  ${snapshotDate}`)
@@ -507,7 +540,11 @@ function renderDetail(job: any, cols: number) {
 
     const ref = stagRows || bqRows // prefer staging as denominator for final rows
     const finalKeys = Object.keys(trc).filter(
-      (k) => !k.startsWith('bq:') && !k.startsWith('staging:') && !k.startsWith('progress:'),
+      (k) =>
+        !k.startsWith('bq:') &&
+        !k.startsWith('staging:') &&
+        !k.startsWith('progress:') &&
+        !k.startsWith('meta:'),
     )
     if (finalKeys.length > 0) {
       for (const k of finalKeys) {
@@ -578,6 +615,31 @@ function renderDetail(job: any, cols: number) {
 let jobs: any[] = []
 let tableCounts: Record<string, number> = {}
 let watermarks: Record<string, string> = {}
+let stuckIds: Set<number> = new Set()
+
+// A job is "stuck" when it's still in a non-terminal state but a newer job of the same kind
+// has already completed — meaning the workflow died without ever finishing this job.
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function computeStuckIds(jobList: any[]): Set<number> {
+  const latestDone = new Map<string, number>()
+  for (const job of jobList) {
+    if (['done', 'cleaned'].includes(job.status) && job.started_at) {
+      const t = new Date(job.started_at as string).getTime()
+      if (!latestDone.has(job.job_kind) || t > (latestDone.get(job.job_kind) ?? 0)) {
+        latestDone.set(job.job_kind, t)
+      }
+    }
+  }
+  const result = new Set<number>()
+  for (const job of jobList) {
+    if (['done', 'failed', 'cleaned', 'pending'].includes(job.status)) continue
+    const latest = latestDone.get(job.job_kind)
+    if (latest && job.started_at && new Date(job.started_at as string).getTime() < latest) {
+      result.add(job.id as number)
+    }
+  }
+  return result
+}
 let selected = 0
 let detailOpen = false
 let lastRefresh: string | null = null
@@ -813,6 +875,7 @@ async function refresh() {
     tableCounts = newTableCounts
     watermarks = newWatermarks
     if (selected >= jobs.length) selected = Math.max(0, jobs.length - 1)
+    stuckIds = computeStuckIds(jobs)
     lastRefresh = new Date().toLocaleTimeString()
     error = null
   } catch (e) {
diff --git a/services/libs/data-access-layer/src/osspckgs/ingestJobs.ts b/services/libs/data-access-layer/src/osspckgs/ingestJobs.ts
index ad7305cc3b..7d2b486136 100644
--- a/services/libs/data-access-layer/src/osspckgs/ingestJobs.ts
+++ b/services/libs/data-access-layer/src/osspckgs/ingestJobs.ts
@@ -44,7 +44,7 @@ export interface MarkJobStatusFields {
   rowCountBq?: number
   rowCountStaging?: number
   rowCountPg?: number
-  tableRowCounts?: Record<string, number>
+  tableRowCounts?: Record<string, number | string | string[]>
   bqBytesBilled?: number
   bqJobId?: string
   bqStats?: BqStats
@@ -302,3 +302,18 @@ export async function markJobStatus(
 
   await qx.result(`UPDATE osspckgs_ingest_jobs SET ${sets.join(', ')} WHERE id = $(jobId)`, params)
 }
+
+// Merges key-value pairs into table_row_counts without changing status.
+// Used to write meta:step and similar display-only tracking keys mid-job.
+export async function mergeJobTableRowCounts(
+  qx: QueryExecutor,
+  jobId: number,
+  kv: Record<string, number | string | string[]>,
+): Promise<void> {
+  await qx.result(
+    `UPDATE osspckgs_ingest_jobs
+     SET table_row_counts = COALESCE(table_row_counts, '{}') || $(kv)
+     WHERE id = $(jobId)`,
+    { jobId, kv },
+  )
+}

From 584cf90ad853d29ba9cf52ea6dc28efe41609871 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Wed, 17 Jun 2026 18:47:48 +0200
Subject: [PATCH 05/29] chore: more states
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 services/apps/packages_worker/src/criticality/activities.ts | 1 +
 .../src/deps-dev/workflows/ingestDependentCounts.ts         | 6 ++++++
 2 files changed, 7 insertions(+)

diff --git a/services/apps/packages_worker/src/criticality/activities.ts b/services/apps/packages_worker/src/criticality/activities.ts
index 1f1ef1a1d8..dc0c18d1ec 100644
--- a/services/apps/packages_worker/src/criticality/activities.ts
+++ b/services/apps/packages_worker/src/criticality/activities.ts
@@ -78,6 +78,7 @@ export async function rankPackages(): Promise<{ scoredRows: number; rankedRows:
 
   const jobId = existing?.id ?? (await createIngestJob(qx, 'ranking', 'ranking', null))
   try {
+    await markJobStatus(qx, jobId, 'merging')
     const [result] = await qx.select(`SELECT * FROM rank_packages()`)
     const scoredRows = Number(result.scored_rows ?? 0)
     const rankedRows = Number(result.ranked_rows ?? 0)
diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependentCounts.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependentCounts.ts
index 030ef45869..e9df9b9ad8 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependentCounts.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependentCounts.ts
@@ -29,6 +29,11 @@ const { checkDependentCountsGuard } = proxyActivities<typeof depsDevActivities>(
   retry: { maximumAttempts: 3 },
 })
 
+const { setJobStep } = proxyActivities<typeof depsDevActivities>({
+  startToCloseTimeout: '30 seconds',
+  retry: { maximumAttempts: 3 },
+})
+
 const STAGING_TABLE = 'staging.osspckgs_dependent_counts_raw'
 
 // Two-statement DDL: DROP before CREATE so an existing table with the old dependent_packages_count
@@ -86,6 +91,7 @@ export async function ingestDependentCounts(opts: {
   const totalFiles = fileNames.length
   const totalRows = totalFiles > 0 ? rowCounts.reduce((a, b) => a + b, 0) : 0
 
+  await setJobStep({ jobId: exportResult.jobId, step: 'guard_check' })
   const guard = await checkDependentCountsGuard({
     currentRowCount: totalRows,
     snapshotDate: opts.snapshotDate,

From c4e31ace2c344d8ebd220cdecca7b763b5678084 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Wed, 17 Jun 2026 20:07:58 +0200
Subject: [PATCH 06/29] fix: use the same env as in deployment
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../packages_worker/src/deps-dev/workflows/ingestAdvisories.ts  | 2 +-
 .../src/deps-dev/workflows/ingestDependencies.ts                | 2 +-
 .../packages_worker/src/deps-dev/workflows/ingestVersions.ts    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestAdvisories.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestAdvisories.ts
index 4fc55de34d..543e79b118 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestAdvisories.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestAdvisories.ts
@@ -135,7 +135,7 @@ export async function ingestAdvisories(opts: {
     runId: opts.runId,
     syncMode: opts.syncMode,
     snapshotAt: opts.today,
-    maxBytesGb: 10,
+    maxBytesGb: 20,
     reuseExports: opts.reuseExports,
     exportName: opts.exportName,
     ecosystems: opts.ecosystems,
diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
index 1243d9bf28..df36432472 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
@@ -230,7 +230,7 @@ export async function ingestDependencies(opts: {
     runId: opts.runId,
     syncMode: opts.syncMode,
     snapshotAt: opts.today,
-    maxBytesGb: 10000,
+    maxBytesGb: opts.syncMode === 'full' ? 25000 : 10000,
     reuseExports: opts.reuseExports,
     exportName: opts.exportName,
     ecosystems: opts.ecosystems,
diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestVersions.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestVersions.ts
index 5da1467121..0463c69641 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestVersions.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestVersions.ts
@@ -131,7 +131,7 @@ export async function ingestVersions(opts: {
     runId: opts.runId,
     syncMode: opts.syncMode,
     snapshotAt: opts.today,
-    maxBytesGb: 400,
+    maxBytesGb: opts.syncMode === 'full' ? 800 : 400,
     reuseExports: opts.reuseExports,
     exportName: opts.exportName,
     ecosystems: opts.ecosystems,

From ba9649809008367a55bcd4071e6ffddbfbccfd78 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Wed, 17 Jun 2026 20:29:10 +0200
Subject: [PATCH 07/29] fix: better deploy script and monitor fix
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 scripts/cli                                   | 57 +++++++++++++++++++
 .../src/scripts/monitorOsspckgs.ts            |  2 +-
 2 files changed, 58 insertions(+), 1 deletion(-)

diff --git a/scripts/cli b/scripts/cli
index 0c9ce03ded..0d522c9620 100755
--- a/scripts/cli
+++ b/scripts/cli
@@ -255,6 +255,55 @@ function select_services() {
     echo "Selected services: $selected_services"
 }
 
+function monitor_workflow() {
+    local REPOSITORY="$1"
+    local WORKFLOW_FILE="$2"
+    local EXISTING_IDS="${3:-}"
+
+    yell "Waiting for workflow run to appear..."
+
+    local RUN_ID=""
+    local ATTEMPTS=0
+
+    while [[ -z "$RUN_ID" && $ATTEMPTS -lt 24 ]]; do
+        sleep 5
+        local CANDIDATE
+        CANDIDATE=$(gh run list \
+            --repo "$REPOSITORY" \
+            --workflow "$WORKFLOW_FILE" \
+            --limit 1 \
+            --json databaseId \
+            --jq '.[0].databaseId' 2>/dev/null || echo "")
+
+        if [[ -n "$CANDIDATE" ]] && ! echo "$EXISTING_IDS" | grep -q "^${CANDIDATE}$"; then
+            RUN_ID="$CANDIDATE"
+        fi
+
+        ATTEMPTS=$((ATTEMPTS + 1))
+    done
+
+    if [[ -z "$RUN_ID" ]]; then
+        error "Could not find new workflow run after $((ATTEMPTS * 5)) seconds."
+        return 1
+    fi
+
+    say "Monitoring run #$RUN_ID — https://github.com/$REPOSITORY/actions/runs/$RUN_ID"
+    nl
+
+    gh run watch "$RUN_ID" --repo "$REPOSITORY" || true
+
+    local CONCLUSION
+    CONCLUSION=$(gh run view "$RUN_ID" --repo "$REPOSITORY" --json conclusion --jq '.conclusion')
+
+    nl
+    if [[ "$CONCLUSION" == "success" ]]; then
+        say "✓ Deployment succeeded!"
+    else
+        error "✗ Deployment failed: $CONCLUSION"
+        return 1
+    fi
+}
+
 function deploy_staging() {
     REPOSITORY="CrowdDotDev/crowd.dev"
     WORKFLOW_FILE="lf-oracle-staging-deploy.yaml"
@@ -264,7 +313,11 @@ function deploy_staging() {
         echo "${SELECTED_SERVICES[*]}"
     )"
 
+    local EXISTING_IDS
+    EXISTING_IDS=$(gh run list --repo "$REPOSITORY" --workflow "$WORKFLOW_FILE" --limit 10 --json databaseId --jq '.[].databaseId' 2>/dev/null || echo "")
+
     gh workflow run $WORKFLOW_FILE --repo $REPOSITORY --ref $CURRENT_BRANCH -f services="$SERVICES"
+    monitor_workflow "$REPOSITORY" "$WORKFLOW_FILE" "$EXISTING_IDS"
 }
 
 function deploy_production() {
@@ -276,7 +329,11 @@ function deploy_production() {
         echo "${SELECTED_SERVICES[*]}"
     )"
 
+    local EXISTING_IDS
+    EXISTING_IDS=$(gh run list --repo "$REPOSITORY" --workflow "$WORKFLOW_FILE" --limit 10 --json databaseId --jq '.[].databaseId' 2>/dev/null || echo "")
+
     gh workflow run $WORKFLOW_FILE --repo $REPOSITORY --ref $CURRENT_BRANCH -f services="$SERVICES"
+    monitor_workflow "$REPOSITORY" "$WORKFLOW_FILE" "$EXISTING_IDS"
 }
 
 function reset_selected_services() {
diff --git a/services/apps/packages_worker/src/scripts/monitorOsspckgs.ts b/services/apps/packages_worker/src/scripts/monitorOsspckgs.ts
index 13488ac652..9cdad180a5 100644
--- a/services/apps/packages_worker/src/scripts/monitorOsspckgs.ts
+++ b/services/apps/packages_worker/src/scripts/monitorOsspckgs.ts
@@ -59,7 +59,7 @@ async function fetchJobs() {
     SELECT
       id, job_kind, status, sync_mode,
       snapshot_at, provisional_snapshot_at,
-      gcs_prefix, export_name, ecosystems,
+      gcs_prefix, export_name,
       row_count_bq, row_count_staging, row_count_pg,
       bq_bytes_billed,
       table_row_counts,

From 954a75163fc02728e113bfd3864f1033ab0df85c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Wed, 17 Jun 2026 21:11:44 +0200
Subject: [PATCH 08/29] fix: monitor column spacing fix
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../src/scripts/monitorOsspckgs.ts            | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/services/apps/packages_worker/src/scripts/monitorOsspckgs.ts b/services/apps/packages_worker/src/scripts/monitorOsspckgs.ts
index 9cdad180a5..9ae86ebb21 100644
--- a/services/apps/packages_worker/src/scripts/monitorOsspckgs.ts
+++ b/services/apps/packages_worker/src/scripts/monitorOsspckgs.ts
@@ -199,10 +199,11 @@ function padCell(s: string, len: number, bg: string) {
   // eslint-disable-next-line no-control-regex
   const visible = s.replace(/\x1b\[[0-9;]*m/g, '')
   if (visible.length > len) {
-    // Walk raw string, skip escape sequences, cut at len-1 visible chars then add …
+    // Walk raw string, skip escape sequences, cut at len-2 visible chars then add "… "
+    // (space after ellipsis ensures visual gap between columns even when truncated)
     let vis = 0
     let i = 0
-    while (i < s.length && vis < len - 1) {
+    while (i < s.length && vis < len - 2) {
       if (s[i] === '\x1b') {
         const end = s.indexOf('m', i)
         i = end !== -1 ? end + 1 : i + 1
@@ -211,7 +212,7 @@ function padCell(s: string, len: number, bg: string) {
         i++
       }
     }
-    return s.slice(0, i) + A.reset + bg + '…'
+    return s.slice(0, i) + A.reset + bg + '… '
   }
   const pad = Math.max(0, len - visible.length)
   return s + bg + ' '.repeat(pad)
@@ -287,16 +288,16 @@ function extractEcosystem(job: any) {
 const COL = {
   id: 18,
   kind: 20,
-  eco: 10,
-  status: 18,
+  eco: 15,
+  status: 26,
   mode: 6,
   bq: 10,
-  cost: 10,
+  cost: 8,
   files: 24,
-  staging: 12,
+  staging: 10,
   pg: 14,
   table: 18,
-  elapsed: 12,
+  elapsed: 10,
   chunk: 26,
   total: 24,
 }
@@ -716,7 +717,7 @@ function computeChunkEta(job: any) {
   const hist = chunkMergeHistory.get(job.id)
   if (!hist || hist.mergeStart == null) return null
 
-  let rateRowsPerMs
+  let rateRowsPerMs: number
   if (hist.completedChunks.length > 0) {
     // Exponential recency weighting: chunk i gets weight 2^i (oldest=0, newest=n-1).
     // Most recent chunk contributes ~50% of the rate; history stabilises it.

From 1f3fda05ccad3c03f534e4da8b51c2253fb082a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Wed, 17 Jun 2026 21:32:06 +0200
Subject: [PATCH 09/29] fix: dont be done prematurely
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../src/deps-dev/workflows/ingestDependencies.ts    | 13 ++++++++++++-
 .../src/deps-dev/workflows/ingestVersions.ts        | 13 ++++++++++++-
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
index df36432472..98e3670ec5 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
@@ -273,7 +273,9 @@ export async function ingestDependencies(opts: {
     for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) {
       const start = chunkIndex * filesPerChunk
       const chunk = fileNames.slice(start, start + filesPerChunk)
-      const isFinal = chunkIndex === totalChunks - 1
+      const isLastChunk = chunkIndex === totalChunks - 1
+      // For full-load (non-fill), don't mark done here — rebuild runs after the chunk loop
+      const isFinal = isLastChunk && !(opts.syncMode === 'full' && !isFill)
 
       const { rowsLoaded } = await gcsParquetToStaging({
         jobId: exportResult.jobId,
@@ -315,6 +317,15 @@ export async function ingestDependencies(opts: {
       await rebuildPackageDepsIndexes()
       await setJobStep({ jobId: exportResult.jobId, step: 'rebuild_constraints' })
       await rebuildPackageDepsConstraints()
+      // Finalize after rebuild — marks done with correct finishedAt
+      await mergeStagingToTable({
+        jobId: exportResult.jobId,
+        mergeSql: [],
+        tableNames: [],
+        isFinal: true,
+        priorRowsAffected,
+        priorTableRowCounts,
+      })
     }
   } catch (err) {
     if (opts.syncMode === 'full' && !isFill) {
diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestVersions.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestVersions.ts
index 0463c69641..af0cd716cb 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestVersions.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestVersions.ts
@@ -171,7 +171,9 @@ export async function ingestVersions(opts: {
     for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) {
       const start = chunkIndex * filesPerChunk
       const chunk = fileNames.slice(start, start + filesPerChunk)
-      const isFinal = chunkIndex === totalChunks - 1
+      const isLastChunk = chunkIndex === totalChunks - 1
+      // For full-load, don't mark done here — rebuild runs after the chunk loop
+      const isFinal = isLastChunk && opts.syncMode !== 'full'
 
       const { rowsLoaded } = await gcsParquetToStaging({
         jobId: exportResult.jobId,
@@ -210,6 +212,15 @@ export async function ingestVersions(opts: {
       await rebuildVersionsIndexes()
       await setJobStep({ jobId: exportResult.jobId, step: 'rebuild_constraints' })
       await rebuildVersionsConstraints()
+      // Finalize after rebuild — marks done with correct finishedAt
+      await mergeStagingToTable({
+        jobId: exportResult.jobId,
+        mergeSql: [],
+        tableNames: [],
+        isFinal: true,
+        priorRowsAffected,
+        priorTableRowCounts,
+      })
     }
   } catch (err) {
     if (opts.syncMode === 'full') {

From fd18287933e83eee5780b3285e13c9ae10f44655 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Thu, 18 Jun 2026 09:40:59 +0200
Subject: [PATCH 10/29] fix: index creation improvements
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../activities/managePackageDepsIndexes.ts    | 76 ++++++++++++-------
 .../activities/manageVersionsIndexes.ts       | 27 ++++---
 .../deps-dev/workflows/ingestDependencies.ts  |  6 +-
 .../src/scripts/monitorOsspckgs.ts            | 12 ++-
 4 files changed, 75 insertions(+), 46 deletions(-)

diff --git a/services/apps/packages_worker/src/deps-dev/activities/managePackageDepsIndexes.ts b/services/apps/packages_worker/src/deps-dev/activities/managePackageDepsIndexes.ts
index ed28891567..12ffac79fe 100644
--- a/services/apps/packages_worker/src/deps-dev/activities/managePackageDepsIndexes.ts
+++ b/services/apps/packages_worker/src/deps-dev/activities/managePackageDepsIndexes.ts
@@ -96,18 +96,20 @@ export async function rebuildPackageDepsIndexes(): Promise<{
   const existing: Array<{ indexdef: string }> = await qx.select(NON_CONSTRAINT_INDEXES_SQL)
   const existingDefs = existing.map((r: { indexdef: string }) => r.indexdef.toLowerCase())
 
+  const toRebuild = SECONDARY_INDEXES.filter(
+    (idx) => !existingDefs.some((def: string) => def.includes(`(${idx.columns})`)),
+  )
   const rebuilt: string[] = []
 
-  for (const idx of SECONDARY_INDEXES) {
-    const alreadyExists = existingDefs.some((def: string) => def.includes(`(${idx.columns})`))
-    if (alreadyExists) {
-      log.info({ columns: idx.columns }, 'Index already exists, skipping')
-      continue
-    }
-    log.info({ columns: idx.columns }, 'Creating index on package_dependencies')
-    await qx.result(idx.createSql)
-    rebuilt.push(idx.columns)
-  }
+  // Build indexes in parallel — each on its own connection so they run concurrently.
+  await Promise.all(
+    toRebuild.map(async (idx) => {
+      const conn = await getPackagesDb()
+      log.info({ columns: idx.columns }, 'Creating index on package_dependencies')
+      await conn.result(idx.createSql)
+      rebuilt.push(idx.columns)
+    }),
+  )
 
   // Remove cross-chunk duplicates before rebuilding the UNIQUE constraint.
   // DISTINCT ON deduplicates within a single chunk, but the same (root, dep) pair can appear in
@@ -118,27 +120,43 @@ export async function rebuildPackageDepsIndexes(): Promise<{
   // Run per partition (depends_on_id % 64 = p) so each iteration prunes to one of the 64
   // partitions instead of scanning all 1.15B rows at once. Same total rows read; each pass
   // fits in work_mem and avoids cross-partition sort.
+  // Run in parallel batches of DEDUP_CONCURRENCY to cut wall-clock from ~10h to ~1-2h.
   const NUM_PARTITIONS = 64
+  const DEDUP_CONCURRENCY = 8
   let totalDedupDeleted = 0
-  for (let p = 0; p < NUM_PARTITIONS; p++) {
-    const result = await qx.result(`
-      DELETE FROM package_dependencies pd
-      USING (
-        SELECT id, depends_on_id
-        FROM (
-          SELECT id, depends_on_id,
-                 ROW_NUMBER() OVER (
-                   PARTITION BY version_id, depends_on_id, dependency_kind
-                   ORDER BY id
-                 ) AS rn
-          FROM package_dependencies
-          WHERE depends_on_id % ${NUM_PARTITIONS} = ${p}
-        ) sub
-        WHERE rn > 1
-      ) dupes
-      WHERE pd.id = dupes.id AND pd.depends_on_id = dupes.depends_on_id
-    `)
-    totalDedupDeleted += result
+  for (let batch = 0; batch < NUM_PARTITIONS; batch += DEDUP_CONCURRENCY) {
+    const partitions = Array.from(
+      { length: Math.min(DEDUP_CONCURRENCY, NUM_PARTITIONS - batch) },
+      (_, i) => batch + i,
+    )
+    const counts = await Promise.all(
+      partitions.map(async (p) => {
+        const conn = await getPackagesDb()
+        // work_mem (not maintenance_work_mem) controls sort memory for window functions.
+        // 2GB avoids disk spill during the ROW_NUMBER() sort on each partition.
+        return conn.tx(async (tx) => {
+          await tx.result(`SET LOCAL work_mem = '2GB'`)
+          return tx.result(`
+            DELETE FROM package_dependencies pd
+            USING (
+              SELECT id, depends_on_id
+              FROM (
+                SELECT id, depends_on_id,
+                       ROW_NUMBER() OVER (
+                         PARTITION BY version_id, depends_on_id, dependency_kind
+                         ORDER BY id
+                       ) AS rn
+                FROM package_dependencies
+                WHERE depends_on_id % ${NUM_PARTITIONS} = ${p}
+              ) sub
+              WHERE rn > 1
+            ) dupes
+            WHERE pd.id = dupes.id AND pd.depends_on_id = dupes.depends_on_id
+          `)
+        })
+      }),
+    )
+    totalDedupDeleted += counts.reduce((sum, n) => sum + n, 0)
   }
   log.info(
     { rowsDeleted: totalDedupDeleted },
diff --git a/services/apps/packages_worker/src/deps-dev/activities/manageVersionsIndexes.ts b/services/apps/packages_worker/src/deps-dev/activities/manageVersionsIndexes.ts
index 0d9eda343a..b41b9a3b26 100644
--- a/services/apps/packages_worker/src/deps-dev/activities/manageVersionsIndexes.ts
+++ b/services/apps/packages_worker/src/deps-dev/activities/manageVersionsIndexes.ts
@@ -86,20 +86,23 @@ export async function rebuildVersionsIndexes(): Promise<{
   const existing: Array<{ indexdef: string }> = await qx.select(NON_CONSTRAINT_INDEXES_SQL)
   const existingDefs = existing.map((r) => r.indexdef.toLowerCase())
 
+  const toRebuild = SECONDARY_INDEXES.filter(
+    (idx) => !existingDefs.some((def) => def.includes(idx.matchString ?? `(${idx.columns})`)),
+  )
   const rebuilt: string[] = []
 
-  for (const idx of SECONDARY_INDEXES) {
-    const alreadyExists = existingDefs.some((def) =>
-      def.includes(idx.matchString ?? `(${idx.columns})`),
-    )
-    if (alreadyExists) {
-      log.info({ columns: idx.columns }, 'Index already exists, skipping')
-      continue
-    }
-    log.info({ columns: idx.columns }, 'Creating index on versions')
-    await qx.result(idx.createSql)
-    rebuilt.push(idx.columns)
-  }
+  // Build indexes in parallel — each needs its own connection.
+  // maintenance_work_mem per connection: with 32 partitions and default 64MB, PG spills to
+  // disk on every partition; 2GB lets the sort fit in RAM and cuts build time dramatically.
+  // Build indexes in parallel — each on its own connection so they run concurrently.
+  await Promise.all(
+    toRebuild.map(async (idx) => {
+      const conn = await getPackagesDb()
+      log.info({ columns: idx.columns }, 'Creating index on versions')
+      await conn.result(idx.createSql)
+      rebuilt.push(idx.columns)
+    }),
+  )
 
   // Remove cross-chunk duplicates before rebuilding the UNIQUE constraint.
   // versions is HASH-partitioned by package_id (32 partitions). Loop over each partition table
diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
index 98e3670ec5..195e794394 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
@@ -36,8 +36,8 @@ const { createVersionsLookup } = proxyActivities<typeof depsDevActivities>({
 const { dropPackageDepsIndexes, rebuildPackageDepsIndexes } = proxyActivities<
   typeof depsDevActivities
 >({
-  // Index builds on 1B+ rows can take hours — long timeout required.
-  startToCloseTimeout: '12 hours',
+  // Index builds + parallel dedup on 1B+ rows — 24h covers worst-case sequential retry.
+  startToCloseTimeout: '24 hours',
   retry: { maximumAttempts: 2, initialInterval: '1 minute' },
 })
 
@@ -45,7 +45,7 @@ const { dropPackageDepsConstraints, rebuildPackageDepsConstraints } = proxyActiv
   typeof depsDevActivities
 >({
   // FK validation on 1B+ rows can take hours.
-  startToCloseTimeout: '12 hours',
+  startToCloseTimeout: '24 hours',
   retry: { maximumAttempts: 2, initialInterval: '1 minute' },
 })
 
diff --git a/services/apps/packages_worker/src/scripts/monitorOsspckgs.ts b/services/apps/packages_worker/src/scripts/monitorOsspckgs.ts
index 9ae86ebb21..e2e325daaf 100644
--- a/services/apps/packages_worker/src/scripts/monitorOsspckgs.ts
+++ b/services/apps/packages_worker/src/scripts/monitorOsspckgs.ts
@@ -642,6 +642,7 @@ function computeStuckIds(jobList: any[]): Set<number> {
   return result
 }
 let selected = 0
+let scrollOffset = 0
 let detailOpen = false
 let lastRefresh: string | null = null
 let error: string | null = null
@@ -837,13 +838,19 @@ function render() {
   const detailHeight = detailOpen ? Math.min(detailLines.length + 2, Math.floor(rows * 0.55)) : 0
   const listHeight = rows - listStart - detailHeight - 1
 
+  // Keep scrollOffset in bounds so selected row is always visible
+  if (selected < scrollOffset) scrollOffset = selected
+  if (selected >= scrollOffset + listHeight) scrollOffset = selected - listHeight + 1
+  scrollOffset = Math.max(0, Math.min(scrollOffset, Math.max(0, jobs.length - listHeight)))
+
   const listEnd = listStart + listHeight
   for (let i = 0; i < listHeight; i++) {
-    const job = jobs[i]
+    const jobIndex = scrollOffset + i
+    const job = jobs[jobIndex]
     if (!job) {
       writeln(listStart + i, 1, '')
     } else {
-      writeln(listStart + i, 1, tableRow(job, i === selected))
+      writeln(listStart + i, 1, tableRow(job, jobIndex === selected))
     }
   }
 
@@ -876,6 +883,7 @@ async function refresh() {
     tableCounts = newTableCounts
     watermarks = newWatermarks
     if (selected >= jobs.length) selected = Math.max(0, jobs.length - 1)
+    scrollOffset = Math.max(0, Math.min(scrollOffset, Math.max(0, jobs.length - 1)))
     stuckIds = computeStuckIds(jobs)
     lastRefresh = new Date().toLocaleTimeString()
     error = null

From 755a317abcbec3b7ff967e1b4a86650b23e36e3e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Thu, 18 Jun 2026 10:03:39 +0200
Subject: [PATCH 11/29] chore: script to fix unique index on
 packages_dependents
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 services/apps/packages_worker/package.json    |   2 +
 .../src/scripts/dedupPackageDeps.ts           | 110 ++++++++++++++++++
 2 files changed, 112 insertions(+)
 create mode 100644 services/apps/packages_worker/src/scripts/dedupPackageDeps.ts

diff --git a/services/apps/packages_worker/package.json b/services/apps/packages_worker/package.json
index 52abd880b0..4ef19ef5d0 100644
--- a/services/apps/packages_worker/package.json
+++ b/services/apps/packages_worker/package.json
@@ -36,6 +36,8 @@
     "backfill:stewardship:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=stewardship-backfill LOG_LEVEL=info tsx src/bin/stewardship-backfill.ts",
     "monitor:osspckgs": "SERVICE=bq-dataset-ingest tsx src/scripts/monitorOsspckgs.ts",
     "monitor:osspckgs:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=bq-dataset-ingest tsx src/scripts/monitorOsspckgs.ts",
+    "dedup-package-deps": "SERVICE=bq-dataset-ingest tsx src/scripts/dedupPackageDeps.ts",
+    "dedup-package-deps:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=bq-dataset-ingest tsx src/scripts/dedupPackageDeps.ts",
     "lint": "npx eslint --ext .ts src --max-warnings=0",
     "format": "npx prettier --write \"src/**/*.ts\"",
     "format-check": "npx prettier --check .",
diff --git a/services/apps/packages_worker/src/scripts/dedupPackageDeps.ts b/services/apps/packages_worker/src/scripts/dedupPackageDeps.ts
new file mode 100644
index 0000000000..8ebf6b5c56
--- /dev/null
+++ b/services/apps/packages_worker/src/scripts/dedupPackageDeps.ts
@@ -0,0 +1,110 @@
+#!/usr/bin/env tsx
+
+/**
+ * Dedup cross-chunk duplicate rows from package_dependencies, then rebuild
+ * the UNIQUE constraint (version_id, depends_on_id, dependency_kind).
+ *
+ * Run after a full-load where the constraint was dropped before bulk INSERT.
+ *
+ * Usage:
+ *   pnpm dedup-package-deps [--concurrency <n>] [--dry-run]
+ *
+ *   --concurrency <n>  Partitions to process in parallel (default: 8).
+ *   --dry-run          Count duplicates without deleting.
+ */
+
+import { getServiceChildLogger } from '@crowd/logging'
+
+import { getPackagesDb } from '../db'
+
+const log = getServiceChildLogger('dedupPackageDeps')
+
+const NUM_PARTITIONS = 64
+
+async function processPartition(p: number, dryRun: boolean): Promise<number> {
+  const conn = await getPackagesDb()
+  return conn.tx(async (tx) => {
+    await tx.result(`SET LOCAL work_mem = '2GB'`)
+    if (dryRun) {
+      const row = await tx.selectOne(`
+        SELECT COUNT(*) AS cnt
+        FROM (
+          SELECT ROW_NUMBER() OVER (
+                   PARTITION BY version_id, depends_on_id, dependency_kind
+                   ORDER BY id
+                 ) AS rn
+          FROM package_dependencies
+          WHERE depends_on_id % ${NUM_PARTITIONS} = ${p}
+        ) sub
+        WHERE rn > 1
+      `)
+      return Number(row.cnt)
+    }
+    return tx.result(`
+      DELETE FROM package_dependencies pd
+      USING (
+        SELECT id, depends_on_id FROM (
+          SELECT id, depends_on_id,
+                 ROW_NUMBER() OVER (
+                   PARTITION BY version_id, depends_on_id, dependency_kind
+                   ORDER BY id
+                 ) AS rn
+          FROM package_dependencies
+          WHERE depends_on_id % ${NUM_PARTITIONS} = ${p}
+        ) sub
+        WHERE rn > 1
+      ) dupes
+      WHERE pd.id = dupes.id AND pd.depends_on_id = dupes.depends_on_id
+    `)
+  })
+}
+
+async function main(): Promise<void> {
+  const args = process.argv.slice(2)
+  const concurrencyIdx = args.indexOf('--concurrency')
+  const concurrency = concurrencyIdx !== -1 ? Number(args[concurrencyIdx + 1]) : 8
+  const dryRun = args.includes('--dry-run')
+
+  if (dryRun) log.info('DRY RUN — counting only')
+  log.info({ concurrency, numPartitions: NUM_PARTITIONS }, 'Starting dedup')
+
+  let total = 0
+  for (let batch = 0; batch < NUM_PARTITIONS; batch += concurrency) {
+    const partitions = Array.from(
+      { length: Math.min(concurrency, NUM_PARTITIONS - batch) },
+      (_, i) => batch + i,
+    )
+    const counts = await Promise.all(partitions.map((p) => processPartition(p, dryRun)))
+    const batchTotal = counts.reduce((s, n) => s + n, 0)
+    total += batchTotal
+    log.info({ partitions, batchTotal, total }, dryRun ? 'batch counted' : 'batch done')
+  }
+
+  log.info({ total }, dryRun ? 'duplicates found' : 'dedup complete')
+
+  if (dryRun) return
+
+  const qx = await getPackagesDb()
+  const existing = await qx.selectOneOrNone(`
+    SELECT conname FROM pg_constraint c
+    JOIN pg_class t ON t.oid = c.conrelid
+    WHERE t.relname = 'package_dependencies' AND c.contype = 'u'
+    LIMIT 1
+  `)
+  if (existing) {
+    log.info({ constraint: existing.conname }, 'UNIQUE constraint already exists')
+  } else {
+    log.info('Rebuilding UNIQUE constraint')
+    await qx.result(
+      `ALTER TABLE package_dependencies ADD UNIQUE (version_id, depends_on_id, dependency_kind)`,
+    )
+    log.info('UNIQUE constraint rebuilt')
+  }
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch((err) => {
+    log.error(err, 'Fatal error')
+    process.exit(1)
+  })

From 73ad88b1cd94dd7226d775e4b33f9d538863ce56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Fri, 19 Jun 2026 11:55:25 +0200
Subject: [PATCH 12/29] fix: better index creation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../packages_worker/src/scripts/dedupPackageDeps.ts    | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/services/apps/packages_worker/src/scripts/dedupPackageDeps.ts b/services/apps/packages_worker/src/scripts/dedupPackageDeps.ts
index 8ebf6b5c56..f5f5fbd7f3 100644
--- a/services/apps/packages_worker/src/scripts/dedupPackageDeps.ts
+++ b/services/apps/packages_worker/src/scripts/dedupPackageDeps.ts
@@ -22,8 +22,9 @@ const log = getServiceChildLogger('dedupPackageDeps')
 const NUM_PARTITIONS = 64
 
 async function processPartition(p: number, dryRun: boolean): Promise<number> {
+  log.info({ partition: p }, dryRun ? 'counting partition' : 'deduping partition')
   const conn = await getPackagesDb()
-  return conn.tx(async (tx) => {
+  const count = await conn.tx(async (tx) => {
     await tx.result(`SET LOCAL work_mem = '2GB'`)
     if (dryRun) {
       const row = await tx.selectOne(`
@@ -57,6 +58,8 @@ async function processPartition(p: number, dryRun: boolean): Promise<number> {
       WHERE pd.id = dupes.id AND pd.depends_on_id = dupes.depends_on_id
     `)
   })
+  log.info({ partition: p, count }, dryRun ? 'partition counted' : 'partition done')
+  return count
 }
 
 async function main(): Promise<void> {
@@ -75,9 +78,8 @@ async function main(): Promise<void> {
       (_, i) => batch + i,
     )
     const counts = await Promise.all(partitions.map((p) => processPartition(p, dryRun)))
-    const batchTotal = counts.reduce((s, n) => s + n, 0)
-    total += batchTotal
-    log.info({ partitions, batchTotal, total }, dryRun ? 'batch counted' : 'batch done')
+    total += counts.reduce((s, n) => s + n, 0)
+    log.info({ partitions, total }, dryRun ? 'batch counted' : 'batch done')
   }
 
   log.info({ total }, dryRun ? 'duplicates found' : 'dedup complete')

From 0f33398538bde9663f2aff4c1e8a336df041ad89 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Fri, 19 Jun 2026 13:59:58 +0200
Subject: [PATCH 13/29] fix: comments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 scripts/cli                                                   | 2 +-
 .../src/deps-dev/activities/manageVersionsIndexes.ts          | 4 ++--
 .../src/deps-dev/workflows/ingestDependencies.ts              | 2 +-
 services/apps/packages_worker/src/scripts/dedupPackageDeps.ts | 1 -
 4 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/scripts/cli b/scripts/cli
index 0d522c9620..84b678f83e 100755
--- a/scripts/cli
+++ b/scripts/cli
@@ -273,7 +273,7 @@ function monitor_workflow() {
             --workflow "$WORKFLOW_FILE" \
             --limit 1 \
             --json databaseId \
-            --jq '.[0].databaseId' 2>/dev/null || echo "")
+            --jq '.[0].databaseId // empty' 2>/dev/null || echo "")
 
         if [[ -n "$CANDIDATE" ]] && ! echo "$EXISTING_IDS" | grep -q "^${CANDIDATE}$"; then
             RUN_ID="$CANDIDATE"
diff --git a/services/apps/packages_worker/src/deps-dev/activities/manageVersionsIndexes.ts b/services/apps/packages_worker/src/deps-dev/activities/manageVersionsIndexes.ts
index b41b9a3b26..895e66ae67 100644
--- a/services/apps/packages_worker/src/deps-dev/activities/manageVersionsIndexes.ts
+++ b/services/apps/packages_worker/src/deps-dev/activities/manageVersionsIndexes.ts
@@ -91,13 +91,13 @@ export async function rebuildVersionsIndexes(): Promise<{
   )
   const rebuilt: string[] = []
 
-  // Build indexes in parallel — each needs its own connection.
+  // Build indexes in parallel — each on its own connection so they run concurrently.
   // maintenance_work_mem per connection: with 32 partitions and default 64MB, PG spills to
   // disk on every partition; 2GB lets the sort fit in RAM and cuts build time dramatically.
-  // Build indexes in parallel — each on its own connection so they run concurrently.
   await Promise.all(
     toRebuild.map(async (idx) => {
       const conn = await getPackagesDb()
+      await conn.result(`SET LOCAL maintenance_work_mem = '2GB'`)
       log.info({ columns: idx.columns }, 'Creating index on versions')
       await conn.result(idx.createSql)
       rebuilt.push(idx.columns)
diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
index 195e794394..2af62f027e 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
@@ -230,7 +230,7 @@ export async function ingestDependencies(opts: {
     runId: opts.runId,
     syncMode: opts.syncMode,
     snapshotAt: opts.today,
-    maxBytesGb: opts.syncMode === 'full' ? 25000 : 10000,
+    maxBytesGb: opts.syncMode === 'full' || isFill ? 25000 : 10000,
     reuseExports: opts.reuseExports,
     exportName: opts.exportName,
     ecosystems: opts.ecosystems,
diff --git a/services/apps/packages_worker/src/scripts/dedupPackageDeps.ts b/services/apps/packages_worker/src/scripts/dedupPackageDeps.ts
index f5f5fbd7f3..303a03ff55 100644
--- a/services/apps/packages_worker/src/scripts/dedupPackageDeps.ts
+++ b/services/apps/packages_worker/src/scripts/dedupPackageDeps.ts
@@ -12,7 +12,6 @@
  *   --concurrency <n>  Partitions to process in parallel (default: 8).
  *   --dry-run          Count duplicates without deleting.
  */
-
 import { getServiceChildLogger } from '@crowd/logging'
 
 import { getPackagesDb } from '../db'

From e2ca9511e4668c255a3e4d0f2ce49aa101305685 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Fri, 19 Jun 2026 14:14:46 +0200
Subject: [PATCH 14/29] fix: comments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../src/deps-dev/activities/manageVersionsIndexes.ts      | 8 +++++---
 .../apps/packages_worker/src/scripts/dedupPackageDeps.ts  | 3 +++
 .../libs/data-access-layer/src/osspckgs/ingestJobs.ts     | 2 +-
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/services/apps/packages_worker/src/deps-dev/activities/manageVersionsIndexes.ts b/services/apps/packages_worker/src/deps-dev/activities/manageVersionsIndexes.ts
index 895e66ae67..239385e609 100644
--- a/services/apps/packages_worker/src/deps-dev/activities/manageVersionsIndexes.ts
+++ b/services/apps/packages_worker/src/deps-dev/activities/manageVersionsIndexes.ts
@@ -97,9 +97,11 @@ export async function rebuildVersionsIndexes(): Promise<{
   await Promise.all(
     toRebuild.map(async (idx) => {
       const conn = await getPackagesDb()
-      await conn.result(`SET LOCAL maintenance_work_mem = '2GB'`)
-      log.info({ columns: idx.columns }, 'Creating index on versions')
-      await conn.result(idx.createSql)
+      await conn.tx(async (t) => {
+        await t.result(`SET LOCAL maintenance_work_mem = '2GB'`)
+        log.info({ columns: idx.columns }, 'Creating index on versions')
+        await t.result(idx.createSql)
+      })
       rebuilt.push(idx.columns)
     }),
   )
diff --git a/services/apps/packages_worker/src/scripts/dedupPackageDeps.ts b/services/apps/packages_worker/src/scripts/dedupPackageDeps.ts
index 303a03ff55..64501037b1 100644
--- a/services/apps/packages_worker/src/scripts/dedupPackageDeps.ts
+++ b/services/apps/packages_worker/src/scripts/dedupPackageDeps.ts
@@ -65,6 +65,9 @@ async function main(): Promise<void> {
   const args = process.argv.slice(2)
   const concurrencyIdx = args.indexOf('--concurrency')
   const concurrency = concurrencyIdx !== -1 ? Number(args[concurrencyIdx + 1]) : 8
+  if (!Number.isInteger(concurrency) || concurrency <= 0) {
+    throw new Error(`--concurrency must be a positive integer, got: ${args[concurrencyIdx + 1]}`)
+  }
   const dryRun = args.includes('--dry-run')
 
   if (dryRun) log.info('DRY RUN — counting only')
diff --git a/services/libs/data-access-layer/src/osspckgs/ingestJobs.ts b/services/libs/data-access-layer/src/osspckgs/ingestJobs.ts
index 7d2b486136..9606035e23 100644
--- a/services/libs/data-access-layer/src/osspckgs/ingestJobs.ts
+++ b/services/libs/data-access-layer/src/osspckgs/ingestJobs.ts
@@ -312,7 +312,7 @@ export async function mergeJobTableRowCounts(
 ): Promise<void> {
   await qx.result(
     `UPDATE osspckgs_ingest_jobs
-     SET table_row_counts = COALESCE(table_row_counts, '{}') || $(kv)
+     SET table_row_counts = COALESCE(table_row_counts, '{}'::jsonb) || $(kv)::jsonb
      WHERE id = $(jobId)`,
     { jobId, kv },
   )

From d36917168f4df2acca078d686bfc3d20d9341291 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Fri, 19 Jun 2026 17:51:44 +0200
Subject: [PATCH 15/29] fix: comments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../src/deps-dev/workflows/ingestDependencies.ts                | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
index 2af62f027e..3b128cecf4 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
@@ -233,7 +233,7 @@ export async function ingestDependencies(opts: {
     maxBytesGb: opts.syncMode === 'full' || isFill ? 25000 : 10000,
     reuseExports: opts.reuseExports,
     exportName: opts.exportName,
-    ecosystems: opts.ecosystems,
+    ecosystems,
   })
 
   const { fileNames, rowCounts } = await listParquetFiles({ gcsPrefix: exportResult.gcsPrefix })

From f56818abe38fdc30a4eac25c2088d6d0087525c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Fri, 19 Jun 2026 17:59:04 +0200
Subject: [PATCH 16/29] fix: comments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../src/deps-dev/workflows/ingestDependencies.ts               | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
index 3b128cecf4..02d03c7c91 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
@@ -216,8 +216,9 @@ export async function ingestDependencies(opts: {
   fillConstraints?: boolean // re-export full BQ data, upsert version_constraint only where NULL
 }): Promise<{ rowCountBq: number }> {
   const ecosystems = opts.ecosystems ?? DEPS_DEFAULT_ECOSYSTEMS
-  const tableOption = opts.depsTableOption ?? 'A'
   const isFill = opts.fillConstraints === true
+  // Fill mode forces Option A — Option B selects NULL for version_constraint, making the fill a no-op.
+  const tableOption = isFill ? 'A' : (opts.depsTableOption ?? 'A')
   // Fill mode always uses full SQL — needs all rows to find which have NULL version_constraint in DB.
   const sql =
     opts.syncMode === 'full' || isFill

From 26f0680b691cf3a7a5b6252f31a54e019e702d1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Fri, 19 Jun 2026 18:11:20 +0200
Subject: [PATCH 17/29] fix: comments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../src/deps-dev/queries/depsSql.ts           | 20 +++++++++++--------
 .../deps-dev/workflows/ingestDependencies.ts  |  2 +-
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/services/apps/packages_worker/src/deps-dev/queries/depsSql.ts b/services/apps/packages_worker/src/deps-dev/queries/depsSql.ts
index 37aaa3efcd..b713ed343c 100644
--- a/services/apps/packages_worker/src/deps-dev/queries/depsSql.ts
+++ b/services/apps/packages_worker/src/deps-dev/queries/depsSql.ts
@@ -159,12 +159,12 @@ WHERE l.to_name IS NULL`,
     AND g.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${today}', INTERVAL 1 DAY))
 )`,
       `watermark_go AS (
-  SELECT g.Name, g.Version, d.Name AS to_name
+  SELECT g.Name, g.Version, d.Name AS to_name, d.Requirement
   FROM \`bigquery-public-data.deps_dev_v1.GoRequirements\` g,
   UNNEST(g.DirectDependencies) AS d
   WHERE g.SnapshotAt >= TIMESTAMP('${watermark}')
     AND g.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${watermark}', INTERVAL 1 DAY))
-  GROUP BY g.Name, g.Version, d.Name
+  GROUP BY g.Name, g.Version, d.Name, d.Requirement
 )`,
     )
     selects.push(
@@ -172,6 +172,7 @@ WHERE l.to_name IS NULL`,
 FROM today_go t
 LEFT JOIN watermark_go l
   ON l.Name = t.root_name AND l.Version = t.root_version AND l.to_name = t.to_name
+  AND l.Requirement = t.version_constraint
 WHERE l.to_name IS NULL`,
     )
   }
@@ -193,13 +194,13 @@ WHERE l.to_name IS NULL`,
     AND n.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${today}', INTERVAL 1 DAY))
 )`,
       `watermark_nuget AS (
-  SELECT n.Name, n.Version, dep.Name AS to_name
+  SELECT n.Name, n.Version, dep.Name AS to_name, dep.Requirement
   FROM \`bigquery-public-data.deps_dev_v1.NuGetRequirements\` n,
   UNNEST(n.DependencyGroups) AS grp,
   UNNEST(grp.Dependencies) AS dep
   WHERE n.SnapshotAt >= TIMESTAMP('${watermark}')
     AND n.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${watermark}', INTERVAL 1 DAY))
-  GROUP BY n.Name, n.Version, dep.Name
+  GROUP BY n.Name, n.Version, dep.Name, dep.Requirement
 )`,
     )
     selects.push(
@@ -207,6 +208,7 @@ WHERE l.to_name IS NULL`,
 FROM today_nuget t
 LEFT JOIN watermark_nuget l
   ON l.Name = t.root_name AND l.Version = t.root_version AND l.to_name = t.to_name
+  AND l.Requirement = t.version_constraint
 WHERE l.to_name IS NULL`,
     )
   }
@@ -279,12 +281,12 @@ WHERE l.to_name IS NULL`,
     AND g.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${today}', INTERVAL 1 DAY))
 )`,
       `watermark_go AS (
-  SELECT g.Name, g.Version, d.Name AS to_name
+  SELECT g.Name, g.Version, d.Name AS to_name, d.Requirement
   FROM \`bigquery-public-data.deps_dev_v1.GoRequirements\` g,
   UNNEST(g.DirectDependencies) AS d
   WHERE g.SnapshotAt >= TIMESTAMP('${watermark}')
     AND g.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${watermark}', INTERVAL 1 DAY))
-  GROUP BY g.Name, g.Version, d.Name
+  GROUP BY g.Name, g.Version, d.Name, d.Requirement
 )`,
     )
     selects.push(
@@ -292,6 +294,7 @@ WHERE l.to_name IS NULL`,
 FROM today_go t
 LEFT JOIN watermark_go l
   ON l.Name = t.root_name AND l.Version = t.root_version AND l.to_name = t.to_name
+  AND l.Requirement = t.version_constraint
 WHERE l.to_name IS NULL`,
     )
   }
@@ -313,13 +316,13 @@ WHERE l.to_name IS NULL`,
     AND n.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${today}', INTERVAL 1 DAY))
 )`,
       `watermark_nuget AS (
-  SELECT n.Name, n.Version, dep.Name AS to_name
+  SELECT n.Name, n.Version, dep.Name AS to_name, dep.Requirement
   FROM \`bigquery-public-data.deps_dev_v1.NuGetRequirements\` n,
   UNNEST(n.DependencyGroups) AS grp,
   UNNEST(grp.Dependencies) AS dep
   WHERE n.SnapshotAt >= TIMESTAMP('${watermark}')
     AND n.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${watermark}', INTERVAL 1 DAY))
-  GROUP BY n.Name, n.Version, dep.Name
+  GROUP BY n.Name, n.Version, dep.Name, dep.Requirement
 )`,
     )
     selects.push(
@@ -327,6 +330,7 @@ WHERE l.to_name IS NULL`,
 FROM today_nuget t
 LEFT JOIN watermark_nuget l
   ON l.Name = t.root_name AND l.Version = t.root_version AND l.to_name = t.to_name
+  AND l.Requirement = t.version_constraint
 WHERE l.to_name IS NULL`,
     )
   }
diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
index 02d03c7c91..0309c079bb 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
@@ -252,7 +252,7 @@ export async function ingestDependencies(opts: {
   }
 
   await setJobStep({ jobId: exportResult.jobId, step: 'creating_lookup' })
-  await createVersionsLookup({ ecosystems: opts.ecosystems })
+  await createVersionsLookup({ ecosystems })
 
   if (opts.syncMode === 'full' && !isFill) {
     await setJobStep({ jobId: exportResult.jobId, step: 'drop_constraints' })

From 8656ec633b34777cfc088910011659693dcfded7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Mon, 22 Jun 2026 22:41:11 +0200
Subject: [PATCH 18/29] fix: dedup deps.dev incremental edges, guard corrupt
 snapshots (CM-1281)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../packages_worker/src/deps-dev/README.md    |   2 +-
 .../activities/checkEdgeSnapshotQuality.ts    | 200 +++++++++++
 .../src/deps-dev/activities/index.ts          |   1 +
 .../src/deps-dev/queries/depsSql.ts           | 322 +++++++++---------
 .../deps-dev/workflows/bootstrapOsspckgs.ts   |  42 ++-
 .../deps-dev/workflows/ingestDependencies.ts  |  27 +-
 6 files changed, 407 insertions(+), 187 deletions(-)
 create mode 100644 services/apps/packages_worker/src/deps-dev/activities/checkEdgeSnapshotQuality.ts

diff --git a/services/apps/packages_worker/src/deps-dev/README.md b/services/apps/packages_worker/src/deps-dev/README.md
index 8e283a9e4e..b04b0cb0f5 100644
--- a/services/apps/packages_worker/src/deps-dev/README.md
+++ b/services/apps/packages_worker/src/deps-dev/README.md
@@ -50,7 +50,7 @@ The mode-specific key takes precedence over the generic key. Value must be a pos
 | `BQ_DATASET_INGEST_PACKAGES_FULL_MAX_BQ_GB`        |         6000 | `packages`             | Full only (set in `ingestPackages.ts`)                    |
 | `BQ_DATASET_INGEST_PACKAGES_INCREMENTAL_MAX_BQ_GB` |          400 | `packages`             | Incremental only (set in `ingestPackages.ts`)             |
 | `BQ_DATASET_INGEST_VERSIONS_MAX_BQ_GB`             |          400 | `versions`             |                                                           |
-| `BQ_DATASET_INGEST_PACKAGE_DEPENDENCIES_MAX_BQ_GB` |        10000 | `package_dependencies` | Incremental always scans ~2 full-day partitions (~3.85TB) |
+| `BQ_DATASET_INGEST_PACKAGE_DEPENDENCIES_MAX_BQ_GB` | 25000 full / 10000 incr | `package_dependencies` | Full scans `*Latest`. Incremental is a snapshot edge-diff (today vs watermark partitions of `DependencyGraphEdges` + `GoRequirements` + `NuGetRequirements`), matched on `(root, to_name)` excluding the resolved `to_version` to drop re-resolution churn (~4.1TB, Option A). Mode-specific `…_FULL_…` / `…_INCREMENTAL_…` keys take precedence. |
 | `BQ_DATASET_INGEST_REPOS_MAX_BQ_GB`                |         2000 | `repos`                |                                                           |
 | `BQ_DATASET_INGEST_PACKAGE_REPOS_MAX_BQ_GB`        |         2000 | `package_repos`        |                                                           |
 | `BQ_DATASET_INGEST_ADVISORIES_MAX_BQ_GB`           |           10 | `advisories`           |                                                           |
diff --git a/services/apps/packages_worker/src/deps-dev/activities/checkEdgeSnapshotQuality.ts b/services/apps/packages_worker/src/deps-dev/activities/checkEdgeSnapshotQuality.ts
new file mode 100644
index 0000000000..07aa432140
--- /dev/null
+++ b/services/apps/packages_worker/src/deps-dev/activities/checkEdgeSnapshotQuality.ts
@@ -0,0 +1,200 @@
+import { SlackChannel, SlackPersona, sendSlackNotification } from '@crowd/slack'
+
+import { bigquery } from '../config'
+import { assertSnapshotDate } from '../queries/depsSql'
+
+// deps.dev ships weekly full snapshots of the resolved dependency graph in
+// DependencyGraphEdges (NPM/MAVEN/PYPI/CARGO). On 2026-06-11 and 2026-06-15 that
+// pipeline shipped corrupt: every (package, version) collapsed to 1–2 arbitrary edges,
+// each duplicated exactly ~100×. An incremental package_dependencies run against a
+// corrupt snapshot exports ~550M garbage rows and burns ~5h before PG's
+// ON CONFLICT DO NOTHING discards 99.5% of them — wasteful, though not data-destroying.
+//
+// This guard runs BEFORE the BQ export. It probes a small set of high-fanout canary packages
+// over the today partition and rejects the snapshot when the duplication ratio
+// (rows / distinct edges) is far above the healthy baseline of ~1.0 (corrupt ≈ 100).
+//
+// Cost: the filter is written as `(System = x AND Name IN (...))` OR-groups so it prunes on the
+// table's clustering (System, Name, Version) — measured ~3.8GB billed (≈ $0.02), vs ~1.76TB if
+// the predicate is a `(System, Name) IN UNNEST(structs)` tuple (which defeats cluster pruning).
+//
+// GO/NUGET are sourced from manifest tables (GoRequirements/NuGetRequirements) which are not
+// produced by the resolution pipeline and were unaffected, so they are not probed here.
+
+export interface CheckEdgeSnapshotQualityInput {
+  snapshotDate: string // YYYY-MM-DD — the resolved BQ partition date being ingested
+  ecosystems: string[]
+}
+
+export interface CanaryStat {
+  system: string
+  name: string
+  rows: number
+  edges: number
+  ratio: number | null // rows / distinct edges; ~1.0 healthy, ~100 when corrupt
+  present: boolean
+}
+
+export interface CheckEdgeSnapshotQualityOutput {
+  ok: boolean
+  reason?: string
+  canaries: CanaryStat[]
+}
+
+// Systems whose direct deps come from the resolved graph (DependencyGraphEdges).
+// Mirrors EDGE_SYSTEMS in queries/depsSql.ts — GO/NUGET are manifest-sourced and excluded.
+const EDGE_SYSTEMS = new Set(['NPM', 'MAVEN', 'PYPI', 'CARGO'])
+
+// High-fanout packages that every healthy snapshot resolves with MANY distinct direct edges.
+// All verified present with ratio ≈ 1.0 on the healthy 2026-06-01 snapshot. Deliberately avoid
+// dependency-free packages (e.g. lodash, urllib3) — they have zero direct edges even when healthy,
+// so they'd masquerade as "missing". Keep several per system so a single yank can't trip the guard.
+const CANARIES: ReadonlyArray<{ system: string; name: string }> = [
+  { system: 'NPM', name: 'express' },
+  { system: 'NPM', name: 'webpack' },
+  { system: 'NPM', name: 'react' },
+  { system: 'NPM', name: 'eslint' },
+  { system: 'NPM', name: '@babel/core' },
+  { system: 'MAVEN', name: 'com.google.guava:guava' },
+  { system: 'MAVEN', name: 'org.springframework:spring-context' },
+  { system: 'MAVEN', name: 'org.apache.httpcomponents:httpclient' },
+  { system: 'PYPI', name: 'requests' },
+  { system: 'PYPI', name: 'flask' },
+  { system: 'PYPI', name: 'django' },
+  { system: 'PYPI', name: 'pandas' },
+  { system: 'CARGO', name: 'serde' },
+  { system: 'CARGO', name: 'tokio' },
+  { system: 'CARGO', name: 'clap' },
+  { system: 'CARGO', name: 'reqwest' },
+]
+
+// Healthy ratio is ~1.0 (max observed ≈ 1.8 for serde); the corrupt snapshots ran ~100×.
+// 5 leaves wide margin for benign multiplicity while catching the ×100 collapse unambiguously.
+const DUP_RATIO_REJECT = 5
+
+interface CanaryRow {
+  system: string
+  name: string
+  row_count: number | string | { value: string }
+  edge_count: number | string | { value: string }
+}
+
+// BigQuery returns INT64 columns as number | string | { value }; coerce defensively.
+function toNum(v: number | string | { value: string }): number {
+  if (typeof v === 'number') return v
+  if (typeof v === 'string') return Number(v)
+  return Number(v.value)
+}
+
+export async function checkEdgeSnapshotQuality(
+  input: CheckEdgeSnapshotQualityInput,
+): Promise<CheckEdgeSnapshotQualityOutput> {
+  const inScope = new Set(
+    input.ecosystems.map((e) => e.toUpperCase()).filter((e) => EDGE_SYSTEMS.has(e)),
+  )
+  const activeCanaries = CANARIES.filter((c) => inScope.has(c.system))
+
+  // No resolved-graph ecosystems requested (e.g. GO/NUGET only) → nothing to probe.
+  if (activeCanaries.length === 0) {
+    return { ok: true, canaries: [] }
+  }
+
+  assertSnapshotDate(input.snapshotDate)
+
+  // Group canaries by system into `(System = x AND Name IN (...))` predicates so the filter prunes
+  // on the (System, Name, Version) clustering — keeps the probe in the single-GB / pennies range.
+  const bySystem = new Map<string, string[]>()
+  for (const c of activeCanaries) {
+    const names = bySystem.get(c.system) ?? []
+    names.push(c.name)
+    bySystem.set(c.system, names)
+  }
+  const systemPredicates = [...bySystem.entries()]
+    .map(
+      ([system, names]) =>
+        `(e.System = '${system}' AND e.Name IN (${names.map((n) => `'${n}'`).join(', ')}))`,
+    )
+    .join('\n        OR ')
+
+  // Direct edges only (From = graph root), matching what the ingest reads. %T formats NULLs
+  // as the literal "NULL" so COUNT(DISTINCT ...) isn't nulled out by unresolved To.Version.
+  const query = `
+    SELECT
+      e.System AS system,
+      e.Name   AS name,
+      COUNT(*) AS row_count,
+      COUNT(DISTINCT FORMAT('%T|%T|%T', e.From.Version, e.To.Name, e.To.Version)) AS edge_count
+    FROM \`bigquery-public-data.deps_dev_v1.DependencyGraphEdges\` e
+    WHERE e.SnapshotAt >= TIMESTAMP('${input.snapshotDate}')
+      AND e.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${input.snapshotDate}', INTERVAL 1 DAY))
+      AND e.From.Name = e.Name AND e.From.Version = e.Version
+      AND (
+        ${systemPredicates}
+      )
+    GROUP BY e.System, e.Name
+  `
+
+  const [job] = await bigquery.createQueryJob({ query, location: 'US' })
+  const [rows] = await job.getQueryResults()
+  const resultRows = rows as CanaryRow[]
+
+  const canaries: CanaryStat[] = activeCanaries.map((c) => {
+    const r = resultRows.find((row) => row.system === c.system && row.name === c.name)
+    if (!r) {
+      return { system: c.system, name: c.name, rows: 0, edges: 0, ratio: null, present: false }
+    }
+    const rowCount = toNum(r.row_count)
+    const edgeCount = toNum(r.edge_count)
+    return {
+      system: c.system,
+      name: c.name,
+      rows: rowCount,
+      edges: edgeCount,
+      ratio: edgeCount > 0 ? rowCount / edgeCount : null,
+      present: true,
+    }
+  })
+
+  // Corruption shows up as a ×100 duplication ratio on the canaries that DO resolve, and/or as
+  // canaries vanishing entirely. Base the verdict on the ratio of PRESENT canaries (missingness
+  // alone is noisy — packages get yanked), with a floor against a mass collapse.
+  const present = canaries.filter((c) => c.present)
+  const overDuplicated = present.filter((c) => c.ratio !== null && c.ratio > DUP_RATIO_REJECT)
+  const minPresent = Math.ceil(activeCanaries.length / 2)
+  const massCollapse = present.length < minPresent
+  const majorityOverDuplicated = overDuplicated.length >= Math.ceil(present.length / 2)
+  const ok = !massCollapse && !majorityOverDuplicated
+
+  if (ok) {
+    return { ok: true, canaries }
+  }
+
+  const dupDetail = overDuplicated
+    .map((c) => `${c.system}:${c.name} ratio=${(c.ratio ?? 0).toFixed(1)} (${c.rows}/${c.edges})`)
+    .join(', ')
+  const reason = massCollapse
+    ? `only ${present.length}/${activeCanaries.length} canaries resolved (min ${minPresent}) — snapshot looks collapsed`
+    : `${overDuplicated.length}/${present.length} present canaries over-duplicated (ratio > ${DUP_RATIO_REJECT}; healthy ≈ 1.0): ${dupDetail}`
+
+  sendSlackNotification(
+    SlackChannel.CDP_CRITICAL_ALERTS,
+    SlackPersona.CRITICAL_ALERTER,
+    ':warning: deps.dev edge snapshot quality anomaly detected',
+    [
+      {
+        title: 'Snapshot',
+        text: input.snapshotDate,
+      },
+      {
+        title: 'Canaries',
+        text: reason,
+      },
+      {
+        title: 'Action',
+        text: 'package_dependencies ingest aborted before export — existing rows preserved. The deps.dev resolved-graph snapshot looks corrupt; re-run once a healthy snapshot is published.',
+      },
+    ],
+  )
+
+  return { ok: false, reason, canaries }
+}
diff --git a/services/apps/packages_worker/src/deps-dev/activities/index.ts b/services/apps/packages_worker/src/deps-dev/activities/index.ts
index 11e1ed49c8..3de67ac1d2 100644
--- a/services/apps/packages_worker/src/deps-dev/activities/index.ts
+++ b/services/apps/packages_worker/src/deps-dev/activities/index.ts
@@ -10,5 +10,6 @@ export * from './gcsParquetToStaging'
 export * from './mergeStagingToTable'
 export * from './getLastSnapshot'
 export * from './checkDependentCountsGuard'
+export * from './checkEdgeSnapshotQuality'
 export * from './probePartitionExists'
 export * from './resolveSnapshotDate'
diff --git a/services/apps/packages_worker/src/deps-dev/queries/depsSql.ts b/services/apps/packages_worker/src/deps-dev/queries/depsSql.ts
index b713ed343c..1acc249b2f 100644
--- a/services/apps/packages_worker/src/deps-dev/queries/depsSql.ts
+++ b/services/apps/packages_worker/src/deps-dev/queries/depsSql.ts
@@ -90,9 +90,123 @@ WHERE d.System IN (${filter})
   return parts.join('\nUNION ALL\n')
 }
 
-// --- Incremental SQL helpers ---
-// Uses base tables (GoRequirements, NuGetRequirements, DependencyGraphEdges) with SnapshotAt filter.
-// All CTEs combined in one WITH clause so UNION ALL can reference them freely.
+// --- Incremental SQL helpers (snapshot edge-diff) ---
+//
+// Diff today's direct edges against the watermark (last successfully ingested) snapshot, matched on
+// the edge identity (ecosystem, root_name, root_version, to_name) — deliberately EXCLUDING the
+// resolved dependency version (to_version) and the requirement string from the match key.
+//
+// Why exclude to_version: a version's declared direct dependencies are immutable (manifest), but
+// deps.dev RE-RESOLVES each dependency's concrete version every snapshot (a range like "^4.17.0"
+// resolves to whatever patch is newest that week). Including to_version in the match makes every
+// stable edge look "new" whenever any dependency ships a patch — the re-resolution churn that
+// produced the ~555M-row exports. Matching on (root, to_name) only:
+//   - drops that churn, AND
+//   - still catches genuinely-new edges: brand-new versions' edges AND edges deps.dev resolved
+//     LATE for already-published versions. (A version lands in PackageVersions on publish, but its
+//     resolved graph can appear a snapshot or more later — measured ~14% of edge-bearing versions.
+//     A version-level diff misses those; an edge-level diff catches them, because the edge is
+//     simply new vs the watermark.)
+//
+// Consistent with how PG stores the data: the unique key is (version_id, depends_on_id,
+// dependency_kind) — depends_on_version_id is NOT in it and the merge is ON CONFLICT DO NOTHING, so
+// depends_on_version_id is never updated. to_version / version_constraint are still SELECTed (to
+// populate the columns on first insert) but are not part of the diff key; the today CTEs collapse
+// to one row per edge identity via GROUP BY + MAX (also defuses ×100 duplication if a corrupt
+// snapshot ever slips past the edge-quality guard).
+//
+// All CTEs go into one WITH clause so each branch's UNION ALL SELECT can reference its CTE pair.
+
+const DEPS_DEV = 'bigquery-public-data.deps_dev_v1'
+
+// Snapshot dates always arrive as YYYY-MM-DD (resolveSnapshotDate slices the timestamp;
+// bootstrap uses toISOString().slice(0,10)). Validate before embedding in SQL so a malformed or
+// operator-supplied value fails loudly instead of producing broken — or injectable — SQL.
+export function assertSnapshotDate(date: string): void {
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) {
+    throw new Error(`Invalid snapshot date '${date}' — expected YYYY-MM-DD`)
+  }
+}
+
+// SnapshotAt is not exactly midnight, so filter by a [date, date+1day) range, never `= TIMESTAMP(date)`.
+function snapshotRange(col: string, date: string): string {
+  assertSnapshotDate(date)
+  return `${col} >= TIMESTAMP('${date}')
+    AND ${col} <  TIMESTAMP(DATE_ADD(DATE '${date}', INTERVAL 1 DAY))`
+}
+
+// Anti-join: today edges not present in the watermark snapshot, matched on the edge identity.
+function antiJoinSelect(todayCte: string, watermarkCte: string): string {
+  return `SELECT t.ecosystem, t.root_name, t.root_version, t.to_name, t.to_version, t.version_constraint
+FROM ${todayCte} t
+LEFT JOIN ${watermarkCte} l
+  ON l.ecosystem = t.ecosystem AND l.root_name = t.root_name
+ AND l.root_version = t.root_version AND l.to_name = t.to_name
+WHERE l.to_name IS NULL`
+}
+
+// GO + NUGET come from manifest tables (GoRequirements / NuGetRequirements) regardless of Option
+// A/B, so their incremental branches are shared. Each returns its today + watermark CTEs + select.
+function goIncrementalBranch(today: string, watermark: string): { ctes: string[]; select: string } {
+  return {
+    ctes: [
+      `today_go AS (
+  SELECT
+    'go'                 AS ecosystem,
+    g.Name               AS root_name,
+    g.Version            AS root_version,
+    d.Name               AS to_name,
+    CAST(NULL AS STRING) AS to_version,
+    MAX(d.Requirement)   AS version_constraint
+  FROM \`${DEPS_DEV}.GoRequirements\` g,
+  UNNEST(g.DirectDependencies) AS d
+  WHERE ${snapshotRange('g.SnapshotAt', today)}
+  GROUP BY 1, 2, 3, 4
+)`,
+      `watermark_go AS (
+  SELECT 'go' AS ecosystem, g.Name AS root_name, g.Version AS root_version, d.Name AS to_name
+  FROM \`${DEPS_DEV}.GoRequirements\` g,
+  UNNEST(g.DirectDependencies) AS d
+  WHERE ${snapshotRange('g.SnapshotAt', watermark)}
+  GROUP BY 1, 2, 3, 4
+)`,
+    ],
+    select: antiJoinSelect('today_go', 'watermark_go'),
+  }
+}
+
+function nugetIncrementalBranch(
+  today: string,
+  watermark: string,
+): { ctes: string[]; select: string } {
+  return {
+    ctes: [
+      `today_nuget AS (
+  SELECT
+    'nuget'              AS ecosystem,
+    n.Name               AS root_name,
+    n.Version            AS root_version,
+    dep.Name             AS to_name,
+    CAST(NULL AS STRING) AS to_version,
+    MAX(dep.Requirement) AS version_constraint
+  FROM \`${DEPS_DEV}.NuGetRequirements\` n,
+  UNNEST(n.DependencyGroups) AS grp,
+  UNNEST(grp.Dependencies) AS dep
+  WHERE ${snapshotRange('n.SnapshotAt', today)}
+  GROUP BY 1, 2, 3, 4
+)`,
+      `watermark_nuget AS (
+  SELECT 'nuget' AS ecosystem, n.Name AS root_name, n.Version AS root_version, dep.Name AS to_name
+  FROM \`${DEPS_DEV}.NuGetRequirements\` n,
+  UNNEST(n.DependencyGroups) AS grp,
+  UNNEST(grp.Dependencies) AS dep
+  WHERE ${snapshotRange('n.SnapshotAt', watermark)}
+  GROUP BY 1, 2, 3, 4
+)`,
+    ],
+    select: antiJoinSelect('today_nuget', 'watermark_nuget'),
+  }
+}
 
 export function buildDepsIncrementalSqlA(
   today: string,
@@ -111,106 +225,40 @@ export function buildDepsIncrementalSqlA(
     ctes.push(
       `today_edges AS (
   SELECT
-    LOWER(e.System) AS ecosystem,
-    e.Name          AS root_name,
-    e.Version       AS root_version,
-    e.To.Name       AS to_name,
-    e.To.Version    AS to_version,
-    e.Requirement   AS version_constraint
-  FROM \`bigquery-public-data.deps_dev_v1.DependencyGraphEdges\` e
-  WHERE e.SnapshotAt >= TIMESTAMP('${today}')
-    AND e.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${today}', INTERVAL 1 DAY))
+    LOWER(e.System)    AS ecosystem,
+    e.Name             AS root_name,
+    e.Version          AS root_version,
+    e.To.Name          AS to_name,
+    MAX(e.To.Version)  AS to_version,
+    MAX(e.Requirement) AS version_constraint
+  FROM \`${DEPS_DEV}.DependencyGraphEdges\` e
+  WHERE ${snapshotRange('e.SnapshotAt', today)}
     AND e.System IN (${filter})
     AND e.From.Name = e.Name AND e.From.Version = e.Version
+  GROUP BY 1, 2, 3, 4
 )`,
       `watermark_edges AS (
-  SELECT e.System, e.Name, e.Version, e.To.Name AS to_name, e.To.Version AS to_version
-  FROM \`bigquery-public-data.deps_dev_v1.DependencyGraphEdges\` e
-  WHERE e.SnapshotAt >= TIMESTAMP('${watermark}')
-    AND e.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${watermark}', INTERVAL 1 DAY))
+  SELECT LOWER(e.System) AS ecosystem, e.Name AS root_name, e.Version AS root_version, e.To.Name AS to_name
+  FROM \`${DEPS_DEV}.DependencyGraphEdges\` e
+  WHERE ${snapshotRange('e.SnapshotAt', watermark)}
     AND e.System IN (${filter})
     AND e.From.Name = e.Name AND e.From.Version = e.Version
-  GROUP BY e.System, e.Name, e.Version, e.To.Name, e.To.Version
+  GROUP BY 1, 2, 3, 4
 )`,
     )
-    selects.push(
-      `SELECT t.*
-FROM today_edges t
-LEFT JOIN watermark_edges l
-  ON LOWER(l.System) = t.ecosystem AND l.Name = t.root_name AND l.Version = t.root_version
- AND l.to_name = t.to_name AND l.to_version = t.to_version
-WHERE l.to_name IS NULL`,
-    )
+    selects.push(antiJoinSelect('today_edges', 'watermark_edges'))
   }
 
   if (includeGo) {
-    ctes.push(
-      `today_go AS (
-  SELECT
-    'go'                   AS ecosystem,
-    g.Name                 AS root_name,
-    g.Version              AS root_version,
-    d.Name                 AS to_name,
-    CAST(NULL AS STRING)   AS to_version,
-    d.Requirement          AS version_constraint
-  FROM \`bigquery-public-data.deps_dev_v1.GoRequirements\` g,
-  UNNEST(g.DirectDependencies) AS d
-  WHERE g.SnapshotAt >= TIMESTAMP('${today}')
-    AND g.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${today}', INTERVAL 1 DAY))
-)`,
-      `watermark_go AS (
-  SELECT g.Name, g.Version, d.Name AS to_name, d.Requirement
-  FROM \`bigquery-public-data.deps_dev_v1.GoRequirements\` g,
-  UNNEST(g.DirectDependencies) AS d
-  WHERE g.SnapshotAt >= TIMESTAMP('${watermark}')
-    AND g.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${watermark}', INTERVAL 1 DAY))
-  GROUP BY g.Name, g.Version, d.Name, d.Requirement
-)`,
-    )
-    selects.push(
-      `SELECT t.*
-FROM today_go t
-LEFT JOIN watermark_go l
-  ON l.Name = t.root_name AND l.Version = t.root_version AND l.to_name = t.to_name
-  AND l.Requirement = t.version_constraint
-WHERE l.to_name IS NULL`,
-    )
+    const { ctes: goCtes, select } = goIncrementalBranch(today, watermark)
+    ctes.push(...goCtes)
+    selects.push(select)
   }
 
   if (includeNuget) {
-    ctes.push(
-      `today_nuget AS (
-  SELECT
-    'nuget'                AS ecosystem,
-    n.Name                 AS root_name,
-    n.Version              AS root_version,
-    dep.Name               AS to_name,
-    CAST(NULL AS STRING)   AS to_version,
-    dep.Requirement        AS version_constraint
-  FROM \`bigquery-public-data.deps_dev_v1.NuGetRequirements\` n,
-  UNNEST(n.DependencyGroups) AS grp,
-  UNNEST(grp.Dependencies) AS dep
-  WHERE n.SnapshotAt >= TIMESTAMP('${today}')
-    AND n.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${today}', INTERVAL 1 DAY))
-)`,
-      `watermark_nuget AS (
-  SELECT n.Name, n.Version, dep.Name AS to_name, dep.Requirement
-  FROM \`bigquery-public-data.deps_dev_v1.NuGetRequirements\` n,
-  UNNEST(n.DependencyGroups) AS grp,
-  UNNEST(grp.Dependencies) AS dep
-  WHERE n.SnapshotAt >= TIMESTAMP('${watermark}')
-    AND n.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${watermark}', INTERVAL 1 DAY))
-  GROUP BY n.Name, n.Version, dep.Name, dep.Requirement
-)`,
-    )
-    selects.push(
-      `SELECT t.*
-FROM today_nuget t
-LEFT JOIN watermark_nuget l
-  ON l.Name = t.root_name AND l.Version = t.root_version AND l.to_name = t.to_name
-  AND l.Requirement = t.version_constraint
-WHERE l.to_name IS NULL`,
-    )
+    const { ctes: nugetCtes, select } = nugetIncrementalBranch(today, watermark)
+    ctes.push(...nugetCtes)
+    selects.push(select)
   }
 
   return `WITH\n${ctes.join(',\n')}\n${selects.join('\nUNION ALL\n')}`
@@ -237,102 +285,36 @@ export function buildDepsIncrementalSqlB(
     d.Name                    AS root_name,
     d.Version                 AS root_version,
     d.Dependency.Name         AS to_name,
-    d.Dependency.Version      AS to_version,
+    MAX(d.Dependency.Version) AS to_version,
     CAST(NULL AS STRING)      AS version_constraint
-  FROM \`bigquery-public-data.deps_dev_v1.Dependencies\` d
-  WHERE d.SnapshotAt >= TIMESTAMP('${today}')
-    AND d.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${today}', INTERVAL 1 DAY))
+  FROM \`${DEPS_DEV}.Dependencies\` d
+  WHERE ${snapshotRange('d.SnapshotAt', today)}
     AND d.System IN (${filter})
     AND d.MinimumDepth = 1
+  GROUP BY 1, 2, 3, 4
 )`,
       `watermark_deps AS (
-  SELECT d.System, d.Name, d.Version, d.Dependency.Name AS to_name, d.Dependency.Version AS to_version
-  FROM \`bigquery-public-data.deps_dev_v1.Dependencies\` d
-  WHERE d.SnapshotAt >= TIMESTAMP('${watermark}')
-    AND d.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${watermark}', INTERVAL 1 DAY))
+  SELECT LOWER(d.System) AS ecosystem, d.Name AS root_name, d.Version AS root_version, d.Dependency.Name AS to_name
+  FROM \`${DEPS_DEV}.Dependencies\` d
+  WHERE ${snapshotRange('d.SnapshotAt', watermark)}
     AND d.System IN (${filter})
     AND d.MinimumDepth = 1
-  GROUP BY d.System, d.Name, d.Version, d.Dependency.Name, d.Dependency.Version
+  GROUP BY 1, 2, 3, 4
 )`,
     )
-    selects.push(
-      `SELECT t.*
-FROM today_deps t
-LEFT JOIN watermark_deps l
-  ON LOWER(l.System) = t.ecosystem AND l.Name = t.root_name AND l.Version = t.root_version
- AND l.to_name = t.to_name AND l.to_version = t.to_version
-WHERE l.to_name IS NULL`,
-    )
+    selects.push(antiJoinSelect('today_deps', 'watermark_deps'))
   }
 
   if (includeGo) {
-    ctes.push(
-      `today_go AS (
-  SELECT
-    'go'                   AS ecosystem,
-    g.Name                 AS root_name,
-    g.Version              AS root_version,
-    d.Name                 AS to_name,
-    CAST(NULL AS STRING)   AS to_version,
-    d.Requirement          AS version_constraint
-  FROM \`bigquery-public-data.deps_dev_v1.GoRequirements\` g,
-  UNNEST(g.DirectDependencies) AS d
-  WHERE g.SnapshotAt >= TIMESTAMP('${today}')
-    AND g.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${today}', INTERVAL 1 DAY))
-)`,
-      `watermark_go AS (
-  SELECT g.Name, g.Version, d.Name AS to_name, d.Requirement
-  FROM \`bigquery-public-data.deps_dev_v1.GoRequirements\` g,
-  UNNEST(g.DirectDependencies) AS d
-  WHERE g.SnapshotAt >= TIMESTAMP('${watermark}')
-    AND g.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${watermark}', INTERVAL 1 DAY))
-  GROUP BY g.Name, g.Version, d.Name, d.Requirement
-)`,
-    )
-    selects.push(
-      `SELECT t.*
-FROM today_go t
-LEFT JOIN watermark_go l
-  ON l.Name = t.root_name AND l.Version = t.root_version AND l.to_name = t.to_name
-  AND l.Requirement = t.version_constraint
-WHERE l.to_name IS NULL`,
-    )
+    const { ctes: goCtes, select } = goIncrementalBranch(today, watermark)
+    ctes.push(...goCtes)
+    selects.push(select)
   }
 
   if (includeNuget) {
-    ctes.push(
-      `today_nuget AS (
-  SELECT
-    'nuget'                AS ecosystem,
-    n.Name                 AS root_name,
-    n.Version              AS root_version,
-    dep.Name               AS to_name,
-    CAST(NULL AS STRING)   AS to_version,
-    dep.Requirement        AS version_constraint
-  FROM \`bigquery-public-data.deps_dev_v1.NuGetRequirements\` n,
-  UNNEST(n.DependencyGroups) AS grp,
-  UNNEST(grp.Dependencies) AS dep
-  WHERE n.SnapshotAt >= TIMESTAMP('${today}')
-    AND n.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${today}', INTERVAL 1 DAY))
-)`,
-      `watermark_nuget AS (
-  SELECT n.Name, n.Version, dep.Name AS to_name, dep.Requirement
-  FROM \`bigquery-public-data.deps_dev_v1.NuGetRequirements\` n,
-  UNNEST(n.DependencyGroups) AS grp,
-  UNNEST(grp.Dependencies) AS dep
-  WHERE n.SnapshotAt >= TIMESTAMP('${watermark}')
-    AND n.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${watermark}', INTERVAL 1 DAY))
-  GROUP BY n.Name, n.Version, dep.Name, dep.Requirement
-)`,
-    )
-    selects.push(
-      `SELECT t.*
-FROM today_nuget t
-LEFT JOIN watermark_nuget l
-  ON l.Name = t.root_name AND l.Version = t.root_version AND l.to_name = t.to_name
-  AND l.Requirement = t.version_constraint
-WHERE l.to_name IS NULL`,
-    )
+    const { ctes: nugetCtes, select } = nugetIncrementalBranch(today, watermark)
+    ctes.push(...nugetCtes)
+    selects.push(select)
   }
 
   return `WITH\n${ctes.join(',\n')}\n${selects.join('\nUNION ALL\n')}`
diff --git a/services/apps/packages_worker/src/deps-dev/workflows/bootstrapOsspckgs.ts b/services/apps/packages_worker/src/deps-dev/workflows/bootstrapOsspckgs.ts
index 0258fa40e7..c45c4775bb 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/bootstrapOsspckgs.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/bootstrapOsspckgs.ts
@@ -211,21 +211,33 @@ export async function bootstrapOsspckgs(opts: {
     })
   }
   if (runs('package_dependencies')) {
-    await executeChild(ingestDependencies, {
-      args: [
-        {
-          runId,
-          syncMode: opts.mode,
-          today: snap('package_dependencies'),
-          watermark: wm('package_dependencies'),
-          ecosystems: opts.ecosystems,
-          reuseExports: opts.reuseExports,
-          depsTableOption: opts.depsTableOption,
-          exportName: opts.exportName,
-          fillConstraints: opts.fillConstraints,
-        },
-      ],
-    })
+    try {
+      await executeChild(ingestDependencies, {
+        args: [
+          {
+            runId,
+            syncMode: opts.mode,
+            today: snap('package_dependencies'),
+            watermark: wm('package_dependencies'),
+            ecosystems: opts.ecosystems,
+            reuseExports: opts.reuseExports,
+            depsTableOption: opts.depsTableOption,
+            exportName: opts.exportName,
+            fillConstraints: opts.fillConstraints,
+          },
+        ],
+      })
+    } catch (err) {
+      // Only soft-fail on the edge-snapshot quality guard (corrupt deps.dev resolved-graph
+      // snapshot). Skipping leaves existing package_dependencies untouched and lets the rest
+      // of the bootstrap proceed; the next healthy snapshot ingests naturally. Mirror the
+      // dependent_counts guard: unwrap the ChildWorkflowFailure to inspect the cause; all
+      // other errors propagate.
+      const cause = err instanceof ChildWorkflowFailure ? err.cause : err
+      if (!(cause instanceof ApplicationFailure) || cause.type !== 'EDGE_SNAPSHOT_GUARD') {
+        throw err
+      }
+    }
   }
   if (runs('advisories') || runs('advisory_packages')) {
     await executeChild(ingestAdvisories, {
diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
index 0309c079bb..68e6a72726 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
@@ -1,4 +1,4 @@
-import { proxyActivities } from '@temporalio/workflow'
+import { ApplicationFailure, proxyActivities } from '@temporalio/workflow'
 
 import type * as depsDevActivities from '../activities'
 import {
@@ -54,6 +54,11 @@ const { setJobStep } = proxyActivities<typeof depsDevActivities>({
   retry: { maximumAttempts: 3 },
 })
 
+const { checkEdgeSnapshotQuality } = proxyActivities<typeof depsDevActivities>({
+  startToCloseTimeout: '5 minutes',
+  retry: { maximumAttempts: 3 },
+})
+
 const STAGING_TABLE = 'staging.osspckgs_deps_raw'
 
 const STAGING_DDL = `
@@ -219,6 +224,22 @@ export async function ingestDependencies(opts: {
   const isFill = opts.fillConstraints === true
   // Fill mode forces Option A — Option B selects NULL for version_constraint, making the fill a no-op.
   const tableOption = isFill ? 'A' : (opts.depsTableOption ?? 'A')
+
+  // Guard against corrupt deps.dev resolved-graph snapshots BEFORE the (multi-hour) export.
+  // Skip when reusing a prior export — we're re-importing already-validated parquet, not
+  // scanning the live snapshot. Both full (*Latest = newest snapshot) and incremental can hit
+  // a bad snapshot, so the guard runs for both. Probes only resolved-graph ecosystems; a clean
+  // GO/NUGET-only run finds no canaries and passes through.
+  if (!opts.reuseExports) {
+    const guard = await checkEdgeSnapshotQuality({ snapshotDate: opts.today, ecosystems })
+    if (!guard.ok) {
+      throw ApplicationFailure.nonRetryable(
+        `edge snapshot quality guard failed for ${opts.today}: ${guard.reason}. ` +
+          `Slack alert sent. Aborting before export to preserve existing package_dependencies and compute.`,
+        'EDGE_SNAPSHOT_GUARD',
+      )
+    }
+  }
   // Fill mode always uses full SQL — needs all rows to find which have NULL version_constraint in DB.
   const sql =
     opts.syncMode === 'full' || isFill
@@ -231,6 +252,10 @@ export async function ingestDependencies(opts: {
     runId: opts.runId,
     syncMode: opts.syncMode,
     snapshotAt: opts.today,
+    // Full/fill scan the *Latest views (everything) → 25000. Incremental is a snapshot edge-diff
+    // (today vs watermark partitions of DependencyGraphEdges + GoRequirements + NuGetRequirements);
+    // measured ~4.1TB for Option A. 10000 leaves ~2.4x headroom and still trips a runaway full-table
+    // scan. Overridable via BQ_DATASET_INGEST_PACKAGE_DEPENDENCIES[_INCREMENTAL]_MAX_BQ_GB (see README).
     maxBytesGb: opts.syncMode === 'full' || isFill ? 25000 : 10000,
     reuseExports: opts.reuseExports,
     exportName: opts.exportName,

From 1ca7b1734bd2b6a1278928a59ac38f814bc6df7f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Thu, 25 Jun 2026 14:38:20 +0200
Subject: [PATCH 19/29] style: format deps-dev README (CM-1281)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../packages_worker/src/deps-dev/README.md    | 30 +++++++++----------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/services/apps/packages_worker/src/deps-dev/README.md b/services/apps/packages_worker/src/deps-dev/README.md
index 814b30d507..92e5b70837 100644
--- a/services/apps/packages_worker/src/deps-dev/README.md
+++ b/services/apps/packages_worker/src/deps-dev/README.md
@@ -48,21 +48,21 @@ The mode-specific key takes precedence over the generic key. Value must be a pos
 
 **When adding a new job kind, add a row to this table.**
 
-| Env var override                                   |            Default (GB) | Job kind               | Notes                                                                                                                                                                                                                                                                                                                                             |
-| -------------------------------------------------- | ----------------------: | ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `BQ_DATASET_INGEST_PACKAGES_FULL_MAX_BQ_GB`        |                    6000 | `packages`             | Full only (set in `ingestPackages.ts`)                                                                                                                                                                                                                                                                                                            |
-| `BQ_DATASET_INGEST_PACKAGES_INCREMENTAL_MAX_BQ_GB` |                     400 | `packages`             | Incremental only (set in `ingestPackages.ts`)                                                                                                                                                                                                                                                                                                     |
-| `BQ_DATASET_INGEST_VERSIONS_MAX_BQ_GB`             |                     400 | `versions`             |                                                                                                                                                                                                                                                                                                                                                   |
-| `BQ_DATASET_INGEST_PACKAGE_DEPENDENCIES_MAX_BQ_GB` | 25000 full / 10000 incr | `package_dependencies` | Full scans `*Latest`. Incremental is a snapshot edge-diff (today vs watermark partitions of `DependencyGraphEdges` + `GoRequirements` + `NuGetRequirements`), matched on `(root, to_name)` excluding the resolved `to_version` to drop re-resolution churn (~4.1TB, Option A). Mode-specific `…_FULL_…` / `…_INCREMENTAL_…` keys take precedence. |
-| `BQ_DATASET_INGEST_REPOS_MAX_BQ_GB`                |                    2000 | `repos`                |                                                                                                                                                                                                                                                                                                                                                   |
-| `BQ_DATASET_INGEST_PACKAGE_REPOS_MAX_BQ_GB`        |                    2000 | `package_repos`        |                                                                                                                                                                                                                                                                                                                                                   |
-| `BQ_DATASET_INGEST_ADVISORIES_MAX_BQ_GB`           |                      10 | `advisories`           |                                                                                                                                                                                                                                                                                                                                                   |
-| `BQ_DATASET_INGEST_ADVISORY_PACKAGES_MAX_BQ_GB`    |                    1500 | `advisory_packages`    |                                                                                                                                                                                                                                                                                                                                                   |
-| `BQ_DATASET_INGEST_DEPENDENT_COUNTS_MAX_BQ_GB`     |                    2000 | `dependent_counts`     | Edges only (NPM/MAVEN/PYPI/CARGO) from the `Dependents` reverse index. GO/NUGET are absent from `Dependents` and run as separate kinds below.                                                                                                                                                                                                       |
-| `BQ_DATASET_INGEST_DEPENDENT_COUNTS_GO_MAX_BQ_GB`     |                 5000 | `dependent_counts_go`     | GO exact reverse transitive closure over `GoRequirementsLatest` (script mode). All 3 count columns. Ceiling is a `maximumBytesBilled` runaway cap above the validated full-pipeline spend (2.31 TB incl. repos aggregation), not a dry-run gate.                                                                                                  |
-| `BQ_DATASET_INGEST_DEPENDENT_COUNTS_NUGET_MAX_BQ_GB`  |                  200 | `dependent_counts_nuget`  | NUGET exact reverse transitive closure over `NuGetRequirementsLatest` (script mode). All 3 count columns. `maximumBytesBilled` runaway cap above the measured ~32 GB.                                                                                                                                                                            |
-| `BQ_DATASET_INGEST_SCORECARD_REPOS_MAX_BQ_GB`      |                      50 | `scorecard_repos`      |                                                                                                                                                                                                                                                                                                                                                   |
-| `BQ_DATASET_INGEST_SCORECARD_CHECKS_MAX_BQ_GB`     |                     500 | `scorecard_checks`     |                                                                                                                                                                                                                                                                                                                                                   |
+| Env var override                                     |            Default (GB) | Job kind                 | Notes                                                                                                                                                                                                                                                                                                                                             |
+| ---------------------------------------------------- | ----------------------: | ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `BQ_DATASET_INGEST_PACKAGES_FULL_MAX_BQ_GB`          |                    6000 | `packages`               | Full only (set in `ingestPackages.ts`)                                                                                                                                                                                                                                                                                                            |
+| `BQ_DATASET_INGEST_PACKAGES_INCREMENTAL_MAX_BQ_GB`   |                     400 | `packages`               | Incremental only (set in `ingestPackages.ts`)                                                                                                                                                                                                                                                                                                     |
+| `BQ_DATASET_INGEST_VERSIONS_MAX_BQ_GB`               |                     400 | `versions`               |                                                                                                                                                                                                                                                                                                                                                   |
+| `BQ_DATASET_INGEST_PACKAGE_DEPENDENCIES_MAX_BQ_GB`   | 25000 full / 10000 incr | `package_dependencies`   | Full scans `*Latest`. Incremental is a snapshot edge-diff (today vs watermark partitions of `DependencyGraphEdges` + `GoRequirements` + `NuGetRequirements`), matched on `(root, to_name)` excluding the resolved `to_version` to drop re-resolution churn (~4.1TB, Option A). Mode-specific `…_FULL_…` / `…_INCREMENTAL_…` keys take precedence. |
+| `BQ_DATASET_INGEST_REPOS_MAX_BQ_GB`                  |                    2000 | `repos`                  |                                                                                                                                                                                                                                                                                                                                                   |
+| `BQ_DATASET_INGEST_PACKAGE_REPOS_MAX_BQ_GB`          |                    2000 | `package_repos`          |                                                                                                                                                                                                                                                                                                                                                   |
+| `BQ_DATASET_INGEST_ADVISORIES_MAX_BQ_GB`             |                      10 | `advisories`             |                                                                                                                                                                                                                                                                                                                                                   |
+| `BQ_DATASET_INGEST_ADVISORY_PACKAGES_MAX_BQ_GB`      |                    1500 | `advisory_packages`      |                                                                                                                                                                                                                                                                                                                                                   |
+| `BQ_DATASET_INGEST_DEPENDENT_COUNTS_MAX_BQ_GB`       |                    2000 | `dependent_counts`       | Edges only (NPM/MAVEN/PYPI/CARGO) from the `Dependents` reverse index. GO/NUGET are absent from `Dependents` and run as separate kinds below.                                                                                                                                                                                                     |
+| `BQ_DATASET_INGEST_DEPENDENT_COUNTS_GO_MAX_BQ_GB`    |                    5000 | `dependent_counts_go`    | GO exact reverse transitive closure over `GoRequirementsLatest` (script mode). All 3 count columns. Ceiling is a `maximumBytesBilled` runaway cap above the validated full-pipeline spend (2.31 TB incl. repos aggregation), not a dry-run gate.                                                                                                  |
+| `BQ_DATASET_INGEST_DEPENDENT_COUNTS_NUGET_MAX_BQ_GB` |                     200 | `dependent_counts_nuget` | NUGET exact reverse transitive closure over `NuGetRequirementsLatest` (script mode). All 3 count columns. `maximumBytesBilled` runaway cap above the measured ~32 GB.                                                                                                                                                                             |
+| `BQ_DATASET_INGEST_SCORECARD_REPOS_MAX_BQ_GB`        |                      50 | `scorecard_repos`        |                                                                                                                                                                                                                                                                                                                                                   |
+| `BQ_DATASET_INGEST_SCORECARD_CHECKS_MAX_BQ_GB`       |                     500 | `scorecard_checks`       |                                                                                                                                                                                                                                                                                                                                                   |
 
 The override logic lives in `src/deps-dev/activities/bqExportToGcs.ts`.
 

From a76d13ebae158b55ed304cdc9e9f6a40c1719167 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Sun, 28 Jun 2026 22:23:29 +0200
Subject: [PATCH 20/29] refactor: clarify edge-snapshot guard threshold naming
 (CM-1281)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../src/deps-dev/activities/checkEdgeSnapshotQuality.ts    | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/services/apps/packages_worker/src/deps-dev/activities/checkEdgeSnapshotQuality.ts b/services/apps/packages_worker/src/deps-dev/activities/checkEdgeSnapshotQuality.ts
index 07aa432140..35097f7d7d 100644
--- a/services/apps/packages_worker/src/deps-dev/activities/checkEdgeSnapshotQuality.ts
+++ b/services/apps/packages_worker/src/deps-dev/activities/checkEdgeSnapshotQuality.ts
@@ -162,8 +162,11 @@ export async function checkEdgeSnapshotQuality(
   const overDuplicated = present.filter((c) => c.ratio !== null && c.ratio > DUP_RATIO_REJECT)
   const minPresent = Math.ceil(activeCanaries.length / 2)
   const massCollapse = present.length < minPresent
-  const majorityOverDuplicated = overDuplicated.length >= Math.ceil(present.length / 2)
-  const ok = !massCollapse && !majorityOverDuplicated
+  // Reject at HALF-OR-MORE (not strict majority): if half the high-fanout canaries show ×100
+  // duplication the snapshot is corrupt enough to abort. Bias toward rejecting — a false reject
+  // just skips one cycle (existing rows preserved), a false accept burns ~5h on ~550M garbage rows.
+  const halfOrMoreOverDuplicated = overDuplicated.length >= Math.ceil(present.length / 2)
+  const ok = !massCollapse && !halfOrMoreOverDuplicated
 
   if (ok) {
     return { ok: true, canaries }

From 3b39fd3008fc969208d6aac0e798c88f7bce9ad8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Sun, 28 Jun 2026 23:01:17 +0200
Subject: [PATCH 21/29] fix: comments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../src/deps-dev/queries/dependentCountsSql.ts                | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/services/apps/packages_worker/src/deps-dev/queries/dependentCountsSql.ts b/services/apps/packages_worker/src/deps-dev/queries/dependentCountsSql.ts
index 51a35a21de..75c2b36ce5 100644
--- a/services/apps/packages_worker/src/deps-dev/queries/dependentCountsSql.ts
+++ b/services/apps/packages_worker/src/deps-dev/queries/dependentCountsSql.ts
@@ -1,3 +1,5 @@
+import { assertSnapshotDate } from './depsSql'
+
 // The deps.dev `Dependents` reverse index only covers the resolved-graph ecosystems
 // {NPM, MAVEN, PYPI, CARGO} — GO and NUGET are entirely absent from the table (verified via BQ
 // 2026-06-17). GO/NUGET dependent counts are produced separately by inverting their manifest tables
@@ -11,6 +13,7 @@
 const EDGE_DEPENDENT_SYSTEMS = `('NPM', 'MAVEN', 'PYPI', 'CARGO')`
 
 export function buildDependentCountsSql(snapshotDate: string): string {
+  assertSnapshotDate(snapshotDate)
   return `
 WITH purl_map AS (
   SELECT System, Name, ANY_VALUE(REGEXP_REPLACE(Purl, r'@[^@]+$', '')) AS purl
@@ -69,6 +72,7 @@ function buildClosureScript(
   edgesSql: string,
   snapshotDate: string,
 ): string {
+  assertSnapshotDate(snapshotDate)
   return `
 DECLARE new_pairs INT64 DEFAULT 1;
 DECLARE iter INT64 DEFAULT 0;

From 4768874ea68828463e5f723b28b917e83e45207b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Sun, 28 Jun 2026 23:04:03 +0200
Subject: [PATCH 22/29] fix: small cleanup
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../src/bin/bq-dataset-ingest.ts              |  2 -
 .../src/deps-dev/workflows/cleanupOsspckgs.ts |  6 ---
 .../src/deps-dev/workflows/index.ts           |  1 -
 .../packages_worker/src/schedules/cleanup.ts  | 40 -------------------
 .../packages_worker/src/workflows/index.ts    |  1 -
 5 files changed, 50 deletions(-)
 delete mode 100644 services/apps/packages_worker/src/deps-dev/workflows/cleanupOsspckgs.ts
 delete mode 100644 services/apps/packages_worker/src/schedules/cleanup.ts

diff --git a/services/apps/packages_worker/src/bin/bq-dataset-ingest.ts b/services/apps/packages_worker/src/bin/bq-dataset-ingest.ts
index a81c14914a..141973a626 100644
--- a/services/apps/packages_worker/src/bin/bq-dataset-ingest.ts
+++ b/services/apps/packages_worker/src/bin/bq-dataset-ingest.ts
@@ -1,10 +1,8 @@
 import { scheduleOsspckgsBootstrap } from '../deps-dev/schedules/bootstrap'
-import { scheduleOsspckgsCleanup } from '../schedules/cleanup'
 import { svc } from '../service'
 
 setImmediate(async () => {
   await svc.init()
   await scheduleOsspckgsBootstrap()
-  await scheduleOsspckgsCleanup()
   await svc.start()
 })
diff --git a/services/apps/packages_worker/src/deps-dev/workflows/cleanupOsspckgs.ts b/services/apps/packages_worker/src/deps-dev/workflows/cleanupOsspckgs.ts
deleted file mode 100644
index 3ef98d74b1..0000000000
--- a/services/apps/packages_worker/src/deps-dev/workflows/cleanupOsspckgs.ts
+++ /dev/null
@@ -1,6 +0,0 @@
-// GCS objects are covered by a 7-day lifecycle rule on the bucket — no explicit cleanup needed.
-// This workflow is reserved for staging-table GC (orphaned tables older than 24h) once
-// versioned staging tables are implemented (B4).
-export async function cleanupOsspckgs(): Promise<void> {
-  // no-op until versioned staging table GC is wired
-}
diff --git a/services/apps/packages_worker/src/deps-dev/workflows/index.ts b/services/apps/packages_worker/src/deps-dev/workflows/index.ts
index 8ef97c9e3f..642653ca2c 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/index.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/index.ts
@@ -1,5 +1,4 @@
 export * from './bootstrapOsspckgs'
-export * from './cleanupOsspckgs'
 export * from './ingestAdvisories'
 export * from './ingestDependentCounts'
 export * from './ingestDependencies'
diff --git a/services/apps/packages_worker/src/schedules/cleanup.ts b/services/apps/packages_worker/src/schedules/cleanup.ts
deleted file mode 100644
index b4e165b118..0000000000
--- a/services/apps/packages_worker/src/schedules/cleanup.ts
+++ /dev/null
@@ -1,40 +0,0 @@
-import { ScheduleAlreadyRunning, ScheduleOverlapPolicy } from '@temporalio/client'
-
-import { cleanupOsspckgs } from '../deps-dev/workflows'
-import { svc } from '../service'
-
-export async function scheduleOsspckgsCleanup(): Promise<void> {
-  const { temporal } = svc
-  if (!temporal) throw new Error('Temporal client not initialized')
-
-  try {
-    await temporal.schedule.create({
-      scheduleId: 'osspckgs-cleanup-daily',
-      spec: {
-        cronExpressions: ['30 3 * * *'],
-      },
-      policies: {
-        overlap: ScheduleOverlapPolicy.SKIP,
-        catchupWindow: '30 minutes',
-      },
-      action: {
-        type: 'startWorkflow',
-        workflowType: cleanupOsspckgs,
-        taskQueue: 'bq-dataset-ingest',
-        workflowExecutionTimeout: '1 hour',
-        retry: {
-          initialInterval: '1 minute',
-          backoffCoefficient: 2,
-          maximumAttempts: 3,
-        },
-        args: [],
-      },
-    })
-  } catch (err) {
-    if (err instanceof ScheduleAlreadyRunning) {
-      svc.log.info('Schedule osspckgs-cleanup-daily already registered.')
-    } else {
-      throw err
-    }
-  }
-}
diff --git a/services/apps/packages_worker/src/workflows/index.ts b/services/apps/packages_worker/src/workflows/index.ts
index 219aa742fc..09416fdaba 100644
--- a/services/apps/packages_worker/src/workflows/index.ts
+++ b/services/apps/packages_worker/src/workflows/index.ts
@@ -6,7 +6,6 @@ export {
 } from '../npm/workflows'
 export {
   bootstrapOsspckgs,
-  cleanupOsspckgs,
   ingestPackages,
   ingestVersions,
   ingestRepos,

From dee16ec73d587fe6c34d789a3630d6e59e9d445b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Sun, 28 Jun 2026 23:12:06 +0200
Subject: [PATCH 23/29] fix: comments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../src/deps-dev/queries/dependentCountsSql.ts        | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/services/apps/packages_worker/src/deps-dev/queries/dependentCountsSql.ts b/services/apps/packages_worker/src/deps-dev/queries/dependentCountsSql.ts
index 75c2b36ce5..f34d4c9b39 100644
--- a/services/apps/packages_worker/src/deps-dev/queries/dependentCountsSql.ts
+++ b/services/apps/packages_worker/src/deps-dev/queries/dependentCountsSql.ts
@@ -110,6 +110,17 @@ WHILE new_pairs > 0 AND iter < ${MAX_CLOSURE_ITERATIONS} DO
   SET iter = iter + 1;
 END WHILE;
 
+-- 5b. The loop exits on convergence (new_pairs = 0) OR the iteration cap. If it stopped on the cap
+--     with pairs still pending, the closure is INCOMPLETE: transitive_dependent_count would be
+--     silently undercounted and the row-count guard wouldn't catch it (row volume stays ~flat).
+--     Fail the job loudly instead of exporting partial counts. Raising the cap is the fix only if
+--     the graph legitimately deepened past it.
+IF new_pairs > 0 THEN
+  RAISE USING MESSAGE = FORMAT(
+    '${system} reverse-dependent closure did not converge within ${MAX_CLOSURE_ITERATIONS} iterations (new_pairs=%d still pending) - aborting to avoid undercounting transitive dependents',
+    new_pairs);
+END IF;
+
 -- 6. repo mapping for the all-depth dependent_repos_count: latest source repo per package within a
 --    60-day window anchored on the run date. Literal date bounds prune partitions (a subquery-derived
 --    SnapshotAt = (SELECT MAX ...) would NOT prune and scans the whole table).

From 561766e55be130c86c87eea80a2cf057f4ad8fda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Sun, 28 Jun 2026 23:40:44 +0200
Subject: [PATCH 24/29] fix: comments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../src/deps-dev/activities/bqExportToGcs.ts  |  4 ++-
 .../activities/checkEdgeSnapshotQuality.ts    | 31 ++++++++++++++-----
 .../deps-dev/workflows/ingestDependencies.ts  | 12 ++++---
 .../workflows/ingestDependentCounts.ts        |  2 +-
 4 files changed, 35 insertions(+), 14 deletions(-)

diff --git a/services/apps/packages_worker/src/deps-dev/activities/bqExportToGcs.ts b/services/apps/packages_worker/src/deps-dev/activities/bqExportToGcs.ts
index f979dff2e2..50d57b08f5 100644
--- a/services/apps/packages_worker/src/deps-dev/activities/bqExportToGcs.ts
+++ b/services/apps/packages_worker/src/deps-dev/activities/bqExportToGcs.ts
@@ -196,7 +196,9 @@ export async function bqExportToGcs(input: BqExportToGcsInput): Promise<BqExport
     // M4: explicit location to avoid cross-region error when account default != US
     const [dryRunJob] = await bigquery.createQueryJob({ query: sql, dryRun: true, location: 'US' })
     const dryRunBytes = Number(dryRunJob.metadata.statistics.totalBytesProcessed ?? 0)
-    log.info({ jobKind, dryRunBytes, maxBytesGb }, 'BQ dry-run complete')
+    // Log the effective ceiling (env override may differ from the default maxBytesGb) and the
+    // computed byte ceiling, so ops can see what the abort decision is actually compared against.
+    log.info({ jobKind, dryRunBytes, maxBytesGb, effectiveMaxBytesGb, ceiling }, 'BQ dry-run complete')
     if (dryRunBytes > ceiling) {
       throw new Error(
         `BQ dry-run for ${jobKind} reports ${dryRunBytes} bytes > ceiling ${ceiling} — aborting`,
diff --git a/services/apps/packages_worker/src/deps-dev/activities/checkEdgeSnapshotQuality.ts b/services/apps/packages_worker/src/deps-dev/activities/checkEdgeSnapshotQuality.ts
index 35097f7d7d..5f2b86214d 100644
--- a/services/apps/packages_worker/src/deps-dev/activities/checkEdgeSnapshotQuality.ts
+++ b/services/apps/packages_worker/src/deps-dev/activities/checkEdgeSnapshotQuality.ts
@@ -22,8 +22,13 @@ import { assertSnapshotDate } from '../queries/depsSql'
 // produced by the resolution pipeline and were unaffected, so they are not probed here.
 
 export interface CheckEdgeSnapshotQualityInput {
-  snapshotDate: string // YYYY-MM-DD — the resolved BQ partition date being ingested
+  snapshotDate: string // YYYY-MM-DD — the BQ partition the incremental diff reads (ignored when fullScan)
   ecosystems: string[]
+  // Probe the SAME source the ingest reads, so the guard validates the snapshot that actually gets
+  // ingested. Full/fill scan the *Latest views (newest snapshot, no date filter) → probe *Latest.
+  // Incremental reads the `snapshotDate` partition → probe that partition. Without this, a full run
+  // with an older --snapshot-date would validate a stale partition while ingesting a corrupt *Latest.
+  fullScan: boolean
 }
 
 export interface CanaryStat {
@@ -99,8 +104,6 @@ export async function checkEdgeSnapshotQuality(
     return { ok: true, canaries: [] }
   }
 
-  assertSnapshotDate(input.snapshotDate)
-
   // Group canaries by system into `(System = x AND Name IN (...))` predicates so the filter prunes
   // on the (System, Name, Version) clustering — keeps the probe in the single-GB / pennies range.
   const bySystem = new Map<string, string[]>()
@@ -116,6 +119,22 @@ export async function checkEdgeSnapshotQuality(
     )
     .join('\n        OR ')
 
+  // Probe the same source the ingest reads (see fullScan): full/fill scan the *Latest view (newest
+  // snapshot, no date filter); incremental reads the `snapshotDate` partition. The date is only
+  // interpolated (and validated) on the partition path; *Latest needs no date.
+  let edgesTable: string
+  let snapshotFilter: string
+  if (input.fullScan) {
+    edgesTable = 'bigquery-public-data.deps_dev_v1.DependencyGraphEdgesLatest'
+    snapshotFilter = ''
+  } else {
+    assertSnapshotDate(input.snapshotDate)
+    edgesTable = 'bigquery-public-data.deps_dev_v1.DependencyGraphEdges'
+    snapshotFilter = `e.SnapshotAt >= TIMESTAMP('${input.snapshotDate}')
+      AND e.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${input.snapshotDate}', INTERVAL 1 DAY))
+      AND `
+  }
+
   // Direct edges only (From = graph root), matching what the ingest reads. %T formats NULLs
   // as the literal "NULL" so COUNT(DISTINCT ...) isn't nulled out by unresolved To.Version.
   const query = `
@@ -124,10 +143,8 @@ export async function checkEdgeSnapshotQuality(
       e.Name   AS name,
       COUNT(*) AS row_count,
       COUNT(DISTINCT FORMAT('%T|%T|%T', e.From.Version, e.To.Name, e.To.Version)) AS edge_count
-    FROM \`bigquery-public-data.deps_dev_v1.DependencyGraphEdges\` e
-    WHERE e.SnapshotAt >= TIMESTAMP('${input.snapshotDate}')
-      AND e.SnapshotAt <  TIMESTAMP(DATE_ADD(DATE '${input.snapshotDate}', INTERVAL 1 DAY))
-      AND e.From.Name = e.Name AND e.From.Version = e.Version
+    FROM \`${edgesTable}\` e
+    WHERE ${snapshotFilter}e.From.Name = e.Name AND e.From.Version = e.Version
       AND (
         ${systemPredicates}
       )
diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
index 68e6a72726..25b15bddae 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
@@ -224,6 +224,9 @@ export async function ingestDependencies(opts: {
   const isFill = opts.fillConstraints === true
   // Fill mode forces Option A — Option B selects NULL for version_constraint, making the fill a no-op.
   const tableOption = isFill ? 'A' : (opts.depsTableOption ?? 'A')
+  // Full/fill scan the *Latest views; incremental reads the `today` partition. Drives both the SQL
+  // source below and which source the guard probes — they must match.
+  const fullScan = opts.syncMode === 'full' || isFill
 
   // Guard against corrupt deps.dev resolved-graph snapshots BEFORE the (multi-hour) export.
   // Skip when reusing a prior export — we're re-importing already-validated parquet, not
@@ -231,7 +234,7 @@ export async function ingestDependencies(opts: {
   // a bad snapshot, so the guard runs for both. Probes only resolved-graph ecosystems; a clean
   // GO/NUGET-only run finds no canaries and passes through.
   if (!opts.reuseExports) {
-    const guard = await checkEdgeSnapshotQuality({ snapshotDate: opts.today, ecosystems })
+    const guard = await checkEdgeSnapshotQuality({ snapshotDate: opts.today, ecosystems, fullScan })
     if (!guard.ok) {
       throw ApplicationFailure.nonRetryable(
         `edge snapshot quality guard failed for ${opts.today}: ${guard.reason}. ` +
@@ -241,10 +244,9 @@ export async function ingestDependencies(opts: {
     }
   }
   // Fill mode always uses full SQL — needs all rows to find which have NULL version_constraint in DB.
-  const sql =
-    opts.syncMode === 'full' || isFill
-      ? buildDepsFullSql(ecosystems, tableOption)
-      : buildDepsIncrementalSql(opts.today, opts.watermark ?? '', ecosystems, tableOption)
+  const sql = fullScan
+    ? buildDepsFullSql(ecosystems, tableOption)
+    : buildDepsIncrementalSql(opts.today, opts.watermark ?? '', ecosystems, tableOption)
 
   const exportResult = await bqExportToGcs({
     jobKind: 'package_dependencies',
diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependentCounts.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependentCounts.ts
index 86e3cd2757..aa50eb1b81 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependentCounts.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependentCounts.ts
@@ -64,7 +64,7 @@ interface VariantConfig {
   // When true, buildSql returns a multi-statement BQ script (the GO/NUGET exact reverse transitive
   // closure) that ends by creating TEMP TABLE _export_data, rather than a single SELECT. The export
   // activity then appends only EXPORT DATA and enforces the byte ceiling via maximumBytesBilled
-  // instead of a dry-run. See ADR-0004. Left unset (single-SELECT) until the closure SQL lands.
+  // instead of a dry-run. See ADR-0004. Unset (single-SELECT) for the edges variant; set for GO/NUGET.
   isScript?: boolean
 }
 

From 053aefe2162eb5f93b67e9bb92f7f2a3f3545994 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Sun, 28 Jun 2026 23:54:48 +0200
Subject: [PATCH 25/29] fix: comments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../src/deps-dev/activities/checkEdgeSnapshotQuality.ts      | 3 ++-
 .../src/deps-dev/workflows/bootstrapOsspckgs.ts              | 5 ++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/services/apps/packages_worker/src/deps-dev/activities/checkEdgeSnapshotQuality.ts b/services/apps/packages_worker/src/deps-dev/activities/checkEdgeSnapshotQuality.ts
index 5f2b86214d..955e91d273 100644
--- a/services/apps/packages_worker/src/deps-dev/activities/checkEdgeSnapshotQuality.ts
+++ b/services/apps/packages_worker/src/deps-dev/activities/checkEdgeSnapshotQuality.ts
@@ -203,7 +203,8 @@ export async function checkEdgeSnapshotQuality(
     [
       {
         title: 'Snapshot',
-        text: input.snapshotDate,
+        // Full/fill probe *Latest (no partition filter), so the literal date isn't what was scanned.
+        text: input.fullScan ? `${input.snapshotDate} (probed *Latest)` : input.snapshotDate,
       },
       {
         title: 'Canaries',
diff --git a/services/apps/packages_worker/src/deps-dev/workflows/bootstrapOsspckgs.ts b/services/apps/packages_worker/src/deps-dev/workflows/bootstrapOsspckgs.ts
index 69be6e08b1..c12edceed8 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/bootstrapOsspckgs.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/bootstrapOsspckgs.ts
@@ -201,7 +201,10 @@ export async function bootstrapOsspckgs(opts: {
         args: [
           {
             runId,
-            snapshotDate: today,
+            // Honor --snapshot-date like the partition kinds do (snap()). The closure reads *Latest
+            // manifests regardless, but anchors the dependent_repos 60-day window on this date, so a
+            // recovery run with an override must use it for a consistent window.
+            snapshotDate: opts.snapshotDate ?? today,
             variant,
             reuseExports: opts.reuseExports,
             exportName: opts.exportName,

From c7a6dd024364e0db484b87606e634f2aa409641d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Mon, 29 Jun 2026 08:42:36 +0200
Subject: [PATCH 26/29] fix: comments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../packages_worker/src/deps-dev/README.md    | 19 +++++++++----------
 .../src/deps-dev/activities/bqExportToGcs.ts  |  5 ++++-
 .../packages_worker/src/deps-dev/config.ts    |  4 ----
 .../deps-dev/workflows/ingestDependencies.ts  |  9 ++++++++-
 4 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/services/apps/packages_worker/src/deps-dev/README.md b/services/apps/packages_worker/src/deps-dev/README.md
index 92e5b70837..9701055aab 100644
--- a/services/apps/packages_worker/src/deps-dev/README.md
+++ b/services/apps/packages_worker/src/deps-dev/README.md
@@ -68,13 +68,12 @@ The override logic lives in `src/deps-dev/activities/bqExportToGcs.ts`.
 
 ## Environment variables
 
-| Variable                       | Required  | Purpose                                                                                            |
-| ------------------------------ | --------- | -------------------------------------------------------------------------------------------------- |
-| `OSSPCKGS_GCP_PROJECT`         | yes       | GCP project ID for BQ and GCS                                                                      |
-| `OSSPCKGS_GCS_BUCKET`          | yes       | GCS bucket for Parquet exports                                                                     |
-| `OSSPCKGS_GCP_CREDENTIALS_B64` | yes       | Base64-encoded GCP service account JSON                                                            |
-| `OSSPCKGS_DEPS_TABLE`          | no        | Set to `B` to use `DependenciesLatest` (ADR-0003 Option B) instead of `DependencyGraphEdgesLatest` |
-| `CROWD_TEMPORAL_SERVER_URL`    | yes       | Temporal server address                                                                            |
-| `CROWD_TEMPORAL_NAMESPACE`     | yes       | Temporal namespace (overrides `backend-config` default)                                            |
-| `CROWD_TEMPORAL_CERTIFICATE`   | prod only | Base64-encoded mTLS client certificate                                                             |
-| `CROWD_TEMPORAL_PRIVATE_KEY`   | prod only | Base64-encoded mTLS private key                                                                    |
+| Variable                       | Required  | Purpose                                                 |
+| ------------------------------ | --------- | ------------------------------------------------------- |
+| `OSSPCKGS_GCP_PROJECT`         | yes       | GCP project ID for BQ and GCS                           |
+| `OSSPCKGS_GCS_BUCKET`          | yes       | GCS bucket for Parquet exports                          |
+| `OSSPCKGS_GCP_CREDENTIALS_B64` | yes       | Base64-encoded GCP service account JSON                 |
+| `CROWD_TEMPORAL_SERVER_URL`    | yes       | Temporal server address                                 |
+| `CROWD_TEMPORAL_NAMESPACE`     | yes       | Temporal namespace (overrides `backend-config` default) |
+| `CROWD_TEMPORAL_CERTIFICATE`   | prod only | Base64-encoded mTLS client certificate                  |
+| `CROWD_TEMPORAL_PRIVATE_KEY`   | prod only | Base64-encoded mTLS private key                         |
diff --git a/services/apps/packages_worker/src/deps-dev/activities/bqExportToGcs.ts b/services/apps/packages_worker/src/deps-dev/activities/bqExportToGcs.ts
index 50d57b08f5..019de5fc86 100644
--- a/services/apps/packages_worker/src/deps-dev/activities/bqExportToGcs.ts
+++ b/services/apps/packages_worker/src/deps-dev/activities/bqExportToGcs.ts
@@ -198,7 +198,10 @@ export async function bqExportToGcs(input: BqExportToGcsInput): Promise<BqExport
     const dryRunBytes = Number(dryRunJob.metadata.statistics.totalBytesProcessed ?? 0)
     // Log the effective ceiling (env override may differ from the default maxBytesGb) and the
     // computed byte ceiling, so ops can see what the abort decision is actually compared against.
-    log.info({ jobKind, dryRunBytes, maxBytesGb, effectiveMaxBytesGb, ceiling }, 'BQ dry-run complete')
+    log.info(
+      { jobKind, dryRunBytes, maxBytesGb, effectiveMaxBytesGb, ceiling },
+      'BQ dry-run complete',
+    )
     if (dryRunBytes > ceiling) {
       throw new Error(
         `BQ dry-run for ${jobKind} reports ${dryRunBytes} bytes > ceiling ${ceiling} — aborting`,
diff --git a/services/apps/packages_worker/src/deps-dev/config.ts b/services/apps/packages_worker/src/deps-dev/config.ts
index c9565ebc94..b934e9c499 100644
--- a/services/apps/packages_worker/src/deps-dev/config.ts
+++ b/services/apps/packages_worker/src/deps-dev/config.ts
@@ -12,10 +12,6 @@ export const GCS_BUCKET = requireEnv('OSSPCKGS_GCS_BUCKET')
 export const DEPS_DEV_DATASET = 'bigquery-public-data.deps_dev_v1'
 export const SCORECARD_DATASET = 'openssf.scorecardcron'
 
-// ADR-0003: Option A = DependencyGraphEdgesLatest (prod default, has version_constraint).
-// Set OSSPCKGS_DEPS_TABLE=B locally to use DependenciesLatest (cheaper, no version_constraint).
-export const DEPS_TABLE_OPTION: 'A' | 'B' = process.env.OSSPCKGS_DEPS_TABLE === 'B' ? 'B' : 'A'
-
 const credentials = JSON.parse(
   Buffer.from(requireEnv('OSSPCKGS_GCP_CREDENTIALS_B64'), 'base64').toString('utf8'),
 )
diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
index 25b15bddae..22fbe14710 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
@@ -233,7 +233,14 @@ export async function ingestDependencies(opts: {
   // scanning the live snapshot. Both full (*Latest = newest snapshot) and incremental can hit
   // a bad snapshot, so the guard runs for both. Probes only resolved-graph ecosystems; a clean
   // GO/NUGET-only run finds no canaries and passes through.
-  if (!opts.reuseExports) {
+  //
+  // Option A only: the guard's canary ratios are DependencyGraphEdges-schema-specific. Option B
+  // ingests the separate Dependencies/DependenciesLatest table, which the 2026-06 corruption was
+  // never observed in and has no calibrated baseline — probing Edges there would "validate" a table
+  // we don't ingest (false confidence) and could abort a healthy Option B run when only Edges is bad.
+  // Option B is a manual, non-scheduled cost-experiment path (--deps-table-b); leave it unguarded by
+  // design rather than invent a guard for an unproven threat. Option A is the production default.
+  if (!opts.reuseExports && tableOption === 'A') {
     const guard = await checkEdgeSnapshotQuality({ snapshotDate: opts.today, ecosystems, fullScan })
     if (!guard.ok) {
       throw ApplicationFailure.nonRetryable(

From 0fdd3010209021a08cff6ed35bd32a313055214b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Mon, 29 Jun 2026 09:05:20 +0200
Subject: [PATCH 27/29] fix: comments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../src/deps-dev/activities/bqExportToGcs.ts  |  97 ++++++++-------
 .../activities/gcsParquetToStaging.ts         | 116 ++++++++++--------
 .../activities/mergeStagingToTable.ts         |  85 +++++++------
 3 files changed, 164 insertions(+), 134 deletions(-)

diff --git a/services/apps/packages_worker/src/deps-dev/activities/bqExportToGcs.ts b/services/apps/packages_worker/src/deps-dev/activities/bqExportToGcs.ts
index 019de5fc86..b47e66865d 100644
--- a/services/apps/packages_worker/src/deps-dev/activities/bqExportToGcs.ts
+++ b/services/apps/packages_worker/src/deps-dev/activities/bqExportToGcs.ts
@@ -217,14 +217,18 @@ export async function bqExportToGcs(input: BqExportToGcsInput): Promise<BqExport
     ...(ecosystems ? { tableRowCounts: { 'meta:ecosystems': ecosystems } } : {}),
   })
 
-  // Both modes finish by exporting from a `_export_data` TEMP table — materializing first forces
-  // BQ to shard the parquet output properly (direct EXPORT DATA from a subquery emits O(rows)
-  // ~1 KB micro-files).
-  // - Single-SELECT: wrap `sql` in SELECT * FROM (...) so QUALIFY / top-level set ops don't break
-  //   the CREATE TEMP TABLE, then export.
-  // - Script (isScript): the script already builds `_export_data` as its final statement, so we
-  //   append only the EXPORT DATA — its temp tables run inline and auto-drop when the session ends.
-  const exportTail = `
+  // From here the row is 'exporting'; any BQ failure (incl. script-mode maximumBytesBilled aborts)
+  // must flip it to 'failed' with the reason, else it stays stuck 'exporting' forever and the
+  // failure reason is lost to monitoring. Mirrors the rankPackages pattern (criticality/activities).
+  try {
+    // Both modes finish by exporting from a `_export_data` TEMP table — materializing first forces
+    // BQ to shard the parquet output properly (direct EXPORT DATA from a subquery emits O(rows)
+    // ~1 KB micro-files).
+    // - Single-SELECT: wrap `sql` in SELECT * FROM (...) so QUALIFY / top-level set ops don't break
+    //   the CREATE TEMP TABLE, then export.
+    // - Script (isScript): the script already builds `_export_data` as its final statement, so we
+    //   append only the EXPORT DATA — its temp tables run inline and auto-drop when the session ends.
+    const exportTail = `
 EXPORT DATA OPTIONS(
   uri='${gcsPrefix}*.parquet',
   format='PARQUET',
@@ -232,45 +236,52 @@ EXPORT DATA OPTIONS(
   overwrite=true
 ) AS SELECT * FROM _export_data;
 `
-  const exportSql = isScript
-    ? `${sql.replace(/;\s*$/, '')};\n${exportTail}`
-    : `\nCREATE TEMP TABLE _export_data AS SELECT * FROM (${sql.replace(/;\s*$/, '')});\n${exportTail}`
+    const exportSql = isScript
+      ? `${sql.replace(/;\s*$/, '')};\n${exportTail}`
+      : `\nCREATE TEMP TABLE _export_data AS SELECT * FROM (${sql.replace(/;\s*$/, '')});\n${exportTail}`
 
-  log.info({ jobKind, jobId, gcsPrefix, isScript: Boolean(isScript) }, 'Starting BQ export')
+    log.info({ jobKind, jobId, gcsPrefix, isScript: Boolean(isScript) }, 'Starting BQ export')
 
-  const [job] = await bigquery.createQueryJob({
-    query: exportSql,
-    location: 'US',
-    // Server-side runaway guard for script mode — aborts the job if a statement scans beyond the
-    // ceiling. Single-SELECT mode is already gated by the dry-run check above.
-    ...(isScript ? { maximumBytesBilled: String(Math.floor(ceiling)) } : {}),
-  })
-  await job.promise()
-  const bqStats = await extractBqStats(job, bigquery)
-
-  const rowCount = bqStats.outputRows ?? 0
+    const [job] = await bigquery.createQueryJob({
+      query: exportSql,
+      location: 'US',
+      // Server-side runaway guard for script mode — aborts the job if a statement scans beyond the
+      // ceiling. Single-SELECT mode is already gated by the dry-run check above.
+      ...(isScript ? { maximumBytesBilled: String(Math.floor(ceiling)) } : {}),
+    })
+    await job.promise()
+    const bqStats = await extractBqStats(job, bigquery)
 
-  await markJobStatus(qx, jobId, 'exported', {
-    gcsPrefix,
-    rowCountBq: rowCount,
-    bqBytesBilled: bqStats.bqBytesBilled,
-    bqJobId: bqStats.bqJobId,
-    bqStats,
-    tableRowCounts: { 'bq:export': rowCount },
-  })
+    const rowCount = bqStats.outputRows ?? 0
 
-  log.info(
-    {
-      jobKind,
-      jobId,
-      rowCount,
+    await markJobStatus(qx, jobId, 'exported', {
+      gcsPrefix,
+      rowCountBq: rowCount,
+      bqBytesBilled: bqStats.bqBytesBilled,
       bqJobId: bqStats.bqJobId,
-      totalBytesProcessed: bqStats.totalBytesProcessed,
-      totalSlotMs: bqStats.totalSlotMs,
-      durationMs: bqStats.durationMs,
-    },
-    'BQ export complete',
-  )
+      bqStats,
+      tableRowCounts: { 'bq:export': rowCount },
+    })
 
-  return { gcsPrefix, rowCount, bqBytesBilled: bqStats.bqBytesBilled, jobId }
+    log.info(
+      {
+        jobKind,
+        jobId,
+        rowCount,
+        bqJobId: bqStats.bqJobId,
+        totalBytesProcessed: bqStats.totalBytesProcessed,
+        totalSlotMs: bqStats.totalSlotMs,
+        durationMs: bqStats.durationMs,
+      },
+      'BQ export complete',
+    )
+
+    return { gcsPrefix, rowCount, bqBytesBilled: bqStats.bqBytesBilled, jobId }
+  } catch (err) {
+    await markJobStatus(qx, jobId, 'failed', {
+      errorMessage: (err as Error).message,
+      finishedAt: new Date(),
+    })
+    throw err
+  }
 }
diff --git a/services/apps/packages_worker/src/deps-dev/activities/gcsParquetToStaging.ts b/services/apps/packages_worker/src/deps-dev/activities/gcsParquetToStaging.ts
index e4efbdf4de..37b001d066 100644
--- a/services/apps/packages_worker/src/deps-dev/activities/gcsParquetToStaging.ts
+++ b/services/apps/packages_worker/src/deps-dev/activities/gcsParquetToStaging.ts
@@ -114,66 +114,76 @@ export async function gcsParquetToStaging(input: GcsToStagingInput): Promise<Gcs
   const qx = await getPackagesDb()
 
   await markJobStatus(qx, jobId, 'loading')
-  if (stagingDdl) {
-    const stmts = Array.isArray(stagingDdl) ? stagingDdl : [stagingDdl]
-    for (const stmt of stmts) await qx.result(stmt)
-  }
-  await qx.result(`TRUNCATE ${stagingTable}`)
-  // Reset progress at the start of a fresh run (first chunk). This handles job ID reuse
-  // via --export-name: a previous run's 193/193 would otherwise persist due to GREATEST.
-  if (filesOffset === 0 && input.totalFiles != null) {
-    await updateLoadingProgress(qx, jobId, 0, input.totalFiles, true)
-  }
-
-  let parquetFileNames: string[]
-  if (input.fileNames) {
-    parquetFileNames = input.fileNames
-  } else if (input.gcsPrefix) {
-    const objectPrefix = gcsPrefixToObjectPrefix(input.gcsPrefix)
-    const [files] = await bucket.getFiles({ prefix: objectPrefix })
-    parquetFileNames = files.filter((f) => f.name.endsWith('.parquet')).map((f) => f.name)
-  } else {
-    throw new Error('gcsParquetToStaging: must provide either fileNames or gcsPrefix')
-  }
-
-  const totalFiles = input.totalFiles ?? parquetFileNames.length
+  // Row is 'loading' from here; a staging-load failure must flip it to 'failed' with the reason
+  // rather than leave it stuck 'loading'. Mirrors the rankPackages pattern (criticality/activities).
+  try {
+    if (stagingDdl) {
+      const stmts = Array.isArray(stagingDdl) ? stagingDdl : [stagingDdl]
+      for (const stmt of stmts) await qx.result(stmt)
+    }
+    await qx.result(`TRUNCATE ${stagingTable}`)
+    // Reset progress at the start of a fresh run (first chunk). This handles job ID reuse
+    // via --export-name: a previous run's 193/193 would otherwise persist due to GREATEST.
+    if (filesOffset === 0 && input.totalFiles != null) {
+      await updateLoadingProgress(qx, jobId, 0, input.totalFiles, true)
+    }
 
-  log.info(
-    { jobId, stagingTable, fileCount: parquetFileNames.length, filesOffset, totalFiles },
-    'Loading parquet files into staging',
-  )
+    let parquetFileNames: string[]
+    if (input.fileNames) {
+      parquetFileNames = input.fileNames
+    } else if (input.gcsPrefix) {
+      const objectPrefix = gcsPrefixToObjectPrefix(input.gcsPrefix)
+      const [files] = await bucket.getFiles({ prefix: objectPrefix })
+      parquetFileNames = files.filter((f) => f.name.endsWith('.parquet')).map((f) => f.name)
+    } else {
+      throw new Error('gcsParquetToStaging: must provide either fileNames or gcsPrefix')
+    }
 
-  let totalLoaded = 0
+    const totalFiles = input.totalFiles ?? parquetFileNames.length
 
-  for (let i = 0; i < parquetFileNames.length; i += MAX_CONCURRENT) {
-    const chunk = parquetFileNames.slice(i, i + MAX_CONCURRENT)
-    const counts = await Promise.all(
-      chunk.map((name) => loadParquetFile(qx, stagingTable, pgColumns, name, tsCols, decCols)),
-    )
-    totalLoaded += counts.reduce((a, b) => a + b, 0)
-    const doneInBatch = i + chunk.length
-    const doneGlobal = filesOffset + doneInBatch
     log.info(
-      {
-        jobId,
-        totalLoaded,
-        progress: `${doneGlobal}/${totalFiles} (${Math.round((doneGlobal / totalFiles) * 100)}%)`,
-      },
-      'Staging load progress',
+      { jobId, stagingTable, fileCount: parquetFileNames.length, filesOffset, totalFiles },
+      'Loading parquet files into staging',
     )
-    Context.current().heartbeat({ done: doneGlobal, total: totalFiles })
-    await updateLoadingProgress(qx, jobId, doneGlobal, totalFiles)
-  }
 
-  const cumulativeStagingRows = (input.priorStagingRows ?? 0) + totalLoaded
-  await markJobStatus(qx, jobId, 'loading', {
-    rowCountStaging: cumulativeStagingRows,
-    tableRowCounts: { [`staging:${stagingTable}`]: totalLoaded },
-  })
+    let totalLoaded = 0
+
+    for (let i = 0; i < parquetFileNames.length; i += MAX_CONCURRENT) {
+      const chunk = parquetFileNames.slice(i, i + MAX_CONCURRENT)
+      const counts = await Promise.all(
+        chunk.map((name) => loadParquetFile(qx, stagingTable, pgColumns, name, tsCols, decCols)),
+      )
+      totalLoaded += counts.reduce((a, b) => a + b, 0)
+      const doneInBatch = i + chunk.length
+      const doneGlobal = filesOffset + doneInBatch
+      log.info(
+        {
+          jobId,
+          totalLoaded,
+          progress: `${doneGlobal}/${totalFiles} (${Math.round((doneGlobal / totalFiles) * 100)}%)`,
+        },
+        'Staging load progress',
+      )
+      Context.current().heartbeat({ done: doneGlobal, total: totalFiles })
+      await updateLoadingProgress(qx, jobId, doneGlobal, totalFiles)
+    }
+
+    const cumulativeStagingRows = (input.priorStagingRows ?? 0) + totalLoaded
+    await markJobStatus(qx, jobId, 'loading', {
+      rowCountStaging: cumulativeStagingRows,
+      tableRowCounts: { [`staging:${stagingTable}`]: totalLoaded },
+    })
 
-  await qx.result(`ANALYZE ${stagingTable}`)
+    await qx.result(`ANALYZE ${stagingTable}`)
 
-  log.info({ jobId, stagingTable, totalLoaded }, 'Staging load complete')
+    log.info({ jobId, stagingTable, totalLoaded }, 'Staging load complete')
 
-  return { rowsLoaded: totalLoaded }
+    return { rowsLoaded: totalLoaded }
+  } catch (err) {
+    await markJobStatus(qx, jobId, 'failed', {
+      errorMessage: (err as Error).message,
+      finishedAt: new Date(),
+    })
+    throw err
+  }
 }
diff --git a/services/apps/packages_worker/src/deps-dev/activities/mergeStagingToTable.ts b/services/apps/packages_worker/src/deps-dev/activities/mergeStagingToTable.ts
index fdbb481d5c..06ed0e1705 100644
--- a/services/apps/packages_worker/src/deps-dev/activities/mergeStagingToTable.ts
+++ b/services/apps/packages_worker/src/deps-dev/activities/mergeStagingToTable.ts
@@ -50,51 +50,60 @@ export async function mergeStagingToTable(input: MergeStagingInput): Promise<Mer
   const qx = await getPackagesDb()
 
   await markJobStatus(qx, jobId, 'merging')
+  // Row is 'merging' from here; a merge failure must flip it to 'failed' with the reason rather
+  // than leave it stuck 'merging'. Mirrors the rankPackages pattern (criticality/activities).
+  try {
+    let rowsAffected = 0
+    const tableRowCounts: Record<string, number> = {}
 
-  let rowsAffected = 0
-  const tableRowCounts: Record<string, number> = {}
+    await qx.tx(async (tx) => {
+      for (const sql of prepareStatements) {
+        await tx.result(sql)
+      }
+      for (let i = 0; i < statements.length; i++) {
+        const count = await tx.result(statements[i])
+        rowsAffected += count
+        const table = names[i] ?? `table_${i}`
+        tableRowCounts[table] = (tableRowCounts[table] ?? 0) + count
+      }
+    })
 
-  await qx.tx(async (tx) => {
-    for (const sql of prepareStatements) {
-      await tx.result(sql)
-    }
-    for (let i = 0; i < statements.length; i++) {
-      const count = await tx.result(statements[i])
-      rowsAffected += count
-      const table = names[i] ?? `table_${i}`
-      tableRowCounts[table] = (tableRowCounts[table] ?? 0) + count
+    if (chunkInfo) {
+      log.info(
+        { jobId, rowsAffected, tableRowCounts, chunk: `${chunkInfo.index + 1}/${chunkInfo.total}` },
+        'Chunk merge complete',
+      )
     }
-  })
 
-  if (chunkInfo) {
-    log.info(
-      { jobId, rowsAffected, tableRowCounts, chunk: `${chunkInfo.index + 1}/${chunkInfo.total}` },
-      'Chunk merge complete',
-    )
-  }
-
-  if (!isFinal) {
-    await markJobStatus(qx, jobId, 'merging', {
-      rowCountPg: priorRowsAffected + rowsAffected,
-    })
-  }
+    if (!isFinal) {
+      await markJobStatus(qx, jobId, 'merging', {
+        rowCountPg: priorRowsAffected + rowsAffected,
+      })
+    }
 
-  if (isFinal) {
-    const totalRowsAffected = priorRowsAffected + rowsAffected
-    const totalTableRowCounts: Record<string, number> = { ...priorTableRowCounts }
-    for (const [table, count] of Object.entries(tableRowCounts)) {
-      totalTableRowCounts[table] = (totalTableRowCounts[table] ?? 0) + count
+    if (isFinal) {
+      const totalRowsAffected = priorRowsAffected + rowsAffected
+      const totalTableRowCounts: Record<string, number> = { ...priorTableRowCounts }
+      for (const [table, count] of Object.entries(tableRowCounts)) {
+        totalTableRowCounts[table] = (totalTableRowCounts[table] ?? 0) + count
+      }
+      await markJobStatus(qx, jobId, 'done', {
+        finishedAt: new Date(),
+        rowCountPg: totalRowsAffected,
+        tableRowCounts: totalTableRowCounts,
+      })
+      log.info(
+        { jobId, rowsAffected: totalRowsAffected, tableRowCounts: totalTableRowCounts },
+        'Merge complete',
+      )
     }
-    await markJobStatus(qx, jobId, 'done', {
+
+    return { rowsAffected, tableRowCounts }
+  } catch (err) {
+    await markJobStatus(qx, jobId, 'failed', {
+      errorMessage: (err as Error).message,
       finishedAt: new Date(),
-      rowCountPg: totalRowsAffected,
-      tableRowCounts: totalTableRowCounts,
     })
-    log.info(
-      { jobId, rowsAffected: totalRowsAffected, tableRowCounts: totalTableRowCounts },
-      'Merge complete',
-    )
+    throw err
   }
-
-  return { rowsAffected, tableRowCounts }
 }

From 4a7e5ca46e5a4a0f3271b228a0058cc89cf34c11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Mon, 29 Jun 2026 14:40:42 +0200
Subject: [PATCH 28/29] fix: comments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../src/deps-dev/workflows/ingestDependencies.ts          | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
index 22fbe14710..111f77534f 100644
--- a/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
+++ b/services/apps/packages_worker/src/deps-dev/workflows/ingestDependencies.ts
@@ -259,13 +259,17 @@ export async function ingestDependencies(opts: {
     jobKind: 'package_dependencies',
     sql,
     runId: opts.runId,
-    syncMode: opts.syncMode,
+    // Report the PHYSICAL scan mode, not the requested one. A fill run (fillConstraints) forces a
+    // full *Latest scan even when opts.syncMode is 'incremental'; passing the raw mode would record
+    // the job as incremental and make bqExportToGcs pick the INCREMENTAL byte-ceiling env override
+    // for a query that's actually full. fullScan already gates the SQL + maxBytesGb below.
+    syncMode: fullScan ? 'full' : opts.syncMode,
     snapshotAt: opts.today,
     // Full/fill scan the *Latest views (everything) → 25000. Incremental is a snapshot edge-diff
     // (today vs watermark partitions of DependencyGraphEdges + GoRequirements + NuGetRequirements);
     // measured ~4.1TB for Option A. 10000 leaves ~2.4x headroom and still trips a runaway full-table
     // scan. Overridable via BQ_DATASET_INGEST_PACKAGE_DEPENDENCIES[_INCREMENTAL]_MAX_BQ_GB (see README).
-    maxBytesGb: opts.syncMode === 'full' || isFill ? 25000 : 10000,
+    maxBytesGb: fullScan ? 25000 : 10000,
     reuseExports: opts.reuseExports,
     exportName: opts.exportName,
     ecosystems,

From f8c4463ab39adb3c1aa215b0cbfb9bcb84b293ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Uro=C5=A1=20Marolt?= <uros@marolt.me>
Date: Mon, 29 Jun 2026 19:56:51 +0200
Subject: [PATCH 29/29] fix: comments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Uroš Marolt <uros@marolt.me>
---
 .../src/deps-dev/activities/bqExportToGcs.ts                | 2 +-
 .../src/deps-dev/activities/checkDependentCountsGuard.ts    | 6 ++----
 .../src/deps-dev/activities/gcsParquetToStaging.ts          | 2 +-
 .../src/deps-dev/activities/mergeStagingToTable.ts          | 2 +-
 4 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/services/apps/packages_worker/src/deps-dev/activities/bqExportToGcs.ts b/services/apps/packages_worker/src/deps-dev/activities/bqExportToGcs.ts
index b47e66865d..ebdc531bea 100644
--- a/services/apps/packages_worker/src/deps-dev/activities/bqExportToGcs.ts
+++ b/services/apps/packages_worker/src/deps-dev/activities/bqExportToGcs.ts
@@ -279,7 +279,7 @@ EXPORT DATA OPTIONS(
     return { gcsPrefix, rowCount, bqBytesBilled: bqStats.bqBytesBilled, jobId }
   } catch (err) {
     await markJobStatus(qx, jobId, 'failed', {
-      errorMessage: (err as Error).message,
+      errorMessage: err instanceof Error ? err.message : String(err),
       finishedAt: new Date(),
     })
     throw err
diff --git a/services/apps/packages_worker/src/deps-dev/activities/checkDependentCountsGuard.ts b/services/apps/packages_worker/src/deps-dev/activities/checkDependentCountsGuard.ts
index 92bc1661bc..3285d88a16 100644
--- a/services/apps/packages_worker/src/deps-dev/activities/checkDependentCountsGuard.ts
+++ b/services/apps/packages_worker/src/deps-dev/activities/checkDependentCountsGuard.ts
@@ -1,7 +1,5 @@
-import {
-  OsspckgsJobKind,
-  getLastCompletedJobRowCount as dalGetLastCompletedJobRowCount,
-} from '@crowd/data-access-layer'
+import { getLastCompletedJobRowCount as dalGetLastCompletedJobRowCount } from '@crowd/data-access-layer'
+import type { OsspckgsJobKind } from '@crowd/data-access-layer'
 import { SlackChannel, SlackPersona, sendSlackNotification } from '@crowd/slack'
 
 import { getPackagesDb } from '../../db'
diff --git a/services/apps/packages_worker/src/deps-dev/activities/gcsParquetToStaging.ts b/services/apps/packages_worker/src/deps-dev/activities/gcsParquetToStaging.ts
index 37b001d066..8ec9923b0e 100644
--- a/services/apps/packages_worker/src/deps-dev/activities/gcsParquetToStaging.ts
+++ b/services/apps/packages_worker/src/deps-dev/activities/gcsParquetToStaging.ts
@@ -181,7 +181,7 @@ export async function gcsParquetToStaging(input: GcsToStagingInput): Promise<Gcs
     return { rowsLoaded: totalLoaded }
   } catch (err) {
     await markJobStatus(qx, jobId, 'failed', {
-      errorMessage: (err as Error).message,
+      errorMessage: err instanceof Error ? err.message : String(err),
       finishedAt: new Date(),
     })
     throw err
diff --git a/services/apps/packages_worker/src/deps-dev/activities/mergeStagingToTable.ts b/services/apps/packages_worker/src/deps-dev/activities/mergeStagingToTable.ts
index 06ed0e1705..4a144a8c85 100644
--- a/services/apps/packages_worker/src/deps-dev/activities/mergeStagingToTable.ts
+++ b/services/apps/packages_worker/src/deps-dev/activities/mergeStagingToTable.ts
@@ -101,7 +101,7 @@ export async function mergeStagingToTable(input: MergeStagingInput): Promise<Mer
     return { rowsAffected, tableRowCounts }
   } catch (err) {
     await markJobStatus(qx, jobId, 'failed', {
-      errorMessage: (err as Error).message,
+      errorMessage: err instanceof Error ? err.message : String(err),
       finishedAt: new Date(),
     })
     throw err