feat: add rubygems ecosystem to bq-dataset-ingest (CM-1296)

services/apps/packages_worker/src/deps-dev/activities/createVersionsLookup.ts

@@ -4,7 +4,7 @@ import { getPackagesDb } from '../../db'
 
 const log = getServiceChildLogger('createVersionsLookup')
 
-const VALID_ECOSYSTEMS = new Set(['npm', 'go', 'maven', 'pypi', 'nuget', 'cargo'])
+const VALID_ECOSYSTEMS = new Set(['npm', 'go', 'maven', 'pypi', 'nuget', 'cargo', 'rubygems'])
 
 // Builds a persistent UNLOGGED lookup table in the staging schema for the root-version JOIN in
 // ingestDependencies. Using a temp table per chunk would rebuild 4GB for every chunk on npm

services/apps/packages_worker/src/deps-dev/activities/getResumeExport.ts

@@ -0,0 +1,42 @@
+import { getIngestJobForResume } from '@crowd/data-access-layer'
+
+import { getPackagesDb } from '../../db'
+
+export interface GetResumeExportInput {
+  jobId: number
+}
+
+export interface GetResumeExportOutput {
+  jobId: number
+  jobKind: string
+  status: string
+  syncMode: string
+  gcsPrefix: string | null
+  progressDone: number
+  progressTotal: number
+  rowCountPg: number
+}
+
+// Pure fetch of a prior job's resume-relevant fields (export path, status, file-load progress,
+// rows merged). Returns null if the job id does not exist. All validation — kind, status, presence
+// of an export, and that the parquet files still exist — is done by the caller (ingestDependencies)
+// so it can fail fast with non-retryable errors instead of retrying a bad-input activity.
+export async function getResumeExport(
+  input: GetResumeExportInput,
+): Promise<GetResumeExportOutput | null> {
+  const qx = await getPackagesDb()
+  const job = await getIngestJobForResume(qx, input.jobId)
+  if (!job) {
+    return null
+  }
+  return {
+    jobId: job.id,
+    jobKind: job.jobKind,
+    status: job.status,
+    syncMode: job.syncMode,
+    gcsPrefix: job.gcsPrefix,
+    progressDone: job.progressDone,
+    progressTotal: job.progressTotal,
+    rowCountPg: job.rowCountPg,
+  }
+}

services/apps/packages_worker/src/deps-dev/activities/index.ts

@@ -1,14 +1,11 @@
 export * from './bqExportToGcs'
 export * from './setJobStep'
 export * from './createVersionsLookup'
-export * from './managePackageDepsConstraints'
-export * from './managePackageDepsIndexes'
-export * from './manageVersionsConstraints'
-export * from './manageVersionsIndexes'
 export * from './listParquetFiles'
 export * from './gcsParquetToStaging'
 export * from './mergeStagingToTable'
 export * from './getLastSnapshot'
+export * from './getResumeExport'
 export * from './checkDependentCountsGuard'
 export * from './checkEdgeSnapshotQuality'
 export * from './probePartitionExists'