chore: deprecate rubygems-dependents-ingest workflow (#4339)

Signed-off-by: Mouad BANI <mouad-mb@outlook.com>

Author: mbani01

services/apps/packages_worker/src/activities.ts

@@ -33,11 +33,7 @@ export {
 } from './pypi/activities'
 export { getCriticalPypiCount } from './pypi/downloads/getCriticalPypiCount'
 export { processNuGetBatch } from './nuget/activities'
-export {
-  processRubyGemsCoreBatch,
-  processRubyGemsCriticalBatch,
-  processRubyGemsDependentsBatch,
-} from './rubygems/activities'
+export { processRubyGemsCoreBatch, processRubyGemsCriticalBatch } from './rubygems/activities'
 export {
   processSecurityContactsBatch,
   ingestSecurityContactsForPurlActivity,

services/apps/packages_worker/src/bin/rubygems-worker.ts

@@ -1,14 +1,9 @@
-import {
-  scheduleRubyGemsCriticalIngestion,
-  scheduleRubyGemsDependentsIngestion,
-  scheduleRubyGemsIngestion,
-} from '../rubygems/schedule'
+import { scheduleRubyGemsCriticalIngestion, scheduleRubyGemsIngestion } from '../rubygems/schedule'
 import { svc } from '../service'
 
 setImmediate(async () => {
   await svc.init()
   await scheduleRubyGemsIngestion()
   await scheduleRubyGemsCriticalIngestion()
-  await scheduleRubyGemsDependentsIngestion()
   await svc.start()
 })

services/apps/packages_worker/src/config.ts

@@ -100,13 +100,6 @@ export function getRubyGemsCriticalConfig() {
   }
 }
 
-export function getRubyGemsDependentsConfig() {
-  return {
-    batchSize: parseInt(process.env.RUBYGEMS_DEPENDENTS_BATCH_SIZE ?? '10000', 10),
-    concurrency: parseInt(process.env.RUBYGEMS_DEPENDENTS_CONCURRENCY ?? '50', 10),
-  }
-}
-
 export function getDockerhubConfig() {
   return {
     hubBaseUrl: requireEnv('DOCKERHUB_API_BASE_URL'),

services/apps/packages_worker/src/rubygems/activities.ts

@@ -1,18 +1,10 @@
 import { getServiceChildLogger } from '@crowd/logging'
 
-import {
-  getRubyGemsConfig,
-  getRubyGemsCriticalConfig,
-  getRubyGemsDependentsConfig,
-} from '../config'
+import { getRubyGemsConfig, getRubyGemsCriticalConfig } from '../config'
 import { getPackagesDb } from '../db'
 
 import { processBatch as processCoreBatch } from './runRubyGemsCoreLoop'
 import { processBatch as processCriticalBatch } from './runRubyGemsCriticalLoop'
-import {
-  DependentsBatchResult,
-  processBatch as processDependentsBatch,
-} from './runRubyGemsDependentsLoop'
 import { BatchResult } from './types'
 
 const log = getServiceChildLogger('rubygems-activity')
@@ -35,13 +27,3 @@ export async function processRubyGemsCriticalBatch(
   log.info({ ...result }, 'RubyGems critical batch complete')
   return result
 }
-
-export async function processRubyGemsDependentsBatch(
-  afterId = '0',
-): Promise<DependentsBatchResult> {
-  const config = getRubyGemsDependentsConfig()
-  const qx = await getPackagesDb()
-  const result = await processDependentsBatch(qx, config, afterId)
-  log.info({ ...result }, 'RubyGems dependents batch complete')
-  return result
-}

services/apps/packages_worker/src/rubygems/client.ts

@@ -52,9 +52,3 @@ export function fetchOwners(name: string): Promise<RubyGemsFetchResult<RubyGemsO
     `https://rubygems.org/api/v1/gems/${encodeURIComponent(name)}/owners.json`,
   )
 }
-
-export function fetchReverseDependencies(name: string): Promise<RubyGemsFetchResult<string[]>> {
-  return rubyGemsGet<string[]>(
-    `https://rubygems.org/api/v1/gems/${encodeURIComponent(name)}/reverse_dependencies.json`,
-  )
-}

services/apps/packages_worker/src/rubygems/runRubyGemsDependentsLoop.ts

@@ -1,11 +1,7 @@
 import { ScheduleAlreadyRunning, ScheduleOverlapPolicy } from '@temporalio/client'
 
 import { svc } from '../service'
-import {
-  ingestRubyGemsCriticalDetails,
-  ingestRubyGemsDependents,
-  ingestRubyGemsPackages,
-} from '../workflows'
+import { ingestRubyGemsCriticalDetails, ingestRubyGemsPackages } from '../workflows'
 
 export async function scheduleRubyGemsIngestion(): Promise<void> {
   const { temporal } = svc
@@ -80,40 +76,3 @@ export async function scheduleRubyGemsCriticalIngestion(): Promise<void> {
     }
   }
 }
-
-export async function scheduleRubyGemsDependentsIngestion(): Promise<void> {
-  const { temporal } = svc
-  if (!temporal) throw new Error('Temporal client not initialized')
-
-  try {
-    await temporal.schedule.create({
-      scheduleId: 'rubygems-dependents-ingest',
-      spec: {
-        cronExpressions: ['0 3 * * 0'],
-      },
-      policies: {
-        overlap: ScheduleOverlapPolicy.SKIP,
-        catchupWindow: '1 hour',
-      },
-      action: {
-        type: 'startWorkflow',
-        workflowType: ingestRubyGemsDependents,
-        workflowId: 'rubygems-weekly-dependents',
-        taskQueue: 'rubygems-worker',
-        workflowRunTimeout: '24 hours',
-        retry: {
-          initialInterval: '30 seconds',
-          backoffCoefficient: 2,
-          maximumAttempts: 5,
-        },
-        args: [],
-      },
-    })
-  } catch (err) {
-    if (err instanceof ScheduleAlreadyRunning) {
-      svc.log.info('Schedule rubygems-dependents-ingest already exists, skipping creation.')
-    } else {
-      throw err
-    }
-  }
-}

services/apps/packages_worker/src/rubygems/schedule.ts

@@ -24,12 +24,3 @@ export async function ingestRubyGemsCriticalDetails(afterId = '0'): Promise<void
   }
   await continueAsNew<typeof ingestRubyGemsCriticalDetails>(result.lastId)
 }
-
-export async function ingestRubyGemsDependents(afterId = '0'): Promise<void> {
-  const result = await acts.processRubyGemsDependentsBatch(afterId)
-  if (result.lastId === null) {
-    log.info('RubyGems dependents ingestion complete — no more work, exiting.', { ...result })
-    return
-  }
-  await continueAsNew<typeof ingestRubyGemsDependents>(result.lastId)
-}

services/apps/packages_worker/src/rubygems/workflows.ts

@@ -25,11 +25,7 @@ export {
   ingestPypiDownloadsDaily,
 } from '../pypi/downloads/ingestPypiDownloads'
 export { ingestNuGetPackages } from '../nuget/workflows'
-export {
-  ingestRubyGemsCriticalDetails,
-  ingestRubyGemsDependents,
-  ingestRubyGemsPackages,
-} from '../rubygems/workflows'
+export { ingestRubyGemsCriticalDetails, ingestRubyGemsPackages } from '../rubygems/workflows'
 export {
   ingestSecurityContacts,
   ingestSecurityContactsForPurlWorkflow,

services/apps/packages_worker/src/workflows/index.ts

@@ -62,41 +62,3 @@ export async function listRubyGemsCriticalPackagesToSync(
     { limit, afterId },
   )
 }
-
-export type RubyGemsPackageForDependents = {
-  id: string
-  name: string
-}
-
-export async function listRubyGemsPackagesForDependents(
-  qx: QueryExecutor,
-  options: { limit: number; afterId?: string },
-): Promise<RubyGemsPackageForDependents[]> {
-  const { limit, afterId = '0' } = options
-  return qx.select(
-    `
-    SELECT p.id, p.name
-    FROM packages p
-    WHERE p.ecosystem = 'rubygems'
-      AND p.id > $(afterId)::bigint
-    ORDER BY p.id ASC
-    LIMIT $(limit)
-    `,
-    { limit, afterId },
-  )
-}
-
-export async function updateRubyGemsDependentCount(
-  qx: QueryExecutor,
-  packageId: string,
-  dependentCount: number,
-): Promise<void> {
-  await qx.result(
-    `UPDATE packages
-        SET dependent_count = $(dependentCount),
-            last_synced_at = NOW()
-      WHERE id = $(packageId)::bigint
-        AND dependent_count IS DISTINCT FROM $(dependentCount)`,
-    { packageId, dependentCount },
-  )
-}