chore: cleanup enricher config

Signed-off-by: Mouad BANI <mouad-mb@outlook.com>

Author: mbani01

backend/.env.dist.local

@@ -174,3 +174,9 @@ CROWD_PACKAGES_DB_PORT=5434
 CROWD_PACKAGES_DB_USERNAME=postgres
 CROWD_PACKAGES_DB_PASSWORD=example
 CROWD_PACKAGES_DB_DATABASE=packages-db
+
+# github-repos-enricher
+ENRICHER_GITHUB_TOKENS=
+ENRICHER_BATCH_SIZE=100
+ENRICHER_REPO_UPDATE_INTERVAL_HOURS=24
+ENRICHER_IDLE_SLEEP_SEC=60

scripts/services/github-repos-enricher.yaml

@@ -6,11 +6,9 @@ x-env-args: &env-args
   SERVICE: github-repos-enricher
   SHELL: /bin/sh
   SUPPRESS_NO_CONFIG_WARNING: 'true'
-  PAGE_SIZE: '200'
-  BATCH_SIZE: '50'
-  MAX_RETRIES: '3'
-  UPDATE_INTERVAL_HOURS: '24'
-  IDLE_SLEEP_SEC: '60'
+  ENRICHER_BATCH_SIZE: '100'
+  ENRICHER_REPO_UPDATE_INTERVAL_HOURS: '24'
+  ENRICHER_IDLE_SLEEP_SEC: '60'
 
 services:
   github-repos-enricher:

services/apps/packages_worker/src/bin/github-repos-enricher.ts

@@ -1,6 +1,3 @@
-import fs from 'fs'
-import path from 'path'
-
 import { getServiceLogger } from '@crowd/logging'
 
 import { getEnricherConfig } from '../config'
@@ -9,9 +6,6 @@ import { runEnrichmentLoop } from '../enricher/runEnrichmentLoop'
 
 const log = getServiceLogger()
 
-const liveFilePath = path.join(__dirname, '../tmp/github-repos-enricher-live.tmp')
-const readyFilePath = path.join(__dirname, '../tmp/github-repos-enricher-ready.tmp')
-
 let shuttingDown = false
 
 const shutdown = async () => {
@@ -37,28 +31,13 @@ const main = async () => {
   await qx.selectOne('SELECT 1')
   log.info('Connected to packages-db.')
 
-  fs.mkdirSync(path.dirname(liveFilePath), { recursive: true })
-
-  const healthInterval = setInterval(async () => {
-    if (shuttingDown) return
-    try {
-      await Promise.all([
-        fs.promises.open(liveFilePath, 'a').then((f) => f.close()),
-        fs.promises.open(readyFilePath, 'a').then((f) => f.close()),
-      ])
-    } catch (err) {
-      log.warn({ err }, 'Failed to write health probe files')
-    }
-  }, 5000)
-
   log.info(
     { tokens: config.tokens.length, pageSize: config.pageSize, batchSize: config.batchSize },
     'Starting enrichment loop',
   )
 
   await runEnrichmentLoop(qx, config, () => shuttingDown)
 
-  clearInterval(healthInterval)
   log.info('github-repos-enricher stopped.')
   process.exit(0)
 }

services/apps/packages_worker/src/config.ts

@@ -24,10 +24,8 @@ export function getEnricherConfig() {
 
   return {
     tokens,
-    pageSize: requireEnvInt('PAGE_SIZE'),
-    batchSize: requireEnvInt('BATCH_SIZE'),
-    maxRetries: requireEnvInt('MAX_RETRIES'),
-    updateIntervalHours: requireEnvInt('UPDATE_INTERVAL_HOURS'),
-    idleSleepSec: requireEnvInt('IDLE_SLEEP_SEC'),
+    batchSize: requireEnvInt('ENRICHER_BATCH_SIZE'),
+    updateIntervalHours: requireEnvInt('ENRICHER_REPO_UPDATE_INTERVAL_HOURS'),
+    idleSleepSec: requireEnvInt('ENRICHER_IDLE_SLEEP_SEC'),
   }
 }

services/apps/packages_worker/src/enricher/runEnrichmentLoop.ts

@@ -8,12 +8,13 @@ import { updateEnrichedRepos } from './updateEnrichedRepos'
 
 const log = getServiceChildLogger('github-repos-enricher')
 
+const MAX_RETRIES = 3
+
 async function fetchWithRetries(
   url: string,
   token: string,
-  maxRetries: number,
 ): Promise<LightRepoResult | null> {
-  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+  for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
     try {
       return await fetchLightRepo(url, token)
     } catch (err) {
@@ -26,12 +27,12 @@ async function fetchWithRetries(
 
       if (err.kind === 'RATE_LIMIT') throw err
 
-      if (attempt < maxRetries) {
+      if (attempt < MAX_RETRIES) {
         const backoffMs = 1000 * 2 ** attempt
         log.warn({ url, attempt, backoffMs }, `Transient error, retrying: ${err.message}`)
         await new Promise((r) => setTimeout(r, backoffMs))
       } else {
-        log.error({ url }, `Gave up after ${maxRetries} retries: ${err.message}`)
+        log.error({ url }, `Gave up after ${MAX_RETRIES} retries: ${err.message}`)
         return null
       }
     }
@@ -42,7 +43,7 @@ async function fetchWithRetries(
 async function fetchPage(
   qx: QueryExecutor,
   cursor: string | null,
-  pageSize: number,
+  batchSize: number,
   updateIntervalHours: number,
 ): Promise<{ rows: Array<{ id: string; url: string }>; urls: string[] }> {
   const rows = await qx.select(
@@ -53,9 +54,9 @@ async function fetchPage(
       AND (last_synced_at IS NULL OR last_synced_at < NOW() - INTERVAL '$(updateIntervalHours) hours')
       AND ($(cursor) IS NULL OR id > $(cursor))
     ORDER BY id
-    LIMIT $(pageSize)
+    LIMIT $(batchSize)
     `,
-    { cursor, pageSize, updateIntervalHours },
+    { cursor, batchSize, updateIntervalHours },
   )
   return {
     rows,
@@ -103,7 +104,7 @@ async function processPage(
         const url = validUrls[idx]
 
         try {
-          const result = await fetchWithRetries(url, token, config.maxRetries)
+          const result = await fetchWithRetries(url, token)
           if (result) {
             buffer.push(result)
             if (buffer.length >= config.batchSize) {
@@ -161,7 +162,7 @@ export async function runEnrichmentLoop(
   while (!isShuttingDown()) {
     pageNum++
 
-    const { rows, urls } = await fetchPage(qx, cursor, config.pageSize, config.updateIntervalHours)
+    const { rows, urls } = await fetchPage(qx, cursor, config.batchSize, config.updateIntervalHours)
 
     if (urls.length === 0) {
       log.info('No more repos to process, sleeping')