feat: repo known files tracking [CM-1329] (#4396)

mbani01 · web-flow · commit ad749810e3fb · 2026-07-27T12:32:28.000+01:00
Signed-off-by: Mouad BANI &lt;mouad-mb@outlook.com&gt;
diff --git a/backend/src/osspckgs/migrations/V1784905809__repo_well_known_files.sql b/backend/src/osspckgs/migrations/V1784905809__repo_well_known_files.sql
@@ -0,0 +1,22 @@
+-- Well-known files inventory per repo, populated by the github-repos-enricher.
+CREATE TABLE IF NOT EXISTS repo_well_known_files (
+    id bigserial PRIMARY KEY,
+    repo_id bigint NOT NULL REFERENCES repos (id) ON DELETE CASCADE,
+    file_type text NOT NULL, -- security | contributing | governance | maintainers | code_of_conduct | readme
+    directory text NOT NULL, -- root | .github | docs
+    path text NOT NULL,
+    blob_oid text NOT NULL,
+    first_seen_at timestamptz NOT NULL DEFAULT NOW(),
+    -- observation time, not commit time; bumped on content change, disappearance, reappearance
+    change_detected_at timestamptz NOT NULL DEFAULT NOW(),
+    checked_at timestamptz NOT NULL,
+    deleted_at timestamptz,
+    UNIQUE (repo_id, path)
+);
+
+CREATE INDEX IF NOT EXISTS repo_well_known_files_change_detected_at_idx
+    ON repo_well_known_files (change_detected_at);
+
+CREATE INDEX IF NOT EXISTS repo_well_known_files_file_type_idx
+    ON repo_well_known_files (file_type)
+    WHERE deleted_at IS NULL;
diff --git a/services/apps/packages_worker/src/enricher/fetchLightRepo.ts b/services/apps/packages_worker/src/enricher/fetchLightRepo.ts
@@ -1,6 +1,12 @@
 import { getServiceChildLogger } from '@crowd/logging'
 
 import { FetchError, LightRepoResult } from './types'
+import {
+  RepoTrees,
+  TreeEntryNode,
+  classifyWellKnownFiles,
+  deriveSecurityFileEnabled,
+} from './wellKnownFiles'
 
 const log = getServiceChildLogger('fetch-light-repo')
 
@@ -24,6 +30,15 @@ const REPO_QUERY = `
       createdAt
       isSecurityPolicyEnabled
       defaultBranchRef { name }
+      rootTree: object(expression: "HEAD:") {
+        ... on Tree { entries { name type oid } }
+      }
+      githubTree: object(expression: "HEAD:.github") {
+        ... on Tree { entries { name type oid } }
+      }
+      docsTree: object(expression: "HEAD:docs") {
+        ... on Tree { entries { name type oid } }
+      }
     }
   }
 `
@@ -34,67 +49,6 @@ export function parseGithubUrl(url: string): { owner: string; name: string } {
   return { owner: match[1], name: match[2] }
 }
 
-// community/profile API doesn't reliably return files.security — use Contents API instead.
-async function fetchSecurityFileEnabled(
-  url: string,
-  owner: string,
-  name: string,
-  token: string,
-  timeoutMs: number,
-): Promise<boolean | null> {
-  const headers = { Authorization: `bearer ${token}`, Accept: 'application/vnd.github+json' }
-  const check = async (path: string): Promise<boolean> => {
-    const controller = new AbortController()
-    const timeoutId = setTimeout(() => controller.abort(), timeoutMs)
-    try {
-      const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${name}/contents/${path}`, {
-        headers,
-        signal: controller.signal,
-      })
-      if (response.status === 200) return true
-      if (response.status === 404) return false
-      if (response.status === 403) {
-        const body = await response.text()
-        if (body.toLowerCase().includes('rate limit')) {
-          // REST secondary limits send retry-after; primary limits send x-ratelimit-reset
-          const retryAfterSec = parseInt(response.headers.get('retry-after') ?? '0', 10)
-          const resetSec = parseInt(response.headers.get('x-ratelimit-reset') ?? '0', 10)
-          const resetMs = retryAfterSec
-            ? Date.now() + retryAfterSec * 1000
-            : resetSec
-              ? resetSec * 1000 + 5_000
-              : Date.now() + 65_000
-          throw new FetchError('RATE_LIMIT', `Contents API rate limited on ${path}`, resetMs)
-        }
-      }
-      throw new Error(`Unexpected status ${response.status} for ${path}`)
-    } finally {
-      clearTimeout(timeoutId)
-    }
-  }
-
-  try {
-    const [root, dotGithub] = await Promise.all([
-      check('SECURITY.md'),
-      check('.github/SECURITY.md'),
-    ])
-    return root || dotGithub
-  } catch (err) {
-    // Rate limits propagate so the caller can park the installation and requeue the repo
-    if (err instanceof FetchError && err.kind === 'RATE_LIMIT') throw err
-    log.warn(
-      {
-        url,
-        errName: (err as Error).name,
-        errMsg: (err as Error).message,
-        errStack: (err as Error).stack,
-      },
-      'Security file check failed — securityFileEnabled will be null',
-    )
-    return null
-  }
-}
-
 interface BranchProtection {
   enabled: boolean | null
   requiredReviews: number | null
@@ -214,6 +168,9 @@ interface RepoGraphqlResponse {
       createdAt: string
       isSecurityPolicyEnabled: boolean
       defaultBranchRef: { name: string } | null
+      rootTree: { entries?: TreeEntryNode[] } | null
+      githubTree: { entries?: TreeEntryNode[] } | null
+      docsTree: { entries?: TreeEntryNode[] } | null
     } | null
   }
   errors?: Array<{ type?: string; message?: string }>
@@ -262,10 +219,7 @@ export async function fetchLightRepo(
   if (response.status === 404) throw new FetchError('NOT_FOUND', `404 for ${url}`)
   if (response.status >= 500) throw new FetchError('TRANSIENT', `${response.status} for ${url}`)
 
-  const [json, securityFileEnabled] = await Promise.all([
-    response.json() as Promise<RepoGraphqlResponse>,
-    fetchSecurityFileEnabled(url, owner, name, token, timeoutMs),
-  ])
+  const json = (await response.json()) as RepoGraphqlResponse
 
   if (json.errors?.length) {
     const err = json.errors[0]
@@ -280,6 +234,12 @@ export async function fetchLightRepo(
   const repo = json.data?.repository
   if (!repo) throw new FetchError('NOT_FOUND', `No repository data for ${url}`)
 
+  const trees: RepoTrees = {
+    root: repo.rootTree?.entries ?? null,
+    github: repo.githubTree?.entries ?? null,
+    docs: repo.docsTree?.entries ?? null,
+  }
+
   const defaultBranch = repo.defaultBranchRef?.name ?? null
   const branchProtection = defaultBranch
     ? await fetchBranchProtection(url, owner, name, defaultBranch, token, timeoutMs)
@@ -305,7 +265,8 @@ export async function fetchLightRepo(
     isFork: repo.isFork ?? null,
     createdAt: repo.createdAt ?? null,
     securityPolicyEnabled: repo.isSecurityPolicyEnabled ?? null,
-    securityFileEnabled,
+    securityFileEnabled: deriveSecurityFileEnabled(trees),
+    wellKnownFiles: classifyWellKnownFiles(trees),
     branchProtectionEnabled: branchProtection.enabled,
     branchProtectionRequiredReviews: branchProtection.requiredReviews,
     branchProtectionRequiresStatusChecks: branchProtection.requiresStatusChecks,
diff --git a/services/apps/packages_worker/src/enricher/runEnrichmentLoop.ts b/services/apps/packages_worker/src/enricher/runEnrichmentLoop.ts
@@ -7,9 +7,15 @@ import { fetchActivitySnapshot } from './fetchActivitySnapshot'
 import { fetchLightRepo, parseGithubUrl } from './fetchLightRepo'
 import { GithubAppConfig, getInstallationToken } from './githubAppAuth'
 import { InstallationPool } from './installationPool'
-import { FetchError, LightRepoResult, RepoActivitySnapshot } from './types'
+import {
+  FetchError,
+  LightRepoResult,
+  RepoActivitySnapshot,
+  RepoWellKnownFilesUpdate,
+} from './types'
 import { bulkUpdateEnrichedRepos, markReposSkipped } from './updateEnrichedRepos'
 import { bulkUpsertRepoActivitySnapshot } from './updateRepoActivitySnapshot'
+import { bulkUpsertRepoWellKnownFiles } from './updateWellKnownFiles'
 
 const log = getServiceChildLogger('github-repos-enricher')
 
@@ -67,6 +73,7 @@ class WriteBuffer {
   private results: LightRepoResult[] = []
   private snapshots: RepoActivitySnapshot[] = []
   private skipUrls: string[] = []
+  private wellKnownFiles: RepoWellKnownFilesUpdate[] = []
   private lastFlushAt = Date.now()
   private flushing = false
   private flushFailures = 0
@@ -81,6 +88,10 @@ class WriteBuffer {
     this.snapshots.push(snapshot)
   }
 
+  addWellKnownFiles(update: RepoWellKnownFilesUpdate): void {
+    this.wellKnownFiles.push(update)
+  }
+
   addSkip(url: string): void {
     this.skipUrls.push(url)
   }
@@ -94,22 +105,34 @@ class WriteBuffer {
     )
   }
 
-  private clearBatch(resultCount: number, snapshotCount: number, skipCount: number): void {
+  private clearBatch(
+    resultCount: number,
+    snapshotCount: number,
+    skipCount: number,
+    wellKnownFilesCount: number,
+  ): void {
     this.results.splice(0, resultCount)
     this.snapshots.splice(0, snapshotCount)
     this.skipUrls.splice(0, skipCount)
+    this.wellKnownFiles.splice(0, wellKnownFilesCount)
     this.flushFailures = 0
   }
 
   async flush(): Promise<number> {
     this.lastFlushAt = Date.now()
-    if (this.results.length === 0 && this.snapshots.length === 0 && this.skipUrls.length === 0) {
+    if (
+      this.results.length === 0 &&
+      this.snapshots.length === 0 &&
+      this.skipUrls.length === 0 &&
+      this.wellKnownFiles.length === 0
+    ) {
       return 0
     }
 
     const batch = [...this.results]
     const snapshotBatch = [...this.snapshots]
     const skips = [...this.skipUrls]
+    const wellKnownFilesBatch = [...this.wellKnownFiles]
     this.flushing = true
     try {
       // The snapshot upsert also updates repos rows — run in one transaction to avoid
@@ -118,8 +141,9 @@ class WriteBuffer {
         await bulkUpdateEnrichedRepos(tx, batch)
         await markReposSkipped(tx, skips)
         await bulkUpsertRepoActivitySnapshot(tx, snapshotBatch)
+        await bulkUpsertRepoWellKnownFiles(tx, wellKnownFilesBatch)
       })
-      this.clearBatch(batch.length, snapshotBatch.length, skips.length)
+      this.clearBatch(batch.length, snapshotBatch.length, skips.length, wellKnownFilesBatch.length)
       return batch.length
     } catch (err) {
       this.flushFailures++
@@ -139,7 +163,13 @@ class WriteBuffer {
           ? 'Flush failed repeatedly — dropping batch, repos will be re-enriched next sweep'
           : 'Flush failed — will retry on next cycle',
       )
-      if (dropBatch) this.clearBatch(batch.length, snapshotBatch.length, skips.length)
+      if (dropBatch)
+        this.clearBatch(
+          batch.length,
+          snapshotBatch.length,
+          skips.length,
+          wellKnownFilesBatch.length,
+        )
       return 0
     } finally {
       this.flushing = false
@@ -360,6 +390,11 @@ async function processRepo(row: RepoRow, ctx: WorkerContext): Promise<void> {
     if (outcome.kind === 'success') {
       metrics.totalFetched++
       writeBuffer.add(outcome.data)
+      writeBuffer.addWellKnownFiles({
+        repoId: row.id,
+        checkedAt: new Date().toISOString(),
+        files: outcome.data.wellKnownFiles,
+      })
       pool.parkIfBudgetLow(
         installationId,
         outcome.data.rateLimit?.remaining,
diff --git a/services/apps/packages_worker/src/enricher/types.ts b/services/apps/packages_worker/src/enricher/types.ts
@@ -1,3 +1,5 @@
+import { WellKnownFileEntry } from './wellKnownFiles'
+
 export interface LightRepoResult {
   url: string
   host: 'github'
@@ -17,6 +19,7 @@ export interface LightRepoResult {
   createdAt: string | null
   securityPolicyEnabled: boolean | null
   securityFileEnabled: boolean | null
+  wellKnownFiles: WellKnownFileEntry[]
   branchProtectionEnabled: boolean | null
   branchProtectionRequiredReviews: number | null
   branchProtectionRequiresStatusChecks: boolean | null
@@ -29,6 +32,12 @@ export interface LightRepoResult {
   } | null
 }
 
+export interface RepoWellKnownFilesUpdate {
+  repoId: string
+  checkedAt: string
+  files: WellKnownFileEntry[]
+}
+
 export interface RepoActivitySnapshot {
   repoId: string
   snapshotAt: string
diff --git a/services/apps/packages_worker/src/enricher/updateWellKnownFiles.ts b/services/apps/packages_worker/src/enricher/updateWellKnownFiles.ts
@@ -0,0 +1,63 @@
+import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor'
+
+import { RepoWellKnownFilesUpdate } from './types'
+
+export async function bulkUpsertRepoWellKnownFiles(
+  qx: QueryExecutor,
+  updates: RepoWellKnownFilesUpdate[],
+): Promise<void> {
+  if (updates.length === 0) return
+
+  // dedupe by repoId: ON CONFLICT DO UPDATE cannot affect the same row twice in one statement
+  const byRepo = new Map(updates.map((u) => [u.repoId, u]))
+  const batch = [...byRepo.values()]
+
+  await qx.result(
+    `
+    WITH input AS (
+      SELECT
+        (j->>'repoId')::bigint        AS repo_id,
+        (j->>'checkedAt')::timestamptz AS checked_at,
+        j->'files'                     AS files
+      FROM jsonb_array_elements($1::jsonb) j
+    ),
+    found AS (
+      SELECT
+        i.repo_id,
+        f->>'fileType'  AS file_type,
+        f->>'directory' AS directory,
+        f->>'path'      AS path,
+        f->>'blobOid'   AS blob_oid,
+        i.checked_at
+      FROM input i, jsonb_array_elements(i.files) f
+    ),
+    soft_deleted AS (
+      UPDATE repo_well_known_files w
+      SET deleted_at = i.checked_at,
+          change_detected_at = i.checked_at
+      FROM input i
+      WHERE w.repo_id = i.repo_id
+        AND w.deleted_at IS NULL
+        AND NOT EXISTS (
+          SELECT 1 FROM found f WHERE f.repo_id = w.repo_id AND f.path = w.path
+        )
+    )
+    INSERT INTO repo_well_known_files (repo_id, file_type, directory, path, blob_oid, checked_at, change_detected_at)
+    SELECT repo_id, file_type, directory, path, blob_oid, checked_at, checked_at
+    FROM found
+    ON CONFLICT (repo_id, path) DO UPDATE SET
+      blob_oid   = EXCLUDED.blob_oid,
+      file_type  = EXCLUDED.file_type,
+      directory  = EXCLUDED.directory,
+      checked_at = EXCLUDED.checked_at,
+      deleted_at = NULL,
+      change_detected_at = CASE
+        WHEN repo_well_known_files.blob_oid <> EXCLUDED.blob_oid
+          OR repo_well_known_files.deleted_at IS NOT NULL
+        THEN EXCLUDED.checked_at
+        ELSE repo_well_known_files.change_detected_at
+      END
+    `,
+    [JSON.stringify(batch)],
+  )
+}
diff --git a/services/apps/packages_worker/src/enricher/wellKnownFiles.ts b/services/apps/packages_worker/src/enricher/wellKnownFiles.ts
