Merge branch 'fix/osspckgs-monitor' into feat/add-rubygems-ingest-CM-1296

Author: themarolt

backend/src/database/repositories/memberRepository.ts

@@ -244,6 +244,10 @@ class MemberRepository {
 
     const subprojectIds = await getSegmentSubprojectIds(qx, currentSegments)
 
+    if (subprojectIds.length === 0) {
+      return
+    }
+
     await seq.query(bulkDeleteMemberSegments, {
       replacements: {
         memberIds,

backend/src/database/repositories/organizationRepository.ts

@@ -192,6 +192,10 @@ class OrganizationRepository {
     const qx = SequelizeRepository.getQueryExecutor(options)
     const subprojectIds = await getSegmentSubprojectIds(qx, currentSegments)
 
+    if (subprojectIds.length === 0) {
+      return
+    }
+
     await seq.query(bulkDeleteOrganizationSegments, {
       replacements: {
         organizationIds,

backend/src/osspckgs/migrations/V1782978597__package_maintainers_ingestion_source.sql

@@ -0,0 +1,2 @@
+ALTER TABLE package_maintainers
+    ADD COLUMN IF NOT EXISTS ingestion_source text;

backend/src/osspckgs/migrations/V1783123200__sonatype_popularity.sql

@@ -0,0 +1,12 @@
+-- Sonatype popularity signal (first delivery: Maven P0, top 500 components).
+-- Sonatype could not provide raw Maven download counts, so instead they deliver a
+-- popularity score derived from their SCA telemetry: normalized 0-100 per ecosystem,
+-- tiered P0-P3. Stored in dedicated columns (not JSONB) because score/rank/tier are
+-- queried independently. Only sonatype_popularity_score feeds rank_packages(); rank and
+-- tier are kept for display/audit. No index on first pass (dataset is small, 500-2k rows).
+ALTER TABLE packages
+    ADD COLUMN IF NOT EXISTS sonatype_popularity_score numeric,      -- 0-100, normalized per ecosystem; consumed by rank_packages()
+    ADD COLUMN IF NOT EXISTS sonatype_rank              int,         -- rank within the Sonatype list; display/audit only
+    ADD COLUMN IF NOT EXISTS sonatype_tier              text,        -- 'P0' | 'P1' | 'P2' | 'P3'
+    ADD COLUMN IF NOT EXISTS sonatype_snapshot_at       timestamptz, -- timestamp of the Sonatype snapshot the row came from
+    ADD COLUMN IF NOT EXISTS sonatype_updated_at        timestamptz; -- our last upsert of the sonatype_* fields

services/apps/git_integration/src/crowdgit/services/clone/clone_service.py

@@ -47,7 +47,7 @@
 
 retry_on_clone_error = retry(
     retry=retry_if_exception_type(RETRYABLE_CLONE_ERRORS),
-    stop=stop_after_attempt(6),
+    stop=stop_after_attempt(3),
     wait=wait_exponential(multiplier=5, min=5, max=120),
     reraise=True,
 )
@@ -190,9 +190,14 @@ async def _perform_minimal_clone(self, path: str, remote: str) -> None:
         Perform minimal clone of depth=1
         """
         self.logger.info("Initializing minimal clone")
-        await run_shell_command(
-            ["git", "clone", "--depth=1", "--no-tags", "--single-branch", remote, "."], cwd=path
-        )
+        base_cmd = ["git", "clone", "--depth=1", "--no-tags", "--single-branch"]
+        try:
+            await run_shell_command([*base_cmd, f"{remote}.git", "."], cwd=path)
+        except Exception:
+            self.logger.warning(
+                f"Clone with .git suffix failed, falling back to bare URL: {remote}"
+            )
+            await run_shell_command([*base_cmd, remote, "."], cwd=path)
         self.logger.info("Minimal clone initialized successfully")
 
     async def _get_repo_size_mb(self, repo_path: str) -> float:
@@ -377,9 +382,14 @@ async def _cleanup_working_directory(self, repo_path: str) -> None:
     async def _perform_full_clone(self, repo_path: str, remote: str):
         """Perform full repository clone"""
         self.logger.info(f"Performing full clone for repo {remote}...")
-        await run_shell_command(
-            ["git", "clone", "--no-tags", "--single-branch", remote, "."], cwd=repo_path
-        )
+        base_cmd = ["git", "clone", "--no-tags", "--single-branch"]
+        try:
+            await run_shell_command([*base_cmd, f"{remote}.git", "."], cwd=repo_path)
+        except Exception:
+            self.logger.warning(
+                f"Clone with .git suffix failed, falling back to bare URL: {remote}"
+            )
+            await run_shell_command([*base_cmd, remote, "."], cwd=repo_path)
         self.logger.info(f"Successfully completed full clone of repository: {remote}")
 
     async def has_default_branch_changed(self, remote: str, saved_branch: str | None) -> bool:

services/apps/packages_worker/package.json

@@ -51,6 +51,10 @@
     "dev:nuget-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9242 src/bin/nuget-worker.ts",
     "backfill:maven": "SERVICE=maven tsx src/bin/maven-backfill.ts",
     "backfill:maven:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven LOG_LEVEL=info tsx src/bin/maven-backfill.ts",
+    "import:maven-maintainers": "SERVICE=maven tsx src/maven/scripts/importMaintainersFromCsv.ts",
+    "import:maven-maintainers:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven tsx src/maven/scripts/importMaintainersFromCsv.ts",
+    "import:sonatype-popularity": "SERVICE=maven tsx src/maven/scripts/importSonatypePopularityFromCsv.ts",
+    "import:sonatype-popularity:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven tsx src/maven/scripts/importSonatypePopularityFromCsv.ts",
     "backfill:stewardship": "SERVICE=stewardship-backfill tsx src/bin/stewardship-backfill.ts",
     "backfill:stewardship:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=stewardship-backfill LOG_LEVEL=info tsx src/bin/stewardship-backfill.ts",
     "monitor:osspckgs": "SERVICE=bq-dataset-ingest tsx src/scripts/monitorOsspckgs.ts",

services/apps/packages_worker/src/maven/scripts/csv.ts

@@ -0,0 +1,56 @@
+export function parseCsv(content: string): Record<string, string>[] {
+  const rows: string[][] = []
+  let cur = ''
+  let inQuote = false
+  let row: string[] = []
+
+  for (let i = 0; i < content.length; i++) {
+    const ch = content[i]
+    const next = content[i + 1]
+
+    if (inQuote) {
+      if (ch === '"' && next === '"') {
+        cur += '"'
+        i++
+      } else if (ch === '"') {
+        inQuote = false
+      } else {
+        cur += ch
+      }
+    } else {
+      if (ch === '"') {
+        inQuote = true
+      } else if (ch === ',') {
+        row.push(cur)
+        cur = ''
+      } else if (ch === '\r' && next === '\n') {
+        row.push(cur)
+        cur = ''
+        rows.push(row)
+        row = []
+        i++
+      } else if (ch === '\n') {
+        row.push(cur)
+        cur = ''
+        rows.push(row)
+        row = []
+      } else {
+        cur += ch
+      }
+    }
+  }
+  if (cur || row.length) {
+    row.push(cur)
+    rows.push(row)
+  }
+
+  if (rows.length === 0) return []
+  const headers = rows[0]
+  return rows.slice(1).map((r) => {
+    const obj: Record<string, string> = {}
+    headers.forEach((h, i) => {
+      obj[h.trim()] = (r[i] ?? '').trim()
+    })
+    return obj
+  })
+}