|
| 1 | +-- ──────────────────────────────────────────────────────────────────────────── |
| 2 | +-- Maven enrichment — data quality scorecard |
| 3 | +-- |
| 4 | +-- One read-only statement. Returns one row per check so it works in any SQL |
| 5 | +-- client (psql, DBeaver, prod read-replica) and via validateDataQuality.ts. |
| 6 | +-- |
| 7 | +-- Scope: the CRITICAL Maven set (packages_universe.is_critical = true) — i.e. the |
| 8 | +-- set the POM fetcher is responsible for. Coverage % is over the whole critical |
| 9 | +-- universe, so packages not yet enriched count as "not covered". |
| 10 | +-- |
| 11 | +-- Columns: |
| 12 | +-- section | metric | value | pct (of critical) | status |
| 13 | +-- status: OK / LOW / POOR for coverage; OK / FAIL for anomalies & integrity. |
| 14 | +-- Coverage thresholds (80% / 50%) are deliberately simple — tune as needed. |
| 15 | +-- ──────────────────────────────────────────────────────────────────────────── |
| 16 | + |
| 17 | +WITH crit AS ( |
| 18 | + -- denominator: the critical universe |
| 19 | + SELECT pu.purl |
| 20 | + FROM packages_universe pu |
| 21 | + WHERE pu.ecosystem = 'maven' |
| 22 | + AND pu.is_critical = true |
| 23 | + AND pu.purl IS NOT NULL |
| 24 | +), |
| 25 | +crit_n AS (SELECT count(*)::numeric AS n FROM crit), |
| 26 | + |
| 27 | +-- per-package version aggregates (only for packages we care about) |
| 28 | +ver AS ( |
| 29 | + SELECT |
| 30 | + v.package_id, |
| 31 | + count(*) AS versions_count, |
| 32 | + count(*) FILTER (WHERE v.is_latest) AS latest_count, |
| 33 | + count(*) FILTER (WHERE v.is_latest AND v.is_prerelease) AS latest_prerelease_count, |
| 34 | + max(v.number) FILTER (WHERE v.is_latest) AS latest_number |
| 35 | + FROM versions v |
| 36 | + GROUP BY v.package_id |
| 37 | +), |
| 38 | + |
| 39 | +-- enriched critical packages with derived quality flags |
| 40 | +pkg AS ( |
| 41 | + SELECT |
| 42 | + pk.id, |
| 43 | + pk.ingestion_source, |
| 44 | + pk.repository_url, |
| 45 | + pk.latest_version, |
| 46 | + pk.last_synced_at, |
| 47 | + (pk.repository_url IS NOT NULL) AS has_repo, |
| 48 | + (pk.licenses IS NOT NULL AND array_length(pk.licenses, 1) >= 1) AS has_license, |
| 49 | + (pk.description IS NOT NULL AND length(btrim(pk.description)) > 0) AS has_description, |
| 50 | + (pk.homepage IS NOT NULL) AS has_homepage, |
| 51 | + COALESCE(ver.versions_count, 0) AS versions_count, |
| 52 | + COALESCE(ver.latest_count, 0) AS latest_count, |
| 53 | + COALESCE(ver.latest_prerelease_count, 0) AS latest_prerelease_count, |
| 54 | + ver.latest_number, |
| 55 | + (pm.package_id IS NOT NULL) AS has_maintainer |
| 56 | + FROM packages pk |
| 57 | + JOIN crit c ON c.purl = pk.purl |
| 58 | + LEFT JOIN ver ON ver.package_id = pk.id |
| 59 | + LEFT JOIN (SELECT DISTINCT package_id FROM package_maintainers) pm ON pm.package_id = pk.id |
| 60 | +), |
| 61 | + |
| 62 | +-- all critical-scoped counts in a single row |
| 63 | +agg AS ( |
| 64 | + SELECT |
| 65 | + (SELECT n FROM crit_n) AS critical_total, |
| 66 | + count(*) AS enriched, |
| 67 | + count(*) FILTER (WHERE ingestion_source = 'maven-registry') AS src_registry, |
| 68 | + count(*) FILTER (WHERE ingestion_source = 'maven_not_on_central') AS src_not_central, |
| 69 | + count(*) FILTER (WHERE ingestion_source = 'maven_no_version') AS src_no_version, |
| 70 | + count(*) FILTER (WHERE ingestion_source = 'maven_error') AS src_error, |
| 71 | + count(*) FILTER (WHERE ingestion_source = 'packages_universe') AS src_universe, |
| 72 | + count(*) FILTER (WHERE has_repo) AS has_repo, |
| 73 | + count(*) FILTER (WHERE has_license) AS has_license, |
| 74 | + count(*) FILTER (WHERE versions_count > 0) AS has_versions, |
| 75 | + count(*) FILTER (WHERE has_maintainer) AS has_maintainer, |
| 76 | + count(*) FILTER (WHERE has_description) AS has_description, |
| 77 | + count(*) FILTER (WHERE has_homepage) AS has_homepage, |
| 78 | + count(*) FILTER (WHERE latest_count > 1) AS multi_latest, |
| 79 | + count(*) FILTER (WHERE versions_count > 0 AND latest_count = 0) AS no_latest, |
| 80 | + count(*) FILTER (WHERE latest_count = 1 |
| 81 | + AND latest_number IS DISTINCT FROM latest_version) AS latest_mismatch, |
| 82 | + count(*) FILTER (WHERE latest_prerelease_count > 0) AS prerelease_latest, |
| 83 | + count(*) FILTER (WHERE has_repo AND repository_url !~ '^https?://') AS repo_not_http, |
| 84 | + count(*) FILTER (WHERE last_synced_at >= now() - interval '24 hours') AS synced_24h, |
| 85 | + count(*) FILTER (WHERE last_synced_at < now() - interval '24 hours' |
| 86 | + AND last_synced_at >= now() - interval '7 days') AS synced_week, |
| 87 | + count(*) FILTER (WHERE last_synced_at < now() - interval '7 days') AS synced_old |
| 88 | + FROM pkg |
| 89 | +), |
| 90 | + |
| 91 | +-- global integrity (not limited to the critical set) |
| 92 | +integ AS ( |
| 93 | + SELECT |
| 94 | + (SELECT count(*) FROM (SELECT purl FROM packages GROUP BY purl HAVING count(*) > 1) d) AS dup_purl, |
| 95 | + (SELECT count(*) FROM versions v LEFT JOIN packages p ON p.id = v.package_id WHERE p.id IS NULL) AS orphan_versions, |
| 96 | + (SELECT count(*) FROM package_repos pr LEFT JOIN packages p ON p.id = pr.package_id WHERE p.id IS NULL) AS orphan_repos, |
| 97 | + (SELECT count(*) FROM package_maintainers pm LEFT JOIN packages p ON p.id = pm.package_id WHERE p.id IS NULL) AS orphan_maintainers, |
| 98 | + (SELECT count(*) FROM maintainers WHERE username IS NULL OR btrim(username) = '') AS maintainer_no_username |
| 99 | +), |
| 100 | + |
| 101 | +-- coverage % + status helper |
| 102 | +cov AS ( |
| 103 | + SELECT |
| 104 | + metric, value, |
| 105 | + round(100.0 * value / nullif((SELECT critical_total FROM agg), 0), 1) AS pct |
| 106 | + FROM (VALUES |
| 107 | + ('enriched (row exists)', (SELECT enriched FROM agg)), |
| 108 | + ('ingestion=maven-registry', (SELECT src_registry FROM agg)), |
| 109 | + ('has repository_url', (SELECT has_repo FROM agg)), |
| 110 | + ('has license', (SELECT has_license FROM agg)), |
| 111 | + ('has versions', (SELECT has_versions FROM agg)), |
| 112 | + ('has maintainer', (SELECT has_maintainer FROM agg)), |
| 113 | + ('has description', (SELECT has_description FROM agg)), |
| 114 | + ('has homepage', (SELECT has_homepage FROM agg)) |
| 115 | + ) AS t(metric, value) |
| 116 | +) |
| 117 | + |
| 118 | +-- ─── report ─────────────────────────────────────────────────────────────────── |
| 119 | +SELECT ord, section, metric, value, pct_txt AS pct, status FROM ( |
| 120 | + -- totals |
| 121 | + SELECT 0 AS ord, '1. TOTALS' AS section, 'critical packages (universe)' AS metric, |
| 122 | + (SELECT critical_total FROM agg) AS value, '100.0%' AS pct_txt, 'INFO' AS status |
| 123 | + UNION ALL |
| 124 | + SELECT 1, '1. TOTALS', 'never enriched (no packages row)', |
| 125 | + (SELECT critical_total - enriched FROM agg), |
| 126 | + to_char(round(100.0 * (SELECT critical_total - enriched FROM agg) |
| 127 | + / nullif((SELECT critical_total FROM agg), 0), 1), 'FM990.0') || '%', |
| 128 | + CASE WHEN (SELECT critical_total - enriched FROM agg) = 0 THEN 'OK' ELSE 'INFO' END |
| 129 | + |
| 130 | + -- coverage |
| 131 | + UNION ALL |
| 132 | + SELECT 10 + row_number() OVER (ORDER BY metric), '2. COVERAGE', metric, value, |
| 133 | + to_char(pct, 'FM990.0') || '%', |
| 134 | + CASE WHEN pct >= 80 THEN 'OK' WHEN pct >= 50 THEN 'LOW' ELSE 'POOR' END |
| 135 | + FROM cov |
| 136 | + |
| 137 | + -- ingestion_source breakdown (enriched rows) |
| 138 | + UNION ALL SELECT 30, '3. INGESTION SOURCE', 'maven-registry (full POM ok)', (SELECT src_registry FROM agg), NULL, 'INFO' |
| 139 | + UNION ALL SELECT 31, '3. INGESTION SOURCE', 'maven_not_on_central', (SELECT src_not_central FROM agg), NULL, 'INFO' |
| 140 | + UNION ALL SELECT 32, '3. INGESTION SOURCE', 'maven_no_version', (SELECT src_no_version FROM agg), NULL, 'INFO' |
| 141 | + UNION ALL SELECT 33, '3. INGESTION SOURCE', 'maven_error', (SELECT src_error FROM agg), NULL, |
| 142 | + CASE WHEN (SELECT src_error FROM agg) = 0 THEN 'OK' ELSE 'WARN' END |
| 143 | + UNION ALL SELECT 34, '3. INGESTION SOURCE', 'packages_universe (unexpected on critical)', (SELECT src_universe FROM agg), NULL, |
| 144 | + CASE WHEN (SELECT src_universe FROM agg) = 0 THEN 'OK' ELSE 'WARN' END |
| 145 | + |
| 146 | + -- anomalies (expect 0) |
| 147 | + UNION ALL SELECT 40, '4. ANOMALIES (expect 0)', 'packages with >1 is_latest version', (SELECT multi_latest FROM agg), NULL, CASE WHEN (SELECT multi_latest FROM agg)=0 THEN 'OK' ELSE 'FAIL' END |
| 148 | + UNION ALL SELECT 41, '4. ANOMALIES (expect 0)', 'has versions but no is_latest', (SELECT no_latest FROM agg), NULL, CASE WHEN (SELECT no_latest FROM agg)=0 THEN 'OK' ELSE 'FAIL' END |
| 149 | + UNION ALL SELECT 42, '4. ANOMALIES (expect 0)', 'latest_version != is_latest version', (SELECT latest_mismatch FROM agg), NULL, CASE WHEN (SELECT latest_mismatch FROM agg)=0 THEN 'OK' ELSE 'FAIL' END |
| 150 | + UNION ALL SELECT 43, '4. ANOMALIES (expect 0)', 'prerelease flagged as latest', (SELECT prerelease_latest FROM agg), NULL, CASE WHEN (SELECT prerelease_latest FROM agg)=0 THEN 'OK' ELSE 'WARN' END |
| 151 | + UNION ALL SELECT 44, '4. ANOMALIES (expect 0)', 'repository_url not http(s)', (SELECT repo_not_http FROM agg), NULL, CASE WHEN (SELECT repo_not_http FROM agg)=0 THEN 'OK' ELSE 'WARN' END |
| 152 | + |
| 153 | + -- global integrity (expect 0) |
| 154 | + UNION ALL SELECT 50, '5. INTEGRITY (expect 0)', 'duplicate purls in packages', (SELECT dup_purl FROM integ), NULL, CASE WHEN (SELECT dup_purl FROM integ)=0 THEN 'OK' ELSE 'FAIL' END |
| 155 | + UNION ALL SELECT 51, '5. INTEGRITY (expect 0)', 'orphan versions', (SELECT orphan_versions FROM integ), NULL, CASE WHEN (SELECT orphan_versions FROM integ)=0 THEN 'OK' ELSE 'FAIL' END |
| 156 | + UNION ALL SELECT 52, '5. INTEGRITY (expect 0)', 'orphan package_repos', (SELECT orphan_repos FROM integ), NULL, CASE WHEN (SELECT orphan_repos FROM integ)=0 THEN 'OK' ELSE 'FAIL' END |
| 157 | + UNION ALL SELECT 53, '5. INTEGRITY (expect 0)', 'orphan package_maintainers', (SELECT orphan_maintainers FROM integ), NULL, CASE WHEN (SELECT orphan_maintainers FROM integ)=0 THEN 'OK' ELSE 'FAIL' END |
| 158 | + UNION ALL SELECT 54, '5. INTEGRITY (expect 0)', 'maintainers without username', (SELECT maintainer_no_username FROM integ), NULL, CASE WHEN (SELECT maintainer_no_username FROM integ)=0 THEN 'OK' ELSE 'FAIL' END |
| 159 | + |
| 160 | + -- freshness (enriched critical) |
| 161 | + UNION ALL SELECT 60, '6. FRESHNESS', 'synced in last 24h', (SELECT synced_24h FROM agg), NULL, 'INFO' |
| 162 | + UNION ALL SELECT 61, '6. FRESHNESS', 'synced 24h–7d ago', (SELECT synced_week FROM agg), NULL, 'INFO' |
| 163 | + UNION ALL SELECT 62, '6. FRESHNESS', 'synced > 7d ago (stale)', (SELECT synced_old FROM agg), NULL, |
| 164 | + CASE WHEN (SELECT synced_old FROM agg) = 0 THEN 'OK' ELSE 'WARN' END |
| 165 | +) report |
| 166 | +ORDER BY ord; |
0 commit comments