Skip to content

Commit fdd2765

Browse files
mbani01mbani01
committed
feat: replace rank_packages() with cumulative-coverage impact scoring
Signed-off-by: Mouad BANI <mouad-mb@outlook.com>
1 parent e5b2d7b commit fdd2765

2 files changed

Lines changed: 131 additions & 22 deletions

File tree

‎backend/src/osspckgs/migrations/V1781262276__rank_packages_cumulative_coverage.sql‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
-- Two related changes bundled here:
2+
--
3+
-- 1. package_criticality_spotlight: replace text-based (ecosystem, namespace, name)
4+
-- matching with a package_id FK so the spotlight join uses an indexed integer key.
5+
--
6+
-- 2. rank_packages(): replace the weighted PERCENT_RANK formula and arbitrary
7+
-- per-ecosystem top-N caps with cumulative-coverage scoring.
8+
-- The smallest set of packages that together account for coverage_cutoff (default 90%)
9+
-- of each signal is critical. impact = average of (1 − cumulative_coverage) across
10+
-- available signals. is_critical = true if in that set for any signal.
11+
12+
-- ── 1. package_criticality_spotlight schema change ───────────────────────────
13+
14+
ALTER TABLE package_criticality_spotlight
15+
ADD COLUMN package_id bigint NOT NULL REFERENCES packages(id),
16+
DROP COLUMN name,
17+
DROP COLUMN namespace;
18+
19+
CREATE UNIQUE INDEX ON package_criticality_spotlight (package_id);
20+
21+
-- ── 2. rank_packages() ───────────────────────────────────────────────────────
22+
23+
DROP FUNCTION IF EXISTS rank_packages(numeric, numeric, numeric, jsonb);
24+
25+
CREATE OR REPLACE FUNCTION rank_packages(
26+
coverage_cutoff numeric DEFAULT 0.90,
27+
ecosystems text[] DEFAULT NULL
28+
)
29+
RETURNS TABLE(processed_rows int)
30+
LANGUAGE plpgsql AS $$
31+
DECLARE
32+
processed_count int;
33+
effective_ecosystems text[];
34+
BEGIN
35+
IF ecosystems IS NULL THEN
36+
SELECT ARRAY_AGG(DISTINCT ecosystem)
37+
INTO effective_ecosystems
38+
FROM packages;
39+
ELSE
40+
effective_ecosystems := ecosystems;
41+
END IF;
42+
43+
WITH base AS (
44+
SELECT
45+
id,
46+
ecosystem,
47+
COALESCE(downloads_last_30d, 0) AS downloads,
48+
COALESCE(dependent_count, 0) AS direct_dependents,
49+
COALESCE(transitive_dependent_count, 0) AS transitive_dependents,
50+
SUM(COALESCE(downloads_last_30d, 0)) OVER (PARTITION BY ecosystem) AS ecosystem_total_downloads,
51+
SUM(COALESCE(dependent_count, 0)) OVER (PARTITION BY ecosystem) AS ecosystem_total_direct_dependents,
52+
SUM(COALESCE(transitive_dependent_count, 0)) OVER (PARTITION BY ecosystem) AS ecosystem_total_transitive_dependents
53+
FROM packages
54+
WHERE ecosystem = ANY(effective_ecosystems)
55+
),
56+
walked AS (
57+
-- One row per (package × signal). Signals with a zero ecosystem total are
58+
-- excluded so they don't factor into the average (e.g. downloads for maven).
59+
-- cumulative_share_exclusive for the top-ranked package equals 0 by arithmetic
60+
-- (sum of rows above it is 0), so the top package is always inside the critical set.
61+
SELECT
62+
id,
63+
ecosystem,
64+
SUM(signal_value) OVER coverage_window / ecosystem_signal_total::numeric AS cumulative_share_inclusive,
65+
(SUM(signal_value) OVER coverage_window - signal_value) / ecosystem_signal_total::numeric AS cumulative_share_exclusive
66+
FROM base
67+
CROSS JOIN LATERAL (VALUES
68+
('downloads', downloads, ecosystem_total_downloads),
69+
('direct_dependents', direct_dependents, ecosystem_total_direct_dependents),
70+
('transitive_dependents', transitive_dependents, ecosystem_total_transitive_dependents)
71+
) AS signal(signal_name, signal_value, ecosystem_signal_total)
72+
WHERE ecosystem_signal_total > 0
73+
WINDOW coverage_window AS (
74+
PARTITION BY ecosystem, signal_name
75+
ORDER BY signal_value DESC, id
76+
ROWS UNBOUNDED PRECEDING
77+
)
78+
),
79+
combined AS (
80+
SELECT
81+
id,
82+
ecosystem,
83+
AVG(
84+
CASE WHEN cumulative_share_exclusive < coverage_cutoff
85+
THEN 1.0 - cumulative_share_inclusive
86+
ELSE 0.0
87+
END
88+
)::numeric(10, 4) AS new_impact,
89+
BOOL_OR(cumulative_share_exclusive < coverage_cutoff) AS new_is_critical
90+
FROM walked
91+
GROUP BY id, ecosystem
92+
),
93+
final AS (
94+
SELECT
95+
combined.id,
96+
combined.new_impact,
97+
combined.new_is_critical OR (spotlight.package_id IS NOT NULL) AS new_is_critical,
98+
ROW_NUMBER() OVER (
99+
PARTITION BY combined.ecosystem
100+
ORDER BY combined.new_impact DESC NULLS LAST, combined.id
101+
) AS new_rank_in_ecosystem
102+
FROM combined
103+
LEFT JOIN package_criticality_spotlight spotlight ON spotlight.package_id = combined.id
104+
)
105+
UPDATE packages p
106+
SET impact = final.new_impact,
107+
is_critical = final.new_is_critical,
108+
rank_in_ecosystem = final.new_rank_in_ecosystem,
109+
last_rank_pass_at = NOW(),
110+
last_synced_at = NOW()
111+
FROM final
112+
WHERE p.id = final.id;
113+
114+
GET DIAGNOSTICS processed_count = ROW_COUNT;
115+
116+
RETURN QUERY SELECT processed_count;
117+
END;
118+
$$;

‎services/apps/packages_worker/src/criticality/run-impact.ts‎

Lines changed: 13 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,12 @@
55
*
66
* Usage (from services/apps/packages_worker):
77
* pnpm run:impact
8-
* pnpm run:impact --top-n '{"cargo":75000,"maven":150000}'
9-
* pnpm run:impact --w-downloads 0.30 --w-dep-pkgs 0.20 --w-transitive 0.50
8+
* pnpm run:impact --cutoff 0.90
9+
* pnpm run:impact --ecosystems npm,go
10+
* pnpm run:impact --cutoff 0.85 --ecosystems npm
1011
*/
1112
import { getPackagesDb } from '../db'
1213

13-
// Env defaults for local dev
1414
process.env.CROWD_PACKAGES_DB_WRITE_HOST ??= 'localhost'
1515
process.env.CROWD_PACKAGES_DB_PORT ??= '5434'
1616
process.env.CROWD_PACKAGES_DB_DATABASE ??= 'packages-db'
@@ -24,40 +24,31 @@ function parseArg(flag: string, fallback: number): number {
2424
return Number.isNaN(v) ? fallback : v
2525
}
2626

27-
function parseJsonArg(flag: string, fallback: string): string {
27+
function parseListArg(flag: string): string[] | null {
2828
const idx = process.argv.indexOf(flag)
29-
return idx !== -1 ? process.argv[idx + 1] : fallback
29+
if (idx === -1) return null
30+
return process.argv[idx + 1].split(',').map((s) => s.trim())
3031
}
3132

32-
const wDownloads = parseArg('--w-downloads', 0.25)
33-
const wDepPkgs = parseArg('--w-dep-pkgs', 0.25)
34-
const wTransitive = parseArg('--w-transitive', 0.5)
35-
const topN = parseJsonArg('--top-n', '{"cargo":75000,"maven":150000}')
33+
const cutoff = parseArg('--cutoff', 0.9)
34+
const ecosystems = parseListArg('--ecosystems')
3635

3736
async function main() {
38-
if (Math.abs(wDownloads + wDepPkgs + wTransitive - 1.0) > 0.001) {
39-
console.error(
40-
`Weights must sum to 1.0, got ${(wDownloads + wDepPkgs + wTransitive).toFixed(3)}`,
41-
)
42-
process.exit(1)
43-
}
44-
4537
console.log(`Running rank_packages()`)
46-
console.log(` weights: downloads=${wDownloads} dep_pkgs=${wDepPkgs} transitive=${wTransitive}`)
47-
console.log(` top-n : ${topN}\n`)
38+
console.log(` cutoff : ${cutoff}`)
39+
console.log(` ecosystems: ${ecosystems ? ecosystems.join(', ') : 'all'}\n`)
4840

4941
const qx = await getPackagesDb()
5042
const t = Date.now()
5143

5244
const [result] = await qx.select(
53-
`SELECT * FROM rank_packages($/wDownloads/, $/wDepPkgs/, $/wTransitive/, $/topN/::jsonb)`,
54-
{ wDownloads, wDepPkgs, wTransitive, topN },
45+
`SELECT * FROM rank_packages($/cutoff/, $/ecosystems/)`,
46+
{ cutoff, ecosystems },
5547
)
5648

5749
const elapsed = ((Date.now() - t) / 1000).toFixed(1)
5850
console.log(`Done in ${elapsed}s`)
59-
console.log(` scored_rows: ${result.scored_rows?.toLocaleString()}`)
60-
console.log(` ranked_rows: ${result.ranked_rows?.toLocaleString()}`)
51+
console.log(` processed_rows: ${result.processed_rows?.toLocaleString()}`)
6152

6253
process.exit(0)
6354
}

0 commit comments

Comments
 (0)