fix: correct JS catalog to 4387 packages; revalidate all language scans

- JS catalog rebuilt from redirect_map tarball keys directly (no registry calls)
  — was 753 (broken), now 4387 with correct versions; 57 vulnerable / 169 CVEs
- fetch_ldpov_npm.py rewritten: extracts versions from S3 redirect_map keys,
  eliminating registry API calls that caused rate-limiting and silent failures
- fetch_ldpov_pypi.py: use S3 prefix listing for package discovery instead of
  root index URL (which returned 403)
- Python re-scanned: 1730 / 31 vuln / 99 CVEs (consistent)
- Java re-scanned: 5357 / 134 vuln / 337 CVEs (consistent)

Author: martinPavesio

public/data/ldpov/java/audit.json

@@ -1,8 +1,9 @@
 #!/usr/bin/env python3
 """Fetch all package names + versions from the LDPoV npm catalog.
 
-Reads LDPOV_ORG_ID, LDPOV_CATALOG_TOKEN, NPM_URL from ../.env or environment.
-Uses S3 redirect_map for package discovery, then fetches versions from the registry.
+Reads LDPOV_ORG_ID from ../.env or environment.
+Fetches the redirect_map from S3 and extracts package names + versions
+directly from the tarball keys — no registry calls needed.
 Writes public/data/ldpov/javascript/catalog.json.
 """
 
@@ -12,9 +13,7 @@
 import re
 import subprocess
 import urllib.parse
-from base64 import b64encode
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from urllib.request import Request, urlopen
+from collections import defaultdict
 
 _env: dict[str, str] = {}
 _env_path = os.path.join(os.path.dirname(__file__), "..", ".env")
@@ -25,76 +24,49 @@
             k, v = line.split("=", 1)
             _env[k.strip()] = v.strip()
 
-ORG_ID  = os.environ.get("LDPOV_ORG_ID",          _env.get("LDPOV_ORG_ID",          ""))
-TOKEN   = os.environ.get("LDPOV_CATALOG_TOKEN",    _env.get("LDPOV_CATALOG_TOKEN",    ""))
-BASE_URL = os.environ.get("NPM_URL",               _env.get("NPM_URL",               ""))
+ORG_ID = os.environ.get("LDPOV_ORG_ID", _env.get("LDPOV_ORG_ID", ""))
 
-if not ORG_ID or not TOKEN:
-    raise SystemExit("ERROR: set LDPOV_ORG_ID and LDPOV_CATALOG_TOKEN in .env or environment")
-if not BASE_URL:
-    raise SystemExit("ERROR: set NPM_URL in .env or environment")
+if not ORG_ID:
+    raise SystemExit("ERROR: set LDPOV_ORG_ID in .env or environment")
 
-AUTH = b64encode(f"x:{TOKEN}".encode()).decode()
+
+def _version_key(v: str) -> list:
+    return [int(x) if x.isdigit() else x for x in re.split(r"[.\-]", v.split("+")[0])]
 
 
-def get_package_list() -> list[str]:
+def main() -> None:
     s3_path = f"s3://curated-catalog/env=prod/org-id={ORG_ID}/repo-type=npm/redirect_map.json"
-    print(f"Fetching npm package list from S3: {s3_path}", flush=True)
+    print(f"Fetching npm redirect_map from S3: {s3_path}", flush=True)
     result = subprocess.run(
         ["aws", "s3", "cp", s3_path, "-"],
         capture_output=True, text=True, check=True,
     )
     data = json.loads(result.stdout)
-    names = sorted(
-        {urllib.parse.unquote(k.split("/-/")[0]) for k in data},
-        key=str.lower,
-    )
-    print(f"Found {len(names)} packages", flush=True)
-    return names
-
-
-def fetch_versions(name: str) -> tuple[str, list[str]]:
-    encoded = urllib.parse.quote(name, safe="@")
-    url = f"{BASE_URL}{encoded}"
-    try:
-        req = Request(url, headers={"Authorization": f"Basic {AUTH}"})
-        with urlopen(req, timeout=30) as r:
-            doc = json.loads(r.read())
-        versions = list(doc.get("versions", {}).keys())
-        return name, sorted(versions, key=_version_key)
-    except Exception:
-        return name, []
-
-
-def _version_key(v: str) -> list:
-    base = v.split("-")[0]
-    nums = re.findall(r"\d+", base)
-    return [int(n) for n in nums[:3]] + [0] * (3 - len(nums[:3]))
-
-
-def main() -> None:
-    names = get_package_list()
-    total = len(names)
-
-    results: list[dict] = []
-    done = 0
-    print(f"Fetching versions with 30 workers...", flush=True)
-
-    with ThreadPoolExecutor(max_workers=30) as pool:
-        futures = {pool.submit(fetch_versions, n): n for n in names}
-        for future in as_completed(futures):
-            name, versions = future.result()
-            if versions:
-                results.append({"name": name, "versions": versions})
-            done += 1
-            if done % 500 == 0 or done == total:
-                print(f"  {done}/{total}", flush=True)
-
-    results.sort(key=lambda x: x["name"].lower())
+    print(f"Found {len(data)} redirect entries", flush=True)
+
+    pkg_versions: dict[str, set[str]] = defaultdict(set)
+    for raw_key in data:
+        key = urllib.parse.unquote(raw_key)
+        if "/-/" not in key:
+            continue
+        name_part, filename = key.split("/-/", 1)
+        filename = filename.split("#")[0]
+        if not filename.endswith(".tgz"):
+            continue
+        stem = filename[:-4]
+        short_name = name_part.split("/")[-1]
+        prefix = short_name + "-"
+        if stem.startswith(prefix):
+            pkg_versions[name_part].add(stem[len(prefix):])
+
+    packages = [
+        {"name": name, "versions": sorted(versions, key=_version_key)}
+        for name, versions in sorted(pkg_versions.items(), key=lambda x: x[0].lower())
+    ]
 
     out = {
         "generated": datetime.datetime.utcnow().isoformat() + "Z",
-        "packages":  results,
+        "packages":  packages,
     }
 
     out_path = os.path.join(
@@ -104,7 +76,7 @@ def main() -> None:
     with open(out_path, "w") as f:
         json.dump(out, f, separators=(",", ":"))
 
-    print(f"Wrote public/data/ldpov/javascript/catalog.json — {len(results)} packages ({total - len(results)} skipped, no versions)")
+    print(f"Wrote public/data/ldpov/javascript/catalog.json — {len(packages)} packages")
 
 
 if __name__ == "__main__":

public/data/ldpov/java/catalog.json

@@ -10,6 +10,7 @@
 import json
 import os
 import re
+import subprocess
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from urllib.parse import urljoin
 from urllib.request import Request, urlopen
@@ -68,20 +69,39 @@ def fetch_versions(pkg_name: str) -> tuple[str, list[str]]:
         return pkg_name, []
 
 
+def get_package_list_from_s3(org_id: str) -> list[str]:
+    s3_path = f"s3://curated-catalog/env=prod/org-id={org_id}/repo-type=pypi/"
+    print(f"Fetching PyPI package list from S3: {s3_path}", flush=True)
+    result = subprocess.run(
+        ["aws", "s3", "ls", s3_path],
+        capture_output=True, text=True, check=True,
+    )
+    packages = []
+    for line in result.stdout.splitlines():
+        # Lines look like: "                           PRE numpy/"
+        parts = line.strip().split()
+        if len(parts) == 2 and parts[0] == "PRE":
+            packages.append(parts[1].rstrip("/"))
+    return sorted(packages, key=str.lower)
+
+
 def main() -> None:
-    print("Fetching LDPoV PyPI package list...", flush=True)
-    html = get(BASE_URL)
-    packages = [m.rstrip("/") for m in HREF_RE.findall(html) if m.endswith("/")]
+    org_id = os.environ.get("LDPOV_ORG_ID", _env.get("LDPOV_ORG_ID", ""))
+    if not org_id:
+        raise SystemExit("ERROR: set LDPOV_ORG_ID in .env or environment")
+
+    packages = get_package_list_from_s3(org_id)
     total = len(packages)
-    print(f"Found {total} packages. Fetching versions with 30 workers...", flush=True)
+    print(f"Found {total} packages. Fetching versions with 10 workers...", flush=True)
 
     results: list[dict] = []
     done = 0
-    with ThreadPoolExecutor(max_workers=30) as pool:
+    with ThreadPoolExecutor(max_workers=10) as pool:
         futures = {pool.submit(fetch_versions, pkg): pkg for pkg in packages}
         for future in as_completed(futures):
             name, versions = future.result()
-            results.append({"name": name, "versions": versions})
+            if versions:
+                results.append({"name": name, "versions": versions})
             done += 1
             if done % 100 == 0 or done == total:
                 print(f"  {done}/{total}", flush=True)
@@ -101,7 +121,7 @@ def main() -> None:
     with open(out_path, "w") as f:
         json.dump(out, f, separators=(",", ":"))
 
-    print(f"Wrote public/data/ldpov/python/catalog.json — {total} packages")
+    print(f"Wrote public/data/ldpov/python/catalog.json — {len(results)} packages ({total - len(results)} skipped, no versions)")
 
 
 if __name__ == "__main__":