fix: checksum verification for Go, Hex, NuGet + stress test

- Go modules: fetch checksums from sum.golang.org (h1: format → SHA256)
- Hex: extract checksum from release-specific API endpoint
- NuGet: fetch sub-pages when registration items aren't inlined
- NuGet: handle nil version gracefully (non-inlined pages)
- Add 100-package stress test across all 10 registries (100% pass)
- 8/10 registries now provide pre-download checksums

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: Jonathan D.A. Jewell

opsm_ex/lib/opsm/registries/go_modules.ex

@@ -35,7 +35,7 @@ defmodule Opsm.Registries.GoModules do
         case VerifiedHttp.get_json(url, receive_timeout: 10_000) do
           {:ok, body} ->
             deps = fetch_go_mod_deps(encoded, ver)
-            ziphash = fetch_ziphash(encoded, ver)
+            ziphash = fetch_ziphash(name, ver)
             {:ok, parse_module(name, body, ver, deps, ziphash)}
 
           {:error, :not_found} ->
@@ -227,26 +227,52 @@ defmodule Opsm.Registries.GoModules do
     |> Enum.join()
   end
 
-  defp fetch_ziphash(encoded, version) do
-    url = "#{@proxy_url}/#{encoded}/@v/#{version}.ziphash"
+  @sum_db "https://sum.golang.org"
+
+  defp fetch_ziphash(name, version) do
+    # Use the Go checksum database (sum.golang.org) for zip hash
+    url = "#{@sum_db}/lookup/#{name}@#{version}"
     case VerifiedHttp.get(url, receive_timeout: 10_000) do
       {:ok, %{body: body}} when is_binary(body) ->
-        parse_ziphash(String.trim(body))
+        parse_sum_db_hash(body, name, version)
       {:ok, body} when is_binary(body) ->
-        parse_ziphash(String.trim(body))
+        parse_sum_db_hash(body, name, version)
       _ -> nil
     end
   end
 
-  # Go ziphash format: "h1:<base64-encoded-sha256>"
-  # We convert to hex-encoded SHA256 for compatibility with our checksum system
-  defp parse_ziphash("h1:" <> b64_hash) do
-    case Base.decode64(b64_hash) do
+  # Parse sum.golang.org lookup response for the zip hash (h1: format)
+  # Format: "module version h1:<base64>\nmodule version/go.mod h1:<base64>\n..."
+  # We want the first line (zip hash), not the go.mod hash
+  defp parse_sum_db_hash(body, name, version) do
+    body
+    |> String.split("\n")
+    |> Enum.find_value(fn line ->
+      prefix = "#{name} #{version} h1:"
+      if String.starts_with?(line, prefix) do
+        "h1:" <> _ = String.trim_leading(line, "#{name} #{version} ")
+        parse_h1_hash(String.trim(line) |> String.split(" ") |> List.last())
+      end
+    end)
+  end
+
+  # Go h1 hash format: "h1:<base64-encoded-sha256>"
+  # Convert to hex-encoded SHA256 for compatibility with our checksum system
+  defp parse_h1_hash("h1:" <> b64_hash) do
+    b64_clean = String.trim_trailing(b64_hash, "=") |> pad_base64()
+    case Base.decode64(b64_clean) do
       {:ok, raw_hash} -> Base.encode16(raw_hash, case: :lower)
       :error -> nil
     end
   end
-  defp parse_ziphash(_), do: nil
+  defp parse_h1_hash(_), do: nil
+
+  defp pad_base64(s) do
+    case rem(String.length(s), 4) do
+      0 -> s
+      n -> s <> String.duplicate("=", 4 - n)
+    end
+  end
 
   # Parsers
 

opsm_ex/lib/opsm/registries/hex.ex

@@ -30,10 +30,12 @@ defmodule Opsm.Registries.Hex do
           version
         end
 
-        # Fetch release-specific data to get dependencies
-        deps = fetch_release_deps(name, target_version)
+        # Fetch release-specific data to get dependencies and checksum
+        {deps, release_checksum} = fetch_release_data(name, target_version)
         release = Enum.find(releases, fn r -> r["version"] == target_version end)
-        {:ok, parse_package(body, release, target_version, deps)}
+        # Prefer checksum from release-specific endpoint (outer_checksum/checksum)
+        checksum = release_checksum || (if release, do: release["checksum"], else: nil)
+        {:ok, parse_package(body, checksum, target_version, deps)}
 
       {:error, :not_found} ->
         {:error, :not_found}
@@ -49,26 +51,28 @@ defmodule Opsm.Registries.Hex do
     end
   end
 
-  # Fetch dependencies from the release-specific API endpoint
-  defp fetch_release_deps(name, version) do
+  # Fetch dependencies and checksum from the release-specific API endpoint
+  defp fetch_release_data(name, version) do
     url = "#{@base_url}/packages/#{URI.encode(name)}/releases/#{version}"
 
     case VerifiedHttp.get_json(url, receive_timeout: 10_000) do
       {:ok, body} ->
         requirements = body["requirements"] || %{}
-        # Convert hex requirements format to simple {name => constraint} map
-        Enum.reduce(requirements, %{}, fn {dep_name, req}, acc ->
+        deps = Enum.reduce(requirements, %{}, fn {dep_name, req}, acc ->
           constraint = req["requirement"] || ">= 0.0.0"
-          # Skip optional dependencies
           if req["optional"] do
             acc
           else
             Map.put(acc, dep_name, constraint)
           end
         end)
 
+        # Extract checksum — Hex uses "checksum" (outer tarball checksum)
+        checksum = body["checksum"]
+        {deps, checksum}
+
       _ ->
-        %{}
+        {%{}, nil}
     end
   end
 
@@ -137,9 +141,8 @@ defmodule Opsm.Registries.Hex do
 
   # Parsers
 
-  defp parse_package(pkg, release, version, deps) do
+  defp parse_package(pkg, checksum, version, deps) do
     meta = pkg["meta"] || %{}
-    checksum = if release, do: release["checksum"], else: nil
 
     %ResolvedPackage{
       package: pkg["name"],

opsm_ex/lib/opsm/registries/nuget.ex

@@ -23,11 +23,23 @@ defmodule Opsm.Registries.NuGet do
     case VerifiedHttp.get_json(url, receive_timeout: 10_000) do
       {:ok, body} ->
         # Registration pages contain version catalogs
+        # Some pages inline items, others have only @id and need separate fetching
         items = body["items"] || []
         all_entries = items
           |> Enum.flat_map(fn page ->
-            # Some pages inline items, others need fetching
-            page["items"] || []
+            case page["items"] do
+              nil ->
+                # Page doesn't inline items — fetch the page by @id
+                case page["@id"] do
+                  nil -> []
+                  page_url ->
+                    case VerifiedHttp.get_json(page_url, receive_timeout: 10_000) do
+                      {:ok, page_body} -> page_body["items"] || []
+                      _ -> []
+                    end
+                end
+              inlined -> inlined
+            end
           end)
 
         target_version = if version == "latest" do
@@ -49,11 +61,15 @@ defmodule Opsm.Registries.NuGet do
           catalog["version"] == target_version
         end)
 
-        catalog = if entry, do: entry["catalogEntry"] || %{}, else: %{}
-        deps = extract_nuget_deps(catalog)
-        {hash, hash_algo} = extract_nuget_hash(catalog)
+        if is_nil(target_version) do
+          {:error, :not_found}
+        else
+          catalog = if entry, do: entry["catalogEntry"] || %{}, else: %{}
+          deps = extract_nuget_deps(catalog)
+          {hash, hash_algo} = extract_nuget_hash(catalog)
 
-        {:ok, parse_nuget_package(name, target_version, catalog, deps, hash, hash_algo)}
+          {:ok, parse_nuget_package(name, target_version, catalog, deps, hash, hash_algo)}
+        end
 
       {:error, :not_found} ->
         {:error, :not_found}

opsm_ex/test/stress_test.exs

@@ -0,0 +1,185 @@
+# SPDX-License-Identifier: PMPL-1.0-or-later
+# Stress test: fetch real packages from all 10 registries
+# Run: mix run test/stress_test.exs
+
+alias Opsm.Registries.Registry
+
+# Test packages per registry — well-known, stable packages
+test_packages = [
+  # npm (10 packages)
+  {:npm, "express", "4.18.2"},
+  {:npm, "lodash", "latest"},
+  {:npm, "chalk", "5.3.0"},
+  {:npm, "axios", "latest"},
+  {:npm, "uuid", "latest"},
+  {:npm, "debug", "latest"},
+  {:npm, "commander", "latest"},
+  {:npm, "minimist", "latest"},
+  {:npm, "glob", "latest"},
+  {:npm, "semver", "latest"},
+
+  # Hex/Elixir (10 packages)
+  {:hex, "jason", "latest"},
+  {:hex, "plug", "latest"},
+  {:hex, "phoenix", "latest"},
+  {:hex, "ecto", "latest"},
+  {:hex, "telemetry", "latest"},
+  {:hex, "nimble_parsec", "latest"},
+  {:hex, "ex_doc", "latest"},
+  {:hex, "earmark_parser", "latest"},
+  {:hex, "decimal", "latest"},
+  {:hex, "mime", "latest"},
+
+  # Cargo/Crates (10 packages)
+  {:cargo, "serde", "latest"},
+  {:cargo, "tokio", "latest"},
+  {:cargo, "rand", "latest"},
+  {:cargo, "clap", "latest"},
+  {:cargo, "regex", "latest"},
+  {:cargo, "log", "latest"},
+  {:cargo, "anyhow", "latest"},
+  {:cargo, "thiserror", "latest"},
+  {:cargo, "chrono", "latest"},
+  {:cargo, "reqwest", "latest"},
+
+  # PyPI (10 packages)
+  {:pypi, "requests", "latest"},
+  {:pypi, "flask", "latest"},
+  {:pypi, "numpy", "latest"},
+  {:pypi, "pytest", "latest"},
+  {:pypi, "click", "latest"},
+  {:pypi, "pyyaml", "latest"},
+  {:pypi, "six", "latest"},
+  {:pypi, "packaging", "latest"},
+  {:pypi, "idna", "latest"},
+  {:pypi, "certifi", "latest"},
+
+  # RubyGems (10 packages)
+  {:gem, "rails", "latest"},
+  {:gem, "rake", "latest"},
+  {:gem, "bundler", "latest"},
+  {:gem, "rspec", "latest"},
+  {:gem, "nokogiri", "latest"},
+  {:gem, "sinatra", "latest"},
+  {:gem, "puma", "latest"},
+  {:gem, "redis", "latest"},
+  {:gem, "json", "latest"},
+  {:gem, "minitest", "latest"},
+
+  # Go Modules (10 packages)
+  {:go, "github.com/gin-gonic/gin", "latest"},
+  {:go, "github.com/fatih/color", "latest"},
+  {:go, "github.com/spf13/cobra", "latest"},
+  {:go, "github.com/spf13/viper", "latest"},
+  {:go, "github.com/stretchr/testify", "latest"},
+  {:go, "github.com/sirupsen/logrus", "latest"},
+  {:go, "github.com/gorilla/mux", "latest"},
+  {:go, "golang.org/x/text", "latest"},
+  {:go, "golang.org/x/net", "latest"},
+  {:go, "golang.org/x/sync", "latest"},
+
+  # Pub.dev/Dart (10 packages)
+  {:pub, "http", "latest"},
+  {:pub, "path", "latest"},
+  {:pub, "json_annotation", "latest"},
+  {:pub, "provider", "latest"},
+  {:pub, "intl", "latest"},
+  {:pub, "collection", "latest"},
+  {:pub, "meta", "latest"},
+  {:pub, "args", "latest"},
+  {:pub, "crypto", "latest"},
+  {:pub, "async", "latest"},
+
+  # Hackage/Haskell (10 packages)
+  {:hackage, "aeson", "latest"},
+  {:hackage, "text", "latest"},
+  {:hackage, "bytestring", "latest"},
+  {:hackage, "containers", "latest"},
+  {:hackage, "mtl", "latest"},
+  {:hackage, "transformers", "latest"},
+  {:hackage, "parsec", "latest"},
+  {:hackage, "vector", "latest"},
+  {:hackage, "lens", "latest"},
+  {:hackage, "attoparsec", "latest"},
+
+  # NuGet/.NET (10 packages)
+  {:nuget, "Newtonsoft.Json", "latest"},
+  {:nuget, "Serilog", "latest"},
+  {:nuget, "AutoMapper", "latest"},
+  {:nuget, "Dapper", "latest"},
+  {:nuget, "FluentValidation", "latest"},
+  {:nuget, "Moq", "latest"},
+  {:nuget, "xunit", "latest"},
+  {:nuget, "Polly", "latest"},
+  {:nuget, "MediatR", "latest"},
+  {:nuget, "Swashbuckle.AspNetCore", "latest"},
+
+  # Maven/Java (10 packages)
+  {:maven, "com.google.guava:guava", "latest"},
+  {:maven, "org.slf4j:slf4j-api", "latest"},
+  {:maven, "junit:junit", "latest"},
+  {:maven, "com.fasterxml.jackson.core:jackson-databind", "latest"},
+  {:maven, "org.apache.commons:commons-lang3", "latest"},
+  {:maven, "commons-io:commons-io", "latest"},
+  {:maven, "org.projectlombok:lombok", "latest"},
+  {:maven, "com.google.code.gson:gson", "latest"},
+  {:maven, "org.mockito:mockito-core", "latest"},
+  {:maven, "ch.qos.logback:logback-classic", "latest"},
+]
+
+IO.puts("=" |> String.duplicate(70))
+IO.puts("OPSM Stress Test: #{length(test_packages)} packages across 10 registries")
+IO.puts("=" |> String.duplicate(70))
+IO.puts("")
+
+# Group by registry
+grouped = Enum.group_by(test_packages, fn {forth, _, _} -> forth end)
+
+results = %{ok: 0, error: 0, total: length(test_packages)}
+errors = []
+
+{results, errors} =
+  Enum.reduce(grouped, {results, errors}, fn {forth, packages}, {res_acc, err_acc} ->
+    IO.puts("--- @#{forth} (#{length(packages)} packages) ---")
+
+    {res, errs} =
+      Enum.reduce(packages, {res_acc, err_acc}, fn {_forth, name, version}, {r, e} ->
+        start = System.monotonic_time(:millisecond)
+
+        case Registry.fetch(forth, name, version) do
+          {:ok, pkg} ->
+            elapsed = System.monotonic_time(:millisecond) - start
+            checksum_status = if pkg.checksum, do: "✓ #{pkg.checksum_algo}", else: "✗ none"
+            dep_count = map_size(pkg.manifest.dependencies || %{})
+            IO.puts("  ✓ #{name}@#{pkg.version} (#{elapsed}ms, deps: #{dep_count}, checksum: #{checksum_status})")
+            {%{r | ok: r.ok + 1}, e}
+
+          {:error, reason} ->
+            elapsed = System.monotonic_time(:millisecond) - start
+            IO.puts("  ✗ #{name} FAILED (#{elapsed}ms): #{inspect(reason)}")
+            {%{r | error: r.error + 1}, [{forth, name, reason} | e]}
+        end
+      end)
+
+    IO.puts("")
+    {res, errs}
+  end)
+
+IO.puts("=" |> String.duplicate(70))
+IO.puts("RESULTS: #{results.ok}/#{results.total} succeeded, #{results.error} failed")
+IO.puts("=" |> String.duplicate(70))
+
+if errors != [] do
+  IO.puts("")
+  IO.puts("FAILURES:")
+  for {forth, name, reason} <- Enum.reverse(errors) do
+    IO.puts("  @#{forth}/#{name}: #{inspect(reason)}")
+  end
+end
+
+# Exit with appropriate code
+if results.error > 0 do
+  IO.puts("\n⚠ #{results.error} package(s) failed to resolve")
+else
+  IO.puts("\n✓ All #{results.total} packages resolved successfully!")
+end