Cleanup and fix the GeneratePackageVersions (#8478)

## Summary of changes

This goes back through the recent changes (#8371 and #8340) to
`GeneratePackageVersions` and attempts to do some removals,
refactorings, and fixes that have come up. Notably this removes the
`nuget_cache`, fixes `IncludePackages` (again 😅), and rewrites the
cooldown logic to not use `supported_versions.json` and `nuget_cache`
and instead reads from the `.g.cs` files in addition to refactoring the
verbage used throughout.

## Reason for change

While working on (#8128) I started running into several issues with
`GeneratePackageVersions` namely that I couldn't get `IncludePackages`
to work and then I started fighting issues with the `nuget_cache` as
well when attempting to add a new integration.

This sets out to do a few things:

- Simplify the logic for NuGet cooldown (automated dependency updates
must adhere to our current cooldown policy for potential supply chain
attacks)
- Remove the `nuget_cache` - this was added so that we wouldn't go and
query NuGet info for packages not listed in the `IncludePackages`
- Change the "verbage" of the cooldown/baseline to be easier to read /
follow
- Do NOT read / rely upon `supported_versions` (this is what I
previously used to determine what the highest version was currently
allowed) - this file is planned to be changed/modified/removed by
another team in the near future hence why we shouldn't use it.
- I didn't account for different timezones in the publish state of the
`nuget_cache` before so if someone ran it depending on their locale it
would change.

## Implementation details

- New `CooldownMode` `enum`: `Normal`, `BypassCooldown`, `Freeze`
  - Normal drops recent versions that are above the previous max
- BypassCooldown accepts all in-range versions (no cooldown) (when
`IncludePackages` is used is an example)
  - Freeze re-emits the previous run's output verbatim
- `XUnitFileGenerator.LoadExistingVersions` reads the previous `.g.cs`
and returns `Dictionary<IntegrationName, List<(Framework, Versions)>>`.
`PackageGroup` loads this once and exposes `TryGetFrozenVersions`
- I think we should probably still try and do something different here
but this seems a bit more reasonable
- Deleted `nuget_version_cache.json` and `NuGetVersionCache.cs`
  - No longer used/needed
- Some readability improvements
  -  `IsWithinCooldown` → `WasPublishedTooRecently`
  - `baseline` → `previousMaxVersions`
- `ApplyCooldown` now uses two named local booleans
(`publishedTooRecently`, `atOrBelowPreviousMax`) so the drop condition
reads positively: "drop the version if it was published too recently
**and** we haven't already shipped against it."

## Test coverage

I ran it locally for a handful of cases I think it is better.

## Other details
<!-- Fixes #{issue} -->

I have ran through several instances with this and it _seems_ to be
good, basically default runs, runs with configurable cooldowns, runs
with `IncludePackages` and runs with `ExcludePackages`. All appeared as
expected to me.


<!--  ⚠️ Note:

Where possible, please obtain 2 approvals prior to merging. Unless
CODEOWNERS specifies otherwise, for external teams it is typically best
to have one review from a team member, and one review from apm-dotnet.
Trivial changes do not require 2 reviews.

MergeQueue is NOT enabled in this repository. If you have write access
to the repo, the PR has 1-2 approvals (see above), and all of the
required checks have passed, you can use the Squash and Merge button to
merge the PR. If you don't have write access, or you need help, reach
out in the #apm-dotnet channel in Slack.
-->

---------

Co-authored-by: Andrew Lock <andrew.lock@datadoghq.com>

Author: bouwkast

tracer/build/_build/Build.Utilities.cs

@@ -59,7 +59,7 @@ partial class Build
     [Parameter("Only update package versions for packages with the following names")]
     readonly string[] IncludePackages;
 
-    [Parameter("Minimum age in days a NuGet package version must have been published before auto-including. Defaults to 2 days, or 0 when --IncludePackages is set")]
+    [Parameter("Minimum age in days a NuGet package version must have been published before auto-including. Defaults to 2. Ignored for packages named in --IncludePackages, which always bypass the cooldown.")]
     readonly int? PackageVersionCooldownDays;
 
     [LazyLocalExecutable(@"C:\Program Files (x86)\Microsoft SDKs\Windows\v10.0A\bin\NETFX 4.8 Tools\gacutil.exe")]
@@ -235,53 +235,56 @@ partial class Build
            var definitionsFile = BuildDirectory / FileNames.DefinitionsJson;
            var supportedVersionsPath = BuildDirectory / "supported_versions.json";
 
-           // Build the shouldQueryNuGet predicate from include/exclude filters
-           Func<string, bool> shouldUpdatePackage = (IncludePackages, ExcludePackages) switch
+           // Decides the cooldown treatment for each package by name. See CooldownMode for what each value does.
+           var getCooldownMode = BuildCooldownModeSelector(IncludePackages, ExcludePackages);
+
+           // Dependabot re-uses the previous entry verbatim for Freeze; Skip and Normal both refresh.
+           Func<string, bool> shouldUpdatePackage = name => getCooldownMode(name) is not CooldownMode.Freeze;
+
+           static Func<string, CooldownMode> BuildCooldownModeSelector(string[] includePackages, string[] excludePackages)
            {
-               ({ } include, _) => name => include.Contains(name, StringComparer.OrdinalIgnoreCase),
-               (_, { } exclude) => name => !exclude.Contains(name, StringComparer.OrdinalIgnoreCase),
-               _ => _ => true
-           };
-
-           // Load caches for both pipelines
-           var cacheFilePath = BuildDirectory / "nuget_version_cache.json";
-           var previousVersionCache = await NuGetVersionCache.Load(cacheFilePath);
-           Logger.Information("Loaded NuGet version cache with {Count} entries", previousVersionCache.Count);
+               // No filter: every package goes through the normal cooldown filter.
+               if (includePackages is null && excludePackages is null)
+               {
+                   return _ => CooldownMode.Normal;
+               }
+
+               // --IncludePackages Foo Bar: update only the listed packages (bypassing cooldown);
+               // freeze every other package so it re-emits its previous output unchanged.
+               if (includePackages is not null)
+               {
+                   var targeted = new HashSet<string>(includePackages, StringComparer.OrdinalIgnoreCase);
+                   return name => targeted.Contains(name) ? CooldownMode.BypassCooldown : CooldownMode.Freeze;
+               }
+
+               // --ExcludePackages Foo Bar: freeze the listed packages; everything else updates normally.
+               var blocked = new HashSet<string>(excludePackages, StringComparer.OrdinalIgnoreCase);
+               return name => blocked.Contains(name) ? CooldownMode.Freeze : CooldownMode.Normal;
+           }
+
+           // supported_versions.json is still loaded for Pipeline B (dependabot + supported_versions.json
+           // regeneration). The cooldown filter in Pipeline A sources its previous-max data from the
+           // generated .g.cs files directly, because those retain per-major version history that
+           // supported_versions.json's single-max-per-package shape cannot represent.
            var previousSupportedVersions = await GenerateSupportMatrix.LoadPreviousVersions(supportedVersionsPath);
            Logger.Information("Loaded previous supported versions with {Count} entries", previousSupportedVersions.Count);
 
-           // Derive baseline from supported_versions.json: the max tested version per package
-           // acts as a floor to prevent cooldown filtering from downgrading previously accepted versions.
-           // We collect all max tested versions per package (not just the global max) so that
-           // split-range packages (e.g., GraphQL 4.x-6.x and 7.x-9.x) get a per-range baseline.
-           var baseline = previousSupportedVersions
-               .Where(kvp => kvp.Value.MaxVersionTestedInclusive is not null)
-               .GroupBy(kvp => kvp.Key.PackageName)
-               .ToDictionary(
-                   g => g.Key,
-                   g => g.Select(kvp => new Version(kvp.Value.MaxVersionTestedInclusive!)).ToList());
-           Logger.Information("Derived version baseline with {Count} entries from supported_versions.json", baseline.Count);
-
-           // Resolve effective cooldown:
-           //  - Explicit --PackageVersionCooldownDays wins
-           //  - --IncludePackages without explicit cooldown defaults to 0
-           var effectiveCooldownDays = PackageVersionCooldownDays ?? (IncludePackages is not null ? 0 : 2);
+           var effectiveCooldownDays = PackageVersionCooldownDays ?? 2;
 
            // Pipeline A: generate .g.props/.g.cs files
            Logger.Information("Using package version cooldown of {Days} days", effectiveCooldownDays);
-           var versionGenerator = new PackageVersionGenerator(TracerDirectory, testDir, shouldUpdatePackage, previousVersionCache, effectiveCooldownDays, baseline);
+           var versionGenerator = new PackageVersionGenerator(TracerDirectory, testDir, getCooldownMode, effectiveCooldownDays);
            var testedVersions = await versionGenerator.GenerateVersions(Solution);
-           await NuGetVersionCache.Save(cacheFilePath, versionGenerator.VersionCache);
 
            // Log version changes: bumps, unchanged, and overridden
-           var versionCache = versionGenerator.VersionCache;
+           var queriedVersions = versionGenerator.QueriedVersions;
            var bumped = 0;
            var unchanged = 0;
            foreach (var tested in testedVersions)
            {
                var packageName = tested.NugetPackageSearchName;
-               baseline.TryGetValue(packageName, out var previousMaxVersions);
-               var previousMax = previousMaxVersions?
+               var previousMax = versionGenerator
+                   .GetPreviouslyTestedVersions(tested.IntegrationName)
                    .Where(v => v >= tested.MinVersion && v <= tested.MaxVersion)
                    .OrderByDescending(v => v)
                    .FirstOrDefault();
@@ -290,12 +293,12 @@ partial class Build
                {
                    bumped++;
                    var publishedDate = "(unknown)";
-                   if (versionCache.TryGetValue(packageName, out var cachedVersions))
+                   if (queriedVersions.TryGetValue(packageName, out var versionsForPackage))
                    {
-                       var match = cachedVersions.FirstOrDefault(v => v.Version == tested.MaxVersion.ToString());
+                       var match = versionsForPackage.FirstOrDefault(v => v.Version == tested.MaxVersion.ToString());
                        if (match?.Published is not null)
                        {
-                           publishedDate = match.Published.Value.ToString("yyyy-MM-dd");
+                           publishedDate = match.Published.Value.UtcDateTime.ToString("yyyy-MM-dd");
                        }
                    }
 
@@ -325,13 +328,11 @@ partial class Build
 
                foreach (var entry in versionGenerator.CooldownReport.Entries)
                {
-                   var resolvedText = entry.ResolvedVersion is not null ? $"using: {entry.ResolvedVersion}" : "skipped";
                    Logger.Warning(
-                       "  {Package} {Version} overridden (published {Date}, {Resolved})",
+                       "  {Package} {Version} overridden (published {Date})",
                        entry.PackageName,
                        entry.OverriddenVersion,
-                       entry.PublishedDate?.ToString("yyyy-MM-dd") ?? "unknown",
-                       resolvedText);
+                       entry.PublishedDate?.UtcDateTime.ToString("yyyy-MM-dd") ?? "unknown");
                }
 
                var reportPath = TemporaryDirectory / "cooldown_report.md";
@@ -347,7 +348,7 @@ partial class Build
            var integrations = GenerateIntegrationDefinitions.GetAllIntegrations(assemblies, definitionsFile);
 
            // Pipeline B: generate dependabot files + supported_versions.json
-           // TestedVersions are cooldown-filtered but the baseline prevents downgrades,
+           // TestedVersions are cooldown-filtered but the previous max pins prevent downgrades,
            // so they accurately reflect what we're testing.
            var distinctIntegrations = await DependabotFileManager.BuildDistinctIntegrationMaps(
                integrations, testedVersions, shouldUpdatePackage, previousSupportedVersions);

tracer/build/_build/GeneratePackageVersions/CooldownMode.cs

@@ -0,0 +1,33 @@
+// <copyright file="CooldownMode.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+namespace GeneratePackageVersions;
+
+/// <summary>
+/// Controls how cooldown filtering is applied to a package.
+/// </summary>
+public enum CooldownMode
+{
+    /// <summary>
+    /// Normal cooldown: versions published within the cooldown window are dropped, unless they
+    /// are at or below the highest version we already tested in a prior run for this entry's range
+    /// (so cooldown never downgrades a version we already shipped against).
+    /// </summary>
+    Normal,
+
+    /// <summary>
+    /// Bypass the cooldown filter: every NuGet version in this entry's range is accepted,
+    /// including those published within the cooldown window. Used for packages explicitly
+    /// targeted via --IncludePackages.
+    /// </summary>
+    BypassCooldown,
+
+    /// <summary>
+    /// Re-emit the previous run's versions verbatim. The generators are not re-run against NuGet;
+    /// the previous .g.cs file is parsed and its versions are fed back through Write(). Used for
+    /// non-targeted packages when --IncludePackages or --ExcludePackages filters are active.
+    /// </summary>
+    Freeze,
+}

tracer/build/_build/GeneratePackageVersions/CooldownReport.cs

@@ -19,7 +19,6 @@ public class CooldownReport
 {
     private readonly int _cooldownDays;
     private readonly List<CooldownEntry> _entries = new();
-    private readonly HashSet<(string PackageName, string Version)> _seen = new();
 
     public CooldownReport(int cooldownDays)
     {
@@ -32,12 +31,7 @@ public CooldownReport(int cooldownDays)
 
     public void Add(CooldownEntry entry)
     {
-        // Deduplicate: the same version gets flagged once per framework and per selection group,
-        // but we only need to report it once.
-        if (_seen.Add((entry.PackageName, entry.OverriddenVersion)))
-        {
-            _entries.Add(entry);
-        }
+        _entries.Add(entry);
     }
 
     public string ToMarkdown()
@@ -53,18 +47,17 @@ public string ToMarkdown()
         sb.AppendLine($"The following versions were published less than **{_cooldownDays} days** ago and have been overridden.");
         sb.AppendLine("These require manual review before inclusion.");
         sb.AppendLine();
-        sb.AppendLine("| Package | Integration | Overridden Version | Published | Age (days) | Using Instead |");
-        sb.AppendLine("|---------|-------------|--------------------|-----------|------------|---------------|");
+        sb.AppendLine("| Package | Integration | Overridden Version | Published | Age (days) |");
+        sb.AppendLine("|---------|-------------|--------------------|-----------|------------|");
 
         foreach (var entry in _entries)
         {
-            var published = entry.PublishedDate?.ToString("yyyy-MM-dd") ?? "unknown";
+            var published = entry.PublishedDate?.UtcDateTime.ToString("yyyy-MM-dd") ?? "unknown";
             var age = entry.PublishedDate.HasValue
                 ? ((int)(DateTimeOffset.UtcNow - entry.PublishedDate.Value).TotalDays).ToString()
                 : "?";
-            var usingInstead = entry.ResolvedVersion ?? "(skipped)";
 
-            sb.AppendLine($"| {entry.PackageName} | {entry.IntegrationName} | {entry.OverriddenVersion} | {published} | {age} | {usingInstead} |");
+            sb.AppendLine($"| {entry.PackageName} | {entry.IntegrationName} | {entry.OverriddenVersion} | {published} | {age} |");
         }
 
         return sb.ToString();
@@ -84,6 +77,5 @@ public record CooldownEntry(
         string PackageName,
         string IntegrationName,
         string OverriddenVersion,
-        DateTimeOffset? PublishedDate,
-        string ResolvedVersion);
+        DateTimeOffset? PublishedDate);
 }