Cleanup and fix the GeneratePackageVersions

Author: bouwkast

tracer/build/_build/Build.Utilities.cs

@@ -59,7 +59,7 @@ partial class Build
     [Parameter("Only update package versions for packages with the following names")]
     readonly string[] IncludePackages;
 
-    [Parameter("Minimum age in days a NuGet package version must have been published before auto-including. Defaults to 2 days, or 0 when --IncludePackages is set")]
+    [Parameter("Minimum age in days a NuGet package version must have been published before auto-including. Defaults to 2. Ignored for packages named in --IncludePackages, which always bypass the cooldown.")]
     readonly int? PackageVersionCooldownDays;
 
     [LazyLocalExecutable(@"C:\Program Files (x86)\Microsoft SDKs\Windows\v10.0A\bin\NETFX 4.8 Tools\gacutil.exe")]
@@ -235,53 +235,64 @@ partial class Build
            var definitionsFile = BuildDirectory / FileNames.DefinitionsJson;
            var supportedVersionsPath = BuildDirectory / "supported_versions.json";
 
-           // Build the shouldQueryNuGet predicate from include/exclude filters
-           Func<string, bool> shouldUpdatePackage = (IncludePackages, ExcludePackages) switch
+           // Decides the cooldown treatment for each package by name. See CooldownMode for what each value does.
+           var getCooldownMode = BuildCooldownModeSelector(IncludePackages, ExcludePackages);
+
+           // Dependabot re-uses the previous entry verbatim for Freeze; Skip and Normal both refresh.
+           Func<string, bool> shouldUpdatePackage = name => getCooldownMode(name) is not CooldownMode.Freeze;
+
+           static Func<string, CooldownMode> BuildCooldownModeSelector(string[] includePackages, string[] excludePackages)
            {
-               ({ } include, _) => name => include.Contains(name, StringComparer.OrdinalIgnoreCase),
-               (_, { } exclude) => name => !exclude.Contains(name, StringComparer.OrdinalIgnoreCase),
-               _ => _ => true
-           };
-
-           // Load caches for both pipelines
-           var cacheFilePath = BuildDirectory / "nuget_version_cache.json";
-           var previousVersionCache = await NuGetVersionCache.Load(cacheFilePath);
-           Logger.Information("Loaded NuGet version cache with {Count} entries", previousVersionCache.Count);
+               // No filter: every package goes through the normal cooldown filter.
+               if (includePackages is null && excludePackages is null)
+               {
+                   return _ => CooldownMode.Normal;
+               }
+
+               // --IncludePackages Foo Bar: update only the listed packages (bypassing cooldown);
+               // freeze every other package so it re-emits its previous output unchanged.
+               if (includePackages is not null)
+               {
+                   var targeted = new HashSet<string>(includePackages, StringComparer.OrdinalIgnoreCase);
+                   return name => targeted.Contains(name) ? CooldownMode.BypassCooldown : CooldownMode.Freeze;
+               }
+
+               // --ExcludePackages Foo Bar: freeze the listed packages; everything else updates normally.
+               var blocked = new HashSet<string>(excludePackages, StringComparer.OrdinalIgnoreCase);
+               return name => blocked.Contains(name) ? CooldownMode.Freeze : CooldownMode.Normal;
+           }
+
            var previousSupportedVersions = await GenerateSupportMatrix.LoadPreviousVersions(supportedVersionsPath);
            Logger.Information("Loaded previous supported versions with {Count} entries", previousSupportedVersions.Count);
 
-           // Derive baseline from supported_versions.json: the max tested version per package
-           // acts as a floor to prevent cooldown filtering from downgrading previously accepted versions.
-           // We collect all max tested versions per package (not just the global max) so that
-           // split-range packages (e.g., GraphQL 4.x-6.x and 7.x-9.x) get a per-range baseline.
-           var baseline = previousSupportedVersions
+           // Pull the max tested version per package from supported_versions.json. The cooldown
+           // filter keeps anything at or below this value so we never downgrade a version we
+           // already shipped against. Collected as a list per package (not a single global max)
+           // so split-range packages (e.g. GraphQL 4.x-6.x and 7.x-9.x) get a per-range value.
+           var previousMaxVersions = previousSupportedVersions
                .Where(kvp => kvp.Value.MaxVersionTestedInclusive is not null)
                .GroupBy(kvp => kvp.Key.PackageName)
                .ToDictionary(
                    g => g.Key,
                    g => g.Select(kvp => new Version(kvp.Value.MaxVersionTestedInclusive!)).ToList());
-           Logger.Information("Derived version baseline with {Count} entries from supported_versions.json", baseline.Count);
+           Logger.Information("Loaded previous max tested versions for {Count} packages from supported_versions.json", previousMaxVersions.Count);
 
-           // Resolve effective cooldown:
-           //  - Explicit --PackageVersionCooldownDays wins
-           //  - --IncludePackages without explicit cooldown defaults to 0
-           var effectiveCooldownDays = PackageVersionCooldownDays ?? (IncludePackages is not null ? 0 : 2);
+           var effectiveCooldownDays = PackageVersionCooldownDays ?? 2;
 
            // Pipeline A: generate .g.props/.g.cs files
            Logger.Information("Using package version cooldown of {Days} days", effectiveCooldownDays);
-           var versionGenerator = new PackageVersionGenerator(TracerDirectory, testDir, shouldUpdatePackage, previousVersionCache, effectiveCooldownDays, baseline);
+           var versionGenerator = new PackageVersionGenerator(TracerDirectory, testDir, getCooldownMode, effectiveCooldownDays, previousMaxVersions);
            var testedVersions = await versionGenerator.GenerateVersions(Solution);
-           await NuGetVersionCache.Save(cacheFilePath, versionGenerator.VersionCache);
 
            // Log version changes: bumps, unchanged, and overridden
-           var versionCache = versionGenerator.VersionCache;
+           var queriedVersions = versionGenerator.QueriedVersions;
            var bumped = 0;
            var unchanged = 0;
            foreach (var tested in testedVersions)
            {
                var packageName = tested.NugetPackageSearchName;
-               baseline.TryGetValue(packageName, out var previousMaxVersions);
-               var previousMax = previousMaxVersions?
+               previousMaxVersions.TryGetValue(packageName, out var previousMaxCandidates);
+               var previousMax = previousMaxCandidates?
                    .Where(v => v >= tested.MinVersion && v <= tested.MaxVersion)
                    .OrderByDescending(v => v)
                    .FirstOrDefault();
@@ -290,12 +301,12 @@ partial class Build
                {
                    bumped++;
                    var publishedDate = "(unknown)";
-                   if (versionCache.TryGetValue(packageName, out var cachedVersions))
+                   if (queriedVersions.TryGetValue(packageName, out var versionsForPackage))
                    {
-                       var match = cachedVersions.FirstOrDefault(v => v.Version == tested.MaxVersion.ToString());
+                       var match = versionsForPackage.FirstOrDefault(v => v.Version == tested.MaxVersion.ToString());
                        if (match?.Published is not null)
                        {
-                           publishedDate = match.Published.Value.ToString("yyyy-MM-dd");
+                           publishedDate = match.Published.Value.UtcDateTime.ToString("yyyy-MM-dd");
                        }
                    }
 
@@ -325,13 +336,11 @@ partial class Build
 
                foreach (var entry in versionGenerator.CooldownReport.Entries)
                {
-                   var resolvedText = entry.ResolvedVersion is not null ? $"using: {entry.ResolvedVersion}" : "skipped";
                    Logger.Warning(
-                       "  {Package} {Version} overridden (published {Date}, {Resolved})",
+                       "  {Package} {Version} overridden (published {Date})",
                        entry.PackageName,
                        entry.OverriddenVersion,
-                       entry.PublishedDate?.ToString("yyyy-MM-dd") ?? "unknown",
-                       resolvedText);
+                       entry.PublishedDate?.UtcDateTime.ToString("yyyy-MM-dd") ?? "unknown");
                }
 
                var reportPath = TemporaryDirectory / "cooldown_report.md";
@@ -347,7 +356,7 @@ partial class Build
            var integrations = GenerateIntegrationDefinitions.GetAllIntegrations(assemblies, definitionsFile);
 
            // Pipeline B: generate dependabot files + supported_versions.json
-           // TestedVersions are cooldown-filtered but the baseline prevents downgrades,
+           // TestedVersions are cooldown-filtered but the previous max pins prevent downgrades,
            // so they accurately reflect what we're testing.
            var distinctIntegrations = await DependabotFileManager.BuildDistinctIntegrationMaps(
                integrations, testedVersions, shouldUpdatePackage, previousSupportedVersions);

tracer/build/_build/GeneratePackageVersions/CooldownMode.cs

@@ -0,0 +1,33 @@
+// <copyright file="CooldownMode.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+namespace GeneratePackageVersions;
+
+/// <summary>
+/// Controls how cooldown filtering is applied to a package.
+/// </summary>
+public enum CooldownMode
+{
+    /// <summary>
+    /// Normal cooldown: versions published within the cooldown window are dropped, unless they
+    /// are at or below the highest version we already tested in a prior run for this entry's range
+    /// (so cooldown never downgrades a version we already shipped against).
+    /// </summary>
+    Normal,
+
+    /// <summary>
+    /// Bypass the cooldown filter: every NuGet version in this entry's range is accepted,
+    /// including those published within the cooldown window. Used for packages explicitly
+    /// targeted via --IncludePackages.
+    /// </summary>
+    BypassCooldown,
+
+    /// <summary>
+    /// Re-emit the previous run's versions verbatim. The generators are not re-run against NuGet;
+    /// the previous .g.cs file is parsed and its versions are fed back through Write(). Used for
+    /// non-targeted packages when --IncludePackages or --ExcludePackages filters are active.
+    /// </summary>
+    Freeze,
+}

tracer/build/_build/GeneratePackageVersions/CooldownReport.cs

@@ -19,7 +19,6 @@ public class CooldownReport
 {
     private readonly int _cooldownDays;
     private readonly List<CooldownEntry> _entries = new();
-    private readonly HashSet<(string PackageName, string Version)> _seen = new();
 
     public CooldownReport(int cooldownDays)
     {
@@ -32,12 +31,7 @@ public CooldownReport(int cooldownDays)
 
     public void Add(CooldownEntry entry)
     {
-        // Deduplicate: the same version gets flagged once per framework and per selection group,
-        // but we only need to report it once.
-        if (_seen.Add((entry.PackageName, entry.OverriddenVersion)))
-        {
-            _entries.Add(entry);
-        }
+        _entries.Add(entry);
     }
 
     public string ToMarkdown()
@@ -53,18 +47,17 @@ public string ToMarkdown()
         sb.AppendLine($"The following versions were published less than **{_cooldownDays} days** ago and have been overridden.");
         sb.AppendLine("These require manual review before inclusion.");
         sb.AppendLine();
-        sb.AppendLine("| Package | Integration | Overridden Version | Published | Age (days) | Using Instead |");
-        sb.AppendLine("|---------|-------------|--------------------|-----------|------------|---------------|");
+        sb.AppendLine("| Package | Integration | Overridden Version | Published | Age (days) |");
+        sb.AppendLine("|---------|-------------|--------------------|-----------|------------|");
 
         foreach (var entry in _entries)
         {
-            var published = entry.PublishedDate?.ToString("yyyy-MM-dd") ?? "unknown";
+            var published = entry.PublishedDate?.UtcDateTime.ToString("yyyy-MM-dd") ?? "unknown";
             var age = entry.PublishedDate.HasValue
                 ? ((int)(DateTimeOffset.UtcNow - entry.PublishedDate.Value).TotalDays).ToString()
                 : "?";
-            var usingInstead = entry.ResolvedVersion ?? "(skipped)";
 
-            sb.AppendLine($"| {entry.PackageName} | {entry.IntegrationName} | {entry.OverriddenVersion} | {published} | {age} | {usingInstead} |");
+            sb.AppendLine($"| {entry.PackageName} | {entry.IntegrationName} | {entry.OverriddenVersion} | {published} | {age} |");
         }
 
         return sb.ToString();
@@ -84,6 +77,5 @@ public record CooldownEntry(
         string PackageName,
         string IntegrationName,
         string OverriddenVersion,
-        DateTimeOffset? PublishedDate,
-        string ResolvedVersion);
+        DateTimeOffset? PublishedDate);
 }