Skip to content

Commit 915cc90

Browse files
lbusselllbussell
authored
Add staging pipeline support to update-dependencies tool (#6473)
1 parent 96b8394 commit 915cc90

13 files changed

Lines changed: 437 additions & 27 deletions

‎eng/update-dependencies/BuildUpdaterService.cs‎

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ public async Task<int> UpdateFrom(Build build, CreatePullRequestOptions pullRequ
5757
string productCommitsJson = await _buildAssetService.GetAssetTextContentsAsync(productCommitsAsset);
5858
ProductCommits productCommits = ProductCommits.FromJson(productCommitsJson);
5959

60-
Version dockerfileVersion = ResolveMajorMinorVersion(productCommits.Sdk.Version);
60+
Version dockerfileVersion = VersionHelper.ResolveMajorMinorVersion(productCommits.Sdk.Version);
6161

6262
// Run old update-dependencies command using the resolved versions
6363
var updateDependencies = new SpecificCommand();
@@ -82,6 +82,7 @@ public async Task<int> UpdateFrom(Build build, CreatePullRequestOptions pullRequ
8282
AzdoOrganization = pullRequestOptions.AzdoOrganization,
8383
AzdoProject = pullRequestOptions.AzdoProject,
8484
AzdoRepo = pullRequestOptions.AzdoRepo,
85+
AzdoToken = pullRequestOptions.AzdoToken,
8586
VersionSourceName = pullRequestOptions.VersionSourceName,
8687
SourceBranch = pullRequestOptions.SourceBranch,
8788
TargetBranch = pullRequestOptions.TargetBranch,
@@ -90,17 +91,6 @@ public async Task<int> UpdateFrom(Build build, CreatePullRequestOptions pullRequ
9091
return await updateDependencies.ExecuteAsync(updateDependenciesOptions);
9192
}
9293

93-
private static Version ResolveMajorMinorVersion(string versionString)
94-
{
95-
string[] versionParts = versionString.Split('.');
96-
if (versionParts.Length < 2)
97-
{
98-
throw new InvalidOperationException($"Could not parse major-minor version from '{versionString}'.");
99-
}
100-
101-
return new Version(major: int.Parse(versionParts[0]), minor: int.Parse(versionParts[1]));
102-
}
103-
10494
private static bool IsVmrBuild(Build build)
10595
{
10696
string repo = build.GitHubRepository ?? build.AzureDevOpsRepository;

‎eng/update-dependencies/CreatePullRequestOptions.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
namespace Dotnet.Docker;
88

9-
public abstract class CreatePullRequestOptions
9+
public abstract record CreatePullRequestOptions
1010
{
1111
private string? _targetBranch = null;
1212

‎eng/update-dependencies/FromBuildOptions.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
namespace Dotnet.Docker;
88

9-
internal class FromBuildOptions : CreatePullRequestOptions, IOptions
9+
internal record FromBuildOptions : CreatePullRequestOptions, IOptions
1010
{
1111
public required int Id { get; init; }
1212

‎eng/update-dependencies/FromChannelOptions.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
namespace Dotnet.Docker;
88

9-
internal class FromChannelOptions : CreatePullRequestOptions, IOptions
9+
internal record FromChannelOptions : CreatePullRequestOptions, IOptions
1010
{
1111
public required int Channel { get; init; }
1212
public required string Repo { get; init; }

‎eng/update-dependencies/FromStagingPipelineCommand.cs‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.Threading.Tasks;
7+
using Microsoft.Extensions.Logging;
8+
using Dotnet.Docker.Model.Release;
9+
using System.Linq;
10+
using System.Text.RegularExpressions;
11+
12+
namespace Dotnet.Docker;
13+
14+
internal partial class FromStagingPipelineCommand(ILogger<FromStagingPipelineCommand> logger)
15+
: BaseCommand<FromStagingPipelineOptions>
16+
{
17+
private readonly ILogger<FromStagingPipelineCommand> _logger = logger;
18+
19+
public override async Task<int> ExecuteAsync(FromStagingPipelineOptions options)
20+
{
21+
if (options.Internal)
22+
{
23+
throw new NotImplementedException("Updating Dockerfiles for internal builds is not implemented yet.");
24+
}
25+
26+
ArgumentException.ThrowIfNullOrWhiteSpace(options.AzdoOrganization);
27+
ArgumentException.ThrowIfNullOrWhiteSpace(options.AzdoProject);
28+
29+
_logger.LogInformation(
30+
"Updating dependencies based on staging pipeline run ID {options.StagingPipelineRunId}",
31+
options.StagingPipelineRunId);
32+
33+
// Each pipeline run has a corresponding blob container named stage-${options.StagingPipelineRunId}.
34+
// Release metadata is stored in metadata/ReleaseManifest.json.
35+
// Release assets are stored individually under in assets/shipping/assets/[Sdk|Runtime|aspnetcore|...].
36+
37+
// Get release manifest from staging storage account
38+
var storageAccount = new StorageAccount(options.StagingStorageAccount);
39+
var releaseManifestJson = await storageAccount.DownloadTextAsync(
40+
containerName: $"stage-{options.StagingPipelineRunId}",
41+
blobPath: "metadata/ReleaseManifest.json");
42+
43+
var buildManifest = BuildManifest.FromJson(releaseManifestJson);
44+
var allAssets = buildManifest.AllAssets.ToList();
45+
46+
// Look through all the assets and get the version of the highest SDK feature band
47+
var sdkAssets = allAssets
48+
.Where(asset => SdkRegex.IsMatch(asset.Name))
49+
.ToList();
50+
var sdkVersions = sdkAssets.Select(asset => asset.Version);
51+
string highestSdkVersion = VersionHelper.GetHighestSdkVersion(sdkVersions);
52+
string dockerfileVersion = VersionHelper.ResolveMajorMinorVersion(highestSdkVersion).ToString();
53+
54+
string runtimeVersion = allAssets
55+
.Where(asset => RuntimeRegex.IsMatch(asset.Name))
56+
.Select(asset => asset.Version)
57+
.First();
58+
59+
string aspnetVersion = allAssets
60+
.Where(asset => AspNetCoreRegex.IsMatch(asset.Name))
61+
.Select(asset => asset.Version)
62+
.First();
63+
64+
_logger.LogInformation(
65+
"""
66+
Resolved .NET versions:
67+
.NET: {dockerfileVersion}
68+
- SDK: {highestSdkVersion}
69+
- Runtime: {runtimeVersion}
70+
- ASP.NET Core: {aspnetVersion}
71+
""",
72+
dockerfileVersion, highestSdkVersion, runtimeVersion, aspnetVersion);
73+
74+
// Run old update-dependencies command using the resolved versions
75+
var updateDependencies = new SpecificCommand();
76+
var updateDependenciesOptions = new SpecificCommandOptions()
77+
{
78+
DockerfileVersion = dockerfileVersion.ToString(),
79+
ProductVersions = new Dictionary<string, string?>()
80+
{
81+
// "dotnet" version is required. It sets the "dotnet|*|product-version"
82+
// variable which is used for runtime-deps, runtime, and aspnet tags.
83+
{ "dotnet", runtimeVersion },
84+
{ "runtime", runtimeVersion },
85+
{ "aspnet", aspnetVersion },
86+
{ "aspnet-composite", aspnetVersion },
87+
{ "sdk", highestSdkVersion },
88+
},
89+
90+
// Pass through all properties of CreatePullRequestOptions
91+
User = options.User,
92+
Email = options.Email,
93+
Password = options.Password,
94+
AzdoOrganization = options.AzdoOrganization,
95+
AzdoProject = options.AzdoProject,
96+
AzdoRepo = options.AzdoRepo,
97+
VersionSourceName = options.VersionSourceName,
98+
SourceBranch = options.SourceBranch,
99+
TargetBranch = options.TargetBranch,
100+
};
101+
102+
return await updateDependencies.ExecuteAsync(updateDependenciesOptions);
103+
}
104+
105+
// Examples:
106+
// - "Sdk/8.0.117-servicing.25269.14/dotnet-sdk-8.0.117-linux-x64.tar.gz"
107+
// - "Sdk/8.0.310-servicing.25113.14/dotnet-sdk-8.0.310-linux-musl-arm64.tar.gz"
108+
[GeneratedRegex(@"Sdk/.*/dotnet-sdk-.*-linux-x64.tar.gz$")]
109+
private static partial Regex SdkRegex { get; }
110+
111+
// "aspnetcore/Runtime/8.0.14-servicing.25112.21/aspnetcore-runtime-8.0.14-linux-x64.tar.gz",
112+
[GeneratedRegex(@"aspnetcore/Runtime/.*/aspnetcore-runtime-.*-linux-x64.tar.gz")]
113+
private static partial Regex AspNetCoreRegex { get; }
114+
115+
// "Runtime/8.0.14-servicing.25111.18/dotnet-runtime-8.0.14-linux-x64.tar.gz"
116+
[GeneratedRegex(@"Runtime/.*/dotnet-runtime-.*-linux-x64.tar.gz")]
117+
private static partial Regex RuntimeRegex { get; }
118+
}

‎eng/update-dependencies/FromStagingPipelineOptions.cs‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System.Collections.Generic;
5+
using System.CommandLine;
6+
7+
namespace Dotnet.Docker;
8+
9+
internal record FromStagingPipelineOptions : CreatePullRequestOptions, IOptions
10+
{
11+
/// <summary>
12+
/// The staging pipeline run ID to use as a source for the update.
13+
/// </summary>
14+
public required int StagingPipelineRunId { get; init; }
15+
16+
/// <summary>
17+
/// Whether or not to use the internal versions of the staged build.
18+
/// </summary>
19+
public bool Internal { get; init; } = false;
20+
21+
/// <summary>
22+
/// This Azure Storage Account will be used as a source for the update.
23+
/// This should be one of two storage accounts: dotnetstagetest or dotnetstage.
24+
/// </summary>
25+
public required string StagingStorageAccount { get; init; }
26+
27+
public static new List<Argument> Arguments { get; } =
28+
[
29+
new Argument<int>("staging-pipeline-run-id")
30+
{
31+
Arity = ArgumentArity.ExactlyOne,
32+
Description = "The staging pipeline run ID to use as a source for the update"
33+
},
34+
new Argument<string>("staging-storage-account")
35+
{
36+
Arity = ArgumentArity.ExactlyOne,
37+
Description = "The Azure Storage Account to use as a source for the update."
38+
+ " This should be one of two storage accounts: dotnetstagetest or dotnetstage."
39+
+ " For example: https://dotnetstagetest.blob.core.windows.net/"
40+
},
41+
..CreatePullRequestOptions.Arguments,
42+
];
43+
44+
public static new List<Option> Options { get; } =
45+
[
46+
new Option<bool>("--internal")
47+
{
48+
Description = "Whether or not to use the internal versions of the staged build. When not using an internal"
49+
+ " build, Dockerfiles will be updated as if the staged build has already been released. When using an"
50+
+ " internal build, Dockerfiles will be updated with internal download links and will only be buildable"
51+
+ " by using an internal Azure DevOps access token."
52+
},
53+
..CreatePullRequestOptions.Options,
54+
];
55+
}

‎eng/update-dependencies/Model/Release/BuildManifest.cs‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.Linq;
7+
using System.Text.Json;
8+
9+
namespace Dotnet.Docker.Model.Release;
10+
11+
// The models in this file are a subset of the BuildManifest models used by the .NET release infrastructure.
12+
// They should stay in sync.
13+
// https://dev.azure.com/dnceng/internal/_git/dotnet-release?path=/src/Microsoft.DotNet.Release/Microsoft.DotNet.ReleaseLib/Models/BuildManifest.cs
14+
15+
/// <summary>
16+
/// Represents a single release of .NET. This can contain multiple channels but typically spans only one major version
17+
/// of .NET. For example, a .NET 8.0 BuildManifest would contain all .NET 8.0.1XX/2XX/3XX SDKs, but no versions of .NET
18+
/// 9.0.
19+
/// </summary>
20+
public record BuildManifest
21+
{
22+
private static readonly JsonSerializerOptions s_jsonOptions = new()
23+
{
24+
WriteIndented = true,
25+
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
26+
};
27+
28+
public required Build[] Builds { get; init; }
29+
public required Asset[] ExtraAssets { get; init; }
30+
31+
public IEnumerable<Asset> AllAssets =>
32+
Builds
33+
.SelectMany(build => build.Assets)
34+
.Concat(ExtraAssets);
35+
36+
public static BuildManifest FromJson(string json) =>
37+
JsonSerializer.Deserialize<BuildManifest>(json, s_jsonOptions)
38+
?? throw new InvalidOperationException($"Failed to deserialize {nameof(BuildManifest)}: " + json);
39+
}
40+
41+
/// <summary>
42+
/// Represents a single, unique build on the .NET build asset registry (BAR). This is a unique build of a single commit
43+
/// of a single repository.
44+
/// </summary>
45+
public record Build
46+
{
47+
public required Uri Repo { get; init; }
48+
public required string Commit { get; init; }
49+
public required string Branch { get; init; }
50+
public required DateTimeOffset Produced { get; init; }
51+
public required string BuildNumber { get; init; }
52+
public required long BarBuildId { get; init; }
53+
public required Channel[] Channels { get; init; }
54+
public required Asset[] Assets { get; init; }
55+
}
56+
57+
/// <summary>
58+
/// Represents one single downloadable asset from a .NET build. Can be a binary, NuGet package, text file, or any other
59+
/// type of file that is produced by a .NET build.
60+
/// </summary>
61+
public record Asset
62+
{
63+
public required string Name { get; init; }
64+
public string? Origin { get; init; } = null;
65+
public bool? DotnetReleaseShipping { get; init; } = false;
66+
public required string Version { get; init; }
67+
public required bool NonShipping { get; init; }
68+
public required Uri Source { get; init; }
69+
public required long BarAssetId { get; init; }
70+
}
71+
72+
/// <summary>
73+
/// Represents a .NET build asset registry (BAR) channel. Channels group together builds of different repositories that
74+
/// must ship together. An example channel might be something like ".NET SDK 10.0.100-preview.1".
75+
/// </summary>
76+
/// <param name="Id">The ID of the channel</param>
77+
/// <param name="Name">The name of the channel</param>
78+
public record Channel(long Id, string Name);

‎eng/update-dependencies/Program.cs‎

Lines changed: 26 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
using Microsoft.DotNet.DarcLib;
1212
using Microsoft.Extensions.DependencyInjection;
1313
using Microsoft.Extensions.Hosting;
14+
using Microsoft.Extensions.Logging;
1415

1516
var rootCommand = new RootCommand()
1617
{
@@ -20,6 +21,9 @@
2021
FromChannelCommand.Create(
2122
name: "from-channel",
2223
description: "Update dependencies using the latest build from a channel"),
24+
FromStagingPipelineCommand.Create(
25+
name: "from-staging-pipeline",
26+
description: "Update dependencies using a specific staging pipeline run"),
2327
SpecificCommand.Create(
2428
name: "specific",
2529
description: "Update dependencies using specific product versions"),
@@ -41,18 +45,29 @@
4145

4246
return Host.CreateDefaultBuilder();
4347
},
44-
configureHost: host => host.ConfigureServices(services =>
45-
{
46-
services.AddSingleton<IBuildUpdaterService, BuildUpdaterService>();
47-
services.AddSingleton<IBasicBarClient>(_ =>
48-
new BarApiClient(null, null, disableInteractiveAuth: true));
49-
services.AddSingleton<IBuildAssetService, BuildAssetService>();
50-
services.AddSingleton<HttpClient>();
48+
configureHost: host => host
49+
.ConfigureLogging(logging =>
50+
{
51+
logging.ClearProviders();
52+
logging.AddSimpleConsole(options =>
53+
{
54+
options.IncludeScopes = true;
55+
options.SingleLine = true;
56+
});
57+
})
58+
.ConfigureServices(services =>
59+
{
60+
services.AddSingleton<IBuildUpdaterService, BuildUpdaterService>();
61+
services.AddSingleton<IBasicBarClient>(_ =>
62+
new BarApiClient(null, null, disableInteractiveAuth: true));
63+
services.AddSingleton<IBuildAssetService, BuildAssetService>();
64+
services.AddSingleton<HttpClient>();
5165

52-
FromBuildCommand.Register<FromBuildCommand>(services);
53-
FromChannelCommand.Register<FromChannelCommand>(services);
54-
SpecificCommand.Register<SpecificCommand>(services);
55-
})
66+
FromBuildCommand.Register<FromBuildCommand>(services);
67+
FromChannelCommand.Register<FromChannelCommand>(services);
68+
FromStagingPipelineCommand.Register<FromStagingPipelineCommand>(services);
69+
SpecificCommand.Register<SpecificCommand>(services);
70+
})
5671
);
5772

5873
return await config.InvokeAsync(args);

0 commit comments

Comments
 (0)