diff --git a/nuget/Dockerfile b/nuget/Dockerfile index 0427b2e766f..dde3618f5d2 100644 --- a/nuget/Dockerfile +++ b/nuget/Dockerfile @@ -87,5 +87,8 @@ RUN chmod +x $DEPENDABOT_HOME/dependabot-updater/bin/run # .NET install targeting packs RUN pwsh $DEPENDABOT_HOME/dependabot-updater/bin/install-targeting-packs.ps1 +# Extract the Dependabot gem version for runtime use +RUN sed -n 's/.*VERSION = "\(.*\)"/\1/p' $DEPENDABOT_HOME/common/lib/dependabot.rb > $DEPENDABOT_HOME/.dependabot-version + # Enable MSBuild operations with a shallow clone for repos that use the Nerdbank.GitVersioning package ENV NBGV_GitEngine=Disabled diff --git a/nuget/helpers/lib/NuGetUpdater/Directory.Packages.props b/nuget/helpers/lib/NuGetUpdater/Directory.Packages.props index f7f4157cb30..4151acdf36b 100644 --- a/nuget/helpers/lib/NuGetUpdater/Directory.Packages.props +++ b/nuget/helpers/lib/NuGetUpdater/Directory.Packages.props @@ -27,6 +27,7 @@ + diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTestHelper.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTestHelper.cs new file mode 100644 index 00000000000..3a64c57810a --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTestHelper.cs @@ -0,0 +1,84 @@ +using System.Text; +using System.Text.Json; + +using NuGetUpdater.Core.Run; +using NuGetUpdater.Core.Run.ApiModel; +using NuGetUpdater.Core.Test; +using NuGetUpdater.Core.Test.Update; + +using Xunit; + +namespace NuGetUpdater.Cli.Test; + +using TestFile = (string Path, string Content); + +internal static class EntryPointTestHelper +{ + internal static async Task RunAsync( + string commandName, + TestFile[] files, + Job job, + string[] expectedUrls, + MockNuGetPackage[]? packages = null, + string? repoContentsPath = null, + int expectedExitCode = 0) + { + using var tempDirectory = new TemporaryDirectory(); + + // write test files + foreach (var testFile in files) + { + var fullPath = Path.Join(tempDirectory.DirectoryPath, testFile.Path); + var directory = Path.GetDirectoryName(fullPath)!; + Directory.CreateDirectory(directory); + await File.WriteAllTextAsync(fullPath, testFile.Content); + } + + // write job file + var jobPath = Path.Combine(tempDirectory.DirectoryPath, "job.json"); + await File.WriteAllTextAsync(jobPath, JsonSerializer.Serialize(new { Job = job }, RunWorker.SerializerOptions)); + + // save packages + await UpdateWorkerTestBase.MockNuGetPackagesInDirectory(packages, tempDirectory.DirectoryPath); + + var actualUrls = new List(); + using var http = TestHttpServer.CreateTestStringServer((method, url) => + { + actualUrls.Add($"{method} {new Uri(url).PathAndQuery}"); + return (200, "ok"); + }); + var args = new List() + { + commandName, + "--job-path", + jobPath, + "--repo-contents-path", + repoContentsPath ?? tempDirectory.DirectoryPath, + "--api-url", + http.BaseUrl, + "--job-id", + "TEST-ID", + "--base-commit-sha", + "BASE-COMMIT-SHA" + }; + + var output = new StringBuilder(); + // redirect stdout + var originalOut = Console.Out; + Console.SetOut(new StringWriter(output)); + int result = -1; + try + { + result = await Program.Main(args.ToArray()); + } + catch + { + // restore stdout + Console.SetOut(originalOut); + throw; + } + + Assert.True(result == expectedExitCode, $"Expected exit code {expectedExitCode} but got {result}.\nSTDOUT:\n" + output.ToString()); + Assert.Equal(expectedUrls, actualUrls); + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Graph.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Graph.cs new file mode 100644 index 00000000000..fa1ab6cfcd8 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Graph.cs @@ -0,0 +1,65 @@ +using NuGetUpdater.Core.Run.ApiModel; +using NuGetUpdater.Core.Test; + +using Xunit; + +namespace NuGetUpdater.Cli.Test; + +using TestFile = (string Path, string Content); + +public partial class EntryPointTests +{ + public class Graph + { + [Fact] + public async Task Graph_Simple() + { + // verify we can pass command line arguments for graph command + await RunAsync( + packages: + [ + MockNuGetPackage.CreateSimplePackage("Some.Package", "1.0.0", "net8.0"), + ], + files: + [ + ("Directory.Build.props", ""), + ("Directory.Build.targets", ""), + ("Directory.Packages.props", """ + + + false + + + """), + ("src/project.csproj", """ + + + net8.0 + + + + + + """) + ], + job: new Job() + { + Source = new() + { + Provider = "github", + Repo = "test", + Directory = "src", + } + }, + expectedUrls: + [ + "POST /update_jobs/TEST-ID/create_dependency_submission", + "PATCH /update_jobs/TEST-ID/mark_as_processed", + ] + ); + } + + private static Task RunAsync(TestFile[] files, Job job, string[] expectedUrls, MockNuGetPackage[]? packages = null, string? repoContentsPath = null, int expectedExitCode = 0) + => EntryPointTestHelper.RunAsync("graph", files, job, expectedUrls, packages, repoContentsPath, expectedExitCode); + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Run.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Run.cs index d678c482ee0..fffb2781c42 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Run.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Run.cs @@ -143,65 +143,7 @@ await RunAsync( ); } - private static async Task RunAsync(TestFile[] files, Job job, string[] expectedUrls, MockNuGetPackage[]? packages = null, string? repoContentsPath = null, int expectedExitCode = 0) - { - using var tempDirectory = new TemporaryDirectory(); - - // write test files - foreach (var testFile in files) - { - var fullPath = Path.Join(tempDirectory.DirectoryPath, testFile.Path); - var directory = Path.GetDirectoryName(fullPath)!; - Directory.CreateDirectory(directory); - await File.WriteAllTextAsync(fullPath, testFile.Content); - } - - // write job file - var jobPath = Path.Combine(tempDirectory.DirectoryPath, "job.json"); - await File.WriteAllTextAsync(jobPath, JsonSerializer.Serialize(new { Job = job }, RunWorker.SerializerOptions)); - - // save packages - await UpdateWorkerTestBase.MockNuGetPackagesInDirectory(packages, tempDirectory.DirectoryPath); - - var actualUrls = new List(); - using var http = TestHttpServer.CreateTestStringServer((method, url) => - { - actualUrls.Add($"{method} {new Uri(url).PathAndQuery}"); - return (200, "ok"); - }); - var args = new List() - { - "run", - "--job-path", - jobPath, - "--repo-contents-path", - repoContentsPath ?? tempDirectory.DirectoryPath, - "--api-url", - http.BaseUrl, - "--job-id", - "TEST-ID", - "--base-commit-sha", - "BASE-COMMIT-SHA" - }; - - var output = new StringBuilder(); - // redirect stdout - var originalOut = Console.Out; - Console.SetOut(new StringWriter(output)); - int result = -1; - try - { - result = await Program.Main(args.ToArray()); - } - catch - { - // restore stdout - Console.SetOut(originalOut); - throw; - } - - Assert.True(result == expectedExitCode, $"Expected exit code {expectedExitCode} but got {result}.\nSTDOUT:\n" + output.ToString()); - Assert.Equal(expectedUrls, actualUrls); - } + private static Task RunAsync(TestFile[] files, Job job, string[] expectedUrls, MockNuGetPackage[]? packages = null, string? repoContentsPath = null, int expectedExitCode = 0) + => EntryPointTestHelper.RunAsync("run", files, job, expectedUrls, packages, repoContentsPath, expectedExitCode); } } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/CloneCommand.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/CloneCommand.cs index 243e870678e..94205f026b9 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/CloneCommand.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/CloneCommand.cs @@ -8,33 +8,24 @@ namespace NuGetUpdater.Cli.Commands; internal static class CloneCommand { - internal static readonly Option JobPathOption = new("--job-path") { Required = true }; - internal static readonly Option RepoContentsPathOption = new("--repo-contents-path") { Required = true }; - internal static readonly Option ApiUrlOption = new("--api-url") - { - Required = true, - CustomParser = (argumentResult) => Uri.TryCreate(argumentResult.Tokens.Single().Value, UriKind.Absolute, out var uri) ? uri : throw new ArgumentException("Invalid API URL format.") - }; - internal static readonly Option JobIdOption = new("--job-id") { Required = true }; - internal static Command GetCommand(Action setExitCode) { var command = new Command("clone", "Clones a repository in preparation for a dependabot job.") { - JobPathOption, - RepoContentsPathOption, - ApiUrlOption, - JobIdOption, + SharedOptions.JobPathOption, + SharedOptions.RepoContentsPathOption, + SharedOptions.ApiUrlOption, + SharedOptions.JobIdOption, }; command.TreatUnmatchedTokensAsErrors = true; command.SetAction(async (parseResult, cancellationToken) => { - var jobPath = parseResult.GetValue(JobPathOption); - var repoContentsPath = parseResult.GetValue(RepoContentsPathOption); - var apiUrl = parseResult.GetValue(ApiUrlOption); - var jobId = parseResult.GetValue(JobIdOption); + var jobPath = parseResult.GetValue(SharedOptions.JobPathOption); + var repoContentsPath = parseResult.GetValue(SharedOptions.RepoContentsPathOption); + var apiUrl = parseResult.GetValue(SharedOptions.ApiUrlOption); + var jobId = parseResult.GetValue(SharedOptions.JobIdOption); var apiHandler = new HttpApiHandler(apiUrl!.ToString(), jobId!); var logger = new OpenTelemetryLogger(); diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/GraphCommand.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/GraphCommand.cs new file mode 100644 index 00000000000..40da88f521a --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/GraphCommand.cs @@ -0,0 +1,47 @@ +using System.CommandLine; + +using NuGetUpdater.Core; +using NuGetUpdater.Core.Discover; +using NuGetUpdater.Core.Graph; +using NuGetUpdater.Core.Run; + +namespace NuGetUpdater.Cli.Commands; + +internal static class GraphCommand +{ + internal static Command GetCommand(Action setExitCode) + { + Command command = new("graph", "Generates a dependency graph for a repository.") + { + SharedOptions.JobPathOption, + SharedOptions.RepoContentsPathOption, + SharedOptions.CaseInsensitiveRepoContentsPathOption, + SharedOptions.ApiUrlOption, + SharedOptions.JobIdOption, + SharedOptions.BaseCommitShaOption + }; + + command.TreatUnmatchedTokensAsErrors = true; + + command.SetAction(async (parseResult, cancellationToken) => + { + var jobPath = parseResult.GetValue(SharedOptions.JobPathOption); + var repoContentsPath = parseResult.GetValue(SharedOptions.RepoContentsPathOption); + var caseInsensitiveRepoContentsPath = parseResult.GetValue(SharedOptions.CaseInsensitiveRepoContentsPathOption); + var apiUrl = parseResult.GetValue(SharedOptions.ApiUrlOption); + var jobId = parseResult.GetValue(SharedOptions.JobIdOption); + var baseCommitSha = parseResult.GetValue(SharedOptions.BaseCommitShaOption); + + var apiHandler = new HttpApiHandler(apiUrl!.ToString(), jobId!); + var (experimentsManager, _errorResult) = await ExperimentsManager.FromJobFileAsync(jobId!, jobPath!.FullName); + var logger = new OpenTelemetryLogger(); + var discoverWorker = new DiscoveryWorker(jobId!, experimentsManager, logger); + var worker = new GraphWorker(jobId!, apiHandler, discoverWorker, logger); + var result = await worker.RunAsync(jobPath!, repoContentsPath!, caseInsensitiveRepoContentsPath, baseCommitSha!); + setExitCode(result); + return 0; + }); + + return command; + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/RunCommand.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/RunCommand.cs index 4d6d76ecd87..4c990ad6ef4 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/RunCommand.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/RunCommand.cs @@ -9,39 +9,28 @@ namespace NuGetUpdater.Cli.Commands; internal static class RunCommand { - internal static readonly Option JobPathOption = new("--job-path") { Required = true }; - internal static readonly Option RepoContentsPathOption = new("--repo-contents-path") { Required = true }; - internal static readonly Option CaseInsensitiveRepoContentsPathOption = new("--case-insensitive-repo-contents-path") { Required = false }; - internal static readonly Option ApiUrlOption = new("--api-url") - { - Required = true, - CustomParser = (argumentResult) => Uri.TryCreate(argumentResult.Tokens.Single().Value, UriKind.Absolute, out var uri) ? uri : throw new ArgumentException("Invalid API URL format.") - }; - internal static readonly Option JobIdOption = new("--job-id") { Required = true }; - internal static readonly Option BaseCommitShaOption = new("--base-commit-sha") { Required = true }; - internal static Command GetCommand(Action setExitCode) { Command command = new("run", "Runs a full dependabot job.") { - JobPathOption, - RepoContentsPathOption, - CaseInsensitiveRepoContentsPathOption, - ApiUrlOption, - JobIdOption, - BaseCommitShaOption + SharedOptions.JobPathOption, + SharedOptions.RepoContentsPathOption, + SharedOptions.CaseInsensitiveRepoContentsPathOption, + SharedOptions.ApiUrlOption, + SharedOptions.JobIdOption, + SharedOptions.BaseCommitShaOption }; command.TreatUnmatchedTokensAsErrors = true; command.SetAction(async (parseResult, cancellationToken) => { - var jobPath = parseResult.GetValue(JobPathOption); - var repoContentsPath = parseResult.GetValue(RepoContentsPathOption); - var caseInsensitiveRepoContentsPath = parseResult.GetValue(CaseInsensitiveRepoContentsPathOption); - var apiUrl = parseResult.GetValue(ApiUrlOption); - var jobId = parseResult.GetValue(JobIdOption); - var baseCommitSha = parseResult.GetValue(BaseCommitShaOption); + var jobPath = parseResult.GetValue(SharedOptions.JobPathOption); + var repoContentsPath = parseResult.GetValue(SharedOptions.RepoContentsPathOption); + var caseInsensitiveRepoContentsPath = parseResult.GetValue(SharedOptions.CaseInsensitiveRepoContentsPathOption); + var apiUrl = parseResult.GetValue(SharedOptions.ApiUrlOption); + var jobId = parseResult.GetValue(SharedOptions.JobIdOption); + var baseCommitSha = parseResult.GetValue(SharedOptions.BaseCommitShaOption); var apiHandler = new HttpApiHandler(apiUrl!.ToString(), jobId!); var (experimentsManager, _errorResult) = await ExperimentsManager.FromJobFileAsync(jobId!, jobPath!.FullName); diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/SharedOptions.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/SharedOptions.cs new file mode 100644 index 00000000000..25e6952c57d --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/SharedOptions.cs @@ -0,0 +1,17 @@ +using System.CommandLine; + +namespace NuGetUpdater.Cli.Commands; + +internal static class SharedOptions +{ + internal static readonly Option JobPathOption = new("--job-path") { Required = true }; + internal static readonly Option RepoContentsPathOption = new("--repo-contents-path") { Required = true }; + internal static readonly Option CaseInsensitiveRepoContentsPathOption = new("--case-insensitive-repo-contents-path") { Required = false }; + internal static readonly Option ApiUrlOption = new("--api-url") + { + Required = true, + CustomParser = (argumentResult) => Uri.TryCreate(argumentResult.Tokens.Single().Value, UriKind.Absolute, out var uri) ? uri : throw new ArgumentException("Invalid API URL format.") + }; + internal static readonly Option JobIdOption = new("--job-id") { Required = true }; + internal static readonly Option BaseCommitShaOption = new("--base-commit-sha") { Required = true }; +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Program.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Program.cs index 1553fef7e27..dce5cb1a334 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Program.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Program.cs @@ -20,6 +20,7 @@ internal static async Task Main(string[] args) { CloneCommand.GetCommand(setExitCode), RunCommand.GetCommand(setExitCode), + GraphCommand.GetCommand(setExitCode), }; command.TreatUnmatchedTokensAsErrors = true; diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTestBase.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTestBase.cs index 8399993e3b3..2444522ebc7 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTestBase.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTestBase.cs @@ -117,6 +117,16 @@ internal static void ValidateProjectResults(ImmutableArray + + + v4.5 + + + + + + + packages\Some.Package.1.0.0\lib\net45\Some.Package.dll + True + + + + + """), + ("src/packages.config", """ + + + + """), + ], + expectedResult: new() + { + Path = "src", + Projects = + [ + new() + { + FilePath = "project.csproj", + TargetFrameworks = ["net45"], + Dependencies = + [ + new Dependency("Some.Package", "1.0.0", DependencyType.PackagesConfig, TargetFrameworks: ["net45"]), + ], + ReferencedProjectPaths = [], + ImportedFiles = [], + AdditionalFiles = [ + "packages.config", + ], + ExpectedDependencyGraph = new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + ["Some.Package/1.0.0"] = [], + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase), + } + ] + } + ); + } + + [Fact] + public async Task TestDependencyGraphIsPopulated() + { + await TestDiscoveryAsync( + packages: + [ + MockNuGetPackage.CreateSimplePackage("Some.Package", "1.0.0", "net8.0", + dependencyGroups: [(null, [("Some.Dependency", "2.0.0")])]), + MockNuGetPackage.CreateSimplePackage("Some.Dependency", "2.0.0", "net8.0"), + ], + workspacePath: "src", + files: + [ + ("src/project.csproj", """ + + + net8.0 + + + + + + """) + ], + expectedResult: new() + { + Path = "src", + Projects = + [ + new() + { + FilePath = "project.csproj", + TargetFrameworks = ["net8.0"], + Dependencies = + [ + new Dependency("Some.Dependency", "2.0.0", DependencyType.Unknown, TargetFrameworks: ["net8.0"], IsTopLevel: false), + new Dependency("Some.Package", "1.0.0", DependencyType.PackageReference, TargetFrameworks: ["net8.0"]), + ], + ReferencedProjectPaths = [], + ImportedFiles = [], + AdditionalFiles = [], + ExpectedDependencyGraph = new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + ["Some.Package/1.0.0"] = ["Some.Dependency/2.0.0"], + ["Some.Dependency/2.0.0"] = [], + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase), + } + ] + } + ); + } } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/ExpectedDiscoveryResults.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/ExpectedDiscoveryResults.cs index aec0fe010ca..e63cdff048c 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/ExpectedDiscoveryResults.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/ExpectedDiscoveryResults.cs @@ -24,6 +24,7 @@ public record ExpectedSdkProjectDiscoveryResult : ExpectedDependencyDiscoveryRes public string? ErrorDetails { get; init; } public PackageManagementKind? ExpectedPackageManagementKind { get; init; } = null; public string? ExpectedPackageManagementSpecialFileRelativePath { get; init; } = null; + public ImmutableDictionary>? ExpectedDependencyGraph { get; init; } } public record ExpectedDependencyDiscoveryResult : IDiscoveryResultWithDependencies diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Graph/GraphWorkerTests.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Graph/GraphWorkerTests.cs new file mode 100644 index 00000000000..40a98f0e7e6 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Graph/GraphWorkerTests.cs @@ -0,0 +1,465 @@ +using System.Collections.Immutable; + +using NuGetUpdater.Core.Discover; +using NuGetUpdater.Core.Graph; +using NuGetUpdater.Core.Run.ApiModel; + +using Xunit; + +namespace NuGetUpdater.Core.Test.Graph; + +public class GraphWorkerTests +{ + [Theory] + [MemberData(nameof(BuildDependencySubmissionTestData))] + public void BuildDependencySubmission_ConvertsDiscoveryResults( + WorkspaceDiscoveryResult discoveryResult, + Job job, + string baseCommitSha, + string repoRoot, + string directory, + string expectedStatus, + string? expectedReason, + int expectedManifestCount) + { + // Arrange + var logger = new TestLogger(); + var apiHandler = new TestApiHandler(); + var discoveryWorker = TestDiscoveryWorker.FromResults(); + var worker = new GraphWorker("test-job-id", apiHandler, discoveryWorker, logger); + + // Act + var result = worker.BuildDependencySubmission(discoveryResult, job, baseCommitSha, repoRoot, directory); + + // Assert + Assert.NotNull(result); + Assert.Equal(1, result.Version); + Assert.Equal(baseCommitSha, result.Sha); + Assert.Equal(expectedStatus, result.Metadata.Status); + Assert.Equal(expectedReason, result.Metadata.Reason); + Assert.Equal(expectedManifestCount, result.Manifests.Count); + Assert.Equal($"nuget::{directory}", result.Metadata.ScannedManifestPath); + } + + public static IEnumerable BuildDependencySubmissionTestData() + { + var job = new Job + { + Source = new JobSource + { + Provider = "github", + Repo = "test/repo", + Directory = "/", + Branch = "main" + } + }; + + // Test case 1: No projects discovered + yield return + [ + // discoveryResult + new WorkspaceDiscoveryResult + { + Path = "/src", + Projects = ImmutableArray.Empty + }, + // job + job, + // baseCommitSha + "abc123", + // repoRoot + "/repo", + // directory + "/src", + // expectedStatus + "skipped", + // expectedReason + "missing manifest files", + // expectedManifestCount + 0 + ]; + + // Test case 2: Project with no dependencies + yield return + [ + // discoveryResult + new WorkspaceDiscoveryResult + { + Path = "/src", + Projects = + [ + new ProjectDiscoveryResult + { + FilePath = "project.csproj", + Dependencies = ImmutableArray.Empty, + ImportedFiles = ImmutableArray.Empty, + AdditionalFiles = ImmutableArray.Empty + } + ] + }, + // job + job, + // baseCommitSha + "abc123", + // repoRoot + "/repo", + // directory + "/src", + // expectedStatus + "skipped", + // expectedReason + "missing manifest files", + // expectedManifestCount + 0 + ]; + + // Test case 3: Project with dependencies + yield return + [ + // discoveryResult + new WorkspaceDiscoveryResult + { + Path = "/src", + Projects = + [ + new ProjectDiscoveryResult + { + FilePath = "project.csproj", + Dependencies = + [ + new Dependency("Some.Package", "1.0.0", DependencyType.PackageReference, IsTopLevel: true), + new Dependency("Some.Dependency", "2.0.0", DependencyType.PackageReference, IsTopLevel: false) + ], + ImportedFiles = ImmutableArray.Empty, + AdditionalFiles = ImmutableArray.Empty + } + ] + }, + // job + job, + // baseCommitSha + "def456", + // repoRoot + "/repo", + // directory + "/src", + // expectedStatus + "ok", + // expectedReason + null, + // expectedManifestCount + 1 + ]; + + // Test case 4: Multiple projects with dependencies + yield return + [ + // discoveryResult + new WorkspaceDiscoveryResult + { + Path = "/src", + Projects = + [ + new ProjectDiscoveryResult + { + FilePath = "project1.csproj", + Dependencies = + [ + new Dependency("Some.Package", "1.0.0", DependencyType.PackageReference, IsTopLevel: true) + ], + ImportedFiles = ImmutableArray.Empty, + AdditionalFiles = ImmutableArray.Empty + }, + new ProjectDiscoveryResult + { + FilePath = "project2.csproj", + Dependencies = + [ + new Dependency("Some.Other.Package", "3.0.0", DependencyType.PackageReference, IsTopLevel: true) + ], + ImportedFiles = ImmutableArray.Empty, + AdditionalFiles = ImmutableArray.Empty + } + ] + }, + // job + job, + // baseCommitSha + "ghi789", + // repoRoot + "/repo", + // directory + "/src", + // expectedStatus + "ok", + // expectedReason + null, + // expectedManifestCount + 2 + ]; + + // Test case 5: Dependencies without versions (should be skipped) + yield return + [ + // discoveryResult + new WorkspaceDiscoveryResult + { + Path = "/src", + Projects = + [ + new ProjectDiscoveryResult + { + FilePath = "project.csproj", + Dependencies = + [ + new Dependency("SomePackage", null, DependencyType.PackageReference, IsTopLevel: true) + ], + ImportedFiles = ImmutableArray.Empty, + AdditionalFiles = ImmutableArray.Empty + } + ] + }, + // job + job, + // baseCommitSha + "jkl012", + // repoRoot + "/repo", + // directory + "/src", + // expectedStatus + "skipped", + // expectedReason + "missing manifest files", + // expectedManifestCount + 0 + ]; + } + + [Theory] + [MemberData(nameof(DependencyConversionTestData))] + public void BuildDependencySubmission_CorrectlyConvertsDependencies( + Dependency dependency, + string expectedPackageUrl, + string expectedRelationship, + string expectedScope) + { + // Arrange + var logger = new TestLogger(); + var apiHandler = new TestApiHandler(); + var discoveryWorker = TestDiscoveryWorker.FromResults(); + var worker = new GraphWorker("test-job-id", apiHandler, discoveryWorker, logger); + + var discoveryResult = new WorkspaceDiscoveryResult + { + Path = "/src", + Projects = + [ + new ProjectDiscoveryResult + { + FilePath = "project.csproj", + Dependencies = [dependency], + ImportedFiles = ImmutableArray.Empty, + AdditionalFiles = ImmutableArray.Empty + } + ] + }; + + var job = new Job + { + Source = new JobSource + { + Provider = "github", + Repo = "test/repo", + Directory = "/" + } + }; + + // Act + var result = worker.BuildDependencySubmission(discoveryResult, job, "abc123", "/repo", "/src"); + + // Assert + Assert.NotNull(result); + Assert.Single(result.Manifests); + + var manifest = result.Manifests.Values.First(); + Assert.Contains(expectedPackageUrl, manifest.Resolved.Keys); + + var resolvedDep = manifest.Resolved[expectedPackageUrl]; + Assert.Equal(expectedPackageUrl, resolvedDep.PackageUrl); + Assert.Equal(expectedRelationship, resolvedDep.Relationship); + Assert.Equal(expectedScope, resolvedDep.Scope); + Assert.Empty(resolvedDep.Dependencies); + } + + public static IEnumerable DependencyConversionTestData() + { + // Direct runtime dependency + yield return + [ + // dependency + new Dependency("Some.Package", "1.0.0", DependencyType.PackageReference, IsTopLevel: true), + // expectedPackageUrl + "pkg:nuget/Some.Package@1.0.0", + // expectedRelationship + "direct", + // expectedScope + "runtime" + ]; + + // Indirect runtime dependency + yield return + [ + // dependency + new Dependency("Some.Dependency", "2.0.0", DependencyType.PackageReference, IsTopLevel: false), + // expectedPackageUrl + "pkg:nuget/Some.Dependency@2.0.0", + // expectedRelationship + "indirect", + // expectedScope + "runtime" + ]; + + // PackageVersion dependency + yield return + [ + // dependency + new Dependency("Some.Other.Package", "3.0.0", DependencyType.PackageVersion, IsTopLevel: true), + // expectedPackageUrl + "pkg:nuget/Some.Other.Package@3.0.0", + // expectedRelationship + "direct", + // expectedScope + "runtime" + ]; + } + + [Fact] + public void BuildDependencySubmission_PopulatesDependenciesFromGraph() + { + // Arrange + var logger = new TestLogger(); + var apiHandler = new TestApiHandler(); + var discoveryWorker = TestDiscoveryWorker.FromResults(); + var worker = new GraphWorker("test-job-id", apiHandler, discoveryWorker, logger); + + var dependencyGraph = new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + ["Some.Package/1.0.0"] = ["Some.Dependency/2.0.0", "Some.Other.Dependency/3.0.0"], + ["Some.Dependency/2.0.0"] = [], + ["Some.Other.Dependency/3.0.0"] = [], + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + + var discoveryResult = new WorkspaceDiscoveryResult + { + Path = "/src", + Projects = + [ + new ProjectDiscoveryResult + { + FilePath = "project.csproj", + Dependencies = + [ + new Dependency("Some.Package", "1.0.0", DependencyType.PackageReference, IsTopLevel: true), + new Dependency("Some.Dependency", "2.0.0", DependencyType.PackageReference, IsTopLevel: false), + new Dependency("Some.Other.Dependency", "3.0.0", DependencyType.PackageReference, IsTopLevel: false), + ], + ImportedFiles = ImmutableArray.Empty, + AdditionalFiles = ImmutableArray.Empty, + DependencyGraph = dependencyGraph, + } + ] + }; + + var job = new Job + { + Source = new JobSource + { + Provider = "github", + Repo = "test/repo", + Directory = "/" + } + }; + + // Act + var result = worker.BuildDependencySubmission(discoveryResult, job, "abc123", "/repo", "/src"); + + // Assert + var manifest = Assert.Single(result.Manifests.Values); + + // Some.Package should have two dependencies + var somePackage = manifest.Resolved["pkg:nuget/Some.Package@1.0.0"]; + Assert.Equal("direct", somePackage.Relationship); + Assert.Equal(2, somePackage.Dependencies.Length); + Assert.Contains("pkg:nuget/Some.Dependency@2.0.0", somePackage.Dependencies); + Assert.Contains("pkg:nuget/Some.Other.Dependency@3.0.0", somePackage.Dependencies); + + // Leaf packages should have no dependencies + var someDependency = manifest.Resolved["pkg:nuget/Some.Dependency@2.0.0"]; + Assert.Empty(someDependency.Dependencies); + + var someOtherDependency = manifest.Resolved["pkg:nuget/Some.Other.Dependency@3.0.0"]; + Assert.Empty(someOtherDependency.Dependencies); + } + + [Fact] + public void BuildDependencySubmission_OmitsGraphDependenciesNotInDiscoveredList() + { + // Arrange + var logger = new TestLogger(); + var apiHandler = new TestApiHandler(); + var discoveryWorker = TestDiscoveryWorker.FromResults(); + var worker = new GraphWorker("test-job-id", apiHandler, discoveryWorker, logger); + + var dependencyGraph = new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + // Graph says Some.Package depends on both, but Undiscovered.Package is not discovered + ["Some.Package/1.0.0"] = ["Some.Dependency/2.0.0", "Undiscovered.Package/4.0.0"], + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + + var discoveryResult = new WorkspaceDiscoveryResult + { + Path = "/src", + Projects = + [ + new ProjectDiscoveryResult + { + FilePath = "project.csproj", + Dependencies = + [ + new Dependency("Some.Package", "1.0.0", DependencyType.PackageReference, IsTopLevel: true), + new Dependency("Some.Dependency", "2.0.0", DependencyType.PackageReference, IsTopLevel: false), + ], + ImportedFiles = ImmutableArray.Empty, + AdditionalFiles = ImmutableArray.Empty, + DependencyGraph = dependencyGraph, + } + ] + }; + + var job = new Job + { + Source = new JobSource + { + Provider = "github", + Repo = "test/repo", + Directory = "/" + } + }; + + // Act + var result = worker.BuildDependencySubmission(discoveryResult, job, "abc123", "/repo", "/src"); + + // Assert + var manifest = Assert.Single(result.Manifests.Values); + var somePackage = manifest.Resolved["pkg:nuget/Some.Package@1.0.0"]; + + // Only Some.Dependency should be listed; Undiscovered.Package is not in the discovered dependencies + Assert.Single(somePackage.Dependencies); + Assert.Contains("pkg:nuget/Some.Dependency@2.0.0", somePackage.Dependencies); + + // Some.Dependency has no transitive dependencies + var someDependency = manifest.Resolved["pkg:nuget/Some.Dependency@2.0.0"]; + Assert.Empty(someDependency.Dependencies); + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/MessageReportTests.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/MessageReportTests.cs index 0f6f9488472..6dbf7030662 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/MessageReportTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/MessageReportTests.cs @@ -58,6 +58,41 @@ public static IEnumerable MessageBaseTestData() """ ]; + yield return + [ + // message + new CreateDependencySubmission() + { + Version = 0, + Sha = "unused", + Ref = "unused", + Job = new CreateDependencySubmission.SubmissionJob() { Correlator = "unused", Id = "unused" }, + Detector = new CreateDependencySubmission.SubmissionDetector() { Name = "unused", Version = "unused", Url = "unused" }, + Manifests = new Dictionary() + { + ["manifest1"] = new CreateDependencySubmission.Manifest() + { + Name = "manifest1", + File = new CreateDependencySubmission.ManifestFile() { SourceLocation = "unused" }, + Metadata = new CreateDependencySubmission.ManifestMetadata() { Ecosystem = "unused" }, + Resolved = new Dictionary(), + }, + }, + Metadata = new CreateDependencySubmission.SubmissionMetadata() + { + Status = "succeeded", + ScannedManifestPath = "path/to/manifest", + }, + }, + // expected + """ + CreateDependencySubmission + - Status: succeeded + - Scanned: path/to/manifest + - Manifests: 1 + """ + ]; + yield return [ // message diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/SerializationTests.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/SerializationTests.cs index 6bbb36a2b55..9b0ec940d25 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/SerializationTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/SerializationTests.cs @@ -978,4 +978,104 @@ public void SerializeRealUnknownErrorWithInnerException() } ]; } + + [Fact] + public void SerializeCreateDependencySubmission() + { + var submission = new CreateDependencySubmission() + { + Version = 1, + Sha = "TEST-SHA", + Ref = "refs/heads/main", + Job = new CreateDependencySubmission.SubmissionJob() + { + Correlator = "dependabot-nuget-MyProject", + Id = "cli" + }, + Detector = new CreateDependencySubmission.SubmissionDetector() + { + Name = "dependabot", + Version = "0.372.0", + Url = "https://github.com/dependabot/dependabot-core" + }, + Manifests = new Dictionary() + { + ["/src/MyProject.csproj"] = new CreateDependencySubmission.Manifest() + { + Name = "/src/MyProject.csproj", + File = new CreateDependencySubmission.ManifestFile() + { + SourceLocation = "src/MyProject.csproj" + }, + Metadata = new CreateDependencySubmission.ManifestMetadata() + { + Ecosystem = "nuget" + }, + Resolved = new Dictionary() + { + ["pkg:nuget/Some.Package@1.0.0"] = new CreateDependencySubmission.ResolvedDependency() + { + PackageUrl = "pkg:nuget/Some.Package@1.0.0", + Relationship = "direct", + Scope = "runtime", + Dependencies = ["pkg:nuget/Some.Dependency@2.0.0"] + }, + ["pkg:nuget/Some.Dependency@2.0.0"] = new CreateDependencySubmission.ResolvedDependency() + { + PackageUrl = "pkg:nuget/Some.Dependency@2.0.0", + Relationship = "indirect", + Scope = "runtime", + Dependencies = [] + } + } + } + }, + Metadata = new CreateDependencySubmission.SubmissionMetadata() + { + Status = "ok", + ScannedManifestPath = "nuget::/src" + } + }; + + var actual = HttpApiHandler.Serialize(submission); + var expected = """ + {"data":{"version":1,"sha":"TEST-SHA","ref":"refs/heads/main","job":{"correlator":"dependabot-nuget-MyProject","id":"cli"},"detector":{"name":"dependabot","version":"0.372.0","url":"https://github.com/dependabot/dependabot-core"},"manifests":{"/src/MyProject.csproj":{"name":"/src/MyProject.csproj","file":{"source_location":"src/MyProject.csproj"},"metadata":{"ecosystem":"nuget"},"resolved":{"pkg:nuget/Some.Package@1.0.0":{"package_url":"pkg:nuget/Some.Package@1.0.0","relationship":"direct","scope":"runtime","dependencies":["pkg:nuget/Some.Dependency@2.0.0"]},"pkg:nuget/Some.Dependency@2.0.0":{"package_url":"pkg:nuget/Some.Dependency@2.0.0","relationship":"indirect","scope":"runtime","dependencies":[]}}}},"metadata":{"status":"ok","scanned_manifest_path":"nuget::/src"}}} + """; + Assert.Equal(expected, actual); + } + + [Fact] + public void SerializeCreateDependencySubmission_Skipped() + { + var submission = new CreateDependencySubmission() + { + Version = 1, + Sha = "TEST-SHA", + Ref = "refs/heads/main", + Job = new CreateDependencySubmission.SubmissionJob() + { + Correlator = "dependabot-nuget-casing", + Id = "cli" + }, + Detector = new CreateDependencySubmission.SubmissionDetector() + { + Name = "dependabot", + Version = "0.372.0", + Url = "https://github.com/dependabot/dependabot-core" + }, + Manifests = new Dictionary(), + Metadata = new CreateDependencySubmission.SubmissionMetadata() + { + Status = "skipped", + ScannedManifestPath = "nuget::/casing", + Reason = "missing manifest files" + } + }; + + var actual = HttpApiHandler.Serialize(submission); + var expected = """ + {"data":{"version":1,"sha":"TEST-SHA","ref":"refs/heads/main","job":{"correlator":"dependabot-nuget-casing","id":"cli"},"detector":{"name":"dependabot","version":"0.372.0","url":"https://github.com/dependabot/dependabot-core"},"manifests":{},"metadata":{"status":"skipped","scanned_manifest_path":"nuget::/casing","reason":"missing manifest files"}}} + """; + Assert.Equal(expected, actual); + } } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/DiscoveryWorker.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/DiscoveryWorker.cs index 5f34adac888..c90438e0381 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/DiscoveryWorker.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/DiscoveryWorker.cs @@ -398,6 +398,13 @@ private async Task> RunForProjectPathsAsy if (packagesConfigResult is not null) { var relativeProjectPath = Path.GetRelativePath(workspacePath, expandedProject).NormalizePathToUnix(); + var dependencyGraph = packagesConfigResult.Dependencies + .Where(d => !string.IsNullOrEmpty(d.Version)) + .OrderBy(d => d.Name, StringComparer.OrdinalIgnoreCase) + .ToImmutableDictionary( + d => $"{d.Name}/{d.Version}", + _ => ImmutableArray.Empty, + StringComparer.OrdinalIgnoreCase); results[relativeProjectPath] = new ProjectDiscoveryResult() { FilePath = relativeProjectPath, @@ -405,6 +412,7 @@ private async Task> RunForProjectPathsAsy TargetFrameworks = packagesConfigResult.TargetFrameworks, ImportedFiles = [], // no imported files resolved for packages.config scenarios AdditionalFiles = packagesConfigResult.AdditionalFiles, + DependencyGraph = dependencyGraph, }; } } @@ -520,6 +528,7 @@ internal static ProjectDiscoveryResult MergeProjectDiscovery(ProjectDiscoveryRes PackageManagementKind = (PackageManagementKind)Math.Max((int)result1.PackageManagementKind, (int)result2.PackageManagementKind), PackageManagementSpecialFileRelativePath = result1.PackageManagementSpecialFileRelativePath ?? result2.PackageManagementSpecialFileRelativePath, HasNoWarnNU1701 = result1.HasNoWarnNU1701 || result2.HasNoWarnNU1701, + DependencyGraph = MergeDependencyGraphs(result1.DependencyGraph, result2.DependencyGraph), }; return mergedResult; } @@ -546,6 +555,29 @@ internal ImmutableArray FilterProjectsInSubmodules( return [.. filtered]; } + private static ImmutableDictionary> MergeDependencyGraphs( + ImmutableDictionary> graph1, + ImmutableDictionary> graph2) + { + var merged = graph1.ToDictionary(kvp => kvp.Key, kvp => kvp.Value, StringComparer.OrdinalIgnoreCase); + foreach (var kvp in graph2) + { + if (merged.TryGetValue(kvp.Key, out var existing)) + { + merged[kvp.Key] = existing + .Union(kvp.Value, StringComparer.OrdinalIgnoreCase) + .OrderBy(n => n, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + } + else + { + merged[kvp.Key] = kvp.Value; + } + } + + return merged.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + } + internal static async Task WriteResultsAsync(string repoRootPath, string outputPath, WorkspaceDiscoveryResult result) { var resultPath = Path.IsPathRooted(outputPath) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/ProjectDiscoveryResult.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/ProjectDiscoveryResult.cs index 28ef30c05cc..bfb2b6dbf98 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/ProjectDiscoveryResult.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/ProjectDiscoveryResult.cs @@ -17,4 +17,8 @@ public record ProjectDiscoveryResult : IDiscoveryResultWithDependencies public required ImmutableArray AdditionalFiles { get; init; } public required ImmutableArray Dependencies { get; init; } public bool HasNoWarnNU1701 { get; init; } = false; + /// + /// Maps each package (keyed as "Name/Version") to its direct dependency package names, as extracted from project.assets.json. + /// + public ImmutableDictionary> DependencyGraph { get; init; } = ImmutableDictionary>.Empty; } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs index fdb8097d6da..930d9866234 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs @@ -635,6 +635,60 @@ async ThreadingTask EnsurePackagesForFileAsync(string fullFilePath) .ThenBy(d => d.Version) .ToImmutableArray(); + // extract dependency graph from project.assets.json + var dependencyGraphBuilder = new Dictionary>(StringComparer.OrdinalIgnoreCase); + if (assetsJson.Value is { } assetsForGraph && + assetsForGraph.TryGetProperty("targets", out var graphTargets)) + { + foreach (var tfmObject in graphTargets.EnumerateObject()) + { + // Build a complete lookup of package name -> resolved version for this TFM. + // This must be a separate pass because the dependency resolution below needs to + // look up any package by name, including ones that appear later in the enumeration. + var resolvedVersions = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var packageObject in tfmObject.Value.EnumerateObject()) + { + var parts = packageObject.Name.Split('/'); + if (parts.Length == 2) + { + resolvedVersions[parts[0]] = parts[1]; + } + } + + // Now that all resolved versions are known, build the dependency graph edges. + foreach (var packageObject in tfmObject.Value.EnumerateObject()) + { + var parts = packageObject.Name.Split('/'); + if (parts.Length == 2) + { + var packageName = parts[0]; + var packageVersion = parts[1]; + var graphKey = $"{packageName}/{packageVersion}"; + var depEntries = packageObject.Value.TryGetProperty("dependencies", out var deps) + ? deps.EnumerateObject() + .Where(d => resolvedVersions.ContainsKey(d.Name)) + .Select(d => $"{d.Name}/{resolvedVersions[d.Name]}") + : []; + if (!dependencyGraphBuilder.TryGetValue(graphKey, out var existing)) + { + dependencyGraphBuilder[graphKey] = depEntries + .OrderBy(n => n, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + } + else + { + dependencyGraphBuilder[graphKey] = existing + .Union(depEntries, StringComparer.OrdinalIgnoreCase) + .OrderBy(n => n, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + } + } + } + } + } + + var dependencyGraph = dependencyGraphBuilder.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + // other values var projectProperties = resolvedProperties[projectPath]; var referenced = referencedProjects.GetOrAdd(projectPath, () => new(PathComparer.Instance)) @@ -702,6 +756,7 @@ PackageManagementKind.CentralPackageManagement or PackageManagementKind = packageManagementKind, PackageManagementSpecialFileRelativePath = packageManagementFile, HasNoWarnNU1701 = hasNoWarnNU1701, + DependencyGraph = dependencyGraph, }; projectDiscoveryResults.Add(projectDiscoveryResult); } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Graph/GraphWorker.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Graph/GraphWorker.cs new file mode 100644 index 00000000000..b9e7022c008 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Graph/GraphWorker.cs @@ -0,0 +1,256 @@ +using System.Security.Cryptography; +using System.Text; + +using NuGetUpdater.Core.Discover; +using NuGetUpdater.Core.Run; +using NuGetUpdater.Core.Run.ApiModel; + +using PackageUrl; + +namespace NuGetUpdater.Core.Graph; + +public class GraphWorker : IGraphWorker +{ + private readonly string _jobId; + private readonly IApiHandler _apiHandler; + private readonly IDiscoveryWorker _discoveryWorker; + private readonly ILogger _logger; + + public GraphWorker(string jobId, IApiHandler apiHandler, IDiscoveryWorker discoveryWorker, ILogger logger) + { + _jobId = jobId; + _apiHandler = apiHandler; + _discoveryWorker = discoveryWorker; + _logger = logger; + } + + public async Task RunAsync(FileInfo jobFilePath, DirectoryInfo repoContentsPath, DirectoryInfo? caseInsensitiveRepoContentsPath, string baseCommitSha) + { + // Deserialize the job file + var jobFileContent = await File.ReadAllTextAsync(jobFilePath.FullName); + var jobWrapper = RunWorker.Deserialize(jobFileContent); + var job = jobWrapper.Job; + + // Use the case-insensitive repo contents path if provided, otherwise use the original + var actualRepoContentsPath = caseInsensitiveRepoContentsPath ?? repoContentsPath; + + int result = 0; + JobErrorBase? error = null; + + try + { + // Process each directory in the job + foreach (var directory in job.GetAllDirectories(actualRepoContentsPath.FullName)) + { + _logger.Info($"Running dependency discovery for directory: {directory}"); + + // Run dependency discovery + var discoveryResult = await _discoveryWorker.RunAsync(actualRepoContentsPath.FullName, directory); + + // Check for discovery errors + if (discoveryResult.Error is not null) + { + _logger.Error($"Discovery error in {directory}: {discoveryResult.Error.GetReport()}"); + await _apiHandler.RecordUpdateJobError(discoveryResult.Error, _logger); + error = discoveryResult.Error; + result = 1; + continue; + } + + // Build the dependency submission from the discovery results + var submission = BuildDependencySubmission( + discoveryResult, + job, + baseCommitSha, + repoContentsPath.FullName, + directory); + + // Submit the dependency graph + _logger.Info($"Submitting dependency graph for {directory}"); + await _apiHandler.CreateDependencySubmission(submission); + } + } + catch (Exception ex) + { + error = JobErrorBase.ErrorFromException(ex, _jobId, actualRepoContentsPath.FullName); + await _apiHandler.RecordUpdateJobError(error, _logger); + result = 1; + } + + // Mark the job as processed + await _apiHandler.MarkAsProcessed(new(baseCommitSha)); + + return result; + } + + internal CreateDependencySubmission BuildDependencySubmission( + WorkspaceDiscoveryResult discoveryResult, + Job job, + string baseCommitSha, + string repoRoot, + string directory) + { + var manifests = new Dictionary(); + string status = "ok"; + string? reason = null; + + // If no projects were discovered, return a skipped submission + if (discoveryResult.Projects.IsEmpty) + { + _logger.Info($"No projects discovered in {directory}"); + status = "skipped"; + reason = "missing manifest files"; + } + else + { + // Process each project + foreach (var project in discoveryResult.Projects) + { + var combinedPath = PathHelper.JoinPath(discoveryResult.Path, project.FilePath).NormalizePathToUnix(); + var manifestPath = $"/{combinedPath}"; + var sourceLocation = combinedPath; + + var resolvedDependencies = new Dictionary(); + + // Build a set of discovered dependency keys for filtering graph edges + var discoveredKeys = project.Dependencies + .Where(d => !string.IsNullOrEmpty(d.Version)) + .Select(d => $"{d.Name}/{d.Version}") + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + // Process each dependency + foreach (var dependency in project.Dependencies) + { + if (string.IsNullOrEmpty(dependency.Version)) + { + continue; // Skip dependencies without versions + } + + var packageUrl = new PackageURL("nuget", null, dependency.Name, dependency.Version, null, null).ToString(); + + // Determine relationship (direct vs indirect) + var relationship = dependency.IsTopLevel ? "direct" : "indirect"; + + // Determine scope (runtime, development, etc.) + var scope = dependency.Type switch + { + DependencyType.PackageReference => "runtime", + DependencyType.PackageVersion => "runtime", + _ => "runtime" + }; + + // Populate direct dependencies from the dependency graph (values are "Name/Version") + var graphKey = $"{dependency.Name}/{dependency.Version}"; + var directDeps = project.DependencyGraph.TryGetValue(graphKey, out var depEntries) + ? depEntries + .Where(entry => discoveredKeys.Contains(entry)) + .Select(entry => entry.Split('/')) + .Where(parts => parts.Length == 2) + .Select(parts => new PackageURL("nuget", null, parts[0], parts[1], null, null).ToString()) + .ToArray() + : []; + + resolvedDependencies[packageUrl] = new CreateDependencySubmission.ResolvedDependency + { + PackageUrl = packageUrl, + Relationship = relationship, + Scope = scope, + Dependencies = directDeps + }; + } + + // Only add manifest if it has dependencies + if (resolvedDependencies.Count > 0) + { + manifests[manifestPath] = new CreateDependencySubmission.Manifest + { + Name = manifestPath, + File = new CreateDependencySubmission.ManifestFile + { + SourceLocation = sourceLocation + }, + Metadata = new CreateDependencySubmission.ManifestMetadata + { + Ecosystem = "nuget" + }, + Resolved = resolvedDependencies + }; + } + } + + // If no manifests with dependencies, mark as skipped + if (manifests.Count == 0) + { + _logger.Info($"No dependencies found in {directory}"); + status = "skipped"; + reason = "missing manifest files"; + } + } + + // Build the submission (always return a submission, even if skipped) + return new CreateDependencySubmission + { + Version = 1, + Sha = baseCommitSha, + Ref = GetSymbolicRef(job.Source.Branch), + Job = new CreateDependencySubmission.SubmissionJob + { + Correlator = GetCorrelator(directory), + Id = _jobId + }, + Detector = new CreateDependencySubmission.SubmissionDetector + { + Name = "dependabot", + Version = GetDetectorVersion(), + Url = "https://github.com/dependabot/dependabot-core" + }, + Manifests = manifests, + Metadata = new CreateDependencySubmission.SubmissionMetadata + { + Status = status, + ScannedManifestPath = $"nuget::{directory}", + Reason = reason + } + }; + } + + internal static string GetSymbolicRef(string? branch) + { + branch = (branch ?? "main").TrimStart('/'); + if (branch.StartsWith("refs/", StringComparison.OrdinalIgnoreCase)) + { + return branch; + } + + return $"refs/heads/{branch}"; + } + + internal static string GetCorrelator(string directory) + { + var sanitized = directory.TrimStart('/').Replace("/", "-").TrimStart('-'); + if (Encoding.UTF8.GetByteCount(sanitized) > 32) + { + sanitized = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(sanitized))); + } + + return string.IsNullOrEmpty(sanitized) ? "dependabot-nuget" : $"dependabot-nuget-{sanitized}"; + } + + internal static string GetDetectorVersion() + { + var version = Environment.GetEnvironmentVariable("DEPENDABOT_VERSION"); + if (string.IsNullOrWhiteSpace(version)) + { + version = "development"; + } + + var sha = Environment.GetEnvironmentVariable("DEPENDABOT_UPDATER_SHA"); + if (!string.IsNullOrEmpty(sha)) + { + version = $"{version}-{sha}"; + } + + return version; + } +} + diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/IGraphWorker.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/IGraphWorker.cs new file mode 100644 index 00000000000..ef26e142856 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/IGraphWorker.cs @@ -0,0 +1,6 @@ +namespace NuGetUpdater.Core; + +public interface IGraphWorker +{ + Task RunAsync(FileInfo jobFilePath, DirectoryInfo repoContentsPath, DirectoryInfo? caseInsensitiveRepoContentsPath, string baseCommitSha); +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj index c06ed563843..e59e6abad4a 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj @@ -28,6 +28,7 @@ + diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/CreateDependencySubmission.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/CreateDependencySubmission.cs new file mode 100644 index 00000000000..21a1207eb1f --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/CreateDependencySubmission.cs @@ -0,0 +1,92 @@ +using System.Text; +using System.Text.Json.Serialization; + +namespace NuGetUpdater.Core.Run.ApiModel; + +public sealed record CreateDependencySubmission : MessageBase +{ + public required int Version { get; init; } + + public required string Sha { get; init; } + + public required string Ref { get; init; } + + public required SubmissionJob Job { get; init; } + + public required SubmissionDetector Detector { get; init; } + + public required Dictionary Manifests { get; init; } + + public required SubmissionMetadata Metadata { get; init; } + + public override string GetReport() + { + var report = new StringBuilder(); + report.AppendLine(nameof(CreateDependencySubmission)); + report.AppendLine($"- Status: {Metadata.Status}"); + report.AppendLine($"- Scanned: {Metadata.ScannedManifestPath}"); + report.AppendLine($"- Manifests: {Manifests.Count}"); + return report.ToString().Trim(); + } + + public sealed record SubmissionJob + { + public required string Correlator { get; init; } + + public required string Id { get; init; } + } + + public sealed record SubmissionDetector + { + public required string Name { get; init; } + + public required string Version { get; init; } + + public required string Url { get; init; } + } + + public sealed record Manifest + { + public required string Name { get; init; } + + public required ManifestFile File { get; init; } + + public required ManifestMetadata Metadata { get; init; } + + public required Dictionary Resolved { get; init; } + } + + public sealed record ManifestFile + { + [JsonPropertyName("source_location")] + public required string SourceLocation { get; init; } + } + + public sealed record ManifestMetadata + { + public required string Ecosystem { get; init; } + } + + public sealed record ResolvedDependency + { + [JsonPropertyName("package_url")] + public required string PackageUrl { get; init; } + + public required string Relationship { get; init; } + + public required string Scope { get; init; } + + public required string[] Dependencies { get; init; } + } + + public sealed record SubmissionMetadata + { + public required string Status { get; init; } + + [JsonPropertyName("scanned_manifest_path")] + public required string ScannedManifestPath { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Reason { get; init; } + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/IApiHandler.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/IApiHandler.cs index 70f7037fa93..6bda4a7395a 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/IApiHandler.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/IApiHandler.cs @@ -46,6 +46,7 @@ public static async Task UpdateDependencyList(this IApiHandler handler, UpdatedD public static Task ClosePullRequest(this IApiHandler handler, ClosePullRequest closePullRequest) => handler.PostAsJson("close_pull_request", closePullRequest); public static Task UpdatePullRequest(this IApiHandler handler, UpdatePullRequest updatePullRequest) => handler.PostAsJson("update_pull_request", updatePullRequest); public static Task MarkAsProcessed(this IApiHandler handler, MarkAsProcessed markAsProcessed) => handler.PatchAsJson("mark_as_processed", markAsProcessed); + public static Task CreateDependencySubmission(this IApiHandler handler, CreateDependencySubmission createDependencySubmission) => handler.PostAsJson("create_dependency_submission", createDependencySubmission); private static Task PostAsJson(this IApiHandler handler, string endpoint, object body) => handler.WithRetries(() => handler.SendAsync(endpoint, body, "POST")); private static Task PatchAsJson(this IApiHandler handler, string endpoint, object body) => handler.WithRetries(() => handler.SendAsync(endpoint, body, "PATCH")); diff --git a/nuget/script/run b/nuget/script/run index 0e26b1133ee..741d3c91bd3 100644 --- a/nuget/script/run +++ b/nuget/script/run @@ -1,4 +1,8 @@ #!/bin/bash # shellcheck disable=all +VERSION_FILE="$DEPENDABOT_HOME/.dependabot-version" +if [ -f "$VERSION_FILE" ] && [ -s "$VERSION_FILE" ]; then + export DEPENDABOT_VERSION=$(cat "$VERSION_FILE") +fi pwsh "$DEPENDABOT_HOME/dependabot-updater/bin/main.ps1" $* diff --git a/nuget/updater/main.ps1 b/nuget/updater/main.ps1 index bee1a729952..4190a59170f 100644 --- a/nuget/updater/main.ps1 +++ b/nuget/updater/main.ps1 @@ -35,7 +35,15 @@ function Get-Files { } } -function Update-Files { +function Get-BaseCommitSha { + Push-Location $env:DEPENDABOT_REPO_CONTENTS_PATH + $baseCommitSha = git rev-parse HEAD + Pop-Location + Write-Host "Base commit SHA: $baseCommitSha" + return $baseCommitSha +} + +function Initialize-UpdateEnvironment { # install relevant SDKs Install-Sdks ` -jobFilePath $env:DEPENDABOT_JOB_PATH ` @@ -45,14 +53,16 @@ function Update-Files { # TODO: install workloads? Set-NuGetConfig +} - Push-Location $env:DEPENDABOT_REPO_CONTENTS_PATH - $baseCommitSha = git rev-parse HEAD - Pop-Location - Write-Host "Base commit SHA: $baseCommitSha" +function Build-UpdaterArguments { + param( + [string]$command, + [string]$baseCommitSha + ) $arguments = @() - $arguments += "run" + $arguments += $command $arguments += "--job-path", $env:DEPENDABOT_JOB_PATH $arguments += "--repo-contents-path", $env:DEPENDABOT_REPO_CONTENTS_PATH $arguments += "--api-url", $env:DEPENDABOT_API_URL @@ -73,25 +83,48 @@ function Update-Files { $env:NUGET_FALLBACK_PACKAGES = "$env:DEPENDABOT_HOME/.nuget/packages" } + return $arguments +} + +function Invoke-Updater { + param( + [Parameter(Mandatory = $true)] + [ValidateSet('run', 'graph')] + [string]$command + ) + + Initialize-UpdateEnvironment + $baseCommitSha = Get-BaseCommitSha + + $arguments = Build-UpdaterArguments -command $command -baseCommitSha $baseCommitSha + $process = Start-Process -FilePath $updaterTool -ArgumentList $arguments -NoNewWindow -Wait -PassThru $process.WaitForExit() $script:operationExitCode = $process.ExitCode } -function Update-Dependencies { +function Invoke-DependencyProcess { + param( + [Parameter(Mandatory = $true)] + [ValidateSet('run', 'graph')] + [string]$command + ) + $env:DEPENDABOT_LOG_MESSAGES = "true" + Get-Files if ($script:operationExitCode -ne 0) { return } - Update-Files + Invoke-Updater -command $command } try { Switch ($args[0]) { "fetch_files" { } - "update_files" { Update-Dependencies } + "update_files" { Invoke-DependencyProcess -command run } + "update_graph" { Invoke-DependencyProcess -command graph } default { throw "unknown command: $args[0]" } } exit $operationExitCode