diff --git a/docs/detectors/go.md b/docs/detectors/go.md index dcf715d1a..e5a8cbfe8 100644 --- a/docs/detectors/go.md +++ b/docs/detectors/go.md @@ -8,22 +8,24 @@ Go detection runs when one of the following files is found in the project: ## Default Detection strategy -Default Go detection depends on the following to successfully run: +### go.mod +- All go.mod are parsed to detect dependencies. This parsing doesn't depend on presence of `go cli`. -- Go v1.11+. +### go cli (go list) or go.sum Parsing +- If a `go.sum` file is found, detector first checks if go version in the adjacent `go.mod` >= `1.17`. If it is `>= 1.17`, the file is skipped. If it is `< 1.17`, the detector proceeds as follows. Read [Go Module Changes in Go 1.17](#go-module-changes-in-go-117) to understand why `1.17` is relevant. +- If `go cli` is found and not [disabled](#environment-variables), `go list` command is preferred over parsing `go.sum` file since `go.sum` files contains history of dependencies and including these dependencies can lead to [over-reporting](#known-limitations). +- If `go list` was not used or did not run successfully, detector falls back to parsing `go.sum` manually. +### Dependency graph generation Full dependency graph generation is supported if Go v1.11+ is present -on the build agent. If no Go v1.11+ is present, fallback detection -strategy is performed. - -Go detection is performed by parsing output from executing -[go list -mod=readonly -m -json all][1]. To generate the graph, the command +on the build agent. To generate the graph, the command [go mod graph][2] is executed. This only adds edges between the components -that were already registered by `go list`. +that were already registered. ## Fallback Detection strategy -The fallback detections trategy is known to overreport (see the +The fallback strategy refers to detector parsing `go.sum` manually. +TThis strategy is known to overreport (see the [known limitations](#known-limitations)). Read through the [troubleshooting section](#troubleshooting-failures-to-run-the-default-go-detection-strategy) for tips on how to ensure that the newer, more accurate default @@ -249,55 +251,6 @@ of the package contents. making it possible to recreate the same build environment consistently. -### Detection Strategy - -The Go Component Detector follows a strategy that involves the -following key steps: - -1. **File Discovery**: The detector searches for go.mod and go.sum files - within the project directory. - -2. **Filtering go.sum Files**: The detector filters out go.sum files when - there is no adjacent go.mod file or when the go.mod file specifies - a Go version lower than 1.17. This filtering reduces the risk of - over-reporting components. More on this later. - -3. **Go CLI Scanning (Optional)**: If the Go CLI (go) is available and not - manually disabled, the detector attempts to use it to scan the - project for dependencies and build a dependency graph. This step can - significantly improve detection speed. - -4. **Fallback Detection**: If Go CLI scanning is not possible or not - successful, the detector falls back to parsing go.mod and go.sum - files directly to identify components. - -5. **Parsing go.mod File**: The detector parses the go.mod file to - identify direct and transitive dependencies, recording their names - and versions. - -6. **Parsing go.sum File**: The detector parses the go.sum file, recording - information about dependencies, including their names, versions, - and hashes. - -7. **Dependency Graph Construction**: If Go CLI scanning was successful, - the detector constructs a dependency graph based on the information - gathered. This graph helps identify relationships between components. - -8. **Recording Components**: Throughout the detection process, the - detector records identified components and their relationships. - -9. **Environment Variable Check**: The detector checks for an environment - variable (DisableGoCliScan) to determine whether Go CLI scanning - should be disabled. - -The logic for checking if the Go version present in the `go.mod` file -is greater than or equal to 1.17 is relevant because it determines -whether the `go.sum` file should be processed for detection. - -This check is essential because Go introduced significant changes in -how it handles dependencies and the `go.sum` file in Go version 1.17, -which have implications for dependency scanning. - ### Go Module Changes in Go 1.17 Prior to Go 1.17, the `go.mod` file primarily contained information @@ -310,6 +263,8 @@ file now includes information about both direct and transitive dependencies. This improvement enhances the clarity and completeness of dependency information within the `go.mod` file. +The completeness of `go.mod` file in `>= 1.17` allows the detector to skip `go.sum` files entirely. + #### Relevance of the Go Version Check 1. **Accuracy of Dependency Detection**: Checking the Go version in diff --git a/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs deleted file mode 100644 index 957b9748a..000000000 --- a/src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs +++ /dev/null @@ -1,241 +0,0 @@ -namespace Microsoft.ComponentDetection.Detectors.Go; - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reactive.Linq; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.ComponentDetection.Common; -using Microsoft.ComponentDetection.Common.Telemetry.Records; -using Microsoft.ComponentDetection.Contracts; -using Microsoft.ComponentDetection.Contracts.Internal; -using Microsoft.ComponentDetection.Contracts.TypedComponent; -using Microsoft.Extensions.Logging; - -public class Go117ComponentDetector : FileComponentDetector, IExperimentalDetector -{ - private readonly HashSet projectRoots = []; - - private readonly ICommandLineInvocationService commandLineInvocationService; - private readonly IGoParserFactory goParserFactory; - private readonly IEnvironmentVariableService envVarService; - - public Go117ComponentDetector( - IComponentStreamEnumerableFactory componentStreamEnumerableFactory, - IObservableDirectoryWalkerFactory walkerFactory, - ICommandLineInvocationService commandLineInvocationService, - IEnvironmentVariableService envVarService, - ILogger logger, - IFileUtilityService fileUtilityService, - IGoParserFactory goParserFactory) - { - this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; - this.Scanner = walkerFactory; - this.commandLineInvocationService = commandLineInvocationService; - this.Logger = logger; - this.goParserFactory = goParserFactory; - this.envVarService = envVarService; - } - - public override string Id => "Go117"; - - public override IEnumerable Categories => [Enum.GetName(typeof(DetectorClass), DetectorClass.GoMod)]; - - public override IList SearchPatterns { get; } = ["go.mod", "go.sum"]; - - public override IEnumerable SupportedComponentTypes { get; } = [ComponentType.Go]; - - public override int Version => 3; - - protected override Task> OnPrepareDetectionAsync( - IObservable processRequests, - IDictionary detectorArgs, - CancellationToken cancellationToken = default) - { - var goModProcessRequests = processRequests.Where(processRequest => - { - if (Path.GetFileName(processRequest.ComponentStream.Location) != "go.sum") - { - return true; - } - - var goModFile = this.FindAdjacentGoModComponentStreams(processRequest).FirstOrDefault(); - - try - { - if (goModFile == null) - { - this.Logger.LogDebug( - "go.sum file found without an adjacent go.mod file. Location: {Location}", - processRequest.ComponentStream.Location); - - return true; - } - - return GoDetectorUtils.ShouldIncludeGoSumFromDetection(goSumFilePath: processRequest.ComponentStream.Location, goModFile, this.Logger); - } - finally - { - goModFile?.Stream.Dispose(); - } - }); - - return Task.FromResult(goModProcessRequests); - } - - protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) - { - var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; - var file = processRequest.ComponentStream; - - var projectRootDirectory = Directory.GetParent(file.Location); - if (this.projectRoots.Any(path => projectRootDirectory.FullName.StartsWith(path))) - { - return; - } - - using var record = new GoGraphTelemetryRecord(); - var wasGoCliDisabled = this.IsGoCliManuallyDisabled(); - record.WasGoCliDisabled = wasGoCliDisabled; - record.WasGoFallbackStrategyUsed = false; - - var fileExtension = Path.GetExtension(file.Location).ToUpperInvariant(); - switch (fileExtension) - { - case ".MOD": - { - this.Logger.LogDebug("Found Go.mod: {Location}", file.Location); - await this.goParserFactory.CreateParser(GoParserType.GoMod, this.Logger).ParseAsync(singleFileComponentRecorder, file, record); - - if (await this.ShouldRunGoGraphAsync()) - { - await GoDependencyGraphUtility.GenerateAndPopulateDependencyGraphAsync( - this.commandLineInvocationService, - this.Logger, - singleFileComponentRecorder, - projectRootDirectory.FullName, - record, - cancellationToken); - } - - break; - } - - case ".SUM": - { - this.Logger.LogDebug("Found Go.sum: {Location}", file.Location); - - // check if we can use Go CLI instead - var wasGoCliScanSuccessful = false; - if (!wasGoCliDisabled) - { - wasGoCliScanSuccessful = await this.goParserFactory.CreateParser(GoParserType.GoCLI, this.Logger).ParseAsync(singleFileComponentRecorder, file, record); - } - - this.Logger.LogDebug("Status of Go CLI scan when considering {GoSumLocation}: {Status}", file.Location, wasGoCliScanSuccessful); - - // If Go CLI scan was not successful/disabled, scan go.sum because this go.sum was recorded due to go.mod - // containing go < 1.17. So go.mod is incomplete. We need to parse go.sum to make list of dependencies complete - if (!wasGoCliScanSuccessful) - { - record.WasGoFallbackStrategyUsed = true; - this.Logger.LogDebug("Go CLI scan when considering {GoSumLocation} was not successful. Falling back to scanning go.sum", file.Location); - await this.goParserFactory.CreateParser(GoParserType.GoSum, this.Logger).ParseAsync(singleFileComponentRecorder, file, record); - } - else - { - this.projectRoots.Add(projectRootDirectory.FullName); - } - - break; - } - - default: - { - throw new InvalidOperationException("Unexpected file type detected in go detector"); - } - } - } - - private bool IsGoCliManuallyDisabled() - { - return this.envVarService.IsEnvironmentVariableValueTrue("DisableGoCliScan"); - } - - private async Task ShouldRunGoGraphAsync() - { - if (this.IsGoCliManuallyDisabled()) - { - return false; - } - - var goVersion = await this.GetGoVersionAsync(); - if (goVersion == null) - { - return false; - } - - return goVersion >= new Version(1, 11); - } - - private async Task GetGoVersionAsync() - { - try - { - var isGoAvailable = await this.commandLineInvocationService.CanCommandBeLocatedAsync("go", null, null, new List { "version" }.ToArray()); - if (!isGoAvailable) - { - this.Logger.LogInformation("Go CLI was not found in the system"); - return null; - } - - var processExecution = await this.commandLineInvocationService.ExecuteCommandAsync("go", null, null, cancellationToken: default, new List { "version" }.ToArray()); - if (processExecution.ExitCode != 0) - { - return null; - } - - // Define the regular expression pattern to match the version number - var versionPattern = @"go version go(\d+\.\d+\.\d+)"; - var match = Regex.Match(processExecution.StdOut, versionPattern); - - if (match.Success) - { - // Extract the version number from the match - var versionStr = match.Groups[1].Value; - return new Version(versionStr); - } - } - catch (Exception e) - { - this.Logger.LogWarning("Failed to get go version: {Exception}", e); - } - - return null; - } - - private IEnumerable FindAdjacentGoModComponentStreams(ProcessRequest processRequest) => - this.ComponentStreamEnumerableFactory.GetComponentStreams( - new FileInfo(processRequest.ComponentStream.Location).Directory, - ["go.mod"], - (_, _) => false, - false) - .Select(x => - { - // The stream will be disposed at the end of this method, so we need to copy it to a new stream. - var memoryStream = new MemoryStream(); - - x.Stream.CopyTo(memoryStream); - memoryStream.Position = 0; - - return new ComponentStream - { - Stream = memoryStream, - Location = x.Location, - Pattern = x.Pattern, - }; - }); -} diff --git a/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs index f2b0f361a..685248bd1 100644 --- a/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs @@ -5,6 +5,7 @@ namespace Microsoft.ComponentDetection.Detectors.Go; using System.IO; using System.Linq; using System.Reactive.Linq; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Microsoft.ComponentDetection.Common; @@ -19,26 +20,8 @@ public class GoComponentDetector : FileComponentDetector private readonly HashSet projectRoots = []; private readonly ICommandLineInvocationService commandLineInvocationService; - private readonly IEnvironmentVariableService envVarService; - private readonly IFileUtilityService fileUtilityService; private readonly IGoParserFactory goParserFactory; - - public GoComponentDetector( - IComponentStreamEnumerableFactory componentStreamEnumerableFactory, - IObservableDirectoryWalkerFactory walkerFactory, - ICommandLineInvocationService commandLineInvocationService, - IEnvironmentVariableService envVarService, - ILogger logger, - IFileUtilityService fileUtilityService) - { - this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; - this.Scanner = walkerFactory; - this.commandLineInvocationService = commandLineInvocationService; - this.envVarService = envVarService; - this.Logger = logger; - this.fileUtilityService = fileUtilityService; - this.goParserFactory = new GoParserFactory(fileUtilityService, commandLineInvocationService); - } + private readonly IEnvironmentVariableService envVarService; public GoComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, @@ -47,15 +30,14 @@ public GoComponentDetector( IEnvironmentVariableService envVarService, ILogger logger, IFileUtilityService fileUtilityService, - IGoParserFactory factory) + IGoParserFactory goParserFactory) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; this.commandLineInvocationService = commandLineInvocationService; - this.envVarService = envVarService; this.Logger = logger; - this.fileUtilityService = fileUtilityService; - this.goParserFactory = factory; + this.goParserFactory = goParserFactory; + this.envVarService = envVarService; } public override string Id => "Go"; @@ -66,14 +48,13 @@ public GoComponentDetector( public override IEnumerable SupportedComponentTypes { get; } = [ComponentType.Go]; - public override int Version => 8; + public override int Version => 9; protected override Task> OnPrepareDetectionAsync( IObservable processRequests, IDictionary detectorArgs, CancellationToken cancellationToken = default) { - // Filter out any go.sum process requests if the adjacent go.mod file is present and has a go version >= 1.17 var goModProcessRequests = processRequests.Where(processRequest => { if (Path.GetFileName(processRequest.ComponentStream.Location) != "go.sum") @@ -105,28 +86,6 @@ protected override Task> OnPrepareDetectionAsync( return Task.FromResult(goModProcessRequests); } - private IEnumerable FindAdjacentGoModComponentStreams(ProcessRequest processRequest) => - this.ComponentStreamEnumerableFactory.GetComponentStreams( - new FileInfo(processRequest.ComponentStream.Location).Directory, - ["go.mod"], - (_, _) => false, - false) - .Select(x => - { - // The stream will be disposed at the end of this method, so we need to copy it to a new stream. - var memoryStream = new MemoryStream(); - - x.Stream.CopyTo(memoryStream); - memoryStream.Position = 0; - - return new ComponentStream - { - Stream = memoryStream, - Location = x.Location, - Pattern = x.Pattern, - }; - }); - protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) { var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; @@ -139,59 +98,71 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID } using var record = new GoGraphTelemetryRecord(); - record.WasGoCliDisabled = false; + var wasGoCliDisabled = this.IsGoCliManuallyDisabled(); + record.WasGoCliDisabled = wasGoCliDisabled; record.WasGoFallbackStrategyUsed = false; - var wasGoCliScanSuccessful = false; - try - { - if (!this.IsGoCliManuallyDisabled()) - { - wasGoCliScanSuccessful = await this.goParserFactory.CreateParser(GoParserType.GoCLI, this.Logger).ParseAsync(singleFileComponentRecorder, file, record); - } - else - { - record.WasGoCliDisabled = true; - this.Logger.LogInformation("Go cli scan was manually disabled, fallback strategy performed." + - " More info: https://github.com/microsoft/component-detection/blob/main/docs/detectors/go.md#fallback-detection-strategy"); - } - } - catch (Exception ex) - { - this.Logger.LogError(ex, "Failed to detect components using go cli. Location: {Location}", file.Location); - record.ExceptionMessage = ex.Message; - } - finally - { - if (wasGoCliScanSuccessful) - { - this.projectRoots.Add(projectRootDirectory.FullName); - } - else - { - record.WasGoFallbackStrategyUsed = true; - await this.ParseGoFileAsync(file, singleFileComponentRecorder, record); - } - } - } - - private async Task ParseGoFileAsync(IComponentStream file, ISingleFileComponentRecorder singleFileComponentRecorder, GoGraphTelemetryRecord record) - { var fileExtension = Path.GetExtension(file.Location).ToUpperInvariant(); switch (fileExtension) { case ".MOD": { this.Logger.LogDebug("Found Go.mod: {Location}", file.Location); - await this.goParserFactory.CreateParser(GoParserType.GoMod, this.Logger).ParseAsync(singleFileComponentRecorder, file, record); + + if (await this.ShouldRunGoGraphAsync()) + { + await GoDependencyGraphUtility.GenerateAndPopulateDependencyGraphAsync( + this.commandLineInvocationService, + this.Logger, + singleFileComponentRecorder, + projectRootDirectory.FullName, + record, + cancellationToken); + } + break; } case ".SUM": { this.Logger.LogDebug("Found Go.sum: {Location}", file.Location); - await this.goParserFactory.CreateParser(GoParserType.GoSum, this.Logger).ParseAsync(singleFileComponentRecorder, file, record); + + // check if we can use Go CLI instead + var wasGoCliScanSuccessful = false; + + try + { + if (!wasGoCliDisabled) + { + wasGoCliScanSuccessful = await this.goParserFactory.CreateParser(GoParserType.GoCLI, this.Logger).ParseAsync(singleFileComponentRecorder, file, record); + } + else + { + this.Logger.LogInformation("Go cli scan was manually disabled, fallback strategy performed."); + } + } + catch (Exception ex) + { + this.Logger.LogError(ex, "Failed to detect components using go cli. Location: {Location}", file.Location); + record.ExceptionMessage = ex.Message; + } + + this.Logger.LogDebug("Status of Go CLI scan when considering {GoSumLocation}: {Status}", file.Location, wasGoCliScanSuccessful); + + // If Go CLI scan was not successful/disabled, scan go.sum because this go.sum was recorded due to go.mod + // containing go < 1.17. So go.mod is incomplete. We need to parse go.sum to make list of dependencies complete + if (!wasGoCliScanSuccessful) + { + record.WasGoFallbackStrategyUsed = true; + this.Logger.LogDebug("Go CLI scan when considering {GoSumLocation} was not successful. Falling back to scanning go.sum", file.Location); + await this.goParserFactory.CreateParser(GoParserType.GoSum, this.Logger).ParseAsync(singleFileComponentRecorder, file, record); + } + else + { + this.projectRoots.Add(projectRootDirectory.FullName); + } + break; } @@ -206,4 +177,78 @@ private bool IsGoCliManuallyDisabled() { return this.envVarService.IsEnvironmentVariableValueTrue("DisableGoCliScan"); } + + private async Task ShouldRunGoGraphAsync() + { + if (this.IsGoCliManuallyDisabled()) + { + return false; + } + + var goVersion = await this.GetGoVersionAsync(); + if (goVersion == null) + { + return false; + } + + return goVersion >= new Version(1, 11); + } + + private async Task GetGoVersionAsync() + { + try + { + var isGoAvailable = await this.commandLineInvocationService.CanCommandBeLocatedAsync("go", null, null, new List { "version" }.ToArray()); + if (!isGoAvailable) + { + this.Logger.LogInformation("Go CLI was not found in the system"); + return null; + } + + var processExecution = await this.commandLineInvocationService.ExecuteCommandAsync("go", null, null, cancellationToken: default, new List { "version" }.ToArray()); + if (processExecution.ExitCode != 0) + { + return null; + } + + // Define the regular expression pattern to match the version number + var versionPattern = @"go version go(\d+\.\d+\.\d+)"; + var match = Regex.Match(processExecution.StdOut, versionPattern); + + if (match.Success) + { + // Extract the version number from the match + var versionStr = match.Groups[1].Value; + return new Version(versionStr); + } + } + catch (Exception e) + { + this.Logger.LogWarning("Failed to get go version: {Exception}", e); + } + + return null; + } + + private IEnumerable FindAdjacentGoModComponentStreams(ProcessRequest processRequest) => + this.ComponentStreamEnumerableFactory.GetComponentStreams( + new FileInfo(processRequest.ComponentStream.Location).Directory, + ["go.mod"], + (_, _) => false, + false) + .Select(x => + { + // The stream will be disposed at the end of this method, so we need to copy it to a new stream. + var memoryStream = new MemoryStream(); + + x.Stream.CopyTo(memoryStream); + memoryStream.Position = 0; + + return new ComponentStream + { + Stream = memoryStream, + Location = x.Location, + Pattern = x.Pattern, + }; + }); } diff --git a/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoModParser.cs b/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoModParser.cs index 689666cc4..27ed96bf1 100644 --- a/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoModParser.cs +++ b/src/Microsoft.ComponentDetection.Detectors/go/Parsers/GoModParser.cs @@ -34,19 +34,36 @@ private static bool IsLocalPath(string path) /// Tries to extract source token from replace directive. /// /// String containing a directive after replace token. - /// HashSet where the token is placed if replace directive substitutes a local path. - private static void TryExtractReplaceDirective(string directiveLine, HashSet replaceDirectives) + /// Hash set containing package+version? that are local references. + /// Dinctionary that maps source package+version? to replaced package+version?. + private static void HandleReplaceDirective( + string directiveLine, + HashSet replacePathDirectives, + Dictionary moduleReplaces) { var parts = directiveLine.Split("=>", StringSplitOptions.RemoveEmptyEntries); - if (parts.Length == 2) + if (parts.Length != 2) { - var source = parts[0].Trim().Split(' ')[0]; - var target = parts[1].Trim(); + return; + } - if (IsLocalPath(target)) - { - replaceDirectives.Add(source); - } + var sourceTokens = parts[0].Trim().Split(' ', StringSplitOptions.RemoveEmptyEntries); + var sourceName = sourceTokens[0]; + var sourceVersion = sourceTokens.Length > 1 ? sourceTokens[1] : null; + + var targetTokens = parts[1].Trim().Split(' ', StringSplitOptions.RemoveEmptyEntries); + var targetName = targetTokens[0]; + var targetVersion = targetTokens.Length > 1 ? targetTokens[1] : null; + + if (IsLocalPath(targetName)) + { + var key = sourceVersion != null ? $"{sourceName}@{sourceVersion}" : sourceName; + replacePathDirectives.Add(key); + } + else + { + var key = sourceVersion != null ? $"{sourceName}@{sourceVersion}" : sourceName; + moduleReplaces[key] = new GoReplaceDirective(sourceName, sourceVersion, targetName, targetVersion); } } @@ -55,8 +72,8 @@ public async Task ParseAsync( IComponentStream file, GoGraphTelemetryRecord record) { - // Collect replace directives that point to a local path - var replaceDirectives = await this.GetAllReplacePathDirectivesAsync(file); + // Collect replace directives + var (replacePathDirectives, moduleReplacements) = await this.GetAllReplaceDirectivesAsync(file); // Rewind stream after reading replace directives file.Stream.Seek(0, SeekOrigin.Begin); @@ -79,7 +96,7 @@ public async Task ParseAsync( // are listed in the require () section if (line.StartsWith(StartString)) { - this.TryRegisterDependencyFromModLine(file, line[StartString.Length..], singleFileComponentRecorder, replaceDirectives); + this.TryRegisterDependencyFromModLine(file, line[StartString.Length..], singleFileComponentRecorder, replacePathDirectives, moduleReplacements); } line = await reader.ReadLineAsync(); @@ -88,14 +105,14 @@ public async Task ParseAsync( // Stopping at the first ) restrict the detection to only the require section. while ((line = await reader.ReadLineAsync()) != null && !line.EndsWith(')')) { - this.TryRegisterDependencyFromModLine(file, line, singleFileComponentRecorder, replaceDirectives); + this.TryRegisterDependencyFromModLine(file, line, singleFileComponentRecorder, replacePathDirectives, moduleReplacements); } } return true; } - private void TryRegisterDependencyFromModLine(IComponentStream file, string line, ISingleFileComponentRecorder singleFileComponentRecorder, HashSet replaceDirectives) + private void TryRegisterDependencyFromModLine(IComponentStream file, string line, ISingleFileComponentRecorder singleFileComponentRecorder, HashSet replacePathDirectives, Dictionary moduleReplacements) { if (line.Trim().StartsWith("//")) { @@ -103,24 +120,36 @@ private void TryRegisterDependencyFromModLine(IComponentStream file, string line return; } - if (this.TryToCreateGoComponentFromModLine(line, out var goComponent)) - { - if (replaceDirectives.Contains(goComponent.Name)) - { - // Skip registering this dependency since it's replaced by a local path - // we will be reading this dependency somewhere else - this.logger.LogInformation("Skipping {GoComponentId} from {Location} because it's a local reference.", goComponent.Id, file.Location); - return; - } - - singleFileComponentRecorder.RegisterUsage(new DetectedComponent(goComponent)); - } - else + if (!this.TryToCreateGoComponentFromModLine(line, out var goComponent)) { var lineTrim = line.Trim(); this.logger.LogWarning("Line could not be parsed for component [{LineTrim}]", lineTrim); singleFileComponentRecorder.RegisterPackageParseFailure(lineTrim); + return; } + + var key = $"{goComponent.Name}@{goComponent.Version}"; + if (replacePathDirectives.Contains(key) || replacePathDirectives.Contains(goComponent.Name)) + { + this.logger.LogInformation( + "Skipping {GoComponentId} from {Location} because it's a local reference.", + goComponent.Id, + file.Location); + return; + } + + if (moduleReplacements.TryGetValue(key, out var replacement) || + moduleReplacements.TryGetValue(goComponent.Name, out replacement)) + { + this.logger.LogInformation( + "go Module {PackageKey} is being replaced with {TargetName}-{TargetVersion}", + key, + replacement.TargetPathOrModule, + replacement.TargetVersion); + goComponent = new GoComponent(replacement.TargetPathOrModule, replacement.TargetVersion ?? goComponent.Version); + } + + singleFileComponentRecorder.RegisterUsage(new DetectedComponent(goComponent)); } private bool TryToCreateGoComponentFromModLine(string line, out GoComponent goComponent) @@ -140,46 +169,48 @@ private bool TryToCreateGoComponentFromModLine(string line, out GoComponent goCo return true; } - private async Task> GetAllReplacePathDirectivesAsync(IComponentStream file) + private async Task<(HashSet ReplacePathDirectives, Dictionary ModuleReplacements)> GetAllReplaceDirectivesAsync(IComponentStream file) { - var replacedDirectives = new HashSet(StringComparer.OrdinalIgnoreCase); + var replacePathDirectives = new HashSet(StringComparer.OrdinalIgnoreCase); + var moduleReplacements = new Dictionary(StringComparer.OrdinalIgnoreCase); const string singleReplaceDirectiveBegin = "replace "; const string multiReplaceDirectiveBegin = "replace ("; - using (var reader = new StreamReader(file.Stream, leaveOpen: true)) + + using var reader = new StreamReader(file.Stream, leaveOpen: true); + while (!reader.EndOfStream) { - while (!reader.EndOfStream) + var line = await reader.ReadLineAsync(); + if (line == null) { - var line = await reader.ReadLineAsync(); - if (line == null) - { - continue; - } + continue; + } - line = line.Trim(); + line = line.Trim(); - // Multiline block: replace ( - if (line.StartsWith(multiReplaceDirectiveBegin)) + // Multiline block: replace ( + if (line.StartsWith(multiReplaceDirectiveBegin)) + { + while ((line = await reader.ReadLineAsync()) != null) { - while ((line = await reader.ReadLineAsync()) != null) + line = line.Trim(); + if (line == ")") { - line = line.Trim(); - if (line == ")") - { - break; - } - - TryExtractReplaceDirective(line, replacedDirectives); + break; } + + HandleReplaceDirective(line, replacePathDirectives, moduleReplacements); } - else if (line.StartsWith(singleReplaceDirectiveBegin)) - { - // single line block: replace - var directiveContent = line[singleReplaceDirectiveBegin.Length..].Trim(); - TryExtractReplaceDirective(directiveContent, replacedDirectives); - } + } + else if (line.StartsWith(singleReplaceDirectiveBegin)) + { + // single line block: replace + var directiveContent = line[singleReplaceDirectiveBegin.Length..].Trim(); + HandleReplaceDirective(directiveContent, replacePathDirectives, moduleReplacements); } } - return replacedDirectives; + return (replacePathDirectives, moduleReplacements); } + + private record GoReplaceDirective(string Source, string Version, string TargetPathOrModule, string TargetVersion); } diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/Go117DetectorExperiment.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/Go117DetectorExperiment.cs deleted file mode 100644 index 080675d39..000000000 --- a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/Go117DetectorExperiment.cs +++ /dev/null @@ -1,22 +0,0 @@ -namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Configs; - -using Microsoft.ComponentDetection.Contracts; -using Microsoft.ComponentDetection.Detectors.Go; - -/// -/// Validating the Go detector for go mod 1.17+. -/// -public class Go117DetectorExperiment : IExperimentConfiguration -{ - /// - public string Name => "Go117Detector"; - - /// - public bool IsInControlGroup(IComponentDetector componentDetector) => componentDetector is GoComponentDetector; - - /// - public bool IsInExperimentGroup(IComponentDetector componentDetector) => componentDetector is Go117ComponentDetector; - - /// - public bool ShouldRecord(IComponentDetector componentDetector, int numComponents) => true; -} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs index 5099e5fa3..c91dee846 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs @@ -67,7 +67,6 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); // Detectors // CocoaPods @@ -87,7 +86,6 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s // Go services.AddSingleton(); - services.AddSingleton(); services.AddSingleton(); // Gradle diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Go117ComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/Go117ComponentDetectorTests.cs deleted file mode 100644 index 57323b780..000000000 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/Go117ComponentDetectorTests.cs +++ /dev/null @@ -1,257 +0,0 @@ -namespace Microsoft.ComponentDetection.Detectors.Tests; - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using FluentAssertions; -using Microsoft.ComponentDetection.Common.Telemetry.Records; -using Microsoft.ComponentDetection.Contracts; -using Microsoft.ComponentDetection.Contracts.BcdeModels; -using Microsoft.ComponentDetection.Detectors.Go; -using Microsoft.ComponentDetection.TestsUtilities; -using Microsoft.Extensions.Logging; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using Moq; - -[TestClass] -[TestCategory("Governance/All")] -[TestCategory("Governance/ComponentDetection")] -public class Go117ComponentDetectorTests : BaseDetectorTest -{ - private readonly Mock commandLineMock; - private readonly Mock envVarService; - private readonly Mock fileUtilityServiceMock; - private readonly Mock> mockLogger; - private readonly Mock mockParserFactory; - - public Go117ComponentDetectorTests() - { - this.commandLineMock = new Mock(); - this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, It.IsAny(), It.IsAny())) - .ReturnsAsync(false); - this.DetectorTestUtility.AddServiceMock(this.commandLineMock); - - var mockGoParser = new Mock(); - this.mockParserFactory = new Mock(); - - this.mockParserFactory.Setup(x => x.CreateParser(It.IsAny(), It.IsAny())).Returns(mockGoParser.Object); - this.envVarService = new Mock(); - this.envVarService.Setup(x => x.IsEnvironmentVariableValueTrue("DisableGoCliScan")).Returns(false); - this.DetectorTestUtility.AddServiceMock(this.envVarService); - this.fileUtilityServiceMock = new Mock(); - this.DetectorTestUtility.AddServiceMock(this.fileUtilityServiceMock); - this.mockLogger = new Mock>(); - this.DetectorTestUtility.AddServiceMock(this.mockLogger); - this.DetectorTestUtility.AddServiceMock(this.mockParserFactory); - } - - [TestMethod] - public async Task Go117ModDetector_GoCliIs1_11OrGreater_GoGraphIsExecutedAsync() - { - var goMod = string.Empty; - - this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, null, It.Is(p => p.SequenceEqual(new List { "version" }.ToArray())))) - .ReturnsAsync(true); - - this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, null, default, It.Is(p => p.SequenceEqual(new List { "version" }.ToArray())))) - .ReturnsAsync(new CommandLineExecutionResult - { - ExitCode = 0, - StdOut = "go version go1.23.6 windows/amd64", - }); - - this.commandLineMock.Setup(service => service.ExecuteCommandAsync( - "go", - null, - It.IsAny(), - default, - It.Is(args => args.Length == 2 && args[0] == "mod" && args[1] == "graph"))) - .ReturnsAsync(new CommandLineExecutionResult - { - ExitCode = 0, - StdOut = string.Empty, - }) - .Verifiable(); - - var (scanResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("go.mod", goMod) - .ExecuteDetectorAsync(); - - scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); - - this.commandLineMock.Verify(); - } - - [TestMethod] - public async Task Go117ModDetector_GoCliIs111OrLessThan1_11_GoGraphIsNotExecutedAsync() - { - var goMod = string.Empty; - - this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, null, It.Is(p => p.SequenceEqual(new List { "version" }.ToArray())))) - .ReturnsAsync(true); - - this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, null, default, It.Is(p => p.SequenceEqual(new List { "version" }.ToArray())))) - .ReturnsAsync(new CommandLineExecutionResult - { - ExitCode = 0, - StdOut = "go version go1.10.6 windows/amd64", - }); - - var (scanResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("go.mod", goMod) - .ExecuteDetectorAsync(); - - scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); - - this.commandLineMock.Verify( - service => service.ExecuteCommandAsync( - "go", - null, - It.IsAny(), - default, - It.Is(args => args.Length == 2 && args[0] == "mod" && args[1] == "graph")), - times: Times.Never); - } - - [TestMethod] - public async Task Go117ModDetector_GoModFileFound_GoModParserIsExecuted() - { - var goModParserMock = new Mock(); - this.mockParserFactory.Setup(x => x.CreateParser(GoParserType.GoMod, It.IsAny())).Returns(goModParserMock.Object); - - this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, null, It.Is(p => p.SequenceEqual(new List { "version" }.ToArray())))) - .ReturnsAsync(true); - - this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, null, default, It.Is(p => p.SequenceEqual(new List { "version" }.ToArray())))) - .ReturnsAsync(new CommandLineExecutionResult - { - ExitCode = 0, - StdOut = "go version go1.10.6 windows/amd64", - }); - - var (scanResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("go.mod", string.Empty) - .ExecuteDetectorAsync(); - - scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); - - goModParserMock.Verify(parser => parser.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); - } - - /// - /// Verifies that if Go CLI is enabled/available and succeeds, go.sum file is not parsed and vice-versa. - /// - /// Task. - [DataTestMethod] - [DataRow(true)] - [DataRow(false)] - public async Task Go117Detector_GoSum_GoSumParserExecuted(bool goCliSucceeds) - { - var nInvocationsOfSumParser = goCliSucceeds ? 0 : 1; - var goSumParserMock = new Mock(); - var goCliParserMock = new Mock(); - this.mockParserFactory.Setup(x => x.CreateParser(GoParserType.GoSum, It.IsAny())).Returns(goSumParserMock.Object); - this.mockParserFactory.Setup(x => x.CreateParser(GoParserType.GoCLI, It.IsAny())).Returns(goCliParserMock.Object); - - // Setup go cli parser to succeed/fail - goCliParserMock.Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(goCliSucceeds); - - // Setup go sum parser to succeed - goSumParserMock.Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(true); - - var (scanResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("go.sum", string.Empty) - .ExecuteDetectorAsync(); - - scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); - this.mockParserFactory.Verify(clm => clm.CreateParser(GoParserType.GoSum, It.IsAny()), nInvocationsOfSumParser == 0 ? Times.Never : Times.Once); - } - - /// - /// Verifies that if Go CLI is disabled, go.sum is parsed. - /// - /// Task. - [TestMethod] - public async Task Go117Detector_GoSum_GoSumParserExecutedIfCliDisabled() - { - var goSumParserMock = new Mock(); - this.mockParserFactory.Setup(x => x.CreateParser(GoParserType.GoSum, It.IsAny())).Returns(goSumParserMock.Object); - - // Setup environment variable to disable CLI scan - this.envVarService.Setup(s => s.IsEnvironmentVariableValueTrue("DisableGoCliScan")).Returns(true); - - // Setup go sum parser to succed - goSumParserMock.Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(true); - - var (scanResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("go.sum", string.Empty) - .ExecuteDetectorAsync(); - - scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); - this.mockParserFactory.Verify(clm => clm.CreateParser(GoParserType.GoSum, It.IsAny()), Times.Once); - } - - [TestMethod] - public async Task Go117ModDetector_ExecutingGoVersionFails_DetectorDoesNotFail() - { - var goModParserMock = new Mock(); - this.mockParserFactory.Setup(x => x.CreateParser(GoParserType.GoMod, It.IsAny())).Returns(goModParserMock.Object); - - this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, null, It.Is(p => p.SequenceEqual(new List { "version" }.ToArray())))) - .ReturnsAsync(true); - - this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, null, default, It.Is(p => p.SequenceEqual(new List { "version" }.ToArray())))) - .Throws(new InvalidOperationException("Failed to execute go version")); - - var (scanResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("go.mod", string.Empty) - .ExecuteDetectorAsync(); - - scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); - - goModParserMock.Verify(parser => parser.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); - } - - [TestMethod] - public async Task Go117ModDetector_VerifyLocalReferencesIgnored() - { - var goModFilePath = "./TestFiles/go_WithLocalReferences.mod"; // Replace with your actual file path - var fileStream = new FileStream(goModFilePath, FileMode.Open, FileAccess.Read); - - var goModParser = new GoModParser(this.mockLogger.Object); - var mockSingleFileComponentRecorder = new Mock(); - - var capturedComponents = new List(); - var expectedComponentIds = new List() - { - "github.com/grafana/grafana-app-sdk v0.23.1 - Go", - "k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f - Go", - }; - - mockSingleFileComponentRecorder - .Setup(m => m.RegisterUsage( - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny())) - .Callback((comp, _, _, _, _, _) => - { - capturedComponents.Add(comp); - }); - - var mockComponentStream = new Mock(); - mockComponentStream.Setup(mcs => mcs.Stream).Returns(fileStream); - mockComponentStream.Setup(mcs => mcs.Location).Returns("Location"); - - var result = await goModParser.ParseAsync(mockSingleFileComponentRecorder.Object, mockComponentStream.Object, new GoGraphTelemetryRecord()); - result.Should().BeTrue(); - capturedComponents - .Select(c => c.Component.Id) - .Should() - .BeEquivalentTo(expectedComponentIds); - } -} diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs index c02e486af..f165c567b 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/GoComponentDetectorTests.cs @@ -1,11 +1,15 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; using System; +using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; using FluentAssertions; +using Microsoft.ComponentDetection.Common.Telemetry.Records; using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.BcdeModels; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.Go; using Microsoft.ComponentDetection.TestsUtilities; @@ -22,14 +26,22 @@ public class GoComponentDetectorTests : BaseDetectorTest private readonly Mock envVarService; private readonly Mock fileUtilityServiceMock; private readonly Mock> mockLogger; + private readonly Mock mockParserFactory; + private readonly Mock mockGoModParser; + private readonly Mock mockGoSumParser; + private readonly Mock mockGoCliParser; public GoComponentDetectorTests() { this.commandLineMock = new Mock(); + this.mockGoModParser = new Mock(); + this.mockGoSumParser = new Mock(); + this.mockGoCliParser = new Mock(); + this.mockParserFactory = new Mock(); + this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, It.IsAny(), It.IsAny())) .ReturnsAsync(false); this.DetectorTestUtility.AddServiceMock(this.commandLineMock); - this.envVarService = new Mock(); this.envVarService.Setup(x => x.IsEnvironmentVariableValueTrue("DisableGoCliScan")).Returns(true); this.DetectorTestUtility.AddServiceMock(this.envVarService); @@ -37,6 +49,32 @@ public GoComponentDetectorTests() this.DetectorTestUtility.AddServiceMock(this.fileUtilityServiceMock); this.mockLogger = new Mock>(); this.DetectorTestUtility.AddServiceMock(this.mockLogger); + this.DetectorTestUtility.AddServiceMock(this.mockParserFactory); + } + + private void SetupMockGoModParser() + { + this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoMod, It.IsAny())).Returns(this.mockGoModParser.Object); + } + + private void SetupMockGoSumParser() + { + this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoSum, It.IsAny())).Returns(this.mockGoSumParser.Object); + } + + private void SetupMockGoCLIParser() + { + this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoCLI, It.IsAny())).Returns(this.mockGoCliParser.Object); + } + + private void SetupActualGoModParser() + { + this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoMod, It.IsAny())).Returns(new GoModParser(this.mockLogger.Object)); + } + + private void SetupActualGoSumParser() + { + this.mockParserFactory.Setup(f => f.CreateParser(GoParserType.GoSum, It.IsAny())).Returns(new GoSumParser(this.mockLogger.Object)); } [TestMethod] @@ -51,6 +89,8 @@ public async Task TestGoModDetectorWithValidFile_ReturnsSuccessfullyAsync() gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 github.com/dgrijalva/jwt-go v3.2.0+incompatible )"; + + this.SetupActualGoModParser(); var (scanResult, componentRecorder) = await this.DetectorTestUtility .WithFile("go.mod", goMod) .ExecuteDetectorAsync(); @@ -77,6 +117,7 @@ public async Task TestGoModDetector_CommentsOnFile_CommentsAreIgnoredAsync() // comment github.com/kr/pretty v0.1.0 // indirect )"; + this.SetupActualGoModParser(); var (scanResult, componentRecorder) = await this.DetectorTestUtility .WithFile("go.mod", goMod) .ExecuteDetectorAsync(); @@ -105,6 +146,7 @@ public async Task TestGoSumDetectorWithValidFile_ReturnsSuccessfullyAsync() github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= )"; + this.SetupActualGoSumParser(); var (scanResult, componentRecorder) = await this.DetectorTestUtility .WithFile("go.sum", goSum) .ExecuteDetectorAsync(); @@ -141,6 +183,7 @@ public async Task TestGoModDetector_MultipleSpaces_ReturnsSuccessfullyAsync() github.com/dgrijalva/jwt-go v3.2.0+incompatible )"; + this.SetupActualGoModParser(); var (scanResult, componentRecorder) = await this.DetectorTestUtility .WithFile("go.mod", goMod) .ExecuteDetectorAsync(); @@ -178,7 +221,7 @@ public async Task TestGoModDetector_ComponentsWithMultipleLocations_ReturnsSucce gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 github.com/Azure/go-autorest v10.15.2+incompatible )"; - + this.SetupActualGoModParser(); var (scanResult, componentRecorder) = await this.DetectorTestUtility .WithFile("go.mod", goMod1) .WithFile("go.mod", goMod2, fileLocation: Path.Join(Path.GetTempPath(), "another-location", "go.mod")) @@ -205,6 +248,7 @@ lorem ipsum four score and seven bugs ago $#26^#25%4"; + this.SetupActualGoModParser(); var (scanResult, componentRecorder) = await this.DetectorTestUtility .WithFile("go.mod", invalidGoMod) .ExecuteDetectorAsync(); @@ -227,6 +271,8 @@ go 1.18 github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U="; + this.SetupActualGoModParser(); + this.SetupActualGoSumParser(); var (scanResult, componentRecorder) = await this.DetectorTestUtility .WithFile("go.mod", goMod) .WithFile("go.mod", goMod, ["go.mod"]) @@ -258,6 +304,7 @@ go 1.18 rsc.io/sampler v1.3.0 // indirect )"; + this.SetupActualGoModParser(); var (scanResult, componentRecorder) = await this.DetectorTestUtility .WithFile("go.mod", goMod) .ExecuteDetectorAsync(); @@ -283,6 +330,7 @@ public async Task TestGoSumDetection_TwoEntriesForTheSameComponent_ReturnsSucces github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= )"; + this.SetupActualGoSumParser(); var (scanResult, componentRecorder) = await this.DetectorTestUtility .WithFile("go.sum", goSum) .ExecuteDetectorAsync(); @@ -308,6 +356,7 @@ public async Task TestGoModDetector_DetectorOnlyDetectInsideRequireSectionAsync( github.com/docker/distribution => github.com/docker/distribution v0.0.0-20191216044856-a8371794149d ) "; + this.SetupActualGoModParser(); var (scanResult, componentRecorder) = await this.DetectorTestUtility .WithFile("go.mod", goMod) .ExecuteDetectorAsync(); @@ -378,58 +427,34 @@ public async Task TestGoDetector_GoGraphCommandThrowsAsync() [TestMethod] public async Task TestGoDetector_GoGraphReplaceAsync() { - var buildDependencies = @"{ - ""Path"": ""some-package"", - ""Version"": ""v1.2.3"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"", - ""Replace"": { - ""Path"": ""some-package"", - ""Version"": ""v1.2.4"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"", - } -}" + "\n" + @"{ - ""Path"": ""test"", - ""Version"": ""v2.0.0"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" -}" + "\n" + @"{ - ""Path"": ""other"", - ""Version"": ""v1.2.0"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" -}" + "\n" + @"{ - ""Path"": ""a"", - ""Version"": ""v1.5.0"", - ""Time"": ""2020-05-19T17:02:07Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" -}"; + var goMod = @" +module example.com/my/module + +go 1.11 + +require ( + some-package v1.2.3 // indirect + test v2.0.0 // indirect + other v1.2.0 // indirect + a v1.5.0 // indirect +) + +replace some-package v1.2.3 => some-package v1.2.4 +;"; var goGraph = "example.com/mainModule some-package@v1.2.3\nsome-package@v1.2.3 other@v1.0.0\nsome-package@v1.2.3 other@v1.2.0\ntest@v2.0.0 a@v1.5.0"; - this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, It.IsAny(), It.IsAny())) + string[] cmdParams = []; + this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, It.IsAny(), cmdParams)) .ReturnsAsync(true); - string[] cmdParams = ["list", "-mod=readonly", "-m", "-json", "all"]; - this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, It.IsAny(), cmdParams)) - .ReturnsAsync(new CommandLineExecutionResult - { - ExitCode = 0, - StdOut = buildDependencies, - }); + cmdParams = ["version"]; + this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, It.IsAny(), cmdParams)) + .ReturnsAsync(true); + this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, null, It.IsAny(), cmdParams)) + .ReturnsAsync(new CommandLineExecutionResult { ExitCode = 0, StdOut = "go version go1.24.3 windows/amd64" }); cmdParams = ["mod", "graph"]; - this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, It.IsAny(), cmdParams)) + this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, It.IsAny(), It.IsAny(), cmdParams)) .ReturnsAsync(new CommandLineExecutionResult { ExitCode = 0, @@ -437,9 +462,9 @@ public async Task TestGoDetector_GoGraphReplaceAsync() }); this.envVarService.Setup(x => x.IsEnvironmentVariableValueTrue("DisableGoCliScan")).Returns(false); - + this.SetupActualGoModParser(); var (scanResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("go.mod", string.Empty) + .WithFile("go.mod", goMod) .ExecuteDetectorAsync(); scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); @@ -456,50 +481,32 @@ public async Task TestGoDetector_GoGraphReplaceAsync() [TestMethod] public async Task TestGoDetector_GoGraphHappyPathAsync() { - var buildDependencies = @"{ - ""Path"": ""some-package"", - ""Version"": ""v1.2.3"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" -}" + "\n" + @"{ - ""Path"": ""test"", - ""Version"": ""v2.0.0"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" -}" + "\n" + @"{ - ""Path"": ""other"", - ""Version"": ""v1.2.0"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" -}" + "\n" + @"{ - ""Path"": ""a"", - ""Version"": ""v1.5.0"", - ""Time"": ""2020-05-19T17:02:07Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" -}"; + var goMod = @" +module example.com/my/module + +go 1.11 + +require ( + some-package v1.2.3 // indirect + test v2.0.0 // indirect + other v1.2.0 // indirect + a v1.5.0 // indirect +); +"; var goGraph = "example.com/mainModule some-package@v1.2.3\nsome-package@v1.2.3 other@v1.0.0\nsome-package@v1.2.3 other@v1.2.0\ntest@v2.0.0 a@v1.5.0"; - this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, It.IsAny(), It.IsAny())) + string[] cmdParams = []; + this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, It.IsAny(), cmdParams)) .ReturnsAsync(true); - string[] cmdParams = ["list", "-mod=readonly", "-m", "-json", "all"]; - this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, It.IsAny(), cmdParams)) - .ReturnsAsync(new CommandLineExecutionResult - { - ExitCode = 0, - StdOut = buildDependencies, - }); + cmdParams = ["version"]; + this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, It.IsAny(), cmdParams)) + .ReturnsAsync(true); + this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, null, It.IsAny(), cmdParams)) + .ReturnsAsync(new CommandLineExecutionResult { ExitCode = 0, StdOut = "go version go1.24.3 windows/amd64" }); cmdParams = ["mod", "graph"]; - this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, It.IsAny(), cmdParams)) + this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, It.IsAny(), It.IsAny(), cmdParams)) .ReturnsAsync(new CommandLineExecutionResult { ExitCode = 0, @@ -507,9 +514,9 @@ public async Task TestGoDetector_GoGraphHappyPathAsync() }); this.envVarService.Setup(x => x.IsEnvironmentVariableValueTrue("DisableGoCliScan")).Returns(false); - + this.SetupActualGoModParser(); var (scanResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("go.mod", string.Empty) + .WithFile("go.mod", goMod) .ExecuteDetectorAsync(); scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); @@ -526,44 +533,32 @@ public async Task TestGoDetector_GoGraphHappyPathAsync() [TestMethod] public async Task TestGoDetector_GoGraphCyclicDependenciesAsync() { - var buildDependencies = @"{ - ""Path"": ""github.com/prometheus/common"", - ""Version"": ""v0.32.1"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" -}" + "\n" + @"{ - ""Path"": ""github.com/prometheus/client_golang"", - ""Version"": ""v1.11.0"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" -}" + "\n" + @"{ - ""Path"": ""github.com/prometheus/client_golang"", - ""Version"": ""v1.12.1"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" -}"; + var goMod = @" + module example.com/my/module + +go 1.11 + +require ( + github.com/prometheus/common v0.32.1 // indirect + github.com/prometheus/client_golang v1.12.1 // indirect +github.com/prometheus/client_golang v1.11.0 // indirect +) +"; var goGraph = @" github.com/prometheus/common@v0.32.1 github.com/prometheus/client_golang@v1.11.0 github.com/prometheus/client_golang@v1.12.1 github.com/prometheus/common@v0.32.1"; - this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, It.IsAny(), It.IsAny())) + string[] cmdParams = []; + this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, It.IsAny(), cmdParams)) .ReturnsAsync(true); - string[] cmdParams = ["list", "-mod=readonly", "-m", "-json", "all"]; - this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, It.IsAny(), cmdParams)) - .ReturnsAsync(new CommandLineExecutionResult - { - ExitCode = 0, - StdOut = buildDependencies, - }); + cmdParams = ["version"]; + this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, It.IsAny(), cmdParams)) + .ReturnsAsync(true); + this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, null, It.IsAny(), cmdParams)) + .ReturnsAsync(new CommandLineExecutionResult { ExitCode = 0, StdOut = "go version go1.24.3 windows/amd64" }); cmdParams = ["mod", "graph"]; - this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, It.IsAny(), cmdParams)) + this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, It.IsAny(), It.IsAny(), cmdParams)) .ReturnsAsync(new CommandLineExecutionResult { ExitCode = 0, @@ -572,8 +567,9 @@ public async Task TestGoDetector_GoGraphCyclicDependenciesAsync() this.envVarService.Setup(x => x.IsEnvironmentVariableValueTrue("DisableGoCliScan")).Returns(false); + this.SetupActualGoModParser(); var (scanResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("go.mod", string.Empty) + .WithFile("go.mod", goMod) .ExecuteDetectorAsync(); scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); @@ -593,57 +589,37 @@ public async Task TestGoDetector_GoCliRequiresEnvVarToRunAsync() [TestMethod] public async Task TestGoDetector_GoGraphReplaceWithRelativePathAsync() { - var buildDependencies = @"{ - ""Path"": ""some-package"", - ""Version"": ""v1.2.3"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" -}" + "\n" + @"{ - ""Path"": ""test"", - ""Version"": ""v2.0.0"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" - -}" + "\n" + @"{ - ""Path"": ""other"", - ""Version"": ""v1.2.0"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" -}" + "\n" + @"{ - ""Path"": ""a"", - ""Version"": ""v1.5.0"", - ""Time"": ""2020-05-19T17:02:07Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"", - ""Replace"": { - ""Path"": ""C:\\test\\module\\"", - ""Version"": null, - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"", - } -}"; + var localPath = OperatingSystem.IsWindows() + ? "C:/test/module/" + : "/home/test/module/"; + var goMod = @"module example.com/project + +go 1.11 + +require ( + some-package v1.2.3 // indirect + test v2.0.0 // indirect + other v1.2.0 // indirect + a v1.5.0 // indirect +) +replace a v1.5.0 => {LOCAL_MODULE_PATH} +"; + + goMod = goMod.Replace("{LOCAL_MODULE_PATH", localPath); var goGraph = "example.com/mainModule some-package@v1.2.3\nsome-package@v1.2.3 other@v1.0.0\nsome-package@v1.2.3 other@v1.2.0\ntest@v2.0.0 a@v1.5.0"; - this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, It.IsAny(), It.IsAny())) + string[] cmdParams = []; + this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, It.IsAny(), cmdParams)) .ReturnsAsync(true); - this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, It.IsAny(), new[] { "list", "-mod=readonly", "-m", "-json", "all" })) - .ReturnsAsync(new CommandLineExecutionResult - { - ExitCode = 0, - StdOut = buildDependencies, - }); + cmdParams = ["version"]; + this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, It.IsAny(), cmdParams)) + .ReturnsAsync(true); + this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, null, It.IsAny(), cmdParams)) + .ReturnsAsync(new CommandLineExecutionResult { ExitCode = 0, StdOut = "go version go1.24.3 windows/amd64" }); - this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, It.IsAny(), new[] { "mod", "graph" })) + cmdParams = ["mod", "graph"]; + this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, It.IsAny(), It.IsAny(), cmdParams)) .ReturnsAsync(new CommandLineExecutionResult { ExitCode = 0, @@ -654,9 +630,9 @@ public async Task TestGoDetector_GoGraphReplaceWithRelativePathAsync() this.fileUtilityServiceMock.Setup(fs => fs.Exists(It.IsAny())) .Returns(true); - + this.SetupActualGoModParser(); var (scanResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("go.mod", string.Empty) + .WithFile("go.mod", goMod) .ExecuteDetectorAsync(); scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); @@ -668,70 +644,40 @@ public async Task TestGoDetector_GoGraphReplaceWithRelativePathAsync() detectedComponents.Should().ContainSingle(component => component.Component.Id == "other v1.2.0 - Go"); detectedComponents.Should().ContainSingle(component => component.Component.Id == "test v2.0.0 - Go"); detectedComponents.Should().ContainSingle(component => component.Component.Id == "some-package v1.2.3 - Go"); - this.mockLogger.Verify( - logger => logger.Log( - LogLevel.Information, - It.IsAny(), - It.Is((v, t) => v.ToString().Contains("go Module a v1.5.0 is being replaced with module at path")), - It.IsAny(), - It.IsAny>()), - Times.Once); } [TestMethod] - public async Task TestGoDetector_GoGraphReplaceWithRelativePathDoesNotContainGoModFileAsync() + public async Task TestGoDetector_GoGraphReplaceMultipleReplaceModulesAsync() { - var buildDependencies = @"{ - ""Path"": ""some-package"", - ""Version"": ""v1.2.3"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" -}" + "\n" + @"{ - ""Path"": ""test"", - ""Version"": ""v2.0.0"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" - -}" + "\n" + @"{ - ""Path"": ""other"", - ""Version"": ""v1.2.0"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" -}" + "\n" + @"{ - ""Path"": ""a"", - ""Version"": ""v1.5.0"", - ""Time"": ""2020-05-19T17:02:07Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"", - ""Replace"": { - ""Path"": ""C:\\test\\module\\"", - ""Version"": null, - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"", - } -}"; + var goMod = @" +module example.com/project - var goGraph = "example.com/mainModule some-package@v1.2.3\nsome-package@v1.2.3 other@v1.0.0\nsome-package@v1.2.3 other@v1.2.0\ntest@v2.0.0 a@v1.5.0"; - this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, It.IsAny(), It.IsAny())) +go 1.17 + +require ( + some-package v1.2.3 // indirect + test v2.0.0 // indirect + other v1.2.0 // indirect + github v1.5.0 // indirect +) + +replace other v1.2.0 => ./component +replace github v1.5.0 => ./module +"; + + var goGraph = "example.com/mainModule some-package@v1.2.3\nsome-package@v1.2.3 other@v1.0.0\nsome-package@v1.2.3 other@v1.2.0\ntest@v2.0.0 github@v1.5.0"; + string[] cmdParams = []; + this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, It.IsAny(), cmdParams)) .ReturnsAsync(true); - this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, It.IsAny(), new[] { "list", "-mod=readonly", "-m", "-json", "all" })) - .ReturnsAsync(new CommandLineExecutionResult - { - ExitCode = 0, - StdOut = buildDependencies, - }); + cmdParams = ["version"]; + this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, It.IsAny(), cmdParams)) + .ReturnsAsync(true); + this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, null, It.IsAny(), cmdParams)) + .ReturnsAsync(new CommandLineExecutionResult { ExitCode = 0, StdOut = "go version go1.24.3 windows/amd64" }); - this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, It.IsAny(), new[] { "mod", "graph" })) + cmdParams = ["mod", "graph"]; + this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, It.IsAny(), It.IsAny(), cmdParams)) .ReturnsAsync(new CommandLineExecutionResult { ExitCode = 0, @@ -741,94 +687,55 @@ public async Task TestGoDetector_GoGraphReplaceWithRelativePathDoesNotContainGoM this.envVarService.Setup(x => x.IsEnvironmentVariableValueTrue("DisableGoCliScan")).Returns(false); this.fileUtilityServiceMock.Setup(fs => fs.Exists(It.IsAny())) - .Returns(false); + .Returns(true); + this.SetupActualGoModParser(); var (scanResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("go.mod", string.Empty) + .WithFile("go.mod", goMod) .ExecuteDetectorAsync(); scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); var detectedComponents = componentRecorder.GetDetectedComponents(); - detectedComponents.Should().HaveCount(4); + detectedComponents.Should().HaveCount(2); + detectedComponents.Should().NotContain(component => component.Component.Id == "a v1.5.0 - Go"); detectedComponents.Should().NotContain(component => component.Component.Id == "other v1.0.0 - Go"); - detectedComponents.Should().ContainSingle(component => component.Component.Id == "a v1.5.0 - Go"); - detectedComponents.Should().ContainSingle(component => component.Component.Id == "other v1.2.0 - Go"); + detectedComponents.Should().NotContain(component => component.Component.Id == "other v1.2.0 - Go"); detectedComponents.Should().ContainSingle(component => component.Component.Id == "test v2.0.0 - Go"); detectedComponents.Should().ContainSingle(component => component.Component.Id == "some-package v1.2.3 - Go"); - this.mockLogger.Verify( - logger => logger.Log( - LogLevel.Warning, - It.IsAny(), - It.Is((v, t) => v.ToString().Contains("does not exist in the relative path given for replacement")), - It.IsAny(), - It.IsAny>()), - Times.Once); } [TestMethod] - public async Task TestGoDetector_GoGraphReplaceMultipleReplaceModulesAsync() + public async Task TestGoDetector_GoGraphReplaceNoPathAsync() { - var buildDependencies = @"{ - ""Path"": ""some-package"", - ""Version"": ""v1.2.3"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" -}" + "\n" + @"{ - ""Path"": ""test"", - ""Version"": ""v2.0.0"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" - -}" + "\n" + @"{ - ""Path"": ""other"", - ""Version"": ""v1.2.0"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"", - ""Replace"": { - ""Path"": ""C:\\test\\component\\"", - ""Version"": null, - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\component\\go.mod"", - ""GoVersion"": ""1.15"", - } - -}" + "\n" + @"{ - ""Path"": ""github"", - ""Version"": ""v1.5.0"", - ""Time"": ""2020-05-19T17:02:07Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"", - ""Replace"": { - ""Path"": ""C:\\test\\module\\"", - ""Version"": null, - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\module\\go.mod"", - ""GoVersion"": ""1.11"", - } -}"; + var goMod = @" +module example.com/project + +go 1.11 + +require ( + some-package v1.2.3 // indirect + test v2.0.0 // indirect + other v1.2.0 // indirect + github v1.5.0 // indirect +) + +replace github v1.5.0 => github v1.18 +"; var goGraph = "example.com/mainModule some-package@v1.2.3\nsome-package@v1.2.3 other@v1.0.0\nsome-package@v1.2.3 other@v1.2.0\ntest@v2.0.0 github@v1.5.0"; - this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, It.IsAny(), It.IsAny())) + string[] cmdParams = []; + this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, It.IsAny(), cmdParams)) .ReturnsAsync(true); - this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, It.IsAny(), new[] { "list", "-mod=readonly", "-m", "-json", "all" })) - .ReturnsAsync(new CommandLineExecutionResult - { - ExitCode = 0, - StdOut = buildDependencies, - }); + cmdParams = ["version"]; + this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, It.IsAny(), cmdParams)) + .ReturnsAsync(true); + this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, null, It.IsAny(), cmdParams)) + .ReturnsAsync(new CommandLineExecutionResult { ExitCode = 0, StdOut = "go version go1.24.3 windows/amd64" }); - this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, It.IsAny(), new[] { "mod", "graph" })) + cmdParams = ["mod", "graph"]; + this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, It.IsAny(), It.IsAny(), cmdParams)) .ReturnsAsync(new CommandLineExecutionResult { ExitCode = 0, @@ -836,213 +743,245 @@ public async Task TestGoDetector_GoGraphReplaceMultipleReplaceModulesAsync() }); this.envVarService.Setup(x => x.IsEnvironmentVariableValueTrue("DisableGoCliScan")).Returns(false); - - this.fileUtilityServiceMock.Setup(fs => fs.Exists(It.IsAny())) - .Returns(true); - + this.SetupActualGoModParser(); var (scanResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("go.mod", string.Empty) + .WithFile("go.mod", goMod) .ExecuteDetectorAsync(); scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); var detectedComponents = componentRecorder.GetDetectedComponents(); - detectedComponents.Should().HaveCount(2); - detectedComponents.Should().NotContain(component => component.Component.Id == "a v1.5.0 - Go"); + detectedComponents.Should().HaveCount(4); detectedComponents.Should().NotContain(component => component.Component.Id == "other v1.0.0 - Go"); - detectedComponents.Should().NotContain(component => component.Component.Id == "other v1.2.0 - Go"); + detectedComponents.Should().ContainSingle(component => component.Component.Id == "github v1.18 - Go"); + detectedComponents.Should().NotContain(component => component.Component.Id == "github v1.5.0 - Go"); + detectedComponents.Should().ContainSingle(component => component.Component.Id == "other v1.2.0 - Go"); detectedComponents.Should().ContainSingle(component => component.Component.Id == "test v2.0.0 - Go"); detectedComponents.Should().ContainSingle(component => component.Component.Id == "some-package v1.2.3 - Go"); this.mockLogger.Verify( logger => logger.Log( LogLevel.Information, It.IsAny(), - It.Is((v, t) => v.ToString().Contains("go Module other v1.2.0 is being replaced with module at path")), + It.Is((v, t) => v.ToString().Equals("go Module github-v1.5.0 being replaced with github-v1.18")), It.IsAny(), It.IsAny>()), - Times.Once); - this.mockLogger.Verify( - logger => logger.Log( - LogLevel.Information, - It.IsAny(), - It.Is((v, t) => v.ToString().Contains("go Module github v1.5.0 is being replaced with module at path")), - It.IsAny(), - It.IsAny>()), - Times.Once); + Times.Never); } [TestMethod] - public async Task TestGoDetector_GoGraphReplaceNoPathAsync() + public async Task GoModDetector_GoCliIs1_11OrGreater_GoGraphIsExecutedAsync() { - var buildDependencies = @"{ - ""Path"": ""some-package"", - ""Version"": ""v1.2.3"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" -}" + "\n" + @"{ - ""Path"": ""test"", - ""Version"": ""v2.0.0"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" - -}" + "\n" + @"{ - ""Path"": ""other"", - ""Version"": ""v1.2.0"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" - -}" + "\n" + @"{ - ""Path"": ""github"", - ""Version"": ""v1.5.0"", - ""Time"": ""2020-05-19T17:02:07Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"", - ""Replace"": { - ""Path"": null, - ""Version"": ""v1.18"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\module\\go.mod"", - ""GoVersion"": ""1.11"", - } -}"; + var goMod = string.Empty; + this.SetupMockGoModParser(); - var goGraph = "example.com/mainModule some-package@v1.2.3\nsome-package@v1.2.3 other@v1.0.0\nsome-package@v1.2.3 other@v1.2.0\ntest@v2.0.0 github@v1.5.0"; - this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, It.IsAny(), It.IsAny())) + this.envVarService.Setup(x => x.IsEnvironmentVariableValueTrue("DisableGoCliScan")).Returns(false); + + string[] cmdParams = []; + this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, null, It.IsAny())) .ReturnsAsync(true); - this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, It.IsAny(), new[] { "list", "-mod=readonly", "-m", "-json", "all" })) + cmdParams = ["version"]; + this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, null, default, It.Is(p => p.SequenceEqual(new List { "version" }.ToArray())))) .ReturnsAsync(new CommandLineExecutionResult { ExitCode = 0, - StdOut = buildDependencies, + StdOut = "go version go1.23.6 windows/amd64", }); - this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, It.IsAny(), new[] { "mod", "graph" })) - .ReturnsAsync(new CommandLineExecutionResult - { - ExitCode = 0, - StdOut = goGraph, - }); + cmdParams = ["mod", "graph"]; + this.commandLineMock.Setup(service => service.ExecuteCommandAsync( + "go", + null, + It.IsAny(), + default, + It.Is(args => args.Length == 2 && args[0] == "mod" && args[1] == "graph"))) + .ReturnsAsync(new CommandLineExecutionResult + { + ExitCode = 0, + StdOut = string.Empty, + }) + .Verifiable(); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("go.mod", goMod) + .ExecuteDetectorAsync(); + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + this.commandLineMock.Verify(); + } + + [TestMethod] + public async Task GoModDetector_GoCliIs111OrLessThan1_11_GoGraphIsNotExecutedAsync() + { + var goMod = string.Empty; + this.SetupMockGoModParser(); + + string[] cmdParams = []; this.envVarService.Setup(x => x.IsEnvironmentVariableValueTrue("DisableGoCliScan")).Returns(false); + this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, null, It.IsAny())) + .ReturnsAsync(true); + + cmdParams = ["version"]; + this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, null, default, cmdParams)) + .ReturnsAsync(new CommandLineExecutionResult + { + ExitCode = 0, + StdOut = "go version go1.10.6 windows/amd64", + }); + var (scanResult, componentRecorder) = await this.DetectorTestUtility - .WithFile("go.mod", string.Empty) + .WithFile("go.mod", goMod) .ExecuteDetectorAsync(); scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); - var detectedComponents = componentRecorder.GetDetectedComponents(); - detectedComponents.Should().HaveCount(4); - detectedComponents.Should().NotContain(component => component.Component.Id == "other v1.0.0 - Go"); - detectedComponents.Should().NotContain(component => component.Component.Id == "github v1.18 - Go"); - detectedComponents.Should().ContainSingle(component => component.Component.Id == "github v1.5.0 - Go"); - detectedComponents.Should().ContainSingle(component => component.Component.Id == "github v1.5.0 - Go"); - detectedComponents.Should().ContainSingle(component => component.Component.Id == "other v1.2.0 - Go"); - detectedComponents.Should().ContainSingle(component => component.Component.Id == "test v2.0.0 - Go"); - detectedComponents.Should().ContainSingle(component => component.Component.Id == "some-package v1.2.3 - Go"); - this.mockLogger.Verify( - logger => logger.Log( - LogLevel.Information, - It.IsAny(), - It.Is((v, t) => v.ToString().Equals("go Module github v1.5.0 being replaced with module github v1.18")), - It.IsAny(), - It.IsAny>()), - Times.Never); + this.commandLineMock.Verify( + service => service.ExecuteCommandAsync( + "go", + null, + It.IsAny(), + default, + It.Is(args => args.Length == 2 && args[0] == "mod" && args[1] == "graph")), + times: Times.Never); } [TestMethod] - public async Task TestGoDetector_GoGraphReplacePathAndVersionAsync() + public async Task GoModDetector_GoModFileFound_GoModParserIsExecuted() { - var buildDependencies = @"{ - ""Path"": ""some-package"", - ""Version"": ""v1.2.3"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" -}" + "\n" + @"{ - ""Path"": ""test"", - ""Version"": ""v2.0.0"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" - -}" + "\n" + @"{ - ""Path"": ""other"", - ""Version"": ""v1.2.0"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"" - -}" + "\n" + @"{ - ""Path"": ""github"", - ""Version"": ""v1.5.0"", - ""Time"": ""2020-05-19T17:02:07Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\go.mod"", - ""GoVersion"": ""1.11"", - ""Replace"": { - ""Path"": ""github"", - ""Version"": ""v1.18"", - ""Time"": ""2021-12-06T23:04:27Z"", - ""Indirect"": true, - ""GoMod"": ""C:\\test\\module\\go.mod"", - ""GoVersion"": ""1.11"", - } -}"; + var goModParserMock = new Mock(); + this.mockParserFactory.Setup(x => x.CreateParser(GoParserType.GoMod, It.IsAny())).Returns(goModParserMock.Object); - var goGraph = "example.com/mainModule some-package@v1.2.3\nsome-package@v1.2.3 other@v1.0.0\nsome-package@v1.2.3 other@v1.2.0\ntest@v2.0.0 github@v1.5.0"; - this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, It.IsAny(), It.IsAny())) - .ReturnsAsync(true); + this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, null, It.Is(p => p.SequenceEqual(new List { "version" }.ToArray())))) + .ReturnsAsync(true); - this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, It.IsAny(), new[] { "list", "-mod=readonly", "-m", "-json", "all" })) - .ReturnsAsync(new CommandLineExecutionResult - { - ExitCode = 0, - StdOut = buildDependencies, - }); + this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, null, default, It.Is(p => p.SequenceEqual(new List { "version" }.ToArray())))) + .ReturnsAsync(new CommandLineExecutionResult + { + ExitCode = 0, + StdOut = "go version go1.10.6 windows/amd64", + }); - this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, It.IsAny(), new[] { "mod", "graph" })) - .ReturnsAsync(new CommandLineExecutionResult - { - ExitCode = 0, - StdOut = goGraph, - }); + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("go.mod", string.Empty) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + goModParserMock.Verify(parser => parser.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + } + + /// + /// Verifies that if Go CLI is enabled/available and succeeds, go.sum file is not parsed and vice-versa. + /// + /// Task. + [DataTestMethod] + [DataRow(true)] + [DataRow(false)] + public async Task GoDetector_GoSum_GoSumParserExecuted(bool goCliSucceeds) + { + var nInvocationsOfSumParser = goCliSucceeds ? 0 : 1; + this.SetupMockGoSumParser(); + this.SetupMockGoCLIParser(); this.envVarService.Setup(x => x.IsEnvironmentVariableValueTrue("DisableGoCliScan")).Returns(false); + // Setup go cli parser to succeed/fail + this.mockGoCliParser.Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(goCliSucceeds); + + // Setup go sum parser to succeed + this.mockGoSumParser.Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(true); + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("go.sum", string.Empty) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + this.mockParserFactory.Verify(clm => clm.CreateParser(GoParserType.GoSum, It.IsAny()), nInvocationsOfSumParser == 0 ? Times.Never : Times.Once); + } + + /// + /// Verifies that if Go CLI is disabled, go.sum is parsed. + /// + /// Task. + [TestMethod] + public async Task GoDetector_GoSum_GoSumParserExecutedIfCliDisabled() + { + var goSumParserMock = new Mock(); + this.mockParserFactory.Setup(x => x.CreateParser(GoParserType.GoSum, It.IsAny())).Returns(goSumParserMock.Object); + + // Setup environment variable to disable CLI scan + this.envVarService.Setup(s => s.IsEnvironmentVariableValueTrue("DisableGoCliScan")).Returns(true); + + // Setup go sum parser to succed + goSumParserMock.Setup(p => p.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(true); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("go.sum", string.Empty) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + this.mockParserFactory.Verify(clm => clm.CreateParser(GoParserType.GoSum, It.IsAny()), Times.Once); + } + + [TestMethod] + public async Task GoModDetector_ExecutingGoVersionFails_DetectorDoesNotFail() + { + this.SetupMockGoModParser(); + + this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, null, It.Is(p => p.SequenceEqual(new List { "version" }.ToArray())))) + .ReturnsAsync(true); + + this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, null, default, It.Is(p => p.SequenceEqual(new List { "version" }.ToArray())))) + .Throws(new InvalidOperationException("Failed to execute go version")); + var (scanResult, componentRecorder) = await this.DetectorTestUtility .WithFile("go.mod", string.Empty) .ExecuteDetectorAsync(); scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); - var detectedComponents = componentRecorder.GetDetectedComponents(); - detectedComponents.Should().HaveCount(4); - detectedComponents.Should().NotContain(component => component.Component.Id == "github v1.5.0 - Go"); - detectedComponents.Should().NotContain(component => component.Component.Id == "other v1.0.0 - Go"); - detectedComponents.Should().ContainSingle(component => component.Component.Id == "github v1.18 - Go"); - detectedComponents.Should().ContainSingle(component => component.Component.Id == "other v1.2.0 - Go"); - detectedComponents.Should().ContainSingle(component => component.Component.Id == "test v2.0.0 - Go"); - detectedComponents.Should().ContainSingle(component => component.Component.Id == "some-package v1.2.3 - Go"); - this.mockLogger.Verify( - logger => logger.Log( - LogLevel.Information, - It.IsAny(), - It.Is((v, t) => v.ToString().Equals("go Module github v1.5.0 being replaced with module github v1.18")), - It.IsAny(), - It.IsAny>()), - Times.Once); + this.mockGoModParser.Verify(parser => parser.ParseAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + } + + [TestMethod] + public async Task GoModDetector_VerifyLocalReferencesIgnored() + { + var goModFilePath = "./TestFiles/go_WithLocalReferences.mod"; // Replace with your actual file path + var fileStream = new FileStream(goModFilePath, FileMode.Open, FileAccess.Read); + + var goModParser = new GoModParser(this.mockLogger.Object); + var mockSingleFileComponentRecorder = new Mock(); + + var capturedComponents = new List(); + var expectedComponentIds = new List() + { + "github.com/grafana/grafana-app-sdk v0.22.1 - Go", + "k8s.io/kube-openapi v1.1.1 - Go", + }; + + mockSingleFileComponentRecorder + .Setup(m => m.RegisterUsage( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Callback((comp, _, _, _, _, _) => + { + capturedComponents.Add(comp); + }); + + var mockComponentStream = new Mock(); + mockComponentStream.Setup(mcs => mcs.Stream).Returns(fileStream); + mockComponentStream.Setup(mcs => mcs.Location).Returns("Location"); + + var result = await goModParser.ParseAsync(mockSingleFileComponentRecorder.Object, mockComponentStream.Object, new GoGraphTelemetryRecord()); + result.Should().BeTrue(); + capturedComponents + .Select(c => c.Component.Id) + .Should() + .BeEquivalentTo(expectedComponentIds); } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/TestFiles/go_WithLocalReferences.mod b/test/Microsoft.ComponentDetection.Detectors.Tests/TestFiles/go_WithLocalReferences.mod index 3b2af55dc..95c996030 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/TestFiles/go_WithLocalReferences.mod +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/TestFiles/go_WithLocalReferences.mod @@ -1,4 +1,4 @@ -module github.com/Go117Tests +module github.com/GoTests go 1.23.5