diff --git a/Directory.Packages.props b/Directory.Packages.props index f0565ac4f..b05540a66 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -17,7 +17,7 @@ - + diff --git a/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj b/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj index adebad6c7..d8afad8cd 100644 --- a/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj +++ b/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj @@ -9,7 +9,7 @@ - + diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/RustSbomDetector.cs b/src/Microsoft.ComponentDetection.Detectors/rust/RustSbomDetector.cs index 9212af429..38dd575bd 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/RustSbomDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/RustSbomDetector.cs @@ -9,11 +9,11 @@ namespace Microsoft.ComponentDetection.Detectors.Rust; using System.Reactive.Threading.Tasks; using System.Threading; using System.Threading.Tasks; -using global::DotNet.Globbing; using Microsoft.ComponentDetection.Common.Telemetry.Records; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.FileSystemGlobbing; using Microsoft.Extensions.Logging; using Tomlyn; using Tomlyn.Model; @@ -408,9 +408,9 @@ private bool ShouldSkip(string directory, FileKind fileKind, string fullPath) var relativePath = this.GetRelativePath(rule.Root, normalizedDir); - // Match against include globs - var matchesInclude = rule.IncludeGlobs.Any(g => - g.IsMatch(relativePath) || g.IsMatch(normalizedFullPath)); + // Match against include globs (try both relative and full path) + var matchesInclude = rule.IncludeMatcher.Match(relativePath).HasMatches + || rule.IncludeMatcher.Match(normalizedFullPath).HasMatches; if (!matchesInclude) { @@ -418,8 +418,8 @@ private bool ShouldSkip(string directory, FileKind fileKind, string fullPath) } // Match against exclude globs - var matchesExclude = rule.ExcludeGlobs.Any(g => - g.IsMatch(relativePath) || g.IsMatch(normalizedFullPath)); + var matchesExclude = rule.ExcludeMatcher.Match(relativePath).HasMatches + || rule.ExcludeMatcher.Match(normalizedFullPath).HasMatches; if (matchesExclude) { @@ -441,7 +441,7 @@ private bool ShouldSkip(string directory, FileKind fileKind, string fullPath) /// Collection of glob patterns to exclude workspace members (e.g., "examples/*", "tests/*"). /// /// This method normalizes all paths and patterns for cross-platform compatibility. - /// On Windows, patterns are evaluated case-insensitively, while on other platforms they are case-sensitive. + /// Patterns are always evaluated case-insensitively. /// The glob rule is used to determine whether files in descendant directories should be skipped during detection. /// private void AddGlobRule(string root, IEnumerable includes, IEnumerable excludes) @@ -450,26 +450,18 @@ private void AddGlobRule(string root, IEnumerable includes, IEnumerable< var includesList = includes?.ToList() ?? []; var excludesList = excludes?.ToList() ?? []; - var globOptions = new GlobOptions - { - Evaluation = new EvaluationOptions - { - CaseInsensitive = true, - }, - }; - - var includeGlobs = new List(); + var includeMatcher = new Matcher(StringComparison.OrdinalIgnoreCase); foreach (var pattern in includesList) { var normalizedPattern = this.pathUtilityService.NormalizePath(pattern); - includeGlobs.Add(Glob.Parse(normalizedPattern, globOptions)); + includeMatcher.AddInclude(normalizedPattern); } - var excludeGlobs = new List(); + var excludeMatcher = new Matcher(StringComparison.OrdinalIgnoreCase); foreach (var pattern in excludesList) { var normalizedPattern = this.pathUtilityService.NormalizePath(pattern); - excludeGlobs.Add(Glob.Parse(normalizedPattern, globOptions)); + excludeMatcher.AddInclude(normalizedPattern); } var rule = new GlobRule @@ -477,8 +469,8 @@ private void AddGlobRule(string root, IEnumerable includes, IEnumerable< Root = normalizedRoot, Includes = includesList, Excludes = excludesList, - IncludeGlobs = includeGlobs, - ExcludeGlobs = excludeGlobs, + IncludeMatcher = includeMatcher, + ExcludeMatcher = excludeMatcher, }; this.visitedGlobRules.Add(rule); @@ -774,8 +766,8 @@ private class GlobRule public List Excludes { get; set; } - public List IncludeGlobs { get; set; } + public Matcher IncludeMatcher { get; set; } - public List ExcludeGlobs { get; set; } + public Matcher ExcludeMatcher { get; set; } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs index 5f743585c..4d7d8dd1b 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs @@ -8,11 +8,11 @@ namespace Microsoft.ComponentDetection.Detectors.Yarn; using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; -using global::DotNet.Globbing; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.Npm; +using Microsoft.Extensions.FileSystemGlobbing; using Microsoft.Extensions.Logging; public class YarnLockComponentDetector : FileComponentDetector @@ -259,21 +259,24 @@ private bool TryReadPeerPackageJsonRequestsAsYarnEntries(ISingleFileComponentRec private void GetWorkspaceDependencies(IList yarnWorkspaces, DirectoryInfo root, IDictionary> dependencies, IDictionary workspaceDependencyVsLocationMap) { - var ignoreCase = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); - - var globOptions = new GlobOptions() - { - Evaluation = new EvaluationOptions() - { - CaseInsensitive = ignoreCase, - }, - }; + var comparison = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? StringComparison.OrdinalIgnoreCase + : StringComparison.Ordinal; foreach (var workspacePattern in yarnWorkspaces) { - var glob = Glob.Parse($"{root.FullName.Replace('\\', '/')}/{workspacePattern}/package.json", globOptions); + var matcher = new Matcher(comparison); + matcher.AddInclude($"{workspacePattern}/package.json"); - var componentStreams = this.ComponentStreamEnumerableFactory.GetComponentStreams(root, (file) => glob.IsMatch(file.FullName.Replace('\\', '/')), null, true); + var componentStreams = this.ComponentStreamEnumerableFactory.GetComponentStreams( + root, + (file) => + { + var relativePath = Path.GetRelativePath(root.FullName, file.FullName).Replace('\\', '/'); + return matcher.Match(relativePath).HasMatches; + }, + null, + true); foreach (var stream in componentStreams) { diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Microsoft.ComponentDetection.Orchestrator.csproj b/src/Microsoft.ComponentDetection.Orchestrator/Microsoft.ComponentDetection.Orchestrator.csproj index cca6e05b2..de8d469fd 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Microsoft.ComponentDetection.Orchestrator.csproj +++ b/src/Microsoft.ComponentDetection.Orchestrator/Microsoft.ComponentDetection.Orchestrator.csproj @@ -9,7 +9,7 @@ - + diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs index d3a7d9fe4..fae8f2ed0 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs @@ -10,7 +10,6 @@ namespace Microsoft.ComponentDetection.Orchestrator.Services; using System.Text.Json; using System.Threading; using System.Threading.Tasks; -using DotNet.Globbing; using Microsoft.ComponentDetection.Common; using Microsoft.ComponentDetection.Common.DependencyGraph; using Microsoft.ComponentDetection.Common.Telemetry.Records; @@ -18,6 +17,7 @@ namespace Microsoft.ComponentDetection.Orchestrator.Services; using Microsoft.ComponentDetection.Contracts.BcdeModels; using Microsoft.ComponentDetection.Orchestrator.Commands; using Microsoft.ComponentDetection.Orchestrator.Experiments; +using Microsoft.Extensions.FileSystemGlobbing; using Microsoft.Extensions.Logging; using Spectre.Console; using static System.Environment; @@ -249,35 +249,49 @@ public ExcludeDirectoryPredicate GenerateDirectoryExclusionPredicate(string orig }; } - var minimatchers = new Dictionary(); + var comparison = ignoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + var matcher = new Matcher(comparison); - var globOptions = new GlobOptions() + foreach (var directoryExclusion in directoryExclusionList) { - Evaluation = new EvaluationOptions() + if (!allowWindowsPaths && directoryExclusion.Contains('\\')) { - CaseInsensitive = ignoreCase, - }, - }; + this.logger.LogDebug("Skipping directory exclusion pattern {Pattern} because it contains backslashes and Windows-style paths are not enabled.", directoryExclusion); + continue; + } - foreach (var directoryExclusion in directoryExclusionList) - { - minimatchers.Add(directoryExclusion, Glob.Parse(allowWindowsPaths ? directoryExclusion : /* [] escapes special chars */ directoryExclusion.Replace("\\", "[\\]"), globOptions)); + var pattern = directoryExclusion.Replace('\\', '/'); + matcher.AddInclude(pattern); + + // FileSystemGlobbing's ** does not match zero trailing segments, + // so **/dir/** won't match "dir" itself. Add **/dir to cover that case. + if (pattern.EndsWith("/**")) + { + matcher.AddInclude(pattern[..^3]); + } } return (name, directoryName) => { - var path = Path.Combine(directoryName.ToString(), name.ToString()); + var path = Path.Combine(directoryName.ToString(), name.ToString()).Replace('\\', '/'); - return minimatchers.Any(minimatcherKeyValue => + // FileSystemGlobbing requires relative paths for matching. + // Strip the leading slash (or drive letter on Windows) so that + // patterns like **/dir/** can match against the full directory path. + var relativePath = path.StartsWith('/') ? path[1..] : path; + if (relativePath.Length > 1 && relativePath[1] == ':') { - if (minimatcherKeyValue.Value.IsMatch(path)) - { - this.logger.LogDebug("Excluding folder {Path} because it matched glob {Glob}.", path, minimatcherKeyValue.Key); - return true; - } + // Windows drive letter, e.g. "C:/foo" → "foo" + relativePath = relativePath[3..]; + } - return false; - }); + if (matcher.Match(relativePath).HasMatches) + { + this.logger.LogDebug("Excluding folder {Path} because it matched a directory exclusion glob.", path); + return true; + } + + return false; }; } diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs index 4ded253ea..9f4bd62ea 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs @@ -419,6 +419,38 @@ public void GenerateDirectoryExclusionPredicate_IgnoreCaseAndAllowWindowsPathsWo exclusionPredicate(dn, dp).Should().BeTrue(); } + [TestMethod] + public void GenerateDirectoryExclusionPredicate_TrailingDoubleStarMatchesDirectoryItself() + { + // FileSystemGlobbing's ** does not match zero trailing segments, + // so the implementation adds a companion pattern (**/dir) alongside **/dir/**. + // This test verifies that the directory itself is excluded, not just its children. + var args = new ScanSettings + { + SourceDirectory = new DirectoryInfo(this.isWin ? @"C:\project" : "/tmp/project"), + DetectorArgs = new Dictionary(), + DirectoryExclusionList = ["**/Source/**"], + }; + + var exclusionPredicate = this.serviceUnderTest.GenerateDirectoryExclusionPredicate( + args.SourceDirectory.FullName, + args.DirectoryExclusionList, + args.DirectoryExclusionListObsolete, + allowWindowsPaths: true, + ignoreCase: true); + + // The directory itself (no trailing segment) should be excluded + var projectPath = this.isWin ? @"C:\project" : "/tmp/project"; + exclusionPredicate("Source", projectPath).Should().BeTrue(); + + // A child under the directory should also be excluded + var sourcePath = this.isWin ? @"C:\project\Source" : "/tmp/project/Source"; + exclusionPredicate("child", sourcePath).Should().BeTrue(); + + // An unrelated directory should not be excluded + exclusionPredicate("Other", projectPath).Should().BeFalse(); + } + [TestMethod] public async Task ProcessDetectorsAsync_DirectoryExclusionPredicateWorksAsExpectedForObsoleteAsync() {