Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
<PackageVersion Include="Microsoft.Extensions.Logging" Version="8.0.1" />
<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
<PackageVersion Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.14.15" />
<PackageVersion Include="DotNet.Glob" Version="2.1.1" />
<PackageVersion Include="Microsoft.Extensions.FileSystemGlobbing" Version="8.0.0" />
<PackageVersion Include="MinVer" Version="7.0.0" />
<PackageVersion Include="Moq" Version="4.18.4" />
<PackageVersion Include="morelinq" Version="4.4.0" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
</ItemGroup>

<ItemGroup>
<PackageReference Include="DotNet.Glob" />
<PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" />
<PackageReference Include="Microsoft.Extensions.Logging" />
<PackageReference Include="morelinq" />
<PackageReference Include="NuGet.ProjectModel" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@ namespace Microsoft.ComponentDetection.Detectors.Rust;
using System.Reactive.Threading.Tasks;
using System.Threading;
using System.Threading.Tasks;
using global::DotNet.Globbing;
using Microsoft.ComponentDetection.Common.Telemetry.Records;
using Microsoft.ComponentDetection.Contracts;
using Microsoft.ComponentDetection.Contracts.Internal;
using Microsoft.ComponentDetection.Contracts.TypedComponent;
using Microsoft.Extensions.FileSystemGlobbing;
using Microsoft.Extensions.Logging;
using Tomlyn;
using Tomlyn.Model;
Expand Down Expand Up @@ -408,18 +408,18 @@ private bool ShouldSkip(string directory, FileKind fileKind, string fullPath)

var relativePath = this.GetRelativePath(rule.Root, normalizedDir);

// Match against include globs
var matchesInclude = rule.IncludeGlobs.Any(g =>
g.IsMatch(relativePath) || g.IsMatch(normalizedFullPath));
// Match against include globs (try both relative and full path)
var matchesInclude = rule.IncludeMatcher.Match(relativePath).HasMatches
|| rule.IncludeMatcher.Match(normalizedFullPath).HasMatches;

if (!matchesInclude)
{
continue;
}

// Match against exclude globs
var matchesExclude = rule.ExcludeGlobs.Any(g =>
g.IsMatch(relativePath) || g.IsMatch(normalizedFullPath));
var matchesExclude = rule.ExcludeMatcher.Match(relativePath).HasMatches
|| rule.ExcludeMatcher.Match(normalizedFullPath).HasMatches;

if (matchesExclude)
{
Expand All @@ -441,7 +441,7 @@ private bool ShouldSkip(string directory, FileKind fileKind, string fullPath)
/// <param name="excludes">Collection of glob patterns to exclude workspace members (e.g., "examples/*", "tests/*").</param>
/// <remarks>
/// This method normalizes all paths and patterns for cross-platform compatibility.
/// On Windows, patterns are evaluated case-insensitively, while on other platforms they are case-sensitive.
/// Patterns are always evaluated case-insensitively.
/// The glob rule is used to determine whether files in descendant directories should be skipped during detection.
/// </remarks>
private void AddGlobRule(string root, IEnumerable<string> includes, IEnumerable<string> excludes)
Expand All @@ -450,35 +450,27 @@ private void AddGlobRule(string root, IEnumerable<string> includes, IEnumerable<
var includesList = includes?.ToList() ?? [];
var excludesList = excludes?.ToList() ?? [];

var globOptions = new GlobOptions
{
Evaluation = new EvaluationOptions
{
CaseInsensitive = true,
},
};

var includeGlobs = new List<Glob>();
var includeMatcher = new Matcher(StringComparison.OrdinalIgnoreCase);
foreach (var pattern in includesList)
{
var normalizedPattern = this.pathUtilityService.NormalizePath(pattern);
includeGlobs.Add(Glob.Parse(normalizedPattern, globOptions));
includeMatcher.AddInclude(normalizedPattern);
}

var excludeGlobs = new List<Glob>();
var excludeMatcher = new Matcher(StringComparison.OrdinalIgnoreCase);
foreach (var pattern in excludesList)
{
var normalizedPattern = this.pathUtilityService.NormalizePath(pattern);
excludeGlobs.Add(Glob.Parse(normalizedPattern, globOptions));
excludeMatcher.AddInclude(normalizedPattern);
}

var rule = new GlobRule
{
Root = normalizedRoot,
Includes = includesList,
Excludes = excludesList,
IncludeGlobs = includeGlobs,
ExcludeGlobs = excludeGlobs,
IncludeMatcher = includeMatcher,
ExcludeMatcher = excludeMatcher,
};

this.visitedGlobRules.Add(rule);
Expand Down Expand Up @@ -774,8 +766,8 @@ private class GlobRule

public List<string> Excludes { get; set; }

public List<Glob> IncludeGlobs { get; set; }
public Matcher IncludeMatcher { get; set; }

public List<Glob> ExcludeGlobs { get; set; }
public Matcher ExcludeMatcher { get; set; }
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@ namespace Microsoft.ComponentDetection.Detectors.Yarn;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using global::DotNet.Globbing;
using Microsoft.ComponentDetection.Contracts;
using Microsoft.ComponentDetection.Contracts.Internal;
using Microsoft.ComponentDetection.Contracts.TypedComponent;
using Microsoft.ComponentDetection.Detectors.Npm;
using Microsoft.Extensions.FileSystemGlobbing;
using Microsoft.Extensions.Logging;

public class YarnLockComponentDetector : FileComponentDetector
Expand Down Expand Up @@ -259,21 +259,24 @@ private bool TryReadPeerPackageJsonRequestsAsYarnEntries(ISingleFileComponentRec

private void GetWorkspaceDependencies(IList<string> yarnWorkspaces, DirectoryInfo root, IDictionary<string, IDictionary<string, bool>> dependencies, IDictionary<string, string> workspaceDependencyVsLocationMap)
{
var ignoreCase = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);

var globOptions = new GlobOptions()
{
Evaluation = new EvaluationOptions()
{
CaseInsensitive = ignoreCase,
},
};
var comparison = RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
? StringComparison.OrdinalIgnoreCase
: StringComparison.Ordinal;

foreach (var workspacePattern in yarnWorkspaces)
{
var glob = Glob.Parse($"{root.FullName.Replace('\\', '/')}/{workspacePattern}/package.json", globOptions);
var matcher = new Matcher(comparison);
matcher.AddInclude($"{workspacePattern}/package.json");

var componentStreams = this.ComponentStreamEnumerableFactory.GetComponentStreams(root, (file) => glob.IsMatch(file.FullName.Replace('\\', '/')), null, true);
var componentStreams = this.ComponentStreamEnumerableFactory.GetComponentStreams(
root,
(file) =>
{
var relativePath = Path.GetRelativePath(root.FullName, file.FullName).Replace('\\', '/');
return matcher.Match(relativePath).HasMatches;
},
null,
true);

foreach (var stream in componentStreams)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

<ItemGroup>
<PackageReference Include="CommandLineParser" />
<PackageReference Include="DotNet.Glob" />
<PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
<PackageReference Include="Microsoft.Extensions.Logging" />
<PackageReference Include="Newtonsoft.Json" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,14 @@ namespace Microsoft.ComponentDetection.Orchestrator.Services;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using DotNet.Globbing;
using Microsoft.ComponentDetection.Common;
using Microsoft.ComponentDetection.Common.DependencyGraph;
using Microsoft.ComponentDetection.Common.Telemetry.Records;
using Microsoft.ComponentDetection.Contracts;
using Microsoft.ComponentDetection.Contracts.BcdeModels;
using Microsoft.ComponentDetection.Orchestrator.Commands;
using Microsoft.ComponentDetection.Orchestrator.Experiments;
using Microsoft.Extensions.FileSystemGlobbing;
using Microsoft.Extensions.Logging;
using Spectre.Console;
using static System.Environment;
Expand Down Expand Up @@ -249,35 +249,49 @@ public ExcludeDirectoryPredicate GenerateDirectoryExclusionPredicate(string orig
};
}

var minimatchers = new Dictionary<string, Glob>();
var comparison = ignoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
var matcher = new Matcher(comparison);

var globOptions = new GlobOptions()
foreach (var directoryExclusion in directoryExclusionList)
{
Comment thread
JamieMagee marked this conversation as resolved.
Evaluation = new EvaluationOptions()
if (!allowWindowsPaths && directoryExclusion.Contains('\\'))
{
CaseInsensitive = ignoreCase,
},
};
this.logger.LogDebug("Skipping directory exclusion pattern {Pattern} because it contains backslashes and Windows-style paths are not enabled.", directoryExclusion);
continue;
}

foreach (var directoryExclusion in directoryExclusionList)
{
minimatchers.Add(directoryExclusion, Glob.Parse(allowWindowsPaths ? directoryExclusion : /* [] escapes special chars */ directoryExclusion.Replace("\\", "[\\]"), globOptions));
var pattern = directoryExclusion.Replace('\\', '/');
matcher.AddInclude(pattern);

// FileSystemGlobbing's ** does not match zero trailing segments,
// so **/dir/** won't match "dir" itself. Add **/dir to cover that case.
if (pattern.EndsWith("/**"))
{
matcher.AddInclude(pattern[..^3]);
}
Comment thread
JamieMagee marked this conversation as resolved.
}

return (name, directoryName) =>
{
var path = Path.Combine(directoryName.ToString(), name.ToString());
var path = Path.Combine(directoryName.ToString(), name.ToString()).Replace('\\', '/');

return minimatchers.Any(minimatcherKeyValue =>
// FileSystemGlobbing requires relative paths for matching.
// Strip the leading slash (or drive letter on Windows) so that
// patterns like **/dir/** can match against the full directory path.
var relativePath = path.StartsWith('/') ? path[1..] : path;
if (relativePath.Length > 1 && relativePath[1] == ':')
{
if (minimatcherKeyValue.Value.IsMatch(path))
{
this.logger.LogDebug("Excluding folder {Path} because it matched glob {Glob}.", path, minimatcherKeyValue.Key);
return true;
}
// Windows drive letter, e.g. "C:/foo" → "foo"
relativePath = relativePath[3..];
}

return false;
});
if (matcher.Match(relativePath).HasMatches)
{
this.logger.LogDebug("Excluding folder {Path} because it matched a directory exclusion glob.", path);
return true;
}

return false;
};
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -419,6 +419,38 @@ public void GenerateDirectoryExclusionPredicate_IgnoreCaseAndAllowWindowsPathsWo
exclusionPredicate(dn, dp).Should().BeTrue();
}

[TestMethod]
public void GenerateDirectoryExclusionPredicate_TrailingDoubleStarMatchesDirectoryItself()
{
// FileSystemGlobbing's ** does not match zero trailing segments,
// so the implementation adds a companion pattern (**/dir) alongside **/dir/**.
// This test verifies that the directory itself is excluded, not just its children.
var args = new ScanSettings
{
SourceDirectory = new DirectoryInfo(this.isWin ? @"C:\project" : "/tmp/project"),
DetectorArgs = new Dictionary<string, string>(),
DirectoryExclusionList = ["**/Source/**"],
};

var exclusionPredicate = this.serviceUnderTest.GenerateDirectoryExclusionPredicate(
args.SourceDirectory.FullName,
args.DirectoryExclusionList,
args.DirectoryExclusionListObsolete,
allowWindowsPaths: true,
ignoreCase: true);

// The directory itself (no trailing segment) should be excluded
var projectPath = this.isWin ? @"C:\project" : "/tmp/project";
exclusionPredicate("Source", projectPath).Should().BeTrue();

// A child under the directory should also be excluded
var sourcePath = this.isWin ? @"C:\project\Source" : "/tmp/project/Source";
exclusionPredicate("child", sourcePath).Should().BeTrue();

// An unrelated directory should not be excluded
exclusionPredicate("Other", projectPath).Should().BeFalse();
}

[TestMethod]
public async Task ProcessDetectorsAsync_DirectoryExclusionPredicateWorksAsExpectedForObsoleteAsync()
{
Expand Down
Loading