Add option to exclude base image components (#1825)

* Add option to exclude base image components

* Address review feedback

* Address feedback

* Remove references to filtered out components from dependency roots

* Handle null layerIndices

* Properly handle components with 0 layer indices

* Use files other than system package manager DBs when possible to determine component to layer map

* Respond to feedback

* Rename flag to ExcludeBaseImageComponents and add docs

* Ensure duplicate layer IDs are handled properly

Author: jasonpaulos

docs/detectors/linux.md

@@ -54,3 +54,15 @@ For example:
 
 - Windows container scanning is not supported
 - Multiplatform images are not supported
+
+## Excluding Base Image Components
+
+When scanning container images, many detected components may originate from the base image rather than from layers added by the user's Dockerfile. The `--ExcludeBaseImageComponents` flag filters out components that exist exclusively in base image layers, so only components introduced by the user's own layers are reported.
+
+```sh
+--ExcludeBaseImageComponents
+```
+
+A component is excluded only if **all** of its associated container layers are marked as base image layers. If a component appears in at least one non-base-image layer, it is retained.
+
+This flag has no effect on non-container scans (i.e., directory-based detection).

src/Microsoft.ComponentDetection.Detectors/linux/LinuxScanner.cs

@@ -30,6 +30,21 @@ internal class LinuxScanner : ILinuxScanner
 
     private static readonly IList<string> ScopeSquashedParameter = ["--scope", "squashed"];
 
+    /// <summary>
+    /// Well-known package manager database paths whose layer attribution should be ignored
+    /// when determining which layer a system package belongs to. These files are shared across
+    /// all packages managed by the same package manager and get updated whenever any package is
+    /// installed or removed, causing unrelated packages to appear as modified in later layers.
+    /// </summary>
+    private static readonly HashSet<string> PackageManagerDatabasePaths = new(StringComparer.OrdinalIgnoreCase)
+    {
+        "/var/lib/dpkg/status",
+        "/lib/apk/db/installed",
+        "/var/lib/rpm/Packages",
+        "/var/lib/rpm/Packages.db",
+        "/var/lib/rpm/rpmdb.sqlite",
+    };
+
     private static readonly SemaphoreSlim ContainerSemaphore = new SemaphoreSlim(2);
 
     /// <summary>
@@ -179,11 +194,16 @@ private IEnumerable<LayerMappedLinuxComponents> ProcessSyftOutputWithTelemetry(
             }
         }
 
+        // Build a file path → layerID map from the top-level files listing.
+        // This allows us to determine layer attribution for files owned by a package
+        // even when the artifact's locations only reference the package manager database.
+        var filePathToLayerId = BuildFilePathToLayerMap(syftOutput.Files);
+
         // Create components using only enabled factories
         var componentsWithLayers = validArtifacts
             .DistinctBy(artifact => (artifact.Name, artifact.Version, artifact.Type))
             .Select(artifact =>
-                this.CreateComponentWithLayers(artifact, syftOutput.Distro, enabledFactories)
+                this.CreateComponentWithLayers(artifact, syftOutput.Distro, enabledFactories, filePathToLayerId)
             )
             .Where(result => result.Component != null)
             .Select(result => (Component: result.Component!, result.LayerIds))
@@ -388,7 +408,8 @@ private async Task<string> RunSyftCoreAsync(
     private (TypedComponent? Component, IEnumerable<string> LayerIds) CreateComponentWithLayers(
         ArtifactElement artifact,
         Distro distro,
-        HashSet<IArtifactComponentFactory> enabledFactories
+        HashSet<IArtifactComponentFactory> enabledFactories,
+        Dictionary<string, string> filePathToLayerId
     )
     {
         if (!this.artifactTypeToFactoryLookup.TryGetValue(artifact.Type, out var factory))
@@ -408,12 +429,96 @@ HashSet<IArtifactComponentFactory> enabledFactories
             return (null, []);
         }
 
-        var layerIds = artifact.Locations?.Select(location => location.LayerId).Distinct() ?? [];
-        return (component, layerIds);
+        // Collect layer IDs from the artifact's locations, filtering out entries with null/empty layer IDs.
+        var locationLayerIds = artifact.Locations?
+            .Where(location => !string.IsNullOrEmpty(location.Path) && !string.IsNullOrEmpty(location.LayerId))
+            .Select(location => (location.Path, location.LayerId))
+            .ToList() ?? [];
+
+        // Also consult the metadata files property to find additional owned files,
+        // and look up their layer IDs from the top-level file listing.
+        if (artifact.Metadata?.Files != null)
+        {
+            foreach (var file in artifact.Metadata.Files)
+            {
+                // The File union type can be either a FileFile object or a plain string path.
+                var filePath = file.FileFile?.Path ?? file.String;
+                if (!string.IsNullOrEmpty(filePath) && filePathToLayerId.TryGetValue(filePath, out var layerId))
+                {
+                    locationLayerIds.Add((filePath, layerId));
+                }
+            }
+        }
+
+        // Exclude well-known package manager database paths from layer attribution,
+        // unless they are the only known locations for this component.
+        var nonDbLayerIds = locationLayerIds
+            .Where(loc => !IsPackageManagerDatabasePath(loc.Path))
+            .Select(loc => loc.LayerId)
+            .Distinct()
+            .ToList();
+
+        if (nonDbLayerIds.Count > 0)
+        {
+            return (component, nonDbLayerIds);
+        }
+
+        // Fall back to database path layer IDs if no other locations are available.
+        var allLayerIds = locationLayerIds
+            .Select(loc => loc.LayerId)
+            .Where(id => !string.IsNullOrEmpty(id))
+            .Distinct()
+            .ToList();
+        return (component, allLayerIds);
     }
 
     /// <summary>
     /// Clears the syft run cache. Intended for test isolation only.
     /// </summary>
     internal static void ResetCache() => SyftRunCache.Clear();
+
+    /// <summary>
+    /// Builds a dictionary mapping file paths to their layer IDs from the top-level
+    /// files listing in the syft output. This enables layer attribution for files
+    /// owned by a package even when the artifact's locations only reference the
+    /// package manager database.
+    /// </summary>
+    private static Dictionary<string, string> BuildFilePathToLayerMap(FileElement[] files)
+    {
+        var map = new Dictionary<string, string>(StringComparer.Ordinal);
+        if (files == null)
+        {
+            return map;
+        }
+
+        foreach (var file in files)
+        {
+            if (file.Location != null && !string.IsNullOrEmpty(file.Location.Path) && !string.IsNullOrEmpty(file.Location.LayerId))
+            {
+                map.TryAdd(file.Location.Path, file.Location.LayerId);
+            }
+        }
+
+        return map;
+    }
+
+    /// <summary>
+    /// Determines whether a file path is a well-known package manager database file
+    /// that should be excluded from layer attribution.
+    /// </summary>
+    private static bool IsPackageManagerDatabasePath(string path)
+    {
+        if (string.IsNullOrEmpty(path))
+        {
+            return false;
+        }
+
+        if (PackageManagerDatabasePaths.Contains(path))
+        {
+            return true;
+        }
+
+        // Cover any file under /var/lib/rpm/ (RPM database can use multiple files)
+        return path.StartsWith("/var/lib/rpm/", StringComparison.OrdinalIgnoreCase);
+    }
 }

src/Microsoft.ComponentDetection.Orchestrator/Commands/ScanSettings.cs

@@ -86,6 +86,10 @@ public class ScanSettings : BaseSettings
     [Description("Whether or not to cleanup files that are created during detection, based on the rules provided in each detector. Defaults to 'true'.")]
     public bool? CleanupCreatedFiles { get; set; }
 
+    [CommandOption("--ExcludeBaseImageComponents")]
+    [Description("When enabled, filters out components that originate exclusively from base image layers when scanning containers.")]
+    public bool ExcludeBaseImageComponents { get; set; }
+
     /// <inheritdoc />
     public override ValidationResult Validate()
     {

src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs

@@ -41,15 +41,175 @@ public ScanResult GenerateScanResultFromProcessingResult(
 
         ReconcileDependencyGraphIds(dependencyGraphs, mergedComponents);
 
+        var componentsToOutput = mergedComponents;
+        if (settings.ExcludeBaseImageComponents)
+        {
+            var originalCount = mergedComponents.Count;
+            componentsToOutput = this.FilterOutBaseImageComponents(componentsToOutput, detectorProcessingResult.ContainersDetailsMap);
+            var filteredCount = originalCount - componentsToOutput.Count;
+
+            if (filteredCount > 0)
+            {
+                this.logger.LogInformation("Filtered out {FilteredCount} of {TotalCount} components that originate exclusively from base image layers. {RetainedCount} components remain.", filteredCount, originalCount, componentsToOutput.Count);
+            }
+            else
+            {
+                this.logger.LogInformation("Base image component filtering is enabled but no components were filtered out ({TotalCount} total).", originalCount);
+            }
+
+            PruneFilteredComponentsFromGraphs(dependencyGraphs, componentsToOutput);
+            PruneFilteredComponentReferrers(componentsToOutput);
+        }
+
         return new DefaultGraphScanResult
         {
-            ComponentsFound = mergedComponents.Select(x => this.ConvertToContract(x)).ToList(),
+            ComponentsFound = componentsToOutput.Select(x => this.ConvertToContract(x)).ToList(),
             ContainerDetailsMap = detectorProcessingResult.ContainersDetailsMap,
             DependencyGraphs = dependencyGraphs,
             SourceDirectory = settings.SourceDirectory.ToString(),
         };
     }
 
+    /// <summary>
+    /// Filters out components that originate exclusively from base image layers.
+    /// A component is removed only if it has container layer references and every referenced
+    /// layer across all containers has <see cref="DockerLayer.IsBaseImage"/> set to true.
+    /// Components with no container references or with at least one non-base-image layer are retained.
+    /// </summary>
+    /// <param name="components">The list of detected components to filter.</param>
+    /// <param name="containerDetailsMap">The map of container details with layer information.</param>
+    /// <returns>A filtered list of components excluding those exclusively from base image layers.</returns>
+    internal List<DetectedComponent> FilterOutBaseImageComponents(
+        List<DetectedComponent> components,
+        Dictionary<int, ContainerDetails> containerDetailsMap)
+    {
+        if (containerDetailsMap == null || containerDetailsMap.Count == 0)
+        {
+            return components;
+        }
+
+        // Build an indexed lookup: containerDetailId → (layerIndex → DockerLayer)
+        var layerLookup = new Dictionary<int, Dictionary<int, DockerLayer>>();
+        foreach (var (id, details) in containerDetailsMap)
+        {
+            if (details.Layers != null)
+            {
+                layerLookup[id] = details.Layers.GroupBy(l => l.LayerIndex).ToDictionary(g => g.Key, g => g.First());
+            }
+        }
+
+        var retained = new List<DetectedComponent>();
+        foreach (var component in components)
+        {
+            if (IsExclusivelyFromBaseImage(component, layerLookup))
+            {
+                this.logger.LogDebug("Filtering out component {ComponentId} because all associated layers are from the base image.", component.Component.Id);
+            }
+            else
+            {
+                retained.Add(component);
+            }
+        }
+
+        return retained;
+    }
+
+    private static bool IsExclusivelyFromBaseImage(DetectedComponent component, Dictionary<int, Dictionary<int, DockerLayer>> layerLookup)
+    {
+        // Components without container layer references are not from a container scan - keep them.
+        if (component.ContainerLayerIds == null || component.ContainerLayerIds.Count == 0)
+        {
+            return false;
+        }
+
+        foreach (var (containerDetailId, layerIndices) in component.ContainerLayerIds)
+        {
+            if (!layerLookup.TryGetValue(containerDetailId, out var layersByIndex))
+            {
+                // If we can't resolve the container details, assume it's not a base image component.
+                return false;
+            }
+
+            if (layerIndices == null || !layerIndices.Any())
+            {
+                // No layer indices for this container detail - keep the component.
+                return false;
+            }
+
+            foreach (var layerIndex in layerIndices)
+            {
+                if (!layersByIndex.TryGetValue(layerIndex, out var layer) || !layer.IsBaseImage)
+                {
+                    // Layer not found or not from base image. Keep this component.
+                    return false;
+                }
+            }
+        }
+
+        return true;
+    }
+
+    /// <summary>
+    /// Removes component IDs from dependency graphs that are no longer present in the output components list.
+    /// </summary>
+    private static void PruneFilteredComponentsFromGraphs(DependencyGraphCollection graphs, List<DetectedComponent> retainedComponents)
+    {
+        if (graphs == null || graphs.Count == 0)
+        {
+            return;
+        }
+
+        var retainedIds = new HashSet<string>(retainedComponents.Select(c => c.Component.Id));
+
+        foreach (var graphWithMetadata in graphs.Values)
+        {
+            var graph = graphWithMetadata.Graph;
+
+            // Remove nodes that are no longer in retained components.
+            var idsToRemove = graph.Keys.Where(id => !retainedIds.Contains(id)).ToList();
+            foreach (var id in idsToRemove)
+            {
+                graph.Remove(id);
+            }
+
+            // Remove references to removed IDs from remaining nodes' dependency sets.
+            // Normalize empty edge sets to null for consistent leaf-node serialization.
+            foreach (var nodeId in graph.Keys.ToList())
+            {
+                var edges = graph[nodeId];
+                if (edges != null)
+                {
+                    edges.RemoveWhere(id => !retainedIds.Contains(id));
+                    if (edges.Count == 0)
+                    {
+                        graph[nodeId] = null;
+                    }
+                }
+            }
+
+            // Clean up metadata sets.
+            graphWithMetadata.ExplicitlyReferencedComponentIds?.RemoveWhere(id => !retainedIds.Contains(id));
+            graphWithMetadata.DevelopmentDependencies?.RemoveWhere(id => !retainedIds.Contains(id));
+            graphWithMetadata.Dependencies?.RemoveWhere(id => !retainedIds.Contains(id));
+        }
+    }
+
+    /// <summary>
+    /// Removes references to filtered-out components from the DependencyRoots and AncestralDependencyRoots
+    /// of retained components, so that TopLevelReferrers and AncestralReferrers in the output don't
+    /// reference components that were removed from ComponentsFound.
+    /// </summary>
+    private static void PruneFilteredComponentReferrers(List<DetectedComponent> retainedComponents)
+    {
+        var retainedIds = new HashSet<string>(retainedComponents.Select(c => c.Component.Id));
+
+        foreach (var component in retainedComponents)
+        {
+            component.DependencyRoots?.RemoveWhere(root => !retainedIds.Contains(root.Id));
+            component.AncestralDependencyRoots?.RemoveWhere(root => !retainedIds.Contains(root.Id));
+        }
+    }
+
     private static ConcurrentHashSet<string> MergeTargetFrameworks(ConcurrentHashSet<string> left, ConcurrentHashSet<string> right)
     {
         if (left == null && right == null)