Merge remote-tracking branch 'origin/master' into fix-sbt-dependency-tree-html-routing

# Conflicts:
#	Changelog.md

Author: zlav

Cargo.lock

@@ -4,12 +4,33 @@
 
 - Scala/sbt: Route projects using `addDependencyTreePlugin` (sbt 1.4+) to the correct `dependencyBrowseTreeHTML` task, restoring deep dependencies. ([#1711](https://github.com/fossas/fossa-cli/pull/1711))
 
+## 3.17.11
+
+- Conan: Handle list-valued license in make_fossa_deps_conan ([#1719](https://github.com/fossas/fossa-cli/pull/1719)).
+- Strip non printable characters from locators ([#1720](https://github.com/fossas/fossa-cli/pull/1720)).
+- Container scanning: Support scanning /var/lib/dpkg/status.d directory ([#1721](https://github.com/fossas/fossa-cli/pull/1721)).
+
+## 3.17.10
+
+- Licensing: Fix bad SPL matches ([#1717](https://github.com/fossas/fossa-cli/pull/1717)).
+
+## 3.17.9
+
+- Cargo: Fix transitive dev-dependency classification. Dependencies reachable only through dev-dep or build-dep roots are now correctly labeled as Development instead of Production ([#1692](https://github.com/fossas/fossa-cli/pull/1692)).
+
+## 3.17.8
+
+- Vendored dependencies: archive uploads with an absolute `path` (as produced by the meta-fossa Yocto layer) no longer crash with a `permission denied` error while writing the tarball. ([#1713](https://github.com/fossas/fossa-cli/pull/1713))
+
+## 3.17.7
+
+- NuGet: PackageReference discovery now analyzes every `.csproj`/`.xproj`/`.vbproj`/`.dbproj`/`.fsproj` in a directory. Previously only the first match returned by the directory listing was analyzed, so sibling project files were silently dropped. ([#1712](https://github.com/fossas/fossa-cli/pull/1712))
+
 ## 3.17.6
 
 - Config: `paths.only` and `paths.exclude` in `.fossa.yml` now accept glob patterns. ([#1703](https://github.com/fossas/fossa-cli/pull/1703))
 - Licensing - Fix two bad GPL matches [No PR]
 
-
 ## 3.17.5
 
 - Vendetta: Debug bundles now include per-file component match data from Vendetta scans, making it easier to diagnose why a vendored dependency was or wasn't detected. ([#1706](https://github.com/fossas/fossa-cli/pull/1706))
@@ -30,8 +51,7 @@
 - Poetry: Support PEP 621 `[project].dependencies` for Poetry 2.x projects. Production dependencies declared in the standard `[project]` section are now correctly detected alongside legacy `[tool.poetry.dependencies]`. ([#1683](https://github.com/fossas/fossa-cli/pull/1683))
 
 ## 3.17.1
-
-- Node.js: Yarn and npm workspace packages now appear as individual build targets (e.g. `yarn@./:my-package`, `npm@./:my-package`), enabling per-package dependency scoping via `.fossa.yml`.
+- Node.js: Yarn and npm workspace packages now appear as individual build targets (e.g. `yarn@./:my-package`, `npm@./:my-package`), enabling per-package dependency scoping via `.fossa.yml`. ([#1643](https://github.com/fossas/fossa-cli/pull/1643))
 - Project edit: Fix 500 error when running `fossa project edit --policy` on existing projects ([#1688](https://github.com/fossas/fossa-cli/pull/1688))
 - UV: Add `directory` source type to uv.lock parser, fixing parse failures on projects with local directory dependencies ([#1690](https://github.com/fossas/fossa-cli/pull/1690))
 

Changelog.md

@@ -107,8 +107,29 @@ def name_version_of(label: str) -> Tuple[str, str]:
     name, version = label.split("/", 1)
     return name, version
 
+# Conan recipes may declare `license` as a single string ("MIT") or as a list/tuple of
+# strings (["MIT", "Apache-2.0"]). The fossa-deps `license` field must be a single string,
+# so a list is joined into one SPDX expression. We use " AND " (every license's obligations
+# apply) as the conservative default; change MULTI_LICENSE_JOINER to " OR " if your packages
+# are dual-licensed (consumer's choice).
+MULTI_LICENSE_JOINER = " AND "
+
+# fossa-deps requires a license string for every custom dependency. When a Conan recipe
+# declares no license, fall back to the SPDX "NOASSERTION" marker so the file stays valid;
+# emitting a bare `license: null` triggers: expected String, but encountered Null.
+NO_LICENSE = "NOASSERTION"
+
 def license_of(node: dict) -> Optional[str]:
-    return node.get("license")
+    raw = node.get("license")
+    if raw is None:
+        return None
+    if isinstance(raw, str):
+        return raw or None
+    if isinstance(raw, (list, tuple)):
+        parts = [str(item).strip() for item in raw if item is not None and str(item).strip()]
+        return MULTI_LICENSE_JOINER.join(parts) if parts else None
+    # Unexpected shape (number, dict, ...): coerce to a string so fossa-deps stays valid.
+    return str(raw)
 
 def homepage_of(node: dict) -> Optional[str]:
     candidate = node.get("homepage")
@@ -158,7 +179,7 @@ def mk_fossa_deps(graph):
             vendored_deps.append(FossaVendorDep(name, version, src_dir))
         else:
             logging.info(f"could not find source code in disk for: {label}, using this as vendored dependency for fossa-deps")
-            custom_deps.append(FossaCustomDep(name, version, license, FossaCustomDepMetadata(homepage, description)))
+            custom_deps.append(FossaCustomDep(name, version, license or NO_LICENSE, FossaCustomDepMetadata(homepage, description)))
 
     fossa_dep_yml = FossaDep(vendored_deps, custom_deps)
     fossa_dep_yml.dump()

docs/walkthroughs/make_fossa_deps_conan.py

@@ -46,7 +46,7 @@ testServiceStackForPkgReferences =
   aroundAll (withAnalysisOf NonStrict $ serviceStack NuGet.discover) $ do
     describe "ServiceStack" $ do
       it "should find targets" $ \(result, _) -> do
-        length result `shouldBe` 64
+        length result `shouldBe` 92
 
 testServiceStackForPkgConfig :: Spec
 testServiceStackForPkgConfig =

integration-test/Analysis/NugetSpec.hs

@@ -692,6 +692,7 @@ test-suite unit-tests
     Node.PackageLockSpec
     Node.PackageLockV3Spec
     NuGet.DirectoryPackagesPropsSpec
+    NuGet.NuGetSpec
     NuGet.NuspecSpec
     NuGet.PackageReferenceSpec
     NuGet.PackagesConfigSpec

spectrometer.cabal

@@ -7,6 +7,7 @@ module App.Fossa.VendoredDependency (
   vendoredDepToLocator,
   forceVendoredToArchive,
   compressFile,
+  safeSeparators,
   hashFile,
   hashBs,
   dedupVendoredDeps,
@@ -50,7 +51,8 @@ import Fossa.API.Types (
 import Path (Abs, Dir, Path)
 import Prettyprinter (Pretty (pretty), vsep)
 import Srclib.Types (Locator (..), ProvidedPackageLabel)
-import System.FilePath.Posix (splitDirectories, (</>))
+import System.FilePath (dropDrive, splitDirectories)
+import System.FilePath.Posix ((</>))
 
 data VendoredDependency = VendoredDependency
   { vendoredName :: Text
@@ -197,8 +199,11 @@ hashFile fileToHash = do
   fileContent <- BS.readFile fileToHash
   pure . toText . show $ md5 fileContent
 
+-- Flatten a path into a single filename component. We use `dropDrive` to
+-- ensure the result is relative, since callers join it with `(</>)`, which
+-- discards the LHS when the RHS is absolute.
 safeSeparators :: FilePath -> FilePath
-safeSeparators = intercalate "_" . splitDirectories
+safeSeparators = intercalate "_" . splitDirectories . dropDrive
 
 skippedDepsDebugLog :: NeedScanningDeps -> SkippableDeps -> VendoredDependencyScanMode -> SkippedDepsLogMsg
 skippedDepsDebugLog needScanningDeps skippedDeps scanMode =

src/App/Fossa/VendoredDependency.hs

@@ -41,6 +41,7 @@ module Srclib.Types (
 ) where
 
 import Data.Aeson
+import Data.Char qualified as Char
 import Data.List.NonEmpty (NonEmpty ((:|)))
 import Data.List.NonEmpty qualified as NE
 import Data.Map (Map)
@@ -497,7 +498,10 @@ instance ToText Locator where
 
 renderLocator :: Locator -> Text
 renderLocator Locator{..} =
-  locatorFetcher <> "+" <> locatorProject <> "$" <> fromMaybe "" locatorRevision
+  stripNonPrintable $
+    locatorFetcher <> "+" <> locatorProject <> "$" <> fromMaybe "" locatorRevision
+  where
+    stripNonPrintable = Text.filter Char.isPrint
 
 -- The projectId is the full locator of the project. E.g. custom+123/someProject (<fetcher>+<orgId>/<project-name>)
 projectId :: Locator -> Text

src/Srclib/Types.hs

@@ -52,8 +52,9 @@ import Data.Foldable (for_, traverse_)
 import Data.Functor (void)
 import Data.List.NonEmpty qualified as NonEmpty
 import Data.Map.Strict qualified as Map
-import Data.Maybe (catMaybes, fromMaybe, isJust)
+import Data.Maybe (catMaybes, fromMaybe, isJust, isNothing)
 import Data.Set (Set)
+import Data.Set qualified as Set
 import Data.String.Conversion (toString, toText)
 import Data.Text (Text, breakOn)
 import Data.Text qualified as Text
@@ -74,7 +75,6 @@ import Effect.Exec (
   execThrow,
  )
 import Effect.Grapher (
-  LabeledGrapher,
   direct,
   edge,
   label,
@@ -485,36 +485,88 @@ toDependency emitGitBackedLocators sourceMap pkg =
         extractGitCommitHash sourceUrl
       _ -> Nothing
 
--- Possible values here are "build", "dev", and null.
--- Null refers to productions, while dev and build refer to development-time dependencies
--- Cargo does not differentiate test dependencies and dev dependencies,
--- so we just simplify it to Development.
-kindToLabel :: Maybe Text.Text -> CargoLabel
-kindToLabel (Just _) = CargoDepKind EnvDevelopment
-kindToLabel Nothing = CargoDepKind EnvProduction
-
-addLabel :: Has (LabeledGrapher PackageId CargoLabel) sig m => NodeDependency -> m ()
-addLabel dep = do
-  let packageId = nodePkg dep
-  traverse_ (label packageId . kindToLabel . nodeDepKind) $ nodeDepKinds dep
-
-addEdge :: Has (LabeledGrapher PackageId CargoLabel) sig m => ResolveNode -> m ()
-addEdge node = do
-  let parentId = resolveNodeId node
-  for_ (resolveNodeDeps node) $ \dep -> do
-    addLabel dep
-    edge parentId $ nodePkg dep
-
+-- A Cargo edge's kind ("build", "dev", or null) reflects the parent's manifest
+-- declaration, not the path taken to reach the parent. We classify each package
+-- by which workspace-rooted paths can reach it:
+--
+--   * Production: reachable from a workspace member via a path of null-kind
+--     edges only. These packages are linked into the release artifact.
+--
+--   * Development: any package reachable (via any edge) from the target of a
+--     non-null-kind edge. A "build" or "dev" edge marks the start of a subtree
+--     that never ships in the release binary, and every descendant of that
+--     subtree inherits Development.
+--
+-- A package can carry both labels when it's reachable by both kinds of paths.
+--
+-- We do not need a separate "dev-deps of prod-deps" case: Cargo only resolves
+-- dev-dependencies for workspace members, so non-workspace edges with kind
+-- "dev" do not appear in 'cargo metadata' output. The only non-null kind we
+-- see on a non-workspace edge is "build".
+--
+-- Cargo is the only strategy with per-edge kinds; others (pnpm, yarn, poetry)
+-- label nodes and propagate with 'hydrateDepEnvs'. That helper walks from a
+-- labeled node to every dependency it declares, regardless of edge kind, so
+-- a Production label on a workspace member would flow through a "dev" or
+-- "build" edge and mislabel the dev/build subtree as Production. We roll our
+-- own edge-filtered reachability here rather than generalize the shared helper.
 buildGraph :: Bool -> CargoMetadata -> Graphing Dependency
--- By construction, workspace members are the root nodes in the graph.
--- Use shrinkRoots to remove them and promote their direct dependencies to the
--- direct dependencies we report for the project.
 buildGraph emitGitBackedLocators meta = shrinkRoots $
   run . withLabeling (toDependency emitGitBackedLocators sourceMap) $ do
-    traverse_ direct $ metadataWorkspaceMembers meta
-    traverse_ addEdge $ resolvedNodes $ metadataResolve meta
+    traverse_ direct (metadataWorkspaceMembers meta)
+    for_ nodes $ \node ->
+      for_ (resolveNodeDeps node) $ \dep ->
+        edge (resolveNodeId node) (nodePkg dep)
+    for_ (Set.toList prodReachable) $ \pkg ->
+      label pkg (CargoDepKind EnvProduction)
+    for_ (Set.toList devReachable) $ \pkg ->
+      label pkg (CargoDepKind EnvDevelopment)
   where
     sourceMap = buildPackageSourceMap $ metadataPackages meta
+    nodes = resolvedNodes (metadataResolve meta)
+    workspaceMembers = Set.fromList (metadataWorkspaceMembers meta)
+
+    -- These predicates are not mutually exclusive: a dep declared in both
+    -- [dependencies] and [dev-dependencies] on the same parent carries both
+    -- a null and a non-null kind, so the edge feeds prodAdj *and* devSeeds.
+    isProdEdge dep = any (isNothing . nodeDepKind) (nodeDepKinds dep)
+    isDevEdge dep = any (isJust . nodeDepKind) (nodeDepKinds dep)
+
+    -- Adjacency containing only edges whose parent declares the child as a
+    -- normal dependency (at least one kind is null). Production reachability
+    -- must only traverse these — a build or dev edge breaks the release chain.
+    prodAdj =
+      Map.fromList $
+        map
+          (\node -> (resolveNodeId node, map nodePkg . filter isProdEdge . resolveNodeDeps $ node))
+          nodes
+
+    -- Every edge in the metadata graph, for Development reachability.
+    allAdj =
+      Map.fromList $
+        map (\node -> (resolveNodeId node, map nodePkg (resolveNodeDeps node))) nodes
+
+    -- Targets of any non-null-kind edge. Each seeds a Development subtree:
+    -- the target and all its transitive descendants are never linked into
+    -- a release build.
+    devSeeds =
+      Set.fromList $
+        map nodePkg $
+          concatMap (filter isDevEdge . resolveNodeDeps) nodes
+
+    prodReachable = reachable prodAdj workspaceMembers
+    devReachable = reachable allAdj devSeeds
+
+reachable :: Map.Map PackageId [PackageId] -> Set PackageId -> Set PackageId
+reachable adj = go Set.empty . Set.toList
+  where
+    go visited [] = visited
+    go visited (x : xs) =
+      if Set.member x visited
+        then go visited xs
+        else
+          let children = fromMaybe [] (Map.lookup x adj)
+           in go (Set.insert x visited) (children ++ xs)
 
 -- | Custom Parsec type alias
 type PkgSpecParser a = Parsec Void Text a

src/Strategy/Cargo.hs

@@ -9,6 +9,7 @@ import Container.OsRelease (OsInfo)
 import Control.Effect.Diagnostics (Diagnostics)
 import Control.Effect.Reader (Reader)
 import Data.Aeson (ToJSON)
+import Data.Foldable (find)
 import Data.String.Conversion (toText)
 import Data.Text qualified as Text
 import Discovery.Filters (AllFilters)
@@ -18,7 +19,7 @@ import Discovery.Walk (
   findFileNamed,
   walkWithFilters',
  )
-import Effect.ReadFS (Has, ReadFS)
+import Effect.ReadFS (Has, ReadFS, listDir)
 import GHC.Generics (Generic)
 import Path (Abs, Dir, File, Path, toFilePath)
 import Strategy.Dpkg.Database (analyze)
@@ -59,13 +60,19 @@ findProjects ::
   OsInfo ->
   Path Abs Dir ->
   m [DpkgDatabase]
-findProjects osInfo = walkWithFilters' $ \dir _ files -> do
-  case findFileNamed "status" files of
-    Nothing -> pure ([], WalkContinue)
-    Just file -> do
-      if (Text.isInfixOf "var/lib/dpkg/" $ toText . toFilePath $ file)
-        then pure ([DpkgDatabase dir file osInfo], WalkContinue)
-        else pure ([], WalkContinue)
+findProjects osInfo = walkWithFilters' $ \dir dirs files -> do
+  let standardDBs = case findFileNamed "status" files of
+        Just file ->
+          if Text.isInfixOf "var/lib/dpkg/" (toText . toFilePath $ file)
+            then [DpkgDatabase dir file osInfo]
+            else []
+        Nothing -> []
+  statusD_DBs <- case find (\f -> toFilePath f == "var/lib/dpkg/status.d/") dirs of
+    Just dir' -> do
+      (_, filesInDir) <- listDir dir'
+      pure $ map (\file -> DpkgDatabase dir' file osInfo) (filter (not . Text.isSuffixOf ".md5sums" . toText) filesInDir)
+    Nothing -> pure []
+  pure (standardDBs ++ statusD_DBs, WalkContinue)
 
 mkProject :: DpkgDatabase -> DiscoveredProject DpkgDatabase
 mkProject project =