Skip to content

Commit 8ae1b2e

Browse files
authored
Merge branch 'master' into pnpm-lock-refactor
2 parents 6414877 + b5a1c53 commit 8ae1b2e

9 files changed

Lines changed: 380 additions & 110 deletions

File tree

.github/workflows/stale.yml

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
name: Close Stale Issues
2+
3+
on:
4+
schedule:
5+
- cron: "0 0 * * *"
6+
workflow_dispatch:
7+
8+
permissions: {}
9+
10+
jobs:
11+
stale:
12+
runs-on: ubuntu-latest
13+
permissions:
14+
issues: write
15+
concurrency:
16+
group: stale-issues-workflow
17+
cancel-in-progress: false
18+
19+
steps:
20+
- uses: actions/stale@v10
21+
with:
22+
repo-token: ${{ secrets.GITHUB_TOKEN }}
23+
days-before-issue-stale: 21
24+
days-before-issue-close: 7
25+
stale-issue-message: "This issue has been marked stale because it has had no activity for 21 days. It will be closed in 7 days if no further activity occurs."
26+
close-issue-message: "This issue has been automatically closed because it has had no activity for 7 days since being marked stale."
27+
stale-issue-label: "stale"
28+
29+
# Don't auto-close PRs
30+
days-before-pr-stale: -1
31+
days-before-pr-close: -1

Changelog.md

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,15 @@
11
# FOSSA CLI Changelog
22

3+
## Unreleased
4+
5+
- Scala/sbt: Route projects using `addDependencyTreePlugin` (sbt 1.4+) to the correct `dependencyBrowseTreeHTML` task, restoring deep dependencies. ([#1711](https://github.com/fossas/fossa-cli/pull/1711))
6+
7+
## 3.17.11
8+
9+
- Conan: Handle list-valued license in make_fossa_deps_conan ([#1719](https://github.com/fossas/fossa-cli/pull/1719)).
10+
- Strip non printable characters from locators ([#1720](https://github.com/fossas/fossa-cli/pull/1720)).
11+
- Container scanning: Support scanning /var/lib/dpkg/status.d directory ([#1721](https://github.com/fossas/fossa-cli/pull/1721)).
12+
313
## 3.17.10
414

515
- Licensing: Fix bad SPL matches ([#1717](https://github.com/fossas/fossa-cli/pull/1717)).

docs/walkthroughs/make_fossa_deps_conan.py

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,8 +107,29 @@ def name_version_of(label: str) -> Tuple[str, str]:
107107
name, version = label.split("/", 1)
108108
return name, version
109109

110+
# Conan recipes may declare `license` as a single string ("MIT") or as a list/tuple of
111+
# strings (["MIT", "Apache-2.0"]). The fossa-deps `license` field must be a single string,
112+
# so a list is joined into one SPDX expression. We use " AND " (every license's obligations
113+
# apply) as the conservative default; change MULTI_LICENSE_JOINER to " OR " if your packages
114+
# are dual-licensed (consumer's choice).
115+
MULTI_LICENSE_JOINER = " AND "
116+
117+
# fossa-deps requires a license string for every custom dependency. When a Conan recipe
118+
# declares no license, fall back to the SPDX "NOASSERTION" marker so the file stays valid;
119+
# emitting a bare `license: null` triggers: expected String, but encountered Null.
120+
NO_LICENSE = "NOASSERTION"
121+
110122
def license_of(node: dict) -> Optional[str]:
111-
return node.get("license")
123+
raw = node.get("license")
124+
if raw is None:
125+
return None
126+
if isinstance(raw, str):
127+
return raw or None
128+
if isinstance(raw, (list, tuple)):
129+
parts = [str(item).strip() for item in raw if item is not None and str(item).strip()]
130+
return MULTI_LICENSE_JOINER.join(parts) if parts else None
131+
# Unexpected shape (number, dict, ...): coerce to a string so fossa-deps stays valid.
132+
return str(raw)
112133

113134
def homepage_of(node: dict) -> Optional[str]:
114135
candidate = node.get("homepage")
@@ -158,7 +179,7 @@ def mk_fossa_deps(graph):
158179
vendored_deps.append(FossaVendorDep(name, version, src_dir))
159180
else:
160181
logging.info(f"could not find source code in disk for: {label}, using this as vendored dependency for fossa-deps")
161-
custom_deps.append(FossaCustomDep(name, version, license, FossaCustomDepMetadata(homepage, description)))
182+
custom_deps.append(FossaCustomDep(name, version, license or NO_LICENSE, FossaCustomDepMetadata(homepage, description)))
162183

163184
fossa_dep_yml = FossaDep(vendored_deps, custom_deps)
164185
fossa_dep_yml.dump()

src/Srclib/Types.hs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ module Srclib.Types (
4141
) where
4242

4343
import Data.Aeson
44+
import Data.Char qualified as Char
4445
import Data.List.NonEmpty (NonEmpty ((:|)))
4546
import Data.List.NonEmpty qualified as NE
4647
import Data.Map (Map)
@@ -497,7 +498,10 @@ instance ToText Locator where
497498

498499
renderLocator :: Locator -> Text
499500
renderLocator Locator{..} =
500-
locatorFetcher <> "+" <> locatorProject <> "$" <> fromMaybe "" locatorRevision
501+
stripNonPrintable $
502+
locatorFetcher <> "+" <> locatorProject <> "$" <> fromMaybe "" locatorRevision
503+
where
504+
stripNonPrintable = Text.filter Char.isPrint
501505

502506
-- The projectId is the full locator of the project. E.g. custom+123/someProject (<fetcher>+<orgId>/<project-name>)
503507
projectId :: Locator -> Text

src/Strategy/Dpkg.hs

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import Container.OsRelease (OsInfo)
99
import Control.Effect.Diagnostics (Diagnostics)
1010
import Control.Effect.Reader (Reader)
1111
import Data.Aeson (ToJSON)
12+
import Data.Foldable (find)
1213
import Data.String.Conversion (toText)
1314
import Data.Text qualified as Text
1415
import Discovery.Filters (AllFilters)
@@ -18,7 +19,7 @@ import Discovery.Walk (
1819
findFileNamed,
1920
walkWithFilters',
2021
)
21-
import Effect.ReadFS (Has, ReadFS)
22+
import Effect.ReadFS (Has, ReadFS, listDir)
2223
import GHC.Generics (Generic)
2324
import Path (Abs, Dir, File, Path, toFilePath)
2425
import Strategy.Dpkg.Database (analyze)
@@ -59,13 +60,19 @@ findProjects ::
5960
OsInfo ->
6061
Path Abs Dir ->
6162
m [DpkgDatabase]
62-
findProjects osInfo = walkWithFilters' $ \dir _ files -> do
63-
case findFileNamed "status" files of
64-
Nothing -> pure ([], WalkContinue)
65-
Just file -> do
66-
if (Text.isInfixOf "var/lib/dpkg/" $ toText . toFilePath $ file)
67-
then pure ([DpkgDatabase dir file osInfo], WalkContinue)
68-
else pure ([], WalkContinue)
63+
findProjects osInfo = walkWithFilters' $ \dir dirs files -> do
64+
let standardDBs = case findFileNamed "status" files of
65+
Just file ->
66+
if Text.isInfixOf "var/lib/dpkg/" (toText . toFilePath $ file)
67+
then [DpkgDatabase dir file osInfo]
68+
else []
69+
Nothing -> []
70+
statusD_DBs <- case find (\f -> toFilePath f == "var/lib/dpkg/status.d/") dirs of
71+
Just dir' -> do
72+
(_, filesInDir) <- listDir dir'
73+
pure $ map (\file -> DpkgDatabase dir' file osInfo) (filter (not . Text.isSuffixOf ".md5sums" . toText) filesInDir)
74+
Nothing -> pure []
75+
pure (standardDBs ++ statusD_DBs, WalkContinue)
6976

7077
mkProject :: DpkgDatabase -> DiscoveredProject DpkgDatabase
7178
mkProject project =

src/Strategy/Scala.hs

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ import Strategy.Maven.Pom.PomFile (RawPom (rawPomArtifact, rawPomGroup, rawPomVe
6060
import Strategy.Maven.Pom.Resolver (buildGlobalClosure)
6161
import Strategy.Scala.Common (mkSbtCommand)
6262
import Strategy.Scala.Errors (FailedToListProjects (FailedToListProjects), MaybeWithoutDependencyTreeTask (..), MissingFullDependencyPlugin (..), sbtDepsGraphPluginUrl, scalaFossaDocUrl)
63-
import Strategy.Scala.Plugin (genTreeJson, hasDependencyPlugins)
63+
import Strategy.Scala.Plugin (DependencyPluginsDetected (..), genTreeJson, hasDependencyPlugins)
6464
import Strategy.Scala.SbtDependencyTree (SbtArtifact (SbtArtifact), analyze, sbtDepTreeCmd)
6565
import Strategy.Scala.SbtDependencyTreeJson qualified as TreeJson
6666
import Types (
@@ -161,10 +161,10 @@ findProjects = walkWithFilters' $ \dir _ files -> do
161161
. context ("Listing sbt projects at " <> pathToText dir)
162162
$ genPoms dir
163163

164-
(miniDepPlugin, depPlugin) <- hasDependencyPlugins dir
165-
case (projectsRes, miniDepPlugin, depPlugin) of
164+
DependencyPluginsDetected{hasMiniDependencyTreePlugin, dependencyTreePlugin} <- hasDependencyPlugins dir
165+
case (projectsRes, hasMiniDependencyTreePlugin, dependencyTreePlugin) of
166166
(Nothing, _, _) -> pure ([], WalkSkipAll)
167-
(Just projects, False, False) -> pure ([SbtTargets Nothing [] projects], WalkSkipAll)
167+
(Just projects, False, Nothing) -> pure ([SbtTargets Nothing [] projects], WalkSkipAll)
168168
(Just projects, True, _) -> do
169169
-- project is using miniature dependency tree plugin,
170170
-- which is included by default with sbt 1.4+
@@ -184,9 +184,13 @@ findProjects = walkWithFilters' $ \dir _ files -> do
184184
(True, _) -> pure ([SbtTargets Nothing [] projects], WalkSkipAll)
185185
(_, Just _) -> pure ([SbtTargets depTreeStdOut [] projects], WalkSkipAll)
186186
(_, _) -> pure ([], WalkSkipAll)
187-
(Just projects, False, True) -> do
188-
-- project is explicitly configured to use dependency-tree-plugin
189-
treeJSONs <- recover $ genTreeJson dir
187+
(Just projects, False, Just pluginKind) -> do
188+
-- project is explicitly configured to use dependency-tree-plugin.
189+
-- The casing of the dependencyBrowseTree task differs between the
190+
-- modern (sbt 1.4+) DependencyTreePlugin and the legacy
191+
-- net.virtualvoid sbt-dependency-graph plugin; pluginKind selects
192+
-- the right one. See TKT-15490 / ANE-2718.
193+
treeJSONs <- recover $ genTreeJson pluginKind dir
190194
pure ([SbtTargets Nothing (fromMaybe [] treeJSONs) projects], WalkSkipAll)
191195

192196
analyzeWithPoms :: (Has Diagnostics sig m) => ScalaProject -> m DependencyResults
@@ -199,13 +203,13 @@ analyzeWithPoms (ScalaProject _ _ closure) = context "Analyzing sbt dependencies
199203
}
200204

201205
analyzeWithDepTreeJson :: (Has ReadFS sig m, Has Diagnostics sig m) => ScalaProject -> m DependencyResults
202-
analyzeWithDepTreeJson (ScalaProject _ treeJson closure) = context "Analyzing sbt dependencies using dependencyBrowseTreeHTML" $ do
206+
analyzeWithDepTreeJson (ScalaProject _ treeJson closure) = context "Analyzing sbt dependencies using dependency tree JSON" $ do
203207
treeJson' <-
204208
errCtx MissingFullDependencyPluginCtx $
205209
errHelp MissingFullDependencyPluginHelp $
206210
errDoc sbtDepsGraphPluginUrl $
207211
errDoc scalaFossaDocUrl $
208-
fromMaybeText "Could not retrieve output from sbt dependencyBrowseTreeHTML" treeJson
212+
fromMaybeText "Could not retrieve dependency tree JSON output from sbt" treeJson
209213
projectGraph <- TreeJson.analyze treeJson'
210214
pure $
211215
DependencyResults

src/Strategy/Scala/Plugin.hs

Lines changed: 95 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,13 @@ module Strategy.Scala.Plugin (
44
hasDependencyPlugins,
55
detectDependencyPlugins,
66
genTreeJson,
7+
DependencyTreePluginKind (..),
8+
DependencyPluginsDetected (..),
79
) where
810

911
import Control.Effect.Diagnostics (Diagnostics, fatalText)
1012
import Control.Effect.Stack (context)
13+
import Data.List (find)
1114
import Data.Maybe (mapMaybe)
1215
import Data.String.Conversion (ConvertUtf8 (decodeUtf8), toString)
1316
import Data.Text (Text)
@@ -22,45 +25,112 @@ import Effect.Exec (
2225
import Path (Abs, Dir, File, Path, mkRelFile, parent, parseAbsFile, (</>))
2326
import Strategy.Scala.Common (mkSbtCommand)
2427

28+
-- | Which non-mini dependency-tree plugin (if any) the project has installed.
29+
--
30+
-- The two plugins differ in the casing of their @dependencyBrowseTree@ task.
31+
-- See 'mkDependencyBrowseTreeCmd' for the command names.
32+
data DependencyTreePluginKind
33+
= -- | @sbt.plugins.DependencyTreePlugin@. Built into sbt 1.4+ and enabled
34+
-- explicitly via @addDependencyTreePlugin@ in @plugins.sbt@. Provides the
35+
-- uppercase @dependencyBrowseTreeHTML@ task.
36+
ModernDependencyTreePlugin
37+
| -- | @net.virtualvoid.sbt.graph.DependencyGraphPlugin@. The third-party
38+
-- @sbt-dependency-graph@ plugin used on sbt < 1.4. Provides the lowercase
39+
-- @dependencyBrowseTreeHtml@ task.
40+
LegacyDependencyGraphPlugin
41+
deriving (Eq, Ord, Show)
42+
43+
-- | What the @sbt plugins@ output told us about dependency-tree plugins.
44+
data DependencyPluginsDetected = DependencyPluginsDetected
45+
{ hasMiniDependencyTreePlugin :: Bool
46+
, dependencyTreePlugin :: Maybe DependencyTreePluginKind
47+
}
48+
deriving (Eq, Ord, Show)
49+
2550
-- | Returns list of plugins used by sbt.
2651
-- Ref: https://www.scala-sbt.org/1.x/docs/Plugins.html
2752
getPlugins :: Command
2853
getPlugins = mkSbtCommand "plugins"
2954

30-
-- | Returns (hasMiniDependencyTreePlugin, hasDependencyTreePlugin) by running sbt plugins.
31-
hasDependencyPlugins :: (Has Exec sig m, Has Diagnostics sig m) => Path Abs Dir -> m (Bool, Bool)
55+
-- | Detect which dependency-tree plugins are loaded by running @sbt plugins@.
56+
hasDependencyPlugins :: (Has Exec sig m, Has Diagnostics sig m) => Path Abs Dir -> m DependencyPluginsDetected
3257
hasDependencyPlugins projectDir = do
3358
stdoutText <- (TextLazy.toStrict . decodeUtf8) <$> context "Identifying plugins" (execThrow projectDir getPlugins)
3459
pure $ detectDependencyPlugins stdoutText
3560

36-
-- | Detect dependency plugins from sbt plugins output.
37-
-- Returns (hasMiniDependencyTreePlugin, hasDependencyTreePlugin).
38-
detectDependencyPlugins :: Text -> (Bool, Bool)
61+
-- | Classify dependency-tree plugins from the @sbt plugins@ output.
62+
--
63+
-- The plugin names mapped here:
64+
--
65+
-- * @sbt.plugins.MiniDependencyTreePlugin@ — bundled with sbt 1.4+, gives
66+
-- us the @dependencyTree@ task used by 'Strategy.Scala.SbtDependencyTree'.
67+
-- * @sbt.plugins.DependencyTreePlugin@ — opt-in on sbt 1.4+ via
68+
-- @addDependencyTreePlugin@. Provides the uppercase
69+
-- @dependencyBrowseTreeHTML@ task.
70+
-- * @net.virtualvoid.sbt.graph.DependencyGraphPlugin@ — third-party plugin
71+
-- used on sbt < 1.4. Provides the lowercase @dependencyBrowseTreeHtml@
72+
-- task.
73+
--
74+
-- @sbt plugins@ groups its output into per-project @Enabled plugins in
75+
-- \<project\>:@ sections, listing one bare plugin FQCN per indented line,
76+
-- followed by a trailing @Plugins that are loaded to the build but not enabled
77+
-- in any subprojects:@ section. A plugin the user disabled via
78+
-- @disablePlugins(...)@ moves into that trailing section. We therefore search
79+
-- only the text *before* that marker, so a disabled (loaded-but-not-enabled)
80+
-- plugin is not mistaken for an active one.
81+
--
82+
-- The plugin FQCN appears on its own line with no @: enabled in@ suffix. An
83+
-- earlier attempt to anchor detection on such a suffix matched nothing in real
84+
-- sbt output, which silently dropped deep dependencies and fell back to poms
85+
-- (regressed TKT-15490). See @test/Scala/PluginSpec.hs@ for fixtures captured
86+
-- from actual @sbt -batch -no-colors plugins@ runs.
87+
--
88+
-- When both modern and legacy non-mini plugins are present we prefer the
89+
-- modern one (sbt 1.4+ wins) since legacy plugin presence on a modern sbt
90+
-- typically means the user has both kinds of declarations in their build.
91+
detectDependencyPlugins :: Text -> DependencyPluginsDetected
3992
detectDependencyPlugins stdoutText =
40-
( Text.count ".MiniDependencyTreePlugin" stdoutText > 0
41-
, Text.count ".DependencyTreePlugin" stdoutText > 0
42-
|| Text.count "net.virtualvoid.sbt.graph.DependencyGraphPlugin" stdoutText > 0 -- sbt < 1.4
43-
)
93+
DependencyPluginsDetected
94+
{ hasMiniDependencyTreePlugin = enabled "sbt.plugins.MiniDependencyTreePlugin"
95+
, dependencyTreePlugin = snd <$> find (enabled . fst) treePlugins
96+
}
97+
where
98+
enabledSection = fst $ Text.breakOn notEnabledMarker stdoutText
99+
notEnabledMarker = "Plugins that are loaded to the build but not enabled"
100+
enabled name = name `Text.isInfixOf` enabledSection
101+
treePlugins =
102+
[ ("sbt.plugins.DependencyTreePlugin", ModernDependencyTreePlugin)
103+
, ("net.virtualvoid.sbt.graph.DependencyGraphPlugin", LegacyDependencyGraphPlugin)
104+
]
44105

45-
-- | Generates Dependency Trees.
46-
-- Ref: https://github.com/sbt/sbt/blob/master/main/src/main/scala/sbt/plugins/DependencyTreeSettings.scala#L101
47-
--
48-
-- This command unlike 'dependencyBrowseTree', does not open
49-
-- the browser when executed.
106+
-- | The sbt task that writes @tree.html@/@tree.json@ alongside its dependency
107+
-- output. Plugin name vs task casing:
50108
--
51-
-- It writes following files in target directory:
52-
-- ./tree.json
53-
-- ./tree.html
54-
-- ./tree.data.js
109+
-- * 'ModernDependencyTreePlugin' (sbt 1.4+, @addDependencyTreePlugin@) →
110+
-- @dependencyBrowseTreeHTML@.
111+
-- * 'LegacyDependencyGraphPlugin' (sbt < 1.4, @sbt-dependency-graph@) →
112+
-- @dependencyBrowseTreeHtml@.
55113
--
56-
-- This command is only used when the plugin is installed explicitly, i.e. sbt < 1.4.
57-
-- Newer versions of sbt will use the built-in dependency graph plugin.
58-
mkDependencyBrowseTreeHTMLCmd :: Command
59-
mkDependencyBrowseTreeHTMLCmd = mkSbtCommand "dependencyBrowseTreeHtml"
114+
-- Picking the wrong casing produces an sbt error like
115+
-- @[error] Not a valid command: dependencyBrowseTreeHTML@ which surfaces to
116+
-- the user as "Could not retrieve output from sbt dependencyBrowseTreeHTML".
117+
-- That regression (CLI 3.8.30) is tracked under TKT-15490 / ANE-2718.
118+
mkDependencyBrowseTreeCmd :: DependencyTreePluginKind -> Command
119+
mkDependencyBrowseTreeCmd ModernDependencyTreePlugin = mkSbtCommand "dependencyBrowseTreeHTML"
120+
mkDependencyBrowseTreeCmd LegacyDependencyGraphPlugin = mkSbtCommand "dependencyBrowseTreeHtml"
60121

61-
genTreeJson :: (Has Exec sig m, Has Diagnostics sig m) => Path Abs Dir -> m [Path Abs File]
62-
genTreeJson projectDir = do
63-
stdoutBL <- context "Generating dependency tree html" $ execThrow projectDir mkDependencyBrowseTreeHTMLCmd
122+
-- | Generates dependency trees by invoking the appropriate
123+
-- @dependencyBrowseTree*@ task. Unlike @dependencyBrowseTree@, this does not
124+
-- open a browser when executed.
125+
--
126+
-- It writes the following files in the target directory:
127+
--
128+
-- * @./tree.json@
129+
-- * @./tree.html@
130+
-- * @./tree.data.js@
131+
genTreeJson :: (Has Exec sig m, Has Diagnostics sig m) => DependencyTreePluginKind -> Path Abs Dir -> m [Path Abs File]
132+
genTreeJson pluginKind projectDir = do
133+
stdoutBL <- context "Generating dependency tree html" $ execThrow projectDir (mkDependencyBrowseTreeCmd pluginKind)
64134

65135
-- stdout for "sbt dependencyBrowseTreeHTML" looks like:
66136
-- -

0 commit comments

Comments
 (0)