Skip to content

Commit 719b9f8

Browse files
committed
WIP
1 parent 618ada4 commit 719b9f8

5 files changed

Lines changed: 227 additions & 40 deletions

File tree

spectrometer.cabal

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,8 @@ library
245245
App.Fossa.DependencyMetadata
246246
App.Fossa.DumpBinaries
247247
App.Fossa.EmbeddedBinary
248+
App.Fossa.Ficus.Analyze
249+
App.Fossa.Ficus.Types
248250
App.Fossa.FirstPartyScan
249251
App.Fossa.Init
250252
App.Fossa.Lernie.Analyze

src/App/Fossa/Analyze.hs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ import Data.String.Conversion (decodeUtf8, toText)
110110
import Data.Text.Extra (showT)
111111
import Data.Traversable (for)
112112
import Diag.Diagnostic as DI
113-
import Diag.Result (resultToMaybe)
113+
import Diag.Result (Result (Success), resultToMaybe)
114114
import Discovery.Archive qualified as Archive
115115
import Discovery.Filters (AllFilters, MavenScopeFilters, applyFilters, filterIsVSIOnly, ignoredPaths, isDefaultNonProductionPath)
116116
import Discovery.Projects (withDiscoveredProjects)
@@ -432,7 +432,7 @@ analyze cfg = Diag.context "fossa-analyze" $ do
432432
$ analyzeForReachability projectScans
433433
let reachabilityUnits = onlyFoundUnits reachabilityUnitsResult
434434

435-
let analysisResult = AnalysisScanResult projectScans vsiResults binarySearchResults manualSrcUnits dynamicLinkedResults maybeLernieResults reachabilityUnitsResult
435+
let analysisResult = AnalysisScanResult projectScans vsiResults binarySearchResults (Success [] Nothing) manualSrcUnits dynamicLinkedResults maybeLernieResults reachabilityUnitsResult
436436
renderScanSummary (severity cfg) maybeEndpointAppVersion analysisResult cfg
437437

438438
-- Need to check if vendored is empty as well, even if its a boolean that vendoredDeps exist

src/App/Fossa/Analyze/ScanSummary.hs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ renderDefaultSkippedTargetHelp =
193193
]
194194

195195
summarize :: Config.AnalyzeConfig -> Text -> AnalysisScanResult -> Maybe ([Doc AnsiStyle])
196-
summarize cfg endpointVersion (AnalysisScanResult dps vsi binary manualDeps dynamicLinkingDeps lernie reachabilityAttempts) =
196+
summarize cfg endpointVersion (AnalysisScanResult dps vsi binary ficus manualDeps dynamicLinkingDeps lernie reachabilityAttempts) =
197197
if (numProjects totalScanCount <= 0)
198198
then Nothing
199199
else
@@ -426,7 +426,7 @@ countWarnings ws =
426426
isIgnoredErrGroup _ = False
427427

428428
dumpResultLogsToTempFile :: (Has (Lift IO) sig m) => Config.AnalyzeConfig -> Text -> AnalysisScanResult -> m (Path Abs File)
429-
dumpResultLogsToTempFile cfg endpointVersion (AnalysisScanResult projects vsi binary manualDeps dynamicLinkingDeps lernieResults reachabilityAttempts) = do
429+
dumpResultLogsToTempFile cfg endpointVersion (AnalysisScanResult projects vsi binary ficus manualDeps dynamicLinkingDeps lernieResults reachabilityAttempts) = do
430430
let doc =
431431
stripAnsiEscapeCodes
432432
. renderStrict
@@ -449,7 +449,7 @@ dumpResultLogsToTempFile cfg endpointVersion (AnalysisScanResult projects vsi bi
449449
pure (tmpDir </> scanSummaryFileName)
450450
where
451451
scanSummary :: [Doc AnsiStyle]
452-
scanSummary = maybeToList (vsep <$> summarize cfg endpointVersion (AnalysisScanResult projects vsi binary manualDeps dynamicLinkingDeps lernieResults reachabilityAttempts))
452+
scanSummary = maybeToList (vsep <$> summarize cfg endpointVersion (AnalysisScanResult projects vsi binary ficus manualDeps dynamicLinkingDeps lernieResults reachabilityAttempts))
453453

454454
renderSourceUnit :: Doc AnsiStyle -> Result (Maybe a) -> Maybe (Doc AnsiStyle)
455455
renderSourceUnit header (Failure ws eg) = Just $ renderFailure ws eg $ vsep $ summarizeSrcUnit header Nothing (Failure ws eg)

src/App/Fossa/Ficus/Analyze.hs

Lines changed: 35 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,20 @@ module App.Fossa.Ficus.Analyze (
1414
import App.Fossa.EmbeddedBinary (BinaryPaths, toPath, withFicusBinary)
1515
import App.Fossa.Ficus.Types (
1616
FicusResults (..),
17+
FicusConfig (..),
18+
FicusRegex (..),
19+
FicusScanType (..),
20+
FicusMessage (..),
21+
FicusMessages (..),
22+
FicusMatch (..),
23+
FicusMatchData (..),
24+
FicusWarning (..),
25+
FicusError (..),
26+
)
27+
import App.Fossa.Lernie.Types (
28+
GrepEntry (..),
29+
GrepOptions (..),
30+
OrgWideCustomLicenseConfigPolicy (..),
1731
)
1832
import App.Types (FileUpload (..))
1933
import Control.Carrier.Debug (Debug)
@@ -65,30 +79,10 @@ analyzeWithFicus ::
6579
m (Maybe FicusResults)
6680
analyzeWithFicus rootDir maybeApiOpts grepOptions filters = do
6781
case (maybeApiOpts, orgWideCustomLicenseScanConfigPolicy grepOptions) of
68-
(_, Ignore) -> analyzeWithFicusMain rootDir grepOptions filters FileUploadMatchData
69-
(Nothing, Use) -> analyzeWithFicusMain rootDir grepOptions filters FileUploadMatchData
82+
(_, Ignore) -> analyzeWithFicusMain rootDir grepOptions filters
83+
(Nothing, Use) -> analyzeWithFicusMain rootDir grepOptions filters
7084
(Just apiOpts, Use) -> runFossaApiClient apiOpts $ analyzeWithFicusWithOrgInfo rootDir grepOptions filters
7185

72-
analyzeWithFicusWithOrgInfo ::
73-
( Has Diagnostics sig m
74-
, Has FossaApiClient sig m
75-
, Has (Lift IO) sig m
76-
, Has Exec sig m
77-
, Has ReadFS sig m
78-
, Has Telemetry sig m
79-
) =>
80-
Path Abs Dir ->
81-
GrepOptions ->
82-
Maybe LicenseScanPathFilters ->
83-
m (Maybe FicusResults)
84-
analyzeWithFicusWithOrgInfo rootDir grepOptions filters = do
85-
org <- getOrganization
86-
let orgWideCustomLicenses = orgCustomLicenseScanConfigs org
87-
uploadKind = orgFileUpload org
88-
89-
let options = grepOptions{customLicenseSearch = nub $ orgWideCustomLicenses <> customLicenseSearch grepOptions}
90-
analyzeWithFicusMain rootDir options filters uploadKind
91-
9286
analyzeWithFicusMain ::
9387
( Has Diagnostics sig m
9488
, Has (Lift IO) sig m
@@ -99,10 +93,9 @@ analyzeWithFicusMain ::
9993
Path Abs Dir ->
10094
GrepOptions ->
10195
Maybe LicenseScanPathFilters ->
102-
FileUpload ->
10396
m (Maybe FicusResults)
104-
analyzeWithFicusMain rootDir grepOptions filters uploadKind = do
105-
let maybeFicusConfig = grepOptionsToFicusConfig rootDir grepOptions filters uploadKind
97+
analyzeWithFicusMain rootDir grepOptions filters = do
98+
let maybeFicusConfig = grepOptionsToFicusConfig rootDir grepOptions filters
10699
case maybeFicusConfig of
107100
Just ficusConfig -> do
108101
unless (null $ customLicenseSearch grepOptions) $ trackUsage CustomLicenseSearchUsage
@@ -116,16 +109,25 @@ grepOptionsToFicusConfig :: Path Abs Dir -> GrepOptions -> Maybe LicenseScanPath
116109
grepOptionsToFicusConfig rootDir grepOptions filters uploadKind =
117110
case (customLicenseSearches <> keywordSearches) of
118111
[] -> Nothing
119-
res -> Just . FicusConfig rootDir res filters $ case uploadKind of
120-
FileUploadMatchData -> False
121-
FileUploadFullContent -> True
112+
res -> Just $ FicusConfig
113+
{ ficusRootDir = rootDir
114+
, ficusRegexes = res
115+
, ficusLicenseScanPathFilters = filters
116+
, ficusFullFiles = case uploadKind of
117+
FileUploadMatchData -> False
118+
FileUploadFullContent -> True
119+
}
122120
where
123-
customLicenseSearches = map (grepEntryToFicusRegex CustomLicense) (customLicenseSearch grepOptions)
124-
keywordSearches = map (grepEntryToFicusRegex KeywordSearch) (keywordSearch grepOptions)
121+
customLicenseSearches = map (grepEntryToFicusRegex FicusCustomLicense) (customLicenseSearch grepOptions)
122+
keywordSearches = map (grepEntryToFicusRegex FicusKeywordSearch) (keywordSearch grepOptions)
125123

126124
grepEntryToFicusRegex :: FicusScanType -> GrepEntry -> FicusRegex
127125
grepEntryToFicusRegex scanType grepEntry =
128-
FicusRegex (grepEntryMatchCriteria grepEntry) (grepEntryName grepEntry) scanType
126+
FicusRegex
127+
{ ficusPat = grepEntryMatchCriteria grepEntry
128+
, ficusName = grepEntryName grepEntry
129+
, ficusScanType = scanType
130+
}
129131

130132
runFicus ::
131133
( Has Diagnostics sig m
@@ -182,8 +184,8 @@ ficusMessagesToFicusResults FicusMessages{..} rootDir =
182184
, ficusResultsSourceUnit = sourceUnit
183185
}
184186
where
185-
keywordSearches = filterFicusMessages ficusMessageMatches KeywordSearch
186-
customLicenses = filterFicusMessages ficusMessageMatches CustomLicense
187+
keywordSearches = filterFicusMessages ficusMessageMatches FicusKeywordSearch
188+
customLicenses = filterFicusMessages ficusMessageMatches FicusCustomLicense
187189
sourceUnit = case NE.nonEmpty customLicenses of
188190
Nothing -> Nothing
189191
Just licenses -> ficusMatchToSourceUnit (NE.toList licenses) rootDir

src/App/Fossa/Ficus/Types.hs

Lines changed: 185 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,190 @@
11
{-# LANGUAGE RecordWildCards #-}
22

33
module App.Fossa.Ficus.Types (
4-
FicusResults(..)
4+
FicusResults (..),
5+
FicusConfig (..),
6+
FicusRegex (..),
7+
FicusScanType (..),
8+
FicusMessage (..),
9+
FicusMessages (..),
10+
FicusMatch (..),
11+
FicusMatchData (..),
12+
FicusWarning (..),
13+
FicusError (..),
514
) where
615

7-
data FicusResults = FicusResults { ficusAnalysisId :: Int }
16+
import Data.Aeson (FromJSON, KeyValue ((.=)), ToJSON (toJSON), Value (Object), object, withObject, withText, (.:?))
17+
import Data.Aeson qualified as A
18+
import Data.Aeson.Types ((.:))
19+
import Data.String.Conversion (ToText (toText))
20+
import Data.Text (Text)
21+
import GHC.Generics (Generic)
22+
import Path (Abs, Dir, Path)
23+
import Srclib.Types (LicenseSourceUnit)
24+
import Types (LicenseScanPathFilters (..))
25+
26+
data FicusResults = FicusResults { ficusAnalysisId :: Int } deriving (Eq, Ord, Show, Generic)
27+
28+
-- FOSSA_API_TOKEN=foobar cargo run -- analyze --exclude ".git/**" --exclude "target/**" --endpoint http://localhost:3000 --locator "pip+flask$1.2.3" --set snippet-scanning:batch-length=123
29+
30+
data FicusConfig = FicusConfig
31+
{ ficusRootDir :: Path Abs Dir
32+
, ficusFossaToken :: Text
33+
, ficusLocator :: Text
34+
, ficusGitignore :: Bool
35+
, ficusSkipHidden :: Bool
36+
, ficusAllExtensions :: Bool
37+
, ficusExcludes :: [Text]
38+
}
39+
deriving (Eq, Ord, Show, Generic)
40+
41+
data FicusRegex = FicusRegex
42+
{ ficusPat :: Text
43+
, ficusName :: Text
44+
, ficusScanType :: FicusScanType
45+
}
46+
deriving (Eq, Ord, Show, Generic)
47+
48+
instance ToJSON FicusRegex where
49+
toJSON FicusRegex{..} =
50+
object
51+
[ "pattern" .= toText ficusPat
52+
, "name" .= toText ficusName
53+
, "scan_type" .= toText ficusScanType
54+
]
55+
56+
data FicusScanType = FicusCustomLicense | FicusKeywordSearch
57+
deriving (Eq, Ord, Show, Generic)
58+
59+
instance ToText FicusScanType where
60+
toText FicusCustomLicense = "CustomLicense"
61+
toText FicusKeywordSearch = "KeywordSearch"
62+
63+
instance ToJSON FicusScanType where
64+
toJSON = toJSON . toText
65+
66+
instance FromJSON FicusScanType
67+
68+
data FicusMessageType = FicusMessageTypeMatch | FicusMessageTypeError | FicusMessageTypeWarning
69+
deriving (Eq, Ord, Show, Generic)
70+
71+
instance ToText FicusMessageType where
72+
toText FicusMessageTypeMatch = "Match"
73+
toText FicusMessageTypeError = "Error"
74+
toText FicusMessageTypeWarning = "Warning"
75+
76+
instance ToJSON FicusMessageType where
77+
toJSON = toJSON . toText
78+
79+
instance FromJSON FicusMessageType where
80+
parseJSON = withText "FicusMessageType" $ \msg -> do
81+
case msg of
82+
"Match" -> pure FicusMessageTypeMatch
83+
"Error" -> pure FicusMessageTypeError
84+
"Warning" -> pure FicusMessageTypeWarning
85+
_ -> fail "invalid Ficus message type"
86+
87+
data FicusMatch = FicusMatch
88+
{ ficusMatchPath :: Text
89+
, ficusMatchMatches :: [FicusMatchData]
90+
, ficusMatchContents :: Maybe Text
91+
}
92+
deriving (Eq, Ord, Show, Generic)
93+
94+
instance FromJSON FicusMatch where
95+
parseJSON = withObject "FicusMatch" $ \obj ->
96+
FicusMatch
97+
<$> (obj .: "path")
98+
<*> (obj .: "matches")
99+
<*> (obj .:? "contents")
100+
101+
data FicusWarning = FicusWarning
102+
{ ficusWarningMessage :: Text
103+
, ficusWarningType :: Text
104+
}
105+
deriving (Eq, Ord, Show, Generic)
106+
107+
instance FromJSON FicusWarning where
108+
parseJSON = withObject "FicusWarning" $ \obj ->
109+
FicusWarning
110+
<$> (obj .: "message")
111+
<*> (obj .: "type")
112+
113+
data FicusError = FicusError
114+
{ ficusErrorMessage :: Text
115+
, ficusErrorType :: Text
116+
}
117+
deriving (Eq, Ord, Show, Generic)
118+
119+
instance FromJSON FicusError where
120+
parseJSON = withObject "FicusError" $ \obj ->
121+
FicusError
122+
<$> (obj .: "message")
123+
<*> (obj .: "type")
124+
125+
data FicusMatchData = FicusMatchData
126+
{ ficusMatchDataPattern :: Text
127+
, ficusMatchDataMatchString :: Text
128+
, ficusMatchDataScanType :: FicusScanType
129+
, ficusMatchDataName :: Text
130+
, ficusMatchDataStartByte :: Integer
131+
, ficusMatchDataEndByte :: Integer
132+
, ficusMatchDataStartLine :: Integer
133+
, ficusMatchDataEndLine :: Integer
134+
}
135+
deriving (Eq, Ord, Show, Generic)
136+
137+
instance FromJSON FicusMatchData where
138+
parseJSON = withObject "FicusMatchData" $ \obj ->
139+
FicusMatchData
140+
<$> (obj .: "pattern")
141+
<*> (obj .: "match_str")
142+
<*> (obj .: "scan_type")
143+
<*> (obj .: "name")
144+
<*> (obj .: "start_byte")
145+
<*> (obj .: "end_byte")
146+
<*> (obj .: "start_line")
147+
<*> (obj .: "end_line")
148+
149+
instance ToJSON FicusMatchData
150+
151+
data FicusMessages = FicusMessages
152+
{ ficusMessageWarnings :: [FicusWarning]
153+
, ficusMessageErrors :: [FicusError]
154+
, ficusMessageMatches :: [FicusMatch]
155+
}
156+
deriving (Eq, Ord, Show, Generic)
157+
158+
instance Semigroup FicusMessages where
159+
FicusMessages w1 e1 m1 <> FicusMessages w2 e2 m2 = FicusMessages (w1 <> w2) (e1 <> e2) (m1 <> m2)
160+
161+
instance Monoid FicusMessages where
162+
mempty = FicusMessages [] [] []
163+
164+
data FicusMessage = FicusMessageFicusMatch FicusMatch | FicusMessageFicusWarning FicusWarning | FicusMessageFicusError FicusError
165+
deriving (Eq, Ord, Show, Generic)
166+
167+
instance FromJSON FicusMessage where
168+
parseJSON (Object o) = do
169+
messageType <- o .: "type"
170+
case messageType of
171+
FicusMessageTypeWarning -> do
172+
Object d <- o .: "data"
173+
let message = d .: "message"
174+
let warningType = d .: "type"
175+
warning <- FicusWarning <$> warningType <*> message
176+
pure $ FicusMessageFicusWarning warning
177+
FicusMessageTypeError -> do
178+
Object d <- o .: "data"
179+
let errorType = d .: "type"
180+
let message = d .: "message"
181+
err <- FicusError <$> errorType <*> message
182+
pure $ FicusMessageFicusError err
183+
FicusMessageTypeMatch -> do
184+
Object d <- o .: "data"
185+
let path = d .: "path"
186+
let matches = d .: "matches"
187+
let contents = d .:? "contents"
188+
match <- FicusMatch <$> path <*> matches <*> contents
189+
pure $ FicusMessageFicusMatch match
190+
parseJSON _ = fail "Invalid schema for FicusMessage. It must be an object"

0 commit comments

Comments
 (0)