Skip to content

Commit 1d7368f

Browse files
james-fossajagonalez
james-fossa
and
jagonalez
authored
[ANE-2484] Download ficus in vendor_download.sh (#1565)
* [ANE-2484] Download ficus in vendor_download.sh * [ANE-2484] Listen to shellcheck's wisdom * [ANE-2484][ANE-2503] Actually call Ficus, use `--x-snippet-scan` as a flag. (#1573) * gradle: exclude constraints when retrieving dependencies (#1563) * [ane-2575] scan all layers for os info (#1566) * scan all layers for os info * add test * lint * accidently on purposed * whitespace * typo * update changelog * no need to log * prepare for release * WIP * WIP * Get ficus wired in and at least vaguely tested. More to do to get tests to be coherent. Now we're cooking with gas: ``` Running Ficus analysis on /Users/jclemer/wam/ [DEBUG] Executing ficus [DEBUG] Ficus returned 4 errors, 0 debug messages, 1 findings [WARN] ERROR fingerprint: Read( Custom { kind: InvalidData, error: "binary file detected: /Users/jclemer/wam/.git/index", }, ) [WARN] ERROR fingerprint: Read( Custom { kind: InvalidData, error: "binary file detected: /Users/jclemer/wam/.git/objects/pack/pack-183ce412024750728f9349e31668d39ee389840e.idx", }, ) [WARN] ERROR fingerprint: Read( Custom { kind: InvalidData, error: "binary file detected: /Users/jclemer/wam/.git/objects/pack/pack-183ce412024750728f9349e31668d39ee389840e.pack", }, ) [WARN] ERROR fingerprint: Read( Custom { kind: InvalidData, error: "binary file detected: /Users/jclemer/wam/.git/objects/pack/pack-183ce412024750728f9349e31668d39ee389840e.rev", }, ) FINDING fingerprint: {"analysis_id":15} Ficus analysis completed successfully with analysis ID: 15 ``` * Fixing up formatting * [ANE-2484] Add ficus to extra-source files * Fix FICUS_ASSET_POSTFIX to match changed release * Fix Windows postfix for changed release * Change themis arch in suffix --------- Co-authored-by: Jeremy Gonzalez <jeremy@fossa.com> * Caveperson debug vendor_download --------- Co-authored-by: Jeremy Gonzalez <jeremy@fossa.com>
1 parent 51add54 commit 1d7368f

23 files changed

Lines changed: 790 additions & 47 deletions

File tree

‎Changelog.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
# FOSSA CLI Changelog
22

3+
## 3.11.0
4+
5+
- Add a dependency on Ficus, a new internal tool.
6+
- Add the `--x-snippet-scan` flag, an experimental flag for using Ficus.
7+
38
## 3.10.14
49

510
- gradle: Do not report version constraints, version contraints are contained within an`DependencyResult`, filter out any constraints by checking [`isConstraint()`](https://docs.gradle.org/current/javadoc/org/gradle/api/artifacts/result/DependencyResult.html#isConstraint()). ([#1563](https://github.com/fossas/fossa-cli/pull/1563))

‎integration-test/Analysis/FicusSpec.hs‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
{-# LANGUAGE OverloadedStrings #-}
2+
{-# LANGUAGE QuasiQuotes #-}
3+
4+
module Analysis.FicusSpec (spec) where
5+
6+
import App.Fossa.Ficus.Analyze (analyzeWithFicus)
7+
import App.Fossa.Ficus.Types (FicusSnippetScanResults (..))
8+
import App.Types (ProjectRevision (..))
9+
import Control.Carrier.Diagnostics (runDiagnostics)
10+
import Control.Carrier.Stack (runStack)
11+
import Control.Carrier.StickyLogger (ignoreStickyLogger)
12+
import Control.Timeout (Duration (Seconds))
13+
import Data.List (isInfixOf)
14+
import Data.String.Conversion (toText)
15+
import Diag.Result (Result (Failure, Success))
16+
import Effect.Exec (runExecIO)
17+
import Effect.Logger (ignoreLogger)
18+
import Effect.ReadFS (runReadFSIO)
19+
import Fossa.API.Types (ApiKey (..), ApiOpts (..))
20+
import Path (Dir, Path, Rel, reldir, (</>))
21+
import Path.IO qualified as PIO
22+
import System.Environment (lookupEnv)
23+
import Test.Hspec
24+
import Text.URI (mkURI)
25+
26+
fixtureDir :: Path Rel Dir
27+
fixtureDir = [reldir|integration-test/Analysis/testdata/ficus|]
28+
29+
spec :: Spec
30+
spec = do
31+
describe "Ficus snippet scanning integration" $ do
32+
it "should run ficus binary successfully" $ do
33+
-- Check for API configuration from environment
34+
maybeApiKey <- lookupEnv "FOSSA_API_KEY"
35+
maybeEndpoint <- lookupEnv "FOSSA_ENDPOINT"
36+
37+
apiOpts <- case (maybeApiKey, maybeEndpoint) of
38+
(Just keyStr, Just endpointStr) -> do
39+
uri <- case mkURI (toText endpointStr) of
40+
Just validUri -> pure validUri
41+
Nothing -> fail $ "Invalid API endpoint URL: " ++ endpointStr
42+
let opts = ApiOpts (Just uri) (ApiKey (toText keyStr)) (Seconds 60)
43+
pure (Just opts)
44+
_ -> pure Nothing
45+
46+
currentDir <- PIO.getCurrentDir
47+
let testDataDir = currentDir </> fixtureDir
48+
revision = ProjectRevision "ficus-integration-test" "testdata-123456" (Just "integration-test")
49+
50+
-- Check if test data exists
51+
testDataExists <- PIO.doesDirExist testDataDir
52+
testDataExists `shouldBe` True
53+
54+
result <- runStack . runDiagnostics . ignoreStickyLogger . ignoreLogger . runExecIO . runReadFSIO $ analyzeWithFicus testDataDir apiOpts revision Nothing
55+
56+
case result of
57+
Success _warnings analysisResult -> do
58+
case analysisResult of
59+
Just (FicusSnippetScanResults analysisId) -> do
60+
analysisId `shouldSatisfy` (> 0)
61+
Nothing -> do
62+
-- No snippet scan results returned - this is acceptable for integration testing
63+
True `shouldBe` True
64+
Failure _warnings errors -> do
65+
let failureMsg = show errors
66+
case apiOpts of
67+
Just _ -> do
68+
-- With API credentials, accept 404/temp-storage errors as valid connectivity tests
69+
if "404" `isInfixOf` failureMsg || "temp-storage" `isInfixOf` failureMsg || "Status(" `isInfixOf` failureMsg
70+
then True `shouldBe` True -- Expected API connectivity issue
71+
else fail ("Ficus integration test failed unexpectedly: " ++ failureMsg)
72+
Nothing -> do
73+
-- Without API credentials, analysis failure is expected
74+
True `shouldBe` True

‎integration-test/Analysis/testdata/ficus/README.md‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Ficus Integration Test Project
2+
3+
This is a test project used for integration testing of the ficus snippet scanning functionality in the FOSSA CLI.
4+
5+
## Files
6+
7+
- `main.c` - Main program with various C constructs for snippet analysis
8+
- `helper.h` - Header file with function declarations and common macros
9+
- `helper.c` - Implementation of helper functions with typical C patterns
10+
- `README.md` - This documentation file
11+
12+
## Purpose
13+
14+
These files contain various C programming patterns that might be detected by ficus fingerprinting:
15+
16+
- Standard library usage (`stdio.h`, `stdlib.h`, `string.h`)
17+
- Memory allocation and deallocation patterns
18+
- String manipulation functions
19+
- Mathematical operations
20+
- Loops and conditionals
21+
- Macro definitions
22+
- Struct definitions
23+
- Common C idioms and patterns
24+
25+
The goal is to provide realistic C code that ficus can analyze for snippets while remaining simple enough for integration testing.
26+
27+
## Building
28+
29+
This is not intended to be built as a real program, but rather analyzed by ficus for snippet detection.
30+
31+
```bash
32+
# This is what ficus would analyze:
33+
ficus analyze --endpoint <endpoint> --secret <api-key> --locator <project-locator>
34+
```

‎integration-test/Analysis/testdata/ficus/helper.c‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#include <stdio.h>
2+
#include <stdlib.h>
3+
#include <string.h>
4+
#include "helper.h"
5+
6+
/**
7+
* Implementation file for helper functions
8+
* Contains common C patterns for snippet analysis
9+
*/
10+
11+
int helper_function(int value) {
12+
// Simple mathematical operation
13+
if (value < 0) {
14+
return -1;
15+
}
16+
17+
// Common algorithm pattern
18+
int result = 1;
19+
for (int i = 1; i <= value; i++) {
20+
result += i * 2;
21+
}
22+
23+
return result % 1000; // Keep result manageable
24+
}
25+
26+
void process_data(const char* input, char* output, size_t max_len) {
27+
if (input == NULL || output == NULL || max_len == 0) {
28+
return;
29+
}
30+
31+
// String processing pattern
32+
size_t input_len = strlen(input);
33+
size_t copy_len = MIN(input_len, max_len - 1);
34+
35+
strncpy(output, input, copy_len);
36+
output[copy_len] = '\0';
37+
38+
// Convert to uppercase
39+
for (size_t i = 0; i < copy_len; i++) {
40+
if (output[i] >= 'a' && output[i] <= 'z') {
41+
output[i] = output[i] - 'a' + 'A';
42+
}
43+
}
44+
}
45+
46+
double calculate_average(int* numbers, int count) {
47+
if (numbers == NULL || count <= 0) {
48+
return 0.0;
49+
}
50+
51+
long long sum = 0;
52+
for (int i = 0; i < count; i++) {
53+
sum += numbers[i];
54+
}
55+
56+
return (double)sum / count;
57+
}

‎integration-test/Analysis/testdata/ficus/helper.h‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#ifndef HELPER_H
2+
#define HELPER_H
3+
4+
/**
5+
* Helper header file for ficus integration testing
6+
* Contains function declarations and common patterns
7+
* that might be detected by snippet analysis.
8+
*/
9+
10+
// Function declarations
11+
int helper_function(int value);
12+
void process_data(const char* input, char* output, size_t max_len);
13+
double calculate_average(int* numbers, int count);
14+
15+
// Common macros that might be fingerprinted
16+
#define MAX_BUFFER_SIZE 1024
17+
#define MIN(a, b) ((a) < (b) ? (a) : (b))
18+
#define MAX(a, b) ((a) > (b) ? (a) : (b))
19+
20+
// Struct definition
21+
typedef struct {
22+
int id;
23+
char name[64];
24+
double value;
25+
} data_record_t;
26+
27+
// Function-like macro
28+
#define SAFE_FREE(ptr) do { \
29+
if ((ptr) != NULL) { \
30+
free(ptr); \
31+
(ptr) = NULL; \
32+
} \
33+
} while(0)
34+
35+
#endif // HELPER_H

‎integration-test/Analysis/testdata/ficus/main.c‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#include <stdio.h>
2+
#include <stdlib.h>
3+
#include <string.h>
4+
#include "helper.h"
5+
6+
/*
7+
* Simple C program for ficus snippet analysis testing
8+
* This file contains various C constructs that might be detected
9+
* by ficus fingerprinting algorithms.
10+
*/
11+
12+
int main(int argc, char *argv[]) {
13+
printf("Hello, World!\n");
14+
15+
// String manipulation that might be fingerprinted
16+
char buffer[256];
17+
strncpy(buffer, "Test string for analysis", sizeof(buffer) - 1);
18+
buffer[sizeof(buffer) - 1] = '\0';
19+
20+
// Function call to helper
21+
int result = helper_function(42);
22+
23+
// Memory allocation pattern
24+
int *data = malloc(10 * sizeof(int));
25+
if (data != NULL) {
26+
for (int i = 0; i < 10; i++) {
27+
data[i] = i * i;
28+
}
29+
free(data);
30+
}
31+
32+
printf("Result: %d\n", result);
33+
return EXIT_SUCCESS;
34+
}

‎spectrometer.cabal‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ extra-source-files:
2222
target/release/millhone
2323
target/release/millhone.exe
2424
vendor-bins/circe
25+
vendor-bins/ficus
2526
vendor-bins/index.gob.xz
2627
vendor-bins/lernie
2728
vendor-bins/themis-cli
@@ -245,6 +246,8 @@ library
245246
App.Fossa.DependencyMetadata
246247
App.Fossa.DumpBinaries
247248
App.Fossa.EmbeddedBinary
249+
App.Fossa.Ficus.Analyze
250+
App.Fossa.Ficus.Types
248251
App.Fossa.FirstPartyScan
249252
App.Fossa.Init
250253
App.Fossa.Lernie.Analyze
@@ -752,6 +755,7 @@ test-suite integration-tests
752755
Analysis.CocoapodsSpec
753756
Analysis.ElixirSpec
754757
Analysis.ErlangSpec
758+
Analysis.FicusSpec
755759
Analysis.FixtureExpectationUtils
756760
Analysis.FixtureUtils
757761
Analysis.GoSpec

‎src/App/Fossa/Analyze.hs‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ import App.Fossa.Config.Analyze (
5050
)
5151
import App.Fossa.Config.Analyze qualified as Config
5252
import App.Fossa.Config.Common (DestinationMeta (..), destinationApiOpts, destinationMetadata)
53+
import App.Fossa.Ficus.Analyze (analyzeWithFicus)
5354
import App.Fossa.FirstPartyScan (runFirstPartyScan)
5455
import App.Fossa.Lernie.Analyze (analyzeWithLernie)
5556
import App.Fossa.Lernie.Types (LernieResults (..))
@@ -108,7 +109,7 @@ import Data.String.Conversion (decodeUtf8, toText)
108109
import Data.Text.Extra (showT)
109110
import Data.Traversable (for)
110111
import Diag.Diagnostic as DI
111-
import Diag.Result (resultToMaybe)
112+
import Diag.Result (Result (Success), resultToMaybe)
112113
import Discovery.Archive qualified as Archive
113114
import Discovery.Filters (AllFilters, MavenScopeFilters, applyFilters, filterIsVSIOnly, ignoredPaths, isDefaultNonProductionPath)
114115
import Discovery.Projects (withDiscoveredProjects)
@@ -297,6 +298,7 @@ analyze cfg = Diag.context "fossa-analyze" $ do
297298
shouldAnalyzePathDependencies = resolvePathDependencies $ Config.experimental cfg
298299
allowedTactics = Config.allowedTacticTypes cfg
299300
withoutDefaultFilters = Config.withoutDefaultFilters cfg
301+
enableSnippetScan = Config.xSnippetScan cfg
300302

301303
manualSrcUnits <-
302304
Diag.errorBoundaryIO . diagToDebug $
@@ -335,6 +337,19 @@ analyze cfg = Diag.context "fossa-analyze" $ do
335337
if (fromFlag BinaryDiscovery $ Config.binaryDiscoveryEnabled $ Config.vsiOptions cfg)
336338
then analyzeDiscoverBinaries basedir filters
337339
else pure Nothing
340+
maybeFicusResults <-
341+
Diag.errorBoundaryIO . diagToDebug $
342+
if not enableSnippetScan
343+
then do
344+
logInfo "Skipping ficus snippet scanning (--x-snippet-scan not set)"
345+
pure Nothing
346+
else
347+
if filterIsVSIOnly filters
348+
then do
349+
logInfo "Running in VSI only mode, skipping snippet-scan"
350+
pure Nothing
351+
else Diag.context "snippet-scanning" . runStickyLogger SevInfo $ analyzeWithFicus basedir maybeApiOpts revision $ Config.licenseScanPathFilters vendoredDepsOptions
352+
let ficusResults = join . resultToMaybe $ maybeFicusResults
338353
maybeLernieResults <-
339354
Diag.errorBoundaryIO . diagToDebug $
340355
if filterIsVSIOnly filters
@@ -422,7 +437,7 @@ analyze cfg = Diag.context "fossa-analyze" $ do
422437
$ analyzeForReachability projectScans
423438
let reachabilityUnits = onlyFoundUnits reachabilityUnitsResult
424439

425-
let analysisResult = AnalysisScanResult projectScans vsiResults binarySearchResults manualSrcUnits dynamicLinkedResults maybeLernieResults reachabilityUnitsResult
440+
let analysisResult = AnalysisScanResult projectScans vsiResults binarySearchResults (Success [] Nothing) manualSrcUnits dynamicLinkedResults maybeLernieResults reachabilityUnitsResult
426441
renderScanSummary (severity cfg) maybeEndpointAppVersion analysisResult cfg
427442

428443
-- Need to check if vendored is empty as well, even if its a boolean that vendoredDeps exist
@@ -442,16 +457,16 @@ analyze cfg = Diag.context "fossa-analyze" $ do
442457
(False, FilteredAll) -> Diag.warn ErrFilteredAllProjects $> emptyScanUnits
443458
(True, FilteredAll) -> Diag.warn ErrOnlyKeywordSearchResultsFound $> emptyScanUnits
444459
(_, CountedScanUnits scanUnits) -> pure scanUnits
445-
sendToDestination outputResult iatAssertion destination basedir jsonOutput revision scanUnits reachabilityUnits
460+
sendToDestination outputResult iatAssertion destination basedir jsonOutput revision scanUnits reachabilityUnits ficusResults
446461

447462
pure outputResult
448463
where
449-
sendToDestination result iatAssertion destination basedir jsonOutput revision scanUnits reachabilityUnits =
464+
sendToDestination result iatAssertion destination basedir jsonOutput revision scanUnits reachabilityUnits ficusResults =
450465
let doUpload (DestinationMeta (apiOpts, metadata)) =
451466
Diag.context "upload-results"
452467
. runFossaApiClient apiOpts
453468
$ do
454-
locator <- uploadSuccessfulAnalysis (BaseDir basedir) metadata jsonOutput revision scanUnits reachabilityUnits
469+
locator <- uploadSuccessfulAnalysis (BaseDir basedir) metadata jsonOutput revision scanUnits reachabilityUnits ficusResults
455470
doAssertRevisionBinaries iatAssertion locator
456471
in case destination of
457472
OutputStdout -> logStdout . decodeUtf8 $ Aeson.encode result

‎src/App/Fossa/Analyze/ScanSummary.hs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ renderDefaultSkippedTargetHelp =
193193
]
194194

195195
summarize :: Config.AnalyzeConfig -> Text -> AnalysisScanResult -> Maybe ([Doc AnsiStyle])
196-
summarize cfg endpointVersion (AnalysisScanResult dps vsi binary manualDeps dynamicLinkingDeps lernie reachabilityAttempts) =
196+
summarize cfg endpointVersion (AnalysisScanResult dps vsi binary _ manualDeps dynamicLinkingDeps lernie reachabilityAttempts) =
197197
if (numProjects totalScanCount <= 0)
198198
then Nothing
199199
else
@@ -426,7 +426,7 @@ countWarnings ws =
426426
isIgnoredErrGroup _ = False
427427

428428
dumpResultLogsToTempFile :: (Has (Lift IO) sig m) => Config.AnalyzeConfig -> Text -> AnalysisScanResult -> m (Path Abs File)
429-
dumpResultLogsToTempFile cfg endpointVersion (AnalysisScanResult projects vsi binary manualDeps dynamicLinkingDeps lernieResults reachabilityAttempts) = do
429+
dumpResultLogsToTempFile cfg endpointVersion (AnalysisScanResult projects vsi binary ficus manualDeps dynamicLinkingDeps lernieResults reachabilityAttempts) = do
430430
let doc =
431431
stripAnsiEscapeCodes
432432
. renderStrict
@@ -449,7 +449,7 @@ dumpResultLogsToTempFile cfg endpointVersion (AnalysisScanResult projects vsi bi
449449
pure (tmpDir </> scanSummaryFileName)
450450
where
451451
scanSummary :: [Doc AnsiStyle]
452-
scanSummary = maybeToList (vsep <$> summarize cfg endpointVersion (AnalysisScanResult projects vsi binary manualDeps dynamicLinkingDeps lernieResults reachabilityAttempts))
452+
scanSummary = maybeToList (vsep <$> summarize cfg endpointVersion (AnalysisScanResult projects vsi binary ficus manualDeps dynamicLinkingDeps lernieResults reachabilityAttempts))
453453

454454
renderSourceUnit :: Doc AnsiStyle -> Result (Maybe a) -> Maybe (Doc AnsiStyle)
455455
renderSourceUnit header (Failure ws eg) = Just $ renderFailure ws eg $ vsep $ summarizeSrcUnit header Nothing (Failure ws eg)

‎src/App/Fossa/Analyze/Types.hs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ module App.Fossa.Analyze.Types (
1212

1313
import App.Fossa.Analyze.Project (ProjectResult)
1414
import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig)
15+
import App.Fossa.Ficus.Types (FicusSnippetScanResults)
1516
import App.Fossa.Lernie.Types (LernieResults)
1617
import App.Fossa.Reachability.Types (SourceUnitReachability (..))
1718
import App.Types (Mode)
@@ -80,6 +81,7 @@ data AnalysisScanResult = AnalysisScanResult
8081
{ analyzersScanResult :: [DiscoveredProjectScan]
8182
, vsiScanResult :: Result (Maybe [SourceUnit])
8283
, binaryDepsScanResult :: Result (Maybe SourceUnit)
84+
, ficusResult :: Result (Maybe FicusSnippetScanResults)
8385
, fossaDepsScanResult :: Result (Maybe SourceUnit)
8486
, dynamicLinkingResult :: Result (Maybe SourceUnit)
8587
, lernieResult :: Result (Maybe LernieResults)

0 commit comments

Comments
 (0)