Get ficus wired in and at least vaguely tested. More to do to get tests to be coherent.

Author: james-fossa

integration-test/Analysis/FicusSpec.hs

@@ -0,0 +1,73 @@
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE QuasiQuotes #-}
+
+module Analysis.FicusSpec (spec) where
+
+import App.Fossa.Ficus.Analyze (analyzeWithFicus)
+import App.Fossa.Ficus.Types (FicusSnippetScanResults (..))
+import App.Types (ProjectRevision (..))
+import Control.Carrier.Diagnostics (runDiagnostics)
+import Control.Carrier.Stack (runStack)
+import Control.Carrier.StickyLogger (ignoreStickyLogger)
+import Control.Timeout (Duration (Seconds))
+import Data.List (isInfixOf)
+import Data.String.Conversion (toText)
+import Diag.Result (Result (Failure, Success))
+import Effect.Exec (runExecIO)
+import Effect.ReadFS (runReadFSIO)
+import Fossa.API.Types (ApiKey (..), ApiOpts (..))
+import Path (Dir, Path, Rel, reldir, (</>))
+import Path.IO qualified as PIO
+import System.Environment (lookupEnv)
+import Test.Hspec
+import Text.URI (mkURI)
+
+fixtureDir :: Path Rel Dir
+fixtureDir = [reldir|integration-test/Analysis/testdata/ficus|]
+
+spec :: Spec
+spec = do
+  describe "Ficus snippet scanning integration" $ do
+    it "should run ficus binary successfully" $ do
+      -- Check for API configuration from environment
+      maybeApiKey <- lookupEnv "FOSSA_API_KEY"
+      maybeEndpoint <- lookupEnv "FOSSA_ENDPOINT"
+
+      apiOpts <- case (maybeApiKey, maybeEndpoint) of
+        (Just keyStr, Just endpointStr) -> do
+          uri <- case mkURI (toText endpointStr) of
+            Just validUri -> pure validUri
+            Nothing -> fail $ "Invalid API endpoint URL: " ++ endpointStr
+          let opts = ApiOpts (Just uri) (ApiKey (toText keyStr)) (Seconds 60)
+          pure (Just opts)
+        _ -> pure Nothing
+
+      currentDir <- PIO.getCurrentDir
+      let testDataDir = currentDir </> fixtureDir
+          revision = ProjectRevision "ficus-integration-test" "testdata-123456" (Just "integration-test")
+
+      -- Check if test data exists
+      testDataExists <- PIO.doesDirExist testDataDir
+      testDataExists `shouldBe` True
+
+      result <- runStack . runDiagnostics . ignoreStickyLogger . runExecIO . runReadFSIO $ analyzeWithFicus testDataDir apiOpts revision Nothing
+
+      case result of
+        Success _warnings analysisResult -> do
+          case analysisResult of
+            Just (FicusSnippetScanResults analysisId) -> do
+              analysisId `shouldSatisfy` (> 0)
+            Nothing -> do
+              -- No snippet scan results returned - this is acceptable for integration testing
+              True `shouldBe` True
+        Failure _warnings errors -> do
+          let failureMsg = show errors
+          case apiOpts of
+            Just _ -> do
+              -- With API credentials, accept 404/temp-storage errors as valid connectivity tests
+              if "404" `isInfixOf` failureMsg || "temp-storage" `isInfixOf` failureMsg || "Status(" `isInfixOf` failureMsg
+                then True `shouldBe` True -- Expected API connectivity issue
+                else fail ("Ficus integration test failed unexpectedly: " ++ failureMsg)
+            Nothing -> do
+              -- Without API credentials, analysis failure is expected
+              True `shouldBe` True

integration-test/Analysis/testdata/ficus/README.md

@@ -0,0 +1,34 @@
+# Ficus Integration Test Project
+
+This is a test project used for integration testing of the ficus snippet scanning functionality in the FOSSA CLI.
+
+## Files
+
+- `main.c` - Main program with various C constructs for snippet analysis
+- `helper.h` - Header file with function declarations and common macros
+- `helper.c` - Implementation of helper functions with typical C patterns
+- `README.md` - This documentation file
+
+## Purpose
+
+These files contain various C programming patterns that might be detected by ficus fingerprinting:
+
+- Standard library usage (`stdio.h`, `stdlib.h`, `string.h`)
+- Memory allocation and deallocation patterns
+- String manipulation functions
+- Mathematical operations
+- Loops and conditionals
+- Macro definitions
+- Struct definitions
+- Common C idioms and patterns
+
+The goal is to provide realistic C code that ficus can analyze for snippets while remaining simple enough for integration testing.
+
+## Building
+
+This is not intended to be built as a real program, but rather analyzed by ficus for snippet detection.
+
+```bash
+# This is what ficus would analyze:
+ficus analyze --endpoint <endpoint> --secret <api-key> --locator <project-locator>
+```

integration-test/Analysis/testdata/ficus/helper.c

@@ -0,0 +1,57 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "helper.h"
+
+/**
+ * Implementation file for helper functions
+ * Contains common C patterns for snippet analysis
+ */
+
+int helper_function(int value) {
+    // Simple mathematical operation
+    if (value < 0) {
+        return -1;
+    }
+    
+    // Common algorithm pattern
+    int result = 1;
+    for (int i = 1; i <= value; i++) {
+        result += i * 2;
+    }
+    
+    return result % 1000;  // Keep result manageable
+}
+
+void process_data(const char* input, char* output, size_t max_len) {
+    if (input == NULL || output == NULL || max_len == 0) {
+        return;
+    }
+    
+    // String processing pattern
+    size_t input_len = strlen(input);
+    size_t copy_len = MIN(input_len, max_len - 1);
+    
+    strncpy(output, input, copy_len);
+    output[copy_len] = '\0';
+    
+    // Convert to uppercase
+    for (size_t i = 0; i < copy_len; i++) {
+        if (output[i] >= 'a' && output[i] <= 'z') {
+            output[i] = output[i] - 'a' + 'A';
+        }
+    }
+}
+
+double calculate_average(int* numbers, int count) {
+    if (numbers == NULL || count <= 0) {
+        return 0.0;
+    }
+    
+    long long sum = 0;
+    for (int i = 0; i < count; i++) {
+        sum += numbers[i];
+    }
+    
+    return (double)sum / count;
+}

integration-test/Analysis/testdata/ficus/helper.h

@@ -0,0 +1,35 @@
+#ifndef HELPER_H
+#define HELPER_H
+
+/**
+ * Helper header file for ficus integration testing
+ * Contains function declarations and common patterns
+ * that might be detected by snippet analysis.
+ */
+
+// Function declarations
+int helper_function(int value);
+void process_data(const char* input, char* output, size_t max_len);
+double calculate_average(int* numbers, int count);
+
+// Common macros that might be fingerprinted
+#define MAX_BUFFER_SIZE 1024
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+
+// Struct definition
+typedef struct {
+    int id;
+    char name[64];
+    double value;
+} data_record_t;
+
+// Function-like macro
+#define SAFE_FREE(ptr) do { \
+    if ((ptr) != NULL) { \
+        free(ptr); \
+        (ptr) = NULL; \
+    } \
+} while(0)
+
+#endif // HELPER_H

integration-test/Analysis/testdata/ficus/main.c

@@ -0,0 +1,34 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "helper.h"
+
+/* 
+ * Simple C program for ficus snippet analysis testing
+ * This file contains various C constructs that might be detected
+ * by ficus fingerprinting algorithms.
+ */
+
+int main(int argc, char *argv[]) {
+    printf("Hello, World!\n");
+    
+    // String manipulation that might be fingerprinted
+    char buffer[256];
+    strncpy(buffer, "Test string for analysis", sizeof(buffer) - 1);
+    buffer[sizeof(buffer) - 1] = '\0';
+    
+    // Function call to helper
+    int result = helper_function(42);
+    
+    // Memory allocation pattern
+    int *data = malloc(10 * sizeof(int));
+    if (data != NULL) {
+        for (int i = 0; i < 10; i++) {
+            data[i] = i * i;
+        }
+        free(data);
+    }
+    
+    printf("Result: %d\n", result);
+    return EXIT_SUCCESS;
+}

spectrometer.cabal

@@ -754,6 +754,7 @@ test-suite integration-tests
     Analysis.CocoapodsSpec
     Analysis.ElixirSpec
     Analysis.ErlangSpec
+    Analysis.FicusSpec
     Analysis.FixtureExpectationUtils
     Analysis.FixtureUtils
     Analysis.GoSpec

src/App/Fossa/Analyze.hs

@@ -51,7 +51,6 @@ import App.Fossa.Config.Analyze (
 import App.Fossa.Config.Analyze qualified as Config
 import App.Fossa.Config.Common (DestinationMeta (..), destinationApiOpts, destinationMetadata)
 import App.Fossa.Ficus.Analyze (analyzeWithFicus)
-import App.Fossa.Ficus.Types (FicusResults (..))
 import App.Fossa.FirstPartyScan (runFirstPartyScan)
 import App.Fossa.Lernie.Analyze (analyzeWithLernie)
 import App.Fossa.Lernie.Types (LernieResults (..))
@@ -299,6 +298,7 @@ analyze cfg = Diag.context "fossa-analyze" $ do
       shouldAnalyzePathDependencies = resolvePathDependencies $ Config.experimental cfg
       allowedTactics = Config.allowedTacticTypes cfg
       withoutDefaultFilters = Config.withoutDefaultFilters cfg
+      enableSnippetScan = Config.xSnippetScan cfg
 
   manualSrcUnits <-
     Diag.errorBoundaryIO . diagToDebug $
@@ -339,11 +339,16 @@ analyze cfg = Diag.context "fossa-analyze" $ do
           else pure Nothing
   maybeFicusResults <-
     Diag.errorBoundaryIO . diagToDebug $
-      if filterIsVSIOnly filters
+      if not enableSnippetScan
         then do
-          logInfo "Running in VSI only mode, skipping snippet-scan"
+          logInfo "Skipping ficus snippet scanning (--x-snippet-scan not set)"
           pure Nothing
-        else Diag.context "snippet-scanning" . runStickyLogger SevInfo $ analyzeWithFicus basedir maybeApiOpts grepOptions $ Config.licenseScanPathFilters vendoredDepsOptions --TODO: fix these options
+        else
+          if filterIsVSIOnly filters
+            then do
+              logInfo "Running in VSI only mode, skipping snippet-scan"
+              pure Nothing
+            else Diag.context "snippet-scanning" . runStickyLogger SevInfo $ analyzeWithFicus basedir maybeApiOpts revision $ Config.licenseScanPathFilters vendoredDepsOptions
   let ficusResults = join . resultToMaybe $ maybeFicusResults
   maybeLernieResults <-
     Diag.errorBoundaryIO . diagToDebug $
@@ -452,16 +457,16 @@ analyze cfg = Diag.context "fossa-analyze" $ do
       (False, FilteredAll) -> Diag.warn ErrFilteredAllProjects $> emptyScanUnits
       (True, FilteredAll) -> Diag.warn ErrOnlyKeywordSearchResultsFound $> emptyScanUnits
       (_, CountedScanUnits scanUnits) -> pure scanUnits
-  sendToDestination outputResult iatAssertion destination basedir jsonOutput revision scanUnits reachabilityUnits
+  sendToDestination outputResult iatAssertion destination basedir jsonOutput revision scanUnits reachabilityUnits ficusResults
 
   pure outputResult
   where
-    sendToDestination result iatAssertion destination basedir jsonOutput revision scanUnits reachabilityUnits =
+    sendToDestination result iatAssertion destination basedir jsonOutput revision scanUnits reachabilityUnits ficusResults =
       let doUpload (DestinationMeta (apiOpts, metadata)) =
             Diag.context "upload-results"
               . runFossaApiClient apiOpts
               $ do
-                locator <- uploadSuccessfulAnalysis (BaseDir basedir) metadata jsonOutput revision scanUnits reachabilityUnits
+                locator <- uploadSuccessfulAnalysis (BaseDir basedir) metadata jsonOutput revision scanUnits reachabilityUnits ficusResults
                 doAssertRevisionBinaries iatAssertion locator
        in case destination of
             OutputStdout -> logStdout . decodeUtf8 $ Aeson.encode result

src/App/Fossa/Analyze/Types.hs

@@ -12,7 +12,7 @@ module App.Fossa.Analyze.Types (
 
 import App.Fossa.Analyze.Project (ProjectResult)
 import App.Fossa.Config.Analyze (ExperimentalAnalyzeConfig)
-import App.Fossa.Ficus.Types (FicusResults)
+import App.Fossa.Ficus.Types (FicusSnippetScanResults)
 import App.Fossa.Lernie.Types (LernieResults)
 import App.Fossa.Reachability.Types (SourceUnitReachability (..))
 import App.Types (Mode)
@@ -81,7 +81,7 @@ data AnalysisScanResult = AnalysisScanResult
   { analyzersScanResult :: [DiscoveredProjectScan]
   , vsiScanResult :: Result (Maybe [SourceUnit])
   , binaryDepsScanResult :: Result (Maybe SourceUnit)
-  , ficusResult :: Result (Maybe FicusResults)
+  , ficusResult :: Result (Maybe FicusSnippetScanResults)
   , fossaDepsScanResult :: Result (Maybe SourceUnit)
   , dynamicLinkingResult :: Result (Maybe SourceUnit)
   , lernieResult :: Result (Maybe LernieResults)

src/App/Fossa/Analyze/Upload.hs

@@ -10,6 +10,7 @@ module App.Fossa.Analyze.Upload (
 import App.Docs (vulnReachabilityProductDocsUrl)
 import App.Fossa.API.BuildLink (getFossaBuildUrl)
 import App.Fossa.Config.Analyze (JsonOutput (JsonOutput))
+import App.Fossa.Ficus.Types (FicusSnippetScanResults)
 import App.Fossa.Reachability.Types (SourceUnitReachability)
 import App.Fossa.Reachability.Upload (upload)
 import App.Types (
@@ -107,8 +108,9 @@ uploadSuccessfulAnalysis ::
   ProjectRevision ->
   ScanUnits ->
   [SourceUnitReachability] ->
+  Maybe FicusSnippetScanResults ->
   m Locator
-uploadSuccessfulAnalysis (BaseDir basedir) metadata jsonOutput revision scanUnits reachabilityUnits =
+uploadSuccessfulAnalysis (BaseDir basedir) metadata jsonOutput revision scanUnits reachabilityUnits ficusResults =
   context "Uploading analysis" $ do
     dieOnMonorepoUpload revision
     org <- getOrganization
@@ -126,13 +128,13 @@ uploadSuccessfulAnalysis (BaseDir basedir) metadata jsonOutput revision scanUnit
     logInfo ("Using branch: `" <> pretty branchText <> "`")
 
     uploadResult <- case scanUnits of
-      SourceUnitOnly units -> uploadAnalysis revision metadata units
+      SourceUnitOnly units -> uploadAnalysis revision metadata units ficusResults
       LicenseSourceUnitOnly licenseSourceUnit -> do
         let mergedUnits = mergeSourceAndLicenseUnits [] licenseSourceUnit
-        runStickyLogger SevInfo . uploadAnalysisWithFirstPartyLicensesToS3AndCore revision metadata mergedUnits $ orgFileUpload org
+        runStickyLogger SevInfo . uploadAnalysisWithFirstPartyLicensesToS3AndCore revision metadata mergedUnits ficusResults $ orgFileUpload org
       SourceAndLicenseUnits sourceUnits licenseSourceUnit -> do
         let mergedUnits = mergeSourceAndLicenseUnits sourceUnits licenseSourceUnit
-        runStickyLogger SevInfo . uploadAnalysisWithFirstPartyLicensesToS3AndCore revision metadata mergedUnits $ orgFileUpload org
+        runStickyLogger SevInfo . uploadAnalysisWithFirstPartyLicensesToS3AndCore revision metadata mergedUnits ficusResults $ orgFileUpload org
 
     emitBuildWarnings uploadResult
 
@@ -167,11 +169,12 @@ uploadAnalysisWithFirstPartyLicensesToS3AndCore ::
   ProjectRevision ->
   ProjectMetadata ->
   NE.NonEmpty FullSourceUnit ->
+  Maybe FicusSnippetScanResults ->
   FileUpload ->
   m UploadResponse
-uploadAnalysisWithFirstPartyLicensesToS3AndCore revision metadata mergedUnits uploadKind = do
+uploadAnalysisWithFirstPartyLicensesToS3AndCore revision metadata mergedUnits ficusResults uploadKind = do
   _ <- uploadAnalysisWithFirstPartyLicensesToS3 revision mergedUnits
-  uploadAnalysisWithFirstPartyLicenses revision metadata uploadKind
+  uploadAnalysisWithFirstPartyLicenses revision metadata uploadKind ficusResults
 
 uploadAnalysisWithFirstPartyLicensesToS3 ::
   ( Has Diagnostics sig m