[backend] feat(stix): artifact management (#3511)

openaev-api/src/main/java/io/openaev/migration/V4_75__Add_artifacts_column_to_security_coverage.java

@@ -0,0 +1,86 @@
+package io.openaev.migration;
+
+import java.sql.Statement;
+import org.flywaydb.core.api.migration.BaseJavaMigration;
+import org.flywaydb.core.api.migration.Context;
+import org.springframework.stereotype.Component;
+
+@Component
+public class V4_75__Add_artifacts_column_to_security_coverage extends BaseJavaMigration {
+  @Override
+  public void migrate(Context context) throws Exception {
+    try (Statement stmt = context.getConnection().createStatement()) {
+      stmt.execute(
+          "ALTER TABLE security_coverages ADD COLUMN IF NOT EXISTS security_coverage_artifacts_refs JSONB;");
+
+      stmt.execute(
+          """
+        UPDATE security_coverages
+        SET security_coverage_vulnerabilities_refs = (
+            CASE
+                WHEN security_coverage_vulnerabilities_refs IS NULL\s
+                     OR jsonb_array_length(security_coverage_vulnerabilities_refs) = 0\s
+                THEN security_coverage_vulnerabilities_refs
+
+                ELSE (
+                    SELECT jsonb_agg(
+                        jsonb_set(
+                            elem - 'external_ref',
+                            '{external_refs}',
+                            jsonb_build_array(elem->>'external_ref')
+                        )
+                    )
+                    FROM jsonb_array_elements(security_coverage_vulnerabilities_refs) AS elem
+                )
+            END
+        );
+      """);
+
+      stmt.execute(
+          """
+        UPDATE security_coverages
+        SET security_coverage_attack_pattern_refs = (
+            CASE
+                WHEN security_coverage_attack_pattern_refs IS NULL\s
+                     OR jsonb_array_length(security_coverage_attack_pattern_refs) = 0\s
+                THEN security_coverage_attack_pattern_refs
+
+                ELSE (
+                    SELECT jsonb_agg(
+                        jsonb_set(
+                            elem - 'external_ref',
+                            '{external_refs}',
+                            jsonb_build_array(elem->>'external_ref')
+                        )
+                    )
+                    FROM jsonb_array_elements(security_coverage_attack_pattern_refs) AS elem
+                )
+            END
+        );
+      """);
+
+      stmt.execute(
+          """
+        UPDATE security_coverages
+        SET security_coverage_indicators_refs = (
+            CASE
+                WHEN security_coverage_indicators_refs IS NULL\s
+                     OR jsonb_array_length(security_coverage_indicators_refs) = 0\s
+                THEN security_coverage_indicators_refs
+
+                ELSE (
+                    SELECT jsonb_agg(
+                        jsonb_set(
+                            elem - 'external_ref',
+                            '{external_refs}',
+                            jsonb_build_array(elem->>'external_ref')
+                        )
+                    )
+                    FROM jsonb_array_elements(security_coverage_indicators_refs) AS elem
+                )
+            END
+        );
+      """);
+    }
+  }
+}

openaev-api/src/main/java/io/openaev/opencti/client/OpenCTIClient.java

@@ -8,22 +8,27 @@
 import io.openaev.authorisation.HttpClientFactory;
 import io.openaev.opencti.client.mutations.Mutation;
 import io.openaev.opencti.client.response.Response;
+import io.openaev.opencti.client.response.ResponseFile;
 import io.openaev.opencti.client.response.fields.Error;
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.hc.client5.http.ClientProtocolException;
+import org.apache.hc.client5.http.classic.methods.HttpGet;
 import org.apache.hc.client5.http.classic.methods.HttpPost;
 import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
 import org.apache.hc.core5.http.ClassicHttpRequest;
+import org.apache.hc.core5.http.HttpEntity;
 import org.apache.hc.core5.http.io.entity.EntityUtils;
 import org.apache.hc.core5.http.io.entity.StringEntity;
 import org.apache.http.HttpHeaders;
 import org.springframework.stereotype.Component;
 
 @Component
+@Slf4j
 @RequiredArgsConstructor
 public class OpenCTIClient {
   private final HttpClientFactory httpClientFactory;
@@ -49,6 +54,30 @@ public Response execute(String url, String authToken, String mutationBody, JsonN
     return execute(req);
   }
 
+  public ResponseFile download(String url, String authToken) throws IOException {
+    try (CloseableHttpClient client = httpClientFactory.httpClientCustom()) {
+      HttpGet req = new HttpGet(url);
+      req.addHeader(HttpHeaders.AUTHORIZATION, "Bearer %s".formatted(authToken));
+
+      try (CloseableHttpResponse res = client.execute(req)) {
+        int statusCode = res.getCode();
+        if (statusCode != 200) {
+          log.warn(
+              String.format("Error downloading file from %s with status code %s", url, statusCode));
+          return null;
+        }
+
+        HttpEntity entity = res.getEntity();
+        byte[] content = entity.getContent().readAllBytes();
+
+        ResponseFile responseFile = new ResponseFile();
+        responseFile.setInputStream(new ByteArrayInputStream(content));
+        responseFile.setSize(content.length);
+        return responseFile;
+      }
+    }
+  }
+
   public record ExtractedData(int status, String body) {}
 
   private Response execute(ClassicHttpRequest request) throws IOException {

openaev-api/src/main/java/io/openaev/opencti/client/response/ResponseFile.java

@@ -0,0 +1,14 @@
+package io.openaev.opencti.client.response;
+
+import java.io.InputStream;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+public class ResponseFile {
+
+  private long size;
+
+  private InputStream inputStream;
+}

openaev-api/src/main/java/io/openaev/opencti/config/OpenCTIConfig.java

@@ -43,4 +43,8 @@ public String getApiUrl() {
 
     return String.join("/", urlStripped, GRAPHQL_ENDPOINT_URI);
   }
+
+  public String getFormattedUrl() {
+    return url.endsWith("/") ? url : url + "/";
+  }
 }

openaev-api/src/main/java/io/openaev/opencti/service/OpenCTIService.java

@@ -8,16 +8,25 @@
 import io.openaev.opencti.client.OpenCTIClient;
 import io.openaev.opencti.client.mutations.*;
 import io.openaev.opencti.client.response.Response;
+import io.openaev.opencti.client.response.ResponseFile;
 import io.openaev.opencti.client.response.fields.Error;
 import io.openaev.opencti.config.OpenCTIConfig;
 import io.openaev.opencti.connectors.ConnectorBase;
 import io.openaev.opencti.connectors.impl.SecurityCoverageConnector;
 import io.openaev.opencti.connectors.service.PrivilegeService;
 import io.openaev.opencti.errors.ConnectorError;
+import io.openaev.rest.document.DocumentService;
+import io.openaev.rest.document.form.DocumentCreateInput;
+import io.openaev.rest.tag.TagService;
+import io.openaev.rest.tag.form.TagCreateInput;
 import io.openaev.stix.objects.Bundle;
 import java.io.IOException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 import java.time.Instant;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 import java.util.stream.Collectors;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -32,6 +41,8 @@ public class OpenCTIService {
   private final OpenCTIClient openCTIClient;
   private final ObjectMapper mapper;
   private final PrivilegeService privilegeService;
+  private final TagService tagService;
+  private final DocumentService documentService;
 
   private void applyJwksIfApplicable(ConnectorBase connector, String jwks) {
     if (connector instanceof SecurityCoverageConnector scc) {
@@ -244,4 +255,53 @@ public void createReport(
       execution.addTrace(getNewErrorTrace("Fail to POST", ExecutionTraceAction.COMPLETE));
     }
   }
+
+  /**
+   * Download and save file from OpenCTI
+   *
+   * @param uri of the file
+   * @param name of the file to download
+   * @param mimeType of the file to download
+   * @return the document created from downloaded file
+   */
+  public Document downloadAndSaveFile(String uri, String name, String mimeType) {
+    try {
+      ResponseFile octiResponseFile = downloadFile(uri);
+
+      if (octiResponseFile != null) {
+        Tag openCtiTag = getOpenCTITag();
+        DocumentCreateInput documentCreateInput = new DocumentCreateInput();
+        documentCreateInput.setDescription(name);
+        if (openCtiTag != null) {
+          documentCreateInput.setTagIds(new ArrayList<>(Set.of(openCtiTag.getId())));
+        }
+
+        return documentService.upsert(
+            name,
+            octiResponseFile.getInputStream(),
+            octiResponseFile.getSize(),
+            mimeType,
+            documentCreateInput);
+      }
+    } catch (Exception e) {
+      log.error(
+          String.format(
+              "Error while upserting document from OpenCTI file (uri=%s, name=%s, mimeType=%s)",
+              uri, name, mimeType),
+          e);
+    }
+    return null;
+  }
+
+  private ResponseFile downloadFile(String uri) throws IOException {
+    return openCTIClient.download(
+        classicOpenCTIConfig.getFormattedUrl() + URLEncoder.encode(uri, StandardCharsets.UTF_8),
+        classicOpenCTIConfig.getToken());
+  }
+
+  private Tag getOpenCTITag() {
+    TagCreateInput tagCreateInput = new TagCreateInput();
+    tagCreateInput.setName(Tag.OPENCTI_TAG_NAME);
+    return tagService.upsertTag(tagCreateInput);
+  }
 }

openaev-api/src/main/java/io/openaev/rest/attack_pattern/service/AttackPatternService.java

@@ -1,7 +1,6 @@
 package io.openaev.rest.attack_pattern.service;
 
 import static io.openaev.helper.StreamHelper.fromIterable;
-import static io.openaev.utils.SecurityCoverageUtils.getExternalIds;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -12,6 +11,7 @@
 import io.openaev.ee.EnterpriseEditionService;
 import io.openaev.rest.attack_pattern.form.AnalysisResultFromTTPExtractionAIWebserviceOutput;
 import io.openaev.rest.exception.ElementNotFoundException;
+import io.openaev.utils.SecurityCoverageUtils;
 import jakarta.annotation.Resource;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -44,6 +44,7 @@ public class AttackPatternService {
   private final AttackPatternRepository attackPatternRepository;
   private final EnterpriseEditionService enterpriseEditionService;
   private final RestTemplate restTemplate;
+  private final SecurityCoverageUtils securityCoverageUtils;
 
   /**
    * Call the TTP Extraction AI Webservice to analyze files and text input.
@@ -245,7 +246,7 @@ public List<String> searchAttackPatternWithTTPAIWebservice(
    */
   public Map<String, AttackPattern> fetchInternalAttackPatternIds(
       Set<StixRefToExternalRef> stixRefs) {
-    return getAttackPatternsByExternalIds(getExternalIds(stixRefs)).stream()
+    return getAttackPatternsByExternalIds(securityCoverageUtils.getExternalIds(stixRefs)).stream()
         .collect(Collectors.toMap(attack -> attack.getId(), Function.identity()));
   }