diff --git a/.github/skills/review-code/SKILL.md b/.github/skills/review-code/SKILL.md index f170200a4fd..675e5167a18 100644 --- a/.github/skills/review-code/SKILL.md +++ b/.github/skills/review-code/SKILL.md @@ -98,7 +98,17 @@ grep -rn "TenantBase\|tenant_id\|TenantContext" --include="*.java" $(git diff -- git diff --name-only HEAD~1 | grep -E "\.tsx$|\.ts$" | head -10 ``` -## Step 8 — Compile review +## Step 8 — Check common anti-patterns (learned from reviews) + +Apply these checks based on past review feedback: + +- **Root cause vs workaround**: If a bug is backend-originated, don't add frontend workarounds (onError handlers, fallback states). Fix the root cause in the correct layer. +- **String/value duplication**: When introducing new methods that share data with existing methods (e.g., logo filenames, config keys), extract the shared value to a private method or constant — never compute the same formatted string in multiple places. +- **Non-critical operations**: Startup operations that interact with external services (MinIO, S3, etc.) for non-critical assets (logos, thumbnails) should be best-effort: wrap in try-catch with `log.warn` so failures don't block application startup. +- **Test proportionality**: Don't require tests for small mechanical changes (1-line additions, parameter threading). Tests should be proportionate to the risk and complexity of the change. +- **Pre-existing issues**: Don't fix unrelated pre-existing issues (e.g., alt text, parameter naming) in a bug fix PR. Track them as follow-up. + +## Step 9 — Compile review Generate the Code Review Summary following the output format defined in `code-reviewer.agent.md`. diff --git a/openaev-api/src/main/java/io/openaev/integration/IntegrationFactory.java b/openaev-api/src/main/java/io/openaev/integration/IntegrationFactory.java index 5f4c88253e3..340f166aeda 100644 --- a/openaev-api/src/main/java/io/openaev/integration/IntegrationFactory.java +++ b/openaev-api/src/main/java/io/openaev/integration/IntegrationFactory.java @@ -25,11 +25,29 @@ public abstract class IntegrationFactory { protected abstract String getClassName(); + /** + * Ensures the catalog logo exists in MinIO. Called on every startup (best-effort), even when the + * catalog entry already exists in the database. Override in subclasses that upload a logo during + * {@link #insertCatalogEntry()}. + * + *

Default implementation is a no-op for factories that do not manage a catalog logo (e.g. + * built-in executors). + */ + protected void ensureCatalogLogo() throws Exception { + // no-op by default + } + @Transactional(rollbackFor = Exception.class) public void initialise() throws Exception { String className = this.getClassName(); if (catalogConnectorService.findByFactoryClassName(className).isEmpty()) { insertCatalogEntry(); + } else { + try { + ensureCatalogLogo(); + } catch (Exception e) { + log.warn("Failed to ensure catalog logo for {}: {}", className, e.getMessage()); + } } runMigrations(); diff --git a/openaev-api/src/main/java/io/openaev/integration/impl/executors/caldera/CalderaExecutorIntegrationFactory.java b/openaev-api/src/main/java/io/openaev/integration/impl/executors/caldera/CalderaExecutorIntegrationFactory.java index fe5418e7eef..224c20b3b93 100644 --- a/openaev-api/src/main/java/io/openaev/integration/impl/executors/caldera/CalderaExecutorIntegrationFactory.java +++ b/openaev-api/src/main/java/io/openaev/integration/impl/executors/caldera/CalderaExecutorIntegrationFactory.java @@ -83,13 +83,22 @@ protected void runMigrations() throws Exception { calderaExecutorConfigurationMigration.migrate(); } + private String getLogoFilename() { + return "%s-logo.png".formatted(CALDERA_EXECUTOR_TYPE); + } + @Override - protected void insertCatalogEntry() throws Exception { - String logoFilename = "%s-logo.png".formatted(CALDERA_EXECUTOR_TYPE); + protected void ensureCatalogLogo() throws Exception { fileService.uploadCatalogLogo( FileService.CONNECTORS_LOGO_PATH, - logoFilename, + getLogoFilename(), getClass().getResourceAsStream("/img/icon-caldera.png")); + } + + @Override + protected void insertCatalogEntry() throws Exception { + ensureCatalogLogo(); + String logoFilename = getLogoFilename(); CatalogConnector connector = new CatalogConnector(); connector.setTitle("Caldera Executor"); connector.setSlug(CALDERA_EXECUTOR_TYPE); diff --git a/openaev-api/src/main/java/io/openaev/integration/impl/executors/crowdstrike/CrowdStrikeExecutorIntegrationFactory.java b/openaev-api/src/main/java/io/openaev/integration/impl/executors/crowdstrike/CrowdStrikeExecutorIntegrationFactory.java index 82a2d6334f4..7f0f6f82429 100644 --- a/openaev-api/src/main/java/io/openaev/integration/impl/executors/crowdstrike/CrowdStrikeExecutorIntegrationFactory.java +++ b/openaev-api/src/main/java/io/openaev/integration/impl/executors/crowdstrike/CrowdStrikeExecutorIntegrationFactory.java @@ -86,13 +86,22 @@ protected void runMigrations() throws Exception { crowdStrikeExecutorConfigurationMigration.migrate(); } + private String getLogoFilename() { + return "%s-logo.png".formatted(CROWDSTRIKE_EXECUTOR_TYPE); + } + @Override - protected void insertCatalogEntry() throws Exception { - String logoFilename = "%s-logo.png".formatted(CROWDSTRIKE_EXECUTOR_TYPE); + protected void ensureCatalogLogo() throws Exception { fileService.uploadCatalogLogo( FileService.CONNECTORS_LOGO_PATH, - logoFilename, + getLogoFilename(), getClass().getResourceAsStream("/img/icon-crowdstrike.png")); + } + + @Override + protected void insertCatalogEntry() throws Exception { + ensureCatalogLogo(); + String logoFilename = getLogoFilename(); CatalogConnector connector = new CatalogConnector(); connector.setTitle("CrowdStrike Executor"); connector.setSlug(CROWDSTRIKE_EXECUTOR_TYPE); diff --git a/openaev-api/src/main/java/io/openaev/integration/impl/executors/paloaltocortex/PaloAltoCortexExecutorIntegrationFactory.java b/openaev-api/src/main/java/io/openaev/integration/impl/executors/paloaltocortex/PaloAltoCortexExecutorIntegrationFactory.java index dd6aebad4ed..f7438a7f451 100644 --- a/openaev-api/src/main/java/io/openaev/integration/impl/executors/paloaltocortex/PaloAltoCortexExecutorIntegrationFactory.java +++ b/openaev-api/src/main/java/io/openaev/integration/impl/executors/paloaltocortex/PaloAltoCortexExecutorIntegrationFactory.java @@ -80,13 +80,22 @@ protected void runMigrations() throws Exception { // No } + private String getLogoFilename() { + return "%s-logo.png".formatted(PALOALTOCORTEX_EXECUTOR_TYPE); + } + @Override - protected void insertCatalogEntry() throws Exception { - String logoFilename = "%s-logo.png".formatted(PALOALTOCORTEX_EXECUTOR_TYPE); + protected void ensureCatalogLogo() throws Exception { fileService.uploadCatalogLogo( FileService.CONNECTORS_LOGO_PATH, - logoFilename, + getLogoFilename(), getClass().getResourceAsStream("/img/icon-paloaltocortex.png")); + } + + @Override + protected void insertCatalogEntry() throws Exception { + ensureCatalogLogo(); + String logoFilename = getLogoFilename(); CatalogConnector connector = new CatalogConnector(); connector.setTitle("Palo Alto Cortex Executor"); connector.setSlug(PALOALTOCORTEX_EXECUTOR_TYPE); diff --git a/openaev-api/src/main/java/io/openaev/integration/impl/executors/sentinelone/SentinelOneExecutorIntegrationFactory.java b/openaev-api/src/main/java/io/openaev/integration/impl/executors/sentinelone/SentinelOneExecutorIntegrationFactory.java index 81a09efcd60..19b3f3690e2 100644 --- a/openaev-api/src/main/java/io/openaev/integration/impl/executors/sentinelone/SentinelOneExecutorIntegrationFactory.java +++ b/openaev-api/src/main/java/io/openaev/integration/impl/executors/sentinelone/SentinelOneExecutorIntegrationFactory.java @@ -84,13 +84,22 @@ protected void runMigrations() throws Exception { sentinelOneExecutorConfigurationMigration.migrate(); } + private String getLogoFilename() { + return "%s-logo.png".formatted(SENTINELONE_EXECUTOR_TYPE); + } + @Override - protected void insertCatalogEntry() throws Exception { - String logoFilename = "%s-logo.png".formatted(SENTINELONE_EXECUTOR_TYPE); + protected void ensureCatalogLogo() throws Exception { fileService.uploadCatalogLogo( FileService.CONNECTORS_LOGO_PATH, - logoFilename, + getLogoFilename(), getClass().getResourceAsStream("/img/icon-sentinelone.png")); + } + + @Override + protected void insertCatalogEntry() throws Exception { + ensureCatalogLogo(); + String logoFilename = getLogoFilename(); CatalogConnector connector = new CatalogConnector(); connector.setTitle("SentinelOne Executor"); connector.setSlug(SENTINELONE_EXECUTOR_TYPE); diff --git a/openaev-api/src/main/java/io/openaev/integration/impl/executors/tanium/TaniumExecutorIntegrationFactory.java b/openaev-api/src/main/java/io/openaev/integration/impl/executors/tanium/TaniumExecutorIntegrationFactory.java index 4028cbe58d1..a148ac4bbb8 100644 --- a/openaev-api/src/main/java/io/openaev/integration/impl/executors/tanium/TaniumExecutorIntegrationFactory.java +++ b/openaev-api/src/main/java/io/openaev/integration/impl/executors/tanium/TaniumExecutorIntegrationFactory.java @@ -87,13 +87,22 @@ protected void runMigrations() throws Exception { taniumExecutorConfigurationMigration.migrate(); } + private String getLogoFilename() { + return "%s-logo.png".formatted(TANIUM_EXECUTOR_TYPE); + } + @Override - protected void insertCatalogEntry() throws Exception { - String logoFilename = "%s-logo.png".formatted(TANIUM_EXECUTOR_TYPE); + protected void ensureCatalogLogo() throws Exception { fileService.uploadCatalogLogo( FileService.CONNECTORS_LOGO_PATH, - logoFilename, + getLogoFilename(), getClass().getResourceAsStream("/img/icon-tanium.png")); + } + + @Override + protected void insertCatalogEntry() throws Exception { + ensureCatalogLogo(); + String logoFilename = getLogoFilename(); CatalogConnector connector = new CatalogConnector(); connector.setTitle("Tanium Executor"); connector.setSlug(TANIUM_EXECUTOR_TYPE); diff --git a/openaev-api/src/main/java/io/openaev/integration/impl/injectors/opencti/OpenCTIInjectorIntegrationFactory.java b/openaev-api/src/main/java/io/openaev/integration/impl/injectors/opencti/OpenCTIInjectorIntegrationFactory.java index 2b980aed503..be92cb3a37b 100644 --- a/openaev-api/src/main/java/io/openaev/integration/impl/injectors/opencti/OpenCTIInjectorIntegrationFactory.java +++ b/openaev-api/src/main/java/io/openaev/integration/impl/injectors/opencti/OpenCTIInjectorIntegrationFactory.java @@ -79,13 +79,22 @@ protected void runMigrations() throws Exception { openctiInjectorConfigurationMigration.migrate(); } + private String getLogoFilename() { + return "%s-logo.png".formatted(openCTIContract.TYPE); + } + @Override - protected void insertCatalogEntry() throws Exception { - String logoFilename = "%s-logo.png".formatted(openCTIContract.TYPE); + protected void ensureCatalogLogo() throws Exception { fileService.uploadCatalogLogo( FileService.CONNECTORS_LOGO_PATH, - logoFilename, + getLogoFilename(), getClass().getResourceAsStream("/img/icon-opencti.png")); + } + + @Override + protected void insertCatalogEntry() throws Exception { + ensureCatalogLogo(); + String logoFilename = getLogoFilename(); CatalogConnector connector = new CatalogConnector(); connector.setTitle(OPENCTI_INJECTOR_NAME); connector.setSlug(openCTIContract.TYPE); diff --git a/openaev-api/src/main/java/io/openaev/integration/impl/injectors/ovh/OvhInjectorIntegrationFactory.java b/openaev-api/src/main/java/io/openaev/integration/impl/injectors/ovh/OvhInjectorIntegrationFactory.java index c264570ce5e..d63b148e461 100644 --- a/openaev-api/src/main/java/io/openaev/integration/impl/injectors/ovh/OvhInjectorIntegrationFactory.java +++ b/openaev-api/src/main/java/io/openaev/integration/impl/injectors/ovh/OvhInjectorIntegrationFactory.java @@ -75,14 +75,23 @@ protected void runMigrations() throws Exception { ovhInjectorConfigurationMigration.migrate(); } + private String getLogoFilename() { + return "%s-logo.png".formatted(ovhSmsContract.getType()); + } + @Override - protected void insertCatalogEntry() throws Exception { - CatalogConnector connector = new CatalogConnector(); - String logoFilename = "%s-logo.png".formatted(ovhSmsContract.getType()); + protected void ensureCatalogLogo() throws Exception { fileService.uploadCatalogLogo( FileService.CONNECTORS_LOGO_PATH, - logoFilename, + getLogoFilename(), getClass().getResourceAsStream("/img/icon-ovh-sms.png")); + } + + @Override + protected void insertCatalogEntry() throws Exception { + ensureCatalogLogo(); + CatalogConnector connector = new CatalogConnector(); + String logoFilename = getLogoFilename(); connector.setTitle("OVHCloud SMS Platform"); connector.setSlug(ovhSmsContract.getType()); connector.setLogoUrl(logoFilename); diff --git a/openaev-api/src/main/java/io/openaev/migration/V5_35__Add_indexing_last_id.java b/openaev-api/src/main/java/io/openaev/migration/V5_35__Add_indexing_last_id.java new file mode 100644 index 00000000000..d20e908c2e0 --- /dev/null +++ b/openaev-api/src/main/java/io/openaev/migration/V5_35__Add_indexing_last_id.java @@ -0,0 +1,18 @@ +package io.openaev.migration; + +import java.sql.Statement; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.springframework.stereotype.Component; + +@Component +public class V5_35__Add_indexing_last_id extends BaseJavaMigration { + + @Override + public void migrate(Context context) throws Exception { + try (Statement statement = context.getConnection().createStatement()) { + statement.execute( + "ALTER TABLE indexing_status ADD COLUMN IF NOT EXISTS indexing_last_id TEXT NULL"); + } + } +} diff --git a/openaev-api/src/test/java/io/openaev/integration/local_fixtures/factory_throws/TestIntegrationFactoryInitThrows.java b/openaev-api/src/test/java/io/openaev/integration/local_fixtures/factory_throws/TestIntegrationFactoryInitThrows.java index f89c91046c9..ed975b7cfaa 100644 --- a/openaev-api/src/test/java/io/openaev/integration/local_fixtures/factory_throws/TestIntegrationFactoryInitThrows.java +++ b/openaev-api/src/test/java/io/openaev/integration/local_fixtures/factory_throws/TestIntegrationFactoryInitThrows.java @@ -51,13 +51,22 @@ protected void runMigrations() throws Exception { throw new RuntimeException("%s: deliberate throw!".formatted(this.getClassName())); } + private String getLogoFilename() { + return "%s-logo.png".formatted(getClassName()); + } + @Override - protected void insertCatalogEntry() throws Exception { - String logoFilename = "%s-logo.png".formatted(getClassName()); + protected void ensureCatalogLogo() throws Exception { fileService.uploadCatalogLogo( FileService.CONNECTORS_LOGO_PATH, - logoFilename, + getLogoFilename(), getClass().getResourceAsStream("/img/icon-default.png")); + } + + @Override + protected void insertCatalogEntry() throws Exception { + ensureCatalogLogo(); + String logoFilename = getLogoFilename(); CatalogConnector connector = new CatalogConnector(); connector.setTitle("Test Integration Init Throws"); connector.setSlug(getClassName()); diff --git a/openaev-api/src/test/java/io/openaev/integration/local_fixtures/regular/TestIntegrationFactory.java b/openaev-api/src/test/java/io/openaev/integration/local_fixtures/regular/TestIntegrationFactory.java index abbd294ca13..dff9b49141b 100644 --- a/openaev-api/src/test/java/io/openaev/integration/local_fixtures/regular/TestIntegrationFactory.java +++ b/openaev-api/src/test/java/io/openaev/integration/local_fixtures/regular/TestIntegrationFactory.java @@ -50,13 +50,22 @@ protected void runMigrations() throws Exception { testIntegrationConfigurationMigration.migrate(); } + private String getLogoFilename() { + return "%s-logo.png".formatted(getClassName()); + } + @Override - protected void insertCatalogEntry() throws Exception { - String logoFilename = "%s-logo.png".formatted(getClassName()); + protected void ensureCatalogLogo() throws Exception { fileService.uploadCatalogLogo( FileService.CONNECTORS_LOGO_PATH, - logoFilename, + getLogoFilename(), getClass().getResourceAsStream("/img/icon-default.png")); + } + + @Override + protected void insertCatalogEntry() throws Exception { + ensureCatalogLogo(); + String logoFilename = getLogoFilename(); CatalogConnector connector = new CatalogConnector(); connector.setTitle("Test Integration"); connector.setSlug(getClassName()); diff --git a/openaev-model/src/main/java/io/openaev/database/model/IndexingStatus.java b/openaev-model/src/main/java/io/openaev/database/model/IndexingStatus.java index 2c3cff7bcc8..4f185a25a95 100644 --- a/openaev-model/src/main/java/io/openaev/database/model/IndexingStatus.java +++ b/openaev-model/src/main/java/io/openaev/database/model/IndexingStatus.java @@ -23,4 +23,10 @@ public class IndexingStatus { @Column(name = "indexing_status_indexing_date") @JsonProperty("indexing_status_indexing_date") private Instant lastIndexing; + + /** Last processed entity ID within the current indexing timestamp window (compound cursor). */ + @Getter + @Column(name = "indexing_last_id") + @JsonProperty("indexing_last_id") + private String lastId; } diff --git a/openaev-model/src/main/java/io/openaev/database/repository/InjectExpectationRepository.java b/openaev-model/src/main/java/io/openaev/database/repository/InjectExpectationRepository.java index 05c7d4a0df7..d96bf4fdf87 100644 --- a/openaev-model/src/main/java/io/openaev/database/repository/InjectExpectationRepository.java +++ b/openaev-model/src/main/java/io/openaev/database/repository/InjectExpectationRepository.java @@ -381,6 +381,85 @@ LEFT JOIN LATERAL jsonb_array_elements(ie.inject_expectation_results::jsonb) AS List findForIndexing( @Param("from") Instant from, @Param("limit") int limit); + @Query( + value = + """ + WITH changed_expectations AS ( + SELECT ie.inject_expectation_id FROM injects_expectations ie + WHERE ie.inject_expectation_updated_at > :from + OR (ie.inject_expectation_updated_at = :from AND ie.inject_expectation_id > :lastId) + UNION + SELECT ie.inject_expectation_id FROM injects_expectations ie + JOIN injects i ON i.inject_id = ie.inject_id + WHERE i.inject_updated_at > :from + UNION + SELECT ie.inject_expectation_id FROM injects_expectations ie + JOIN injects i ON i.inject_id = ie.inject_id + JOIN injectors_contracts ic ON ic.injector_contract_id = i.inject_injector_contract + WHERE ic.injector_contract_updated_at > :from + ), + inject_expectation_data AS ( + SELECT + ie.inject_expectation_id, + ie.inject_expectation_name, + ie.inject_expectation_description, + ie.inject_expectation_type, + ie.inject_expectation_results, + ie.inject_expectation_score, + ie.inject_expectation_expected_score, + ie.inject_expiration_time, + ie.inject_expectation_group, + ie.inject_expectation_created_at, + GREATEST(ie.inject_expectation_updated_at, max(i.inject_updated_at), max(ic.injector_contract_updated_at)) as inject_expectation_updated_at, + ie.exercise_id, + ie.inject_id, + ie.user_id, + ie.team_id, + ie.agent_id, + ie.asset_id, + ie.asset_group_id, + i.tenant_id, + i.inject_title as inject_title, + MAX(ins.tracking_sent_date) AS tracking_sent_date, + array_agg(DISTINCT ap.attack_pattern_id) FILTER ( WHERE ap.attack_pattern_id IS NOT NULL ) AS attack_pattern_ids, + array_agg(DISTINCT ic_d.domain_id) FILTER (WHERE ic_d.domain_id IS NOT NULL ) AS domain_ids, + MAX(se.scenario_id) AS scenario_id, + array_agg(DISTINCT c.collector_security_platform) FILTER ( WHERE c.collector_security_platform IS NOT NULL ) || + array_agg(DISTINCT a.asset_id) FILTER ( WHERE a.asset_id IS NOT NULL ) AS security_platform_ids + FROM injects_expectations ie + JOIN changed_expectations ce ON ie.inject_expectation_id = ce.inject_expectation_id + LEFT JOIN exercises ex ON ex.exercise_id = ie.exercise_id + LEFT JOIN injects i ON i.inject_id = ie.inject_id + LEFT JOIN injects_statuses ins ON ins.status_inject = i.inject_id + LEFT JOIN injectors_contracts ic ON ic.injector_contract_id = i.inject_injector_contract + LEFT JOIN injectors_contracts_attack_patterns ic_ap ON ic_ap.injector_contract_id = ic.injector_contract_id + LEFT JOIN attack_patterns ap ON ap.attack_pattern_id = ic_ap.attack_pattern_id + LEFT JOIN injectors_contracts_domains ic_d ON ic_d.injector_contract_id = ic.injector_contract_id + LEFT JOIN users u ON u.user_id = ie.user_id + LEFT JOIN teams t ON t.team_id = ie.team_id + LEFT JOIN assets asset ON asset.asset_id = ie.asset_id + LEFT JOIN asset_groups ag ON ag.asset_group_id = ie.asset_group_id + LEFT JOIN scenarios_exercises se ON se.exercise_id = ie.exercise_id + LEFT JOIN LATERAL jsonb_array_elements(ie.inject_expectation_results::jsonb) AS r(elem) ON true + LEFT JOIN collectors c ON r.elem->>'sourceId' = c.collector_id::text + LEFT JOIN assets a ON r.elem->>'sourceId' = a.asset_id::text + GROUP BY + ie.inject_expectation_id, + ic.injector_contract_id, + i.inject_title, + i.tenant_id + ) + SELECT * FROM inject_expectation_data ied + WHERE ied.agent_id IS NULL + ORDER BY ied.inject_expectation_updated_at ASC, ied.inject_expectation_id ASC + LIMIT :limit + """, + nativeQuery = true) + List findForIndexingAfter( + @Param("from") Instant from, + @Param("lastId") String lastId, + @Param("limit") int limit); + /** * Retrieves a set of distinct inject IDs associated with the specified inject expectation IDs. * diff --git a/openaev-model/src/main/java/io/openaev/engine/Handler.java b/openaev-model/src/main/java/io/openaev/engine/Handler.java index 161b79eb0bb..413e6a4e618 100644 --- a/openaev-model/src/main/java/io/openaev/engine/Handler.java +++ b/openaev-model/src/main/java/io/openaev/engine/Handler.java @@ -21,4 +21,25 @@ public interface Handler { * @return list data to index */ List fetch(Instant from, int limit); + + /** + * Variant of {@link #fetch(Instant, int)} using a compound keyset cursor (timestamp + last entity + * ID) to avoid missing items that share the same {@code updated_at} timestamp when the previous + * batch was full. + * + *

The default implementation ignores {@code lastId} and falls back to the basic cursor, + * providing backward-compatible behaviour for handlers that do not need compound pagination. + * Override in handlers where the same-millisecond duplicate issue is observable (e.g. + * InjectExpectationHandler). + * + * @param from lower-bound timestamp (exclusive unless lastId is provided) + * @param lastId ID of the last successfully indexed entity at {@code from}; when non-null the + * query returns items at {@code from} with ID strictly greater than this value, plus all + * items strictly after {@code from} + * @param limit maximum number of records to fetch per batch + * @return list data to index + */ + default List fetch(Instant from, String lastId, int limit) { + return fetch(from, limit); + } } diff --git a/openaev-model/src/main/java/io/openaev/engine/model/injectexpectation/InjectExpectationHandler.java b/openaev-model/src/main/java/io/openaev/engine/model/injectexpectation/InjectExpectationHandler.java index 27b412c59e5..b9c150003fc 100644 --- a/openaev-model/src/main/java/io/openaev/engine/model/injectexpectation/InjectExpectationHandler.java +++ b/openaev-model/src/main/java/io/openaev/engine/model/injectexpectation/InjectExpectationHandler.java @@ -16,6 +16,124 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; +@Service +@RequiredArgsConstructor +public class InjectExpectationHandler implements Handler { + + private final InjectExpectationRepository injectExpectationRepository; + + @Override + public List fetch(Instant from, int limit) { + Instant queryFrom = from != null ? from : Instant.ofEpochMilli(0); + return this.injectExpectationRepository.findForIndexing(queryFrom, limit).stream() + .map(this::map) + .toList(); + } + + @Override + public List fetch(Instant from, String lastId, int limit) { + Instant queryFrom = from != null ? from : Instant.ofEpochMilli(0); + return this.injectExpectationRepository.findForIndexingAfter(queryFrom, lastId, limit).stream() + .map(this::map) + .toList(); + } + + private EsInjectExpectation map(RawInjectExpectationIndexing injectExpectation) { + EsInjectExpectation esInjectExpectation = new EsInjectExpectation(); + // Base + esInjectExpectation.setBase_id(injectExpectation.getInject_expectation_id()); + esInjectExpectation.setBase_representative(injectExpectation.getInject_expectation_name()); + esInjectExpectation.setBase_created_at(injectExpectation.getInject_expectation_created_at()); + esInjectExpectation.setBase_updated_at(injectExpectation.getInject_expectation_updated_at()); + esInjectExpectation.setBase_restrictions( + buildRestrictions(injectExpectation.getExercise_id(), injectExpectation.getInject_id())); + esInjectExpectation.setBase_tenant_side(injectExpectation.getTenant_id()); + // Specific + esInjectExpectation.setInject_expectation_name(injectExpectation.getInject_expectation_name()); + esInjectExpectation.setInject_expectation_description( + injectExpectation.getInject_expectation_description()); + esInjectExpectation.setInject_expectation_type(injectExpectation.getInject_expectation_type()); + esInjectExpectation.setInject_expectation_results( + injectExpectation.getInject_expectation_results()); + esInjectExpectation.setInject_title(injectExpectation.getInject_title()); + esInjectExpectation.setExecution_date(injectExpectation.getTracking_sent_date()); + esInjectExpectation.setInject_expectation_score(injectExpectation.getInject_expectation_score()); + esInjectExpectation.setInject_expectation_expected_score( + injectExpectation.getInject_expectation_expected_score()); + esInjectExpectation.setInject_expectation_expiration_time( + injectExpectation.getInject_expiration_time()); + esInjectExpectation.setInject_expectation_group(injectExpectation.getInject_expectation_group()); + // Dependencies (see base_dependencies in EsBase) + List dependencies = new ArrayList<>(); + if (hasText(injectExpectation.getExercise_id())) { + dependencies.add(injectExpectation.getExercise_id()); + esInjectExpectation.setBase_simulation_side(injectExpectation.getExercise_id()); + } else { + esInjectExpectation.setBase_simulation_side(null); + } + if (hasText(injectExpectation.getScenario_id())) { + dependencies.add(injectExpectation.getScenario_id()); + esInjectExpectation.setBase_scenario_side(injectExpectation.getScenario_id()); + } else { + esInjectExpectation.setBase_scenario_side(null); + } + if (hasText(injectExpectation.getInject_id())) { + dependencies.add(injectExpectation.getInject_id()); + esInjectExpectation.setBase_inject_side(injectExpectation.getInject_id()); + } else { + esInjectExpectation.setBase_inject_side(null); + } + if (hasText(injectExpectation.getUser_id())) { + dependencies.add(injectExpectation.getUser_id()); + esInjectExpectation.setBase_user_side(injectExpectation.getUser_id()); + } else { + esInjectExpectation.setBase_user_side(null); + } + if (hasText(injectExpectation.getTeam_id())) { + dependencies.add(injectExpectation.getTeam_id()); + esInjectExpectation.setBase_team_side(injectExpectation.getTeam_id()); + } else { + esInjectExpectation.setBase_team_side(null); + } + if (hasText(injectExpectation.getAsset_id())) { + dependencies.add(injectExpectation.getAsset_id()); + esInjectExpectation.setBase_asset_side(injectExpectation.getAsset_id()); + } else { + esInjectExpectation.setBase_asset_side(null); + } + if (hasText(injectExpectation.getAsset_group_id())) { + dependencies.add(injectExpectation.getAsset_group_id()); + esInjectExpectation.setBase_asset_group_side(injectExpectation.getAsset_group_id()); + } else { + esInjectExpectation.setBase_asset_group_side(null); + } + if (!isEmpty(injectExpectation.getAttack_pattern_ids())) { + esInjectExpectation.setBase_attack_patterns_side(injectExpectation.getAttack_pattern_ids()); + } else { + esInjectExpectation.setBase_attack_patterns_side(Set.of()); + } + if (!isEmpty(injectExpectation.getDomain_ids())) { + esInjectExpectation.setBase_security_domains_side(injectExpectation.getDomain_ids()); + } else { + esInjectExpectation.setBase_security_domains_side(Set.of()); + } + if (!isEmpty(injectExpectation.getSecurity_platform_ids())) { + esInjectExpectation.setBase_security_platforms_side( + injectExpectation.getSecurity_platform_ids()); + } else { + esInjectExpectation.setBase_security_platforms_side(Set.of()); + } + esInjectExpectation.setInject_expectation_status( + valueOf( + computeStatus( + injectExpectation.getInject_expectation_score(), + injectExpectation.getInject_expectation_expected_score()))); + esInjectExpectation.setBase_dependencies(dependencies); + return esInjectExpectation; + } +} + + @Service @RequiredArgsConstructor public class InjectExpectationHandler implements Handler { @@ -135,4 +253,115 @@ public List fetch(Instant from, int limit) { }) .toList(); } + + @Override + public List fetch(Instant from, String lastId, int limit) { + Instant queryFrom = from != null ? from : Instant.ofEpochMilli(0); + List forIndexing = + this.injectExpectationRepository.findForIndexingAfter(queryFrom, lastId, limit); + return forIndexing.stream() + .map( + injectExpectation -> { + EsInjectExpectation esInjectExpectation = new EsInjectExpectation(); + esInjectExpectation.setBase_id(injectExpectation.getInject_expectation_id()); + esInjectExpectation.setBase_representative( + injectExpectation.getInject_expectation_name()); + esInjectExpectation.setBase_created_at( + injectExpectation.getInject_expectation_created_at()); + esInjectExpectation.setBase_updated_at( + injectExpectation.getInject_expectation_updated_at()); + esInjectExpectation.setBase_restrictions( + buildRestrictions( + injectExpectation.getExercise_id(), injectExpectation.getInject_id())); + esInjectExpectation.setBase_tenant_side(injectExpectation.getTenant_id()); + esInjectExpectation.setInject_expectation_name( + injectExpectation.getInject_expectation_name()); + esInjectExpectation.setInject_expectation_description( + injectExpectation.getInject_expectation_description()); + esInjectExpectation.setInject_expectation_type( + injectExpectation.getInject_expectation_type()); + esInjectExpectation.setInject_expectation_results( + injectExpectation.getInject_expectation_results()); + esInjectExpectation.setInject_title(injectExpectation.getInject_title()); + esInjectExpectation.setExecution_date(injectExpectation.getTracking_sent_date()); + esInjectExpectation.setInject_expectation_score( + injectExpectation.getInject_expectation_score()); + esInjectExpectation.setInject_expectation_expected_score( + injectExpectation.getInject_expectation_expected_score()); + esInjectExpectation.setInject_expectation_expiration_time( + injectExpectation.getInject_expiration_time()); + esInjectExpectation.setInject_expectation_group( + injectExpectation.getInject_expectation_group()); + List dependencies = new ArrayList<>(); + if (hasText(injectExpectation.getExercise_id())) { + dependencies.add(injectExpectation.getExercise_id()); + esInjectExpectation.setBase_simulation_side(injectExpectation.getExercise_id()); + } else { + esInjectExpectation.setBase_simulation_side(null); + } + if (hasText(injectExpectation.getScenario_id())) { + dependencies.add(injectExpectation.getScenario_id()); + esInjectExpectation.setBase_scenario_side(injectExpectation.getScenario_id()); + } else { + esInjectExpectation.setBase_scenario_side(null); + } + if (hasText(injectExpectation.getInject_id())) { + dependencies.add(injectExpectation.getInject_id()); + esInjectExpectation.setBase_inject_side(injectExpectation.getInject_id()); + } else { + esInjectExpectation.setBase_inject_side(null); + } + if (hasText(injectExpectation.getUser_id())) { + dependencies.add(injectExpectation.getUser_id()); + esInjectExpectation.setBase_user_side(injectExpectation.getUser_id()); + } else { + esInjectExpectation.setBase_user_side(null); + } + if (hasText(injectExpectation.getTeam_id())) { + dependencies.add(injectExpectation.getTeam_id()); + esInjectExpectation.setBase_team_side(injectExpectation.getTeam_id()); + } else { + esInjectExpectation.setBase_team_side(null); + } + if (hasText(injectExpectation.getAsset_id())) { + dependencies.add(injectExpectation.getAsset_id()); + esInjectExpectation.setBase_asset_side(injectExpectation.getAsset_id()); + } else { + esInjectExpectation.setBase_asset_side(null); + } + if (hasText(injectExpectation.getAsset_group_id())) { + dependencies.add(injectExpectation.getAsset_group_id()); + esInjectExpectation.setBase_asset_group_side( + injectExpectation.getAsset_group_id()); + } else { + esInjectExpectation.setBase_asset_group_side(null); + } + if (!isEmpty(injectExpectation.getAttack_pattern_ids())) { + esInjectExpectation.setBase_attack_patterns_side( + injectExpectation.getAttack_pattern_ids()); + } else { + esInjectExpectation.setBase_attack_patterns_side(Set.of()); + } + if (!isEmpty(injectExpectation.getDomain_ids())) { + esInjectExpectation.setBase_security_domains_side( + injectExpectation.getDomain_ids()); + } else { + esInjectExpectation.setBase_security_domains_side(Set.of()); + } + if (!isEmpty(injectExpectation.getSecurity_platform_ids())) { + esInjectExpectation.setBase_security_platforms_side( + injectExpectation.getSecurity_platform_ids()); + } else { + esInjectExpectation.setBase_security_platforms_side(Set.of()); + } + esInjectExpectation.setInject_expectation_status( + valueOf( + computeStatus( + injectExpectation.getInject_expectation_score(), + injectExpectation.getInject_expectation_expected_score()))); + esInjectExpectation.setBase_dependencies(dependencies); + return esInjectExpectation; + }) + .toList(); + } } diff --git a/openaev-model/src/main/java/io/openaev/service/ElasticService.java b/openaev-model/src/main/java/io/openaev/service/ElasticService.java index fba464b7d20..29c647f004e 100644 --- a/openaev-model/src/main/java/io/openaev/service/ElasticService.java +++ b/openaev-model/src/main/java/io/openaev/service/ElasticService.java @@ -341,8 +341,9 @@ public void bulkProcessing(Stream> models) { String index = model.getIndex(engineConfig); Instant fetchInstant = indexingStatus.map(IndexingStatus::getLastIndexing).orElse(null); + String lastId = indexingStatus.map(IndexingStatus::getLastId).orElse(null); List results = - handler.fetch(fetchInstant, engineConfig.getIndexingBatchSize()); + handler.fetch(fetchInstant, lastId, engineConfig.getIndexingBatchSize()); if (!results.isEmpty()) { // Create bulk for the data BulkRequest.Builder br = new BulkRequest.Builder(); @@ -367,14 +368,23 @@ public void bulkProcessing(Stream> models) { } } else { // Update the status for the next round + EsBase lastResult = results.getLast(); + // When the batch is full, more items may share the same timestamp; + // track lastId so the compound cursor skips already-processed ones. + String nextLastId = + results.size() == engineConfig.getIndexingBatchSize() + ? lastResult.getBase_id() + : null; if (indexingStatus.isPresent()) { IndexingStatus status = indexingStatus.get(); - status.setLastIndexing(results.getLast().getBase_updated_at()); + status.setLastIndexing(lastResult.getBase_updated_at()); + status.setLastId(nextLastId); return status; } else { IndexingStatus status = new IndexingStatus(); status.setType(model.getName()); - status.setLastIndexing(results.getLast().getBase_updated_at()); + status.setLastIndexing(lastResult.getBase_updated_at()); + status.setLastId(nextLastId); return status; } } diff --git a/openaev-model/src/main/java/io/openaev/service/OpenSearchService.java b/openaev-model/src/main/java/io/openaev/service/OpenSearchService.java index 22034a44c3d..5077e25ff18 100644 --- a/openaev-model/src/main/java/io/openaev/service/OpenSearchService.java +++ b/openaev-model/src/main/java/io/openaev/service/OpenSearchService.java @@ -408,8 +408,9 @@ public void bulkProcessing(Stream> models) { String index = model.getIndex(engineConfig); Instant fetchInstant = indexingStatus.map(IndexingStatus::getLastIndexing).orElse(null); + String lastId = indexingStatus.map(IndexingStatus::getLastId).orElse(null); List results = - handler.fetch(fetchInstant, engineConfig.getIndexingBatchSize()); + handler.fetch(fetchInstant, lastId, engineConfig.getIndexingBatchSize()); if (!results.isEmpty()) { // Create bulk for the data BulkRequest.Builder br = new BulkRequest.Builder(); @@ -434,14 +435,23 @@ public void bulkProcessing(Stream> models) { } } else { // Update the status for the next round + EsBase lastResult = results.getLast(); + // When the batch is full, more items may share the same timestamp; + // track lastId so the compound cursor skips already-processed ones. + String nextLastId = + results.size() == engineConfig.getIndexingBatchSize() + ? lastResult.getBase_id() + : null; if (indexingStatus.isPresent()) { IndexingStatus status = indexingStatus.get(); - status.setLastIndexing(results.getLast().getBase_updated_at()); + status.setLastIndexing(lastResult.getBase_updated_at()); + status.setLastId(nextLastId); return status; } else { IndexingStatus status = new IndexingStatus(); status.setType(model.getName()); - status.setLastIndexing(results.getLast().getBase_updated_at()); + status.setLastIndexing(lastResult.getBase_updated_at()); + status.setLastId(nextLastId); return status; } }