Add module-execution config on the host level (#3594)

Author: AntoxaAntoxic

docs/config-app.md

@@ -370,9 +370,6 @@ contain 'WHERE last_updated > ?' for MySQL and 'WHERE last_updated > $1' for Pos
 For targeting available next options:
 - `settings.targeting.truncate-attr-chars` - set the max length for names of targeting keywords (0 means no truncation).
 
-For modules:
-- `settings.modules.require-config-to-invoke` - when enabled it requires a runtime config to exist for a module.
-
 ## Host Cookie
 - `host-cookie.optout-cookie.name` - set the cookie name for optout checking.
 - `host-cookie.optout-cookie.value` - set the cookie value for optout checking.
@@ -443,6 +440,10 @@ If not defined in config all other Health Checkers would be disabled and endpoin
 - `analytics.pubstack.buffers.count` - threshold in events count for buffer to send events
 - `analytics.pubstack.buffers.report-ttl-ms` - max period between two reports.
 
+## Modules
+- `hooks.admin.module-execution` - a key-value map, where a key is a module name and a value is a boolean, that defines whether modules hooks should/should not be always executed; if the module is not specified it is executed by default when it's present in the execution plan
+- `settings.modules.require-config-to-invoke` - when enabled it requires a runtime config to exist for a module.
+
 ## Debugging
 - `debug.override-token` - special string token for overriding Prebid Server account and/or adapter debug information presence in the auction response.
 

src/main/java/org/prebid/server/hooks/execution/GroupExecutor.java

@@ -81,7 +81,7 @@ public Future<GroupResult<PAYLOAD>> execute() {
         Future<GroupResult<PAYLOAD>> groupFuture = Future.succeededFuture(initialGroupResult);
 
         for (final HookId hookId : group.getHookSequence()) {
-            if (!modulesExecution.getOrDefault(hookId.getModuleCode(), true)) {
+            if (!modulesExecution.get(hookId.getModuleCode())) {
                 continue;
             }
 

src/main/java/org/prebid/server/hooks/execution/HookStageExecutor.java

@@ -81,6 +81,7 @@ public class HookStageExecutor {
 
     private final ExecutionPlan hostExecutionPlan;
     private final ExecutionPlan defaultAccountExecutionPlan;
+    private final Map<String, Boolean> hostModuleExecution;
     private final HookCatalog hookCatalog;
     private final TimeoutFactory timeoutFactory;
     private final Vertx vertx;
@@ -90,6 +91,7 @@ public class HookStageExecutor {
 
     private HookStageExecutor(ExecutionPlan hostExecutionPlan,
                               ExecutionPlan defaultAccountExecutionPlan,
+                              Map<String, Boolean> hostModuleExecution,
                               HookCatalog hookCatalog,
                               TimeoutFactory timeoutFactory,
                               Vertx vertx,
@@ -105,10 +107,12 @@ private HookStageExecutor(ExecutionPlan hostExecutionPlan,
         this.clock = clock;
         this.mapper = mapper;
         this.isConfigToInvokeRequired = isConfigToInvokeRequired;
+        this.hostModuleExecution = hostModuleExecution;
     }
 
     public static HookStageExecutor create(String hostExecutionPlan,
                                            String defaultAccountExecutionPlan,
+                                           Map<String, Boolean> hostModuleExecution,
                                            HookCatalog hookCatalog,
                                            TimeoutFactory timeoutFactory,
                                            Vertx vertx,
@@ -122,6 +126,7 @@ public static HookStageExecutor create(String hostExecutionPlan,
         return new HookStageExecutor(
                 parseAndValidateExecutionPlan(hostExecutionPlan, mapper, hookCatalog),
                 parseAndValidateExecutionPlan(defaultAccountExecutionPlan, mapper, hookCatalog),
+                hostModuleExecution,
                 hookCatalog,
                 Objects.requireNonNull(timeoutFactory),
                 Objects.requireNonNull(vertx),
@@ -185,7 +190,7 @@ public Future<HookStageExecutionResult<EntrypointPayload>> executeEntrypointStag
                 .withHookProvider(hookProviderForEntrypointStage(context))
                 .withInitialPayload(EntrypointPayloadImpl.of(queryParams, headers, body))
                 .withInvocationContextProvider(invocationContextProvider(endpoint))
-                .withModulesExecution(Collections.emptyMap())
+                .withModulesExecution(DefaultedMap.defaultedMap(hostModuleExecution, true))
                 .withRejectAllowed(true)
                 .execute();
     }
@@ -374,6 +379,7 @@ private Map<String, Boolean> modulesExecutionForAccount(Account account) {
                     .forEach(module -> resultModulesExecution.computeIfAbsent(module, key -> true));
         }
 
+        resultModulesExecution.putAll(hostModuleExecution);
         return DefaultedMap.defaultedMap(resultModulesExecution, !isConfigToInvokeRequired);
     }
 

src/main/java/org/prebid/server/spring/config/HooksConfiguration.java

@@ -8,6 +8,7 @@
 import org.prebid.server.hooks.execution.HookStageExecutor;
 import org.prebid.server.hooks.v1.Module;
 import org.prebid.server.json.JacksonMapper;
+import org.prebid.server.settings.model.HooksAdminConfig;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.context.annotation.Bean;
@@ -16,6 +17,8 @@
 
 import java.time.Clock;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.Optional;
 
 @Configuration
 public class HooksConfiguration {
@@ -38,6 +41,9 @@ HookStageExecutor hookStageExecutor(HooksConfigurationProperties hooksConfigurat
         return HookStageExecutor.create(
                 hooksConfiguration.getHostExecutionPlan(),
                 hooksConfiguration.getDefaultAccountExecutionPlan(),
+                Optional.ofNullable(hooksConfiguration.getAdmin())
+                        .map(HooksAdminConfig::getModuleExecution)
+                        .orElseGet(Collections::emptyMap),
                 hookCatalog,
                 timeoutFactory,
                 vertx,
@@ -60,5 +66,7 @@ private static class HooksConfigurationProperties {
         String hostExecutionPlan;
 
         String defaultAccountExecutionPlan;
+
+        HooksAdminConfig admin;
     }
 }

src/test/groovy/org/prebid/server/functional/testcontainers/container/PrebidServerContainer.groovy

@@ -91,7 +91,6 @@ class PrebidServerContainer extends GenericContainer<PrebidServerContainer> {
 
     private static String normalizeProperty(String property) {
         property.replace(".", "_")
-                .replace("-", "")
                 .replace("[", "_")
                 .replace("]", "_")
     }

src/test/groovy/org/prebid/server/functional/tests/module/GeneralModuleSpec.groovy

@@ -15,7 +15,6 @@ import org.prebid.server.functional.model.request.auction.TraceLevel
 import org.prebid.server.functional.model.response.auction.InvocationResult
 import org.prebid.server.functional.service.PrebidServerService
 import org.prebid.server.functional.util.PBSUtils
-import spock.lang.PendingFeature
 
 import static org.prebid.server.functional.model.ModuleName.ORTB2_BLOCKING
 import static org.prebid.server.functional.model.ModuleName.PB_RICHMEDIA_FILTER
@@ -293,7 +292,6 @@ class GeneralModuleSpec extends ModuleBaseSpec {
         modulesConfig << [null, new PbsModulesConfig()]
     }
 
-    @PendingFeature
     def "PBS should call all modules without account config when modules enabled in module-execution host config"() {
         given: "PBS service with module-execution config"
         def pbsConfig = MULTI_MODULE_CONFIG + ENABLED_INVOKE_CONFIG +
@@ -333,6 +331,37 @@ class GeneralModuleSpec extends ModuleBaseSpec {
         pbsServiceFactory.removeContainer(pbsConfig)
     }
 
+    def "PBS shouldn't call any module without account config when modules disabled in module-execution host config"() {
+        given: "PBS service with module-execution config"
+        def pbsConfig = MULTI_MODULE_CONFIG + ENABLED_INVOKE_CONFIG +
+                [("hooks.admin.module-execution.${ORTB2_BLOCKING.code}".toString()): 'false']
+        def pbsServiceWithMultipleModules = pbsServiceFactory.getService(pbsConfig)
+
+        and: "Default bid request with verbose trace"
+        def bidRequest = defaultBidRequest.tap {
+            ext.prebid.trace = TraceLevel.VERBOSE
+        }
+
+        and: "Flush metrics"
+        flushMetrics(pbsServiceWithMultipleModules)
+
+        when: "PBS processes auction request"
+        def response = pbsServiceWithMultipleModules.sendAuctionRequest(bidRequest)
+
+        then: "PBS response shouldn't include trace information about no-called modules"
+        assert !response?.ext?.prebid?.modules?.trace?.stages?.outcomes?.groups?.invocationResults?.flatten()
+
+        and: "Ortb2blocking module call metrics shouldn't be updated"
+        def metrics = pbsServiceWithMultipleModuleWithRequireInvoke.sendCollectedMetricsRequest()
+        assert !metrics[CALL_METRIC.formatted(ORTB2_BLOCKING.code, BIDDER_REQUEST.metricValue, ORTB2_BLOCKING_BIDDER_REQUEST.code)]
+        assert !metrics[CALL_METRIC.formatted(ORTB2_BLOCKING.code, RAW_BIDDER_RESPONSE.metricValue, ORTB2_BLOCKING_RAW_BIDDER_RESPONSE.code)]
+        assert !metrics[NOOP_METRIC.formatted(ORTB2_BLOCKING.code, BIDDER_REQUEST.metricValue, ORTB2_BLOCKING_BIDDER_REQUEST.code)]
+        assert !metrics[NOOP_METRIC.formatted(ORTB2_BLOCKING.code, RAW_BIDDER_RESPONSE.metricValue, ORTB2_BLOCKING_RAW_BIDDER_RESPONSE.code)]
+
+        cleanup: "Stop and remove pbs container"
+        pbsServiceFactory.removeContainer(pbsConfig)
+    }
+
     def "PBS should call module without account config when default-account module-execution config enabled module"() {
         given: "PBS service with module-execution and default account configs"
         def defaultAccountConfigSettings = AccountConfig.defaultAccountConfig.tap {
@@ -358,19 +387,15 @@ class GeneralModuleSpec extends ModuleBaseSpec {
         when: "PBS processes auction request"
         def response = pbsServiceWithMultipleModules.sendAuctionRequest(bidRequest)
 
-        then: "PBS response should include trace information about called modules"
-        verifyAll(response?.ext?.prebid?.modules?.trace?.stages?.outcomes?.groups?.invocationResults?.flatten() as List<InvocationResult>) {
-            it.status == [SUCCESS, SUCCESS]
-            it.action == [NO_ACTION, NO_ACTION]
-            it.hookId.moduleCode.sort() == [ORTB2_BLOCKING, ORTB2_BLOCKING].code.sort()
-        }
+        then: "PBS response shouldn't include trace information about no-called modules"
+        assert !response?.ext?.prebid?.modules?.trace?.stages?.outcomes?.groups?.invocationResults?.flatten()
 
-        and: "Ortb2blocking module call metrics should be updated"
-        def metrics = pbsServiceWithMultipleModules.sendCollectedMetricsRequest()
-        assert metrics[CALL_METRIC.formatted(ORTB2_BLOCKING.code, BIDDER_REQUEST.metricValue, ORTB2_BLOCKING_BIDDER_REQUEST.code)] == 1
-        assert metrics[CALL_METRIC.formatted(ORTB2_BLOCKING.code, RAW_BIDDER_RESPONSE.metricValue, ORTB2_BLOCKING_RAW_BIDDER_RESPONSE.code)] == 1
-        assert metrics[NOOP_METRIC.formatted(ORTB2_BLOCKING.code, BIDDER_REQUEST.metricValue, ORTB2_BLOCKING_BIDDER_REQUEST.code)] == 1
-        assert metrics[NOOP_METRIC.formatted(ORTB2_BLOCKING.code, RAW_BIDDER_RESPONSE.metricValue, ORTB2_BLOCKING_RAW_BIDDER_RESPONSE.code)] == 1
+        and: "Ortb2blocking module call metrics shouldn't be updated"
+        def metrics = pbsServiceWithMultipleModuleWithRequireInvoke.sendCollectedMetricsRequest()
+        assert !metrics[CALL_METRIC.formatted(ORTB2_BLOCKING.code, BIDDER_REQUEST.metricValue, ORTB2_BLOCKING_BIDDER_REQUEST.code)]
+        assert !metrics[CALL_METRIC.formatted(ORTB2_BLOCKING.code, RAW_BIDDER_RESPONSE.metricValue, ORTB2_BLOCKING_RAW_BIDDER_RESPONSE.code)]
+        assert !metrics[NOOP_METRIC.formatted(ORTB2_BLOCKING.code, BIDDER_REQUEST.metricValue, ORTB2_BLOCKING_BIDDER_REQUEST.code)]
+        assert !metrics[NOOP_METRIC.formatted(ORTB2_BLOCKING.code, RAW_BIDDER_RESPONSE.metricValue, ORTB2_BLOCKING_RAW_BIDDER_RESPONSE.code)]
 
         and: "RB-Richmedia-Filter module call metrics shouldn't be updated"
         assert !metrics[CALL_METRIC.formatted(PB_RICHMEDIA_FILTER.code, ALL_PROCESSED_BID_RESPONSES.metricValue, PB_RICHMEDIA_FILTER_ALL_PROCESSED_RESPONSES.code)]