diff --git a/docs/config-app.md b/docs/config-app.md index a661f5a74a2..5c5fbaa95e2 100644 --- a/docs/config-app.md +++ b/docs/config-app.md @@ -457,6 +457,8 @@ If not defined in config all other Health Checkers would be disabled and endpoin - `gdpr.special-features.sfN.vendor-exceptions[]` - bidder names that will be treated opposite to `sfN.enforce` value. - `gdpr.purpose-one-treatment-interpretation` - option that allows to skip the Purpose one enforcement workflow. - `gdpr.vendorlist.default-timeout-ms` - default operation timeout for obtaining new vendor list. +- `gdpr.vendorlist.live-gvl-url` - URL of the latest TCF GVL used to detect vendors with a past `deletedDate`. Default `https://vendor-list.consensu.org/v3/vendor-list.json`. +- `gdpr.vendorlist.live-gvl-refresh-period-ms` - how often to refresh the live GVL deleted-vendor set, in milliseconds. Default `86400000` (24 hours). - `gdpr.vendorlist.v2.http-endpoint-template` - template string for vendor list url version 2. - `gdpr.vendorlist.v2.refresh-missing-list-period-ms` - time to wait between attempts to fetch vendor list version that previously was reported to be missing by origin. Default `3600000` (one hour). - `gdpr.vendorlist.v2.fallback-vendor-list-path` - location on the file system of the fallback vendor list that will be used in place of missing vendor list versions. Optional. diff --git a/docs/metrics.md b/docs/metrics.md index c07e0660598..e5844413032 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -131,6 +131,7 @@ Following metrics are collected and submitted if account is configured with `det - `privacy.tcf.(v1,v2).in-geo` - number of requests received from TCF-concerned geo region with consent string of particular version - `privacy.tcf.(v1,v2).out-geo` - number of requests received outside of TCF-concerned geo region with consent string of particular version - `privacy.tcf.(v1,v2).vendorlist.(missing|ok|err|fallback)` - number of processed vendor lists of particular version +- `privacy.tcf.vendorlist.latest.(ok|err)` - number of successful or failed refreshes of the live GVL used for deleted-vendor detection - `privacy.usp.specified` - number of requests with a valid US Privacy string (CCPA) - `privacy.usp.opt-out` - number of requests that required privacy enforcement according to CCPA rules - `privacy.lmt` - number of requests that required privacy enforcement according to LMT flag diff --git a/src/main/java/org/prebid/server/metric/Metrics.java b/src/main/java/org/prebid/server/metric/Metrics.java index 517fbb36bd5..e533ba53636 100644 --- a/src/main/java/org/prebid/server/metric/Metrics.java +++ b/src/main/java/org/prebid/server/metric/Metrics.java @@ -554,6 +554,14 @@ public void updatePrivacyTcfVendorListFallbackMetric(int version) { updatePrivacyTcfVendorListMetric(version, MetricName.fallback); } + public void updatePrivacyTcfVendorListLatestOkMetric() { + privacy().tcf().vendorListLatest().incCounter(MetricName.ok); + } + + public void updatePrivacyTcfVendorListLatestErrorMetric() { + privacy().tcf().vendorListLatest().incCounter(MetricName.err); + } + private void updatePrivacyTcfVendorListMetric(int version, MetricName metricName) { final TcfMetrics tcfMetrics = privacy().tcf(); tcfMetrics.fromVersion(version).vendorList().incCounter(metricName); diff --git a/src/main/java/org/prebid/server/metric/TcfMetrics.java b/src/main/java/org/prebid/server/metric/TcfMetrics.java index 9fd5a811562..ba5d8ed86f3 100644 --- a/src/main/java/org/prebid/server/metric/TcfMetrics.java +++ b/src/main/java/org/prebid/server/metric/TcfMetrics.java @@ -16,6 +16,7 @@ class TcfMetrics extends UpdatableMetrics { private final TcfVersionMetrics tcfVersion1Metrics; private final TcfVersionMetrics tcfVersion2Metrics; + private final VendorListLatestMetrics vendorListLatestMetrics; TcfMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) { super( @@ -25,6 +26,7 @@ class TcfMetrics extends UpdatableMetrics { tcfVersion1Metrics = new TcfVersionMetrics(metricRegistry, counterType, createTcfPrefix(prefix), "v1"); tcfVersion2Metrics = new TcfVersionMetrics(metricRegistry, counterType, createTcfPrefix(prefix), "v2"); + vendorListLatestMetrics = new VendorListLatestMetrics(metricRegistry, counterType, createTcfPrefix(prefix)); } TcfVersionMetrics fromVersion(int version) { @@ -35,6 +37,10 @@ TcfVersionMetrics fromVersion(int version) { }; } + VendorListLatestMetrics vendorListLatest() { + return vendorListLatestMetrics; + } + private static String createTcfPrefix(String prefix) { return prefix + ".tcf"; } @@ -87,4 +93,22 @@ private static Function nameCreator(String prefix) { return metricName -> "%s.%s".formatted(prefix, metricName); } } + + static class VendorListLatestMetrics extends UpdatableMetrics { + + VendorListLatestMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) { + super( + metricRegistry, + counterType, + nameCreator(createLatestPrefix(prefix))); + } + + private static String createLatestPrefix(String prefix) { + return prefix + ".vendorlist.latest"; + } + + private static Function nameCreator(String prefix) { + return metricName -> "%s.%s".formatted(prefix, metricName); + } + } } diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java new file mode 100644 index 00000000000..c0d388dfac2 --- /dev/null +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java @@ -0,0 +1,131 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import io.vertx.core.Promise; +import io.vertx.core.Vertx; +import org.prebid.server.exception.PreBidException; +import org.prebid.server.json.JacksonMapper; +import org.prebid.server.log.Logger; +import org.prebid.server.log.LoggerFactory; +import org.prebid.server.metric.Metrics; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; +import org.prebid.server.util.HttpUtil; +import org.prebid.server.vertx.Initializable; +import org.prebid.server.vertx.httpclient.HttpClient; +import org.prebid.server.vertx.httpclient.model.HttpClientResponse; + +import java.time.Clock; +import java.time.Instant; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +public class LiveVendorListService implements Initializable { + + private static final Logger logger = LoggerFactory.getLogger(LiveVendorListService.class); + + private final String cacheDir; + private final String liveGvlUrl; + private final long refreshPeriodMs; + private final int defaultTimeoutMs; + private final Vertx vertx; + private final HttpClient httpClient; + private final VendorListFileStore vendorListFileStore; + private final Metrics metrics; + private final JacksonMapper mapper; + private final Clock clock; + + private volatile Set deletedVendorIds = Set.of(); + + public LiveVendorListService(String cacheDir, + String liveGvlUrl, + long refreshPeriodMs, + int defaultTimeoutMs, + Vertx vertx, + HttpClient httpClient, + VendorListFileStore vendorListFileStore, + Metrics metrics, + JacksonMapper mapper, + Clock clock) { + + this.cacheDir = Objects.requireNonNull(cacheDir); + this.liveGvlUrl = HttpUtil.validateUrl(Objects.requireNonNull(liveGvlUrl)); + this.refreshPeriodMs = refreshPeriodMs; + this.defaultTimeoutMs = defaultTimeoutMs; + this.vertx = Objects.requireNonNull(vertx); + this.httpClient = Objects.requireNonNull(httpClient); + this.vendorListFileStore = Objects.requireNonNull(vendorListFileStore); + this.metrics = Objects.requireNonNull(metrics); + this.mapper = Objects.requireNonNull(mapper); + this.clock = Objects.requireNonNull(clock); + } + + public boolean isDeleted(Integer id) { + final Set ids = deletedVendorIds; + return !ids.isEmpty() && ids.contains(id); + } + + @Override + public void initialize(Promise initializePromise) { + initializeWithLatestCachedVersion(); + vertx.setPeriodic(0, refreshPeriodMs, ignored -> refresh()); + + initializePromise.tryComplete(); + } + + private void initializeWithLatestCachedVersion() { + vendorListFileStore.getLatestVendorListFromCache(cacheDir).ifPresent(vendorList -> { + saveDeletedVendorsFromVendorList(vendorList); + logger.info("Initialized live GVL from cache with version %d".formatted(vendorList.getVendorListVersion())); + }); + } + + void refresh() { + httpClient.get(liveGvlUrl, defaultTimeoutMs) + .map(this::processResponse) + .map(this::saveDeletedVendorsFromVendorList) + .otherwise(this::handleError); + } + + private Void saveDeletedVendorsFromVendorList(VendorList vendorList) { + updateDeletedVendorIds(extractDeletedVendorIds(vendorList)); + return null; + } + + private VendorList processResponse(HttpClientResponse response) { + final int statusCode = response.getStatusCode(); + if (statusCode != 200) { + throw new PreBidException("HTTP status code " + statusCode); + } + + final String body = response.getBody(); + final VendorList vendorList = VendorListUtil.parseVendorList(body, mapper); + + if (!VendorListUtil.vendorListIsValid(vendorList)) { + throw new PreBidException("Fetched vendor list parsed but has invalid data: " + body); + } + + return vendorList; + } + + Set extractDeletedVendorIds(VendorList vendorList) { + final Instant now = clock.instant(); + return vendorList.getVendors().values().stream() + .filter(vendor -> VendorListUtil.vendorIsDeletedAt(vendor, now)) + .map(Vendor::getId) + .filter(Objects::nonNull) + .collect(Collectors.toUnmodifiableSet()); + } + + private Void updateDeletedVendorIds(Set ids) { + deletedVendorIds = ids; + metrics.updatePrivacyTcfVendorListLatestOkMetric(); + return null; + } + + private Void handleError(Throwable exception) { + logger.warn("Error occurred while fetching live GVL", exception); + metrics.updatePrivacyTcfVendorListLatestErrorMetric(); + return null; + } +} diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java new file mode 100644 index 00000000000..acf3ffb11ce --- /dev/null +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java @@ -0,0 +1,134 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import com.github.benmanes.caffeine.cache.Caffeine; +import io.vertx.core.Future; +import io.vertx.core.Promise; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.file.FileProps; +import io.vertx.core.file.FileSystem; +import io.vertx.core.file.FileSystemException; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.prebid.server.exception.PreBidException; +import org.prebid.server.json.JacksonMapper; +import org.prebid.server.log.ConditionalLogger; +import org.prebid.server.log.Logger; +import org.prebid.server.log.LoggerFactory; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Comparator; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +public class VendorListFileStore { + + private static final Logger logger = LoggerFactory.getLogger(VendorListFileStore.class); + private static final ConditionalLogger conditionalLogger = new ConditionalLogger(logger); + + private static final String JSON_SUFFIX = ".json"; + + private final double logSamplingRate; + private final FileSystem fileSystem; + private final JacksonMapper mapper; + + public VendorListFileStore(double logSamplingRate, + FileSystem fileSystem, + JacksonMapper mapper) { + + this.logSamplingRate = logSamplingRate; + this.fileSystem = Objects.requireNonNull(fileSystem); + this.mapper = Objects.requireNonNull(mapper); + } + + Map> createCacheFromDisk(String cacheDir) { + createAndCheckWritePermissionsForCacheDir(cacheDir); + final Map versionToFileContent = readFileSystemCache(cacheDir); + + final Map> cache = Caffeine.newBuilder() + .>build() + .asMap(); + + for (Map.Entry versionAndFileContent : versionToFileContent.entrySet()) { + final VendorList vendorList = VendorListUtil.parseVendorList(versionAndFileContent.getValue(), mapper); + + cache.put(versionAndFileContent.getKey(), vendorList.getVendors()); + } + return cache; + } + + private void createAndCheckWritePermissionsForCacheDir(String cacheDir) { + final FileProps props = fileSystem.existsBlocking(cacheDir) ? fileSystem.propsBlocking(cacheDir) : null; + if (props == null || !props.isDirectory()) { + try { + fileSystem.mkdirsBlocking(cacheDir); + } catch (FileSystemException e) { + throw new PreBidException("Cannot create directory: " + cacheDir, e); + } + } else if (!Files.isWritable(Paths.get(cacheDir))) { + throw new PreBidException("No write permissions for directory: " + cacheDir); + } + } + + private Map readFileSystemCache(String cacheDir) { + return fileSystem.readDirBlocking(cacheDir).stream() + .filter(filepath -> filepath.endsWith(JSON_SUFFIX)) + .collect(Collectors.toMap(VendorListFileStore::parseCachedFileVersion, + filename -> fileSystem.readFileBlocking(filename).toString())); + } + + Optional getLatestVendorListFromCache(String cacheDir) { + createAndCheckWritePermissionsForCacheDir(cacheDir); + return fileSystem.readDirBlocking(cacheDir).stream() + .filter(filepath -> filepath.endsWith(JSON_SUFFIX)) + .max(Comparator.comparing(VendorListFileStore::parseCachedFileVersion)) + .map(fileSystem::readFileBlocking) + .map(Buffer::toString) + .map(content -> VendorListUtil.parseVendorList(content, mapper)); + } + + private static Integer parseCachedFileVersion(String filepath) { + final String filename = new File(filepath).getName(); + final String filenameWithoutExtension = StringUtils.removeEnd(filename, JSON_SUFFIX); + return Integer.valueOf(filenameWithoutExtension); + } + + Future saveToFile(VendorListResult vendorListResult, String cacheDir, String generationVersion) { + final Promise promise = Promise.promise(); + final int version = vendorListResult.getVersion(); + final String filepath = new File(cacheDir, version + JSON_SUFFIX).getPath(); + + fileSystem.writeFile(filepath, Buffer.buffer(vendorListResult.getVendorListAsString()), result -> { + if (result.succeeded()) { + promise.complete(vendorListResult); + } else { + conditionalLogger.error( + "Could not create new vendor list for version %s.%s, file: %s, trace: %s".formatted( + generationVersion, version, filepath, ExceptionUtils.getStackTrace(result.cause())), + logSamplingRate); + promise.fail(result.cause()); + } + }); + + return promise.future(); + } + + Map readFallbackVendorList(String fallbackVendorListPath) { + if (StringUtils.isBlank(fallbackVendorListPath)) { + return null; + } + + final String vendorListContent = fileSystem.readFileBlocking(fallbackVendorListPath).toString(); + final VendorList vendorList = VendorListUtil.parseVendorList(vendorListContent, mapper); + if (!VendorListUtil.vendorListIsValid(vendorList)) { + throw new PreBidException("Fallback vendor list parsed but has invalid data: " + vendorListContent); + } + + return vendorList.getVendors(); + } +} diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListResult.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListResult.java new file mode 100644 index 00000000000..827300f90c2 --- /dev/null +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListResult.java @@ -0,0 +1,14 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import lombok.Value; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; + +@Value(staticConstructor = "of") +class VendorListResult { + + int version; + + String vendorListAsString; + + VendorList vendorList; +} diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListService.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListService.java index 51c9b3e9252..694ab0ed01c 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListService.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListService.java @@ -1,17 +1,8 @@ package org.prebid.server.privacy.gdpr.vendorlist; -import com.github.benmanes.caffeine.cache.Caffeine; import io.netty.handler.codec.http.HttpResponseStatus; import io.vertx.core.Future; -import io.vertx.core.Promise; import io.vertx.core.Vertx; -import io.vertx.core.buffer.Buffer; -import io.vertx.core.file.FileProps; -import io.vertx.core.file.FileSystem; -import io.vertx.core.file.FileSystemException; -import lombok.Value; -import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.prebid.server.exception.PreBidException; import org.prebid.server.json.JacksonMapper; @@ -24,16 +15,10 @@ import org.prebid.server.vertx.httpclient.HttpClient; import org.prebid.server.vertx.httpclient.model.HttpClientResponse; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Collection; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; /** * Works with GDPR Vendor List. @@ -54,7 +39,6 @@ public class VendorListService { private static final int TCF_VERSION = 2; - private static final String JSON_SUFFIX = ".json"; private static final String VERSION_PLACEHOLDER = "{VERSION}"; private final double logSamplingRate; @@ -64,21 +48,18 @@ public class VendorListService { private final long refreshMissingListPeriodMs; private final boolean deprecated; private final Vertx vertx; - private final FileSystem fileSystem; private final HttpClient httpClient; private final Metrics metrics; private final String generationVersion; - protected final JacksonMapper mapper; + private final VendorListFetchThrottler fetchThrottler; + private final VendorListFileStore vendorListFileStore; + private final JacksonMapper mapper; - /** - * This is memory/performance optimized model slice: - * map of vendor list version -> map of vendor ID -> Vendors - */ + // Map of vendor list version -> map of vendor ID -> Vendors private final Map> cache; private final Map fallbackVendorList; private final Set versionsToFallback; - private final VendorListFetchThrottler fetchThrottler; public VendorListService(double logSamplingRate, String cacheDir, @@ -88,12 +69,12 @@ public VendorListService(double logSamplingRate, boolean deprecated, String fallbackVendorListPath, Vertx vertx, - FileSystem fileSystem, HttpClient httpClient, Metrics metrics, String generationVersion, - JacksonMapper mapper, - VendorListFetchThrottler fetchThrottler) { + VendorListFetchThrottler fetchThrottler, + VendorListFileStore vendorListFileStore, + JacksonMapper mapper) { this.logSamplingRate = logSamplingRate; this.cacheDir = Objects.requireNonNull(cacheDir); @@ -103,17 +84,15 @@ public VendorListService(double logSamplingRate, this.deprecated = deprecated; this.generationVersion = generationVersion; this.vertx = Objects.requireNonNull(vertx); - this.fileSystem = Objects.requireNonNull(fileSystem); this.httpClient = Objects.requireNonNull(httpClient); this.metrics = Objects.requireNonNull(metrics); - this.mapper = Objects.requireNonNull(mapper); this.fetchThrottler = Objects.requireNonNull(fetchThrottler); + this.vendorListFileStore = Objects.requireNonNull(vendorListFileStore); + this.mapper = Objects.requireNonNull(mapper); - createAndCheckWritePermissionsFor(fileSystem, cacheDir); - cache = Objects.requireNonNull(createCache(fileSystem, cacheDir)); + cache = Objects.requireNonNull(vendorListFileStore.createCacheFromDisk(cacheDir)); - fallbackVendorList = StringUtils.isNotBlank(fallbackVendorListPath) - ? readFallbackVendorList(fallbackVendorListPath) : null; + fallbackVendorList = vendorListFileStore.readFallbackVendorList(fallbackVendorListPath); if (deprecated) { validateFallbackVendorListIfDeprecatedVersion(); } @@ -161,50 +140,6 @@ public Future> forVersion(int version) { .formatted(tcf, generationVersion, version)); } - /** - * Creates vendorList object from string content or throw {@link PreBidException}. - */ - private VendorList toVendorList(String content) { - try { - return mapper.mapper().readValue(content, VendorList.class); - } catch (IOException e) { - final String message = "Cannot parse vendor list from: " + content; - - logger.error(message, e); - throw new PreBidException(message, e); - } - } - - /** - * Returns a Map of vendor id to Vendors. - */ - private Map filterVendorIdToVendors(VendorList vendorList) { - return vendorList.getVendors().entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - /** - * Verifies all significant fields of given {@link VendorList} object. - */ - private boolean isValid(VendorList vendorList) { - return vendorList.getVendorListVersion() != null - && vendorList.getLastUpdated() != null - && MapUtils.isNotEmpty(vendorList.getVendors()) - && isValidVendors(vendorList.getVendors().values()); - } - - private static boolean isValidVendors(Collection vendors) { - return vendors.stream() - .allMatch(vendor -> vendor != null - && vendor.getId() != null - && vendor.getPurposes() != null - && vendor.getLegIntPurposes() != null - && vendor.getFlexiblePurposes() != null - && vendor.getSpecialPurposes() != null - && vendor.getFeatures() != null - && vendor.getSpecialFeatures() != null); - } - /** * Returns the version of TCF which {@link VendorListService} implementation deals with. */ @@ -212,62 +147,6 @@ private int getTcfVersion() { return TCF_VERSION; } - /** - * Creates if doesn't exists and checks write permissions for the given directory. - */ - private static void createAndCheckWritePermissionsFor(FileSystem fileSystem, String dir) { - final FileProps props = fileSystem.existsBlocking(dir) ? fileSystem.propsBlocking(dir) : null; - if (props == null || !props.isDirectory()) { - try { - fileSystem.mkdirsBlocking(dir); - } catch (FileSystemException e) { - throw new PreBidException("Cannot create directory: " + dir, e); - } - } else if (!Files.isWritable(Paths.get(dir))) { - throw new PreBidException("No write permissions for directory: " + dir); - } - } - - /** - * Creates the cache from previously downloaded vendor lists. - */ - private Map> createCache(FileSystem fileSystem, String cacheDir) { - final Map versionToFileContent = readFileSystemCache(fileSystem, cacheDir); - - final Map> cache = Caffeine.newBuilder() - .>build() - .asMap(); - - for (Map.Entry versionAndFileContent : versionToFileContent.entrySet()) { - final VendorList vendorList = toVendorList(versionAndFileContent.getValue()); - final Map vendorIdToVendors = filterVendorIdToVendors(vendorList); - - cache.put(Integer.valueOf(versionAndFileContent.getKey()), vendorIdToVendors); - } - return cache; - } - - /** - * Reads files with .json extension in configured directory and - * returns a {@link Map} where key is a file name without .json extension and value is file content. - */ - private Map readFileSystemCache(FileSystem fileSystem, String dir) { - return fileSystem.readDirBlocking(dir).stream() - .filter(filepath -> filepath.endsWith(JSON_SUFFIX)) - .collect(Collectors.toMap(filepath -> StringUtils.removeEnd(new File(filepath).getName(), JSON_SUFFIX), - filename -> fileSystem.readFileBlocking(filename).toString())); - } - - private Map readFallbackVendorList(String fallbackVendorListPath) { - final String vendorListContent = fileSystem.readFileBlocking(fallbackVendorListPath).toString(); - final VendorList vendorList = toVendorList(vendorListContent); - if (!isValid(vendorList)) { - throw new PreBidException("Fallback vendor list parsed but has invalid data: " + vendorListContent); - } - - return filterVendorIdToVendors(vendorList); - } - private boolean shouldFallback(int version) { return deprecated || (versionsToFallback != null && versionsToFallback.contains(version)); } @@ -290,7 +169,7 @@ private void fetchNewVendorListFor(int version) { * and creates {@link Future} with {@link VendorListResult} from body content * or throws {@link PreBidException} in case of errors. */ - private VendorListResult processResponse(HttpClientResponse response, int version) { + private VendorListResult processResponse(HttpClientResponse response, int version) { final int statusCode = response.getStatusCode(); if (statusCode == HttpResponseStatus.NOT_FOUND.code()) { @@ -301,11 +180,11 @@ private VendorListResult processResponse(HttpClientResponse response } final String body = response.getBody(); - final VendorList vendorList = toVendorList(body); + final VendorList vendorList = VendorListUtil.parseVendorList(body, mapper); // we should care on obtained vendor list, because it'll be saved and never be downloaded again // while application is running - if (!isValid(vendorList)) { + if (!VendorListUtil.vendorListIsValid(vendorList)) { throw new PreBidException("Fetched vendor list parsed but has invalid data: " + body); } @@ -313,33 +192,14 @@ private VendorListResult processResponse(HttpClientResponse response return VendorListResult.of(version, body, vendorList); } - /** - * Saves given vendor list on file system. - */ - private Future> saveToFile(VendorListResult vendorListResult) { - final Promise> promise = Promise.promise(); - final int version = vendorListResult.getVersion(); - final String filepath = new File(cacheDir, version + JSON_SUFFIX).getPath(); - - fileSystem.writeFile(filepath, Buffer.buffer(vendorListResult.getVendorListAsString()), result -> { - if (result.succeeded()) { - promise.complete(vendorListResult); - } else { - conditionalLogger.error( - "Could not create new vendor list for version %s.%s, file: %s, trace: %s".formatted( - generationVersion, version, filepath, ExceptionUtils.getStackTrace(result.cause())), - logSamplingRate); - promise.fail(result.cause()); - } - }); - - return promise.future(); + private Future saveToFile(VendorListResult vendorListResult) { + return vendorListFileStore.saveToFile(vendorListResult, cacheDir, generationVersion); } - private Void updateCache(VendorListResult vendorListResult) { + private Void updateCache(VendorListResult vendorListResult) { final int version = vendorListResult.getVersion(); - cache.put(version, filterVendorIdToVendors(vendorListResult.getVendorList())); + cache.put(version, vendorListResult.getVendorList().getVendors()); final int tcf = getTcfVersion(); @@ -395,16 +255,6 @@ private void stopUsingFallbackForVersion(int version) { versionsToFallback.remove(version); } - @Value(staticConstructor = "of") - private static class VendorListResult { - - int version; - - String vendorListAsString; - - T vendorList; - } - private static class MissingVendorListException extends RuntimeException { MissingVendorListException(String message) { diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtil.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtil.java new file mode 100644 index 00000000000..371a90e1c55 --- /dev/null +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtil.java @@ -0,0 +1,56 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import org.apache.commons.collections4.MapUtils; +import org.prebid.server.exception.PreBidException; +import org.prebid.server.json.JacksonMapper; +import org.prebid.server.log.Logger; +import org.prebid.server.log.LoggerFactory; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; + +import java.io.IOException; +import java.time.Instant; +import java.util.Collection; + +public class VendorListUtil { + + private static final Logger logger = LoggerFactory.getLogger(VendorListUtil.class); + + private VendorListUtil() { + } + + public static VendorList parseVendorList(String content, JacksonMapper mapper) { + try { + return mapper.mapper().readValue(content, VendorList.class); + } catch (IOException e) { + final String message = "Cannot parse vendor list from: " + content; + + logger.error(message, e); + throw new PreBidException(message, e); + } + } + + public static boolean vendorListIsValid(VendorList vendorList) { + return vendorList.getVendorListVersion() != null + && vendorList.getLastUpdated() != null + && MapUtils.isNotEmpty(vendorList.getVendors()) + && vendorsAreValid(vendorList.getVendors().values()); + } + + private static boolean vendorsAreValid(Collection vendors) { + return vendors.stream() + .allMatch(vendor -> vendor != null + && vendor.getId() != null + && vendor.getPurposes() != null + && vendor.getLegIntPurposes() != null + && vendor.getFlexiblePurposes() != null + && vendor.getSpecialPurposes() != null + && vendor.getFeatures() != null + && vendor.getSpecialFeatures() != null); + } + + public static boolean vendorIsDeletedAt(Vendor vendor, Instant now) { + final Instant deletedDate = vendor.getDeletedDate(); + return deletedDate != null && deletedDate.isBefore(now); + } +} diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java index 5e261d9b6b4..8f6487ff60e 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java @@ -4,25 +4,46 @@ import io.vertx.core.Future; import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import java.time.Clock; +import java.time.Instant; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; public class VersionedVendorListService { private final VendorListService vendorListServiceV2; private final VendorListService vendorListServiceV3; + private final LiveVendorListService liveVendorListService; + private final Clock clock; + + public VersionedVendorListService(VendorListService vendorListServiceV2, + VendorListService vendorListServiceV3, + LiveVendorListService liveVendorListService, + Clock clock) { - public VersionedVendorListService(VendorListService vendorListServiceV2, VendorListService vendorListServiceV3) { this.vendorListServiceV2 = Objects.requireNonNull(vendorListServiceV2); this.vendorListServiceV3 = Objects.requireNonNull(vendorListServiceV3); + this.liveVendorListService = Objects.requireNonNull(liveVendorListService); + this.clock = Objects.requireNonNull(clock); } public Future> forConsent(TCString consent) { final int tcfPolicyVersion = consent.getTcfPolicyVersion(); final int vendorListVersion = consent.getVendorListVersion(); - return tcfPolicyVersion < 4 + final Future> vendorListFuture = tcfPolicyVersion < 4 ? vendorListServiceV2.forVersion(vendorListVersion) : vendorListServiceV3.forVersion(vendorListVersion); + + return vendorListFuture.map(this::filterDeletedVendors); + } + + private Map filterDeletedVendors(Map vendors) { + final Instant now = clock.instant(); + return vendors.entrySet().stream() + .filter(entry -> !VendorListUtil.vendorIsDeletedAt(entry.getValue(), now)) + .filter(entry -> !liveVendorListService.isDeleted(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } } diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/proto/Vendor.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/proto/Vendor.java index 6bb2be9dddb..336a5d4cdae 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/proto/Vendor.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/proto/Vendor.java @@ -6,6 +6,7 @@ import lombok.Data; import lombok.NoArgsConstructor; +import java.time.Instant; import java.util.EnumSet; @AllArgsConstructor @@ -16,6 +17,9 @@ public class Vendor { Integer id; + @JsonProperty("deletedDate") + Instant deletedDate; + EnumSet purposes; @JsonProperty("legIntPurposes") diff --git a/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java b/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java index b8175828c3b..9ba7da3d839 100644 --- a/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java @@ -37,7 +37,9 @@ import org.prebid.server.privacy.gdpr.tcfstrategies.purpose.typestrategies.PurposeTwoBasicEnforcePurposeStrategy; import org.prebid.server.privacy.gdpr.tcfstrategies.specialfeature.SpecialFeaturesOneStrategy; import org.prebid.server.privacy.gdpr.tcfstrategies.specialfeature.SpecialFeaturesStrategy; +import org.prebid.server.privacy.gdpr.vendorlist.LiveVendorListService; import org.prebid.server.privacy.gdpr.vendorlist.VendorListFetchThrottler; +import org.prebid.server.privacy.gdpr.vendorlist.VendorListFileStore; import org.prebid.server.privacy.gdpr.vendorlist.VendorListService; import org.prebid.server.privacy.gdpr.vendorlist.VersionedVendorListService; import org.prebid.server.settings.model.GdprConfig; @@ -65,6 +67,15 @@ @Configuration public class PrivacyServiceConfiguration { + @Bean + VendorListFileStore vendorListFileStore( + @Value("${logging.sampling-rate:0.01}") double logSamplingRate, + FileSystem fileSystem, + JacksonMapper mapper) { + + return new VendorListFileStore(logSamplingRate, fileSystem, mapper); + } + @Bean VendorListService vendorListServiceV2( @Value("${logging.sampling-rate:0.01}") double logSamplingRate, @@ -72,9 +83,9 @@ VendorListService vendorListServiceV2( VendorListServiceConfigurationProperties vendorListServiceV2Properties, Vertx vertx, Clock clock, - FileSystem fileSystem, HttpClient httpClient, Metrics metrics, + VendorListFileStore vendorListFileStore, JacksonMapper mapper) { return new VendorListService( @@ -86,12 +97,12 @@ VendorListService vendorListServiceV2( vendorListServiceV2Properties.getDeprecated(), vendorListServiceV2Properties.getFallbackVendorListPath(), vertx, - fileSystem, httpClient, metrics, "v2", - mapper, - new VendorListFetchThrottler(vendorListServiceV2Properties.getRetryPolicy().toPolicy(), clock)); + new VendorListFetchThrottler(vendorListServiceV2Properties.getRetryPolicy().toPolicy(), clock), + vendorListFileStore, + mapper); } @Bean @@ -107,9 +118,9 @@ VendorListService vendorListServiceV3( VendorListServiceConfigurationProperties vendorListServiceV3Properties, Vertx vertx, Clock clock, - FileSystem fileSystem, HttpClient httpClient, Metrics metrics, + VendorListFileStore vendorListFileStore, JacksonMapper mapper) { return new VendorListService( @@ -121,12 +132,12 @@ VendorListService vendorListServiceV3( vendorListServiceV3Properties.getDeprecated(), vendorListServiceV3Properties.getFallbackVendorListPath(), vertx, - fileSystem, httpClient, metrics, "v3", - mapper, - new VendorListFetchThrottler(vendorListServiceV3Properties.getRetryPolicy().toPolicy(), clock)); + new VendorListFetchThrottler(vendorListServiceV3Properties.getRetryPolicy().toPolicy(), clock), + vendorListFileStore, + mapper); } @Bean @@ -135,11 +146,40 @@ VendorListServiceConfigurationProperties vendorListServiceV3Properties() { return new VendorListServiceConfigurationProperties(); } + @Bean + LiveVendorListService liveVendorListService( + VendorListServiceConfigurationProperties vendorListServiceV3Properties, + @Value("${gdpr.vendorlist.live-gvl-url}") String liveGvlUrl, + @Value("${gdpr.vendorlist.live-gvl-refresh-period-ms}") long refreshPeriodMs, + @Value("${gdpr.vendorlist.default-timeout-ms}") int defaultTimeoutMs, + Vertx vertx, + HttpClient httpClient, + VendorListFileStore vendorListFileStore, + Metrics metrics, + JacksonMapper mapper, + Clock clock) { + + return new LiveVendorListService( + vendorListServiceV3Properties.getCacheDir(), + liveGvlUrl, + refreshPeriodMs, + defaultTimeoutMs, + vertx, + httpClient, + vendorListFileStore, + metrics, + mapper, + clock); + } + @Bean VersionedVendorListService versionedVendorListService(VendorListService vendorListServiceV2, - VendorListService vendorListServiceV3) { + VendorListService vendorListServiceV3, + LiveVendorListService liveVendorListService, + Clock clock) { - return new VersionedVendorListService(vendorListServiceV2, vendorListServiceV3); + return new VersionedVendorListService( + vendorListServiceV2, vendorListServiceV3, liveVendorListService, clock); } @Bean diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 55822047954..28fc07808f8 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -207,6 +207,8 @@ gdpr: eea-countries: at,bg,be,cy,cz,dk,ee,fi,fr,de,gr,hu,ie,it,lv,lt,lu,mt,nl,pl,pt,ro,sk,si,es,se,gb,is,no,li,ai,aw,pt,bm,aq,io,vg,ic,ky,fk,re,mw,gp,gf,yt,pf,tf,gl,pt,ms,an,bq,cw,sx,nc,pn,sh,pm,gs,tc,uk,wf vendorlist: default-timeout-ms: 2000 + live-gvl-refresh-period-ms: 86400000 + live-gvl-url: https://vendor-list.consensu.org/v3/vendor-list.json v2: http-endpoint-template: https://vendor-list.consensu.org/v2/archives/vendor-list-v{VERSION}.json refresh-missing-list-period-ms: 3600000 diff --git a/src/main/resources/metrics-config/prometheus-labels.yaml b/src/main/resources/metrics-config/prometheus-labels.yaml index b40139d6a8a..6941b1d7cb0 100644 --- a/src/main/resources/metrics-config/prometheus-labels.yaml +++ b/src/main/resources/metrics-config/prometheus-labels.yaml @@ -85,6 +85,10 @@ mappers: labels: tcf: ${0} status: ${1} + - match: privacy.tcf.vendorlist.latest.* + name: privacy.tcf.vendorlist.latest + labels: + status: ${0} - match: privacy.tcf.*.* name: privacy.tcf.${1} labels: diff --git a/src/test/java/org/prebid/server/metric/MetricsTest.java b/src/test/java/org/prebid/server/metric/MetricsTest.java index 8a56e279d30..1df98cedd68 100644 --- a/src/test/java/org/prebid/server/metric/MetricsTest.java +++ b/src/test/java/org/prebid/server/metric/MetricsTest.java @@ -1004,6 +1004,24 @@ public void updatePrivacyTcfVendorListFallbackMetricShouldIncrementMetric() { assertThat(metricRegistry.counter("privacy.tcf.v1.vendorlist.fallback").getCount()).isEqualTo(1); } + @Test + public void updatePrivacyTcfVendorListLatestOkMetricShouldIncrementMetric() { + // when + metrics.updatePrivacyTcfVendorListLatestOkMetric(); + + // then + assertThat(metricRegistry.counter("privacy.tcf.vendorlist.latest.ok").getCount()).isOne(); + } + + @Test + public void updatePrivacyTcfVendorListLatestErrorMetricShouldIncrementMetric() { + // when + metrics.updatePrivacyTcfVendorListLatestErrorMetric(); + + // then + assertThat(metricRegistry.counter("privacy.tcf.vendorlist.latest.err").getCount()).isOne(); + } + @Test public void shouldNotUpdateAccountMetricsIfVerbosityIsNone() { // given diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java new file mode 100644 index 00000000000..66676ed51ca --- /dev/null +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java @@ -0,0 +1,294 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import com.fasterxml.jackson.core.JsonProcessingException; +import io.vertx.core.Future; +import io.vertx.core.Promise; +import io.vertx.core.Vertx; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.prebid.server.VertxTest; +import org.prebid.server.metric.Metrics; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Feature; +import org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode; +import org.prebid.server.privacy.gdpr.vendorlist.proto.SpecialFeature; +import org.prebid.server.privacy.gdpr.vendorlist.proto.SpecialPurpose; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; +import org.prebid.server.vertx.httpclient.HttpClient; +import org.prebid.server.vertx.httpclient.model.HttpClientResponse; + +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.Arrays; +import java.util.Date; +import java.util.EnumSet; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode.ONE; +import static org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode.TWO; + +@ExtendWith(MockitoExtension.class) +public class LiveVendorListServiceTest extends VertxTest { + + private static final Instant NOW = Instant.parse("2024-06-01T12:00:00Z"); + private static final String CACHE_DIR = "/cache/dir"; + private static final String LIVE_GVL_URL = "https://example.com"; + private static final long REFRESH_PERIOD_MS = 1000; + + @Mock + private Vertx vertx; + @Mock + private HttpClient httpClient; + @Mock + private VendorListFileStore vendorListFileStore; + @Mock + private Metrics metrics; + + private LiveVendorListService target; + + @BeforeEach + public void setUp() { + target = new LiveVendorListService( + CACHE_DIR, + LIVE_GVL_URL, + REFRESH_PERIOD_MS, + 1000, + vertx, + httpClient, + vendorListFileStore, + metrics, + jacksonMapper, + Clock.fixed(NOW, ZoneOffset.UTC)); + } + + @Test + public void isDeletedShouldReturnFalseWhenFetchNeverSucceeded() { + // when and then + assertThat(target.isDeleted(1)).isFalse(); + assertThat(target.isDeleted(null)).isFalse(); + } + + @Test + public void isDeletedShouldReturnTrueWhenVendorIsDeletedInLiveVendorList() throws JsonProcessingException { + // given + final Vendor vendor = givenVendor(42, "2024-01-01T00:00:00Z"); + final String responseBody = mapper.writeValueAsString(givenVendorList(vendor)); + given(httpClient.get(anyString(), anyLong())) + .willReturn(Future.succeededFuture(HttpClientResponse.of(200, null, responseBody))); + + // when + target.refresh(); + + // then + assertThat(target.isDeleted(42)).isTrue(); + assertThat(target.isDeleted(99)).isFalse(); + } + + @Test + public void extractDeletedVendorIdsShouldReturnOnlyVendorsWithPastDeletedDate() { + // given + final VendorList vendorList = givenVendorList( + givenVendor(1, "2024-01-01T00:00:00Z"), + givenVendor(2, null), + givenVendor(3, "2025-01-01T00:00:00Z"), + givenVendor(4, "2024-06-01T12:00:00Z")); + + // when + final var deletedIds = target.extractDeletedVendorIds(vendorList); + + // then + assertThat(deletedIds).containsExactly(1); + } + + @Test + public void refreshShouldUpdateDeletedVendorIdsAndIncrementOkMetric() throws JsonProcessingException { + // given + final Vendor vendor = givenVendor(1, "2024-01-01T00:00:00Z"); + final String responseBody = mapper.writeValueAsString(givenVendorList(vendor)); + givenHttpClientReturnsResponse(200, responseBody); + + // when + target.refresh(); + + // then + assertThat(target.isDeleted(1)).isTrue(); + verify(metrics).updatePrivacyTcfVendorListLatestOkMetric(); + verify(metrics, never()).updatePrivacyTcfVendorListLatestErrorMetric(); + } + + @Test + public void refreshShouldReplaceDeletedVendorIdsOnSubsequentSuccessfulFetch() throws JsonProcessingException { + // given + final Vendor vendor = givenVendor(1, "2024-01-01T00:00:00Z"); + final String responseBody = mapper.writeValueAsString(givenVendorList(vendor)); + final Vendor vendor2 = givenVendor(2, "2024-02-01T00:00:00Z"); + final String responseBody2 = mapper.writeValueAsString(givenVendorList(vendor2)); + given(httpClient.get(anyString(), anyLong())) + .willReturn( + Future.succeededFuture(HttpClientResponse.of(200, null, responseBody)), + Future.succeededFuture(HttpClientResponse.of(200, null, responseBody2))); + + // when + target.refresh(); + target.refresh(); + + // then + assertThat(target.isDeleted(1)).isFalse(); + assertThat(target.isDeleted(2)).isTrue(); + } + + @Test + public void refreshShouldIncrementErrorMetricOnHttpFailure() { + // given + given(httpClient.get(anyString(), anyLong())) + .willReturn(Future.failedFuture(new RuntimeException("connection failed"))); + + // when + target.refresh(); + + // then + assertThat(target.isDeleted(1)).isFalse(); + verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); + verify(metrics, never()).updatePrivacyTcfVendorListLatestOkMetric(); + } + + @Test + public void refreshShouldIncrementErrorMetricOnNonOkStatus() { + // given + givenHttpClientReturnsResponse(503, "{}"); + + // when + target.refresh(); + + // then + assertThat(target.isDeleted(1)).isFalse(); + verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); + } + + @Test + public void refreshShouldIncrementErrorMetricOnInvalidJson() { + // given + givenHttpClientReturnsResponse(200, "invalid-json"); + + // when + target.refresh(); + + // then + assertThat(target.isDeleted(1)).isFalse(); + verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); + } + + @Test + public void refreshShouldIncrementErrorMetricOnInvalidVendorList() throws JsonProcessingException { + // given + final Vendor vendor = givenVendor(42, "2024-01-01T00:00:00Z") + .toBuilder().features(null).build(); + final String responseBody = mapper.writeValueAsString(givenVendorList(vendor)); + givenHttpClientReturnsResponse(200, responseBody); + + // when + target.refresh(); + + // then + assertThat(target.isDeleted(1)).isFalse(); + verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); + } + + @Test + public void initializeShouldLoadDeletedVendorsFromCachedVendorList() { + // given + final VendorList vendorList = givenVendorList(givenVendor(42, "2024-01-01T00:00:00Z")); + given(vendorListFileStore.getLatestVendorListFromCache(eq(CACHE_DIR))).willReturn(Optional.of(vendorList)); + + // when + target.initialize(Promise.promise()); + + // then + assertThat(target.isDeleted(42)).isTrue(); + assertThat(target.isDeleted(99)).isFalse(); + } + + @Test + public void initializeShouldSchedulePeriodicRefresh() { + // given + given(vendorListFileStore.getLatestVendorListFromCache(eq(CACHE_DIR))).willReturn(Optional.empty()); + + // when + target.initialize(Promise.promise()); + + // then + verify(vertx).setPeriodic(eq(0L), eq(REFRESH_PERIOD_MS), any()); + } + + @Test + public void initializeShouldCompleteInitializePromise() { + // given + given(vendorListFileStore.getLatestVendorListFromCache(eq(CACHE_DIR))).willReturn(Optional.empty()); + final Promise promise = Promise.promise(); + + // when + target.initialize(promise); + + // then + assertThat(promise.future().succeeded()).isTrue(); + } + + @Test + public void refreshShouldKeepLastGoodSetOnFailureAfterSuccessfulFetch() throws JsonProcessingException { + // given + final Vendor vendor = givenVendor(1, "2024-01-01T00:00:00Z"); + final String responseBody = mapper.writeValueAsString(givenVendorList(vendor)); + given(httpClient.get(anyString(), anyLong())) + .willReturn( + Future.succeededFuture(HttpClientResponse.of(200, null, responseBody)), + Future.failedFuture(new RuntimeException("connection failed"))); + + // when + target.refresh(); + target.refresh(); + + // then + assertThat(target.isDeleted(1)).isTrue(); + verify(metrics).updatePrivacyTcfVendorListLatestOkMetric(); + verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); + } + + private void givenHttpClientReturnsResponse(int statusCode, String response) { + given(httpClient.get(anyString(), anyLong())) + .willReturn(Future.succeededFuture(HttpClientResponse.of(statusCode, null, response))); + } + + private static Vendor givenVendor(int id, String deletedDate) { + return Vendor.builder() + .id(id) + .deletedDate(deletedDate != null ? Instant.parse(deletedDate) : null) + .purposes(EnumSet.of(ONE)) + .legIntPurposes(EnumSet.of(TWO)) + .flexiblePurposes(EnumSet.noneOf(PurposeCode.class)) + .specialPurposes(EnumSet.noneOf(SpecialPurpose.class)) + .features(EnumSet.noneOf(Feature.class)) + .specialFeatures(EnumSet.noneOf(SpecialFeature.class)) + .build(); + } + + private static VendorList givenVendorList(Vendor... vendors) { + return VendorList.of( + 1, + Date.from(Instant.parse("2020-08-20T16:05:24Z")), + Arrays.stream(vendors).collect(Collectors.toMap(Vendor::getId, Function.identity()))); + } +} diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStoreTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStoreTest.java new file mode 100644 index 00000000000..4a96f8f6dd3 --- /dev/null +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStoreTest.java @@ -0,0 +1,410 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import com.fasterxml.jackson.core.JsonProcessingException; +import io.vertx.core.AsyncResult; +import io.vertx.core.Future; +import io.vertx.core.Handler; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.file.FileProps; +import io.vertx.core.file.FileSystem; +import io.vertx.core.file.FileSystemException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.prebid.server.VertxTest; +import org.prebid.server.exception.PreBidException; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Feature; +import org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode; +import org.prebid.server.privacy.gdpr.vendorlist.proto.SpecialFeature; +import org.prebid.server.privacy.gdpr.vendorlist.proto.SpecialPurpose; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; + +import java.io.File; +import java.nio.file.Path; +import java.util.Date; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonMap; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode.ONE; +import static org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode.TWO; + +@ExtendWith(MockitoExtension.class) +public class VendorListFileStoreTest extends VertxTest { + + private static final String CACHE_DIR = "/cache/dir"; + private static final String FALLBACK_VENDOR_LIST_PATH = "fallback.json"; + private static final String GENERATION_VERSION = "v0"; + + @Mock + private FileSystem fileSystem; + + @TempDir + Path tempDir; + + private VendorListFileStore target; + + @BeforeEach + public void setUp() { + target = new VendorListFileStore(0, fileSystem, jacksonMapper); + } + + @Test + public void createCacheFromDiskShouldCreateCacheDirWhenItDoesNotExist() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of()); + + // when + target.createCacheFromDisk(CACHE_DIR); + + // then + verify(fileSystem).mkdirsBlocking(eq(CACHE_DIR)); + } + + @Test + public void createCacheFromDiskShouldCreateCacheDirWhenPathExistsButIsNotADirectory() { + // given + final FileProps fileProps = mock(FileProps.class); + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(true); + given(fileSystem.propsBlocking(eq(CACHE_DIR))).willReturn(fileProps); + given(fileProps.isDirectory()).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of()); + + // when + target.createCacheFromDisk(CACHE_DIR); + + // then + verify(fileSystem).mkdirsBlocking(eq(CACHE_DIR)); + } + + @Test + public void createCacheFromDiskShouldFailIfCannotCreateCacheDir() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.mkdirsBlocking(eq(CACHE_DIR))) + .willThrow(new FileSystemException("dir creation error")); + + // when and then + assertThatThrownBy(() -> target.createCacheFromDisk(CACHE_DIR)) + .isInstanceOf(PreBidException.class) + .hasMessage("Cannot create directory: " + CACHE_DIR); + } + + @Test + public void createCacheFromDiskShouldFailIfNoWritePermissionsForCacheDir() { + // given + final String cacheDir = tempDir.toString(); + tempDir.toFile().setWritable(false, false); + final FileProps fileProps = mock(FileProps.class); + given(fileSystem.existsBlocking(eq(cacheDir))).willReturn(true); + given(fileSystem.propsBlocking(eq(cacheDir))).willReturn(fileProps); + given(fileProps.isDirectory()).willReturn(true); + + // when and then + assertThatThrownBy(() -> target.createCacheFromDisk(cacheDir)) + .isInstanceOf(PreBidException.class) + .hasMessage("No write permissions for directory: " + cacheDir); + } + + @Test + public void createCacheFromDiskShouldReturnCacheLoadedFromJsonFiles() throws JsonProcessingException { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of("/cache/dir/1.json")); + given(fileSystem.readFileBlocking(eq("/cache/dir/1.json"))) + .willReturn(Buffer.buffer(mapper.writeValueAsString(givenVendorList()))); + + // when + final Map> cache = target.createCacheFromDisk(CACHE_DIR); + + // then + assertThat(cache).isEqualTo(singletonMap(1, givenVendorMap())); + } + + @Test + public void createCacheFromDiskShouldReturnEmptyCacheWhenCacheDirHasNoJsonFiles() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of()); + + // when + final Map> cache = target.createCacheFromDisk(CACHE_DIR); + + // then + assertThat(cache).isEmpty(); + } + + @Test + public void createCacheFromDiskShouldFailIfCannotReadCacheDir() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willThrow(new RuntimeException("read error")); + + // when and then + assertThatThrownBy(() -> target.createCacheFromDisk(CACHE_DIR)) + .isInstanceOf(RuntimeException.class) + .hasMessage("read error"); + } + + @Test + public void createCacheFromDiskShouldFailIfCannotReadAtLeastOneVendorListFile() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of("1.json")); + given(fileSystem.readFileBlocking(eq("1.json"))).willThrow(new RuntimeException("read error")); + + // when and then + assertThatThrownBy(() -> target.createCacheFromDisk(CACHE_DIR)) + .isInstanceOf(RuntimeException.class) + .hasMessage("read error"); + } + + @Test + public void createCacheFromDiskShouldFailIfAtLeastOneVendorListFileCannotBeParsed() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of("1.json")); + given(fileSystem.readFileBlocking(eq("1.json"))).willReturn(Buffer.buffer("invalid")); + + // when and then + assertThatThrownBy(() -> target.createCacheFromDisk(CACHE_DIR)) + .isInstanceOf(PreBidException.class) + .hasMessage("Cannot parse vendor list from: invalid"); + } + + @Test + public void saveToFileShouldCompleteWithVendorListResultWhenWriteSucceeds() throws JsonProcessingException { + // given + final VendorList vendorList = givenVendorList(); + final String vendorListAsString = mapper.writeValueAsString(vendorList); + final VendorListResult vendorListResult = VendorListResult.of(1, vendorListAsString, vendorList); + givenWriteFileSucceeds(); + + // when + final Future future = target.saveToFile(vendorListResult, CACHE_DIR, GENERATION_VERSION); + + // then + org.prebid.server.assertion.FutureAssertion.assertThat(future).succeededWith(vendorListResult); + } + + @Test + public void saveToFileShouldWriteToExpectedPathWithExpectedContent() throws JsonProcessingException { + // given + final VendorList vendorList = givenVendorList(); + final String vendorListAsString = mapper.writeValueAsString(vendorList); + final VendorListResult vendorListResult = VendorListResult.of(1, vendorListAsString, vendorList); + givenWriteFileSucceeds(); + + // when + target.saveToFile(vendorListResult, CACHE_DIR, GENERATION_VERSION); + + // then + final ArgumentCaptor pathCaptor = ArgumentCaptor.forClass(String.class); + final ArgumentCaptor bufferCaptor = ArgumentCaptor.forClass(Buffer.class); + verify(fileSystem).writeFile(pathCaptor.capture(), bufferCaptor.capture(), any()); + assertThat(pathCaptor.getValue()).isEqualTo(new File(CACHE_DIR, "1.json").getPath()); + assertThat(bufferCaptor.getValue().toString()).isEqualTo(vendorListAsString); + } + + @Test + public void saveToFileShouldFailWhenWriteFails() throws JsonProcessingException { + // given + final VendorList vendorList = givenVendorList(); + final String vendorListAsString = mapper.writeValueAsString(vendorList); + final VendorListResult vendorListResult = VendorListResult.of(1, vendorListAsString, vendorList); + final RuntimeException exception = new RuntimeException("write error"); + givenWriteFileFails(exception); + + // when + final Future future = target.saveToFile(vendorListResult, CACHE_DIR, GENERATION_VERSION); + + // then + org.prebid.server.assertion.FutureAssertion.assertThat(future) + .isFailed() + .isInstanceOf(RuntimeException.class) + .hasMessage("write error"); + } + + @Test + public void readFallbackVendorListShouldReturnNullWhenPathIsBlank() { + // when and then + assertThat(target.readFallbackVendorList(null)).isNull(); + assertThat(target.readFallbackVendorList("")).isNull(); + assertThat(target.readFallbackVendorList(" ")).isNull(); + } + + @Test + public void readFallbackVendorListShouldReturnVendorsWhenPathIsValid() throws JsonProcessingException { + // given + given(fileSystem.readFileBlocking(eq(FALLBACK_VENDOR_LIST_PATH))) + .willReturn(Buffer.buffer(mapper.writeValueAsString(givenVendorList()))); + + // when + final Map vendors = target.readFallbackVendorList(FALLBACK_VENDOR_LIST_PATH); + + // then + assertThat(vendors).isEqualTo(givenVendorMap()); + } + + @Test + public void readFallbackVendorListShouldFailIfCannotReadFallbackFile() { + // given + given(fileSystem.readFileBlocking(eq(FALLBACK_VENDOR_LIST_PATH))) + .willThrow(new RuntimeException("read error")); + + // when and then + assertThatThrownBy(() -> target.readFallbackVendorList(FALLBACK_VENDOR_LIST_PATH)) + .isInstanceOf(RuntimeException.class) + .hasMessage("read error"); + } + + @Test + public void readFallbackVendorListShouldFailIfFallbackCannotBeParsed() { + // given + given(fileSystem.readFileBlocking(eq(FALLBACK_VENDOR_LIST_PATH))) + .willReturn(Buffer.buffer("invalid")); + + // when and then + assertThatThrownBy(() -> target.readFallbackVendorList(FALLBACK_VENDOR_LIST_PATH)) + .isInstanceOf(PreBidException.class) + .hasMessage("Cannot parse vendor list from: invalid"); + } + + @Test + public void getLatestVendorListFromCacheShouldReturnEmptyWhenCacheDirHasNoJsonFiles() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of()); + + // when + final Optional result = target.getLatestVendorListFromCache(CACHE_DIR); + + // then + assertThat(result).isEmpty(); + } + + @Test + public void getLatestVendorListFromCacheShouldReturnVendorListWithHighestVersion() throws JsonProcessingException { + // given + final VendorList vendorList = givenVendorList(10); + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of( + "/cache/dir/2.json", + "/cache/dir/10.json")); + given(fileSystem.readFileBlocking(eq("/cache/dir/10.json"))) + .willReturn(Buffer.buffer(mapper.writeValueAsString(vendorList))); + + // when + final Optional result = target.getLatestVendorListFromCache(CACHE_DIR); + + // then + assertThat(result).hasValue(vendorList); + verify(fileSystem, never()).readFileBlocking(eq("/cache/dir/2.json")); + } + + @Test + public void getLatestVendorListFromCacheShouldCreateCacheDirWhenItDoesNotExist() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of()); + + // when + target.getLatestVendorListFromCache(CACHE_DIR); + + // then + verify(fileSystem).mkdirsBlocking(eq(CACHE_DIR)); + } + + @Test + public void getLatestVendorListFromCacheShouldFailIfCannotReadCacheDir() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willThrow(new RuntimeException("read error")); + + // when and then + assertThatThrownBy(() -> target.getLatestVendorListFromCache(CACHE_DIR)) + .isInstanceOf(RuntimeException.class) + .hasMessage("read error"); + } + + @Test + public void getLatestVendorListFromCacheShouldFailIfLatestVendorListFileCannotBeParsed() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of("/cache/dir/1.json")); + given(fileSystem.readFileBlocking(eq("/cache/dir/1.json"))).willReturn(Buffer.buffer("invalid")); + + // when and then + assertThatThrownBy(() -> target.getLatestVendorListFromCache(CACHE_DIR)) + .isInstanceOf(PreBidException.class) + .hasMessage("Cannot parse vendor list from: invalid"); + } + + @Test + public void readFallbackVendorListShouldFailIfFallbackHasInvalidData() throws JsonProcessingException { + // given + final VendorList invalidVendorList = VendorList.of(1, new Date(), emptyMap()); + given(fileSystem.readFileBlocking(eq(FALLBACK_VENDOR_LIST_PATH))) + .willReturn(Buffer.buffer(mapper.writeValueAsString(invalidVendorList))); + + // when and then + assertThatThrownBy(() -> target.readFallbackVendorList(FALLBACK_VENDOR_LIST_PATH)) + .isInstanceOf(PreBidException.class) + .hasMessageStartingWith("Fallback vendor list parsed but has invalid data:"); + } + + private void givenWriteFileSucceeds() { + given(fileSystem.writeFile(anyString(), any(Buffer.class), any())).willAnswer(invocation -> { + final Handler> handler = invocation.getArgument(2); + handler.handle(Future.succeededFuture()); + return null; + }); + } + + private void givenWriteFileFails(Throwable throwable) { + given(fileSystem.writeFile(anyString(), any(Buffer.class), any())).willAnswer(invocation -> { + final Handler> handler = invocation.getArgument(2); + handler.handle(Future.failedFuture(throwable)); + return null; + }); + } + + private static VendorList givenVendorList() { + return givenVendorList(1); + } + + private static VendorList givenVendorList(int version) { + return VendorList.of(version, new Date(), givenVendorMap()); + } + + private static Map givenVendorMap() { + final Vendor vendor = Vendor.builder() + .id(52) + .purposes(EnumSet.of(ONE)) + .legIntPurposes(EnumSet.of(TWO)) + .flexiblePurposes(EnumSet.noneOf(PurposeCode.class)) + .specialPurposes(EnumSet.noneOf(SpecialPurpose.class)) + .features(EnumSet.noneOf(Feature.class)) + .specialFeatures(EnumSet.noneOf(SpecialFeature.class)) + .build(); + return singletonMap(52, vendor); + } +} diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListServiceTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListServiceTest.java index 82ef85987d2..9be7e32d546 100644 --- a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListServiceTest.java +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListServiceTest.java @@ -2,17 +2,13 @@ import com.fasterxml.jackson.core.JsonProcessingException; import io.vertx.core.Future; -import io.vertx.core.Handler; import io.vertx.core.Vertx; -import io.vertx.core.buffer.Buffer; -import io.vertx.core.file.FileSystem; import org.apache.commons.lang3.StringUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.stubbing.Answer; import org.prebid.server.VertxTest; import org.prebid.server.bidder.BidderCatalog; import org.prebid.server.exception.PreBidException; @@ -26,7 +22,6 @@ import org.prebid.server.vertx.httpclient.HttpClient; import org.prebid.server.vertx.httpclient.model.HttpClientResponse; -import java.io.File; import java.util.Date; import java.util.EnumSet; import java.util.HashMap; @@ -34,7 +29,6 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.singleton; -import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; @@ -42,9 +36,11 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.BDDMockito.given; import static org.mockito.Mock.Strictness.LENIENT; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.prebid.server.assertion.FutureAssertion.assertThat; @@ -61,8 +57,6 @@ public class VendorListServiceTest extends VertxTest { @Mock private Vertx vertx; - @Mock - private FileSystem fileSystem; @Mock(strictness = LENIENT) private HttpClient httpClient; @Mock @@ -71,19 +65,17 @@ public class VendorListServiceTest extends VertxTest { private BidderCatalog bidderCatalog; @Mock(strictness = LENIENT) private VendorListFetchThrottler fetchThrottler; + @Mock(strictness = LENIENT) + private VendorListFileStore vendorListFileStore; private VendorListService target; @BeforeEach public void setUp() throws JsonProcessingException { - given(fileSystem.existsBlocking(anyString())).willReturn(false); // always create cache dir - given(bidderCatalog.knownVendorIds()).willReturn(singleton(52)); - - given(fileSystem.readFileBlocking(eq(FALLBACK_VENDOR_LIST_PATH))) - .willReturn(Buffer.buffer(mapper.writeValueAsString(givenVendorList()))); - given(fetchThrottler.registerFetchAttempt(anyInt())).willReturn(true); + given(vendorListFileStore.readFallbackVendorList(isNull())).willReturn(null); + given(vendorListFileStore.readFallbackVendorList(eq(FALLBACK_VENDOR_LIST_PATH))).willReturn(givenVendorMap()); target = new VendorListService( 0, @@ -94,44 +86,72 @@ public void setUp() throws JsonProcessingException { false, FALLBACK_VENDOR_LIST_PATH, vertx, - fileSystem, httpClient, metrics, GENERATION_VERSION, - jacksonMapper, - fetchThrottler); + fetchThrottler, + vendorListFileStore, + jacksonMapper); } // Creation related tests @Test - public void creationShouldFailsIfCannotCreateCacheDir() { - // given - given(fileSystem.mkdirsBlocking(anyString())).willThrow(new RuntimeException("dir creation error")); + public void creationShouldCreateCacheFromDisk() { + reset(vendorListFileStore); + + // given and when + new VendorListService( + 0, + CACHE_DIR, + "http://vendorlist/%s", + 0, + REFRESH_MISSING_LIST_PERIOD_MS, + false, + FALLBACK_VENDOR_LIST_PATH, + vertx, + httpClient, + metrics, + GENERATION_VERSION, + fetchThrottler, + vendorListFileStore, + jacksonMapper); // then - assertThatThrownBy( - () -> new VendorListService( - 0, - CACHE_DIR, - "http://vendorlist/%s", - 0, - REFRESH_MISSING_LIST_PERIOD_MS, - false, - FALLBACK_VENDOR_LIST_PATH, - vertx, - fileSystem, - httpClient, - metrics, - GENERATION_VERSION, - jacksonMapper, - fetchThrottler)) - .hasMessage("dir creation error"); + verify(vendorListFileStore).createCacheFromDisk(eq(CACHE_DIR)); + } + + @Test + public void creationShouldFailIfCannotCreateCache() { + // given + given(vendorListFileStore.createCacheFromDisk(anyString())) + .willThrow(new RuntimeException("error creating cache")); + + // when and then + assertThatThrownBy(() -> new VendorListService( + 0, + CACHE_DIR, + "http://vendorlist/{VERSION}", + 0, + REFRESH_MISSING_LIST_PERIOD_MS, + true, + null, + vertx, + httpClient, + metrics, + GENERATION_VERSION, + fetchThrottler, + vendorListFileStore, + jacksonMapper)) + .isInstanceOf(RuntimeException.class) + .hasMessage("error creating cache"); } @Test public void shouldStartUsingFallbackVersionIfDeprecatedIsTrue() { // given + given(vendorListFileStore.readFallbackVendorList(anyString())).willReturn(givenVendorMap()); + target = new VendorListService( 0, CACHE_DIR, @@ -141,12 +161,12 @@ public void shouldStartUsingFallbackVersionIfDeprecatedIsTrue() { true, FALLBACK_VENDOR_LIST_PATH, vertx, - fileSystem, httpClient, metrics, GENERATION_VERSION, - jacksonMapper, - fetchThrottler); + fetchThrottler, + vendorListFileStore, + jacksonMapper); // when final Future> future = target.forVersion(1); @@ -177,96 +197,16 @@ public void shouldThrowExceptionIfVersionIsDeprecatedAndNoFallbackPresent() { true, null, vertx, - fileSystem, httpClient, metrics, GENERATION_VERSION, - jacksonMapper, - fetchThrottler)) + fetchThrottler, + vendorListFileStore, + jacksonMapper)) .isInstanceOf(PreBidException.class) .hasMessage("No fallback vendorList for deprecated version present"); } - @Test - public void creationShouldFailsIfCannotReadFiles() { - // given - given(fileSystem.readDirBlocking(anyString())).willThrow(new RuntimeException("read error")); - - // then - assertThatThrownBy( - () -> new VendorListService( - 0, - CACHE_DIR, - "http://vendorlist/%s", - 0, - REFRESH_MISSING_LIST_PERIOD_MS, - false, - FALLBACK_VENDOR_LIST_PATH, - vertx, - fileSystem, - httpClient, - metrics, - GENERATION_VERSION, - jacksonMapper, - fetchThrottler)) - .isInstanceOf(RuntimeException.class) - .hasMessage("read error"); - } - - @Test - public void creationShouldFailsIfCannotReadAtLeastOneVendorListFile() { - // given - given(fileSystem.readDirBlocking(anyString())).willReturn(singletonList("1.json")); - given(fileSystem.readFileBlocking(anyString())).willThrow(new RuntimeException("read error")); - - // then - assertThatThrownBy( - () -> new VendorListService( - 0, - CACHE_DIR, - "http://vendorlist/%s", - 0, - REFRESH_MISSING_LIST_PERIOD_MS, - false, - FALLBACK_VENDOR_LIST_PATH, - vertx, - fileSystem, - httpClient, - metrics, - GENERATION_VERSION, - jacksonMapper, - fetchThrottler)) - .isInstanceOf(RuntimeException.class) - .hasMessage("read error"); - } - - @Test - public void creationShouldFailsIfAtLeastOneVendorListFileCannotBeParsed() { - // given - given(fileSystem.readDirBlocking(anyString())).willReturn(singletonList("1.json")); - given(fileSystem.readFileBlocking(anyString())).willReturn(Buffer.buffer("invalid")); - - // then - assertThatThrownBy( - () -> new VendorListService( - 0, - CACHE_DIR, - "http://vendorlist/%s", - 0, - REFRESH_MISSING_LIST_PERIOD_MS, - false, - FALLBACK_VENDOR_LIST_PATH, - vertx, - fileSystem, - httpClient, - metrics, - GENERATION_VERSION, - jacksonMapper, - fetchThrottler)) - .isInstanceOf(PreBidException.class) - .hasMessage("Cannot parse vendor list from: invalid"); - } - // Http related tests @Test @@ -293,11 +233,11 @@ public void shouldNotPerformHttpRequestIfVendorListNotFoundAndFetchNotAllowed() // then verify(httpClient, never()).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfReadingHttpResponseFails() { + public void shouldNotTryToSaveFileIfReadingHttpResponseFails() { // given givenHttpClientProducesException(new RuntimeException("Response exception")); @@ -306,11 +246,11 @@ public void shouldNotAskToSaveFileIfReadingHttpResponseFails() { // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfResponseCodeIsNot200() { + public void shouldNotTryToSaveFileIfResponseCodeIsNot200() { // given givenHttpClientReturnsResponse(503, null); @@ -319,11 +259,11 @@ public void shouldNotAskToSaveFileIfResponseCodeIsNot200() { // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfResponseBodyCouldNotBeParsed() { + public void shouldNotTryToSaveFileIfResponseBodyCouldNotBeParsed() { // given givenHttpClientReturnsResponse(200, "response"); @@ -332,11 +272,11 @@ public void shouldNotAskToSaveFileIfResponseBodyCouldNotBeParsed() { // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfFetchedVendorListHasInvalidVendorListVersion() throws JsonProcessingException { + public void shouldNotTryToSaveFileIfFetchedVendorListHasInvalidVendorListVersion() throws JsonProcessingException { // given final VendorList vendorList = VendorList.of(null, null, null); givenHttpClientReturnsResponse(200, mapper.writeValueAsString(vendorList)); @@ -346,11 +286,11 @@ public void shouldNotAskToSaveFileIfFetchedVendorListHasInvalidVendorListVersion // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfFetchedVendorListHasInvalidLastUpdated() throws JsonProcessingException { + public void shouldNotTryToSaveFileIfFetchedVendorListHasInvalidLastUpdated() throws JsonProcessingException { // given final VendorList vendorList = VendorList.of(1, null, null); givenHttpClientReturnsResponse(200, mapper.writeValueAsString(vendorList)); @@ -360,11 +300,11 @@ public void shouldNotAskToSaveFileIfFetchedVendorListHasInvalidLastUpdated() thr // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfFetchedVendorListHasNoVendors() throws JsonProcessingException { + public void shouldNotTryToSaveFileIfFetchedVendorListHasNoVendors() throws JsonProcessingException { // given final VendorList vendorList = VendorList.of(1, new Date(), null); givenHttpClientReturnsResponse(200, mapper.writeValueAsString(vendorList)); @@ -374,11 +314,11 @@ public void shouldNotAskToSaveFileIfFetchedVendorListHasNoVendors() throws JsonP // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfFetchedVendorListHasEmptyVendors() throws JsonProcessingException { + public void shouldNotTryToSaveFileIfFetchedVendorListHasEmptyVendors() throws JsonProcessingException { // given final VendorList vendorList = VendorList.of(1, new Date(), emptyMap()); givenHttpClientReturnsResponse(200, mapper.writeValueAsString(vendorList)); @@ -388,11 +328,11 @@ public void shouldNotAskToSaveFileIfFetchedVendorListHasEmptyVendors() throws Js // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfFetchedVendorListHasAtLeastOneInvalidVendor() throws JsonProcessingException { + public void shouldNotTryToSaveFileIfFetchedVendorListHasAtLeastOneInvalidVendor() throws JsonProcessingException { // given final VendorList vendorList = VendorList.of(1, new Date(), singletonMap(1, Vendor.builder().build())); givenHttpClientReturnsResponse(200, mapper.writeValueAsString(vendorList)); @@ -402,7 +342,7 @@ public void shouldNotAskToSaveFileIfFetchedVendorListHasAtLeastOneInvalidVendor( // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } // File system related tests @@ -410,16 +350,20 @@ public void shouldNotAskToSaveFileIfFetchedVendorListHasAtLeastOneInvalidVendor( @Test public void shouldSaveFileWithExpectedPathAndContentIfVendorListNotFound() throws JsonProcessingException { // given - final String vendorListAsString = mapper.writeValueAsString(givenVendorList()); + final VendorList vendorList = givenVendorList(); + final String vendorListAsString = mapper.writeValueAsString(vendorList); givenHttpClientReturnsResponse(200, vendorListAsString); - // generate file path to avoid conflicts with path separators in different OS - final String filePath = new File("/cache/dir/1.json").getPath(); + given(vendorListFileStore.saveToFile(any(), anyString(), anyString())) + .willAnswer(inv -> Future.succeededFuture(inv.getArgument(0))); // when target.forVersion(1); // then - verify(fileSystem).writeFile(eq(filePath), eq(Buffer.buffer(vendorListAsString)), any()); + verify(vendorListFileStore).saveToFile( + eq(VendorListResult.of(1, vendorListAsString, vendorList)), + eq(CACHE_DIR), + eq(GENERATION_VERSION)); } // In-memory cache related tests @@ -456,8 +400,8 @@ public void shouldReturnVendorListFromCache() throws JsonProcessingException { // given givenHttpClientReturnsResponse(200, mapper.writeValueAsString(givenVendorList())); - given(fileSystem.writeFile(anyString(), any(), any())) - .willAnswer(withSelfAndPassObjectToHandler(Future.succeededFuture())); + given(vendorListFileStore.saveToFile(any(), anyString(), anyString())) + .willAnswer(inv -> Future.succeededFuture(inv.getArgument(0))); // when target.forVersion(1); // populate cache @@ -504,8 +448,8 @@ public void shouldKeepPurposesForAllVendors() throws JsonProcessingException { final VendorList vendorList = VendorList.of(1, new Date(), idToVendor); givenHttpClientReturnsResponse(200, mapper.writeValueAsString(vendorList)); - given(fileSystem.writeFile(anyString(), any(), any())) - .willAnswer(withSelfAndPassObjectToHandler(Future.succeededFuture())); + given(vendorListFileStore.saveToFile(any(), anyString(), anyString())) + .willAnswer(inv -> Future.succeededFuture(inv.getArgument(0))); // when target.forVersion(1); // populate cache @@ -572,8 +516,8 @@ public void shouldIncrementVendorListErrorMetricWhenFileIsNotSaved() throws Json // given givenHttpClientReturnsResponse(200, mapper.writeValueAsString(givenVendorList())); - given(fileSystem.writeFile(anyString(), any(), any())) - .willAnswer(withSelfAndPassObjectToHandler(Future.failedFuture("error"))); + given(vendorListFileStore.saveToFile(any(), anyString(), anyString())) + .willReturn(Future.failedFuture("error")); // when target.forVersion(1); @@ -587,8 +531,8 @@ public void shouldIncrementVendorListOkMetric() throws JsonProcessingException { // given givenHttpClientReturnsResponse(200, mapper.writeValueAsString(givenVendorList())); - given(fileSystem.writeFile(anyString(), any(), any())) - .willAnswer(withSelfAndPassObjectToHandler(Future.succeededFuture())); + given(vendorListFileStore.saveToFile(any(), anyString(), anyString())) + .willAnswer(inv -> Future.succeededFuture(inv.getArgument(0))); // when target.forVersion(1); @@ -614,6 +558,10 @@ public void shouldIncrementVendorListFallbackMetric() { } private static VendorList givenVendorList() { + return VendorList.of(1, new Date(), givenVendorMap()); + } + + private static Map givenVendorMap() { final Vendor vendor = Vendor.builder() .id(52) .purposes(EnumSet.of(ONE)) @@ -623,7 +571,7 @@ private static VendorList givenVendorList() { .features(EnumSet.noneOf(Feature.class)) .specialFeatures(EnumSet.noneOf(SpecialFeature.class)) .build(); - return VendorList.of(1, new Date(), singletonMap(52, vendor)); + return singletonMap(52, vendor); } private void givenHttpClientReturnsResponse(int statusCode, String response) { @@ -635,13 +583,4 @@ private void givenHttpClientProducesException(Throwable throwable) { given(httpClient.get(anyString(), anyLong())) .willReturn(Future.failedFuture(throwable)); } - - @SuppressWarnings("unchecked") - private static Answer withSelfAndPassObjectToHandler(T obj) { - return inv -> { - // invoking handler right away passing mock to it - ((Handler) inv.getArgument(2)).handle(obj); - return inv.getMock(); - }; - } } diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtilTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtilTest.java new file mode 100644 index 00000000000..8ff016e924b --- /dev/null +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtilTest.java @@ -0,0 +1,192 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.junit.jupiter.api.Test; +import org.prebid.server.VertxTest; +import org.prebid.server.exception.PreBidException; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Feature; +import org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode; +import org.prebid.server.privacy.gdpr.vendorlist.proto.SpecialFeature; +import org.prebid.server.privacy.gdpr.vendorlist.proto.SpecialPurpose; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; + +import java.util.Date; +import java.util.EnumSet; +import java.util.Map; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonMap; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode.ONE; +import static org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode.TWO; + +public class VendorListUtilTest extends VertxTest { + + @Test + public void parseVendorListShouldReturnVendorListWhenContentIsValid() throws JsonProcessingException { + // given + final VendorList vendorList = givenVendorList(); + final String content = mapper.writeValueAsString(vendorList); + + // when + final VendorList result = VendorListUtil.parseVendorList(content, jacksonMapper); + + // then + assertThat(result).isEqualTo(vendorList); + } + + @Test + public void parseVendorListShouldThrowExceptionWhenContentCannotBeParsed() { + // when and then + assertThatThrownBy(() -> VendorListUtil.parseVendorList("invalid", jacksonMapper)) + .isInstanceOf(PreBidException.class) + .hasMessage("Cannot parse vendor list from: invalid"); + } + + @Test + public void vendorListIsValidShouldReturnTrueWhenVendorListIsValid() { + // when and then + assertThat(VendorListUtil.vendorListIsValid(givenVendorList())).isTrue(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorListVersionIsNull() { + // given + final VendorList vendorList = VendorList.of(null, new Date(), givenVendorMap()); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenLastUpdatedIsNull() { + // given + final VendorList vendorList = VendorList.of(1, null, givenVendorMap()); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorsIsNull() { + // given + final VendorList vendorList = VendorList.of(1, new Date(), null); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorsIsEmpty() { + // given + final VendorList vendorList = VendorList.of(1, new Date(), emptyMap()); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorIsNull() { + // given + final VendorList vendorList = VendorList.of(1, new Date(), singletonMap(1, null)); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorIdIsNull() { + // given + final Vendor vendor = givenVendor().toBuilder().id(null).build(); + final VendorList vendorList = givenVendorListWithVendor(vendor); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorPurposesIsNull() { + // given + final Vendor vendor = givenVendor().toBuilder().purposes(null).build(); + final VendorList vendorList = givenVendorListWithVendor(vendor); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorLegIntPurposesIsNull() { + // given + final Vendor vendor = givenVendor().toBuilder().legIntPurposes(null).build(); + final VendorList vendorList = givenVendorListWithVendor(vendor); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorFlexiblePurposesIsNull() { + // given + final Vendor vendor = givenVendor().toBuilder().flexiblePurposes(null).build(); + final VendorList vendorList = givenVendorListWithVendor(vendor); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorSpecialPurposesIsNull() { + // given + final Vendor vendor = givenVendor().toBuilder().specialPurposes(null).build(); + final VendorList vendorList = givenVendorListWithVendor(vendor); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorFeaturesIsNull() { + // given + final Vendor vendor = givenVendor().toBuilder().features(null).build(); + final VendorList vendorList = givenVendorListWithVendor(vendor); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorSpecialFeaturesIsNull() { + // given + final Vendor vendor = givenVendor().toBuilder().specialFeatures(null).build(); + final VendorList vendorList = givenVendorListWithVendor(vendor); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + private static VendorList givenVendorList() { + return VendorList.of(1, new Date(), givenVendorMap()); + } + + private static VendorList givenVendorListWithVendor(Vendor vendor) { + return VendorList.of(1, new Date(), singletonMap(vendor.getId() != null ? vendor.getId() : 1, vendor)); + } + + private static Map givenVendorMap() { + return singletonMap(52, givenVendor()); + } + + private static Vendor givenVendor() { + return Vendor.builder() + .id(52) + .purposes(EnumSet.of(ONE)) + .legIntPurposes(EnumSet.of(TWO)) + .flexiblePurposes(EnumSet.noneOf(PurposeCode.class)) + .specialPurposes(EnumSet.noneOf(SpecialPurpose.class)) + .features(EnumSet.noneOf(Feature.class)) + .specialFeatures(EnumSet.noneOf(SpecialFeature.class)) + .build(); + } +} diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java index 8437698b9f0..2f20041192f 100644 --- a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java @@ -2,29 +2,45 @@ import com.iabtcf.decoder.TCString; import com.iabtcf.encoder.TCStringEncoder; +import io.vertx.core.Future; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.Map; import java.util.concurrent.ThreadLocalRandom; +import static java.util.Collections.emptyMap; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.BDDMockito.given; import static org.mockito.Mockito.verify; +import static org.prebid.server.assertion.FutureAssertion.assertThat; @ExtendWith(MockitoExtension.class) public class VersionedVendorListServiceTest { - private VersionedVendorListService versionedVendorListService; + private static final Instant NOW = Instant.parse("2024-06-01T12:00:00Z"); + + private VersionedVendorListService target; @Mock private VendorListService vendorListServiceV2; @Mock private VendorListService vendorListServiceV3; + @Mock + private LiveVendorListService liveVendorListService; @BeforeEach public void setUp() { - versionedVendorListService = new VersionedVendorListService(vendorListServiceV2, vendorListServiceV3); + target = new VersionedVendorListService( + vendorListServiceV2, vendorListServiceV3, liveVendorListService, Clock.fixed(NOW, ZoneOffset.UTC)); } @Test @@ -36,9 +52,10 @@ public void versionedVendorListServiceShouldTreatTcfPolicyLessThanFourAsVendorLi .tcfPolicyVersion(tcfPolicyVersion) .vendorListVersion(12) .toTCString(); + given(vendorListServiceV2.forVersion(anyInt())).willReturn(Future.succeededFuture(emptyMap())); // when - versionedVendorListService.forConsent(consent); + target.forConsent(consent); // then verify(vendorListServiceV2).forVersion(12); @@ -53,11 +70,65 @@ public void versionedVendorListServiceShouldTreatTcfPolicyGreaterOrEqualFourAsVe .tcfPolicyVersion(tcfPolicyVersion) .vendorListVersion(12) .toTCString(); + given(vendorListServiceV3.forVersion(anyInt())).willReturn(Future.succeededFuture(emptyMap())); // when - versionedVendorListService.forConsent(consent); + target.forConsent(consent); // then verify(vendorListServiceV3).forVersion(12); } + + @Test + public void forConsentShouldRemoveVendorsMarkedDeletedInRequestedGvl() { + // given + final Vendor deletedVendor = Vendor.empty(1).toBuilder() + .deletedDate(Instant.parse("2024-01-01T00:00:00Z")) + .build(); + final Vendor activeVendor = Vendor.empty(52); + final Map vendorList = Map.of(1, deletedVendor, 52, activeVendor); + final TCString consent = TCStringEncoder.newBuilder() + .version(2) + .tcfPolicyVersion(3) + .vendorListVersion(12) + .toTCString(); + + given(vendorListServiceV2.forVersion(anyInt())).willReturn(Future.succeededFuture(vendorList)); + given(liveVendorListService.isDeleted(anyInt())).willReturn(false); + + // when and then + assertThat(target.forConsent(consent)) + .isSucceeded() + .unwrap() + .satisfies(result -> { + assertThat(result).containsOnlyKeys(52); + assertThat(result.get(52)).isSameAs(activeVendor); + }); + } + + @Test + public void forConsentShouldRemoveVendorsMarkedDeletedInLiveGvl() { + // given + final Vendor deletedVendor = Vendor.empty(1); + final Vendor activeVendor = Vendor.empty(52); + final Map vendorList = Map.of(1, deletedVendor, 52, activeVendor); + final TCString consent = TCStringEncoder.newBuilder() + .version(2) + .tcfPolicyVersion(3) + .vendorListVersion(12) + .toTCString(); + + given(vendorListServiceV2.forVersion(anyInt())).willReturn(Future.succeededFuture(vendorList)); + given(liveVendorListService.isDeleted(1)).willReturn(true); + given(liveVendorListService.isDeleted(52)).willReturn(false); + + // when and then + assertThat(target.forConsent(consent)) + .isSucceeded() + .unwrap() + .satisfies(result -> { + assertThat(result).containsOnlyKeys(52); + assertThat(result.get(52)).isSameAs(activeVendor); + }); + } }