Skip to content

Commit 22555dd

Browse files
Lightwood13Lightwood13
committed
Added loading of live vendor list from disk cache on startup
1 parent 6c750b4 commit 22555dd

6 files changed

Lines changed: 182 additions & 17 deletions

File tree

‎src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,33 +24,39 @@ public class LiveVendorListService implements Initializable {
2424

2525
private static final Logger logger = LoggerFactory.getLogger(LiveVendorListService.class);
2626

27+
private final String cacheDir;
2728
private final String liveGvlUrl;
2829
private final long refreshPeriodMs;
2930
private final int defaultTimeoutMs;
3031
private final Vertx vertx;
3132
private final HttpClient httpClient;
32-
private final JacksonMapper mapper;
33+
private final VendorListFileStore vendorListFileStore;
3334
private final Metrics metrics;
35+
private final JacksonMapper mapper;
3436
private final Clock clock;
3537

3638
private volatile Set<Integer> deletedVendorIds = Set.of();
3739

38-
public LiveVendorListService(String liveGvlUrl,
40+
public LiveVendorListService(String cacheDir,
41+
String liveGvlUrl,
3942
long refreshPeriodMs,
4043
int defaultTimeoutMs,
4144
Vertx vertx,
4245
HttpClient httpClient,
43-
JacksonMapper mapper,
46+
VendorListFileStore vendorListFileStore,
4447
Metrics metrics,
48+
JacksonMapper mapper,
4549
Clock clock) {
4650

51+
this.cacheDir = Objects.requireNonNull(cacheDir);
4752
this.liveGvlUrl = HttpUtil.validateUrl(Objects.requireNonNull(liveGvlUrl));
4853
this.refreshPeriodMs = refreshPeriodMs;
4954
this.defaultTimeoutMs = defaultTimeoutMs;
5055
this.vertx = Objects.requireNonNull(vertx);
5156
this.httpClient = Objects.requireNonNull(httpClient);
52-
this.mapper = Objects.requireNonNull(mapper);
57+
this.vendorListFileStore = Objects.requireNonNull(vendorListFileStore);
5358
this.metrics = Objects.requireNonNull(metrics);
59+
this.mapper = Objects.requireNonNull(mapper);
5460
this.clock = Objects.requireNonNull(clock);
5561
}
5662

@@ -61,19 +67,31 @@ public boolean isDeleted(Integer id) {
6167

6268
@Override
6369
public void initialize(Promise<Void> initializePromise) {
70+
initializeWithLatestCachedVersion();
6471
vertx.setPeriodic(0, refreshPeriodMs, ignored -> refresh());
6572

6673
initializePromise.tryComplete();
6774
}
6875

76+
private void initializeWithLatestCachedVersion() {
77+
vendorListFileStore.getLatestVendorListFromCache(cacheDir).ifPresent(vendorList -> {
78+
saveDeletedVendorsFromVendorList(vendorList);
79+
logger.info("Initialized live GVL from cache with version %d".formatted(vendorList.getVendorListVersion()));
80+
});
81+
}
82+
6983
void refresh() {
7084
httpClient.get(liveGvlUrl, defaultTimeoutMs)
7185
.map(this::processResponse)
72-
.map(this::extractDeletedVendorIds)
73-
.map(this::updateDeletedVendorIds)
86+
.map(this::saveDeletedVendorsFromVendorList)
7487
.otherwise(this::handleError);
7588
}
7689

90+
private Void saveDeletedVendorsFromVendorList(VendorList vendorList) {
91+
updateDeletedVendorIds(extractDeletedVendorIds(vendorList));
92+
return null;
93+
}
94+
7795
private VendorList processResponse(HttpClientResponse response) {
7896
final int statusCode = response.getStatusCode();
7997
if (statusCode != 200) {

‎src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,10 @@
2020
import java.io.File;
2121
import java.nio.file.Files;
2222
import java.nio.file.Paths;
23+
import java.util.Comparator;
2324
import java.util.Map;
2425
import java.util.Objects;
26+
import java.util.Optional;
2527
import java.util.stream.Collectors;
2628

2729
public class VendorListFileStore {
@@ -46,16 +48,16 @@ public VendorListFileStore(double logSamplingRate,
4648

4749
Map<Integer, Map<Integer, Vendor>> createCacheFromDisk(String cacheDir) {
4850
createAndCheckWritePermissionsForCacheDir(cacheDir);
49-
final Map<String, String> versionToFileContent = readFileSystemCache(cacheDir);
51+
final Map<Integer, String> versionToFileContent = readFileSystemCache(cacheDir);
5052

5153
final Map<Integer, Map<Integer, Vendor>> cache = Caffeine.newBuilder()
5254
.<Integer, Map<Integer, Vendor>>build()
5355
.asMap();
5456

55-
for (Map.Entry<String, String> versionAndFileContent : versionToFileContent.entrySet()) {
57+
for (Map.Entry<Integer, String> versionAndFileContent : versionToFileContent.entrySet()) {
5658
final VendorList vendorList = VendorListUtil.parseVendorList(versionAndFileContent.getValue(), mapper);
5759

58-
cache.put(Integer.valueOf(versionAndFileContent.getKey()), vendorList.getVendors());
60+
cache.put(versionAndFileContent.getKey(), vendorList.getVendors());
5961
}
6062
return cache;
6163
}
@@ -73,13 +75,29 @@ private void createAndCheckWritePermissionsForCacheDir(String cacheDir) {
7375
}
7476
}
7577

76-
private Map<String, String> readFileSystemCache(String cacheDir) {
78+
private Map<Integer, String> readFileSystemCache(String cacheDir) {
7779
return fileSystem.readDirBlocking(cacheDir).stream()
7880
.filter(filepath -> filepath.endsWith(JSON_SUFFIX))
79-
.collect(Collectors.toMap(filepath -> StringUtils.removeEnd(new File(filepath).getName(), JSON_SUFFIX),
81+
.collect(Collectors.toMap(VendorListFileStore::parseCachedFileVersion,
8082
filename -> fileSystem.readFileBlocking(filename).toString()));
8183
}
8284

85+
Optional<VendorList> getLatestVendorListFromCache(String cacheDir) {
86+
createAndCheckWritePermissionsForCacheDir(cacheDir);
87+
return fileSystem.readDirBlocking(cacheDir).stream()
88+
.filter(filepath -> filepath.endsWith(JSON_SUFFIX))
89+
.max(Comparator.comparing(VendorListFileStore::parseCachedFileVersion))
90+
.map(fileSystem::readFileBlocking)
91+
.map(Buffer::toString)
92+
.map(content -> VendorListUtil.parseVendorList(content, mapper));
93+
}
94+
95+
private static Integer parseCachedFileVersion(String filepath) {
96+
final String filename = new File(filepath).getName();
97+
final String filenameWithoutExtension = StringUtils.removeEnd(filename, JSON_SUFFIX);
98+
return Integer.valueOf(filenameWithoutExtension);
99+
}
100+
83101
Future<VendorListResult> saveToFile(VendorListResult vendorListResult, String cacheDir, String generationVersion) {
84102
final Promise<VendorListResult> promise = Promise.promise();
85103
final int version = vendorListResult.getVersion();

‎src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ public Future<Map<Integer, Vendor>> forConsent(TCString consent) {
4040
}
4141

4242
private Map<Integer, Vendor> filterDeletedVendors(Map<Integer, Vendor> vendors) {
43-
Instant now = clock.instant();
43+
final Instant now = clock.instant();
4444
return vendors.entrySet().stream()
4545
.filter(entry -> !VendorListUtil.vendorIsDeletedAt(entry.getValue(), now))
4646
.filter(entry -> !liveVendorListService.isDeleted(entry.getKey()))

‎src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -148,23 +148,27 @@ VendorListServiceConfigurationProperties vendorListServiceV3Properties() {
148148

149149
@Bean
150150
LiveVendorListService liveVendorListService(
151+
VendorListServiceConfigurationProperties vendorListServiceV3Properties,
151152
@Value("${gdpr.vendorlist.live-gvl-url}") String liveGvlUrl,
152153
@Value("${gdpr.vendorlist.live-gvl-refresh-period-ms}") long refreshPeriodMs,
153154
@Value("${gdpr.vendorlist.default-timeout-ms}") int defaultTimeoutMs,
154155
Vertx vertx,
155156
HttpClient httpClient,
156-
JacksonMapper mapper,
157+
VendorListFileStore vendorListFileStore,
157158
Metrics metrics,
159+
JacksonMapper mapper,
158160
Clock clock) {
159161

160162
return new LiveVendorListService(
163+
vendorListServiceV3Properties.getCacheDir(),
161164
liveGvlUrl,
162165
refreshPeriodMs,
163166
defaultTimeoutMs,
164167
vertx,
165168
httpClient,
166-
mapper,
169+
vendorListFileStore,
167170
metrics,
171+
mapper,
168172
clock);
169173
}
170174

‎src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java‎

Lines changed: 51 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import com.fasterxml.jackson.core.JsonProcessingException;
44
import io.vertx.core.Future;
5+
import io.vertx.core.Promise;
56
import io.vertx.core.Vertx;
67
import org.junit.jupiter.api.BeforeEach;
78
import org.junit.jupiter.api.Test;
@@ -25,12 +26,15 @@
2526
import java.util.Arrays;
2627
import java.util.Date;
2728
import java.util.EnumSet;
29+
import java.util.Optional;
2830
import java.util.function.Function;
2931
import java.util.stream.Collectors;
3032

3133
import static org.assertj.core.api.Assertions.assertThat;
34+
import static org.mockito.ArgumentMatchers.any;
3235
import static org.mockito.ArgumentMatchers.anyLong;
3336
import static org.mockito.ArgumentMatchers.anyString;
37+
import static org.mockito.ArgumentMatchers.eq;
3438
import static org.mockito.BDDMockito.given;
3539
import static org.mockito.Mockito.never;
3640
import static org.mockito.Mockito.verify;
@@ -41,27 +45,33 @@
4145
public class LiveVendorListServiceTest extends VertxTest {
4246

4347
private static final Instant NOW = Instant.parse("2024-06-01T12:00:00Z");
48+
private static final String CACHE_DIR = "/cache/dir";
4449
private static final String LIVE_GVL_URL = "https://example.com";
50+
private static final long REFRESH_PERIOD_MS = 1000;
4551

4652
@Mock
4753
private Vertx vertx;
4854
@Mock
4955
private HttpClient httpClient;
5056
@Mock
57+
private VendorListFileStore vendorListFileStore;
58+
@Mock
5159
private Metrics metrics;
5260

5361
private LiveVendorListService target;
5462

5563
@BeforeEach
5664
public void setUp() {
5765
target = new LiveVendorListService(
66+
CACHE_DIR,
5867
LIVE_GVL_URL,
59-
0,
68+
REFRESH_PERIOD_MS,
6069
1000,
6170
vertx,
6271
httpClient,
63-
jacksonMapper,
72+
vendorListFileStore,
6473
metrics,
74+
jacksonMapper,
6575
Clock.fixed(NOW, ZoneOffset.UTC));
6676
}
6777

@@ -198,6 +208,45 @@ public void refreshShouldIncrementErrorMetricOnInvalidVendorList() throws JsonPr
198208
verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric();
199209
}
200210

211+
@Test
212+
public void initializeShouldLoadDeletedVendorsFromCachedVendorList() {
213+
// given
214+
final VendorList vendorList = givenVendorList(givenVendor(42, "2024-01-01T00:00:00Z"));
215+
given(vendorListFileStore.getLatestVendorListFromCache(eq(CACHE_DIR))).willReturn(Optional.of(vendorList));
216+
217+
// when
218+
target.initialize(Promise.promise());
219+
220+
// then
221+
assertThat(target.isDeleted(42)).isTrue();
222+
assertThat(target.isDeleted(99)).isFalse();
223+
}
224+
225+
@Test
226+
public void initializeShouldSchedulePeriodicRefresh() {
227+
// given
228+
given(vendorListFileStore.getLatestVendorListFromCache(eq(CACHE_DIR))).willReturn(Optional.empty());
229+
230+
// when
231+
target.initialize(Promise.promise());
232+
233+
// then
234+
verify(vertx).setPeriodic(eq(0L), eq(REFRESH_PERIOD_MS), any());
235+
}
236+
237+
@Test
238+
public void initializeShouldCompleteInitializePromise() {
239+
// given
240+
given(vendorListFileStore.getLatestVendorListFromCache(eq(CACHE_DIR))).willReturn(Optional.empty());
241+
final Promise<Void> promise = Promise.promise();
242+
243+
// when
244+
target.initialize(promise);
245+
246+
// then
247+
assertThat(promise.future().succeeded()).isTrue();
248+
}
249+
201250
@Test
202251
public void refreshShouldKeepLastGoodSetOnFailureAfterSuccessfulFetch() throws JsonProcessingException {
203252
// given

‎src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStoreTest.java‎

Lines changed: 77 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import java.util.EnumSet;
3131
import java.util.List;
3232
import java.util.Map;
33+
import java.util.Optional;
3334

3435
import static java.util.Collections.emptyMap;
3536
import static java.util.Collections.singletonMap;
@@ -40,6 +41,7 @@
4041
import static org.mockito.ArgumentMatchers.eq;
4142
import static org.mockito.BDDMockito.given;
4243
import static org.mockito.Mockito.mock;
44+
import static org.mockito.Mockito.never;
4345
import static org.mockito.Mockito.verify;
4446
import static org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode.ONE;
4547
import static org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode.TWO;
@@ -286,6 +288,76 @@ public void readFallbackVendorListShouldFailIfFallbackCannotBeParsed() {
286288
.hasMessage("Cannot parse vendor list from: invalid");
287289
}
288290

291+
@Test
292+
public void getLatestVendorListFromCacheShouldReturnEmptyWhenCacheDirHasNoJsonFiles() {
293+
// given
294+
given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false);
295+
given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of());
296+
297+
// when
298+
final Optional<VendorList> result = target.getLatestVendorListFromCache(CACHE_DIR);
299+
300+
// then
301+
assertThat(result).isEmpty();
302+
}
303+
304+
@Test
305+
public void getLatestVendorListFromCacheShouldReturnVendorListWithHighestVersion() throws JsonProcessingException {
306+
// given
307+
final VendorList vendorList = givenVendorList(10);
308+
given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false);
309+
given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of(
310+
"/cache/dir/2.json",
311+
"/cache/dir/10.json"));
312+
given(fileSystem.readFileBlocking(eq("/cache/dir/10.json")))
313+
.willReturn(Buffer.buffer(mapper.writeValueAsString(vendorList)));
314+
315+
// when
316+
final Optional<VendorList> result = target.getLatestVendorListFromCache(CACHE_DIR);
317+
318+
// then
319+
assertThat(result).hasValue(vendorList);
320+
verify(fileSystem, never()).readFileBlocking(eq("/cache/dir/2.json"));
321+
}
322+
323+
@Test
324+
public void getLatestVendorListFromCacheShouldCreateCacheDirWhenItDoesNotExist() {
325+
// given
326+
given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false);
327+
given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of());
328+
329+
// when
330+
target.getLatestVendorListFromCache(CACHE_DIR);
331+
332+
// then
333+
verify(fileSystem).mkdirsBlocking(eq(CACHE_DIR));
334+
}
335+
336+
@Test
337+
public void getLatestVendorListFromCacheShouldFailIfCannotReadCacheDir() {
338+
// given
339+
given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false);
340+
given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willThrow(new RuntimeException("read error"));
341+
342+
// when and then
343+
assertThatThrownBy(() -> target.getLatestVendorListFromCache(CACHE_DIR))
344+
.isInstanceOf(RuntimeException.class)
345+
.hasMessage("read error");
346+
}
347+
348+
@Test
349+
public void getLatestVendorListFromCacheShouldFailIfLatestVendorListFileCannotBeParsed() {
350+
// given
351+
given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false);
352+
given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of("/cache/dir/1.json"));
353+
given(fileSystem.readFileBlocking(eq("/cache/dir/1.json"))).willReturn(Buffer.buffer("invalid"));
354+
355+
// when and then
356+
assertThatThrownBy(() -> target.getLatestVendorListFromCache(CACHE_DIR))
357+
.isInstanceOf(PreBidException.class)
358+
.hasMessage("Cannot parse vendor list from: invalid");
359+
}
360+
289361
@Test
290362
public void readFallbackVendorListShouldFailIfFallbackHasInvalidData() throws JsonProcessingException {
291363
// given
@@ -316,7 +388,11 @@ private void givenWriteFileFails(Throwable throwable) {
316388
}
317389

318390
private static VendorList givenVendorList() {
319-
return VendorList.of(1, new Date(), givenVendorMap());
391+
return givenVendorList(1);
392+
}
393+
394+
private static VendorList givenVendorList(int version) {
395+
return VendorList.of(version, new Date(), givenVendorMap());
320396
}
321397

322398
private static Map<Integer, Vendor> givenVendorMap() {

0 commit comments

Comments
 (0)