Skip to content

Commit 53f60be

Browse files
pidoubleyoupidoubleyou
authored
Merge pull request #1143 from mediathekview/feature/1140
ard: search compilations on topic page
2 parents 00aec69 + 6367318 commit 53f60be

12 files changed

Lines changed: 4945 additions & 34 deletions

‎src/main/java/de/mediathekview/mserver/crawler/ard/ArdCrawler.java‎

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
import org.apache.logging.log4j.LogManager;
1414
import org.apache.logging.log4j.Logger;
1515

16-
import java.util.Arrays;
1716
import java.util.Collection;
1817
import java.util.HashSet;
1918
import java.util.List;
@@ -61,9 +60,9 @@ protected RecursiveTask<Set<Film>> createCrawlerTask() {
6160
ServerMessages.DEBUG_ALL_SENDUNG_FOLGEN_COUNT, getSender().getName(), shows.size());
6261

6362
if (Boolean.TRUE.equals(crawlerConfig.getTopicsSearchEnabled())) {
63+
final Set<CrawlerUrlDTO> senderTopicUrls = new HashSet<>();
6464
final Set<ForkJoinTask<Set<CrawlerUrlDTO>>> senderTopicTasks = createSenderTopicTasks();
6565

66-
final Set<CrawlerUrlDTO> senderTopicUrls = new HashSet<>();
6766
for (final ForkJoinTask<Set<CrawlerUrlDTO>> senderTopicTask : senderTopicTasks) {
6867
senderTopicUrls.addAll(senderTopicTask.get());
6968
}
@@ -75,11 +74,28 @@ protected RecursiveTask<Set<Film>> createCrawlerTask() {
7574

7675
final ArdTopicPageTask topicTask =
7776
new ArdTopicPageTask(this, new ConcurrentLinkedQueue<>(assitUrls));
78-
77+
78+
final Set<ArdFilmInfoDto> ardFilmInfosWithCompilations = forkJoinPool.submit(topicTask).get();
79+
80+
// add filmInfos without compilation
7981
final int showsCountBefore = shows.size();
80-
shows.addAll(forkJoinPool.submit(topicTask).get());
82+
shows.addAll(ardFilmInfosWithCompilations.stream().filter(filmInfo -> !filmInfo.isCompilation()).toList());
8183
LOG.debug(
82-
"ARD crawler found {} topics for all sub-sender.", shows.size() - showsCountBefore);
84+
"ARD crawler found {} topics excluding compilations for all sub-sender.", shows.size() - showsCountBefore);
85+
86+
// search compilations
87+
final List<ArdFilmInfoDto> compilations = ardFilmInfosWithCompilations.stream().filter(ArdFilmInfoDto::isCompilation).toList();
88+
LOG.debug(
89+
"ARD crawler found {} compilations for all sub-sender.", compilations.size());
90+
final ArdTopicCompilationTask compilationTask =
91+
new ArdTopicCompilationTask(this, new ConcurrentLinkedQueue<>(compilations));
92+
final Set<ArdFilmInfoDto> ardCompilationEntries = forkJoinPool.submit(compilationTask).get();
93+
LOG.debug(
94+
"ARD crawler found {} entries in compilations for all sub-sender.", ardCompilationEntries.size());
95+
96+
final int sizeBefore = shows.size();
97+
shows.addAll(ardCompilationEntries.stream().filter(filmInfo -> !filmInfo.isCompilation()).toList());
98+
LOG.debug("ARD crawler added {} entries from compilations to shows.", shows.size() - sizeBefore);
8399
}
84100
//
85101
final Queue<ArdFilmInfoDto> showsFiltered = this.filterExistingFilms(shows, ArdFilmInfoDto::getId);
@@ -116,13 +132,12 @@ private Set<ForkJoinTask<Set<CrawlerUrlDTO>>> createSenderTopicTasks() {
116132

117133
private ForkJoinTask<Set<CrawlerUrlDTO>> getTopicEntriesBySender(final String sender) throws ExecutionException, InterruptedException {
118134
Set<CrawlerUrlDTO> senderSingleLetterUrls = forkJoinPool.submit(
119-
new ArdTopicsTask(this, sender, CreateLetterUrlQuery(sender))).get();
135+
new ArdTopicsTask(this, sender, createLetterUrlQuery(sender))).get();
120136

121-
//LOG.debug("topics task result {}", senderSingleLetterUrls.size());
122137
return forkJoinPool.submit(new ArdTopicsLetterTask(this, sender, new ConcurrentLinkedQueue<>(senderSingleLetterUrls)));
123138
}
124139

125-
private Queue<CrawlerUrlDTO> CreateLetterUrlQuery(final String client) {
140+
private Queue<CrawlerUrlDTO> createLetterUrlQuery(final String client) {
126141
final Queue<CrawlerUrlDTO> urls = new ConcurrentLinkedQueue<>();
127142

128143
final String url = String.format(ArdConstants.TOPICS_URL, client);

‎src/main/java/de/mediathekview/mserver/crawler/ard/ArdFilmInfoDto.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,14 @@ public class ArdFilmInfoDto extends CrawlerUrlDTO {
88

99
private final String id;
1010
private final int numberOfClips;
11+
private final boolean isCompilation;
1112

12-
public ArdFilmInfoDto(final String id, final String aUrl, final int numberOfClips) {
13+
public ArdFilmInfoDto(final String id, final String aUrl, final int numberOfClips, final boolean isCompilation) {
1314
super(aUrl);
1415

1516
this.id = id;
1617
this.numberOfClips = numberOfClips;
18+
this.isCompilation = isCompilation;
1719
}
1820

1921
public String getId() {
@@ -24,6 +26,8 @@ public int getNumberOfClips() {
2426
return numberOfClips;
2527
}
2628

29+
public boolean isCompilation() { return isCompilation; }
30+
2731
@Override
2832
public boolean equals(final Object o) {
2933
if (this == o) {
@@ -35,11 +39,11 @@ public boolean equals(final Object o) {
3539
if (!super.equals(o)) {
3640
return false;
3741
}
38-
return numberOfClips == that.numberOfClips && Objects.equals(id, that.id);
42+
return numberOfClips == that.numberOfClips && Objects.equals(id, that.id) && isCompilation == that.isCompilation;
3943
}
4044

4145
@Override
4246
public int hashCode() {
43-
return Objects.hash(super.hashCode(), id, numberOfClips);
47+
return Objects.hash(super.hashCode(), id, numberOfClips, isCompilation);
4448
}
4549
}

‎src/main/java/de/mediathekview/mserver/crawler/ard/json/ArdDayPageDeserializer.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ private Set<ArdFilmInfoDto> parseChannels(JsonArray channels) {
4949

5050
private ArdFilmInfoDto createFilmInfo(final String id, final int numberOfClips) {
5151
final String url = String.format(ArdConstants.ITEM_URL, id);
52-
return new ArdFilmInfoDto(id, url, numberOfClips);
52+
return new ArdFilmInfoDto(id, url, numberOfClips, false);
5353
}
5454

5555
private Optional<String> toId(final JsonObject teaserObject) {

‎src/main/java/de/mediathekview/mserver/crawler/ard/json/ArdTeasersDeserializer.java‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,12 @@ abstract class ArdTeasersDeserializer {
2020

2121
private static final String ATTRIBUTE_ID = "id";
2222
private static final String ATTRIBUTE_NUMBER_OF_CLIPS = "numberOfClips";
23+
private static final String ATTRIBUTE_TYPE = "type";
2324

2425
private static final String ELEMENT_PUBLICATION_SERVICE = "publicationService";
2526
private static final String ATTRIBUTE_PARTNER = "partner";
26-
27+
private static final String ATTRIBUTE_HREF = "href";
28+
2729
Set<ArdFilmInfoDto> parseTeasers(final JsonArray teasers) {
2830
return StreamSupport.stream(teasers.spliterator(), true)
2931
.map(JsonElement::getAsJsonObject)
@@ -34,9 +36,23 @@ Set<ArdFilmInfoDto> parseTeasers(final JsonArray teasers) {
3436
}
3537

3638
private ArdFilmInfoDto toFilmInfo(final JsonObject teaserObject) {
37-
return toId(teaserObject)
38-
.map(id -> createFilmInfo(id, getNumberOfClips(teaserObject)))
39-
.orElse(null);
39+
final boolean compilation = isCompilation(teaserObject);
40+
if (compilation) {
41+
final Optional<String> url = JsonUtils.getElementValueAsString(teaserObject, ELEMENT_LINKS, ELEMENT_TARGET, ATTRIBUTE_HREF);
42+
final Optional<String> id = toId(teaserObject);
43+
return url.map(s -> new ArdFilmInfoDto(id.orElse(""), s, getNumberOfClips(teaserObject), compilation)).orElse(null);
44+
} else {
45+
return toId(teaserObject)
46+
.map(id -> createFilmInfo(id, getNumberOfClips(teaserObject), compilation))
47+
.orElse(null);
48+
}
49+
}
50+
51+
private boolean isCompilation(final JsonObject teaserObject) {
52+
if (teaserObject.has(ATTRIBUTE_TYPE)) {
53+
return "compilation".equals(teaserObject.get(ATTRIBUTE_TYPE).getAsString());
54+
}
55+
return false;
4056
}
4157

4258
private int getNumberOfClips(final JsonObject teaserObject) {
@@ -55,13 +71,13 @@ private Optional<String> toId(final JsonObject teaserObject) {
5571
return JsonUtils.getAttributeAsString(teaserObject, ATTRIBUTE_ID);
5672
}
5773

58-
private ArdFilmInfoDto createFilmInfo(final String id, final int numberOfClips) {
74+
private ArdFilmInfoDto createFilmInfo(final String id, final int numberOfClips, final boolean isCompilation) {
5975
String refId = id;
6076
if(id.contains(":")) {
6177
refId = id.replace(":", "%3A");
6278
}
6379
final String url = String.format(ArdConstants.ITEM_URL, refId);
64-
return new ArdFilmInfoDto(id, url, numberOfClips);
80+
return new ArdFilmInfoDto(id, url, numberOfClips, isCompilation);
6581
}
6682

6783
private boolean isRelevant(final JsonObject teaserObject) {

‎src/main/java/de/mediathekview/mserver/crawler/ard/json/ArdTopicCompilationDeserializer.java‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
package de.mediathekview.mserver.crawler.ard.json;
2+
3+
import com.google.gson.*;
4+
import de.mediathekview.mserver.crawler.ard.ArdFilmInfoDto;
5+
import de.mediathekview.mserver.crawler.ard.ArdTopicInfoDto;
6+
import java.lang.reflect.Type;
7+
import java.util.HashSet;
8+
import java.util.Set;
9+
10+
public class ArdTopicCompilationDeserializer extends ArdTeasersDeserializer
11+
implements JsonDeserializer<ArdTopicInfoDto> {
12+
13+
private static final String ELEMENT_WIDGETS = "widgets";
14+
private static final String ELEMENT_TEASERS = "teasers";
15+
private static final String ELEMENT_PAGE_NUMBER = "pageNumber";
16+
private static final String ELEMENT_TOTAL_ELEMENTS = "totalElements";
17+
private static final String ELEMENT_PAGE_SIZE = "pageSize";
18+
private static final String ELEMENT_PAGINATION = "pagination";
19+
20+
@Override
21+
public ArdTopicInfoDto deserialize(
22+
final JsonElement showPageElement, final Type type, final JsonDeserializationContext context) {
23+
final Set<ArdFilmInfoDto> results = new HashSet<>();
24+
final ArdTopicInfoDto ardTopicInfoDto = new ArdTopicInfoDto(results);
25+
26+
final JsonObject showPageObject = showPageElement.getAsJsonObject();
27+
if (showPageObject.has(ELEMENT_WIDGETS)) {
28+
final JsonArray widgets = showPageObject.get(ELEMENT_WIDGETS).getAsJsonArray();
29+
widgets.forEach(widget -> {
30+
if (widget.getAsJsonObject().has(ELEMENT_TEASERS)) {
31+
final JsonArray teasers = widget.getAsJsonObject().get(ELEMENT_TEASERS).getAsJsonArray();
32+
results.addAll(parseTeasers(teasers));
33+
}
34+
});
35+
}
36+
37+
final JsonElement paginationElement = showPageObject.get(ELEMENT_PAGINATION);
38+
final int pageNumber = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_NUMBER);
39+
final int totalElements = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_TOTAL_ELEMENTS);
40+
final int pageSize = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_SIZE);
41+
ardTopicInfoDto.setPageNumber(pageNumber);
42+
ardTopicInfoDto.setPageSize(pageSize);
43+
ardTopicInfoDto.setTotalElements(totalElements);
44+
return ardTopicInfoDto;
45+
}
46+
47+
private int getChildElementAsIntOrNullIfNotExist(
48+
final JsonElement parentElement, final String childElementName) {
49+
if (parentElement == null || parentElement.isJsonNull()) {
50+
return 0;
51+
}
52+
return getJsonElementAsIntOrNullIfNotExist(
53+
parentElement.getAsJsonObject().get(childElementName));
54+
}
55+
56+
private int getJsonElementAsIntOrNullIfNotExist(final JsonElement element) {
57+
if (element.isJsonNull()) {
58+
return 0;
59+
}
60+
return element.getAsInt();
61+
}
62+
}

‎src/main/java/de/mediathekview/mserver/crawler/ard/tasks/ArdTopicCompilationTask.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
package de.mediathekview.mserver.crawler.ard.tasks;
2+
3+
import com.google.gson.reflect.TypeToken;
4+
import de.mediathekview.mserver.crawler.ard.ArdFilmInfoDto;
5+
import de.mediathekview.mserver.crawler.ard.ArdTopicInfoDto;
6+
import de.mediathekview.mserver.crawler.ard.json.ArdTopicCompilationDeserializer;
7+
import de.mediathekview.mserver.crawler.basic.AbstractCrawler;
8+
import de.mediathekview.mserver.crawler.basic.AbstractRecursiveConverterTask;
9+
import de.mediathekview.mserver.crawler.basic.CrawlerUrlDTO;
10+
import jakarta.ws.rs.client.WebTarget;
11+
import java.lang.reflect.Type;
12+
import java.util.Queue;
13+
14+
public class ArdTopicCompilationTask extends ArdTaskBase<ArdFilmInfoDto, CrawlerUrlDTO> {
15+
16+
private static final Type ARDTOPICINFODTO_TYPE_TOKEN =
17+
new TypeToken<ArdTopicInfoDto>() {}.getType();
18+
19+
public ArdTopicCompilationTask(
20+
final AbstractCrawler aCrawler, final Queue<CrawlerUrlDTO> aUrlToCrawlDtos) {
21+
super(aCrawler, aUrlToCrawlDtos);
22+
23+
registerJsonDeserializer(ARDTOPICINFODTO_TYPE_TOKEN, new ArdTopicCompilationDeserializer());
24+
}
25+
26+
@Override
27+
protected void processRestTarget(final CrawlerUrlDTO aDTO, final WebTarget aTarget) {
28+
final ArdTopicInfoDto topicInfo = deserialize(aTarget, ARDTOPICINFODTO_TYPE_TOKEN, aDTO);
29+
if (topicInfo != null
30+
&& topicInfo.getFilmInfos() != null
31+
&& !topicInfo.getFilmInfos().isEmpty()) {
32+
taskResults.addAll(topicInfo.getFilmInfos());
33+
}
34+
}
35+
36+
@Override
37+
protected AbstractRecursiveConverterTask<ArdFilmInfoDto, CrawlerUrlDTO> createNewOwnInstance(
38+
final Queue<CrawlerUrlDTO> aElementsToProcess) {
39+
return new ArdTopicCompilationTask(crawler, aElementsToProcess);
40+
}
41+
}

‎src/test/java/de/mediathekview/mserver/crawler/ard/json/ArdDayPageDeserializerTest.java‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package de.mediathekview.mserver.crawler.ard.json;
22

3-
import com.google.gson.JsonArray;
43
import com.google.gson.JsonElement;
54
import de.mediathekview.mserver.crawler.ard.ArdConstants;
65
import de.mediathekview.mserver.crawler.ard.ArdFilmInfoDto;
@@ -26,55 +25,55 @@ public void testDeserialize() {
2625
String.format(
2726
ArdConstants.ITEM_URL,
2827
"Y3JpZDovL3JiYl8xY2RjODJjMy01ZTIyLTQ0MDctODEwZi0yMWMwYTBhY2NjMmNfcHVibGljYXRpb24"),
29-
1),
28+
1, false),
3029
new ArdFilmInfoDto(
3130
"Y3JpZDovL3JiYl9hN2RkMDNjMC0yMmU5LTRmYzEtYmNiOC1kYTg0Y2RjOWMxMWZfcHVibGljYXRpb24",
3231
String.format(
3332
ArdConstants.ITEM_URL,
3433
"Y3JpZDovL3JiYl9hN2RkMDNjMC0yMmU5LTRmYzEtYmNiOC1kYTg0Y2RjOWMxMWZfcHVibGljYXRpb24"),
35-
1),
34+
1, false),
3635
new ArdFilmInfoDto(
3736
"Y3JpZDovL21kci5kZS9zZW5kdW5nLzI4MjA0MC80MDQ4MzMtMzg1Mjgw",
3837
String.format(
3938
ArdConstants.ITEM_URL,
4039
"Y3JpZDovL21kci5kZS9zZW5kdW5nLzI4MjA0MC80MDQ4MzMtMzg1Mjgw"),
41-
1),
40+
1, false),
4241
new ArdFilmInfoDto(
4342
"Y3JpZDovL21kci5kZS9zZW5kdW5nLzI4MjA0MC80MDQ4MzQtMzg1Mjgx",
4443
String.format(
4544
ArdConstants.ITEM_URL,
4645
"Y3JpZDovL21kci5kZS9zZW5kdW5nLzI4MjA0MC80MDQ4MzQtMzg1Mjgx"),
47-
1),
46+
1, false),
4847
new ArdFilmInfoDto(
4948
"Y3JpZDovL2Rhc2Vyc3RlLmRlL2Zlcm5zZWhmaWxtZSBpbSBlcnN0ZW4vMjAyNC0wOS0yOF8xNC0wMC1NRVNa",
5049
String.format(
5150
ArdConstants.ITEM_URL,
5251
"Y3JpZDovL2Rhc2Vyc3RlLmRlL2Zlcm5zZWhmaWxtZSBpbSBlcnN0ZW4vMjAyNC0wOS0yOF8xNC0wMC1NRVNa"),
53-
1),
52+
1, false),
5453
new ArdFilmInfoDto(
5554
"Y3JpZDovL2Rhc2Vyc3RlLmRlL2Zlcm5zZWhmaWxtZSBpbSBlcnN0ZW4vMjAyNC0wOS0yOF8xNS0zMC1NRVNa",
5655
String.format(
5756
ArdConstants.ITEM_URL,
5857
"Y3JpZDovL2Rhc2Vyc3RlLmRlL2Zlcm5zZWhmaWxtZSBpbSBlcnN0ZW4vMjAyNC0wOS0yOF8xNS0zMC1NRVNa"),
59-
1),
58+
1, false),
6059
new ArdFilmInfoDto(
6160
"Y3JpZDovL3dkci5kZS9CZWl0cmFnLXNvcGhvcmEtMmIwZDg4NDMtMzQ0YS00OTZmLTlhNDYtNGY3ODk5MjE2MmFi",
6261
String.format(
6362
ArdConstants.ITEM_URL,
6463
"Y3JpZDovL3dkci5kZS9CZWl0cmFnLXNvcGhvcmEtMmIwZDg4NDMtMzQ0YS00OTZmLTlhNDYtNGY3ODk5MjE2MmFi"),
65-
1),
64+
1, false),
6665
new ArdFilmInfoDto(
6766
"Y3JpZDovL2Rhc2Vyc3RlLmRlL2RpZS1zdGlsbGVuLW1vZXJkZXIvMjAyNC0wOS0yOF8yMC0xNS1NRVNa",
6867
String.format(
6968
ArdConstants.ITEM_URL,
7069
"Y3JpZDovL2Rhc2Vyc3RlLmRlL2RpZS1zdGlsbGVuLW1vZXJkZXIvMjAyNC0wOS0yOF8yMC0xNS1NRVNa"),
71-
1),
70+
1, false),
7271
new ArdFilmInfoDto(
7372
"Y3JpZDovL2Rhc2Vyc3RlLmRlL2hhcnR3aWctc2VlbGVyLzIwMjQtMDktMjhfMjEtNDUtTUVTWg",
7473
String.format(
7574
ArdConstants.ITEM_URL,
7675
"Y3JpZDovL2Rhc2Vyc3RlLmRlL2hhcnR3aWctc2VlbGVyLzIwMjQtMDktMjhfMjEtNDUtTUVTWg"),
77-
1)
76+
1, false)
7877
};
7978

8079
final ArdDayPageDeserializer instance = new ArdDayPageDeserializer();

‎src/test/java/de/mediathekview/mserver/crawler/ard/json/ArdTopicCompilationDeserializerTest.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
package de.mediathekview.mserver.crawler.ard.json;
2+
3+
import com.google.gson.JsonElement;
4+
import de.mediathekview.mserver.crawler.ard.ArdTopicInfoDto;
5+
import de.mediathekview.mserver.testhelper.JsonFileReader;
6+
import org.junit.jupiter.api.Test;
7+
8+
import static org.hamcrest.CoreMatchers.equalTo;
9+
import static org.hamcrest.MatcherAssert.assertThat;
10+
11+
class ArdTopicCompilationDeserializerTest {
12+
13+
@Test
14+
void deserialize(){
15+
final JsonElement jsonElement = JsonFileReader.readJson("/ard/ard_compilation_page.json");
16+
17+
final ArdTopicCompilationDeserializer instance = new ArdTopicCompilationDeserializer();
18+
19+
final ArdTopicInfoDto filmInfos = instance.deserialize(jsonElement, null, null);
20+
21+
assertThat(filmInfos.getFilmInfos().size(), equalTo(24));
22+
}
23+
}

0 commit comments

Comments
 (0)