Skip to content

Commit 42736a9

Browse files
pidoubleyoupidoubleyou
committed
add compilation subpage handling
1 parent 9ca4c62 commit 42736a9

6 files changed

Lines changed: 131 additions & 31 deletions

File tree

‎src/main/java/mServer/crawler/sender/ard/ArdConstants.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,13 @@ public class ArdConstants {
1111

1212
public static final String TOPICS_URL = API_URL + "/page-gateway/pages/%s/editorial/experiment-a-z?embedded=false";
1313
public static final String TOPICS_COMPILATION_URL = API_URL + "/page-gateway/widgets/%s/editorials/%s?pageNumber=0&pageSize=%s";
14+
public static final String TOPIC_COMPILATION_PAGE_URL = API_URL + "/page-gateway/widgets/%s/compilation/%s?pageNumber=%d&pageSize=%d";
1415
public static final String TOPIC_URL = API_URL + "/page-gateway/widgets/ard/asset/%s?pageSize=%d";
1516
public static final String DAY_PAGE_URL = "https://programm-api.ard.de/program/api/program?day=%s&channelIds=%s&mode=channel";
1617

1718
public static final int TOPICS_COMPILATION_PAGE_SIZE = 100;
1819
public static final int TOPIC_PAGE_SIZE = 50;
20+
public static final int MAX_COMPILATION_SUB_PAGES = 4;
1921

2022
public static final String DEFAULT_CLIENT = "ard";
2123

‎src/main/java/mServer/crawler/sender/ard/ArdTopicInfoDto.java‎

Lines changed: 48 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -6,53 +6,77 @@
66
public class ArdTopicInfoDto {
77
private final Set<ArdFilmInfoDto> filmInfos;
88
private String id;
9-
private int subPageNumber;
9+
private int pageNumber;
10+
private int pageSize;
11+
private int totalElements;
1012
private int maxSubPageNumber;
1113

14+
1215
public ArdTopicInfoDto(final Set<ArdFilmInfoDto> filmInfos) {
1316
this.filmInfos = filmInfos;
14-
subPageNumber = 0;
15-
maxSubPageNumber = 0;
17+
setPageNumber(0);
18+
setPageSize(0);
19+
setTotalElements(0);
20+
setMaxSubPageNumber(0);
1621
}
1722

1823
public Set<ArdFilmInfoDto> getFilmInfos() {
1924
return filmInfos;
2025
}
2126

22-
public String getId() {
23-
return id;
27+
@Override
28+
public boolean equals(final Object o) {
29+
if (this == o) {
30+
return true;
31+
}
32+
if (!(o instanceof final ArdTopicInfoDto that)) {
33+
return false;
34+
}
35+
return getId() == that.getId()
36+
&& getPageNumber() == that.getPageNumber()
37+
&& getPageSize() == that.getPageSize()
38+
&& getTotalElements() == that.getTotalElements()
39+
&& Objects.equals(filmInfos, that.filmInfos);
2440
}
2541

26-
public void setId(String id) {
27-
this.id = id;
42+
@Override
43+
public int hashCode() {
44+
return Objects.hash(filmInfos, getId(), getPageNumber(), getPageSize(), getTotalElements());
2845
}
2946

30-
public int getSubPageNumber() {
31-
return subPageNumber;
47+
public String getId() { return id; }
48+
49+
public int getPageNumber() {
50+
return pageNumber;
3251
}
3352

34-
public void setSubPageNumber(final int subPageNumber) {
35-
this.subPageNumber = subPageNumber;
53+
public void setPageNumber(int pageNumber) {
54+
this.pageNumber = pageNumber;
3655
}
3756

38-
public int getMaxSubPageNumber() {
39-
return maxSubPageNumber;
57+
public int getPageSize() {
58+
return pageSize;
4059
}
4160

42-
public void setMaxSubPageNumber(final int maxSubPageNumber) {
43-
this.maxSubPageNumber = maxSubPageNumber;
61+
public void setPageSize(int pageSize) {
62+
this.pageSize = pageSize;
4463
}
4564

46-
@Override
47-
public boolean equals(Object o) {
48-
if (this == o) return true;
49-
if (o == null || getClass() != o.getClass()) return false;
50-
ArdTopicInfoDto that = (ArdTopicInfoDto) o;
51-
return Objects.equals(id, that.id) && subPageNumber == that.subPageNumber && maxSubPageNumber == that.maxSubPageNumber && Objects.equals(filmInfos, that.filmInfos);
65+
public int getTotalElements() {
66+
return totalElements;
5267
}
5368

54-
@Override
55-
public int hashCode() {
56-
return Objects.hash(id, filmInfos, subPageNumber, maxSubPageNumber);
69+
public void setTotalElements(int totalElements) {
70+
this.totalElements = totalElements;
71+
}
72+
73+
public void setId(String id) { this.id = id; }
74+
75+
public int getMaxSubPageNumber() {
76+
return maxSubPageNumber;
77+
}
78+
79+
public void setMaxSubPageNumber(final int maxSubPageNumber) {
80+
this.maxSubPageNumber = maxSubPageNumber;
5781
}
5882
}

‎src/main/java/mServer/crawler/sender/ard/json/ArdTopicCompilationDeserializer.java‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,19 +33,39 @@ public ArdTopicInfoDto deserialize(
3333
results.addAll(parseTeasers(teasers));
3434
}
3535
});
36+
} else if (showPageObject.getAsJsonObject().has(ELEMENT_TEASERS)) {
37+
final JsonArray teasers = showPageObject.getAsJsonObject().get(ELEMENT_TEASERS).getAsJsonArray();
38+
results.addAll(parseTeasers(teasers));
3639
}
3740

38-
final JsonElement paginationElement = showPageObject.get(ELEMENT_PAGINATION);
41+
final JsonElement paginationElement = findPaginationElement(showPageObject);
3942
final int pageNumber = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_NUMBER);
4043
final int totalElements = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_TOTAL_ELEMENTS);
44+
ardTopicInfoDto.setTotalElements(totalElements);
4145
final int pageSize = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_SIZE);
46+
ardTopicInfoDto.setPageSize(pageSize);
4247
ardTopicInfoDto.setMaxSubPageNumber(pageSize == 0 ? 0 :
4348
(totalElements + pageSize - 1) / pageSize);
44-
ardTopicInfoDto.setSubPageNumber(pageNumber);
49+
ardTopicInfoDto.setPageNumber(pageNumber);
4550

4651
return ardTopicInfoDto;
4752
}
4853

54+
private JsonElement findPaginationElement(JsonObject showPageObject) {
55+
if (showPageObject.has(ELEMENT_WIDGETS)) {
56+
final JsonElement widgetElement = showPageObject.get(ELEMENT_WIDGETS);
57+
if (widgetElement.isJsonArray()) {
58+
final JsonArray widgetArray = widgetElement.getAsJsonArray();
59+
if (!widgetArray.isEmpty()) {
60+
return widgetArray.get(0).getAsJsonObject().get(ELEMENT_PAGINATION);
61+
}
62+
}
63+
} else if (showPageObject.has(ELEMENT_PAGINATION)) {
64+
return showPageObject.get(ELEMENT_PAGINATION);
65+
}
66+
return null;
67+
}
68+
4969
private int getChildElementAsIntOrNullIfNotExist(
5070
final JsonElement parentElement, final String childElementName) {
5171
if (parentElement == null || parentElement.isJsonNull()) {

‎src/main/java/mServer/crawler/sender/ard/json/ArdTopicPageDeserializer.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,12 @@ public ArdTopicInfoDto deserialize(
3535
id.ifPresent(ardTopicInfoDto::setId);
3636

3737
final JsonElement paginationElement = showPageObject.get(ELEMENT_PAGINATION);
38-
ardTopicInfoDto.setSubPageNumber(
38+
ardTopicInfoDto.setPageNumber(
3939
getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_NUMBER));
4040
final int totalElements = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_TOTAL_ELEMENTS);
41+
ardTopicInfoDto.setTotalElements(totalElements);
4142
final int pageSize = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_SIZE);
42-
ardTopicInfoDto.setMaxSubPageNumber(pageSize == 0 ? 0 :
43-
(totalElements + pageSize - 1) / pageSize);
43+
ardTopicInfoDto.setPageSize(pageSize);
4444

4545
return ardTopicInfoDto;
4646
}

‎src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicCompilationTask.java‎

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,28 @@
33
import com.google.gson.reflect.TypeToken;
44
import jakarta.ws.rs.client.WebTarget;
55
import mServer.crawler.sender.MediathekReader;
6+
import mServer.crawler.sender.ard.ArdConstants;
67
import mServer.crawler.sender.ard.ArdFilmInfoDto;
78
import mServer.crawler.sender.ard.ArdTopicInfoDto;
89
import mServer.crawler.sender.ard.json.ArdTopicCompilationDeserializer;
910
import mServer.crawler.sender.base.AbstractRecursivConverterTask;
1011
import mServer.crawler.sender.base.CrawlerUrlDTO;
12+
import org.apache.logging.log4j.LogManager;
13+
import org.apache.logging.log4j.Logger;
1114

1215
import java.lang.reflect.Type;
1316
import java.util.concurrent.ConcurrentLinkedQueue;
17+
import java.util.regex.Matcher;
18+
import java.util.regex.Pattern;
1419

1520
public class ArdTopicCompilationTask extends ArdTaskBase<ArdFilmInfoDto, CrawlerUrlDTO> {
21+
private static final Logger LOG = LogManager.getLogger(ArdTopicCompilationTask.class);
22+
private static final Pattern UUID_PATTERN = Pattern.compile("/compilation/([^/?]+)([/?])");
23+
private static final Pattern WIDGETS_SENDER_PATTERN = Pattern.compile("/pages/([^/]+)/"); // captures the segment after /widgets/ (e.g. "ard")
1624

1725
private static final Type ARDTOPICINFODTO_TYPE_TOKEN =
18-
new TypeToken<ArdTopicInfoDto>() {}.getType();
26+
new TypeToken<ArdTopicInfoDto>() {
27+
}.getType();
1928

2029
public ArdTopicCompilationTask(
2130
final MediathekReader aCrawler, final ConcurrentLinkedQueue<CrawlerUrlDTO> aUrlToCrawlDtos) {
@@ -31,7 +40,52 @@ protected void processRestTarget(final CrawlerUrlDTO aDTO, final WebTarget aTarg
3140
&& topicInfo.getFilmInfos() != null
3241
&& !topicInfo.getFilmInfos().isEmpty()) {
3342
taskResults.addAll(topicInfo.getFilmInfos());
43+
44+
final ConcurrentLinkedQueue<CrawlerUrlDTO> subpages = createSubPageUrls(aDTO, topicInfo);
45+
if (!subpages.isEmpty()) {
46+
taskResults.addAll(createNewOwnInstance(subpages).fork().join());
47+
}
48+
}
49+
}
50+
51+
private ConcurrentLinkedQueue<CrawlerUrlDTO> createSubPageUrls(CrawlerUrlDTO urlDto, ArdTopicInfoDto topicInfo) {
52+
final ConcurrentLinkedQueue<CrawlerUrlDTO> subpages = new ConcurrentLinkedQueue<>();
53+
if (topicInfo.getTotalElements() < topicInfo.getPageSize() || topicInfo.getPageNumber() > 0) {
54+
return subpages;
55+
}
56+
//
57+
final String sender = extractSender(urlDto.getUrl());
58+
final String id = extractCompilation(urlDto.getUrl());
59+
60+
int index = 0;
61+
while ((topicInfo.getPageSize() + (index * topicInfo.getPageSize())) < topicInfo.getTotalElements()) {
62+
subpages.add(new CrawlerUrlDTO(String.format(ArdConstants.TOPIC_COMPILATION_PAGE_URL, sender, id, index + 1, topicInfo.getPageSize())));
63+
index++;
64+
if (index >= ArdConstants.MAX_COMPILATION_SUB_PAGES) {
65+
LOG.debug("ignore more subpage due to limit of {} pages but found {}", ArdConstants.MAX_COMPILATION_SUB_PAGES, Integer.valueOf(topicInfo.getTotalElements() / topicInfo.getPageSize()));
66+
break;
67+
}
68+
}
69+
return subpages;
70+
}
71+
72+
private String extractCompilation(String url) {
73+
final Matcher m = UUID_PATTERN.matcher(url);
74+
if (m.find()) {
75+
return m.group(1);
76+
}
77+
78+
LOG.warn("no compilation pattern found in url {}", url);
79+
return "";
80+
}
81+
82+
private String extractSender(String url) {
83+
final Matcher m = WIDGETS_SENDER_PATTERN.matcher(url);
84+
if (m.find()) {
85+
return m.group(1);
3486
}
87+
LOG.warn("no sender pattern found in url {}", url);
88+
return "";
3589
}
3690

3791
@Override

‎src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ private ConcurrentLinkedQueue<CrawlerUrlDTO> createSubPageUrls(
6868
final WebTarget aTarget, final ArdTopicInfoDto topicInfo) {
6969
final ConcurrentLinkedQueue<CrawlerUrlDTO> subpages = new ConcurrentLinkedQueue<>();
7070

71-
final int actualSubPageNumber = topicInfo.getSubPageNumber();
71+
final int actualSubPageNumber = topicInfo.getPageNumber();
7272
final int maximumAllowedSubpages = getMaximumSubpages(topicInfo.getId());
7373
if (actualSubPageNumber != 0) {
7474
LOG.debug("Sub page {} is already the maximum allowed sub page.", actualSubPageNumber);

0 commit comments

Comments
 (0)