Skip to content

Commit 31c07b7

Browse files
pidoubleyoupidoubleyou
committed
add tagesschau vor 20 Jahren
1 parent dfb97bd commit 31c07b7

7 files changed

Lines changed: 469 additions & 0 deletions

File tree

‎src/main/java/mServer/crawler/FilmeSuchen.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import mServer.crawler.sender.phoenix.PhoenixCrawler;
3636
import mServer.crawler.sender.sr.SrCrawler;
3737
import mServer.crawler.sender.srf.SrfCrawler;
38+
import mServer.crawler.sender.tagesschau.TagesschauCrawler;
3839
import mServer.crawler.sender.zdf.ZdfCrawler;
3940
import mServer.tool.MserverDaten;
4041
import mServer.tool.MserverKonstanten;
@@ -119,6 +120,9 @@ public FilmeSuchen() {
119120
if (crawlerList.contains("PHONIX")) {
120121
mediathekListe.add(new PhoenixCrawler(this, 0));
121122
}
123+
if (crawlerList.contains("TAGESSCHAU")) {
124+
mediathekListe.add(new TagesschauCrawler(this, 1));
125+
}
122126

123127
}
124128

‎src/main/java/mServer/crawler/sender/tagesschau/EntryUrlDto.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package mServer.crawler.sender.tagesschau;
2+
3+
import mServer.crawler.sender.base.CrawlerUrlDTO;
4+
5+
import java.util.HashSet;
6+
import java.util.Set;
7+
8+
public class EntryUrlDto {
9+
private final Set<CrawlerUrlDTO> videos;
10+
private final Set<CrawlerUrlDTO> subPages;
11+
12+
public EntryUrlDto() {
13+
this.videos = new HashSet<>();
14+
this.subPages = new HashSet<>();
15+
}
16+
17+
public Set<CrawlerUrlDTO> getVideos() {
18+
return videos;
19+
}
20+
21+
public Set<CrawlerUrlDTO> getSubPages() {
22+
return subPages;
23+
}
24+
25+
public void addVideo(CrawlerUrlDTO videoUrl) {
26+
this.videos.add(videoUrl);
27+
}
28+
29+
public void addSubPage(CrawlerUrlDTO subPageUrl) {
30+
this.subPages.add(subPageUrl);
31+
}
32+
}

‎src/main/java/mServer/crawler/sender/tagesschau/TagesschauConstants.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
package mServer.crawler.sender.tagesschau;
2+
3+
/**
4+
* Constants for the Tagesschau crawler.
5+
* Handles the "vor 20 Jahren" (20 years ago) archive with daily news broadcasts.
6+
*/
7+
public final class TagesschauConstants {
8+
9+
// Starting point: Tagesschau vor 20 Jahren (20 years ago)
10+
public static final String ARCHIVE_START_URL = "https://www.tagesschau.de/inland/tsvorzwanzigjahren-ts-142.html";
11+
12+
// Private constructor to hide the implicit public one
13+
private TagesschauConstants() {
14+
// Utility class, do not instantiate
15+
}
16+
}
17+
18+

‎src/main/java/mServer/crawler/sender/tagesschau/TagesschauCrawler.java‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
package mServer.crawler.sender.tagesschau;
2+
3+
import de.mediathekview.mlib.Const;
4+
import de.mediathekview.mlib.daten.DatenFilm;
5+
import de.mediathekview.mlib.tool.Log;
6+
import mServer.crawler.CrawlerTool;
7+
import mServer.crawler.FilmeSuchen;
8+
import mServer.crawler.sender.MediathekCrawler;
9+
import mServer.crawler.sender.base.CrawlerUrlDTO;
10+
import mServer.crawler.sender.tagesschau.tasks.TagesschauEntriesTask;
11+
import mServer.crawler.sender.tagesschau.tasks.TagesschauVideoTask;
12+
import org.apache.logging.log4j.LogManager;
13+
import org.apache.logging.log4j.Logger;
14+
15+
import java.util.HashSet;
16+
import java.util.Set;
17+
import java.util.concurrent.ConcurrentLinkedQueue;
18+
import java.util.concurrent.ExecutionException;
19+
import java.util.concurrent.RecursiveTask;
20+
21+
public class TagesschauCrawler extends MediathekCrawler {
22+
23+
private static final Logger LOG = LogManager.getLogger(TagesschauCrawler.class);
24+
25+
public TagesschauCrawler(FilmeSuchen ssearch, int startPrio) {
26+
super(ssearch, Const.TAGESSCHAU24, 0, 1, startPrio);
27+
}
28+
29+
@Override
30+
protected RecursiveTask<Set<DatenFilm>> createCrawlerTask() {
31+
try {
32+
Set<CrawlerUrlDTO> videos = new HashSet<>();
33+
ConcurrentLinkedQueue<CrawlerUrlDTO> inputQueue = createArchiveUrl();
34+
35+
// short run uses 2 recursion -> only the actual month is included
36+
int recursionMax = CrawlerTool.loadLongMax() ? 10 : 2;
37+
int recursionCount = 0;
38+
39+
while (!inputQueue.isEmpty() && recursionCount < recursionMax) {
40+
LOG.debug("processing {} sub pages", inputQueue.size());
41+
TagesschauEntriesTask round1 = new TagesschauEntriesTask(this, inputQueue);
42+
final Set<EntryUrlDto> results = this.forkJoinPool.submit(round1).get();
43+
44+
Set<CrawlerUrlDTO> subPages = new HashSet<>();
45+
results.forEach(
46+
result -> {
47+
videos.addAll(result.getVideos());
48+
subPages.addAll(result.getSubPages());
49+
});
50+
inputQueue = new ConcurrentLinkedQueue<>(subPages);
51+
recursionCount++;
52+
}
53+
54+
Log.sysLog("Tagesschau 20 Jahre Anzahl topics: " + videos.size());
55+
meldungAddMax(videos.size());
56+
57+
return new TagesschauVideoTask(this, new ConcurrentLinkedQueue<>(videos));
58+
59+
} catch (final InterruptedException ex) {
60+
LOG.fatal("Exception in Tagesschau crawler.", ex);
61+
Thread.currentThread().interrupt();
62+
} catch (final ExecutionException ex) {
63+
LOG.fatal("Exception in Tagesschau crawler.", ex);
64+
}
65+
return new RecursiveTask<>() {
66+
@Override
67+
protected Set<DatenFilm> compute() {
68+
return Set.of();
69+
}
70+
};
71+
}
72+
73+
private ConcurrentLinkedQueue<CrawlerUrlDTO> createArchiveUrl() {
74+
ConcurrentLinkedQueue<CrawlerUrlDTO> urls = new ConcurrentLinkedQueue<>();
75+
urls.add(new CrawlerUrlDTO(TagesschauConstants.ARCHIVE_START_URL));
76+
return urls;
77+
}
78+
}
79+
80+

‎src/main/java/mServer/crawler/sender/tagesschau/json/TagesschauVideoDeserializer.java‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
package mServer.crawler.sender.tagesschau.json;
2+
3+
import com.google.gson.JsonDeserializationContext;
4+
import com.google.gson.JsonDeserializer;
5+
import com.google.gson.JsonElement;
6+
import com.google.gson.JsonObject;
7+
import java.lang.reflect.Type;
8+
import java.time.*;
9+
import java.time.format.DateTimeFormatter;
10+
import java.time.format.DateTimeFormatterBuilder;
11+
import java.time.format.DateTimeParseException;
12+
import java.util.*;
13+
import java.util.regex.Matcher;
14+
import java.util.regex.Pattern;
15+
16+
import de.mediathekview.mlib.Const;
17+
import de.mediathekview.mlib.daten.DatenFilm;
18+
import mServer.crawler.CrawlerTool;
19+
import mServer.crawler.sender.base.JsonUtils;
20+
import mServer.crawler.sender.base.Qualities;
21+
import org.apache.logging.log4j.LogManager;
22+
import org.apache.logging.log4j.Logger;
23+
24+
public class TagesschauVideoDeserializer implements JsonDeserializer<List<DatenFilm>> {
25+
private static final String ELEMENT_MC = "mc";
26+
private static final String ELEMENT_MEDIA = "media";
27+
private static final String ELEMENT_META = "meta";
28+
private static final String ELEMENT_PLUG_IN_DATA = "pluginData";
29+
private static final String ELEMENT_SHARING_WEB = "sharing@web";
30+
private static final String ELEMENT_STREAMS = "streams";
31+
32+
private static final String ATTRIBUTE_DATE = "broadcastedOnDateTime";
33+
private static final String ATTRIBUTE_DURATION = "durationSeconds";
34+
private static final String ATTRIBUTE_TOPIC = "seriesTitle";
35+
private static final String ATTRIBUTE_TITLE = "title";
36+
37+
private static final String ATTRIBUTE_WIDTH = "maxHResolutionPx";
38+
private static final String ATTRIBUTE_MIMETYPE = "mimeType";
39+
private static final String ATTRIBUTE_URL = "url";
40+
private static final String ATTRIBUTE_LINK = "link";
41+
private static final String[] SUPPORTED_MIME_TYPES = new String[] { "video/mp4" };
42+
43+
private static final DateTimeFormatter DATE_FORMAT
44+
= DateTimeFormatter.ofPattern("dd.MM.yyyy");
45+
private static final DateTimeFormatter TIME_FORMAT
46+
= DateTimeFormatter.ofPattern("HH:mm:ss");
47+
48+
private static final Pattern LONG_MONTH_PATTERN =
49+
Pattern.compile("(\\d{1,2}(?:\\.|\\s)\\s*[A-Za-zÄÖÜäöüß]+\\s+\\d{4})");
50+
private static final DateTimeFormatter GERMAN_LONG = new DateTimeFormatterBuilder()
51+
.parseCaseInsensitive()
52+
.appendPattern("d. MMMM uuuu")
53+
.toFormatter(Locale.GERMAN);
54+
private static final DateTimeFormatter GERMAN_LONG_NO_SPACE = new DateTimeFormatterBuilder()
55+
.parseCaseInsensitive()
56+
.appendPattern("d.MMMM uuuu")
57+
.toFormatter(Locale.GERMAN);
58+
private static final DateTimeFormatter GERMAN_LONG_NO_DOT = new DateTimeFormatterBuilder()
59+
.parseCaseInsensitive()
60+
.appendPattern("d MMMM uuuu")
61+
.toFormatter(Locale.GERMAN);
62+
private static final DateTimeFormatter DATE_TIME_FORMATTER =
63+
DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssZ", Locale.GERMANY);
64+
private static final String GERMAN_TIME_ZONE = "Europe/Berlin";
65+
private static final Logger LOG = LogManager.getLogger(TagesschauVideoDeserializer.class);
66+
67+
private static Optional<LocalDateTime> parseDate(final JsonObject metaObject, final Optional<LocalDate> titleDate) {
68+
final Optional<String> dateValue =
69+
JsonUtils.getAttributeAsString(metaObject, ATTRIBUTE_DATE);
70+
if (dateValue.isPresent()) {
71+
try {
72+
final OffsetDateTime inputDateTime = OffsetDateTime.parse(dateValue.get(), DATE_TIME_FORMATTER);
73+
LocalDateTime localDateTime = inputDateTime.atZoneSameInstant(ZoneId.of(GERMAN_TIME_ZONE)).toLocalDateTime();
74+
if (titleDate.isPresent() && titleDate.get().getYear() != localDateTime.getYear()) {
75+
localDateTime = localDateTime.withYear(titleDate.get().getYear());
76+
}
77+
return Optional.of(localDateTime);
78+
} catch (final DateTimeParseException ex) {
79+
LOG.warn("Error parsing date time value {}", dateValue.get(), ex);
80+
}
81+
}
82+
83+
return titleDate.map(localDate -> LocalDateTime.of(localDate, LocalTime.of(20, 0)));
84+
}
85+
86+
@Override
87+
public List<DatenFilm> deserialize(
88+
JsonElement jsonElement, Type type, JsonDeserializationContext jsonDeserializationContext) {
89+
90+
final List<DatenFilm> results = new ArrayList<>();
91+
92+
Optional<JsonElement> mcElement = JsonUtils.getElement(jsonElement, ELEMENT_MC);
93+
if (mcElement.isPresent()) {
94+
final Optional<JsonElement> metaElement = JsonUtils.getElement(mcElement.get(), ELEMENT_META);
95+
if (metaElement.isPresent()) {
96+
final JsonObject metaObject = metaElement.get().getAsJsonObject();
97+
final Optional<String> topic = JsonUtils.getAttributeAsString(metaObject, ATTRIBUTE_TOPIC);
98+
final Optional<String> title = JsonUtils.getAttributeAsString(metaObject, ATTRIBUTE_TITLE);
99+
final Optional<Integer> duration = JsonUtils.getAttributeAsInt(metaObject, ATTRIBUTE_DURATION);
100+
final LocalDateTime date = parseDate(metaObject, parseDateFromTitle(title.orElse(""))).orElse(LocalDateTime.now());
101+
final Map<Qualities, String> urls = parseUrls(mcElement.get().getAsJsonObject());
102+
final String website = parseWebsite(mcElement.get().getAsJsonObject());
103+
104+
String dateValue = date.format(DATE_FORMAT);
105+
String timeValue = date.format(TIME_FORMAT);
106+
107+
DatenFilm film = new DatenFilm(Const.TAGESSCHAU24, topic.orElse(""), website, title.orElse(""), urls.get(Qualities.NORMAL), "",
108+
dateValue, timeValue, duration.orElse(0), "");
109+
110+
if (urls.containsKey(Qualities.SMALL)) {
111+
CrawlerTool.addUrlKlein(film, urls.get(Qualities.SMALL));
112+
}
113+
if (urls.containsKey(Qualities.HD)) {
114+
CrawlerTool.addUrlHd(film, urls.get(Qualities.HD));
115+
}
116+
117+
results.add(film);
118+
}
119+
}
120+
121+
return results;
122+
}
123+
124+
private Optional<LocalDate> parseDateFromTitle(String title) {
125+
Matcher m = LONG_MONTH_PATTERN.matcher(title);
126+
if (m.find()) {
127+
String datePart = m.group(1).replaceAll("\\s+", " ").trim();
128+
try {
129+
return Optional.of(LocalDate.parse(datePart, GERMAN_LONG));
130+
} catch (DateTimeParseException ignored) {
131+
// try other conversion
132+
}
133+
try {
134+
return Optional.of(LocalDate.parse(datePart, GERMAN_LONG_NO_SPACE));
135+
} catch (DateTimeParseException ignored) {
136+
// try other conversion
137+
}
138+
try {
139+
return Optional.of(LocalDate.parse(datePart, GERMAN_LONG_NO_DOT));
140+
} catch (DateTimeParseException ex) {
141+
LOG.debug("no valid date converted", ex);
142+
}
143+
}
144+
return Optional.empty();
145+
}
146+
147+
private String parseWebsite(JsonObject mcObject) {
148+
return JsonUtils.getElementValueAsString(mcObject, ELEMENT_PLUG_IN_DATA, ELEMENT_SHARING_WEB, ATTRIBUTE_LINK).orElse("");
149+
}
150+
151+
private Map<Qualities, String> parseUrls(final JsonObject mcObject) {
152+
final Map<Qualities, String> urls = new EnumMap<>(Qualities.class);
153+
if (mcObject.has(ELEMENT_STREAMS) && mcObject.get(ELEMENT_STREAMS).isJsonArray()) {
154+
mcObject
155+
.get(ELEMENT_STREAMS)
156+
.getAsJsonArray()
157+
.forEach(
158+
stream -> {
159+
final JsonObject streamObject = stream.getAsJsonObject();
160+
if (streamObject.has(ELEMENT_MEDIA)
161+
&& streamObject.get(ELEMENT_MEDIA).isJsonArray()) {
162+
streamObject
163+
.get(ELEMENT_MEDIA)
164+
.getAsJsonArray()
165+
.forEach(
166+
media -> {
167+
final Optional<String> mimeType =
168+
JsonUtils.getElementValueAsString(media, ATTRIBUTE_MIMETYPE);
169+
if (mimeType.isPresent()
170+
&& Arrays.stream(SUPPORTED_MIME_TYPES)
171+
.anyMatch(type -> type.equals(mimeType.get()))) {
172+
final Optional<Integer> width =
173+
JsonUtils.getAttributeAsInt(
174+
media.getAsJsonObject(), ATTRIBUTE_WIDTH);
175+
final Optional<String> url =
176+
JsonUtils.getElementValueAsString(media, ATTRIBUTE_URL);
177+
178+
if (width.isPresent() && url.isPresent()) {
179+
final Qualities resolution =
180+
Qualities.getResolutionFromWidth(width.get());
181+
urls.put(resolution, url.get());
182+
}
183+
}
184+
});
185+
}
186+
});
187+
}
188+
return urls;
189+
}
190+
}

0 commit comments

Comments
 (0)