Update youtube data extractor

Author: duncte123

build.gradle.kts

@@ -25,7 +25,7 @@ plugins {
 }
 
 project.group = "com.dunctebot"
-project.version = "1.5.0"
+project.version = "1.5.1"
 val archivesBaseName = "sourcemanagers"
 
 repositories {

src/main/java/com/dunctebot/sourcemanagers/extra/YoutubeContextFilterOverride.java

@@ -35,9 +35,12 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 public class YoutubeContextFilterOverride extends YoutubeHttpContextFilter implements Closeable {
     private static final Logger logger = LoggerFactory.getLogger(YoutubeContextFilterOverride.class);
+    private static final Pattern YOUTUBE_INFO_REGEX = Pattern.compile("window\\.ytplayer=\\{\\};\\n?ytcfg\\.set\\((.*)\\);");
     private YoutubeVersionData youtubeVersionData = null;
     private final HttpInterface httpInterface;
     private final ScheduledExecutorService dataUpdateThread = Executors.newSingleThreadScheduledExecutor((r) -> {
@@ -71,6 +74,7 @@ private void updateYoutubeData() {
             logger.info("New youtube version data {}", this.youtubeVersionData);
         } catch (IOException e) {
             Sentry.capture(e);
+            logger.error("Failed to capture youtube data", e);
         }
     }
 
@@ -81,12 +85,18 @@ private YoutubeVersionData getYoutubeHeaderDetails() throws IOException {
         try (final CloseableHttpResponse response = this.httpInterface.execute(httpGet)) {
             final String html = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8);
 
-            final String extracted = DataFormatTools.extractBetween(html,
-                "window.ytplayer = {};ytcfg.set(",
-                ");ytcfg.set(");
-            final JsonBrowser json = JsonBrowser.parse(extracted);
+            final Matcher matcher = YOUTUBE_INFO_REGEX.matcher(html);
 
-            return YoutubeVersionData.fromBrowser(json);
+            if (matcher.find()) {
+                final String extracted = matcher.group(matcher.groupCount())
+                    .trim()
+                    .replaceAll("undefined", "null");
+                final JsonBrowser json = JsonBrowser.parse(extracted);
+
+                return YoutubeVersionData.fromBrowser(json);
+            }
+
+            return null;
         }
     }
 
@@ -101,7 +111,7 @@ public void onRequest(HttpClientContext context, HttpUriRequest request, boolean
         request.setHeader("x-youtube-client-version", youtubeVersionData.getVersion());
         request.setHeader("x-youtube-page-cl", youtubeVersionData.getPageCl());
         request.setHeader("x-youtube-page-label", youtubeVersionData.getLabel());
-        request.setHeader("x-youtube-variants-checksum", youtubeVersionData.getChecksum());
+        request.setHeader("x-youtube-identity-token", youtubeVersionData.getIdToken());
     }
 
     @Override

src/main/java/com/dunctebot/sourcemanagers/extra/YoutubeVersionData.java

@@ -23,19 +23,19 @@ public class YoutubeVersionData {
     /* INNERTUBE_CONTEXT_CLIENT_VERSION  x-youtube-client-version */
     private final String version;
     /* VARIANTS_CHECKSUM  x-youtube-variants-checksum */
-    private final String checksum;
+    //private final String checksum;
     /* PAGE_BUILD_LABEL  x-youtube-page-label */
     private final String label;
     /* ID_TOKEN  x-youtube-identity-token */
-//    private final String idToken;
+    private final String idToken;
     /* PAGE_CL  x-youtube-page-cl */
     private final String pageCl;
     /* DEVICE  x-youtube-device */
 //    private final String device;
 
-    public YoutubeVersionData(String version, String checksum, String label, String pageCl) {
+    public YoutubeVersionData(String version, String idToken, String label, String pageCl) {
         this.version = version;
-        this.checksum = checksum;
+        this.idToken = idToken;
         this.label = label;
         this.pageCl = pageCl;
     }
@@ -44,8 +44,8 @@ public String getVersion() {
         return version;
     }
 
-    public String getChecksum() {
-        return checksum;
+    public String getIdToken() {
+        return idToken;
     }
 
     public String getLabel() {
@@ -60,7 +60,7 @@ public String getPageCl() {
     public String toString() {
         return "YoutubeVersionData{" +
                 "version='" + version + '\'' +
-                ", checksum='" + checksum + '\'' +
+                ", idToken='" + idToken + '\'' +
                 ", label='" + label + '\'' +
                 ", pageCl='" + pageCl + '\'' +
                 '}';
@@ -69,9 +69,9 @@ public String toString() {
     public static YoutubeVersionData fromBrowser(JsonBrowser json) {
         return new YoutubeVersionData(
                 json.get("INNERTUBE_CONTEXT_CLIENT_VERSION").safeText(),
-                json.get("VARIANTS_CHECKSUM").safeText(),
+                json.get("ID_TOKEN").safeText(),
                 json.get("PAGE_BUILD_LABEL").safeText(),
                 json.get("PAGE_CL").safeText()
         );
     }
-}
+}