fix: store YouTube visitor data for channel tabs

extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ChannelTabHandler.java

@@ -1,5 +1,7 @@
 package org.schabi.newpipe.extractor.linkhandler;
 
+import javax.annotation.Nullable;
+
 public class ChannelTabHandler extends ListLinkHandler {
     public enum Tab {
         Playlists,
@@ -11,12 +13,36 @@ public class ChannelTabHandler extends ListLinkHandler {
 
     private final Tab tab;
 
+    /**
+     * Since YouTube is currently A/B testing a new tab layout,
+     * we need to store the visitor data cookie when fetching a channel and pass it to
+     * YouTube when requesting channel tabs. Otherwise YouTube may not enable the A/B test
+     * on subsequent requests and return empty tabs.
+     * <p>
+     * This may be removed when the new layout is made permanent.
+     */
+    @Nullable
+    private final String visitorData;
+
+    public ChannelTabHandler(final ListLinkHandler linkHandler, final Tab tab,
+                             @Nullable final String visitorData) {
+        super(linkHandler);
+        this.tab = tab;
+        this.visitorData = visitorData;
+    }
+
     public ChannelTabHandler(final ListLinkHandler linkHandler, final Tab tab) {
         super(linkHandler);
         this.tab = tab;
+        this.visitorData = null;
     }
 
     public Tab getTab() {
         return tab;
     }
+
+    @Nullable
+    public String getVisitorData() {
+        return visitorData;
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -1142,7 +1142,8 @@ public final class YoutubeParsingHelper {
     @Nonnull
     public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
             @Nonnull final Localization localization,
-            @Nonnull final ContentCountry contentCountry)
+            @Nonnull final ContentCountry contentCountry,
+            @Nullable final String vData)
             throws IOException, ExtractionException {
         // @formatter:off
         final JsonBuilder<JsonObject> builder = JsonObject.builder()
@@ -1155,7 +1156,10 @@ public final class YoutubeParsingHelper {
                         .value("originalUrl", "https://www.youtube.com")
                         .value("platform", "DESKTOP");
 
-        if (visitorData != null) {
+        // Use specified visitor data, otherwise fall back to the configured value
+        if (vData != null) {
+            builder.value("visitorData", vData);
+        } else if (visitorData != null) {
             builder.value("visitorData", visitorData);
         }
 
@@ -1176,6 +1180,14 @@ public final class YoutubeParsingHelper {
         return builder;
     }
 
+    @Nonnull
+    public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
+            @Nonnull final Localization localization,
+            @Nonnull final ContentCountry contentCountry)
+            throws IOException, ExtractionException {
+        return prepareDesktopJsonBuilder(localization, contentCountry, visitorData);
+    }
+
     @Nonnull
     public static JsonBuilder<JsonObject> prepareAndroidMobileJsonBuilder(
             @Nonnull final Localization localization,
@@ -1692,7 +1704,8 @@ public final class YoutubeParsingHelper {
     public static ChannelResponseData getChannelResponse(final String channelId,
                                                          final String params,
                                                          final Localization loc,
-                                                         final ContentCountry country)
+                                                         final ContentCountry country,
+                                                         @Nullable final String vData)
             throws ExtractionException, IOException {
         String id = channelId;
         JsonObject ajaxJson = null;
@@ -1700,7 +1713,7 @@ public final class YoutubeParsingHelper {
         int level = 0;
         while (level < 3) {
             final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
-                            loc, country)
+                            loc, country, vData)
                             .value("browseId", id)
                             .value("params", params) // Equal to videos
                             .done())

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -93,7 +93,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
         final String channelPath = super.getId();
         final String id = resolveChannelId(channelPath);
         final ChannelResponseData data = getChannelResponse(id, "EgZ2aWRlb3M%3D",
-                getExtractorLocalization(), getExtractorContentCountry());
+                getExtractorLocalization(), getExtractorContentCountry(), null);
 
         initialData = data.responseJson;
         redirectedChannelId = data.channelId;
@@ -389,8 +389,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
         JsonObject foundVideoTab = null;
         tabs = new ArrayList<>();
 
+        final String visitorData = initialData.getObject("responseContext")
+                .getString("visitorData");
+
         final Consumer<ChannelTabHandler.Tab> addTab = tab ->
-                tabs.add(new ChannelTabHandler(getLinkHandler(), tab));
+                tabs.add(new ChannelTabHandler(getLinkHandler(), tab, visitorData));
 
         for (final Object tab : responseTabs) {
             if (((JsonObject) tab).has("tabRenderer")) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java

@@ -83,7 +83,8 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
         final String params = getParams();
         final String id = resolveChannelId(super.getId());
         final ChannelResponseData data = getChannelResponse(id, params,
-                getExtractorLocalization(), getExtractorContentCountry());
+                getExtractorLocalization(), getExtractorContentCountry(),
+                getLinkHandler().getVisitorData());
 
         initialData = data.responseJson;
         redirectedChannelId = data.channelId;
@@ -312,7 +313,8 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
                 .getString("token");
 
         final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
-                        getExtractorContentCountry())
+                        getExtractorContentCountry(),
+                        getLinkHandler().getVisitorData())
                         .value("continuation", continuation)
                         .done())
                 .getBytes(StandardCharsets.UTF_8);

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java

@@ -20,11 +20,6 @@
 
 package org.schabi.newpipe.extractor.services.youtube.stream;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -47,6 +42,7 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamSegment;
 import org.schabi.newpipe.extractor.stream.StreamType;
 
+import javax.annotation.Nullable;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
@@ -54,7 +50,8 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
-import javax.annotation.Nullable;
+import static org.junit.jupiter.api.Assertions.*;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 
 public class YoutubeStreamExtractorDefaultTest {
     private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";