Use the youtubei API for YouTube mixes + update the corresponding test + do some improvements

Use the youtubei API for YouTube mixes. The corresponding has been updated because the new API breaks the tests of YoutubeMixPlaylistExtractorTest.
Remove some deprecated code (the old search code with the pbj JSON) and do some other improvements.

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -64,11 +64,12 @@ public class YoutubeParsingHelper {
     private YoutubeParsingHelper() {
     }
 
+    public static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/";
+
     private static final String HARDCODED_CLIENT_VERSION = "2.20210526.07.00";
     private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
     private static final String[] MOBILE_YOUTUBE_KEYS = {"AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w",
             "16.20.35"};
-    private static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/";
     private static String clientVersion;
     private static String key;
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -24,15 +24,13 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 import javax.annotation.Nonnull;
 
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder;
 import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
 import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@@ -349,9 +347,13 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
 
         // Unfortunately, we have to fetch the page even if we are only getting next streams,
         // as they don't deliver enough information on their own (the channel name, for example).
-        fetchPage();
 
-        StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+        if (!isPageFetched()) fetchPage();
+
+        final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+        final Map<String, List<String>> headers = new HashMap<>();
+        addClientInfoHeaders(headers);
+
         final Response response = getDownloader().post(page.getUrl(), null, page.getBody(),
                 getExtractorLocalization());
 
@@ -383,7 +385,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
                 .done())
                 .getBytes(UTF_8);
 
-        return new Page("https://www.youtube.com/youtubei/v1/browse?key=" + getKey(), body);
+        return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), body);
     }
 
     /**

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java

@@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 
+import com.grack.nanojson.JsonWriter;
 import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
@@ -11,6 +12,7 @@ import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
+import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
@@ -19,19 +21,14 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 
 import java.io.IOException;
-import java.util.Collections;
+import java.net.URL;
-import java.util.HashMap;
+import java.util.*;
-import java.util.List;
-import java.util.Map;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
+import static org.schabi.newpipe.extractor.utils.Utils.*;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getResponse;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
-import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 /**
  * A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist).
@@ -58,13 +55,51 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
     @Override
     public void onFetchPage(@Nonnull final Downloader downloader)
             throws IOException, ExtractionException {
-        final String url = getUrl() + "&pbj=1";
+        final Localization localization = getExtractorLocalization();
-        final Response response = getResponse(url, getExtractorLocalization());
+        final URL url = stringToURL(getUrl());
-        final JsonArray ajaxJson = JsonUtils.toJsonArray(response.responseBody());
+        final String mixPlaylistId = getId();
-        initialData = ajaxJson.getObject(3).getObject("response");
+        final String videoId = getQueryValue(url, "v");
+        final String playlistIndexString = getQueryValue(url, "index");
+
+        final byte[] body;
+        if (videoId != null) {
+            if (playlistIndexString != null) {
+                body = JsonWriter.string(prepareJsonBuilder(localization,
+                        getExtractorContentCountry())
+                        .value("videoId", videoId)
+                        .value("playlistId", mixPlaylistId)
+                        .value("playlistIndex", Integer.parseInt(playlistIndexString))
+                        .done())
+                        .getBytes(UTF_8);
+            } else {
+                body = JsonWriter.string(prepareJsonBuilder(localization,
+                        getExtractorContentCountry())
+                        .value("videoId", videoId)
+                        .value("playlistId", mixPlaylistId)
+                        .done())
+                        .getBytes(UTF_8);
+            }
+        } else {
+            body = JsonWriter.string(prepareJsonBuilder(localization,
+                    getExtractorContentCountry())
+                    .value("playlistId", mixPlaylistId)
+                    .done())
+                    .getBytes(UTF_8);
+        }
+
+        final Map<String, List<String>> headers = new HashMap<>();
+        addClientInfoHeaders(headers);
+
+        final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey(),
+                headers, body, localization);
+
+        initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
         playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
                 .getObject("playlist").getObject("playlist");
+        if (isNullOrEmpty(playlistData)) throw new ExtractionException(
+                "Could not get playlistData");
         cookieValue = extractCookieValue(COOKIE_NAME, response);
+
     }
 
     @Nonnull
@@ -83,10 +118,9 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
             return getThumbnailUrlFromPlaylistId(playlistData.getString("playlistId"));
         } catch (final Exception e) {
             try {
-                //fallback to thumbnail of current video. Always the case for channel mix
+                // Fallback to thumbnail of current video. Always the case for channel mix
-                return getThumbnailUrlFromVideoId(
+                return getThumbnailUrlFromVideoId(initialData.getObject("currentVideoEndpoint")
-                        initialData.getObject("currentVideoEndpoint").getObject("watchEndpoint")
+                        .getObject("watchEndpoint").getString("videoId"));
-                                .getString("videoId"));
             } catch (final Exception ignored) {
             }
             throw new ParsingException("Could not get playlist thumbnail", e);
@@ -100,19 +134,19 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
 
     @Override
     public String getUploaderUrl() {
-        //Youtube mix are auto-generated
+        // YouTube mixes are auto-generated by YouTube
         return "";
     }
 
     @Override
     public String getUploaderName() {
-        //Youtube mix are auto-generated by YouTube
+        // YouTube mixes are auto-generated by YouTube
         return "YouTube";
     }
 
     @Override
     public String getUploaderAvatarUrl() {
-        //Youtube mix are auto-generated by YouTube
+        // YouTube mixes are auto-generated by YouTube
         return "";
     }
 
@@ -123,64 +157,81 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
 
     @Override
     public long getStreamCount() {
-        // Auto-generated playlist always start with 25 videos and are endless
+        // Auto-generated playlists always start with 25 videos and are endless
         return ListExtractor.ITEM_COUNT_INFINITE;
     }
 
     @Nonnull
     @Override
-    public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
+    public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException,
+            ExtractionException {
         final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
         collectStreamsFrom(collector, playlistData.getArray("contents"));
 
         final Map<String, String> cookies = new HashMap<>();
         cookies.put(COOKIE_NAME, cookieValue);
 
-        return new InfoItemsPage<>(collector, new Page(getNextPageUrlFrom(playlistData), cookies));
+        return new InfoItemsPage<>(collector, getNextPageFrom(playlistData, cookies));
     }
 
-    private String getNextPageUrlFrom(final JsonObject playlistJson) throws ExtractionException {
+    private Page getNextPageFrom(final JsonObject playlistJson,
+                                 final Map<String, String> cookies) throws IOException,
+            ExtractionException {
         final JsonObject lastStream = ((JsonObject) playlistJson.getArray("contents")
                 .get(playlistJson.getArray("contents").size() - 1));
         if (lastStream == null || lastStream.getObject("playlistPanelVideoRenderer") == null) {
             throw new ExtractionException("Could not extract next page url");
         }
 
-        return getUrlFromNavigationEndpoint(
+        final JsonObject watchEndpoint = lastStream.getObject("playlistPanelVideoRenderer")
-                lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint"))
+                .getObject("navigationEndpoint").getObject("watchEndpoint");
-                + "&pbj=1";
+        final String playlistId = watchEndpoint.getString("playlistId");
+        final String videoId = watchEndpoint.getString("videoId");
+        final int index = watchEndpoint.getInt("index");
+        final String params = watchEndpoint.getString("params");
+        final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(),
+                getExtractorContentCountry())
+                .value("videoId", videoId)
+                .value("playlistId", playlistId)
+                .value("playlistIndex", index)
+                .value("params", params)
+                .done())
+                .getBytes(UTF_8);
+
+        return new Page(YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, cookies, body);
     }
 
     @Override
-    public InfoItemsPage<StreamInfoItem> getPage(final Page page)
+    public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
-            throws ExtractionException, IOException {
+            ExtractionException {
         if (page == null || isNullOrEmpty(page.getUrl())) {
-            throw new IllegalArgumentException("Page url is empty or null");
+            throw new IllegalArgumentException("Page doesn't contain an URL");
         }
         if (!page.getCookies().containsKey(COOKIE_NAME)) {
-            throw new IllegalArgumentException("Cooke '" + COOKIE_NAME + "' is missing");
+            throw new IllegalArgumentException("Cookie '" + COOKIE_NAME + "' is missing");
         }
 
-        final JsonArray ajaxJson = getJsonResponse(page, getExtractorLocalization());
+        final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-        final JsonObject playlistJson =
+        final Map<String, List<String>> headers = new HashMap<>();
-                ajaxJson.getObject(3).getObject("response").getObject("contents")
+        addClientInfoHeaders(headers);
-                        .getObject("twoColumnWatchNextResults").getObject("playlist")
+
-                        .getObject("playlist");
+        final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(),
+                getExtractorLocalization());
+        final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
+        final JsonObject playlistJson = ajaxJson.getObject("contents")
+                .getObject("twoColumnWatchNextResults").getObject("playlist").getObject("playlist");
         final JsonArray allStreams = playlistJson.getArray("contents");
-        // Sublist because youtube returns up to 24 previous streams in the mix
+        // Sublist because YouTube returns up to 24 previous streams in the mix
         // +1 because the stream of "currentIndex" was already extracted in previous request
         final List<Object> newStreams =
                 allStreams.subList(playlistJson.getInt("currentIndex") + 1, allStreams.size());
 
-        final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
         collectStreamsFrom(collector, newStreams);
-        return new InfoItemsPage<>(collector,
+        return new InfoItemsPage<>(collector, getNextPageFrom(playlistJson, page.getCookies()));
-                new Page(getNextPageUrlFrom(playlistJson), page.getCookies()));
     }
 
-    private void collectStreamsFrom(
+    private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
-            @Nonnull final StreamInfoItemsCollector collector,
+                                    @Nullable final List<Object> streams) {
-            @Nullable final List<Object> streams) {
 
         if (streams == null) {
             return;
@@ -193,7 +244,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
                 final JsonObject streamInfo = ((JsonObject) stream)
                         .getObject("playlistPanelVideoRenderer");
                 if (streamInfo != null) {
-                    collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser));
+                    collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo,
+                            timeAgoParser));
                 }
             }
         }
@@ -204,7 +256,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
         if (playlistId.startsWith("RDMM")) {
             videoId = playlistId.substring(4);
         } else if (playlistId.startsWith("RDCMUC")) {
-            throw new ParsingException("is channel mix");
+            throw new ParsingException("This playlist is a channel mix");
         } else {
             videoId = playlistId.substring(2);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -21,16 +21,13 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 import javax.annotation.Nonnull;
 
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder;
 import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
@@ -224,14 +221,15 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
         }
 
         final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+        final Map<String, List<String>> headers = new HashMap<>();
+        addClientInfoHeaders(headers);
 
-        final Response response = getDownloader().post(page.getUrl(), null, page.getBody(),
+        final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(),
                 getExtractorLocalization());
         final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
 
         final JsonArray continuation = ajaxJson.getArray("onResponseReceivedActions")
-                .getObject(0)
+                .getObject(0).getObject("appendContinuationItemsAction")
-                .getObject("appendContinuationItemsAction")
                 .getArray("continuationItems");
 
         collectStreamsFrom(collector, continuation);
@@ -259,7 +257,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
                     .done())
                     .getBytes(UTF_8);
 
-            return new Page("https://www.youtube.com/youtubei/v1/browse?key=" + getKey(), body);
+            return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), body);
         } else {
             return null;
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java

@@ -153,10 +153,8 @@ public class YoutubeSearchExtractor extends SearchExtractor {
                         .getObject("itemSectionRenderer");
 
                 collectStreamsFrom(collector, itemSectionRenderer.getArray("contents"));
-
-                nextPage = getNextPageFrom(itemSectionRenderer.getArray("continuations"));
             } else if (((JsonObject) section).has("continuationItemRenderer")) {
-                nextPage = getNewNextPageFrom(((JsonObject) section)
+                nextPage = getNextPageFrom(((JsonObject) section)
                         .getObject("continuationItemRenderer"));
             }
         }
@@ -174,46 +172,34 @@ public class YoutubeSearchExtractor extends SearchExtractor {
         final Localization localization = getExtractorLocalization();
         final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
 
-        if (page.getId() == null) {
+        // @formatter:off
-            final JsonArray ajaxJson = getJsonResponse(page.getUrl(), localization);
+        final byte[] json = JsonWriter.string(prepareJsonBuilder(localization,
+                getExtractorContentCountry())
+                .value("continuation", page.getId())
+                .done())
+                .getBytes(UTF_8);
+        // @formatter:on
 
-            final JsonObject itemSectionContinuation = ajaxJson.getObject(1).getObject("response")
+        final String responseBody = getValidJsonResponseBody(getDownloader().post(
-                    .getObject("continuationContents").getObject("itemSectionContinuation");
+                page.getUrl(), new HashMap<>(), json));
 
-            collectStreamsFrom(collector, itemSectionContinuation.getArray("contents"));
+        final JsonObject ajaxJson;
-            final JsonArray continuations = itemSectionContinuation.getArray("continuations");
+        try {
-
+            ajaxJson = JsonParser.object().from(responseBody);
-            return new InfoItemsPage<>(collector, getNextPageFrom(continuations));
+        } catch (JsonParserException e) {
-        } else {
+            throw new ParsingException("Could not parse JSON", e);
-            // @formatter:off
-            final byte[] json = JsonWriter.string(prepareJsonBuilder(localization,
-                    getExtractorContentCountry())
-                    .value("continuation", page.getId())
-                    .done())
-                    .getBytes(UTF_8);
-            // @formatter:on
-
-            final String responseBody = getValidJsonResponseBody(getDownloader().post(
-                    page.getUrl(), new HashMap<>(), json));
-
-            final JsonObject ajaxJson;
-            try {
-                ajaxJson = JsonParser.object().from(responseBody);
-            } catch (JsonParserException e) {
-                throw new ParsingException("Could not parse JSON", e);
-            }
-
-            final JsonArray continuationItems = ajaxJson.getArray("onResponseReceivedCommands")
-                    .getObject(0).getObject("appendContinuationItemsAction")
-                    .getArray("continuationItems");
-
-            final JsonArray contents = continuationItems.getObject(0)
-                    .getObject("itemSectionRenderer").getArray("contents");
-            collectStreamsFrom(collector, contents);
-
-            return new InfoItemsPage<>(collector, getNewNextPageFrom(continuationItems.getObject(1)
-                    .getObject("continuationItemRenderer")));
         }
+
+        final JsonArray continuationItems = ajaxJson.getArray("onResponseReceivedCommands")
+                .getObject(0).getObject("appendContinuationItemsAction")
+                .getArray("continuationItems");
+
+        final JsonArray contents = continuationItems.getObject(0)
+                .getObject("itemSectionRenderer").getArray("contents");
+        collectStreamsFrom(collector, contents);
+
+        return new InfoItemsPage<>(collector, getNextPageFrom(continuationItems.getObject(1)
+                .getObject("continuationItemRenderer")));
     }
 
     private void collectStreamsFrom(final InfoItemsSearchCollector collector,
@@ -239,22 +225,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
         }
     }
 
-    private Page getNextPageFrom(final JsonArray continuations) throws ParsingException {
+    private Page getNextPageFrom(final JsonObject continuationItemRenderer) throws IOException,
-        if (isNullOrEmpty(continuations)) {
-            return null;
-        }
-
-        final JsonObject nextContinuationData = continuations.getObject(0)
-                .getObject("nextContinuationData");
-        final String continuation = nextContinuationData.getString("continuation");
-        final String clickTrackingParams = nextContinuationData
-                .getString("clickTrackingParams");
-
-        return new Page(getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation="
-                + continuation + "&itct=" + clickTrackingParams);
-    }
-
-    private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) throws IOException,
             ExtractionException {
         if (isNullOrEmpty(continuationItemRenderer)) {
             return null;
@@ -263,7 +234,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
         final String token = continuationItemRenderer.getObject("continuationEndpoint")
                 .getObject("continuationCommand").getString("token");
 
-        final String url = "https://www.youtube.com/youtubei/v1/search?key=" + getKey();
+        final String url = YOUTUBEI_V1_URL + "search?key=" + getKey();
 
         return new Page(url, token);
     }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java

@@ -1,8 +1,8 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
+import com.grack.nanojson.JsonWriter;
 import org.hamcrest.MatcherAssert;
 import org.junit.BeforeClass;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Suite;
@@ -32,12 +32,13 @@ import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
+import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
 
 @RunWith(Suite.class)
 @SuiteClasses({Mix.class, MixWithIndex.class, MyMix.class, Invalid.class, ChannelMix.class})
 public class YoutubeMixPlaylistExtractorTest {
 
-    public static final String PBJ = "&pbj=1";
     private static final String VIDEO_ID = "_AzeUSL9lZc";
     private static final String VIDEO_TITLE =
             "Most Beautiful And Emotional  Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO";
@@ -55,8 +56,8 @@ public class YoutubeMixPlaylistExtractorTest {
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "mix"));
             dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
             extractor = (YoutubeMixPlaylistExtractor) YouTube
-                    .getPlaylistExtractor(
+                    .getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID
-                            "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID);
+                            + "&list=RD" + VIDEO_ID);
             extractor.fetchPage();
         }
 
@@ -89,9 +90,16 @@ public class YoutubeMixPlaylistExtractorTest {
 
         @Test
         public void getPage() throws Exception {
-            final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
+            final byte[] body = JsonWriter.string(prepareJsonBuilder(
-                    new Page("https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID
+                    NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
-                            + PBJ, dummyCookie));
+                    .value("videoId", VIDEO_ID)
+                    .value("playlistId", "RD" + VIDEO_ID)
+                    .value("params", "OAE%3D")
+                    .done())
+                    .getBytes(UTF_8);
+
+            final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
+                    YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
         }
@@ -127,7 +135,7 @@ public class YoutubeMixPlaylistExtractorTest {
     @Ignore
     public static class MixWithIndex {
 
-        private static final String INDEX = "&index=13";
+        private static final int INDEX = 13;
         private static final String VIDEO_ID_NUMBER_13 = "qHtzO49SDmk";
 
         @BeforeClass
@@ -137,9 +145,8 @@ public class YoutubeMixPlaylistExtractorTest {
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "mixWithIndex"));
             dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
             extractor = (YoutubeMixPlaylistExtractor) YouTube
-                    .getPlaylistExtractor(
+                    .getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13
-                            "https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
+                            + "&list=RD" + VIDEO_ID + "&index=" + INDEX);
-                                    + VIDEO_ID + INDEX);
             extractor.fetchPage();
         }
 
@@ -167,9 +174,17 @@ public class YoutubeMixPlaylistExtractorTest {
 
         @Test
         public void getPage() throws Exception {
-            final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
+            final byte[] body = JsonWriter.string(prepareJsonBuilder(
-                    new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
+                    NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
-                            + VIDEO_ID + INDEX + PBJ, dummyCookie));
+                    .value("videoId", VIDEO_ID)
+                    .value("playlistId", "RD" + VIDEO_ID)
+                    .value("playlistIndex", INDEX)
+                    .value("params", "OAE%3D")
+                    .done())
+                    .getBytes(UTF_8);
+
+            final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
+                    YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
         }
@@ -210,9 +225,8 @@ public class YoutubeMixPlaylistExtractorTest {
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "myMix"));
             dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
             extractor = (YoutubeMixPlaylistExtractor) YouTube
-                    .getPlaylistExtractor(
+                    .getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID
-                            "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RDMM"
+                            + "&list=RDMM" + VIDEO_ID);
-                                    + VIDEO_ID);
             extractor.fetchPage();
         }
 
@@ -243,9 +257,16 @@ public class YoutubeMixPlaylistExtractorTest {
 
         @Test
         public void getPage() throws Exception {
-            final InfoItemsPage<StreamInfoItem> streams =
+            final byte[] body = JsonWriter.string(prepareJsonBuilder(
-                    extractor.getPage(new Page("https://www.youtube.com/watch?v=" + VIDEO_ID
+                    NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
-                            + "&list=RDMM" + VIDEO_ID + PBJ, dummyCookie));
+                    .value("videoId", VIDEO_ID)
+                    .value("playlistId", "RDMM" + VIDEO_ID)
+                    .value("params", "OAE%3D")
+                    .done())
+                    .getBytes(UTF_8);
+
+            final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
+                    YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
         }
@@ -291,8 +312,8 @@ public class YoutubeMixPlaylistExtractorTest {
         @Test(expected = IllegalArgumentException.class)
         public void getPageEmptyUrl() throws Exception {
             extractor = (YoutubeMixPlaylistExtractor) YouTube
-                    .getPlaylistExtractor(
+                    .getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID
-                            "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID);
+                            + "&list=RD" + VIDEO_ID);
             extractor.fetchPage();
             extractor.getPage(new Page(""));
         }
@@ -300,8 +321,8 @@ public class YoutubeMixPlaylistExtractorTest {
         @Test(expected = ExtractionException.class)
         public void invalidVideoId() throws Exception {
             extractor = (YoutubeMixPlaylistExtractor) YouTube
-                    .getPlaylistExtractor(
+                    .getPlaylistExtractor("https://www.youtube.com/watch?v=" + "abcde"
-                            "https://www.youtube.com/watch?v=" + "abcde" + "&list=RD" + "abcde");
+                            + "&list=RD" + "abcde");
             extractor.fetchPage();
             extractor.getName();
         }
@@ -321,9 +342,8 @@ public class YoutubeMixPlaylistExtractorTest {
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "channelMix"));
             dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
             extractor = (YoutubeMixPlaylistExtractor) YouTube
-                    .getPlaylistExtractor(
+                    .getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
-                            "https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
+                            + "&list=RDCM" + CHANNEL_ID);
-                                    + "&list=RDCM" + CHANNEL_ID);
             extractor.fetchPage();
         }
 
@@ -350,9 +370,16 @@ public class YoutubeMixPlaylistExtractorTest {
 
         @Test
         public void getPage() throws Exception {
-            final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
+            final byte[] body = JsonWriter.string(prepareJsonBuilder(
-                    new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
+                    NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
-                            + "&list=RDCM" + CHANNEL_ID + PBJ, dummyCookie));
+                    .value("videoId", VIDEO_ID_OF_CHANNEL)
+                    .value("playlistId", "RDCM" + CHANNEL_ID)
+                    .value("params", "OAE%3D")
+                    .done())
+                    .getBytes(UTF_8);
+
+            final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
+                    YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
             assertFalse(streams.getItems().isEmpty());
             assertTrue(streams.hasNextPage());
         }