Merge 2e3f6b4ee3 into eac850ca10

extractor/build.gradle

@@ -18,6 +18,9 @@ checkstyle {
     toolVersion checkstyleVersion
 }
 
+// Exclude Protobuf generated files from Checkstyle
+checkstyleMain.exclude('org/schabi/newpipe/extractor/services/youtube/protos')
+
 checkstyleTest {
     enabled false // do not checkstyle test files
 }
@@ -28,6 +31,11 @@ dependencies {
     implementation "com.github.TeamNewPipe:nanojson:$nanojsonVersion"
     implementation 'org.jsoup:jsoup:1.16.2'
     implementation "com.github.spotbugs:spotbugs-annotations:$spotbugsVersion"
+    implementation "com.google.protobuf:protobuf-javalite:3.24.3"
+
+    // TODO: remove this dependency used for Base64 encoding and use Java's Base64 once its
+    //  Android desugarging support has been added
+    implementation 'commons-codec:commons-codec:1.16.0'
 
     // do not upgrade to 1.7.14, since in 1.7.14 Rhino uses the `SourceVersion` class, which is not
     // available on Android (even when using desugaring), and `NoClassDefFoundError` is thrown

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -15,6 +15,7 @@ import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonWriter;
 
+import org.apache.commons.codec.binary.Base64;
 import org.schabi.newpipe.extractor.Image;
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
@@ -27,12 +28,17 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.ContinuationParams;
+import org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.PlaylistContentFiltersParams;
+import org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.PlaylistContinuation;
+import org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.PlaylistContinuationProperties;
 import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 
@@ -40,6 +46,25 @@ import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 public class YoutubePlaylistExtractor extends PlaylistExtractor {
+
+    private static final String PLAYLIST_CONTINUATION_PROPERTIES_BASE64;
+
+    static {
+        try {
+            PLAYLIST_CONTINUATION_PROPERTIES_BASE64 = Utils.encodeUrlUtf8(
+                    Base64.encodeBase64String(
+                            PlaylistContinuationProperties.newBuilder()
+                                    .setRequestCount(0)
+                                    .setContentFilters(PlaylistContentFiltersParams.newBuilder()
+                                            .setHideUnavailableVideos(false)
+                                            .build())
+                                    .build()
+                                    .toByteArray()));
+        } catch (final UnsupportedEncodingException e) {
+            throw new RuntimeException("Couldn't encode playlist continuation properties", e);
+        }
+    }
+
     // Names of some objects in JSON response frequently used in this class
     private static final String PLAYLIST_VIDEO_RENDERER = "playlistVideoRenderer";
     private static final String PLAYLIST_VIDEO_LIST_RENDERER = "playlistVideoListRenderer";
@@ -50,6 +75,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     private static final String VIDEO_OWNER_RENDERER = "videoOwnerRenderer";
 
     private JsonObject browseResponse;
+    private JsonObject initialContinuationResponse;
 
     private JsonObject playlistInfo;
     private JsonObject uploaderInfo;
@@ -330,6 +356,17 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
             if (videoPlaylistObject.has(PLAYLIST_VIDEO_LIST_RENDERER)) {
                 renderer = videoPlaylistObject.getObject(PLAYLIST_VIDEO_LIST_RENDERER);
             } else if (videoPlaylistObject.has(RICH_GRID_RENDERER)) {
+                // richGridRenderer objects are returned for playlists with a Shorts UI
+                // As of 09/12/2023 (American English date format), an initial continuation allows
+                // to get continuations of playlists with a Shorts UI and regular playlist video
+                // renderers
+                final InfoItemsPage<StreamInfoItem> continuationPage = getInitialContinuationPage();
+                if (!continuationPage.getItems().isEmpty()) {
+                    return continuationPage;
+                }
+
+                // If no items could be extracted from the continuation, fall back to the shorts UI
+                // renderers, no continuation is provided
                 renderer = videoPlaylistObject.getObject(RICH_GRID_RENDERER);
             } else {
                 return new InfoItemsPage<>(collector, null);
@@ -375,25 +412,66 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
 
         final JsonObject lastElement = contents.getObject(contents.size() - 1);
         if (lastElement.has("continuationItemRenderer")) {
-            final String continuation = lastElement
-                    .getObject("continuationItemRenderer")
+            return getPageFromContinuation(lastElement.getObject("continuationItemRenderer")
                     .getObject("continuationEndpoint")
                     .getObject("continuationCommand")
-                    .getString("token");
-
-            final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
-                            getExtractorLocalization(), getExtractorContentCountry())
-                            .value("continuation", continuation)
-                            .done())
-                    .getBytes(StandardCharsets.UTF_8);
-
-            return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
-                    + DISABLE_PRETTY_PRINT_PARAMETER, body);
+                    .getString("token"));
         } else {
             return null;
         }
     }
 
+    @Nonnull
+    private Page getPageFromContinuation(@Nonnull final String continuation)
+            throws IOException, ExtractionException {
+        final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
+                        getExtractorLocalization(), getExtractorContentCountry())
+                        .value("continuation", continuation)
+                        .done())
+                .getBytes(StandardCharsets.UTF_8);
+
+        return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+                + DISABLE_PRETTY_PRINT_PARAMETER, body);
+    }
+
+    @Nonnull
+    private InfoItemsPage<StreamInfoItem> getInitialContinuationPage()
+            throws IOException, ExtractionException {
+        if (initialContinuationResponse == null) {
+            final String playlistId = getId();
+            final PlaylistContinuation playlistContinuation = PlaylistContinuation.newBuilder()
+                    .setParameters(ContinuationParams.newBuilder()
+                            .setBrowseId("VL" + playlistId)
+                            .setPlaylistId(playlistId)
+                            .setContinuationProperties(PLAYLIST_CONTINUATION_PROPERTIES_BASE64)
+                            .build())
+                    .build();
+
+            final String initialContinuation = Utils.encodeUrlUtf8(
+                    Base64.encodeBase64String(playlistContinuation.toByteArray()));
+            final Page page = getPageFromContinuation(initialContinuation);
+
+            try {
+                initialContinuationResponse = getJsonPostResponse("browse", page.getBody(),
+                        getExtractorLocalization());
+            } catch (final Exception e) {
+                return InfoItemsPage.emptyPage();
+            }
+        }
+
+        final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+
+        final JsonArray initialItems = initialContinuationResponse
+                .getArray("onResponseReceivedActions")
+                .getObject(0)
+                .getObject("reloadContinuationItemsCommand")
+                .getArray("continuationItems");
+
+        collectStreamsFrom(collector, initialItems);
+
+        return new InfoItemsPage<>(collector, getNextPageFrom(initialItems));
+    }
+
     private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
                                     @Nonnull final JsonArray videos) {
         final TimeAgoParser timeAgoParser = getTimeAgoParser();

extractor/src/main/proto/youtube/playlist/playlist_continuation_schema.proto

@@ -0,0 +1,47 @@
+syntax = "proto3";
+
+package youtube.playlists;
+
+option java_outer_classname = "PlaylistProtobufContinuation";
+option java_multiple_files = false;
+option java_package = "org.schabi.newpipe.extractor.services.youtube.protos.playlist";
+
+message PlaylistContinuation {
+  ContinuationParams parameters = 80226972;
+}
+
+message ContinuationParams {
+  // The playlist ID as a browse one (it should be "VL" + playlist ID)
+  string browseId = 2;
+  // A PlaylistContinuationProperties message safe-encoded as a Base64 string
+  string continuationProperties = 3;
+  string playlistId = 35;
+}
+
+message PlaylistContinuationProperties {
+  // Optional field which should be used to avoid difference behavior with
+  // continuations returned by YouTube, starts at 0
+  int32 requestCount = 1;
+  // The playlist response starts at 0 if this field is omitted
+  // An optional string "PT:" with a PlaylistIndexContinuationParameter message
+  // safe-encoded in Base64 appended at the end
+  string playlistIndexParam = 15;
+  PlaylistContentFiltersParams contentFilters = 104;
+}
+
+message PlaylistIndexContinuationParam {
+  // The index cannot be 0 and should be a 100 multiplier (YouTube backends
+  // return that requests contain an invalid argument otherwise)
+  int32 index = 1;
+}
+
+// Content filters parameters for playlists.
+// Parameters hideUnavailableVideos and showOnlyVideos can be used at the same
+// time, but no videos would be returned in this case
+message PlaylistContentFiltersParams {
+  bool hideUnavailableVideos = 1;
+  bool showOnlyVideos = 2;
+  // Parameter works but doesn't return a Shorts UI as of 09/12/23 (American
+  // English date format)
+  bool showOnlyShorts = 3;
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java

@@ -452,10 +452,6 @@ public class YoutubePlaylistExtractorTest {
             defaultTestRelatedItems(extractor);
         }
 
-        // TODO: enable test when continuations are available
-        @Disabled("Shorts UI doesn't return any continuation, even if when there are more than 100 "
-                + "items: this is a bug on YouTube's side, which is not related to the requirement "
-                + "of a valid visitorData like it is for Shorts channel tab")
         @Test
         @Override
         public void testMoreRelatedItems() throws Exception {

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/playlist/shortsUI/generated_mock_0.json

@@ -41,10 +41,10 @@
         "same-origin; report-to\u003d\"youtube_main\""
       ],
       "date": [
-        "Mon, 07 Aug 2023 17:06:40 GMT"
+        "Tue, 12 Sep 2023 20:51:32 GMT"
       ],
       "expires": [
-        "Mon, 07 Aug 2023 17:06:40 GMT"
+        "Tue, 12 Sep 2023 20:51:32 GMT"
       ],
       "origin-trial": [
         "AvC9UlR6RDk2crliDsFl66RWLnTbHrDbp+DiY6AYz/PNQ4G4tdUTjrHYr2sghbkhGQAVxb7jaPTHpEVBz0uzQwkAAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTcxOTUzMjc5OSwiaXNTdWJkb21haW4iOnRydWV9"
@@ -62,9 +62,10 @@
         "ESF"
       ],
       "set-cookie": [
-        "YSC\u003d9y_BcrNyhW0; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dTue, 10-Nov-2020 17:06:40 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "CONSENT\u003dPENDING+034; expires\u003dWed, 06-Aug-2025 17:06:40 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
+        "YSC\u003doyO6LlCVbXw; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dWed, 16-Dec-2020 20:51:32 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "VISITOR_PRIVACY_METADATA\u003dCgJGUhIA; Domain\u003d.youtube.com; Expires\u003dSun, 10-Mar-2024 20:51:32 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dlax",
+        "CONSENT\u003dPENDING+972; expires\u003dThu, 11-Sep-2025 20:51:32 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
       ],
       "strict-transport-security": [
         "max-age\u003d31536000"