Merge 2e3f6b4ee3
into eac850ca10
This commit is contained in:
commit
6d40683aed
|
@ -18,6 +18,9 @@ checkstyle {
|
|||
toolVersion checkstyleVersion
|
||||
}
|
||||
|
||||
// Exclude Protobuf generated files from Checkstyle
|
||||
checkstyleMain.exclude('org/schabi/newpipe/extractor/services/youtube/protos')
|
||||
|
||||
checkstyleTest {
|
||||
enabled false // do not checkstyle test files
|
||||
}
|
||||
|
@ -28,6 +31,11 @@ dependencies {
|
|||
implementation "com.github.TeamNewPipe:nanojson:$nanojsonVersion"
|
||||
implementation 'org.jsoup:jsoup:1.16.2'
|
||||
implementation "com.github.spotbugs:spotbugs-annotations:$spotbugsVersion"
|
||||
implementation "com.google.protobuf:protobuf-javalite:3.24.3"
|
||||
|
||||
// TODO: remove this dependency used for Base64 encoding and use Java's Base64 once its
|
||||
// Android desugarging support has been added
|
||||
implementation 'commons-codec:commons-codec:1.16.0'
|
||||
|
||||
// do not upgrade to 1.7.14, since in 1.7.14 Rhino uses the `SourceVersion` class, which is not
|
||||
// available on Android (even when using desugaring), and `NoClassDefFoundError` is thrown
|
||||
|
|
|
@ -15,6 +15,7 @@ import com.grack.nanojson.JsonArray;
|
|||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonWriter;
|
||||
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
import org.schabi.newpipe.extractor.Image;
|
||||
import org.schabi.newpipe.extractor.Page;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
|
@ -27,12 +28,17 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser;
|
|||
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
|
||||
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
|
||||
import org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.ContinuationParams;
|
||||
import org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.PlaylistContentFiltersParams;
|
||||
import org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.PlaylistContinuation;
|
||||
import org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.PlaylistContinuationProperties;
|
||||
import org.schabi.newpipe.extractor.stream.Description;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
|
||||
|
@ -40,6 +46,25 @@ import javax.annotation.Nonnull;
|
|||
import javax.annotation.Nullable;
|
||||
|
||||
public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||
|
||||
private static final String PLAYLIST_CONTINUATION_PROPERTIES_BASE64;
|
||||
|
||||
static {
|
||||
try {
|
||||
PLAYLIST_CONTINUATION_PROPERTIES_BASE64 = Utils.encodeUrlUtf8(
|
||||
Base64.encodeBase64String(
|
||||
PlaylistContinuationProperties.newBuilder()
|
||||
.setRequestCount(0)
|
||||
.setContentFilters(PlaylistContentFiltersParams.newBuilder()
|
||||
.setHideUnavailableVideos(false)
|
||||
.build())
|
||||
.build()
|
||||
.toByteArray()));
|
||||
} catch (final UnsupportedEncodingException e) {
|
||||
throw new RuntimeException("Couldn't encode playlist continuation properties", e);
|
||||
}
|
||||
}
|
||||
|
||||
// Names of some objects in JSON response frequently used in this class
|
||||
private static final String PLAYLIST_VIDEO_RENDERER = "playlistVideoRenderer";
|
||||
private static final String PLAYLIST_VIDEO_LIST_RENDERER = "playlistVideoListRenderer";
|
||||
|
@ -50,6 +75,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
private static final String VIDEO_OWNER_RENDERER = "videoOwnerRenderer";
|
||||
|
||||
private JsonObject browseResponse;
|
||||
private JsonObject initialContinuationResponse;
|
||||
|
||||
private JsonObject playlistInfo;
|
||||
private JsonObject uploaderInfo;
|
||||
|
@ -330,6 +356,17 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
if (videoPlaylistObject.has(PLAYLIST_VIDEO_LIST_RENDERER)) {
|
||||
renderer = videoPlaylistObject.getObject(PLAYLIST_VIDEO_LIST_RENDERER);
|
||||
} else if (videoPlaylistObject.has(RICH_GRID_RENDERER)) {
|
||||
// richGridRenderer objects are returned for playlists with a Shorts UI
|
||||
// As of 09/12/2023 (American English date format), an initial continuation allows
|
||||
// to get continuations of playlists with a Shorts UI and regular playlist video
|
||||
// renderers
|
||||
final InfoItemsPage<StreamInfoItem> continuationPage = getInitialContinuationPage();
|
||||
if (!continuationPage.getItems().isEmpty()) {
|
||||
return continuationPage;
|
||||
}
|
||||
|
||||
// If no items could be extracted from the continuation, fall back to the shorts UI
|
||||
// renderers, no continuation is provided
|
||||
renderer = videoPlaylistObject.getObject(RICH_GRID_RENDERER);
|
||||
} else {
|
||||
return new InfoItemsPage<>(collector, null);
|
||||
|
@ -375,25 +412,66 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
|||
|
||||
final JsonObject lastElement = contents.getObject(contents.size() - 1);
|
||||
if (lastElement.has("continuationItemRenderer")) {
|
||||
final String continuation = lastElement
|
||||
.getObject("continuationItemRenderer")
|
||||
return getPageFromContinuation(lastElement.getObject("continuationItemRenderer")
|
||||
.getObject("continuationEndpoint")
|
||||
.getObject("continuationCommand")
|
||||
.getString("token");
|
||||
|
||||
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
|
||||
getExtractorLocalization(), getExtractorContentCountry())
|
||||
.value("continuation", continuation)
|
||||
.done())
|
||||
.getBytes(StandardCharsets.UTF_8);
|
||||
|
||||
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
|
||||
+ DISABLE_PRETTY_PRINT_PARAMETER, body);
|
||||
.getString("token"));
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private Page getPageFromContinuation(@Nonnull final String continuation)
|
||||
throws IOException, ExtractionException {
|
||||
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
|
||||
getExtractorLocalization(), getExtractorContentCountry())
|
||||
.value("continuation", continuation)
|
||||
.done())
|
||||
.getBytes(StandardCharsets.UTF_8);
|
||||
|
||||
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
|
||||
+ DISABLE_PRETTY_PRINT_PARAMETER, body);
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private InfoItemsPage<StreamInfoItem> getInitialContinuationPage()
|
||||
throws IOException, ExtractionException {
|
||||
if (initialContinuationResponse == null) {
|
||||
final String playlistId = getId();
|
||||
final PlaylistContinuation playlistContinuation = PlaylistContinuation.newBuilder()
|
||||
.setParameters(ContinuationParams.newBuilder()
|
||||
.setBrowseId("VL" + playlistId)
|
||||
.setPlaylistId(playlistId)
|
||||
.setContinuationProperties(PLAYLIST_CONTINUATION_PROPERTIES_BASE64)
|
||||
.build())
|
||||
.build();
|
||||
|
||||
final String initialContinuation = Utils.encodeUrlUtf8(
|
||||
Base64.encodeBase64String(playlistContinuation.toByteArray()));
|
||||
final Page page = getPageFromContinuation(initialContinuation);
|
||||
|
||||
try {
|
||||
initialContinuationResponse = getJsonPostResponse("browse", page.getBody(),
|
||||
getExtractorLocalization());
|
||||
} catch (final Exception e) {
|
||||
return InfoItemsPage.emptyPage();
|
||||
}
|
||||
}
|
||||
|
||||
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
|
||||
final JsonArray initialItems = initialContinuationResponse
|
||||
.getArray("onResponseReceivedActions")
|
||||
.getObject(0)
|
||||
.getObject("reloadContinuationItemsCommand")
|
||||
.getArray("continuationItems");
|
||||
|
||||
collectStreamsFrom(collector, initialItems);
|
||||
|
||||
return new InfoItemsPage<>(collector, getNextPageFrom(initialItems));
|
||||
}
|
||||
|
||||
private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
|
||||
@Nonnull final JsonArray videos) {
|
||||
final TimeAgoParser timeAgoParser = getTimeAgoParser();
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,47 @@
|
|||
syntax = "proto3";
|
||||
|
||||
package youtube.playlists;
|
||||
|
||||
option java_outer_classname = "PlaylistProtobufContinuation";
|
||||
option java_multiple_files = false;
|
||||
option java_package = "org.schabi.newpipe.extractor.services.youtube.protos.playlist";
|
||||
|
||||
message PlaylistContinuation {
|
||||
ContinuationParams parameters = 80226972;
|
||||
}
|
||||
|
||||
message ContinuationParams {
|
||||
// The playlist ID as a browse one (it should be "VL" + playlist ID)
|
||||
string browseId = 2;
|
||||
// A PlaylistContinuationProperties message safe-encoded as a Base64 string
|
||||
string continuationProperties = 3;
|
||||
string playlistId = 35;
|
||||
}
|
||||
|
||||
message PlaylistContinuationProperties {
|
||||
// Optional field which should be used to avoid difference behavior with
|
||||
// continuations returned by YouTube, starts at 0
|
||||
int32 requestCount = 1;
|
||||
// The playlist response starts at 0 if this field is omitted
|
||||
// An optional string "PT:" with a PlaylistIndexContinuationParameter message
|
||||
// safe-encoded in Base64 appended at the end
|
||||
string playlistIndexParam = 15;
|
||||
PlaylistContentFiltersParams contentFilters = 104;
|
||||
}
|
||||
|
||||
message PlaylistIndexContinuationParam {
|
||||
// The index cannot be 0 and should be a 100 multiplier (YouTube backends
|
||||
// return that requests contain an invalid argument otherwise)
|
||||
int32 index = 1;
|
||||
}
|
||||
|
||||
// Content filters parameters for playlists.
|
||||
// Parameters hideUnavailableVideos and showOnlyVideos can be used at the same
|
||||
// time, but no videos would be returned in this case
|
||||
message PlaylistContentFiltersParams {
|
||||
bool hideUnavailableVideos = 1;
|
||||
bool showOnlyVideos = 2;
|
||||
// Parameter works but doesn't return a Shorts UI as of 09/12/23 (American
|
||||
// English date format)
|
||||
bool showOnlyShorts = 3;
|
||||
}
|
|
@ -452,10 +452,6 @@ public class YoutubePlaylistExtractorTest {
|
|||
defaultTestRelatedItems(extractor);
|
||||
}
|
||||
|
||||
// TODO: enable test when continuations are available
|
||||
@Disabled("Shorts UI doesn't return any continuation, even if when there are more than 100 "
|
||||
+ "items: this is a bug on YouTube's side, which is not related to the requirement "
|
||||
+ "of a valid visitorData like it is for Shorts channel tab")
|
||||
@Test
|
||||
@Override
|
||||
public void testMoreRelatedItems() throws Exception {
|
||||
|
|
|
@ -41,10 +41,10 @@
|
|||
"same-origin; report-to\u003d\"youtube_main\""
|
||||
],
|
||||
"date": [
|
||||
"Mon, 07 Aug 2023 17:06:40 GMT"
|
||||
"Tue, 12 Sep 2023 20:51:32 GMT"
|
||||
],
|
||||
"expires": [
|
||||
"Mon, 07 Aug 2023 17:06:40 GMT"
|
||||
"Tue, 12 Sep 2023 20:51:32 GMT"
|
||||
],
|
||||
"origin-trial": [
|
||||
"AvC9UlR6RDk2crliDsFl66RWLnTbHrDbp+DiY6AYz/PNQ4G4tdUTjrHYr2sghbkhGQAVxb7jaPTHpEVBz0uzQwkAAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTcxOTUzMjc5OSwiaXNTdWJkb21haW4iOnRydWV9"
|
||||
|
@ -62,9 +62,10 @@
|
|||
"ESF"
|
||||
],
|
||||
"set-cookie": [
|
||||
"YSC\u003d9y_BcrNyhW0; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
|
||||
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dTue, 10-Nov-2020 17:06:40 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
|
||||
"CONSENT\u003dPENDING+034; expires\u003dWed, 06-Aug-2025 17:06:40 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
|
||||
"YSC\u003doyO6LlCVbXw; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
|
||||
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dWed, 16-Dec-2020 20:51:32 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
|
||||
"VISITOR_PRIVACY_METADATA\u003dCgJGUhIA; Domain\u003d.youtube.com; Expires\u003dSun, 10-Mar-2024 20:51:32 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dlax",
|
||||
"CONSENT\u003dPENDING+972; expires\u003dThu, 11-Sep-2025 20:51:32 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
|
||||
],
|
||||
"strict-transport-security": [
|
||||
"max-age\u003d31536000"
|
||||
|
|
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue