2020-02-02 14:19:48 +01:00
|
|
|
package org.schabi.newpipe.extractor.services.youtube.extractors;
|
|
|
|
|
2020-03-06 20:40:40 +01:00
|
|
|
import com.grack.nanojson.JsonArray;
|
2021-06-24 18:39:16 +02:00
|
|
|
import com.grack.nanojson.JsonBuilder;
|
2020-03-06 20:40:40 +01:00
|
|
|
import com.grack.nanojson.JsonObject;
|
2021-03-04 18:58:51 +01:00
|
|
|
|
2021-05-30 17:23:51 +02:00
|
|
|
import com.grack.nanojson.JsonWriter;
|
2020-03-17 14:04:46 +01:00
|
|
|
import org.schabi.newpipe.extractor.ListExtractor;
|
2020-04-16 19:28:27 +02:00
|
|
|
import org.schabi.newpipe.extractor.Page;
|
2020-02-02 14:19:48 +01:00
|
|
|
import org.schabi.newpipe.extractor.StreamingService;
|
|
|
|
import org.schabi.newpipe.extractor.downloader.Downloader;
|
2020-04-16 19:28:27 +02:00
|
|
|
import org.schabi.newpipe.extractor.downloader.Response;
|
2020-02-02 14:19:48 +01:00
|
|
|
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
|
|
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
|
|
|
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
|
2021-05-30 17:23:51 +02:00
|
|
|
import org.schabi.newpipe.extractor.localization.Localization;
|
2020-02-02 14:19:48 +01:00
|
|
|
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
|
|
|
|
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
2021-03-25 21:47:10 +01:00
|
|
|
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
|
2020-02-02 14:19:48 +01:00
|
|
|
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
|
|
|
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
2021-03-04 18:58:51 +01:00
|
|
|
import org.schabi.newpipe.extractor.utils.JsonUtils;
|
2020-02-02 14:19:48 +01:00
|
|
|
|
2020-05-21 14:52:13 +02:00
|
|
|
import java.io.IOException;
|
2021-05-30 17:23:51 +02:00
|
|
|
import java.net.URL;
|
|
|
|
import java.util.*;
|
2020-05-21 14:52:13 +02:00
|
|
|
|
2021-03-04 18:58:51 +01:00
|
|
|
import javax.annotation.Nonnull;
|
|
|
|
import javax.annotation.Nullable;
|
|
|
|
|
2021-05-30 17:23:51 +02:00
|
|
|
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
|
|
|
|
import static org.schabi.newpipe.extractor.utils.Utils.*;
|
2020-05-21 14:52:13 +02:00
|
|
|
|
2020-02-02 18:15:47 +01:00
|
|
|
/**
|
2020-04-16 19:28:27 +02:00
|
|
|
* A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist).
|
|
|
|
* It handles URLs in the format of
|
|
|
|
* {@code youtube.com/watch?v=videoId&list=playlistId}
|
2020-02-02 18:15:47 +01:00
|
|
|
*/
|
2020-02-02 14:19:48 +01:00
|
|
|
public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
|
|
|
|
|
2020-04-16 19:28:27 +02:00
|
|
|
/**
|
|
|
|
* YouTube identifies mixes based on this cookie. With this information it can generate
|
|
|
|
* continuations without duplicates.
|
|
|
|
*/
|
2020-09-26 11:22:24 +02:00
|
|
|
public static final String COOKIE_NAME = "VISITOR_INFO1_LIVE";
|
2020-04-16 19:28:27 +02:00
|
|
|
|
2020-03-21 18:48:12 +01:00
|
|
|
private JsonObject initialData;
|
2020-03-07 16:03:12 +01:00
|
|
|
private JsonObject playlistData;
|
2020-04-16 19:28:27 +02:00
|
|
|
private String cookieValue;
|
2020-03-07 16:03:12 +01:00
|
|
|
|
2020-04-16 19:28:27 +02:00
|
|
|
public YoutubeMixPlaylistExtractor(final StreamingService service,
|
|
|
|
final ListLinkHandler linkHandler) {
|
2020-03-07 16:03:12 +01:00
|
|
|
super(service, linkHandler);
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2020-04-16 19:28:27 +02:00
|
|
|
public void onFetchPage(@Nonnull final Downloader downloader)
|
|
|
|
throws IOException, ExtractionException {
|
2021-05-30 17:23:51 +02:00
|
|
|
final Localization localization = getExtractorLocalization();
|
|
|
|
final URL url = stringToURL(getUrl());
|
|
|
|
final String mixPlaylistId = getId();
|
|
|
|
final String videoId = getQueryValue(url, "v");
|
|
|
|
final String playlistIndexString = getQueryValue(url, "index");
|
|
|
|
|
2021-06-24 18:39:16 +02:00
|
|
|
final JsonBuilder<JsonObject> jsonBody = prepareJsonBuilder(localization,
|
|
|
|
getExtractorContentCountry()).value("playlistId", mixPlaylistId);
|
2021-05-30 17:23:51 +02:00
|
|
|
if (videoId != null) {
|
2021-06-24 18:39:16 +02:00
|
|
|
jsonBody.value("videoId", videoId);
|
2021-05-30 17:23:51 +02:00
|
|
|
}
|
2021-06-24 18:39:16 +02:00
|
|
|
if (playlistIndexString != null) {
|
|
|
|
jsonBody.value("playlistIndex", Integer.parseInt(playlistIndexString));
|
|
|
|
}
|
|
|
|
|
|
|
|
final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(UTF_8);
|
2021-05-30 17:23:51 +02:00
|
|
|
|
|
|
|
final Map<String, List<String>> headers = new HashMap<>();
|
|
|
|
addClientInfoHeaders(headers);
|
|
|
|
|
|
|
|
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey(),
|
|
|
|
headers, body, localization);
|
|
|
|
|
|
|
|
initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
|
2020-03-21 18:48:12 +01:00
|
|
|
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
|
2020-04-16 19:28:27 +02:00
|
|
|
.getObject("playlist").getObject("playlist");
|
2021-05-30 17:23:51 +02:00
|
|
|
if (isNullOrEmpty(playlistData)) throw new ExtractionException(
|
|
|
|
"Could not get playlistData");
|
2020-04-16 19:28:27 +02:00
|
|
|
cookieValue = extractCookieValue(COOKIE_NAME, response);
|
2020-03-07 16:03:12 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Nonnull
|
|
|
|
@Override
|
|
|
|
public String getName() throws ParsingException {
|
2021-03-25 21:47:10 +01:00
|
|
|
final String name = YoutubeParsingHelper.getTextAtKey(playlistData, "title");
|
|
|
|
if (isNullOrEmpty(name)) {
|
2020-03-17 14:04:46 +01:00
|
|
|
throw new ParsingException("Could not get playlist name");
|
2020-03-07 16:03:12 +01:00
|
|
|
}
|
2020-03-17 14:04:46 +01:00
|
|
|
return name;
|
2020-03-07 16:03:12 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public String getThumbnailUrl() throws ParsingException {
|
|
|
|
try {
|
2020-04-16 19:28:27 +02:00
|
|
|
return getThumbnailUrlFromPlaylistId(playlistData.getString("playlistId"));
|
|
|
|
} catch (final Exception e) {
|
2020-03-21 18:48:12 +01:00
|
|
|
try {
|
2021-05-30 17:23:51 +02:00
|
|
|
// Fallback to thumbnail of current video. Always the case for channel mix
|
|
|
|
return getThumbnailUrlFromVideoId(initialData.getObject("currentVideoEndpoint")
|
|
|
|
.getObject("watchEndpoint").getString("videoId"));
|
2020-04-16 19:28:27 +02:00
|
|
|
} catch (final Exception ignored) {
|
2020-03-21 18:48:12 +01:00
|
|
|
}
|
2020-03-07 16:03:12 +01:00
|
|
|
throw new ParsingException("Could not get playlist thumbnail", e);
|
|
|
|
}
|
2020-02-02 14:19:48 +01:00
|
|
|
}
|
2020-03-07 16:03:12 +01:00
|
|
|
|
|
|
|
@Override
|
|
|
|
public String getBannerUrl() {
|
|
|
|
return "";
|
2020-02-02 14:19:48 +01:00
|
|
|
}
|
2020-03-07 16:03:12 +01:00
|
|
|
|
|
|
|
@Override
|
|
|
|
public String getUploaderUrl() {
|
2021-05-30 17:23:51 +02:00
|
|
|
// YouTube mixes are auto-generated by YouTube
|
2020-03-07 16:03:12 +01:00
|
|
|
return "";
|
2020-03-07 08:42:26 +01:00
|
|
|
}
|
|
|
|
|
2020-03-07 16:03:12 +01:00
|
|
|
@Override
|
|
|
|
public String getUploaderName() {
|
2021-05-30 17:23:51 +02:00
|
|
|
// YouTube mixes are auto-generated by YouTube
|
2020-03-17 14:04:46 +01:00
|
|
|
return "YouTube";
|
2020-02-02 14:19:48 +01:00
|
|
|
}
|
|
|
|
|
2020-03-07 16:03:12 +01:00
|
|
|
@Override
|
|
|
|
public String getUploaderAvatarUrl() {
|
2021-05-30 17:23:51 +02:00
|
|
|
// YouTube mixes are auto-generated by YouTube
|
2020-03-07 16:03:12 +01:00
|
|
|
return "";
|
|
|
|
}
|
2020-02-02 14:19:48 +01:00
|
|
|
|
2021-01-22 01:44:58 +01:00
|
|
|
@Override
|
|
|
|
public boolean isUploaderVerified() throws ParsingException {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2020-03-07 16:03:12 +01:00
|
|
|
@Override
|
|
|
|
public long getStreamCount() {
|
2021-05-30 17:23:51 +02:00
|
|
|
// Auto-generated playlists always start with 25 videos and are endless
|
2020-03-17 14:04:46 +01:00
|
|
|
return ListExtractor.ITEM_COUNT_INFINITE;
|
2020-03-07 16:03:12 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Nonnull
|
|
|
|
@Override
|
2021-05-30 17:23:51 +02:00
|
|
|
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException,
|
|
|
|
ExtractionException {
|
2020-04-16 19:28:27 +02:00
|
|
|
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
2020-03-21 18:48:12 +01:00
|
|
|
collectStreamsFrom(collector, playlistData.getArray("contents"));
|
2021-04-07 12:25:59 +02:00
|
|
|
|
|
|
|
final Map<String, String> cookies = new HashMap<>();
|
|
|
|
cookies.put(COOKIE_NAME, cookieValue);
|
|
|
|
|
2021-05-30 17:23:51 +02:00
|
|
|
return new InfoItemsPage<>(collector, getNextPageFrom(playlistData, cookies));
|
2020-03-21 18:48:12 +01:00
|
|
|
}
|
|
|
|
|
2021-05-30 17:23:51 +02:00
|
|
|
private Page getNextPageFrom(final JsonObject playlistJson,
|
|
|
|
final Map<String, String> cookies) throws IOException,
|
|
|
|
ExtractionException {
|
2020-04-16 19:28:27 +02:00
|
|
|
final JsonObject lastStream = ((JsonObject) playlistJson.getArray("contents")
|
|
|
|
.get(playlistJson.getArray("contents").size() - 1));
|
2020-03-21 18:48:12 +01:00
|
|
|
if (lastStream == null || lastStream.getObject("playlistPanelVideoRenderer") == null) {
|
2020-03-07 16:03:12 +01:00
|
|
|
throw new ExtractionException("Could not extract next page url");
|
|
|
|
}
|
2020-04-16 19:28:27 +02:00
|
|
|
|
2021-05-30 17:23:51 +02:00
|
|
|
final JsonObject watchEndpoint = lastStream.getObject("playlistPanelVideoRenderer")
|
|
|
|
.getObject("navigationEndpoint").getObject("watchEndpoint");
|
|
|
|
final String playlistId = watchEndpoint.getString("playlistId");
|
|
|
|
final String videoId = watchEndpoint.getString("videoId");
|
|
|
|
final int index = watchEndpoint.getInt("index");
|
|
|
|
final String params = watchEndpoint.getString("params");
|
|
|
|
final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(),
|
|
|
|
getExtractorContentCountry())
|
|
|
|
.value("videoId", videoId)
|
|
|
|
.value("playlistId", playlistId)
|
|
|
|
.value("playlistIndex", index)
|
|
|
|
.value("params", params)
|
|
|
|
.done())
|
|
|
|
.getBytes(UTF_8);
|
|
|
|
|
|
|
|
return new Page(YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, cookies, body);
|
2020-03-07 16:03:12 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2021-05-30 17:23:51 +02:00
|
|
|
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
|
|
|
|
ExtractionException {
|
2020-09-26 11:22:24 +02:00
|
|
|
if (page == null || isNullOrEmpty(page.getUrl())) {
|
2021-05-30 17:23:51 +02:00
|
|
|
throw new IllegalArgumentException("Page doesn't contain an URL");
|
2020-09-26 11:22:24 +02:00
|
|
|
}
|
|
|
|
if (!page.getCookies().containsKey(COOKIE_NAME)) {
|
2021-05-30 17:23:51 +02:00
|
|
|
throw new IllegalArgumentException("Cookie '" + COOKIE_NAME + "' is missing");
|
2020-03-07 08:42:26 +01:00
|
|
|
}
|
2020-03-07 16:03:12 +01:00
|
|
|
|
2021-05-30 17:23:51 +02:00
|
|
|
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
|
|
|
final Map<String, List<String>> headers = new HashMap<>();
|
|
|
|
addClientInfoHeaders(headers);
|
|
|
|
|
|
|
|
final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(),
|
|
|
|
getExtractorLocalization());
|
|
|
|
final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
|
|
|
|
final JsonObject playlistJson = ajaxJson.getObject("contents")
|
|
|
|
.getObject("twoColumnWatchNextResults").getObject("playlist").getObject("playlist");
|
2020-04-16 19:28:27 +02:00
|
|
|
final JsonArray allStreams = playlistJson.getArray("contents");
|
2021-05-30 17:23:51 +02:00
|
|
|
// Sublist because YouTube returns up to 24 previous streams in the mix
|
2020-04-16 19:28:27 +02:00
|
|
|
// +1 because the stream of "currentIndex" was already extracted in previous request
|
|
|
|
final List<Object> newStreams =
|
|
|
|
allStreams.subList(playlistJson.getInt("currentIndex") + 1, allStreams.size());
|
|
|
|
|
|
|
|
collectStreamsFrom(collector, newStreams);
|
2021-05-30 17:23:51 +02:00
|
|
|
return new InfoItemsPage<>(collector, getNextPageFrom(playlistJson, page.getCookies()));
|
2020-03-07 08:42:26 +01:00
|
|
|
}
|
2020-02-02 18:04:15 +01:00
|
|
|
|
2021-05-30 17:23:51 +02:00
|
|
|
private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
|
|
|
|
@Nullable final List<Object> streams) {
|
2020-03-07 16:03:12 +01:00
|
|
|
|
|
|
|
if (streams == null) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
final TimeAgoParser timeAgoParser = getTimeAgoParser();
|
|
|
|
|
2020-04-16 19:28:27 +02:00
|
|
|
for (final Object stream : streams) {
|
2020-03-07 16:03:12 +01:00
|
|
|
if (stream instanceof JsonObject) {
|
2020-04-16 19:28:27 +02:00
|
|
|
final JsonObject streamInfo = ((JsonObject) stream)
|
2021-01-22 01:44:58 +01:00
|
|
|
.getObject("playlistPanelVideoRenderer");
|
2020-03-07 16:03:12 +01:00
|
|
|
if (streamInfo != null) {
|
2021-05-30 17:23:51 +02:00
|
|
|
collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo,
|
|
|
|
timeAgoParser));
|
2020-03-07 16:03:12 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-04-16 19:28:27 +02:00
|
|
|
private String getThumbnailUrlFromPlaylistId(final String playlistId) throws ParsingException {
|
2020-03-17 14:04:46 +01:00
|
|
|
final String videoId;
|
|
|
|
if (playlistId.startsWith("RDMM")) {
|
|
|
|
videoId = playlistId.substring(4);
|
2020-03-21 18:48:12 +01:00
|
|
|
} else if (playlistId.startsWith("RDCMUC")) {
|
2021-05-30 17:23:51 +02:00
|
|
|
throw new ParsingException("This playlist is a channel mix");
|
2020-03-17 14:04:46 +01:00
|
|
|
} else {
|
|
|
|
videoId = playlistId.substring(2);
|
|
|
|
}
|
|
|
|
if (videoId.isEmpty()) {
|
|
|
|
throw new ParsingException("videoId is empty");
|
|
|
|
}
|
2020-03-21 18:48:12 +01:00
|
|
|
return getThumbnailUrlFromVideoId(videoId);
|
|
|
|
}
|
|
|
|
|
2020-04-16 19:28:27 +02:00
|
|
|
private String getThumbnailUrlFromVideoId(final String videoId) {
|
2020-03-07 16:03:12 +01:00
|
|
|
return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg";
|
|
|
|
}
|
2020-05-21 14:52:13 +02:00
|
|
|
|
|
|
|
@Nonnull
|
|
|
|
@Override
|
|
|
|
public String getSubChannelName() {
|
|
|
|
return "";
|
|
|
|
}
|
|
|
|
|
|
|
|
@Nonnull
|
|
|
|
@Override
|
|
|
|
public String getSubChannelUrl() {
|
|
|
|
return "";
|
|
|
|
}
|
|
|
|
|
|
|
|
@Nonnull
|
|
|
|
@Override
|
|
|
|
public String getSubChannelAvatarUrl() {
|
|
|
|
return "";
|
|
|
|
}
|
2020-02-02 14:19:48 +01:00
|
|
|
}
|