Merge pull request #567 from XiangRongLin/playlist_continuations

Playlist continuations
This commit is contained in:
XiangRongLin 2021-03-04 19:05:32 +01:00 committed by GitHub
commit 9256b3b848
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 1849 additions and 273 deletions

View File

@ -1,6 +1,11 @@
package org.schabi.newpipe.extractor.services.youtube;
import com.grack.nanojson.*;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.downloader.Response;
@ -10,6 +15,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
@ -33,7 +39,12 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.utils.Utils.*;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.join;
/*
* Created by Christian Schabesberger on 02.03.16.
@ -638,7 +649,7 @@ public class YoutubeParsingHelper {
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
final Response response = getDownloader().get(url, headers, localization);
return toJsonArray(getValidJsonResponseBody(response));
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
}
public static JsonArray getJsonResponse(final Page page, final Localization localization)
@ -652,15 +663,7 @@ public class YoutubeParsingHelper {
final Response response = getDownloader().get(page.getUrl(), headers, localization);
return toJsonArray(getValidJsonResponseBody(response));
}
public static JsonArray toJsonArray(final String responseBody) throws ParsingException {
try {
return JsonParser.array().from(responseBody);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse JSON", e);
}
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
}
/**

View File

@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
@ -14,14 +15,19 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/**
@ -51,7 +57,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
throws IOException, ExtractionException {
final String url = getUrl() + "&pbj=1";
final Response response = getResponse(url, getExtractorLocalization());
final JsonArray ajaxJson = toJsonArray(response.responseBody());
final JsonArray ajaxJson = JsonUtils.toJsonArray(response.responseBody());
initialData = ajaxJson.getObject(3).getObject("response");
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("playlist").getObject("playlist");

View File

@ -2,9 +2,12 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@ -19,11 +22,20 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.utils.JsonUtils.toJsonObject;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@SuppressWarnings("WeakerAccess")
@ -168,7 +180,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() {
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Page nextPage = null;
@ -205,12 +217,27 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
// @formatter:off
byte[] json = JsonWriter.string()
.object()
.object("context")
.object("client")
.value("clientName", "1")
.value("clientVersion", getClientVersion())
.end()
.end()
.value("continuation", page.getId())
.end()
.done()
.getBytes(UTF_8);
// @formatter:on
final JsonArray continuation = ajaxJson.getObject(1)
.getObject("response")
.getArray("onResponseReceivedActions")
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final Response response = getDownloader().post(page.getUrl(), null, json, getExtractorLocalization());
final JsonObject ajaxJson = toJsonObject(getValidJsonResponseBody(response));
final JsonArray continuation = ajaxJson.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("appendContinuationItemsAction")
.getArray("continuationItems");
@ -220,7 +247,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
}
private Page getNextPageFrom(final JsonArray contents) {
private Page getNextPageFrom(final JsonArray contents) throws IOException, ExtractionException {
if (isNullOrEmpty(contents)) {
return null;
}
@ -232,7 +259,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
.getObject("continuationEndpoint")
.getObject("continuationCommand")
.getString("token");
return new Page("https://www.youtube.com/browse_ajax?continuation=" + continuation);
return new Page(
"https://www.youtube.com/youtubei/v1/browse?key=" + getKey(),
continuation);
} else {
return null;
}

View File

@ -2,14 +2,18 @@ package org.schabi.newpipe.extractor.utils;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
public class JsonUtils {
public static final JsonObject EMPTY_OBJECT = new JsonObject();
public static final JsonArray EMPTY_ARRAY = new JsonArray();
@ -99,4 +103,19 @@ public class JsonUtils {
return result;
}
public static JsonArray toJsonArray(final String responseBody) throws ParsingException {
try {
return JsonParser.array().from(responseBody);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse JSON", e);
}
}
public static JsonObject toJsonObject(final String responseBody) throws ParsingException {
try {
return JsonParser.object().from(responseBody);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse JSON", e);
}
}
}

View File

@ -13,7 +13,11 @@ import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.*;
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.ContinuationsTests;
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.HugePlaylist;
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.LearningPlaylist;
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.NotAvailable;
import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.TimelessPopHits;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
@ -410,7 +414,7 @@ public class YoutubePlaylistExtractorTest {
public void testOnlySingleContinuation() throws Exception {
final YoutubePlaylistExtractor extractor = (YoutubePlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/playlist?list=PLjgwFL8urN2DFRuRkFTkmtHjyoNWHHdZX");
"https://www.youtube.com/playlist?list=PLoumn5BIsUDeGF1vy5Nylf_RJKn5aL_nr");
extractor.fetchPage();
final ListExtractor.InfoItemsPage<StreamInfoItem> page = defaultTestMoreItems(