lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Support YouTube's learning playlists

This commit is contained in:
wb9688 2020-05-13 17:26:07 +02:00
parent 0b4977bb0c
commit 4d683e7655
2 changed files with 196 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -8,16 +8,21 @@ import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
@ -152,34 +157,47 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Nonnull
@Override
public String getSubChannelName() throws ParsingException {
public String getSubChannelName() {
return "";
}
@Nonnull
@Override
public String getSubChannelUrl() throws ParsingException {
public String getSubChannelUrl() {
return "";
}
@Nonnull
@Override
public String getSubChannelAvatarUrl() throws ParsingException {
public String getSubChannelAvatarUrl() {
return "";
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
JsonArray videos = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
final JsonArray contents = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content")
.getObject("sectionListRenderer").getArray("contents").getObject(0)
.getObject("itemSectionRenderer").getArray("contents").getObject(0)
.getObject("playlistVideoListRenderer").getArray("contents");
.getObject("itemSectionRenderer").getArray("contents");
if (contents.getObject(0).has("playlistSegmentRenderer")) {
for (final Object segment : contents) {
if (((JsonObject) segment).getObject("playlistSegmentRenderer").has("trailer")) {
collectTrailerFrom(collector, ((JsonObject) segment));
} else if (((JsonObject) segment).getObject("playlistSegmentRenderer").has("videoList")) {
collectStreamsFrom(collector, ((JsonObject) segment).getObject("playlistSegmentRenderer")
.getObject("videoList").getObject("playlistVideoListRenderer").getArray("contents"));
}
}
} else if (contents.getObject(0).has("playlistVideoListRenderer")) {
final JsonArray videos = contents.getObject(0)
.getObject("playlistVideoListRenderer").getArray("contents");
collectStreamsFrom(collector, videos);
}
collectStreamsFrom(collector, videos);
return new InfoItemsPage<>(collector, getNextPageUrl());
}
@ -189,10 +207,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
final JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
.getObject("continuationContents").getObject("playlistVideoListContinuation");
collectStreamsFrom(collector, sectionListContinuation.getArray("contents"));
@ -200,7 +218,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
return new InfoItemsPage<>(collector, getNextPageUrlFrom(sectionListContinuation.getArray("continuations")));
}
private String getNextPageUrlFrom(JsonArray continuations) {
private String getNextPageUrlFrom(final JsonArray continuations) {
if (isNullOrEmpty(continuations)) {
return "";
}
@ -212,9 +230,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
+ "&itct=" + clickTrackingParams;
}
private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) {
collector.reset();
private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonArray videos) {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object video : videos) {
@ -228,4 +244,72 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
}
}
private void collectTrailerFrom(final StreamInfoItemsCollector collector,
final JsonObject segment) {
collector.commit(new StreamInfoItemExtractor() {
@Override
public String getName() throws ParsingException {
return getTextFromObject(segment.getObject("playlistSegmentRenderer")
.getObject("title"));
}
@Override
public String getUrl() throws ParsingException {
return YoutubeStreamLinkHandlerFactory.getInstance()
.fromId(segment.getObject("playlistSegmentRenderer").getObject("trailer")
.getObject("playlistVideoPlayerRenderer").getString("videoId"))
.getUrl();
}
@Override
public String getThumbnailUrl() {
return null;
}
@Override
public StreamType getStreamType() {
return StreamType.VIDEO_STREAM;
}
@Override
public boolean isAd() {
return false;
}
@Override
public long getDuration() throws ParsingException {
return YoutubeParsingHelper.parseDurationString(
getTextFromObject(segment.getObject("playlistSegmentRenderer")
.getObject("segmentAnnotation")).split("•")[0]);
}
@Override
public long getViewCount() {
return -1;
}
@Override
public String getUploaderName() throws ParsingException {
return YoutubePlaylistExtractor.this.getUploaderName();
}
@Override
public String getUploaderUrl() throws ParsingException {
return YoutubePlaylistExtractor.this.getUploaderUrl();
}
@Nullable
@Override
public String getTextualUploadDate() {
return null;
}
@Nullable
@Override
public DateWrapper getUploadDate() {
return null;
}
});
}
}

98
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View File

@ -254,4 +254,102 @@ public class YoutubePlaylistExtractorTest {
assertTrue("Error in the streams count", extractor.getStreamCount() > 100);
}
}
public static class LearningPlaylist implements BasePlaylistExtractorTest {
private static YoutubePlaylistExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubePlaylistExtractor) YouTube
.getPlaylistExtractor("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8");
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testServiceId() {
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
}
@Test
public void testName() throws Exception {
String name = extractor.getName();
assertTrue(name, name.startsWith("Anatomy & Physiology"));
}
@Test
public void testId() throws Exception {
assertEquals("PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8", extractor.getId());
}
@Test
public void testUrl() throws ParsingException {
assertEquals("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Ignore
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
// PlaylistExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testThumbnailUrl() throws Exception {
final String thumbnailUrl = extractor.getThumbnailUrl();
assertIsSecureUrl(thumbnailUrl);
assertTrue(thumbnailUrl, thumbnailUrl.contains("yt"));
}
@Ignore
@Test
public void testBannerUrl() throws Exception {
final String bannerUrl = extractor.getBannerUrl();
assertIsSecureUrl(bannerUrl);
assertTrue(bannerUrl, bannerUrl.contains("yt"));
}
@Test
public void testUploaderUrl() throws Exception {
assertEquals("https://www.youtube.com/channel/UCX6b17PVsYBQ0ip5gyeme-Q", extractor.getUploaderUrl());
}
@Test
public void testUploaderName() throws Exception {
final String uploaderName = extractor.getUploaderName();
assertTrue(uploaderName, uploaderName.contains("CrashCourse"));
}
@Test
public void testUploaderAvatarUrl() throws Exception {
final String uploaderAvatarUrl = extractor.getUploaderAvatarUrl();
assertTrue(uploaderAvatarUrl, uploaderAvatarUrl.contains("yt"));
}
@Test
public void testStreamCount() throws Exception {
assertTrue("Error in the streams count", extractor.getStreamCount() > 40);
}
}
}