From 3ff8619bcc703920705431e217d549726cefb830 Mon Sep 17 00:00:00 2001 From: Xiang Rong Lin <41164160+XiangRongLin@users.noreply.github.com> Date: Sat, 21 Mar 2020 18:48:12 +0100 Subject: [PATCH] [Youtube] apply wb9688 suggestion (mix) Channel mix adjusments and test Don't accept youtube music mix urls as playlist Don't override playlistData to keep getInitialPage() Remove json constants Indentation --- .../youtube/YoutubeParsingHelper.java | 23 +++++-- .../services/youtube/YoutubeService.java | 2 +- .../YoutubeMixPlaylistExtractor.java | 67 +++++++++++-------- .../YoutubePlaylistLinkHandlerFactory.java | 5 +- .../YoutubeMixPlaylistExtractorTest.java | 19 +----- 5 files changed, 64 insertions(+), 52 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 258799692..4ac9724b2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -193,13 +193,23 @@ public class YoutubeParsingHelper { } /** - * Checks if the given playlist id is a mix (auto-generated playlist) - * Ids from a mix start with "RD" + * Checks if the given playlist id is a youtube mix (auto-generated playlist) + * Ids from a youtube mix start with "RD" * @param playlistId - * @return Whether given id belongs to a mix + * @return Whether given id belongs to a youtube mix */ public static boolean isYoutubeMixId(String playlistId) { - return playlistId.startsWith("RD"); + return playlistId.startsWith("RD") && !isYoutubeMusicMixId(playlistId); + } + + /** + * Checks if the given playlist id is a youtube music mix (auto-generated playlist) + * Ids from a youtube music mix start with "RD" + * @param playlistId + * @return Whether given id belongs to a youtube music mix + */ + public static boolean isYoutubeMusicMixId(String playlistId) { + return playlistId.startsWith("RDAMVM"); } public static JsonObject getInitialData(String html) throws ParsingException { @@ -427,9 +437,9 @@ public class YoutubeParsingHelper { StringBuilder url = new StringBuilder(); url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId")); if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) - url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId")); + url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId")); if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) - url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds")); + url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds")); return url.toString(); } else if (navigationEndpoint.has("watchPlaylistEndpoint")) { return "https://www.youtube.com/playlist?list=" + @@ -457,6 +467,7 @@ public class YoutubeParsingHelper { if (html && ((JsonObject) textPart).has("navigationEndpoint")) { String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint")); if (!isNullOrEmpty(url)) { + url = url.replaceAll("&", "&"); textBuilder.append("").append(text).append(""); continue; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 7d7a83eba..fce5a1d46 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -144,7 +144,7 @@ public class YoutubeService extends StreamingService { public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String id) - throws ExtractionException { + throws ExtractionException { return new YoutubeTrendingExtractor(YoutubeService.this, new YoutubeTrendingLinkHandlerFactory().fromUrl(url), id); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 060edc20b..0aedafd0e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -25,13 +26,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; */ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { - - private final static String CONTENTS = "contents"; - private final static String RESPONSE = "response"; - private final static String PLAYLIST = "playlist"; - private final static String TWO_COLUMN_WATCH_NEXT_RESULTS = "twoColumnWatchNextResults"; - private final static String PLAYLIST_PANEL_VIDEO_RENDERER = "playlistPanelVideoRenderer"; - + private JsonObject initialData; private JsonObject playlistData; public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { @@ -43,9 +38,9 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { throws IOException, ExtractionException { final String url = getUrl() + "&pbj=1"; final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); - JsonObject initialData = ajaxJson.getObject(3).getObject(RESPONSE); - playlistData = initialData.getObject(CONTENTS).getObject(TWO_COLUMN_WATCH_NEXT_RESULTS) - .getObject(PLAYLIST).getObject(PLAYLIST); + initialData = ajaxJson.getObject(3).getObject("response"); + playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults") + .getObject("playlist").getObject("playlist"); } @Nonnull @@ -62,7 +57,14 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { public String getThumbnailUrl() throws ParsingException { try { final String playlistId = playlistData.getString("playlistId"); - return getThumbnailUrlFromId(playlistId); + try { + return getThumbnailUrlFromPlaylistId(playlistId); + } catch (ParsingException e) { + //fallback to thumbnail of current video. Always the case for channel mix + return getThumbnailUrlFromVideoId( + initialData.getObject("currentVideoEndpoint").getObject("watchEndpoint") + .getString("videoId")); + } } catch (Exception e) { throw new ParsingException("Could not get playlist thumbnail", e); } @@ -101,20 +103,26 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public InfoItemsPage getInitialPage() throws ExtractionException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - collectStreamsFrom(collector, playlistData.getArray(CONTENTS)); + collectStreamsFrom(collector, playlistData.getArray("contents")); return new InfoItemsPage<>(collector, getNextPageUrl()); } @Override public String getNextPageUrl() throws ExtractionException { - final JsonObject lastStream = ((JsonObject) playlistData.getArray(CONTENTS) - .get(playlistData.getArray(CONTENTS).size() - 1)); - if (lastStream == null || lastStream.getObject(PLAYLIST_PANEL_VIDEO_RENDERER) == null) { + return getNextPageUrlFrom(playlistData); + } + + private String getNextPageUrlFrom(JsonObject playlistData) throws ExtractionException { + final JsonObject lastStream = ((JsonObject) playlistData.getArray("contents") + .get(playlistData.getArray("contents").size() - 1)); + if (lastStream == null || lastStream.getObject("playlistPanelVideoRenderer") == null) { throw new ExtractionException("Could not extract next page url"); } - return "https://youtube.com" + lastStream.getObject(PLAYLIST_PANEL_VIDEO_RENDERER) - .getObject("navigationEndpoint").getObject("commandMetadata") - .getObject("webCommandMetadata").getString("url") + "&pbj=1"; + //Index of video in mix is missing, but adding it doesn't appear to have any effect. + //And since the index needs to be tracked by us, it is left out + return getUrlFromNavigationEndpoint( + lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint")) + + "&pbj=1"; } @Override @@ -127,21 +135,20 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); - playlistData = - ajaxJson.getObject(3).getObject(RESPONSE).getObject(CONTENTS) - .getObject(TWO_COLUMN_WATCH_NEXT_RESULTS).getObject(PLAYLIST) - .getObject(PLAYLIST); - final JsonArray streams = playlistData.getArray(CONTENTS); + JsonObject playlistData = + ajaxJson.getObject(3).getObject("response").getObject("contents") + .getObject("twoColumnWatchNextResults").getObject("playlist") + .getObject("playlist"); + final JsonArray streams = playlistData.getArray("contents"); //Because continuation requests are created with the last video of previous request as start streams.remove(0); collectStreamsFrom(collector, streams); - return new InfoItemsPage<>(collector, getNextPageUrl()); + return new InfoItemsPage<>(collector, getNextPageUrlFrom(playlistData)); } private void collectStreamsFrom( @Nonnull StreamInfoItemsCollector collector, @Nullable JsonArray streams) { - collector.reset(); if (streams == null) { return; @@ -152,7 +159,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { for (Object stream : streams) { if (stream instanceof JsonObject) { JsonObject streamInfo = ((JsonObject) stream) - .getObject(PLAYLIST_PANEL_VIDEO_RENDERER); + .getObject("playlistPanelVideoRenderer"); if (streamInfo != null) { collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser)); } @@ -160,16 +167,22 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { } } - private String getThumbnailUrlFromId(String playlistId) throws ParsingException { + private String getThumbnailUrlFromPlaylistId(String playlistId) throws ParsingException { final String videoId; if (playlistId.startsWith("RDMM")) { videoId = playlistId.substring(4); + } else if (playlistId.startsWith("RDCMUC")) { + throw new ParsingException("is channel mix"); } else { videoId = playlistId.substring(2); } if (videoId.isEmpty()) { throw new ParsingException("videoId is empty"); } + return getThumbnailUrlFromVideoId(videoId); + } + + private String getThumbnailUrlFromVideoId(String videoId) { return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index 787372dcb..b565fde62 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -64,11 +64,12 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { @Override public boolean onAcceptUrl(final String url) { try { - getId(url); + String playlistId = getId(url); + //Because youtube music mix are not supported yet. + return !YoutubeParsingHelper.isYoutubeMusicMixId(playlistId); } catch (ParsingException e) { return false; } - return true; } /** diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java index d18e4fc55..b612827a7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; @@ -93,11 +94,6 @@ public class YoutubeMixPlaylistExtractorTest { public void getStreamCount() throws Exception { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } - - @Test - public void getStreamCount() throws Exception { - assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); - } } public static class MixWithIndex { @@ -166,11 +162,6 @@ public class YoutubeMixPlaylistExtractorTest { assertFalse(streams.getItems().isEmpty()); } - @Test - public void getStreamCount() { - assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); - } - @Test public void getStreamCount() throws Exception { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); @@ -264,12 +255,13 @@ public class YoutubeMixPlaylistExtractorTest { extractor.getPage(""); } - @Test(expected = NullPointerException.class) + @Test(expected = ExtractionException.class) public void invalidVideoId() throws Exception { extractor = (YoutubeMixPlaylistExtractor) YouTube .getPlaylistExtractor( "https://www.youtube.com/watch?v=" + "abcde" + "&list=RD" + "abcde"); extractor.fetchPage(); + extractor.getName(); } } @@ -329,10 +321,5 @@ public class YoutubeMixPlaylistExtractorTest { public void getStreamCount() throws Exception { assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); } - - @Test - public void getStreamCount() throws Exception { - assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); - } } } \ No newline at end of file