From 50db871d8982f93086a642080e34350f0c25b220 Mon Sep 17 00:00:00 2001 From: Stypox Date: Wed, 2 Feb 2022 20:23:11 +0100 Subject: [PATCH] [YouTube] Extract mixes from streams related items --- .../youtube/YoutubeParsingHelper.java | 24 +++++- .../YoutubeMixPlaylistExtractor.java | 4 +- .../YoutubeMixPlaylistInfoItemExtractor.java | 85 +++++++++++++++++++ .../extractors/YoutubeStreamExtractor.java | 18 ++-- .../YoutubeStreamInfoItemExtractor.java | 10 +-- 5 files changed, 122 insertions(+), 19 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistInfoItemExtractor.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 48ffbac0b..cef3db86e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -249,6 +249,17 @@ public class YoutubeParsingHelper { return playlistId.startsWith("RD") && !isYoutubeMusicMixId(playlistId); } + /** + * Checks if the given playlist id is a YouTube My Mix (auto-generated playlist) + * Ids from a YouTube My Mix start with "RDMM" + * + * @param playlistId the playlist id + * @return Whether given id belongs to a YouTube My Mix + */ + public static boolean isYoutubeMyMixId(@Nonnull final String playlistId) { + return playlistId.startsWith("RDMM"); + } + /** * Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist) * Ids from a YouTube Music Mix start with "RDAMVM" or "RDCLAK" @@ -278,7 +289,7 @@ public class YoutubeParsingHelper { @Nonnull public static String extractVideoIdFromMixId(@Nonnull final String playlistId) throws ParsingException { - if (playlistId.startsWith("RDMM")) { // My Mix + if (isYoutubeMyMixId(playlistId)) { // My Mix return playlistId.substring(4); } else if (isYoutubeMusicMixId(playlistId)) { // starts with "RDAMVM" or "RDCLAK" @@ -705,6 +716,17 @@ public class YoutubeParsingHelper { return thumbnailUrl; } + public static String getThumbnailUrlFromInfoItem(final JsonObject infoItem) + throws ParsingException { + // TODO: Don't simply get the first item, but look at all thumbnails and their resolution + try { + return fixThumbnailUrl(infoItem.getObject("thumbnail").getArray("thumbnails") + .getObject(0).getString("url")); + } catch (final Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + @Nonnull public static String getValidJsonResponseBody(@Nonnull final Response response) throws ParsingException, MalformedURLException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index a98fa617a..b59384ed1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -234,9 +234,9 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Nonnull private String getThumbnailUrlFromPlaylistId(@Nonnull final String playlistId) throws ParsingException { final String videoId; - if (playlistId.startsWith("RDMM")) { + if (isYoutubeMyMixId(playlistId)) { videoId = playlistId.substring(4); - } else if (playlistId.startsWith("RDCMUC")) { + } else if (isYoutubeChannelMixId(playlistId)) { throw new ParsingException("This playlist is a channel mix"); } else { videoId = playlistId.substring(2); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistInfoItemExtractor.java new file mode 100644 index 000000000..279d55ef3 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistInfoItemExtractor.java @@ -0,0 +1,85 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailUrlFromInfoItem; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeChannelMixId; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeMusicMixId; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + +import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.playlist.PlaylistInfo; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; +import org.schabi.newpipe.extractor.utils.Utils; + +import java.net.MalformedURLException; + +import javax.annotation.Nonnull; + +public class YoutubeMixPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { + private final JsonObject mixInfoItem; + + public YoutubeMixPlaylistInfoItemExtractor(final JsonObject mixInfoItem) { + this.mixInfoItem = mixInfoItem; + } + + @Override + public String getName() throws ParsingException { + final String name = getTextFromObject(mixInfoItem.getObject("title")); + if (isNullOrEmpty(name)) { + throw new ParsingException("Could not get name"); + } + return name; + } + + @Override + public String getUrl() throws ParsingException { + final String url = mixInfoItem.getString("shareUrl"); + if (isNullOrEmpty(url)) { + throw new ParsingException("Could not get url"); + } + return url; + } + + @Override + public String getThumbnailUrl() throws ParsingException { + return getThumbnailUrlFromInfoItem(mixInfoItem); + } + + @Override + public String getUploaderName() throws ParsingException { + // YouTube mixes are auto-generated by YouTube + return "YouTube"; + } + + @Override + public long getStreamCount() throws ParsingException { + // Auto-generated playlists always start with 25 videos and are endless + return ListExtractor.ITEM_COUNT_INFINITE; + } + + @Nonnull + @Override + public PlaylistInfo.PlaylistType getPlaylistType() throws ParsingException { + try { + final String url = getUrl(); + final String mixPlaylistId = Utils.getQueryValue(Utils.stringToURL(url), "list"); + if (isNullOrEmpty(mixPlaylistId)) { + throw new ParsingException("Mix playlist id was null or empty for url " + url); + } + + if (isYoutubeMusicMixId(mixPlaylistId)) { + return PlaylistInfo.PlaylistType.MIX_MUSIC; + } else if (isYoutubeChannelMixId(mixPlaylistId)) { + return PlaylistInfo.PlaylistType.MIX_CHANNEL; + } else { + // either a normal mix based on a stream, or a "my mix" (still based on a stream) + return PlaylistInfo.PlaylistType.MIX_STREAM; + } + } catch (final MalformedURLException e) { + throw new ParsingException("Could not obtain mix playlist id", e); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 836a3976a..feb1218da 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -10,6 +10,7 @@ import org.mozilla.javascript.ScriptableObject; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.MetaInfo; +import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException; @@ -618,7 +619,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nullable @Override - public StreamInfoItemsCollector getRelatedItems() throws ExtractionException { + public MultiInfoItemsCollector getRelatedItems() throws ExtractionException { assertPageFetched(); if (getAgeLimit() != NO_AGE_LIMIT) { @@ -626,8 +627,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } try { - final StreamInfoItemsCollector collector = new StreamInfoItemsCollector( - getServiceId()); + final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); final JsonArray results = nextResponse.getObject("contents") .getObject("twoColumnWatchNextResults").getObject("secondaryResults") @@ -635,10 +635,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for (final Object ul : results) { - if (((JsonObject) ul).has("compactVideoRenderer")) { - collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) ul) - .getObject("compactVideoRenderer"), timeAgoParser)); + for (final Object resultObject : results) { + final JsonObject result = (JsonObject) resultObject; + if (result.has("compactVideoRenderer")) { + collector.commit(new YoutubeStreamInfoItemExtractor( + result.getObject("compactVideoRenderer"), timeAgoParser)); + } else if (result.has("compactRadioRenderer")) { + collector.commit(new YoutubeMixPlaylistInfoItemExtractor( + result.getObject("compactRadioRenderer"))); } } return collector; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index e46a63a53..199c24525 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -252,15 +252,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public String getThumbnailUrl() throws ParsingException { - try { - // TODO: Don't simply get the first item, but look at all thumbnails and their resolution - String url = videoInfo.getObject("thumbnail").getArray("thumbnails") - .getObject(0).getString("url"); - - return fixThumbnailUrl(url); - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } + return getThumbnailUrlFromInfoItem(videoInfo); } private boolean isPremium() {