diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java index 9099cb1a7..cea663edb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java @@ -1,6 +1,8 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems; import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -23,7 +25,7 @@ public class MediaCCCConferenceInfoItemExtractor implements ChannelInfoItemExtra @Override public long getStreamCount() { - return -1; + return ListExtractor.ITEM_COUNT_UNKNOWN; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 6ea588341..835ddd309 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -5,8 +5,10 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; + import org.jsoup.Jsoup; import org.jsoup.nodes.Document; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -21,6 +23,7 @@ import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.OffsetDateTime; import java.time.ZoneOffset; @@ -35,6 +38,7 @@ import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.HTTP; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.utils.Utils.join; /* * Created by Christian Schabesberger on 02.03.16. @@ -61,6 +65,12 @@ public class YoutubeParsingHelper { private YoutubeParsingHelper() { } + /** + * The official youtube app supports intents in this format, where after the ':' is the videoId. + * Accordingly there are other apps sharing streams in this format. + */ + public final static String BASE_YOUTUBE_INTENT_URL = "vnd.youtube"; + private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; private static String clientVersion; @@ -192,6 +202,57 @@ public class YoutubeParsingHelper { } } + /** + * Checks if the given playlist id is a YouTube Mix (auto-generated playlist) + * Ids from a YouTube Mix start with "RD" + * @param playlistId + * @return Whether given id belongs to a YouTube Mix + */ + public static boolean isYoutubeMixId(final String playlistId) { + return playlistId.startsWith("RD") && !isYoutubeMusicMixId(playlistId); + } + + /** + * Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist) + * Ids from a YouTube Music Mix start with "RDAMVM" + * @param playlistId + * @return Whether given id belongs to a YouTube Music Mix + */ + public static boolean isYoutubeMusicMixId(final String playlistId) { + return playlistId.startsWith("RDAMVM"); + } + /** + * Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist) + * Ids from a YouTube channel Mix start with "RDCM" + * @return Whether given id belongs to a YouTube Channel Mix + */ + public static boolean isYoutubeChannelMixId(final String playlistId) { + return playlistId.startsWith("RDCM"); + } + + /** + * Extracts the video id from the playlist id for Mixes. + * @throws ParsingException If the playlistId is a Channel Mix or not a mix. + */ + public static String extractVideoIdFromMixId(final String playlistId) throws ParsingException { + if (playlistId.startsWith("RDMM")) { //My Mix + return playlistId.substring(4); + + } else if (playlistId.startsWith("RDAMVM")) { //Music mix + return playlistId.substring(6); + + } else if (playlistId.startsWith("RMCM")) { //Channel mix + //Channel mix are build with RMCM{channelId}, so videoId can't be determined + throw new ParsingException("Video id could not be determined from mix id: " + playlistId); + + } else if (playlistId.startsWith("RD")) { // Normal mix + return playlistId.substring(2); + + } else { //not a mix + throw new ParsingException("Video id could not be determined from mix id: " + playlistId); + } + } + public static JsonObject getInitialData(String html) throws ParsingException { try { try { @@ -416,10 +477,14 @@ public class YoutubeParsingHelper { } else if (navigationEndpoint.has("watchEndpoint")) { StringBuilder url = new StringBuilder(); url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId")); - if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) - url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId")); - if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) - url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds")); + if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) { + url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint") + .getString("playlistId")); + } + if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) { + url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint") + .getInt("startTimeSeconds")); + } return url.toString(); } else if (navigationEndpoint.has("watchPlaylistEndpoint")) { return "https://www.youtube.com/playlist?list=" + @@ -485,8 +550,8 @@ public class YoutubeParsingHelper { public static String getValidJsonResponseBody(final Response response) throws ParsingException, MalformedURLException { if (response.responseCode() == 404) { - throw new ContentNotAvailableException("Not found" + - " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); + throw new ContentNotAvailableException("Not found" + + " (\"" + response.responseCode() + " " + response.responseMessage() + "\")"); } final String responseBody = response.responseBody(); @@ -506,13 +571,39 @@ public class YoutubeParsingHelper { final String responseContentType = response.getHeader("Content-Type"); if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { - throw new ParsingException("Got HTML document, expected JSON response" + - " (latest url was: \"" + response.latestUrl() + "\")"); + throw new ParsingException("Got HTML document, expected JSON response" + + " (latest url was: \"" + response.latestUrl() + "\")"); } return responseBody; } + public static Response getResponse(final String url, final Localization localization) + throws IOException, ExtractionException { + final Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); + + final Response response = getDownloader().get(url, headers, localization); + getValidJsonResponseBody(response); + + return response; + } + + public static String extractCookieValue(final String cookieName, final Response response) { + final List cookies = response.responseHeaders().get("Set-Cookie"); + int startIndex; + String result = ""; + for (final String cookie : cookies) { + startIndex = cookie.indexOf(cookieName); + if (startIndex != -1) { + result = cookie.substring(startIndex + cookieName.length() + "=".length(), + cookie.indexOf(";", startIndex)); + } + } + return result; + } + public static JsonArray getJsonResponse(final String url, final Localization localization) throws IOException, ExtractionException { Map> headers = new HashMap<>(); @@ -520,8 +611,24 @@ public class YoutubeParsingHelper { headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); final Response response = getDownloader().get(url, headers, localization); - final String responseBody = getValidJsonResponseBody(response); + return toJsonArray(getValidJsonResponseBody(response)); + } + public static JsonArray getJsonResponse(final Page page, final Localization localization) + throws IOException, ExtractionException { + final Map> headers = new HashMap<>(); + if (!isNullOrEmpty(page.getCookies())) { + headers.put("Cookie", Collections.singletonList(join(";", "=", page.getCookies()))); + } + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); + + final Response response = getDownloader().get(page.getUrl(), headers, localization); + + return toJsonArray(getValidJsonResponseBody(response)); + } + + public static JsonArray toJsonArray(final String responseBody) throws ParsingException { try { return JsonParser.array().from(responseBody); } catch (JsonParserException e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 519672141..997fd0b73 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -20,6 +20,7 @@ import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMusicSearchExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; @@ -109,8 +110,12 @@ public class YoutubeService extends StreamingService { } @Override - public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) { - return new YoutubePlaylistExtractor(this, linkHandler); + public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) { + if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) { + return new YoutubeMixPlaylistExtractor(this, linkHandler); + } else { + return new YoutubePlaylistExtractor(this, linkHandler); + } } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 881fbd794..fbab74d83 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; @@ -86,7 +87,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor try { if (!channelInfoItem.has("videoCountText")) { // Video count is not available, channel probably has no public uploads. - return -1; + return ListExtractor.ITEM_COUNT_UNKNOWN; } return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject( diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java new file mode 100644 index 000000000..d424129ef --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -0,0 +1,227 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.toJsonArray; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + +/** + * A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist). + * It handles URLs in the format of + * {@code youtube.com/watch?v=videoId&list=playlistId} + */ +public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { + + /** + * YouTube identifies mixes based on this cookie. With this information it can generate + * continuations without duplicates. + */ + public static final String COOKIE_NAME = "VISITOR_INFO1_LIVE"; + + private JsonObject initialData; + private JsonObject playlistData; + private String cookieValue; + + public YoutubeMixPlaylistExtractor(final StreamingService service, + final ListLinkHandler linkHandler) { + super(service, linkHandler); + } + + @Override + public void onFetchPage(@Nonnull final Downloader downloader) + throws IOException, ExtractionException { + final String url = getUrl() + "&pbj=1"; + final Response response = getResponse(url, getExtractorLocalization()); + final JsonArray ajaxJson = toJsonArray(response.responseBody()); + initialData = ajaxJson.getObject(3).getObject("response"); + playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults") + .getObject("playlist").getObject("playlist"); + cookieValue = extractCookieValue(COOKIE_NAME, response); + } + + @Nonnull + @Override + public String getName() throws ParsingException { + final String name = playlistData.getString("title"); + if (name == null) { + throw new ParsingException("Could not get playlist name"); + } + return name; + } + + @Override + public String getThumbnailUrl() throws ParsingException { + try { + return getThumbnailUrlFromPlaylistId(playlistData.getString("playlistId")); + } catch (final Exception e) { + try { + //fallback to thumbnail of current video. Always the case for channel mix + return getThumbnailUrlFromVideoId( + initialData.getObject("currentVideoEndpoint").getObject("watchEndpoint") + .getString("videoId")); + } catch (final Exception ignored) { + } + throw new ParsingException("Could not get playlist thumbnail", e); + } + } + + @Override + public String getBannerUrl() { + return ""; + } + + @Override + public String getUploaderUrl() { + //Youtube mix are auto-generated + return ""; + } + + @Override + public String getUploaderName() { + //Youtube mix are auto-generated by YouTube + return "YouTube"; + } + + @Override + public String getUploaderAvatarUrl() { + //Youtube mix are auto-generated by YouTube + return ""; + } + + @Override + public long getStreamCount() { + // Auto-generated playlist always start with 25 videos and are endless + return ListExtractor.ITEM_COUNT_INFINITE; + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws ExtractionException { + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + collectStreamsFrom(collector, playlistData.getArray("contents")); + return new InfoItemsPage<>(collector, + new Page(getNextPageUrlFrom(playlistData), Collections.singletonMap(COOKIE_NAME, cookieValue))); + } + + private String getNextPageUrlFrom(final JsonObject playlistJson) throws ExtractionException { + final JsonObject lastStream = ((JsonObject) playlistJson.getArray("contents") + .get(playlistJson.getArray("contents").size() - 1)); + if (lastStream == null || lastStream.getObject("playlistPanelVideoRenderer") == null) { + throw new ExtractionException("Could not extract next page url"); + } + + return getUrlFromNavigationEndpoint( + lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint")) + + "&pbj=1"; + } + + @Override + public InfoItemsPage getPage(final Page page) + throws ExtractionException, IOException { + if (page == null || isNullOrEmpty(page.getUrl())) { + throw new IllegalArgumentException("Page url is empty or null"); + } + if (!page.getCookies().containsKey(COOKIE_NAME)) { + throw new IllegalArgumentException("Cooke '" + COOKIE_NAME + "' is missing"); + } + + final JsonArray ajaxJson = getJsonResponse(page, getExtractorLocalization()); + final JsonObject playlistJson = + ajaxJson.getObject(3).getObject("response").getObject("contents") + .getObject("twoColumnWatchNextResults").getObject("playlist") + .getObject("playlist"); + final JsonArray allStreams = playlistJson.getArray("contents"); + // Sublist because youtube returns up to 24 previous streams in the mix + // +1 because the stream of "currentIndex" was already extracted in previous request + final List newStreams = + allStreams.subList(playlistJson.getInt("currentIndex") + 1, allStreams.size()); + + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + collectStreamsFrom(collector, newStreams); + return new InfoItemsPage<>(collector, + new Page(getNextPageUrlFrom(playlistJson), page.getCookies())); + } + + private void collectStreamsFrom( + @Nonnull final StreamInfoItemsCollector collector, + @Nullable final List streams) { + + if (streams == null) { + return; + } + + final TimeAgoParser timeAgoParser = getTimeAgoParser(); + + for (final Object stream : streams) { + if (stream instanceof JsonObject) { + final JsonObject streamInfo = ((JsonObject) stream) + .getObject("playlistPanelVideoRenderer"); + if (streamInfo != null) { + collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser)); + } + } + } + } + + private String getThumbnailUrlFromPlaylistId(final String playlistId) throws ParsingException { + final String videoId; + if (playlistId.startsWith("RDMM")) { + videoId = playlistId.substring(4); + } else if (playlistId.startsWith("RDCMUC")) { + throw new ParsingException("is channel mix"); + } else { + videoId = playlistId.substring(2); + } + if (videoId.isEmpty()) { + throw new ParsingException("videoId is empty"); + } + return getThumbnailUrlFromVideoId(videoId); + } + + private String getThumbnailUrlFromVideoId(final String videoId) { + return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; + } + + @Nonnull + @Override + public String getSubChannelName() { + return ""; + } + + @Nonnull + @Override + public String getSubChannelUrl() { + return ""; + } + + @Nonnull + @Override + public String getSubChannelAvatarUrl() { + return ""; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index 56abc194b..aa2908e64 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -1,60 +1,72 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.List; import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.Utils; -import java.net.URL; -import java.util.List; - public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { - private static final YoutubePlaylistLinkHandlerFactory instance = new YoutubePlaylistLinkHandlerFactory(); + private static final YoutubePlaylistLinkHandlerFactory INSTANCE = + new YoutubePlaylistLinkHandlerFactory(); public static YoutubePlaylistLinkHandlerFactory getInstance() { - return instance; + return INSTANCE; } @Override - public String getUrl(String id, List contentFilters, String sortFilter) { + public String getUrl(final String id, final List contentFilters, + final String sortFilter) { return "https://www.youtube.com/playlist?list=" + id; } @Override - public String getId(String url) throws ParsingException { + public String getId(final String url) throws ParsingException { try { - URL urlObj = Utils.stringToURL(url); + final URL urlObj = Utils.stringToURL(url); if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj) || YoutubeParsingHelper.isInvidioURL(urlObj))) { throw new ParsingException("the url given is not a Youtube-URL"); } - String path = urlObj.getPath(); + final String path = urlObj.getPath(); if (!path.equals("/watch") && !path.equals("/playlist")) { throw new ParsingException("the url given is neither a video nor a playlist URL"); } - String listID = Utils.getQueryValue(urlObj, "list"); + final String listID = Utils.getQueryValue(urlObj, "list"); if (listID == null) { throw new ParsingException("the url given does not include a playlist"); } if (!listID.matches("[a-zA-Z0-9_-]{10,}")) { - throw new ParsingException("the list-ID given in the URL does not match the list pattern"); + throw new ParsingException( + "the list-ID given in the URL does not match the list pattern"); } - // Don't accept auto-generated "Mix" playlists but auto-generated YouTube Music playlists - if (listID.startsWith("RD") && !listID.startsWith("RDCLAK")) { - throw new ContentNotSupportedException("YouTube Mix playlists are not yet supported"); + if (YoutubeParsingHelper.isYoutubeMusicMixId(listID)) { + throw new ContentNotSupportedException( + "YouTube Music Mix playlists are not yet supported"); + } + + if (YoutubeParsingHelper.isYoutubeChannelMixId(listID) + && Utils.getQueryValue(urlObj, "v") == null) { + //Video id can't be determined from the channel mix id. See YoutubeParsingHelper#extractVideoIdFromMixId + throw new ContentNotSupportedException("Channel Mix without a video id are not supported"); } return listID; } catch (final Exception exception) { - throw new ParsingException("Error could not parse url :" + exception.getMessage(), exception); + throw new ParsingException("Error could not parse url :" + exception.getMessage(), + exception); } } @@ -67,4 +79,33 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { } return true; } + + /** + * * If it is a mix (auto-generated playlist) URL, return a {@link LinkHandler} where the URL is + * like + * https://youtube.com/watch?v=videoId&list=playlistId. + *

Otherwise use super

+ */ + @Override + public ListLinkHandler fromUrl(final String url) throws ParsingException { + try { + final URL urlObj = Utils.stringToURL(url); + final String listID = Utils.getQueryValue(urlObj, "list"); + if (listID != null && YoutubeParsingHelper.isYoutubeMixId(listID)) { + String videoID = Utils.getQueryValue(urlObj, "v"); + if (videoID == null) { + videoID = YoutubeParsingHelper.extractVideoIdFromMixId(listID); + } + final String newUrl = "https://www.youtube.com/watch?v=" + videoID + + "&list=" + listID; + return new ListLinkHandler(new LinkHandler(url, newUrl, listID), + getContentFilter(url), + getSortFilter(url)); + } + } catch (MalformedURLException exception) { + throw new ParsingException("Error could not parse url :" + exception.getMessage(), + exception); + } + return super.fromUrl(url); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 959202700..3c2bc7d9f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -8,6 +8,7 @@ import java.net.URL; import java.net.URLDecoder; import java.util.Collection; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -260,4 +261,15 @@ public class Utils { } return stringBuilder.toString(); } + + public static String join(final String delimiter, final String mapJoin, + final Map elements) { + final List list = new LinkedList<>(); + for (final Map.Entry entry : elements + .entrySet()) { + list.add(entry.getKey() + mapJoin + entry.getValue()); + } + return join(delimiter, list); + } + } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java new file mode 100644 index 000000000..49fb3fe04 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -0,0 +1,345 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import org.hamcrest.MatcherAssert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; +import org.junit.runners.Suite.SuiteClasses; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.ChannelMix; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.Invalid; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.Mix; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.MixWithIndex; +import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractorTest.MyMix; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.startsWith; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + +@RunWith(Suite.class) +@SuiteClasses({Mix.class, MixWithIndex.class, MyMix.class, Invalid.class, ChannelMix.class}) +public class YoutubeMixPlaylistExtractorTest { + + public static final String PBJ = "&pbj=1"; + private static final String VIDEO_ID = "_AzeUSL9lZc"; + private static final String VIDEO_TITLE = + "Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO"; + + private static YoutubeMixPlaylistExtractor extractor; + private static Map dummyCookie + = Collections.singletonMap(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever"); + + public static class Mix { + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID); + extractor.fetchPage(); + } + + @Test + public void getServiceId() { + assertEquals(YouTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void getName() throws Exception { + final String name = extractor.getName(); + assertThat(name, startsWith("Mix")); + assertThat(name, containsString(VIDEO_TITLE)); + } + + @Test + public void getThumbnailUrl() throws Exception { + final String thumbnailUrl = extractor.getThumbnailUrl(); + assertIsSecureUrl(thumbnailUrl); + MatcherAssert.assertThat(thumbnailUrl, containsString("yt")); + assertThat(thumbnailUrl, containsString(VIDEO_ID)); + } + + @Test + public void getInitialPage() throws Exception { + final InfoItemsPage streams = extractor.getInitialPage(); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getPage() throws Exception { + final InfoItemsPage streams = extractor.getPage( + new Page("https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID + + PBJ, dummyCookie)); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getContinuations() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + final Set urls = new HashSet<>(); + + //Should work infinitely, but for testing purposes only 3 times + for (int i = 0; i < 3; i++) { + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + + for (final StreamInfoItem item : streams.getItems()) { + assertFalse(urls.contains(item.getUrl())); + urls.add(item.getUrl()); + } + + streams = extractor.getPage(streams.getNextPage()); + } + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + } + + @Test + public void getStreamCount() { + assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); + } + } + + public static class MixWithIndex { + + private static final String INDEX = "&index=13"; + private static final String VIDEO_ID_NUMBER_13 = "qHtzO49SDmk"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD" + + VIDEO_ID + INDEX); + extractor.fetchPage(); + } + + @Test + public void getName() throws Exception { + final String name = extractor.getName(); + assertThat(name, startsWith("Mix")); + assertThat(name, containsString(VIDEO_TITLE)); + } + + @Test + public void getThumbnailUrl() throws Exception { + final String thumbnailUrl = extractor.getThumbnailUrl(); + assertIsSecureUrl(thumbnailUrl); + assertThat(thumbnailUrl, containsString("yt")); + assertThat(thumbnailUrl, containsString(VIDEO_ID)); + } + + @Test + public void getInitialPage() throws Exception { + final InfoItemsPage streams = extractor.getInitialPage(); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getPage() throws Exception { + final InfoItemsPage streams = extractor.getPage( + new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD" + + VIDEO_ID + INDEX + PBJ, dummyCookie)); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getContinuations() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + final Set urls = new HashSet<>(); + + //Should work infinitely, but for testing purposes only 3 times + for (int i = 0; i < 3; i++) { + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + for (final StreamInfoItem item : streams.getItems()) { + assertFalse(urls.contains(item.getUrl())); + urls.add(item.getUrl()); + } + + streams = extractor.getPage(streams.getNextPage()); + } + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + } + + @Test + public void getStreamCount() { + assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); + } + } + + public static class MyMix { + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RDMM" + + VIDEO_ID); + extractor.fetchPage(); + } + + @Test + public void getServiceId() { + assertEquals(YouTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void getName() throws Exception { + final String name = extractor.getName(); + assertEquals("My Mix", name); + } + + @Test + public void getThumbnailUrl() throws Exception { + final String thumbnailUrl = extractor.getThumbnailUrl(); + assertIsSecureUrl(thumbnailUrl); + assertThat(thumbnailUrl, startsWith("https://i.ytimg.com/vi/_AzeUSL9lZc")); + } + + @Test + public void getInitialPage() throws Exception { + final InfoItemsPage streams = extractor.getInitialPage(); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getPage() throws Exception { + final InfoItemsPage streams = + extractor.getPage(new Page("https://www.youtube.com/watch?v=" + VIDEO_ID + + "&list=RDMM" + VIDEO_ID + PBJ, dummyCookie)); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + @Test + public void getContinuations() throws Exception { + InfoItemsPage streams = extractor.getInitialPage(); + final Set urls = new HashSet<>(); + + //Should work infinitely, but for testing purposes only 3 times + for (int i = 0; i < 3; i++) { + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + + for (final StreamInfoItem item : streams.getItems()) { + assertFalse(urls.contains(item.getUrl())); + urls.add(item.getUrl()); + } + + streams = extractor.getPage(streams.getNextPage()); + } + assertTrue(streams.hasNextPage()); + assertFalse(streams.getItems().isEmpty()); + } + + @Test + public void getStreamCount() { + assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); + } + } + + public static class Invalid { + + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test(expected = IllegalArgumentException.class) + public void getPageEmptyUrl() throws Exception { + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID); + extractor.fetchPage(); + extractor.getPage(new Page("")); + } + + @Test(expected = ExtractionException.class) + public void invalidVideoId() throws Exception { + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + "abcde" + "&list=RD" + "abcde"); + extractor.fetchPage(); + extractor.getName(); + } + } + + public static class ChannelMix { + + private static final String CHANNEL_ID = "UCXuqSBlHAE6Xw-yeJA0Tunw"; + private static final String VIDEO_ID_OF_CHANNEL = "mnk6gnOBYIo"; + private static final String CHANNEL_TITLE = "Linus Tech Tips"; + + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (YoutubeMixPlaylistExtractor) YouTube + .getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL + + "&list=RDCM" + CHANNEL_ID); + extractor.fetchPage(); + } + + @Test + public void getName() throws Exception { + final String name = extractor.getName(); + assertThat(name, startsWith("Mix")); + assertThat(name, containsString(CHANNEL_TITLE)); + } + + @Test + public void getThumbnailUrl() throws Exception { + final String thumbnailUrl = extractor.getThumbnailUrl(); + assertIsSecureUrl(thumbnailUrl); + assertThat(thumbnailUrl, containsString("yt")); + } + + @Test + public void getInitialPage() throws Exception { + final InfoItemsPage streams = extractor.getInitialPage(); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getPage() throws Exception { + final InfoItemsPage streams = extractor.getPage( + new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL + + "&list=RDCM" + CHANNEL_ID + PBJ, dummyCookie)); + assertFalse(streams.getItems().isEmpty()); + assertTrue(streams.hasNextPage()); + } + + @Test + public void getStreamCount() { + assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount()); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java index 636a646f8..e7bec115d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java @@ -56,7 +56,7 @@ public class YoutubePlaylistLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); assertTrue(linkHandler.acceptUrl("https://music.youtube.com/playlist?list=OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM")); assertTrue(linkHandler.acceptUrl("https://www.youtube.com/playlist?list=RDCLAK5uy_ly6s4irLuZAcjEDwJmqcA_UtSipMyGgbQ")); // YouTube Music playlist - assertFalse(linkHandler.acceptUrl("https://www.youtube.com/watch?v=2kZVEUGLgy4&list=RDdoEcQv1wlsI&index=2, ")); // YouTube Mix + assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch?v=2kZVEUGLgy4&list=RDdoEcQv1wlsI&index=2, ")); // YouTube Mix } @Test @@ -105,4 +105,23 @@ public class YoutubePlaylistLinkHandlerFactoryTest { assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId()); } + + @Test + public void fromUrlIsMixVideo() throws Exception { + final String videoId = "_AzeUSL9lZc"; + String url = "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId; + assertEquals(url, linkHandler.fromUrl(url).getUrl()); + + final String mixVideoId = "qHtzO49SDmk"; + url = "https://www.youtube.com/watch?v=" + mixVideoId + "&list=RD" + videoId; + assertEquals(url, linkHandler.fromUrl(url).getUrl()); + } + + @Test + public void fromUrlIsMixPlaylist() throws Exception { + final String videoId = "_AzeUSL9lZc"; + final String url = "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId; + assertEquals(url, + linkHandler.fromUrl("https://www.youtube.com/watch?list=RD" + videoId).getUrl()); + } } \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java index 6de39e5ce..4354fa258 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java @@ -26,9 +26,13 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.kiosk.KioskList; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.YouTube; /** @@ -54,4 +58,30 @@ public class YoutubeServiceTest { public void testGetDefaultKiosk() throws Exception { assertEquals(kioskList.getDefaultKioskExtractor(null).getId(), "Trending"); } + + + @Test + public void getPlayListExtractorIsNormalPlaylist() throws Exception { + final PlaylistExtractor extractor = service.getPlaylistExtractor( + "https://www.youtube.com/watch?v=JhqtYOnNrTs&list=PL-EkZZikQIQVqk9rBWzEo5b-2GeozElS"); + assertTrue(extractor instanceof YoutubePlaylistExtractor); + } + + @Test + public void getPlaylistExtractorIsMix() throws Exception { + final String videoId = "_AzeUSL9lZc"; + PlaylistExtractor extractor = YouTube.getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId); + assertTrue(extractor instanceof YoutubeMixPlaylistExtractor); + + extractor = YouTube.getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + videoId + "&list=RDMM" + videoId); + assertTrue(extractor instanceof YoutubeMixPlaylistExtractor); + + final String mixVideoId = "qHtzO49SDmk"; + + extractor = YouTube.getPlaylistExtractor( + "https://www.youtube.com/watch?v=" + mixVideoId + "&list=RD" + videoId); + assertTrue(extractor instanceof YoutubeMixPlaylistExtractor); + } }