diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 7c448662f..91450d474 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -2,33 +2,27 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; -import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import javax.annotation.Nonnull; -import static org.schabi.newpipe.extractor.utils.Utils.HTTP; -import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; /* * Created by Christian Schabesberger on 25.07.16. @@ -52,11 +46,8 @@ import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; @SuppressWarnings("WeakerAccess") public class YoutubeChannelExtractor extends ChannelExtractor { - /*package-private*/ static final String CHANNEL_URL_BASE = "https://www.youtube.com/channel/"; - private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000"; - - private Document doc; private JsonObject initialData; + private JsonObject videoTab; public YoutubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); @@ -64,23 +55,27 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - String channelUrl = super.getUrl() + CHANNEL_URL_PARAMETERS; - final Response response = downloader.get(channelUrl, getExtractorLocalization()); - doc = YoutubeParsingHelper.parseAndCheckPage(channelUrl, response); - initialData = YoutubeParsingHelper.getInitialData(response.responseBody()); + final String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid"; + + final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); + + initialData = ajaxJson.getObject(1).getObject("response"); } @Override public String getNextPageUrl() throws ExtractionException { - return getNextPageUrlFrom(getVideoTab().getObject("content").getObject("sectionListRenderer").getArray("continuations")); + if (getVideoTab() == null) return ""; + return getNextPageUrlFrom(getVideoTab().getObject("content").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer") + .getArray("contents").getObject(0).getObject("gridRenderer").getArray("continuations")); } @Nonnull @Override public String getUrl() throws ParsingException { try { - return CHANNEL_URL_BASE + getId(); + return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + getId()); } catch (ParsingException e) { return super.getUrl(); } @@ -109,8 +104,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getAvatarUrl() throws ParsingException { try { - return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar") + String url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar") .getArray("thumbnails").getObject(0).getString("url"); + + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get avatar", e); } @@ -127,17 +124,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { if (url == null || url.contains("s.ytimg.com") || url.contains("default_banner")) { return null; } - // the first characters of the banner URLs are different for each channel and some are not even valid URLs - if (url.startsWith("//")) { - url = url.substring(2); - } - if (url.startsWith(HTTP)) { - url = Utils.replaceHttpWithHttps(url); - } else if (!url.startsWith(HTTPS)) { - url = HTTPS + url; - } - return url; + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get banner", e); } @@ -157,13 +145,17 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final JsonObject subscriberInfo = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText"); if (subscriberInfo != null) { try { - return Utils.mixedNumberWordToLong(subscriberInfo.getArray("runs").getObject(0).getString("text")); + return Utils.mixedNumberWordToLong(getTextFromObject(subscriberInfo)); } catch (NumberFormatException e) { throw new ParsingException("Could not get subscriber count", e); } } else { - // If the element is null, the channel have the subscriber count disabled - return -1; + // If there's no subscribe button, the channel has the subscriber count disabled + if (initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscribeButton") == null) { + return -1; + } else { + return 0; + } } } @@ -181,8 +173,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor { public InfoItemsPage getInitialPage() throws ExtractionException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - JsonArray videos = getVideoTab().getObject("content").getObject("sectionListRenderer").getArray("contents"); - collectStreamsFrom(collector, videos); + if (getVideoTab() != null) { + JsonArray videos = getVideoTab().getObject("content").getObject("sectionListRenderer").getArray("contents") + .getObject(0).getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("gridRenderer").getArray("items"); + collectStreamsFrom(collector, videos); + } return new InfoItemsPage<>(collector, getNextPageUrl()); } @@ -198,46 +194,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor { fetchPage(); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - JsonArray ajaxJson; - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - try { - // Use the hardcoded client version first to get JSON with a structure we know - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); - final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - ajaxJson = JsonParser.array().from(response); - } catch (Exception e) { - try { - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString()))); - final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException ignored) { - throw new ParsingException("Could not parse json data for next streams", e); - } - } + final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") - .getObject("continuationContents").getObject("sectionListContinuation"); + .getObject("continuationContents").getObject("gridContinuation"); - collectStreamsFrom(collector, sectionListContinuation.getArray("contents")); + collectStreamsFrom(collector, sectionListContinuation.getArray("items")); return new InfoItemsPage<>(collector, getNextPageUrlFrom(sectionListContinuation.getArray("continuations"))); } private String getNextPageUrlFrom(JsonArray continuations) { - if (continuations == null) { - return ""; - } + if (continuations == null) return ""; JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); String continuation = nextContinuationData.getString("continuation"); @@ -254,10 +223,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final TimeAgoParser timeAgoParser = getTimeAgoParser(); for (Object video : videos) { - JsonObject videoInfo = ((JsonObject) video).getObject("itemSectionRenderer") - .getArray("contents").getObject(0); - if (videoInfo.getObject("videoRenderer") != null) { - collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo.getObject("videoRenderer"), timeAgoParser) { + if (((JsonObject) video).getObject("gridVideoRenderer") != null) { + collector.commit(new YoutubeStreamInfoItemExtractor( + ((JsonObject) video).getObject("gridVideoRenderer"), timeAgoParser) { @Override public String getUploaderName() { return uploaderName; @@ -273,6 +241,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } private JsonObject getVideoTab() throws ParsingException { + if (this.videoTab != null) return this.videoTab; + JsonArray tabs = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") .getArray("tabs"); JsonObject videoTab = null; @@ -290,6 +260,15 @@ public class YoutubeChannelExtractor extends ChannelExtractor { throw new ParsingException("Could not find Videos tab"); } + try { + if (getTextFromObject(videoTab.getObject("content").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer") + .getArray("contents").getObject(0).getObject("messageRenderer") + .getObject("text")).equals("This channel has no videos.")) + return null; + } catch (Exception ignored) {} + + this.videoTab = videoTab; return videoTab; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 483cd894c..29aa045b7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -7,8 +7,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; -import static org.schabi.newpipe.extractor.utils.Utils.HTTP; -import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; /* * Created by Christian Schabesberger on 12.02.17. @@ -41,15 +41,8 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor public String getThumbnailUrl() throws ParsingException { try { String url = channelInfoItem.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); - if (url.startsWith("//")) { - url = url.substring(2); - } - if (url.startsWith(HTTP)) { - url = Utils.replaceHttpWithHttps(url); - } else if (!url.startsWith(HTTPS)) { - url = HTTPS + url; - } - return url; + + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } @@ -58,7 +51,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public String getName() throws ParsingException { try { - return channelInfoItem.getObject("title").getString("simpleText"); + return getTextFromObject(channelInfoItem.getObject("title")); } catch (Exception e) { throw new ParsingException("Could not get name", e); } @@ -67,7 +60,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public String getUrl() throws ParsingException { try { - String id = "channel/" + channelInfoItem.getString("channelId"); // Does prepending 'channel/' always work? + String id = "channel/" + channelInfoItem.getString("channelId"); return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id); } catch (Exception e) { throw new ParsingException("Could not get url", e); @@ -77,7 +70,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public long getSubscriberCount() throws ParsingException { try { - String subscribers = channelInfoItem.getObject("subscriberCountText").getString("simpleText").split(" ")[0]; + String subscribers = getTextFromObject(channelInfoItem.getObject("subscriberCountText")); return Utils.mixedNumberWordToLong(subscribers); } catch (Exception e) { throw new ParsingException("Could not get subscriber count", e); @@ -87,8 +80,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public long getStreamCount() throws ParsingException { try { - return Long.parseLong(Utils.removeNonDigitCharacters(channelInfoItem.getObject("videoCountText") - .getArray("runs").getObject(0).getString("text"))); + return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(channelInfoItem.getObject("videoCountText")))); } catch (Exception e) { throw new ParsingException("Could not get stream count", e); } @@ -97,7 +89,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public String getDescription() throws ParsingException { try { - return channelInfoItem.getObject("descriptionSnippet").getArray("runs").getObject(0).getString("text"); + return getTextFromObject(channelInfoItem.getObject("descriptionSnippet")); } catch (Exception e) { throw new ParsingException("Could not get description", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 0ac2dcf05..0f57a21a7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -2,37 +2,30 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; -import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; + @SuppressWarnings("WeakerAccess") public class YoutubePlaylistExtractor extends PlaylistExtractor { - - private Document doc; private JsonObject initialData; - private JsonObject uploaderInfo; private JsonObject playlistInfo; public YoutubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { @@ -41,11 +34,11 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String url = getUrl(); - final Response response = downloader.get(url, getExtractorLocalization()); - doc = YoutubeParsingHelper.parseAndCheckPage(url, response); - initialData = YoutubeParsingHelper.getInitialData(response.responseBody()); - uploaderInfo = getUploaderInfo(); + final String url = getUrl() + "&pbj=1"; + + final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); + + initialData = ajaxJson.getObject(1).getObject("response"); playlistInfo = getPlaylistInfo(); } @@ -94,7 +87,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getName() throws ParsingException { try { - String name = playlistInfo.getObject("title").getArray("runs").getObject(0).getString("text"); + String name = getTextFromObject(playlistInfo.getObject("title")); if (name != null) return name; } catch (Exception ignored) {} try { @@ -106,16 +99,23 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getThumbnailUrl() throws ParsingException { + String url = null; + try { - return playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer") + url = playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer") .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); } catch (Exception ignored) {} - try { - return initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail") - .getArray("thumbnails").getObject(0).getString("url"); - } catch (Exception e) { - throw new ParsingException("Could not get playlist thumbnail", e); + + if (url == null) { + try { + url = initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail") + .getArray("thumbnails").getObject(0).getString("url"); + } catch (Exception ignored) {} + + if (url == null) throw new ParsingException("Could not get playlist thumbnail"); } + + return fixThumbnailUrl(url); } @Override @@ -127,8 +127,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderUrl() throws ParsingException { try { - return YoutubeChannelExtractor.CHANNEL_URL_BASE + - uploaderInfo.getObject("navigationEndpoint").getObject("browseEndpoint").getString("browseId"); + return getUrlFromNavigationEndpoint(getUploaderInfo().getObject("navigationEndpoint")); } catch (Exception e) { throw new ParsingException("Could not get playlist uploader url", e); } @@ -137,7 +136,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderName() throws ParsingException { try { - return uploaderInfo.getObject("title").getArray("runs").getObject(0).getString("text"); + return getTextFromObject(getUploaderInfo().getObject("title")); } catch (Exception e) { throw new ParsingException("Could not get playlist uploader name", e); } @@ -146,7 +145,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderAvatarUrl() throws ParsingException { try { - return uploaderInfo.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); + String url = getUploaderInfo().getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); + + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get playlist uploader avatar", e); } @@ -155,7 +156,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public long getStreamCount() throws ParsingException { try { - String viewsText = getPlaylistInfo().getArray("stats").getObject(0).getArray("runs").getObject(0).getString("text"); + String viewsText = getTextFromObject(getPlaylistInfo().getArray("stats").getObject(0)); return Long.parseLong(Utils.removeNonDigitCharacters(viewsText)); } catch (Exception e) { throw new ParsingException("Could not get video count from playlist", e); @@ -184,32 +185,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - JsonArray ajaxJson; - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - try { - // Use the hardcoded client version first to get JSON with a structure we know - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); - final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - ajaxJson = JsonParser.array().from(response); - } catch (Exception e) { - try { - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString()))); - final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException ignored) { - throw new ParsingException("Could not parse json data for next streams", e); - } - } + final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") .getObject("continuationContents").getObject("playlistVideoListContinuation"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java index 358fa2e69..5b4e7afc8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java @@ -7,6 +7,9 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; + public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { private JsonObject playlistInfoItem; @@ -17,8 +20,10 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract @Override public String getThumbnailUrl() throws ParsingException { try { - return playlistInfoItem.getArray("thumbnails").getObject(0).getArray("thumbnails") - .getObject(0).getString("url"); + String url = playlistInfoItem.getArray("thumbnails").getObject(0) + .getArray("thumbnails").getObject(0).getString("url"); + + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } @@ -27,7 +32,7 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract @Override public String getName() throws ParsingException { try { - return playlistInfoItem.getObject("title").getString("simpleText"); + return getTextFromObject(playlistInfoItem.getObject("title")); } catch (Exception e) { throw new ParsingException("Could not get name", e); } @@ -46,7 +51,7 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract @Override public String getUploaderName() throws ParsingException { try { - return playlistInfoItem.getObject("longBylineText").getArray("runs").getObject(0).getString("text"); + return getTextFromObject(playlistInfoItem.getObject("longBylineText")); } catch (Exception e) { throw new ParsingException("Could not get uploader name", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index b06699098..0449a9dbe 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -2,30 +2,24 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; -import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; + /* * Created by Christian Schabesberger on 22.07.2018 * @@ -47,8 +41,6 @@ import javax.annotation.Nonnull; */ public class YoutubeSearchExtractor extends SearchExtractor { - - private Document doc; private JsonObject initialData; public YoutubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { @@ -57,10 +49,11 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String url = getUrl(); - final Response response = downloader.get(url, getExtractorLocalization()); - doc = YoutubeParsingHelper.parseAndCheckPage(url, response); - initialData = YoutubeParsingHelper.getInitialData(response.responseBody()); + final String url = getUrl() + "&pbj=1"; + + final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); + + initialData = ajaxJson.getObject(1).getObject("response"); } @Nonnull @@ -79,8 +72,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { if (showingResultsForRenderer == null) { return ""; } else { - return showingResultsForRenderer.getObject("correctedQuery").getArray("runs") - .getObject(0).getString("text"); + return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); } } @@ -88,11 +80,13 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public InfoItemsPage getInitialPage() throws ExtractionException { InfoItemsSearchCollector collector = getInfoItemSearchCollector(); - JsonArray videos = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") - .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents") - .getObject(0).getObject("itemSectionRenderer").getArray("contents"); + JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") + .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"); + + for (Object section : sections) { + collectStreamsFrom(collector, ((JsonObject) section).getObject("itemSectionRenderer").getArray("contents")); + } - collectStreamsFrom(collector, videos); return new InfoItemsPage<>(collector, getNextPageUrl()); } @@ -110,33 +104,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { } InfoItemsSearchCollector collector = getInfoItemSearchCollector(); - JsonArray ajaxJson; - - Map> headers = new HashMap<>(); - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - - try { - // Use the hardcoded client version first to get JSON with a structure we know - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); - final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - ajaxJson = JsonParser.array().from(response); - } catch (Exception e) { - try { - headers.put("X-YouTube-Client-Version", - Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString()))); - final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); - if (response.length() < 50) { // ensure to have a valid response - throw new ParsingException("Could not parse json data for next streams"); - } - ajaxJson = JsonParser.array().from(response); - } catch (JsonParserException ignored) { - throw new ParsingException("Could not parse json data for next streams", e); - } - } + final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization()); JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") .getObject("continuationContents").getObject("itemSectionContinuation"); @@ -153,8 +121,8 @@ public class YoutubeSearchExtractor extends SearchExtractor { for (Object item : videos) { if (((JsonObject) item).getObject("backgroundPromoRenderer") != null) { - throw new NothingFoundException(((JsonObject) item).getObject("backgroundPromoRenderer") - .getObject("bodyText").getArray("runs").getObject(0).getString("text")); + throw new NothingFoundException(getTextFromObject(((JsonObject) item) + .getObject("backgroundPromoRenderer").getObject("bodyText"))); } else if (((JsonObject) item).getObject("videoRenderer") != null) { collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) item).getObject("videoRenderer"), timeAgoParser)); } else if (((JsonObject) item).getObject("channelRenderer") != null) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index b5d03edcc..3a66dd3cb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -4,8 +4,6 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; @@ -13,8 +11,6 @@ import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; -import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; @@ -24,6 +20,7 @@ import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager; import org.schabi.newpipe.extractor.services.youtube.ItagItem; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.Description; @@ -40,8 +37,6 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; @@ -56,6 +51,11 @@ import java.util.Map; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; + /* * Created by Christian Schabesberger on 06.08.15. * @@ -89,19 +89,20 @@ public class YoutubeStreamExtractor extends StreamExtractor { /*//////////////////////////////////////////////////////////////////////////*/ - private Document doc; + private JsonArray initialAjaxJson; @Nullable private JsonObject playerArgs; @Nonnull private final Map videoInfoPage = new HashMap<>(); private JsonObject playerResponse; private JsonObject initialData; + private JsonObject videoPrimaryInfoRenderer; + private JsonObject videoSecondaryInfoRenderer; + private int ageLimit; @Nonnull private List subtitlesInfos = new ArrayList<>(); - private boolean isAgeRestricted; - public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) { super(service, linkHandler); } @@ -115,16 +116,20 @@ public class YoutubeStreamExtractor extends StreamExtractor { public String getName() throws ParsingException { assertPageFetched(); String title = null; + try { - title = getVideoPrimaryInfoRenderer().getObject("title").getArray("runs").getObject(0).getString("text"); + title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title")); } catch (Exception ignored) {} + if (title == null) { try { title = playerResponse.getObject("videoDetails").getString("title"); } catch (Exception ignored) {} + + if (title == null) throw new ParsingException("Could not get name"); } - if (title != null) return title; - throw new ParsingException("Could not get name"); + + return title; } @Override @@ -144,8 +149,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { } catch (Exception ignored) {} try { - if (getVideoPrimaryInfoRenderer().getObject("dateText").getString("simpleText").startsWith("Premiered")) { - String time = getVideoPrimaryInfoRenderer().getObject("dateText").getString("simpleText").substring(10); + if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).startsWith("Premiered")) { + String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).substring(10); try { // Premiered 20 hours ago TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en")); @@ -163,7 +168,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { // TODO this parses English formatted dates only, we need a better approach to parse the textual date Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse( - getVideoPrimaryInfoRenderer().getObject("dateText").getString("simpleText")); + getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"))); return new SimpleDateFormat("yyyy-MM-dd").format(d); } catch (Exception ignored) {} throw new ParsingException("Could not get upload date"); @@ -187,8 +192,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails"); // the last thumbnail is the one with the highest resolution - return thumbnails.getObject(thumbnails.size() - 1).getString("url"); + String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get thumbnail url"); } @@ -201,55 +207,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); // description with more info on links try { - boolean htmlConversionRequired = false; - JsonArray descriptions = getVideoSecondaryInfoRenderer().getObject("description").getArray("runs"); - StringBuilder descriptionBuilder = new StringBuilder(descriptions.size()); - for (Object textObjectHolder : descriptions) { - JsonObject textHolder = (JsonObject) textObjectHolder; - String text = textHolder.getString("text"); - if (textHolder.getObject("navigationEndpoint") != null) { - // The text is a link. Get the URL it points to and generate a HTML link of it - if (textHolder.getObject("navigationEndpoint").getObject("urlEndpoint") != null) { - String internUrl = textHolder.getObject("navigationEndpoint").getObject("urlEndpoint").getString("url"); - if (internUrl.startsWith("/redirect?")) { - // q parameter can be the first parameter - internUrl = internUrl.substring(10); - String[] params = internUrl.split("&"); - for (String param : params) { - if (param.split("=")[0].equals("q")) { - String url = URLDecoder.decode(param.split("=")[1], StandardCharsets.UTF_8.name()); - if (url != null && !url.isEmpty()) { - descriptionBuilder.append("").append(text).append(""); - htmlConversionRequired = true; - } else { - descriptionBuilder.append(text); - } - break; - } - } - } else if (internUrl.startsWith("http")) { - descriptionBuilder.append("").append(text).append(""); - htmlConversionRequired = true; - } - continue; - } - continue; - } - if (text != null) { - descriptionBuilder.append(text); - } - } - - String description = descriptionBuilder.toString(); - - if (!description.isEmpty()) { - if (htmlConversionRequired) { - description = description.replaceAll("\\n", "
"); - description = description.replaceAll(" ", "  "); - return new Description(description, Description.HTML); - } - return new Description(description, Description.PLAIN_TEXT); - } + String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true); + return new Description(description, Description.HTML); } catch (Exception ignored) { } // raw non-html description @@ -261,17 +220,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public int getAgeLimit() throws ParsingException { - assertPageFetched(); - if (!isAgeRestricted) { - return NO_AGE_LIMIT; - } - try { - return Integer.valueOf(doc.select("meta[property=\"og:restrictions:age\"]") - .attr(CONTENT).replace("+", "")); - } catch (Exception e) { - throw new ParsingException("Could not get age restriction"); - } + public int getAgeLimit() { + if (initialData == null || initialData.isEmpty()) throw new IllegalStateException("initialData is not parsed yet"); + + return ageLimit; } @Override @@ -311,24 +263,21 @@ public class YoutubeStreamExtractor extends StreamExtractor { public long getViewCount() throws ParsingException { assertPageFetched(); String views = null; + try { - views = getVideoPrimaryInfoRenderer().getObject("viewCount") - .getObject("videoViewCountRenderer").getObject("viewCount") - .getArray("runs").getObject(0).getString("text"); + views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount") + .getObject("videoViewCountRenderer").getObject("viewCount")); } catch (Exception ignored) {} - if (views == null) { - try { - views = getVideoPrimaryInfoRenderer().getObject("viewCount") - .getObject("videoViewCountRenderer").getObject("viewCount").getString("simpleText"); - } catch (Exception ignored) {} - } + if (views == null) { try { views = playerResponse.getObject("videoDetails").getString("viewCount"); } catch (Exception ignored) {} + + if (views == null) throw new ParsingException("Could not get view count"); } - if (views != null) return Long.parseLong(Utils.removeNonDigitCharacters(views)); - throw new ParsingException("Could not get view count"); + + return Long.parseLong(Utils.removeNonDigitCharacters(views)); } @Override @@ -381,17 +330,16 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getUploaderUrl() throws ParsingException { assertPageFetched(); - String uploaderId = null; try { - uploaderId = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") - .getObject("navigationEndpoint").getObject("browseEndpoint").getString("browseId"); + String uploaderUrl = getUrlFromNavigationEndpoint(getVideoSecondaryInfoRenderer() + .getObject("owner").getObject("videoOwnerRenderer").getObject("navigationEndpoint")); + if (uploaderUrl != null) return uploaderUrl; + } catch (Exception ignored) {} + try { + String uploaderId = playerResponse.getObject("videoDetails").getString("channelId"); + if (uploaderId != null) + return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + uploaderId); } catch (Exception ignored) {} - if (uploaderId == null) { - try { - uploaderId = playerResponse.getObject("videoDetails").getString("channelId"); - } catch (Exception ignored) {} - } - if (uploaderId != null) return "https://www.youtube.com/channel/" + uploaderId; throw new ParsingException("Could not get uploader url"); } @@ -400,44 +348,35 @@ public class YoutubeStreamExtractor extends StreamExtractor { public String getUploaderName() throws ParsingException { assertPageFetched(); String uploaderName = null; + try { - uploaderName = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") - .getObject("title").getArray("runs").getObject(0).getString("text"); + uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner") + .getObject("videoOwnerRenderer").getObject("title")); } catch (Exception ignored) {} + if (uploaderName == null) { try { uploaderName = playerResponse.getObject("videoDetails").getString("author"); } catch (Exception ignored) {} + + if (uploaderName == null) throw new ParsingException("Could not get uploader name"); } - if (uploaderName != null) return uploaderName; - throw new ParsingException("Could not get uploader name"); + + return uploaderName; } @Nonnull @Override public String getUploaderAvatarUrl() throws ParsingException { assertPageFetched(); - - String uploaderAvatarUrl = null; try { - uploaderAvatarUrl = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("secondaryResults") - .getObject("secondaryResults").getArray("results").getObject(0).getObject("compactAutoplayRenderer") - .getArray("contents").getObject(0).getObject("compactVideoRenderer").getObject("channelThumbnail") - .getArray("thumbnails").getObject(0).getString("url"); - if (uploaderAvatarUrl != null && !uploaderAvatarUrl.isEmpty()) { - return uploaderAvatarUrl; - } - } catch (Exception ignored) {} - - try { - uploaderAvatarUrl = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") + String url = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); - } catch (Exception ignored) {} - if (uploaderAvatarUrl == null) { - throw new ParsingException("Could not get uploader avatar url"); + return fixThumbnailUrl(url); + } catch (Exception e) { + throw new ParsingException("Could not get uploader avatar url", e); } - return uploaderAvatarUrl; } @Nonnull @@ -578,9 +517,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public StreamInfoItem getNextStream() throws ExtractionException { assertPageFetched(); - if (isAgeRestricted) { - return null; - } + + if (getAgeLimit() != NO_AGE_LIMIT) return null; + try { final JsonObject videoInfo = initialData.getObject("contents").getObject("twoColumnWatchNextResults") .getObject("secondaryResults").getObject("secondaryResults").getArray("results") @@ -599,9 +538,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public StreamInfoItemsCollector getRelatedStreams() throws ExtractionException { assertPageFetched(); - if (isAgeRestricted) { - return null; - } + + if (getAgeLimit() != NO_AGE_LIMIT) return null; + try { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); JsonArray results = initialData.getObject("contents").getObject("twoColumnWatchNextResults") @@ -625,23 +564,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { */ @Override public String getErrorMessage() { - StringBuilder errorReason; - Element errorElement = doc.select("h1[id=\"unavailable-message\"]").first(); - - if (errorElement == null) { - errorReason = null; - } else { - String errorMessage = errorElement.text(); - if (errorMessage == null || errorMessage.isEmpty()) { - errorReason = null; - } else { - errorReason = new StringBuilder(errorMessage); - errorReason.append(" "); - errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text()); - } - } - - return errorReason != null ? errorReason.toString() : ""; + return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse").getObject("playabilityStatus") + .getObject("errorScreen").getObject("playerErrorMessageRenderer").getObject("reason")); } /*////////////////////////////////////////////////////////////////////////// @@ -651,11 +575,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { private static final String FORMATS = "formats"; private static final String ADAPTIVE_FORMATS = "adaptiveFormats"; private static final String HTTPS = "https:"; - private static final String CONTENT = "content"; private static final String DECRYPTION_FUNC_NAME = "decrypt"; - private static final String VERIFIED_URL_PARAMS = "&has_verified=1&bpctr=9999999999"; - private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX = "([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;"; private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX_2 = @@ -667,32 +588,32 @@ public class YoutubeStreamExtractor extends StreamExtractor { private volatile String decryptionCode = ""; - private String pageHtml = null; - @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String verifiedUrl = getUrl() + VERIFIED_URL_PARAMS; - final Response response = downloader.get(verifiedUrl, getExtractorLocalization()); - pageHtml = response.responseBody(); - doc = YoutubeParsingHelper.parseAndCheckPage(verifiedUrl, response); + final String url = getUrl() + "&pbj=1"; + + initialAjaxJson = getJsonResponse(url, getExtractorLocalization()); final String playerUrl; - // Check if the video is age restricted - if (!doc.select("meta[property=\"og:restrictions:age\"]").isEmpty()) { + + if (initialAjaxJson.getObject(2).getObject("response") != null) { // age-restricted videos + initialData = initialAjaxJson.getObject(2).getObject("response"); + ageLimit = 18; + final EmbeddedInfo info = getEmbeddedInfo(); final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts); final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody(); videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse)); playerUrl = info.url; - isAgeRestricted = true; } else { - final JsonObject ytPlayerConfig = getPlayerConfig(); - playerArgs = getPlayerArgs(ytPlayerConfig); - playerUrl = getPlayerUrl(ytPlayerConfig); - isAgeRestricted = false; + initialData = initialAjaxJson.getObject(3).getObject("response"); + ageLimit = NO_AGE_LIMIT; + + playerArgs = getPlayerArgs(initialAjaxJson.getObject(2).getObject("player")); + playerUrl = getPlayerUrl(initialAjaxJson.getObject(2).getObject("player")); } + playerResponse = getPlayerResponse(); - initialData = YoutubeParsingHelper.getInitialData(pageHtml); if (decryptionCode.isEmpty()) { decryptionCode = loadDecryptionCode(playerUrl); @@ -703,21 +624,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - private JsonObject getPlayerConfig() throws ParsingException { - try { - String ytPlayerConfigRaw = Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageHtml); - return JsonParser.object().from(ytPlayerConfigRaw); - } catch (Parser.RegexException e) { - String errorReason = getErrorMessage(); - if (errorReason.isEmpty()) { - throw new ContentNotAvailableException("Content not available: player config empty", e); - } - throw new ContentNotAvailableException("Content not available", e); - } catch (Exception e) { - throw new ParsingException("Could not parse yt player config", e); - } - } - private JsonObject getPlayerArgs(JsonObject playerConfig) throws ParsingException { JsonObject playerArgs; @@ -869,7 +775,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull private List getAvailableSubtitlesInfo() { // If the video is age restricted getPlayerConfig will fail - if (isAgeRestricted) return Collections.emptyList(); + if (getAgeLimit() != NO_AGE_LIMIT) return Collections.emptyList(); final JsonObject captions; if (!playerResponse.has("captions")) { @@ -939,6 +845,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { //////////////////////////////////////////////////////////////////////////*/ private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException { + if (this.videoPrimaryInfoRenderer != null) return this.videoPrimaryInfoRenderer; + JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults") .getObject("results").getObject("results").getArray("contents"); JsonObject videoPrimaryInfoRenderer = null; @@ -954,10 +862,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { throw new ParsingException("Could not find videoPrimaryInfoRenderer"); } + this.videoPrimaryInfoRenderer = videoPrimaryInfoRenderer; return videoPrimaryInfoRenderer; } private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException { + if (this.videoSecondaryInfoRenderer != null) return this.videoSecondaryInfoRenderer; + JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults") .getObject("results").getObject("results").getArray("contents"); JsonObject videoSecondaryInfoRenderer = null; @@ -973,6 +884,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { throw new ParsingException("Could not find videoSecondaryInfoRenderer"); } + this.videoSecondaryInfoRenderer = videoSecondaryInfoRenderer; return videoSecondaryInfoRenderer; } @@ -1010,9 +922,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { urlAndItags.put(streamUrl, itagItem); } - } catch (UnsupportedEncodingException ignored) { - - } + } catch (UnsupportedEncodingException ignored) {} } } @@ -1023,17 +933,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public List getFrames() throws ExtractionException { try { - final String script = doc.select("#player-api").first().siblingElements().select("script").html(); - int p = script.indexOf("ytplayer.config"); - if (p == -1) { - return Collections.emptyList(); - } - p = script.indexOf('{', p); - int e = script.indexOf("ytplayer.load", p); - if (e == -1) { - return Collections.emptyList(); - } - JsonObject jo = JsonParser.object().from(script.substring(p, e - 1)); + JsonObject jo = initialAjaxJson.getObject(2).getObject("player"); final String resp = jo.getObject("args").getString("player_response"); jo = JsonParser.object().from(resp); final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 2010cfb5e..2ee89e245 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -6,7 +6,6 @@ import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.TimeAgoParser; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; @@ -15,6 +14,10 @@ import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nullable; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint; + /* * Copyright (C) Christian Schabesberger 2016 * YoutubeStreamInfoItemExtractor.java is part of NewPipe. @@ -76,89 +79,95 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public String getName() throws ParsingException { - String name = null; - try { - name = videoInfo.getObject("title").getString("simpleText"); - } catch (Exception ignored) {} - if (name == null) { - try { - name = videoInfo.getObject("title").getArray("runs").getObject(0).getString("text"); - } catch (Exception ignored) {} - } + String name = getTextFromObject(videoInfo.getObject("title")); if (name != null && !name.isEmpty()) return name; throw new ParsingException("Could not get name"); } @Override public long getDuration() throws ParsingException { + if (getStreamType() == StreamType.LIVE_STREAM) return -1; + + String duration = null; + try { - if (getStreamType() == StreamType.LIVE_STREAM) return -1; - return YoutubeParsingHelper.parseDurationString(videoInfo.getObject("lengthText").getString("simpleText")); - } catch (Exception e) { - throw new ParsingException("Could not get duration", e); + duration = getTextFromObject(videoInfo.getObject("lengthText")); + } catch (Exception ignored) {} + + if (duration == null) { + try { + for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) { + if (((JsonObject) thumbnailOverlay).getObject("thumbnailOverlayTimeStatusRenderer") != null) { + duration = getTextFromObject(((JsonObject) thumbnailOverlay) + .getObject("thumbnailOverlayTimeStatusRenderer").getObject("text")); + } + } + } catch (Exception ignored) {} + + if (duration == null) throw new ParsingException("Could not get duration"); } + + return YoutubeParsingHelper.parseDurationString(duration); } @Override public String getUploaderName() throws ParsingException { String name = null; + try { - name = videoInfo.getObject("longBylineText").getArray("runs") - .getObject(0).getString("text"); + name = getTextFromObject(videoInfo.getObject("longBylineText")); } catch (Exception ignored) {} + if (name == null) { try { - name = videoInfo.getObject("ownerText").getArray("runs") - .getObject(0).getString("text"); + name = getTextFromObject(videoInfo.getObject("ownerText")); } catch (Exception ignored) {} + + if (name == null) { + try { + name = getTextFromObject(videoInfo.getObject("shortBylineText")); + } catch (Exception ignored) {} + + if (name == null) throw new ParsingException("Could not get uploader name"); + } } - if (name == null) { - try { - name = videoInfo.getObject("shortBylineText").getArray("runs") - .getObject(0).getString("text"); - } catch (Exception ignored) {} - } - if (name != null && !name.isEmpty()) return name; - throw new ParsingException("Could not get uploader name"); + + return name; } @Override public String getUploaderUrl() throws ParsingException { + String url = null; + try { - String id = null; + url = getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText") + .getArray("runs").getObject(0).getObject("navigationEndpoint")); + } catch (Exception ignored) {} + + if (url == null) { try { - id = videoInfo.getObject("longBylineText").getArray("runs") - .getObject(0).getObject("navigationEndpoint") - .getObject("browseEndpoint").getString("browseId"); + url = getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText") + .getArray("runs").getObject(0).getObject("navigationEndpoint")); } catch (Exception ignored) {} - if (id == null) { + + if (url == null) { try { - id = videoInfo.getObject("ownerText").getArray("runs") - .getObject(0).getObject("navigationEndpoint") - .getObject("browseEndpoint").getString("browseId"); + url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText") + .getArray("runs").getObject(0).getObject("navigationEndpoint")); } catch (Exception ignored) {} + + if (url == null) throw new ParsingException("Could not get uploader url"); } - if (id == null) { - try { - id = videoInfo.getObject("shortBylineText").getArray("runs") - .getObject(0).getObject("navigationEndpoint") - .getObject("browseEndpoint").getString("browseId"); - } catch (Exception ignored) {} - } - if (id == null || id.isEmpty()) { - throw new IllegalArgumentException("is empty"); - } - return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id); - } catch (Exception e) { - throw new ParsingException("Could not get uploader url"); } + + return url; } @Nullable @Override public String getTextualUploadDate() { try { - return videoInfo.getObject("publishedTimeText").getString("simpleText"); + return getTextFromObject(videoInfo.getObject("publishedTimeText")); } catch (Exception e) { // upload date is not always available, e.g. in playlists return null; @@ -185,15 +194,11 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { if (videoInfo.getObject("topStandaloneBadge") != null || isPremium()) { return -1; } - String viewCount; - if (getStreamType() == StreamType.LIVE_STREAM) { - viewCount = videoInfo.getObject("viewCountText") - .getArray("runs").getObject(0).getString("text"); - } else { - viewCount = videoInfo.getObject("viewCountText").getString("simpleText"); - } - if (viewCount.equals("Recommended for you")) return -1; + String viewCount = getTextFromObject(videoInfo.getObject("viewCountText")); + return Long.parseLong(Utils.removeNonDigitCharacters(viewCount)); + } catch (NumberFormatException e) { + return -1; } catch (Exception e) { throw new ParsingException("Could not get view count", e); } @@ -203,8 +208,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { public String getThumbnailUrl() throws ParsingException { try { // TODO: Don't simply get the first item, but look at all thumbnails and their resolution - return videoInfo.getObject("thumbnail").getArray("thumbnails") + String url = videoInfo.getObject("thumbnail").getArray("thumbnails") .getObject(0).getString("url"); + + return fixThumbnailUrl(url); } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index 649cdf4e7..e15463b7f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -25,13 +25,11 @@ import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; @@ -39,6 +37,9 @@ import java.io.IOException; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject; + public class YoutubeTrendingExtractor extends KioskExtractor { private JsonObject initialData; @@ -50,11 +51,12 @@ public class YoutubeTrendingExtractor extends KioskExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String url = getUrl() + - "?gl=" + getExtractorContentCountry().getCountryCode(); + final String url = getUrl() + "?pbj=1&gl=" + + getExtractorContentCountry().getCountryCode(); - final Response response = downloader.get(url, getExtractorLocalization()); - initialData = YoutubeParsingHelper.getInitialData(response.responseBody()); + final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); + + initialData = ajaxJson.getObject(1).getObject("response"); } @Override @@ -72,8 +74,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor { public String getName() throws ParsingException { String name; try { - name = initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title") - .getArray("runs").getObject(0).getString("text"); + name = getTextFromObject(initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title")); } catch (Exception e) { throw new ParsingException("Could not get Trending name", e); } @@ -87,17 +88,21 @@ public class YoutubeTrendingExtractor extends KioskExtractor { @Override public InfoItemsPage getInitialPage() { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - JsonArray firstPageElements = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") - .getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content") - .getObject("sectionListRenderer").getArray("contents").getObject(0).getObject("itemSectionRenderer") - .getArray("contents").getObject(0).getObject("shelfRenderer").getObject("content") - .getObject("expandedShelfContentsRenderer").getArray("items"); - final TimeAgoParser timeAgoParser = getTimeAgoParser(); + JsonArray itemSectionRenderers = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") + .getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content") + .getObject("sectionListRenderer").getArray("contents"); - for (Object ul : firstPageElements) { - final JsonObject videoInfo = ((JsonObject) ul).getObject("videoRenderer"); - collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser)); + for (Object itemSectionRenderer : itemSectionRenderers) { + JsonObject expandedShelfContentsRenderer = ((JsonObject) itemSectionRenderer).getObject("itemSectionRenderer") + .getArray("contents").getObject(0).getObject("shelfRenderer").getObject("content") + .getObject("expandedShelfContentsRenderer"); + if (expandedShelfContentsRenderer != null) { + for (Object ul : expandedShelfContentsRenderer.getArray("items")) { + final JsonObject videoInfo = ((JsonObject) ul).getObject("videoRenderer"); + collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser)); + } + } } return new InfoItemsPage<>(collector, getNextPageUrl()); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java index eb34cf065..77eaf0694 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java @@ -35,6 +35,14 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { return instance; } + /** + * Returns URL to channel from an ID + * + * @param id Channel ID including e.g. 'channel/' + * @param contentFilters + * @param searchFilter + * @return URL to channel + */ @Override public String getUrl(String id, List contentFilters, String searchFilter) { return "https://www.youtube.com/" + id; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 51347d423..643a63138 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -5,18 +5,34 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Utils; +import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.net.URL; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; +import java.util.Collections; import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.schabi.newpipe.extractor.NewPipe.getDownloader; +import static org.schabi.newpipe.extractor.utils.Utils.HTTP; +import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; /* * Created by Christian Schabesberger on 02.03.16. @@ -43,7 +59,9 @@ public class YoutubeParsingHelper { private YoutubeParsingHelper() { } - public static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; + private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; + private static String clientVersion; + private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user="; @@ -160,58 +178,189 @@ public class YoutubeParsingHelper { } } + public static boolean isHardcodedClientVersionValid() throws IOException { + try { + final String url = "https://www.youtube.com/results?search_query=test&pbj=1"; + + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", + Collections.singletonList(HARDCODED_CLIENT_VERSION)); + final String response = getDownloader().get(url, headers).responseBody(); + if (response.length() > 50) { // ensure to have a valid response + return true; + } + } catch (ReCaptchaException ignored) {} + + return false; + } + /** * Get the client version from a page - * @param initialData - * @param html The page HTML * @return * @throws ParsingException */ - public static String getClientVersion(JsonObject initialData, String html) throws ParsingException { - if (initialData == null) initialData = getInitialData(html); - JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams"); - String shortClientVersion = null; + public static String getClientVersion() throws ParsingException, IOException { + if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion; - // try to get version from initial data first - for (Object service : serviceTrackingParams) { - JsonObject s = (JsonObject) service; - if (s.getString("service").equals("CSI")) { - JsonArray params = s.getArray("params"); - for (Object param: params) { - JsonObject p = (JsonObject) param; - String key = p.getString("key"); - if (key != null && key.equals("cver")) { - return p.getString("value"); + if (isHardcodedClientVersionValid()) { + clientVersion = HARDCODED_CLIENT_VERSION; + return clientVersion; + } + + // Try extracting it from YouTube's website otherwise + try { + final String url = "https://www.youtube.com/results?search_query=test"; + final String html = getDownloader().get(url).responseBody(); + JsonObject initialData = getInitialData(html); + JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams"); + String shortClientVersion = null; + + // try to get version from initial data first + for (Object service : serviceTrackingParams) { + JsonObject s = (JsonObject) service; + if (s.getString("service").equals("CSI")) { + JsonArray params = s.getArray("params"); + for (Object param : params) { + JsonObject p = (JsonObject) param; + String key = p.getString("key"); + if (key != null && key.equals("cver")) { + clientVersion = p.getString("value"); + return clientVersion; + } } - } - } else if (s.getString("service").equals("ECATCHER")) { - // fallback to get a shortened client version which does not contain the last do digits - JsonArray params = s.getArray("params"); - for (Object param: params) { - JsonObject p = (JsonObject) param; - String key = p.getString("key"); - if (key != null && key.equals("client.version")) { - shortClientVersion = p.getString("value"); + } else if (s.getString("service").equals("ECATCHER")) { + // fallback to get a shortened client version which does not contain the last two digits + JsonArray params = s.getArray("params"); + for (Object param : params) { + JsonObject p = (JsonObject) param; + String key = p.getString("key"); + if (key != null && key.equals("client.version")) { + shortClientVersion = p.getString("value"); + } } } } - } - String clientVersion; - String[] patterns = { - "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", - "innertube_context_client_version\":\"([0-9\\.]+?)\"", - "client.version=([0-9\\.]+)" - }; - for (String pattern: patterns) { - try { - clientVersion = Parser.matchGroup1(pattern, html); - if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion; - } catch (Exception ignored) {} - } + String contextClientVersion; + String[] patterns = { + "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", + "innertube_context_client_version\":\"([0-9\\.]+?)\"", + "client.version=([0-9\\.]+)" + }; + for (String pattern : patterns) { + try { + contextClientVersion = Parser.matchGroup1(pattern, html); + if (contextClientVersion != null && !contextClientVersion.isEmpty()) { + clientVersion = contextClientVersion; + return clientVersion; + } + } catch (Exception ignored) { + } + } - if (shortClientVersion != null) return shortClientVersion; + if (shortClientVersion != null) { + clientVersion = shortClientVersion; + return clientVersion; + } + } catch (Exception ignored) {} throw new ParsingException("Could not get client version"); } + + public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) { + if (navigationEndpoint.getObject("urlEndpoint") != null) { + String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url"); + if (internUrl.startsWith("/redirect?")) { + // q parameter can be the first parameter + internUrl = internUrl.substring(10); + String[] params = internUrl.split("&"); + for (String param : params) { + if (param.split("=")[0].equals("q")) { + String url; + try { + url = URLDecoder.decode(param.split("=")[1], StandardCharsets.UTF_8.name()); + } catch (UnsupportedEncodingException e) { + return null; + } + return url; + } + } + } else if (internUrl.startsWith("http")) { + return internUrl; + } + } else if (navigationEndpoint.getObject("browseEndpoint") != null) { + return "https://www.youtube.com" + navigationEndpoint.getObject("browseEndpoint").getString("canonicalBaseUrl"); + } else if (navigationEndpoint.getObject("watchEndpoint") != null) { + StringBuilder url = new StringBuilder(); + url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId")); + if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) + url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId")); + if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) + url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds")); + return url.toString(); + } + return null; + } + + public static String getTextFromObject(JsonObject textObject, boolean html) { + if (textObject.has("simpleText")) return textObject.getString("simpleText"); + + StringBuilder textBuilder = new StringBuilder(); + for (Object textPart : textObject.getArray("runs")) { + String text = ((JsonObject) textPart).getString("text"); + if (html && ((JsonObject) textPart).getObject("navigationEndpoint") != null) { + String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint")); + if (url != null && !url.isEmpty()) { + textBuilder.append("").append(text).append(""); + continue; + } + } + textBuilder.append(text); + } + + String text = textBuilder.toString(); + + if (html) { + text = text.replaceAll("\\n", "
"); + text = text.replaceAll(" ", "  "); + } + + return text; + } + + public static String getTextFromObject(JsonObject textObject) { + return getTextFromObject(textObject, false); + } + + public static String fixThumbnailUrl(String thumbnailUrl) { + if (thumbnailUrl.startsWith("//")) { + thumbnailUrl = thumbnailUrl.substring(2); + } + + if (thumbnailUrl.startsWith(HTTP)) { + thumbnailUrl = Utils.replaceHttpWithHttps(thumbnailUrl); + } else if (!thumbnailUrl.startsWith(HTTPS)) { + thumbnailUrl = "https://" + thumbnailUrl; + } + + return thumbnailUrl; + } + + public static JsonArray getJsonResponse(String url, Localization localization) throws IOException, ExtractionException { + Map> headers = new HashMap<>(); + headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); + headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); + final String response = getDownloader().get(url, headers, localization).responseBody(); + + if (response.length() < 50) { // ensure to have a valid response + throw new ParsingException("JSON response is too short"); + } + + try { + return JsonParser.array().from(response); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse JSON", e); + } + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java new file mode 100644 index 000000000..f8ff12358 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java @@ -0,0 +1,24 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; + +import java.io.IOException; + +import static org.junit.Assert.assertTrue; + +public class YoutubeParsingHelperTest { + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void testIsHardcodedClientVersionValid() throws IOException { + assertTrue("Hardcoded client version is not valid anymore", + YoutubeParsingHelper.isHardcodedClientVersionValid()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index d4de9175e..7c9112798 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -99,7 +99,7 @@ public class YoutubePlaylistExtractorTest { @Test public void testUploaderUrl() throws Exception { - assertEquals("https://www.youtube.com/channel/UCs72iRpTEuwV3y6pdWYLgiw", extractor.getUploaderUrl()); + assertEquals("https://www.youtube.com/user/andre0y0you", extractor.getUploaderUrl()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java index 7add41262..cb72622b4 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java @@ -12,10 +12,17 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; -import java.util.regex.Pattern; +import java.net.URL; +import java.net.URLDecoder; +import java.util.LinkedHashMap; +import java.util.Map; import static java.util.Arrays.asList; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.schabi.newpipe.extractor.ServiceList.YouTube; public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtractorBaseTest { @@ -47,18 +54,26 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto } } assertFalse("First and second page are equal", equals); - - assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&gl=GB&page=3", secondPage.getNextPageUrl()); } @Test public void testGetSecondPageUrl() throws Exception { - // check that ctoken, continuation and itct are longer than 5 characters - Pattern pattern = Pattern.compile( - "https:\\/\\/www.youtube.com\\/results\\?search_query=pewdiepie&sp=EgIQAg%253D%253D&gl=GB&pbj=1" - + "&ctoken=[\\w%]{5,}?&continuation=[\\w%]{5,}?&itct=[\\w]{5,}?" - ); - assertTrue(pattern.matcher(extractor.getNextPageUrl()).find()); + URL url = new URL(extractor.getNextPageUrl()); + + assertEquals(url.getHost(), "www.youtube.com"); + assertEquals(url.getPath(), "/results"); + + Map queryPairs = new LinkedHashMap<>(); + for (String queryPair : url.getQuery().split("&")) { + int index = queryPair.indexOf("="); + queryPairs.put(URLDecoder.decode(queryPair.substring(0, index), "UTF-8"), + URLDecoder.decode(queryPair.substring(index + 1), "UTF-8")); + } + + assertEquals("pewdiepie", queryPairs.get("search_query")); + assertEquals(queryPairs.get("ctoken"), queryPairs.get("continuation")); + assertTrue(queryPairs.get("continuation").length() > 5); + assertTrue(queryPairs.get("itct").length() > 5); } @Ignore @@ -77,13 +92,18 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto if (item instanceof ChannelInfoItem) { ChannelInfoItem channel = (ChannelInfoItem) item; - if (channel.getSubscriberCount() > 5e7) { // the real PewDiePie + if (channel.getSubscriberCount() > 1e8) { // the real PewDiePie assertEquals("https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw", item.getUrl()); - } else { - assertThat(item.getUrl(), CoreMatchers.startsWith("https://www.youtube.com/channel/")); + break; } } } + + for (InfoItem item : itemsPage.getItems()) { + if (item instanceof ChannelInfoItem) { + assertThat(item.getUrl(), CoreMatchers.startsWith("https://www.youtube.com/channel/")); + } + } } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java index 3078d8736..65ffe839d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java @@ -10,6 +10,11 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import java.net.URL; +import java.net.URLDecoder; +import java.util.LinkedHashMap; +import java.util.Map; + import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -48,13 +53,28 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas @Test public void testGetUrl() throws Exception { - assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB", extractor.getUrl()); + assertEquals("https://www.youtube.com/results?search_query=pewdiepie&gl=GB", extractor.getUrl()); } @Test public void testGetSecondPageUrl() throws Exception { - assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB&page=2", extractor.getNextPageUrl()); + URL url = new URL(extractor.getNextPageUrl()); + + assertEquals(url.getHost(), "www.youtube.com"); + assertEquals(url.getPath(), "/results"); + + Map queryPairs = new LinkedHashMap<>(); + for (String queryPair : url.getQuery().split("&")) { + int index = queryPair.indexOf("="); + queryPairs.put(URLDecoder.decode(queryPair.substring(0, index), "UTF-8"), + URLDecoder.decode(queryPair.substring(index + 1), "UTF-8")); + } + + assertEquals("pewdiepie", queryPairs.get("search_query")); + assertEquals(queryPairs.get("ctoken"), queryPairs.get("continuation")); + assertTrue(queryPairs.get("continuation").length() > 5); + assertTrue(queryPairs.get("itct").length() > 5); } @Test @@ -101,14 +121,12 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas } } assertFalse("First and second page are equal", equals); - - assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB&page=3", secondPage.getNextPageUrl()); } @Test - public void testSuggestionNotNull() throws Exception { + public void testSuggestionNotNull() { //todo write a real test - assertTrue(extractor.getSearchSuggestion() != null); + assertNotNull(extractor.getSearchSuggestion()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java index 8777cc701..fc6af4c4d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java @@ -11,11 +11,11 @@ public class YoutubeSearchQHTest { @Test public void testRegularValues() throws Exception { - assertEquals("https://www.youtube.com/results?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf").getUrl()); - assertEquals("https://www.youtube.com/results?q=hans", YouTube.getSearchQHFactory().fromQuery("hans").getUrl()); - assertEquals("https://www.youtube.com/results?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); - assertEquals("https://www.youtube.com/results?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); - assertEquals("https://www.youtube.com/results?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); + assertEquals("https://www.youtube.com/results?search_query=asdf", YouTube.getSearchQHFactory().fromQuery("asdf").getUrl()); + assertEquals("https://www.youtube.com/results?search_query=hans", YouTube.getSearchQHFactory().fromQuery("hans").getUrl()); + assertEquals("https://www.youtube.com/results?search_query=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); + assertEquals("https://www.youtube.com/results?search_query=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); + assertEquals("https://www.youtube.com/results?search_query=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index 2798e8db5..d6cf3815f 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -10,17 +10,24 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; -import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.stream.Frameset; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.Utils; -import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.List; import static java.util.Objects.requireNonNull; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -89,7 +96,6 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetFullLinksInDescription() throws ParsingException { assertTrue(extractor.getDescription().getContent().contains("http://adele.com")); - assertFalse(extractor.getDescription().getContent().contains("http://smarturl.it/SubscribeAdele?IQi...")); } @Test @@ -142,12 +148,12 @@ public class YoutubeStreamExtractorDefaultTest { } @Test - public void testGetAudioStreams() throws IOException, ExtractionException { + public void testGetAudioStreams() throws ExtractionException { assertFalse(extractor.getAudioStreams().isEmpty()); } @Test - public void testGetVideoStreams() throws IOException, ExtractionException { + public void testGetVideoStreams() throws ExtractionException { for (VideoStream s : extractor.getVideoStreams()) { assertIsSecureUrl(s.url); assertTrue(s.resolution.length() > 0); @@ -169,7 +175,7 @@ public class YoutubeStreamExtractorDefaultTest { } @Test - public void testGetRelatedVideos() throws ExtractionException, IOException { + public void testGetRelatedVideos() throws ExtractionException { StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); Utils.printErrors(relatedVideos.getErrors()); assertFalse(relatedVideos.getItems().isEmpty()); @@ -177,13 +183,13 @@ public class YoutubeStreamExtractorDefaultTest { } @Test - public void testGetSubtitlesListDefault() throws IOException, ExtractionException { + public void testGetSubtitlesListDefault() { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null assertTrue(extractor.getSubtitlesDefault().isEmpty()); } @Test - public void testGetSubtitlesList() throws IOException, ExtractionException { + public void testGetSubtitlesList() { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); } @@ -223,10 +229,6 @@ public class YoutubeStreamExtractorDefaultTest { assertTrue(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSubmissions/")); assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng")); assertTrue(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/")); - - assertFalse(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSub...")); - assertFalse(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8...")); - assertFalse(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/...")); } } @@ -249,15 +251,11 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); - - assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/X7FLCHVXpsA?list=PL7...")); - assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/Lqv6G0pDNnw?list=PL7...")); - assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/XxaRBPyrnBU?list=PL7...")); - assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/U-9tUEOFKNU?list=PL7...")); + final String description = extractor.getDescription().getContent(); + assertTrue(description.contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); + assertTrue(description.contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); + assertTrue(description.contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); + assertTrue(description.contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34")); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java index b1d6f53a9..f5c7fe104 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java @@ -13,9 +13,12 @@ import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.Utils; -import java.io.IOException; - -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -26,7 +29,7 @@ public class YoutubeStreamExtractorLivestreamTest { public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (YoutubeStreamExtractor) YouTube - .getStreamExtractor("https://www.youtube.com/watch?v=EcEMX-63PKY"); + .getStreamExtractor("https://www.youtube.com/watch?v=5qap5aO4i9A"); extractor.fetchPage(); } @@ -49,8 +52,7 @@ public class YoutubeStreamExtractorLivestreamTest { @Test public void testGetFullLinksInDescription() throws ParsingException { - assertTrue(extractor.getDescription().getContent().contains("https://www.instagram.com/nathalie.baraton/")); - assertFalse(extractor.getDescription().getContent().contains("https://www.instagram.com/nathalie.ba...")); + assertTrue(extractor.getDescription().getContent().contains("https://bit.ly/chilledcow-playlists")); } @Test @@ -119,7 +121,7 @@ public class YoutubeStreamExtractorLivestreamTest { } @Test - public void testGetRelatedVideos() throws ExtractionException, IOException { + public void testGetRelatedVideos() throws ExtractionException { StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); Utils.printErrors(relatedVideos.getErrors()); assertFalse(relatedVideos.getItems().isEmpty()); @@ -127,12 +129,12 @@ public class YoutubeStreamExtractorLivestreamTest { } @Test - public void testGetSubtitlesListDefault() throws IOException, ExtractionException { + public void testGetSubtitlesListDefault() { assertTrue(extractor.getSubtitlesDefault().isEmpty()); } @Test - public void testGetSubtitlesList() throws IOException, ExtractionException { + public void testGetSubtitlesList() { assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); } }