From 11216f361f25fa0652ba65897e41468389ba40bc Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Wed, 28 Feb 2018 21:02:43 -0300 Subject: [PATCH] Fix bug and some re-structure --- .../newpipe/extractor/ListExtractor.java | 32 ++++++-------- .../extractor/channel/ChannelExtractor.java | 13 +++--- .../extractor/channel/ChannelInfo.java | 4 +- .../extractor/kiosk/KioskExtractor.java | 9 ++++ .../newpipe/extractor/kiosk/KioskInfo.java | 9 ++-- .../extractor/playlist/PlaylistExtractor.java | 10 ++--- .../extractor/playlist/PlaylistInfo.java | 3 +- .../SoundcloudChannelExtractor.java | 11 ++--- .../soundcloud/SoundcloudChartsExtractor.java | 18 ++++---- .../SoundcloudPlaylistExtractor.java | 11 ++--- .../youtube/YoutubeChannelExtractor.java | 40 ++++++++--------- .../youtube/YoutubePlaylistExtractor.java | 44 ++++++++----------- .../youtube/YoutubeTrendingExtractor.java | 8 +++- .../SoundcloudChannelExtractorTest.java | 15 ++++--- .../SoundcloudChartsExtractorTest.java | 4 +- .../SoundcloudPlaylistExtractorTest.java | 8 ++-- .../youtube/YoutubeChannelExtractorTest.java | 15 ++++--- .../youtube/YoutubePlaylistExtractorTest.java | 14 +++--- 18 files changed, 134 insertions(+), 134 deletions(-) diff --git a/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java b/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java index 253444451..06cded295 100644 --- a/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java @@ -11,52 +11,46 @@ import java.util.List; */ public abstract class ListExtractor extends Extractor { - /** - * Get a new ListExtractor with the given nextPageUrl set. - */ public ListExtractor(StreamingService service, String url) { super(service, url); } @Nonnull - public abstract InfoItemsCollector getInfoItems() throws IOException, ExtractionException; - + public abstract InfoItemsCollector getInfoItems() throws IOException, ExtractionException; public abstract String getNextPageUrl() throws IOException, ExtractionException; - - public abstract InfoItemPage getPage(final String nextPageUrl) throws IOException, ExtractionException; + public abstract InfoItemPage getPage(final String nextPageUrl) throws IOException, ExtractionException; public boolean hasNextPage() throws IOException, ExtractionException { - return getNextPageUrl() != null && !getNextPageUrl().isEmpty(); + final String nextPageUrl = getNextPageUrl(); + return nextPageUrl != null && !nextPageUrl.isEmpty(); } - - /*////////////////////////////////////////////////////////////////////////// // Inner //////////////////////////////////////////////////////////////////////////*/ - public static class InfoItemPage { + public static class InfoItemPage { /** * The current list of items to this result */ - public final List infoItemList; + private final List itemsList; /** * Next url to fetch more items */ - public final String nextPageUrl; + private final String nextPageUrl; /** * Errors that happened during the extraction */ - public final List errors; + private final List errors; - public InfoItemPage(InfoItemsCollector collector, String nextPageUrl) { + public InfoItemPage(InfoItemsCollector collector, String nextPageUrl) { this(collector.getItemList(), nextPageUrl, collector.getErrors()); } - public InfoItemPage(List infoItemList, String nextPageUrl, List errors) { - this.infoItemList = infoItemList; + public InfoItemPage(List itemsList, String nextPageUrl, List errors) { + this.itemsList = itemsList; this.nextPageUrl = nextPageUrl; this.errors = errors; } @@ -65,8 +59,8 @@ public abstract class ListExtractor extends Extractor { return nextPageUrl != null && !nextPageUrl.isEmpty(); } - public List getItemsList() { - return infoItemList; + public List getItemsList() { + return itemsList; } public String getNextPageUrl() { diff --git a/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java index 4badd5690..49a6b5394 100644 --- a/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java @@ -1,9 +1,12 @@ package org.schabi.newpipe.extractor.channel; import edu.umd.cs.findbugs.annotations.NonNull; -import org.schabi.newpipe.extractor.*; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; @@ -43,12 +46,10 @@ public abstract class ChannelExtractor extends ListExtractor { @NonNull @Override - public InfoItemsCollector getInfoItems() - throws IOException, ExtractionException { - return getStreams(); - } + public abstract StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException; + @Override + public abstract InfoItemPage getPage(String nextPageUrl) throws IOException, ExtractionException; - public abstract StreamInfoItemsCollector getStreams() throws IOException, ExtractionException; public abstract String getAvatarUrl() throws ParsingException; public abstract String getBannerUrl() throws ParsingException; public abstract String getFeedUrl() throws ParsingException; diff --git a/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java index 3a824faf6..7695368b2 100644 --- a/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java +++ b/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java @@ -5,7 +5,7 @@ import org.schabi.newpipe.extractor.ListInfo; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.utils.ExtractorHelper; import java.io.IOException; @@ -37,7 +37,7 @@ public class ChannelInfo extends ListInfo { } - public static InfoItemPage getMoreItems(StreamingService service, String url, String pageUrl) + public static InfoItemPage getMoreItems(StreamingService service, String url, String pageUrl) throws IOException, ExtractionException { return service.getChannelExtractor(url).getPage(pageUrl); } diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java index 1ad59094e..98afe99e1 100644 --- a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java @@ -20,10 +20,13 @@ package org.schabi.newpipe.extractor.kiosk; * along with NewPipe. If not, see . */ +import edu.umd.cs.findbugs.annotations.NonNull; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; import java.io.IOException; @@ -40,6 +43,12 @@ public abstract class KioskExtractor extends ListExtractor { this.id = kioskId; } + @NonNull + @Override + public abstract StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException; + @Override + public abstract InfoItemPage getPage(String nextPageUrl) throws IOException, ExtractionException; + /** * For certain Websites the content of a kiosk will be different depending * on the country you want to poen the website in. Therefore you should diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java index a666590b7..4a60f0bd6 100644 --- a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java @@ -25,6 +25,7 @@ import org.schabi.newpipe.extractor.ListInfo; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.utils.ExtractorHelper; import java.io.IOException; @@ -35,10 +36,10 @@ public class KioskInfo extends ListInfo { super(serviceId, id, url, name); } - public static ListExtractor.InfoItemPage getMoreItems(StreamingService service, - String url, - String pageUrl, - String contentCountry) throws IOException, ExtractionException { + public static ListExtractor.InfoItemPage getMoreItems(StreamingService service, + String url, + String pageUrl, + String contentCountry) throws IOException, ExtractionException { KioskList kl = service.getKioskList(); KioskExtractor extractor = kl.getExtractorByUrl(url, pageUrl); extractor.setContentCountry(contentCountry); diff --git a/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java index c506c5830..621004f86 100644 --- a/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java @@ -1,12 +1,12 @@ package org.schabi.newpipe.extractor.playlist; import edu.umd.cs.findbugs.annotations.NonNull; -import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; @@ -26,12 +26,10 @@ public abstract class PlaylistExtractor extends ListExtractor { @NonNull @Override - public InfoItemsCollector getInfoItems() - throws IOException, ExtractionException { - return getStreams(); - } + public abstract StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException; + @Override + public abstract InfoItemPage getPage(String nextPageUrl) throws IOException, ExtractionException; - public abstract StreamInfoItemsCollector getStreams() throws IOException, ExtractionException; public abstract String getThumbnailUrl() throws ParsingException; public abstract String getBannerUrl() throws ParsingException; diff --git a/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java b/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java index 1fc940f4c..407e4deea 100644 --- a/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java +++ b/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java @@ -5,6 +5,7 @@ import org.schabi.newpipe.extractor.ListInfo; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import java.io.IOException; @@ -16,7 +17,7 @@ public class PlaylistInfo extends ListInfo { super(serviceId, id, url, name); } - public static InfoItemPage getMoreItems(StreamingService service, String url, String pageUrl) throws IOException, ExtractionException { + public static InfoItemPage getMoreItems(StreamingService service, String url, String pageUrl) throws IOException, ExtractionException { return service.getPlaylistExtractor(url).getPage(pageUrl); } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java index c59b7f7b5..d806c44e1 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; @@ -89,7 +90,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { @Nonnull @Override - public StreamInfoItemsCollector getStreams() throws ExtractionException { + public StreamInfoItemsCollector getInfoItems() throws ExtractionException { if(streamInfoItemsCollector == null) { computeNextPageAndGetStreams(); } @@ -120,14 +121,14 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { } @Override - public InfoItemPage getPage(final String pageUrl) throws IOException, ExtractionException { - if (!hasNextPage()) { - throw new ExtractionException("Channel doesn't have more streams"); + public InfoItemPage getPage(final String pageUrl) throws IOException, ExtractionException { + if (pageUrl == null || pageUrl.isEmpty()) { + throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl); - return new InfoItemPage(collector, nextPageUrl); + return new InfoItemPage<>(collector, nextPageUrl); } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java index 0bef85d89..c93120598 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -1,19 +1,17 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; - -import org.schabi.newpipe.extractor.Collector; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; public class SoundcloudChartsExtractor extends KioskExtractor { private String url; @@ -44,15 +42,15 @@ public class SoundcloudChartsExtractor extends KioskExtractor { } @Override - public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException { - if (!hasNextPage()) { - throw new ExtractionException("Chart doesn't have more streams"); + public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException { + if (pageUrl == null || pageUrl.isEmpty()) { + throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, pageUrl, true); - return new InfoItemPage(collector, nextPageUrl); + return new InfoItemPage<>(collector, nextPageUrl); } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index ff247d54b..83f3b08f5 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; @@ -91,7 +92,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { @Nonnull @Override - public StreamInfoItemsCollector getStreams() throws IOException, ExtractionException { + public StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException { if(streamInfoItemsCollector == null) { computeStreamsAndNextPageUrl(); } @@ -119,14 +120,14 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { } @Override - public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException { - if (!hasNextPage()) { - throw new ExtractionException("Playlist doesn't have more streams"); + public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException { + if (pageUrl == null || pageUrl.isEmpty()) { + throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl); - return new InfoItemPage(collector, nextPageUrl); + return new InfoItemPage<>(collector, nextPageUrl); } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java index fbf25cd33..5e49ea44e 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java @@ -13,7 +13,7 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; @@ -150,7 +150,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Nonnull @Override - public StreamInfoItemsCollector getStreams() throws ExtractionException { + public StreamInfoItemsCollector getInfoItems() throws ExtractionException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); Element ul = doc.select("ul[id=\"browse-items-primary\"]").first(); collectStreamsFrom(collector, ul); @@ -158,29 +158,27 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } @Override - public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException { + public InfoItemPage getPage(String pageUrl) throws IOException, ExtractionException { + if (pageUrl == null || pageUrl.isEmpty()) { + throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); + } + + // Unfortunately, we have to fetch the page even if we are only getting next streams, + // as they don't deliver enough information on their own (the channel name, for example). + fetchPage(); + + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + JsonObject ajaxJson; try { - - if (!hasNextPage()) { - throw new ExtractionException("Channel doesn't have more streams"); - } - - fetchPage(); - - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - - final JsonObject ajaxJson = JsonParser.object().from( - NewPipe.getDownloader() - .download(pageUrl)); - - final Document ajaxHtml = Jsoup.parse(ajaxJson.getString("content_html")); - - collectStreamsFrom(collector, ajaxHtml.select("body").first()); - - return new InfoItemPage(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl)); + ajaxJson = JsonParser.object().from(NewPipe.getDownloader().download(pageUrl)); } catch (JsonParserException pe) { throw new ParsingException("Could not parse json data for next streams", pe); } + + final Document ajaxHtml = Jsoup.parse(ajaxJson.getString("content_html")); + collectStreamsFrom(collector, ajaxHtml.select("body").first()); + + return new InfoItemPage<>(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl)); } private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl) diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java index e845c25b0..597b1d9c3 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java @@ -12,8 +12,8 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.Parser; @@ -26,10 +26,6 @@ import java.io.IOException; public class YoutubePlaylistExtractor extends PlaylistExtractor { private Document doc; - /** - * It's lazily initialized (when getInfoItemPage is called) - */ - private Document nextPageAjax; public YoutubePlaylistExtractor(StreamingService service, String url) { super(service, url); @@ -39,8 +35,6 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { String pageContent = downloader.download(getCleanUrl()); doc = Jsoup.parse(pageContent, getCleanUrl()); - - nextPageAjax = null; } @Override @@ -135,7 +129,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Nonnull @Override - public StreamInfoItemsCollector getStreams() throws IOException, ExtractionException { + public StreamInfoItemsCollector getInfoItems() throws IOException, ExtractionException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); Element tbody = doc.select("tbody[id=\"pl-load-more-destination\"]").first(); collectStreamsFrom(collector, tbody); @@ -143,28 +137,26 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } @Override - public InfoItemPage getPage(final String pageUrl) throws IOException, ExtractionException { + public InfoItemPage getPage(final String pageUrl) throws IOException, ExtractionException { + if (pageUrl == null || pageUrl.isEmpty()) { + throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); + } + + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + JsonObject pageJson; try { - if (!hasNextPage()) { - throw new ExtractionException("Playlist doesn't have more streams"); - } - - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - // setupNextStreamsAjax(NewPipe.getDownloader()); - final JsonObject pageJson = JsonParser.object().from(NewPipe.getDownloader() - .download(pageUrl)); - final Document pageHtml = Jsoup.parse("" - + pageJson.getString("content_html") - + "
", pageUrl); - - collectStreamsFrom(collector, pageHtml.select("tbody[id=\"pl-load-more-destination\"]").first()); - - - - return new InfoItemPage(collector, getNextPageUrlFromAjax(pageJson, pageUrl)); + pageJson = JsonParser.object().from(NewPipe.getDownloader().download(pageUrl)); } catch (JsonParserException pe) { throw new ParsingException("Could not parse ajax json", pe); } + + final Document pageHtml = Jsoup.parse("" + + pageJson.getString("content_html") + + "
", pageUrl); + + collectStreamsFrom(collector, pageHtml.select("tbody[id=\"pl-load-more-destination\"]").first()); + + return new InfoItemPage<>(collector, getNextPageUrlFromAjax(pageJson, pageUrl)); } private String getNextPageUrlFromAjax(final JsonObject pageJson, final String pageUrl) diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java index 3628cdfd7..d5342724d 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java @@ -24,10 +24,14 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; -import org.schabi.newpipe.extractor.*; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; @@ -66,7 +70,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor { } @Override - public ListExtractor.InfoItemPage getPage(String pageUrl) { + public ListExtractor.InfoItemPage getPage(String pageUrl) { return null; } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java index 465bcf0e8..f988ecf68 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java @@ -6,6 +6,7 @@ import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; @@ -49,18 +50,18 @@ public class SoundcloudChannelExtractorTest { @Test public void testGetStreams() throws Exception { - assertFalse("no streams are received", extractor.getStreams().getItemList().isEmpty()); + assertFalse("no streams are received", extractor.getInfoItems().getItemList().isEmpty()); } @Test public void testGetStreamsErrors() throws Exception { - assertTrue("errors during stream list extraction", extractor.getStreams().getErrors().isEmpty()); + assertTrue("errors during stream list extraction", extractor.getInfoItems().getErrors().isEmpty()); } @Test public void testHasMoreStreams() throws Exception { // Setup the streams - extractor.getStreams(); + extractor.getInfoItems(); assertTrue("don't have more streams", extractor.hasNextPage()); } @@ -77,10 +78,10 @@ public class SoundcloudChannelExtractorTest { @Test public void testGetPage() throws Exception { // Setup the streams - extractor.getStreams(); - ListExtractor.InfoItemPage nextItemsResult = extractor.getPage(extractor.getNextPageUrl()); - assertTrue("extractor didn't have next streams", !nextItemsResult.infoItemList.isEmpty()); - assertTrue("errors occurred during extraction of the next streams", nextItemsResult.errors.isEmpty()); + extractor.getInfoItems(); + ListExtractor.InfoItemPage nextItemsResult = extractor.getPage(extractor.getNextPageUrl()); + assertTrue("extractor didn't have next streams", !nextItemsResult.getItemsList().isEmpty()); + assertTrue("errors occurred during extraction of the next streams", nextItemsResult.getErrors().isEmpty()); assertTrue("extractor didn't have more streams after getInfoItemPage", extractor.hasNextPage()); } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java index fb084f360..7c1aa16c4 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java @@ -80,9 +80,9 @@ public class SoundcloudChartsExtractorTest { @Test public void testGetNextPage() throws Exception { - extractor.getInfoItems(); + extractor.getInfoItems().getItemList(); assertFalse("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null - || extractor.getPage(extractor.getNextPageUrl()).infoItemList.isEmpty()); + || extractor.getPage(extractor.getNextPageUrl()).getItemsList().isEmpty()); } @Test diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index f6b0d59d1..e029174ad 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -69,25 +69,25 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testGetStreams() throws Exception { - assertTrue("no streams are received", !extractor.getStreams().getItemList().isEmpty()); + assertTrue("no streams are received", !extractor.getInfoItems().getItemList().isEmpty()); } @Test public void testGetStreamsErrors() throws Exception { - assertTrue("errors during stream list extraction", extractor.getStreams().getErrors().isEmpty()); + assertTrue("errors during stream list extraction", extractor.getInfoItems().getErrors().isEmpty()); } @Test public void testHasMoreStreams() throws Exception { // Setup the streams - extractor.getStreams(); + extractor.getInfoItems(); assertTrue("extractor didn't have more streams", !extractor.hasNextPage()); } @Test(expected = ExtractionException.class) public void testGetNextPageNonExistent() throws Exception { // Setup the streams - extractor.getStreams(); + extractor.getInfoItems(); // This playlist don't have more streams, it should throw an error extractor.getPage(extractor.getNextPageUrl()); diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 9fcdcf06f..474288e60 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -6,6 +6,7 @@ import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmptyErrors; @@ -88,18 +89,18 @@ public class YoutubeChannelExtractorTest { @Test public void testGetStreams() throws Exception { - assertTrue("no streams are received", !extractor.getStreams().getItemList().isEmpty()); + assertTrue("no streams are received", !extractor.getInfoItems().getItemList().isEmpty()); } @Test public void testGetStreamsErrors() throws Exception { - assertEmptyErrors("errors during stream list extraction", extractor.getStreams().getErrors()); + assertEmptyErrors("errors during stream list extraction", extractor.getInfoItems().getErrors()); } @Test public void testHasMoreStreams() throws Exception { // Setup the streams - extractor.getStreams(); + extractor.getInfoItems(); assertTrue("don't have more streams", extractor.hasNextPage()); } @@ -116,10 +117,10 @@ public class YoutubeChannelExtractorTest { @Test public void testGetPage() throws Exception { // Setup the streams - extractor.getStreams(); - ListExtractor.InfoItemPage nextItemsResult = extractor.getPage(extractor.getNextPageUrl()); - assertTrue("extractor didn't have next streams", !nextItemsResult.infoItemList.isEmpty()); - assertEmptyErrors("errors occurred during extraction of the next streams", nextItemsResult.errors); + extractor.getInfoItems(); + ListExtractor.InfoItemPage nextItemsResult = extractor.getPage(extractor.getNextPageUrl()); + assertTrue("extractor didn't have next streams", !nextItemsResult.getItemsList().isEmpty()); + assertEmptyErrors("errors occurred during extraction of the next streams", nextItemsResult.getErrors()); assertTrue("extractor didn't have more streams after getInfoItemPage", extractor.hasNextPage()); } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index bbd5dccc0..1e5d1062d 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -81,7 +81,7 @@ public class YoutubePlaylistExtractorTest { @Test public void testGetStreams() throws Exception { - List streams = extractor.getStreams().getItemList(); + List streams = extractor.getInfoItems().getItemList(); assertFalse("no streams are received", streams.isEmpty()); assertTrue(streams.size() > 60); assertFalse(streams.contains(null)); @@ -96,13 +96,13 @@ public class YoutubePlaylistExtractorTest { @Test public void testGetStreamsErrors() throws Exception { - assertEmptyErrors("errors during stream list extraction", extractor.getStreams().getErrors()); + assertEmptyErrors("errors during stream list extraction", extractor.getInfoItems().getErrors()); } @Test public void testHasMoreStreams() throws Exception { // Setup the streams - extractor.getStreams(); + extractor.getInfoItems(); assertTrue("extractor didn't have more streams", extractor.hasNextPage()); } @@ -110,10 +110,10 @@ public class YoutubePlaylistExtractorTest { @Test @Ignore public void testGetNextPage() throws Exception { // Setup the streams - extractor.getStreams(); - ListExtractor.InfoItemPage infoItemPage = extractor.getPage(extractor.getNextPageUrl()); - assertTrue("extractor didn't have next streams", !infoItemPage.infoItemList.isEmpty()); - assertEmptyErrors("errors occurred during extraction of the next streams", infoItemPage.errors); + extractor.getInfoItems(); + ListExtractor.InfoItemPage infoItemPage = extractor.getPage(extractor.getNextPageUrl()); + assertTrue("extractor didn't have next streams", !infoItemPage.getItemsList().isEmpty()); + assertEmptyErrors("errors occurred during extraction of the next streams", infoItemPage.getErrors()); assertTrue("extractor didn't have more streams after getInfoItemPage", extractor.hasNextPage()); }