From 466865385e565cb1777d631efa8bfd262467a04b Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sun, 29 Apr 2018 21:31:24 +0200 Subject: [PATCH 1/7] add support for affiliate --- .../soundcloud/SoundcloudStreamExtractor.java | 5 ++++ .../extractor/services/youtube/ItagItem.java | 3 ++ .../youtube/YoutubeChannelExtractor.java | 2 +- .../youtube/YoutubeStreamExtractor.java | 19 +++++++++++- .../extractor/stream/StreamExtractor.java | 1 + .../newpipe/extractor/stream/StreamInfo.java | 29 +++++++++++++++++++ .../extractor/utils/DonationLinkHelper.java | 16 ++++++++-- .../YoutubeStreamExtractorDonationTest.java | 6 ++++ 8 files changed, 77 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 8de597d88..30478b592 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -214,6 +214,11 @@ public class SoundcloudStreamExtractor extends StreamExtractor { return new String[0]; } + @Override + public String[] getAffiliateLinks() { + return new String[0]; + } + @Override public String getErrorMessage() { return null; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java index 1950c0b5e..3be29c981 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java @@ -43,6 +43,9 @@ public class ItagItem { new ItagItem(139, AUDIO, M4A, 48), new ItagItem(140, AUDIO, M4A, 128), new ItagItem(141, AUDIO, M4A, 256), + new ItagItem(249, AUDIO, OPUS, 50), + new ItagItem(250, AUDIO, OPUS, 70), + new ItagItem(160, AUDIO, OPUS, 160), /// VIDEO ONLY //////////////////////////////////////////// // ID Type Format Resolution FPS /// diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java index 195d20abc..dfcfd6b85 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java @@ -196,7 +196,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } for(Element a : linkHolder.select("a")) { String link = a.attr("abs:href"); - if(DonationLinkHelper.getServiceByLink(link) != DonationLinkHelper.DonationService.NO_DONATION) { + if(DonationLinkHelper.getDonatoinServiceByLink(link) != DonationLinkHelper.DonationService.NO_DONATION) { links.add(link); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index c9839ac69..a5d48a14d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -534,7 +534,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { ArrayList donationLinks = new ArrayList<>(); for (String s : Parser.getLinksFromString(getDescription())) { - if (DonationLinkHelper.getServiceByLink(s) != DonationLinkHelper.DonationService.NO_DONATION) { + if (DonationLinkHelper.getDonatoinServiceByLink(s) != DonationLinkHelper.DonationService.NO_DONATION) { donationLinks.add(s); } } @@ -546,6 +546,23 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } + @Override + public String[] getAffiliateLinks() throws ParsingException { + try { + ArrayList donationLinks = new ArrayList<>(); + for (String s : Parser.getLinksFromString(getDescription())) { + if (DonationLinkHelper.getAffiliateServiceByLink(s) != DonationLinkHelper.AffiliateService.NO_AFILIATE) { + donationLinks.add(s); + } + } + String[] donlret = new String[donationLinks.size()]; + donlret = donationLinks.toArray(donlret); + return donlret; + } catch (Exception e) { + throw new ParsingException("Could not get afiliate links", e); + } + } + /*////////////////////////////////////////////////////////////////////////// // Fetch page diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 0aeedd029..8a2626b7c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -142,6 +142,7 @@ public abstract class StreamExtractor extends Extractor { public abstract StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException; public abstract String[] getDonationLinks() throws ExtractionException; + public abstract String[] getAffiliateLinks() throws ExtractionException; /** * Analyses the webpage's document and extracts any error message there might be. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 716afa4b9..50b2ec0f2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -242,6 +242,16 @@ public class StreamInfo extends Info { } catch (Exception e) { streamInfo.addError(e); } + try { + streamInfo.setAffiliateLinks(extractor.getAffiliateLinks()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setDonationLinks(extractor.getDonationLinks()); + } catch (Exception e) { + streamInfo.addError(e); + } streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor)); return streamInfo; @@ -274,6 +284,9 @@ public class StreamInfo extends Info { private long startPosition = 0; private List subtitles; + private String[] donationLinks; + private String[] affiliateLinks; + /** * Get the stream type * @@ -466,4 +479,20 @@ public class StreamInfo extends Info { public void setSubtitles(List subtitles) { this.subtitles = subtitles; } + + public String[] getDonationLinks() { + return donationLinks; + } + + public void setDonationLinks(String[] donationLinks) { + this.donationLinks = donationLinks; + } + + public String[] getAffiliateLinks() { + return affiliateLinks; + } + + public void setAffiliateLinks(String[] affiliateLinks) { + this.affiliateLinks = affiliateLinks; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java index b62e062f3..049132257 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java @@ -7,11 +7,16 @@ public class DonationLinkHelper { public enum DonationService { NO_DONATION, PATREON, - PAYPAL + PAYPAL, + } + + public enum AffiliateService { + NO_AFILIATE, + AMAZON, } - public static DonationService getServiceByLink(String link) throws MalformedURLException { + public static DonationService getDonatoinServiceByLink(String link) throws MalformedURLException { URL url = new URL(link); switch (url.getHost()) { case "www.patreon.com": @@ -27,5 +32,12 @@ public class DonationLinkHelper { } } + public static AffiliateService getAffiliateServiceByLink(String link) throws MalformedURLException { + URL url = new URL(link); + switch (url.getHost()) { + case "amzn.to": return AffiliateService.AMAZON; + default: return AffiliateService.NO_AFILIATE; + } + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java index 8bc82ded8..9f26a7bd2 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java @@ -49,4 +49,10 @@ public class YoutubeStreamExtractorDonationTest { assertTrue(String.valueOf(extractor.getDonationLinks().length), extractor.getDonationLinks().length == 2); } + + @Test + public void getAffiliateLinksTest() throws Exception { + assertTrue(String.valueOf(extractor.getAffiliateLinks().length), + extractor.getAffiliateLinks().length == 1); + } } From bf1c7716620e1dbc2b01d29b2f5af601596ac621 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Fri, 4 May 2018 16:11:25 +0200 Subject: [PATCH 2/7] fix could not parese metadata despide available error --- .../services/youtube/YoutubeStreamInfoItemExtractor.java | 4 ++++ .../services/youtube/YoutubeTrendingExtractorTest.java | 1 + 2 files changed, 5 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java index 7e792c1af..e4146313d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java @@ -135,7 +135,11 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { Element meta = item.select("div[class=\"yt-lockup-meta\"]").first(); if (meta == null) return -1; + // This case can happen if google releases a special video + if(meta.select("li").size() < 2) return -1; + input = meta.select("li").get(1).text(); + } catch (IndexOutOfBoundsException e) { throw new ParsingException("Could not parse yt-lockup-meta although available: " + getUrl(), e); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java index ea8312a12..d4599e983 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java @@ -48,6 +48,7 @@ public class YoutubeTrendingExtractorTest { .getKioskList() .getExtractorById("Trending", null); extractor.fetchPage(); + extractor.setContentCountry("de"); } @Test From b7c1f251d89f7d5ec8b76e5f7b9bafdf0bcd4f14 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sat, 5 May 2018 20:07:47 +0200 Subject: [PATCH 3/7] add filter to stream lists --- .../newpipe/extractor/ListUrlIdHandler.java | 31 +++++++++++++++++++ .../newpipe/extractor/UrlIdHandler.java | 10 +++--- .../SoundcloudChannelUrlIdHandler.java | 5 +-- .../SoundcloudChartsUrlIdHandler.java | 5 +-- .../SoundcloudPlaylistUrlIdHandler.java | 5 +-- .../SoundcloudStreamUrlIdHandler.java | 2 +- .../youtube/YoutubeChannelUrlIdHandler.java | 5 +-- .../youtube/YoutubePlaylistUrlIdHandler.java | 5 +-- .../youtube/YoutubeStreamUrlIdHandler.java | 2 +- .../youtube/YoutubeTrendingUrlIdHandler.java | 8 +++-- .../SoundcloudChartsUrlIdHandlerTest.java | 2 +- 11 files changed, 59 insertions(+), 21 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/ListUrlIdHandler.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ListUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ListUrlIdHandler.java new file mode 100644 index 000000000..ef56da31d --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ListUrlIdHandler.java @@ -0,0 +1,31 @@ +package org.schabi.newpipe.extractor; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +public abstract class ListUrlIdHandler extends UrlIdHandler { + + public abstract String getUrl(String id, String[] contentFilter, String sortFilter) throws ParsingException; + + @Override + public String getUrl(String id) throws ParsingException { + return getUrl(id, new String[0], null); + } + + /** + * Will returns content filter the corresponding extractor can handle like "channels", "videos", "music", etc. + * + * @return filter that can be applied when building a query for getting a list + */ + public String[] getAvailableContentFilter() { + return new String[0]; + } + + /** + * Will returns sort filter the corresponding extractor can handle like "A-Z", "oldest first", "size", etc. + * + * @return filter that can be applied when building a query for getting a list + */ + public String[] getAvailableSortFilter() { + return new String[0]; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java index 6fe43ec8a..9e327c5fc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java @@ -22,16 +22,16 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; * along with NewPipe. If not, see . */ -public interface UrlIdHandler { +public abstract class UrlIdHandler { - String getUrl(String id) throws ParsingException; - String getId(String url) throws ParsingException; - String cleanUrl(String complexUrl) throws ParsingException; + public abstract String getUrl(String id) throws ParsingException; + public abstract String getId(String url) throws ParsingException; + public abstract String cleanUrl(String complexUrl) throws ParsingException; /** * When a VIEW_ACTION is caught this function will test if the url delivered within the calling * Intent was meant to be watched with this Service. * Return false if this service shall not allow to be called through ACTIONs. */ - boolean acceptUrl(String url); + public abstract boolean acceptUrl(String url); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java index 04902eb3a..9498671ab 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; import org.jsoup.Jsoup; import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -10,7 +11,7 @@ import org.schabi.newpipe.extractor.utils.Utils; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; -public class SoundcloudChannelUrlIdHandler implements UrlIdHandler { +public class SoundcloudChannelUrlIdHandler extends ListUrlIdHandler { private static final SoundcloudChannelUrlIdHandler instance = new SoundcloudChannelUrlIdHandler(); private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" + "(/((tracks|albums|sets|reposts|followers|following)/?)?)?([#?].*)?$"; @@ -20,7 +21,7 @@ public class SoundcloudChannelUrlIdHandler implements UrlIdHandler { } @Override - public String getUrl(String id) throws ParsingException { + public String getUrl(String id, String[] contentFilter, String sortFilter) throws ParsingException { try { return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/" + id); } catch (Exception e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java index b26f0deb0..bc8a16190 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java @@ -1,13 +1,14 @@ package org.schabi.newpipe.extractor.services.soundcloud; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.utils.Parser; -public class SoundcloudChartsUrlIdHandler implements UrlIdHandler { +public class SoundcloudChartsUrlIdHandler extends ListUrlIdHandler { private final String TOP_URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$"; private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$"; - public String getUrl(String id) { + public String getUrl(String id, String[] contentFilter, String sortFilter) { if (id.equals("Top 50")) { return "https://soundcloud.com/charts/top"; } else { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java index 9518f1f9a..ccad52787 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; import org.jsoup.Jsoup; import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -10,7 +11,7 @@ import org.schabi.newpipe.extractor.utils.Utils; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; -public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler { +public class SoundcloudPlaylistUrlIdHandler extends ListUrlIdHandler { private static final SoundcloudPlaylistUrlIdHandler instance = new SoundcloudPlaylistUrlIdHandler(); private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" + "/sets/[0-9a-z_-]+/?([#?].*)?$"; @@ -20,7 +21,7 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler { } @Override - public String getUrl(String id) throws ParsingException { + public String getUrl(String id, String[] contentFilter, String sortFilter) throws ParsingException { try { return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/playlists/" + id); } catch (Exception e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java index b38dac032..0b9dd0286 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java @@ -10,7 +10,7 @@ import org.schabi.newpipe.extractor.utils.Utils; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; -public class SoundcloudStreamUrlIdHandler implements UrlIdHandler { +public class SoundcloudStreamUrlIdHandler extends UrlIdHandler { private static final SoundcloudStreamUrlIdHandler instance = new SoundcloudStreamUrlIdHandler(); private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" + "/(?!(tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?([#?].*)?$"; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandler.java index bc9ce6811..259a5bf30 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandler.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Parser; @@ -24,7 +25,7 @@ import org.schabi.newpipe.extractor.utils.Parser; * along with NewPipe. If not, see . */ -public class YoutubeChannelUrlIdHandler implements UrlIdHandler { +public class YoutubeChannelUrlIdHandler extends ListUrlIdHandler { private static final YoutubeChannelUrlIdHandler instance = new YoutubeChannelUrlIdHandler(); private static final String ID_PATTERN = "/(user/[A-Za-z0-9_-]*|channel/[A-Za-z0-9_-]*)"; @@ -34,7 +35,7 @@ public class YoutubeChannelUrlIdHandler implements UrlIdHandler { } @Override - public String getUrl(String id) { + public String getUrl(String id, String[] contentFilter, String sortFilter) { return "https://www.youtube.com/" + id; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistUrlIdHandler.java index 0c24f993c..e6032803c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistUrlIdHandler.java @@ -1,11 +1,12 @@ package org.schabi.newpipe.extractor.services.youtube; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Parser; -public class YoutubePlaylistUrlIdHandler implements UrlIdHandler { +public class YoutubePlaylistUrlIdHandler extends ListUrlIdHandler { private static final YoutubePlaylistUrlIdHandler instance = new YoutubePlaylistUrlIdHandler(); private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{10,})"; @@ -15,7 +16,7 @@ public class YoutubePlaylistUrlIdHandler implements UrlIdHandler { } @Override - public String getUrl(String id) { + public String getUrl(String id, String[] contentFilter, String sortFilter) { return "https://www.youtube.com/playlist?list=" + id; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java index 9addec48f..25afe98a4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java @@ -37,7 +37,7 @@ import java.net.URLDecoder; * along with NewPipe. If not, see . */ -public class YoutubeStreamUrlIdHandler implements UrlIdHandler { +public class YoutubeStreamUrlIdHandler extends UrlIdHandler { private static final YoutubeStreamUrlIdHandler instance = new YoutubeStreamUrlIdHandler(); private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{11})"; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java index 63a35181e..f6110484f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java @@ -20,12 +20,14 @@ package org.schabi.newpipe.extractor.services.youtube; * along with NewPipe. If not, see . */ +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Parser; -public class YoutubeTrendingUrlIdHandler implements UrlIdHandler { +public class YoutubeTrendingUrlIdHandler extends ListUrlIdHandler { - public String getUrl(String id) { + public String getUrl(String id, String[] contentFilter, String sortFilter) { return "https://www.youtube.com/feed/trending"; } @@ -35,7 +37,7 @@ public class YoutubeTrendingUrlIdHandler implements UrlIdHandler { } @Override - public String cleanUrl(String url) { + public String cleanUrl(String url) throws ParsingException { return getUrl(""); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java index e805b297f..76e52cd78 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java @@ -22,7 +22,7 @@ public class SoundcloudChartsUrlIdHandlerTest { } @Test - public void getUrl() { + public void getUrl() throws Exception { assertEquals(urlIdHandler.getUrl("Top 50"), "https://soundcloud.com/charts/top"); assertEquals(urlIdHandler.getUrl("New & hot"), "https://soundcloud.com/charts/new"); } From 08f0247d73448539484df66a3fa2c1fe92f6fa7c Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sun, 6 May 2018 14:08:50 +0200 Subject: [PATCH 4/7] make UrlIdHandler actual wrapper for the url/id fix tests fix tests --- .../schabi/newpipe/extractor/Extractor.java | 56 ++++---------- .../newpipe/extractor/ListExtractor.java | 4 +- .../newpipe/extractor/ListUrlIdHandler.java | 19 ++++- .../newpipe/extractor/StreamingService.java | 45 +++++++++-- .../newpipe/extractor/UrlIdHandler.java | 45 ++++++++++- .../extractor/channel/ChannelExtractor.java | 6 +- .../extractor/channel/ChannelInfo.java | 2 +- .../extractor/kiosk/KioskExtractor.java | 8 +- .../newpipe/extractor/kiosk/KioskInfo.java | 2 +- .../newpipe/extractor/kiosk/KioskList.java | 2 +- .../extractor/playlist/PlaylistExtractor.java | 6 +- .../extractor/playlist/PlaylistInfo.java | 2 +- .../SoundcloudChannelExtractor.java | 13 +--- .../SoundcloudChannelUrlIdHandler.java | 21 +---- .../soundcloud/SoundcloudChartsExtractor.java | 9 +-- .../SoundcloudChartsUrlIdHandler.java | 33 ++++---- .../SoundcloudPlaylistExtractor.java | 13 +--- .../SoundcloudPlaylistUrlIdHandler.java | 46 ++++------- .../soundcloud/SoundcloudService.java | 20 ++--- .../soundcloud/SoundcloudStreamExtractor.java | 10 +-- .../SoundcloudStreamUrlIdHandler.java | 20 +---- .../SoundcloudSubscriptionExtractor.java | 2 +- .../youtube/YoutubeChannelExtractor.java | 18 ++--- .../youtube/YoutubeChannelUrlIdHandler.java | 13 +--- .../youtube/YoutubePlaylistExtractor.java | 27 ++----- .../youtube/YoutubePlaylistUrlIdHandler.java | 10 +-- .../services/youtube/YoutubeService.java | 20 ++--- .../youtube/YoutubeStreamExtractor.java | 23 ++---- .../youtube/YoutubeStreamUrlIdHandler.java | 16 ++-- .../youtube/YoutubeTrendingExtractor.java | 7 +- .../youtube/YoutubeTrendingUrlIdHandler.java | 11 +-- .../extractor/stream/StreamExtractor.java | 4 +- .../newpipe/extractor/stream/StreamInfo.java | 2 +- .../extractor/services/BaseExtractorTest.java | 2 +- .../services/BasePlaylistExtractorTest.java | 1 - .../SoundcloudChannelExtractorTest.java | 15 ++-- .../SoundcloudChartsExtractorTest.java | 2 +- .../SoundcloudChartsUrlIdHandlerTest.java | 13 ++-- .../SoundcloudPlaylistExtractorTest.java | 32 ++++---- .../SoundcloudStreamUrlIdHandlerTest.java | 26 +++---- .../SoundcloudSubscriptionExtractorTest.java | 3 +- .../youtube/YoutubeChannelExtractorTest.java | 27 +++---- .../YoutubeChannelUrlIdHandlerTest.java | 16 ++-- .../youtube/YoutubePlaylistExtractorTest.java | 15 ++-- .../YoutubeStreamExtractorRestrictedTest.java | 2 +- .../YoutubeStreamUrlIdHandlerTest.java | 76 +++++++++---------- .../youtube/YoutubeTrendingExtractorTest.java | 4 +- .../youtube/YoutubeTrendingKioskInfoTest.java | 2 +- .../YoutubeTrendingUrlIdHandlerTest.java | 5 +- 49 files changed, 361 insertions(+), 415 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java index f904d3338..832ec5624 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor; +import edu.umd.cs.findbugs.annotations.NonNull; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -14,42 +15,28 @@ public abstract class Extractor { */ private final StreamingService service; - /** - * Dirty/original url that was passed in the constructor. - *

- * What makes a url "dirty" or not is, for example, the additional parameters - * (not important as—in this case—the id): - * 

-     *     https://www.youtube.com/watch?v=a9Zf_258aTI&t=4s  →  &t=4s
-     *
- * But as you can imagine, the time parameter is very important when calling {@link org.schabi.newpipe.extractor.stream.StreamExtractor#getTimeStamp()}. - */ - private final String originalUrl; + private final UrlIdHandler urlIdHandler; - /** - * The cleaned url, result of passing the {@link #originalUrl} to the associated urlIdHandler ({@link #getUrlIdHandler()}). - *

- * Is lazily-cleaned by calling {@link #getCleanUrl()} - */ @Nullable - private String cleanUrl; private boolean pageFetched = false; private final Downloader downloader; - public Extractor(final StreamingService service, final String url) { + public Extractor(final StreamingService service, final UrlIdHandler urlIdHandler) { if(service == null) throw new NullPointerException("service is null"); - if(url == null) throw new NullPointerException("url is null"); + if(urlIdHandler == null) throw new NullPointerException("UrlIdHandler is null"); this.service = service; - this.originalUrl = url; + this.urlIdHandler = urlIdHandler; this.downloader = NewPipe.getDownloader(); if(downloader == null) throw new NullPointerException("downloader is null"); } /** - * @return a {@link UrlIdHandler} of the current extractor type (e.g. a ChannelExtractor should return a channel url handler). + * @return The {@link UrlIdHandler} of the current extractor object (e.g. a ChannelExtractor should return a channel url handler). */ @Nonnull - protected abstract UrlIdHandler getUrlIdHandler() throws ParsingException; + protected UrlIdHandler getUrlIdHandler() { + return urlIdHandler; + } /** * Fetch the current page. @@ -79,7 +66,9 @@ public abstract class Extractor { public abstract void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException; @Nonnull - public abstract String getId() throws ParsingException; + public String getId() throws ParsingException { + return urlIdHandler.getId(); + } /** * Get the name @@ -90,26 +79,13 @@ public abstract class Extractor { public abstract String getName() throws ParsingException; @Nonnull - public String getOriginalUrl() { - return originalUrl; + public String getOriginalUrl() throws ParsingException { + return urlIdHandler.getOriginalUrl(); } - /** - * Get a clean url and as a fallback the original url. - * @return the clean url or the original url - */ @Nonnull - public String getCleanUrl() { - if (cleanUrl != null && !cleanUrl.isEmpty()) return cleanUrl; - - try { - cleanUrl = getUrlIdHandler().cleanUrl(originalUrl); - } catch (Exception e) { - cleanUrl = null; - // Fallback to the original url - return originalUrl; - } - return cleanUrl; + public String getUrl() throws ParsingException { + return urlIdHandler.getUrl(); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java index a25f88236..e1a790117 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java @@ -12,8 +12,8 @@ import java.util.List; */ public abstract class ListExtractor extends Extractor { - public ListExtractor(StreamingService service, String url) { - super(service, url); + public ListExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) { + super(service, urlIdHandler); } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ListUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ListUrlIdHandler.java index ef56da31d..43ff19c8b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ListUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ListUrlIdHandler.java @@ -4,11 +4,22 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; public abstract class ListUrlIdHandler extends UrlIdHandler { - public abstract String getUrl(String id, String[] contentFilter, String sortFilter) throws ParsingException; + protected String[] contentFilter; + protected String sortFilter; - @Override - public String getUrl(String id) throws ParsingException { - return getUrl(id, new String[0], null); + public ListUrlIdHandler setQuery(String id, String[] contentFilter, String softFilter) throws ParsingException { + setId(id); + this.contentFilter = contentFilter; + this.sortFilter = softFilter; + return this; + } + + public ListUrlIdHandler setUrl(String url) throws ParsingException { + return (ListUrlIdHandler) super.setUrl(url); + } + + public ListUrlIdHandler setId(String id) throws ParsingException { + return (ListUrlIdHandler) super.setId(id); } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index 7fb667576..fc68dd30d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchEngine; @@ -62,22 +63,52 @@ public abstract class StreamingService { return serviceId + ":" + serviceInfo.getName(); } + //////////////////////////////////////////// + // Url Id handler + //////////////////////////////////////////// public abstract UrlIdHandler getStreamUrlIdHandler(); - public abstract UrlIdHandler getChannelUrlIdHandler(); - public abstract UrlIdHandler getPlaylistUrlIdHandler(); + public abstract ListUrlIdHandler getChannelUrlIdHandler(); + public abstract ListUrlIdHandler getPlaylistUrlIdHandler(); + + //////////////////////////////////////////// + // Extractor + //////////////////////////////////////////// public abstract SearchEngine getSearchEngine(); public abstract SuggestionExtractor getSuggestionExtractor(); - public abstract StreamExtractor getStreamExtractor(String url); - public abstract KioskList getKioskList() throws ExtractionException; - public abstract ChannelExtractor getChannelExtractor(String url); - public abstract PlaylistExtractor getPlaylistExtractor(String url); public abstract SubscriptionExtractor getSubscriptionExtractor(); + public abstract KioskList getKioskList() throws ExtractionException; + + public abstract ChannelExtractor getChannelExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException; + public abstract PlaylistExtractor getPlaylistExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException; + public abstract StreamExtractor getStreamExtractor(UrlIdHandler urlIdHandler) throws ExtractionException; + + public ChannelExtractor getChannelExtractor(String id, String[] contentFilter, String sortFilter) throws ExtractionException { + return getChannelExtractor(getChannelUrlIdHandler().setQuery(id, contentFilter, sortFilter)); + } + + public PlaylistExtractor getPlaylistExtractor(String id, String[] contentFilter, String sortFilter) throws ExtractionException { + return getPlaylistExtractor(getPlaylistUrlIdHandler().setQuery(id, contentFilter, sortFilter)); + } + + public ChannelExtractor getChannelExtractor(String url) throws ExtractionException { + return getChannelExtractor(getChannelUrlIdHandler().setUrl(url)); + } + + public PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException { + return getPlaylistExtractor(getPlaylistUrlIdHandler().setUrl(url)); + } + + public StreamExtractor getStreamExtractor(String url) throws ExtractionException { + return getStreamExtractor(getStreamUrlIdHandler().setUrl(url)); + } + + /** * figure out where the link is pointing to (a channel, video, playlist, etc.) */ - public final LinkType getLinkTypeByUrl(String url) { + public final LinkType getLinkTypeByUrl(String url) throws ParsingException { UrlIdHandler sH = getStreamUrlIdHandler(); UrlIdHandler cH = getChannelUrlIdHandler(); UrlIdHandler pH = getPlaylistUrlIdHandler(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java index 9e327c5fc..a9fda4d31 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; /* @@ -24,14 +25,50 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; public abstract class UrlIdHandler { - public abstract String getUrl(String id) throws ParsingException; - public abstract String getId(String url) throws ParsingException; - public abstract String cleanUrl(String complexUrl) throws ParsingException; + protected String id = ""; + protected String originalUrl = ""; + + public abstract String onGetIdFromUrl(String url) throws ParsingException; + public abstract String getUrl() throws ParsingException; + public abstract boolean onAcceptUrl(final String url) throws ParsingException; + + + public UrlIdHandler setUrl(String url) throws ParsingException { + if(url == null) throw new IllegalArgumentException("url can not be null"); + originalUrl = url; + id = onGetIdFromUrl(url); + return this; + } + + public UrlIdHandler setId(String id) throws ParsingException { + if(id == null) throw new IllegalArgumentException("id can not be null"); + this.id = id; + if(!acceptUrl(getUrl())) { + throw new ParsingException("Malformed unacceptable url: " + getUrl()); + } + return this; + } + + public String getId() { + return id; + } + + public String getOriginalUrl() throws ParsingException { + return (originalUrl == null || originalUrl.isEmpty()) + ? getUrl() + : originalUrl; + } /** * When a VIEW_ACTION is caught this function will test if the url delivered within the calling * Intent was meant to be watched with this Service. * Return false if this service shall not allow to be called through ACTIONs. */ - public abstract boolean acceptUrl(String url); + public boolean acceptUrl(final String url) { + try { + return onAcceptUrl(url); + } catch (Exception e) { + return false; + } + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java index 05a5bc587..625f63810 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java @@ -1,8 +1,10 @@ package org.schabi.newpipe.extractor.channel; import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.StreamInfoItem; @@ -30,8 +32,8 @@ import javax.annotation.Nonnull; public abstract class ChannelExtractor extends ListExtractor { - public ChannelExtractor(StreamingService service, String url) { - super(service, url); + public ChannelExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) { + super(service, urlIdHandler); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java index 0d8a860bd..fd5868b27 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java @@ -54,7 +54,7 @@ public class ChannelInfo extends ListInfo { // important data int serviceId = extractor.getServiceId(); - String url = extractor.getCleanUrl(); + String url = extractor.getUrl(); String originalUrl = extractor.getOriginalUrl(); String id = extractor.getId(); String name = extractor.getName(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java index 31b496b45..95ce34f57 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java @@ -21,6 +21,7 @@ package org.schabi.newpipe.extractor.kiosk; */ import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -33,10 +34,9 @@ public abstract class KioskExtractor extends ListExtractor { private final String id; public KioskExtractor(StreamingService streamingService, - String url, - String kioskId) - throws ExtractionException { - super(streamingService, url); + ListUrlIdHandler urlIdHandler, + String kioskId) { + super(streamingService, urlIdHandler); this.id = kioskId; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java index 550b8b577..f11ebb9f6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java @@ -71,7 +71,7 @@ public class KioskInfo extends ListInfo { int serviceId = extractor.getServiceId(); String name = extractor.getName(); String id = extractor.getId(); - String url = extractor.getCleanUrl(); + String url = extractor.getUrl(); String originalUrl = extractor.getOriginalUrl(); KioskInfo info = new KioskInfo(serviceId, id, url, originalUrl, name); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java index fb8e48680..11c8f1414 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java @@ -73,7 +73,7 @@ public class KioskList { throw new ExtractionException("No kiosk found with the type: " + kioskId); } else { return ke.extractorFactory.createNewKiosk(NewPipe.getService(service_id), - ke.handler.getUrl(kioskId), kioskId); + ke.handler.setId(kioskId).getUrl(), kioskId); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java index 4569e7b51..cc7107f72 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java @@ -1,8 +1,10 @@ package org.schabi.newpipe.extractor.playlist; import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.StreamInfoItem; @@ -10,8 +12,8 @@ import javax.annotation.Nonnull; public abstract class PlaylistExtractor extends ListExtractor { - public PlaylistExtractor(StreamingService service, String url) { - super(service, url); + public PlaylistExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) { + super(service, urlIdHandler); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java index c8910651d..29f7b1448 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java @@ -38,7 +38,7 @@ public class PlaylistInfo extends ListInfo { public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws IOException, ExtractionException { int serviceId = extractor.getServiceId(); - String url = extractor.getCleanUrl(); + String url = extractor.getUrl(); String originalUrl = extractor.getOriginalUrl(); String id = extractor.getId(); String name = extractor.getName(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java index 7232ac5ad..6c2aba0d7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java @@ -5,6 +5,7 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -25,14 +26,14 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { private StreamInfoItemsCollector streamInfoItemsCollector = null; private String nextPageUrl = null; - public SoundcloudChannelExtractor(StreamingService service, String url) { - super(service, url); + public SoundcloudChannelExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) { + super(service, urlIdHandler); } @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - userId = getUrlIdHandler().getId(getOriginalUrl()); + userId = getUrlIdHandler().getId(); String apiUrl = "https://api-v2.soundcloud.com/users/" + userId + "?client_id=" + SoundcloudParsingHelper.clientId(); @@ -44,12 +45,6 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { } } - @Nonnull - @Override - public String getCleanUrl() { - return user.isString("permalink_url") ? replaceHttpWithHttps(user.getString("permalink_url")) : getOriginalUrl(); - } - @Nonnull @Override public String getId() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java index 9498671ab..063ad7aa8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java @@ -20,17 +20,9 @@ public class SoundcloudChannelUrlIdHandler extends ListUrlIdHandler { return instance; } - @Override - public String getUrl(String id, String[] contentFilter, String sortFilter) throws ParsingException { - try { - return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/" + id); - } catch (Exception e) { - throw new ParsingException(e.getMessage(), e); - } - } @Override - public String getId(String url) throws ParsingException { + public String onGetIdFromUrl(String url) throws ParsingException { Utils.checkUrl(URL_PATTERN, url); try { @@ -41,21 +33,16 @@ public class SoundcloudChannelUrlIdHandler extends ListUrlIdHandler { } @Override - public String cleanUrl(String complexUrl) throws ParsingException { - Utils.checkUrl(URL_PATTERN, complexUrl); - + public String getUrl() throws ParsingException { try { - Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl)) - .select("meta[property=og:url]").first(); - - return replaceHttpWithHttps(ogElement.attr("content")); + return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/" + id); } catch (Exception e) { throw new ParsingException(e.getMessage(), e); } } @Override - public boolean acceptUrl(String url) { + public boolean onAcceptUrl(final String url) { return Parser.isMatch(URL_PATTERN, url.toLowerCase()); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java index bfea06e75..655d4c065 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -14,15 +15,11 @@ import java.util.Arrays; import java.util.List; public class SoundcloudChartsExtractor extends KioskExtractor { - private String url; - private StreamInfoItemsCollector collector = null; private String nextPageUrl = null; - public SoundcloudChartsExtractor(StreamingService service, String url, String kioskId) - throws ExtractionException { - super(service, url, kioskId); - this.url = url; + public SoundcloudChartsExtractor(StreamingService service, ListUrlIdHandler urlIdHandler, String kioskId) { + super(service, urlIdHandler, kioskId); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java index bc8a16190..03a846cbf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java @@ -2,13 +2,24 @@ package org.schabi.newpipe.extractor.services.soundcloud; import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Parser; public class SoundcloudChartsUrlIdHandler extends ListUrlIdHandler { private final String TOP_URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$"; private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$"; - public String getUrl(String id, String[] contentFilter, String sortFilter) { + + @Override + public String onGetIdFromUrl(String url) { + if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) { + return "Top 50"; + } else { + return "New & hot"; + } + } + + public String getUrl() { if (id.equals("Top 50")) { return "https://soundcloud.com/charts/top"; } else { @@ -17,25 +28,7 @@ public class SoundcloudChartsUrlIdHandler extends ListUrlIdHandler { } @Override - public String getId(String url) { - if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) { - return "Top 50"; - } else { - return "New & hot"; - } - } - - @Override - public String cleanUrl(String url) { - if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) { - return "https://soundcloud.com/charts/top"; - } else { - return "https://soundcloud.com/charts/new"; - } - } - - @Override - public boolean acceptUrl(String url) { + public boolean onAcceptUrl(final String url) { return Parser.isMatch(URL_PATTERN, url.toLowerCase()); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index 84379b9e4..d6647539b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -4,6 +4,7 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -24,14 +25,14 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { private StreamInfoItemsCollector streamInfoItemsCollector = null; private String nextPageUrl = null; - public SoundcloudPlaylistExtractor(StreamingService service, String url) { - super(service, url); + public SoundcloudPlaylistExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) { + super(service, urlIdHandler); } @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - playlistId = getUrlIdHandler().getId(getOriginalUrl()); + playlistId = getUrlIdHandler().getId(); String apiUrl = "https://api.soundcloud.com/playlists/" + playlistId + "?client_id=" + SoundcloudParsingHelper.clientId() + "&representation=compact"; @@ -44,12 +45,6 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { } } - @Nonnull - @Override - public String getCleanUrl() { - return playlist.isString("permalink_url") ? replaceHttpWithHttps(playlist.getString("permalink_url")) : getOriginalUrl(); - } - @Nonnull @Override public String getId() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java index ccad52787..2855c1d50 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java @@ -1,16 +1,10 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.ListUrlIdHandler; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; -import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; - public class SoundcloudPlaylistUrlIdHandler extends ListUrlIdHandler { private static final SoundcloudPlaylistUrlIdHandler instance = new SoundcloudPlaylistUrlIdHandler(); private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" + @@ -21,7 +15,18 @@ public class SoundcloudPlaylistUrlIdHandler extends ListUrlIdHandler { } @Override - public String getUrl(String id, String[] contentFilter, String sortFilter) throws ParsingException { + public String onGetIdFromUrl(String url) throws ParsingException { + Utils.checkUrl(URL_PATTERN, url); + + try { + return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url); + } catch (Exception e) { + throw new ParsingException("Could not get id of url: " + url + " " + e.getMessage(), e); + } + } + + @Override + public String getUrl() throws ParsingException { try { return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/playlists/" + id); } catch (Exception e) { @@ -30,32 +35,7 @@ public class SoundcloudPlaylistUrlIdHandler extends ListUrlIdHandler { } @Override - public String getId(String url) throws ParsingException { - Utils.checkUrl(URL_PATTERN, url); - - try { - return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url); - } catch (Exception e) { - throw new ParsingException(e.getMessage(), e); - } - } - - @Override - public String cleanUrl(String complexUrl) throws ParsingException { - Utils.checkUrl(URL_PATTERN, complexUrl); - - try { - Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl)) - .select("meta[property=og:url]").first(); - - return replaceHttpWithHttps(ogElement.attr("content")); - } catch (Exception e) { - throw new ParsingException(e.getMessage(), e); - } - } - - @Override - public boolean acceptUrl(String url) { + public boolean onAcceptUrl(final String url) throws ParsingException { return Parser.isMatch(URL_PATTERN, url.toLowerCase()); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index b65be7b23..b3e90e725 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.soundcloud; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.SuggestionExtractor; import org.schabi.newpipe.extractor.UrlIdHandler; @@ -32,29 +33,29 @@ public class SoundcloudService extends StreamingService { } @Override - public UrlIdHandler getChannelUrlIdHandler() { + public ListUrlIdHandler getChannelUrlIdHandler() { return SoundcloudChannelUrlIdHandler.getInstance(); } @Override - public UrlIdHandler getPlaylistUrlIdHandler() { + public ListUrlIdHandler getPlaylistUrlIdHandler() { return SoundcloudPlaylistUrlIdHandler.getInstance(); } @Override - public StreamExtractor getStreamExtractor(String url) { - return new SoundcloudStreamExtractor(this, url); + public StreamExtractor getStreamExtractor(UrlIdHandler urlIdHandler) throws ExtractionException { + return new SoundcloudStreamExtractor(this, urlIdHandler); } @Override - public ChannelExtractor getChannelExtractor(String url) { - return new SoundcloudChannelExtractor(this, url); + public ChannelExtractor getChannelExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException { + return new SoundcloudChannelExtractor(this, urlIdHandler); } @Override - public PlaylistExtractor getPlaylistExtractor(String url) { - return new SoundcloudPlaylistExtractor(this, url); + public PlaylistExtractor getPlaylistExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException { + return new SoundcloudPlaylistExtractor(this, urlIdHandler); } @Override @@ -71,8 +72,7 @@ public class SoundcloudService extends StreamingService { String id) throws ExtractionException { return new SoundcloudChartsExtractor(SoundcloudService.this, - url, - id); + new SoundcloudChartsUrlIdHandler().setUrl(url), id); } }; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 30478b592..46fb76fb6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -22,8 +22,8 @@ import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudStreamExtractor extends StreamExtractor { private JsonObject track; - public SoundcloudStreamExtractor(StreamingService service, String url) { - super(service, url); + public SoundcloudStreamExtractor(StreamingService service, UrlIdHandler urlIdHandler) { + super(service, urlIdHandler); } @Override @@ -36,12 +36,6 @@ public class SoundcloudStreamExtractor extends StreamExtractor { } } - @Nonnull - @Override - public String getCleanUrl() { - return track.isString("permalink_url") ? replaceHttpWithHttps(track.getString("permalink_url")) : getOriginalUrl(); - } - @Nonnull @Override public String getId() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java index 0b9dd0286..7a760a2bb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java @@ -23,7 +23,7 @@ public class SoundcloudStreamUrlIdHandler extends UrlIdHandler { } @Override - public String getUrl(String id) throws ParsingException { + public String getUrl() throws ParsingException { try { return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/tracks/" + id); } catch (Exception e) { @@ -32,7 +32,7 @@ public class SoundcloudStreamUrlIdHandler extends UrlIdHandler { } @Override - public String getId(String url) throws ParsingException { + public String onGetIdFromUrl(String url) throws ParsingException { Utils.checkUrl(URL_PATTERN, url); try { @@ -43,21 +43,7 @@ public class SoundcloudStreamUrlIdHandler extends UrlIdHandler { } @Override - public String cleanUrl(String complexUrl) throws ParsingException { - Utils.checkUrl(URL_PATTERN, complexUrl); - - try { - Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl)) - .select("meta[property=og:url]").first(); - - return replaceHttpWithHttps(ogElement.attr("content")); - } catch (Exception e) { - throw new ParsingException(e.getMessage(), e); - } - } - - @Override - public boolean acceptUrl(String url) { + public boolean onAcceptUrl(final String url) throws ParsingException { return Parser.isMatch(URL_PATTERN, url.toLowerCase()); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java index 49e64414c..d34a4f8d0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java @@ -31,7 +31,7 @@ public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor { String id; try { - id = service.getChannelUrlIdHandler().getId(getUrlFrom(channelUrl)); + id = service.getChannelUrlIdHandler().setUrl(getUrlFrom(channelUrl)).getId(); } catch (ExtractionException e) { throw new InvalidSourceException(e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java index dfcfd6b85..65c26dd18 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java @@ -8,10 +8,7 @@ import com.sun.org.apache.xerces.internal.xs.StringList; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.schabi.newpipe.extractor.Downloader; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.*; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -52,16 +49,15 @@ public class YoutubeChannelExtractor extends ChannelExtractor { private Document doc; - public YoutubeChannelExtractor(StreamingService service, String url) { - super(service, url); + public YoutubeChannelExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) { + super(service, urlIdHandler); } @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - String channelUrl = super.getCleanUrl() + CHANNEL_URL_PARAMETERS; + String channelUrl = super.getUrl() + CHANNEL_URL_PARAMETERS; String pageContent = downloader.download(channelUrl); doc = Jsoup.parse(pageContent, channelUrl); - } @Override @@ -71,11 +67,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Nonnull @Override - public String getCleanUrl() { + public String getUrl() throws ParsingException { try { return "https://www.youtube.com/channel/" + getId(); } catch (ParsingException e) { - return super.getCleanUrl(); + return super.getUrl(); } } @@ -236,7 +232,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { collector.reset(); final String uploaderName = getName(); - final String uploaderUrl = getCleanUrl(); + final String uploaderUrl = getUrl(); for (final Element li : element.children()) { if (li.select("div[class=\"feed-item-dismissable\"]").first() != null) { collector.commit(new YoutubeStreamInfoItemExtractor(li) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandler.java index 259a5bf30..4e7bdea03 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandler.java @@ -35,22 +35,17 @@ public class YoutubeChannelUrlIdHandler extends ListUrlIdHandler { } @Override - public String getUrl(String id, String[] contentFilter, String sortFilter) { - return "https://www.youtube.com/" + id; - } - - @Override - public String getId(String url) throws ParsingException { + public String onGetIdFromUrl(String url) throws ParsingException { return Parser.matchGroup1(ID_PATTERN, url); } @Override - public String cleanUrl(String complexUrl) throws ParsingException { - return getUrl(getId(complexUrl)); + public String getUrl() { + return "https://www.youtube.com/" + id; } @Override - public boolean acceptUrl(String url) { + public boolean onAcceptUrl(String url) { return (url.contains("youtube") || url.contains("youtu.be") || url.contains("hooktube.com")) && (url.contains("/user/") || url.contains("/channel/")); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java index eff7adc1e..a1b7eec58 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java @@ -6,10 +6,7 @@ import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.schabi.newpipe.extractor.Downloader; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.*; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; @@ -26,14 +23,14 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { private Document doc; - public YoutubePlaylistExtractor(StreamingService service, String url) { - super(service, url); + public YoutubePlaylistExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) throws ExtractionException { + super(service, urlIdHandler); } @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - String pageContent = downloader.download(getCleanUrl()); - doc = Jsoup.parse(pageContent, getCleanUrl()); + String pageContent = downloader.download(getUrl()); + doc = Jsoup.parse(pageContent, getUrl()); } @Override @@ -41,16 +38,6 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { return getNextPageUrlFrom(doc); } - @Nonnull - @Override - public String getId() throws ParsingException { - try { - return getUrlIdHandler().getId(getCleanUrl()); - } catch (Exception e) { - throw new ParsingException("Could not get playlist id"); - } - } - @Nonnull @Override public String getName() throws ParsingException { @@ -202,7 +189,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getUrl() throws ParsingException { try { - return streamUrlIdHandler.getUrl(li.attr("data-video-id")); + return streamUrlIdHandler.setId(li.attr("data-video-id")).getUrl(); } catch (Exception e) { throw new ParsingException("Could not get web page url for the video", e); } @@ -267,7 +254,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getThumbnailUrl() throws ParsingException { try { - return "https://i.ytimg.com/vi/" + streamUrlIdHandler.getId(getUrl()) + "/hqdefault.jpg"; + return "https://i.ytimg.com/vi/" + streamUrlIdHandler.setUrl(getUrl()).getId() + "/hqdefault.jpg"; } catch (Exception e) { throw new ParsingException("Could not get thumbnail url", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistUrlIdHandler.java index e6032803c..cf0acb4dc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistUrlIdHandler.java @@ -16,12 +16,12 @@ public class YoutubePlaylistUrlIdHandler extends ListUrlIdHandler { } @Override - public String getUrl(String id, String[] contentFilter, String sortFilter) { + public String getUrl() { return "https://www.youtube.com/playlist?list=" + id; } @Override - public String getId(String url) throws ParsingException { + public String onGetIdFromUrl(String url) throws ParsingException { try { return Parser.matchGroup1("list=" + ID_PATTERN, url); } catch (final Exception exception) { @@ -29,13 +29,9 @@ public class YoutubePlaylistUrlIdHandler extends ListUrlIdHandler { } } - @Override - public String cleanUrl(String complexUrl) throws ParsingException { - return getUrl(getId(complexUrl)); - } @Override - public boolean acceptUrl(String url) { + public boolean onAcceptUrl(final String url) { final boolean hasNotEmptyUrl = url != null && !url.isEmpty(); final boolean isYoutubeDomain = hasNotEmptyUrl && (url.contains("youtube") || url.contains("youtu.be")); return isYoutubeDomain && url.contains("list="); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 2ac5dff29..9fb585055 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.SuggestionExtractor; import org.schabi.newpipe.extractor.UrlIdHandler; @@ -53,28 +54,28 @@ public class YoutubeService extends StreamingService { } @Override - public UrlIdHandler getChannelUrlIdHandler() { + public ListUrlIdHandler getChannelUrlIdHandler() { return YoutubeChannelUrlIdHandler.getInstance(); } @Override - public UrlIdHandler getPlaylistUrlIdHandler() { + public ListUrlIdHandler getPlaylistUrlIdHandler() { return YoutubePlaylistUrlIdHandler.getInstance(); } @Override - public StreamExtractor getStreamExtractor(String url) { - return new YoutubeStreamExtractor(this, url); + public StreamExtractor getStreamExtractor(UrlIdHandler urlIdHandler) throws ExtractionException { + return new YoutubeStreamExtractor(this, urlIdHandler); } @Override - public ChannelExtractor getChannelExtractor(String url) { - return new YoutubeChannelExtractor(this, url); + public ChannelExtractor getChannelExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException { + return new YoutubeChannelExtractor(this, urlIdHandler); } @Override - public PlaylistExtractor getPlaylistExtractor(String url) { - return new YoutubePlaylistExtractor(this, url); + public PlaylistExtractor getPlaylistExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException { + return new YoutubePlaylistExtractor(this, urlIdHandler); } @Override @@ -92,7 +93,8 @@ public class YoutubeService extends StreamingService { @Override public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String id) throws ExtractionException { - return new YoutubeTrendingExtractor(YoutubeService.this, url, id); + return new YoutubeTrendingExtractor(YoutubeService.this, + new YoutubeTrendingUrlIdHandler().setUrl(url), id); } }, new YoutubeTrendingUrlIdHandler(), "Trending"); list.setDefaultKiosk("Trending"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index a5d48a14d..9bbbe8b6c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -10,10 +10,7 @@ import org.jsoup.nodes.Element; import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; -import org.schabi.newpipe.extractor.Downloader; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.Subtitles; +import org.schabi.newpipe.extractor.*; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -86,24 +83,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { private boolean isAgeRestricted; - public YoutubeStreamExtractor(StreamingService service, String url) { - super(service, url); + public YoutubeStreamExtractor(StreamingService service, UrlIdHandler urlIdHandler) throws ExtractionException { + super(service, urlIdHandler); } /*////////////////////////////////////////////////////////////////////////// // Impl //////////////////////////////////////////////////////////////////////////*/ - @Nonnull - @Override - public String getId() throws ParsingException { - try { - return getUrlIdHandler().getId(getCleanUrl()); - } catch (Exception e) { - throw new ParsingException("Could not get stream id"); - } - } - @Nonnull @Override public String getName() throws ParsingException { @@ -581,7 +568,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { private String pageHtml = null; private String getPageHtml(Downloader downloader) throws IOException, ExtractionException { - final String verifiedUrl = getCleanUrl() + VERIFIED_URL_PARAMS; + final String verifiedUrl = getUrl() + VERIFIED_URL_PARAMS; if (pageHtml == null) { pageHtml = downloader.download(verifiedUrl); } @@ -591,7 +578,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { final String pageContent = getPageHtml(downloader); - doc = Jsoup.parse(pageContent, getCleanUrl()); + doc = Jsoup.parse(pageContent, getUrl()); final String playerUrl; // Check if the video is age restricted diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java index 25afe98a4..7dd3e6e92 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java @@ -6,6 +6,7 @@ import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; @@ -50,12 +51,12 @@ public class YoutubeStreamUrlIdHandler extends UrlIdHandler { } @Override - public String getUrl(String id) { + public String getUrl() { return "https://www.youtube.com/watch?v=" + id; } @Override - public String getId(String url) throws ParsingException, IllegalArgumentException { + public String onGetIdFromUrl(String url) throws ParsingException, IllegalArgumentException { if (url.isEmpty()) { throw new IllegalArgumentException("The url parameter should not be empty"); } @@ -167,19 +168,14 @@ public class YoutubeStreamUrlIdHandler extends UrlIdHandler { } @Override - public String cleanUrl(String complexUrl) throws ParsingException { - return getUrl(getId(complexUrl)); - } - - @Override - public boolean acceptUrl(String url) { - String lowercaseUrl = url.toLowerCase(); + public boolean onAcceptUrl(final String url) { + final String lowercaseUrl = url.toLowerCase(); if (lowercaseUrl.contains("youtube") || lowercaseUrl.contains("youtu.be") || lowercaseUrl.contains("hooktube")) { // bad programming I know try { - getId(url); + onGetIdFromUrl(url); return true; } catch (Exception e) { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java index 81c2c2680..59eddb508 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java @@ -25,6 +25,7 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.ListUrlIdHandler; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -40,15 +41,15 @@ public class YoutubeTrendingExtractor extends KioskExtractor { private Document doc; - public YoutubeTrendingExtractor(StreamingService service, String url, String kioskId) + public YoutubeTrendingExtractor(StreamingService service, ListUrlIdHandler urlIdHandler, String kioskId) throws ExtractionException { - super(service, url, kioskId); + super(service, urlIdHandler, kioskId); } @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { final String contentCountry = getContentCountry(); - String url = getCleanUrl(); + String url = getUrl(); if(contentCountry != null && !contentCountry.isEmpty()) { url += "?gl=" + contentCountry; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java index f6110484f..325c776c8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java @@ -27,22 +27,17 @@ import org.schabi.newpipe.extractor.utils.Parser; public class YoutubeTrendingUrlIdHandler extends ListUrlIdHandler { - public String getUrl(String id, String[] contentFilter, String sortFilter) { + public String getUrl() { return "https://www.youtube.com/feed/trending"; } @Override - public String getId(String url) { + public String onGetIdFromUrl(String url) { return "Trending"; } @Override - public String cleanUrl(String url) throws ParsingException { - return getUrl(""); - } - - @Override - public boolean acceptUrl(String url) { + public boolean onAcceptUrl(final String url) { return Parser.isMatch("^(https://|http://|)(www.|m.|)youtube.com/feed/trending(|\\?.*)$", url); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 8a2626b7c..a03bb6131 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -39,8 +39,8 @@ public abstract class StreamExtractor extends Extractor { public static final int NO_AGE_LIMIT = 0; - public StreamExtractor(StreamingService service, String url) { - super(service, url); + public StreamExtractor(StreamingService service, UrlIdHandler urlIdHandler) { + super(service, urlIdHandler); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 50b2ec0f2..9443e1ff4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -85,7 +85,7 @@ public class StreamInfo extends Info { // if one of these is not available an exception is meant to be thrown directly into the frontend. int serviceId = extractor.getServiceId(); - String url = extractor.getCleanUrl(); + String url = extractor.getUrl(); String originalUrl = extractor.getOriginalUrl(); StreamType streamType = extractor.getStreamType(); String id = extractor.getId(); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseExtractorTest.java index 020cceaf3..4ff51295b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseExtractorTest.java @@ -5,6 +5,6 @@ public interface BaseExtractorTest { void testServiceId() throws Exception; void testName() throws Exception; void testId() throws Exception; - void testCleanUrl() throws Exception; + void testUrl() throws Exception; void testOriginalUrl() throws Exception; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BasePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BasePlaylistExtractorTest.java index da5bfdcb4..ea097a3b5 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BasePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BasePlaylistExtractorTest.java @@ -4,7 +4,6 @@ package org.schabi.newpipe.extractor.services; public interface BasePlaylistExtractorTest extends BaseListExtractorTest { void testThumbnailUrl() throws Exception; void testBannerUrl() throws Exception; - void testUploaderUrl() throws Exception; void testUploaderName() throws Exception; void testUploaderAvatarUrl() throws Exception; void testStreamCount() throws Exception; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java index 9266aaa86..30884adcb 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java @@ -5,6 +5,7 @@ import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import static org.junit.Assert.*; @@ -48,12 +49,12 @@ public class SoundcloudChannelExtractorTest { } @Test - public void testCleanUrl() { - assertEquals("https://soundcloud.com/liluzivert", extractor.getCleanUrl()); + public void testUrl() throws ParsingException { + assertEquals("https://soundcloud.com/liluzivert", extractor.getUrl()); } @Test - public void testOriginalUrl() { + public void testOriginalUrl() throws ParsingException { assertEquals("http://soundcloud.com/liluzivert/sets", extractor.getOriginalUrl()); } @@ -118,7 +119,7 @@ public class SoundcloudChannelExtractorTest { @Test public void testGetPageInNewExtractor() throws Exception { - final ChannelExtractor newExtractor = SoundCloud.getChannelExtractor(extractor.getCleanUrl()); + final ChannelExtractor newExtractor = SoundCloud.getChannelExtractor(extractor.getUrl()); defaultTestGetPageInNewExtractor(extractor, newExtractor, SoundCloud.getServiceId()); } @@ -142,12 +143,12 @@ public class SoundcloudChannelExtractorTest { } @Test - public void testCleanUrl() { - assertEquals("https://soundcloud.com/dubmatix", extractor.getCleanUrl()); + public void testUrl() throws ParsingException { + assertEquals("https://soundcloud.com/dubmatix", extractor.getUrl()); } @Test - public void testOriginalUrl() { + public void testOriginalUrl() throws ParsingException { assertEquals("https://soundcloud.com/dubmatix", extractor.getOriginalUrl()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java index f3e720c85..a77230244 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java @@ -88,6 +88,6 @@ public class SoundcloudChartsExtractorTest { @Test public void testGetCleanUrl() throws Exception { - assertEquals(extractor.getCleanUrl(), "https://soundcloud.com/charts/top"); + assertEquals(extractor.getUrl(), "https://soundcloud.com/charts/top"); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java index 76e52cd78..4fa09ffa5 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java @@ -4,6 +4,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import static junit.framework.TestCase.assertFalse; import static org.junit.Assert.assertEquals; @@ -23,18 +24,18 @@ public class SoundcloudChartsUrlIdHandlerTest { @Test public void getUrl() throws Exception { - assertEquals(urlIdHandler.getUrl("Top 50"), "https://soundcloud.com/charts/top"); - assertEquals(urlIdHandler.getUrl("New & hot"), "https://soundcloud.com/charts/new"); + assertEquals(urlIdHandler.setId("Top 50").getUrl(), "https://soundcloud.com/charts/top"); + assertEquals(urlIdHandler.setId("New & hot").getUrl(), "https://soundcloud.com/charts/new"); } @Test - public void getId() { - assertEquals(urlIdHandler.getId("http://soundcloud.com/charts/top?genre=all-music"), "Top 50"); - assertEquals(urlIdHandler.getId("HTTP://www.soundcloud.com/charts/new/?genre=all-music&country=all-countries"), "New & hot"); + public void getId() throws ParsingException { + assertEquals(urlIdHandler.setUrl("http://soundcloud.com/charts/top?genre=all-music").getId(), "Top 50"); + assertEquals(urlIdHandler.setUrl("HTTP://www.soundcloud.com/charts/new/?genre=all-music&country=all-countries").getId(), "New & hot"); } @Test - public void acceptUrl() { + public void acceptUrl() throws ParsingException { assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/charts")); assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/charts/")); assertTrue(urlIdHandler.acceptUrl("https://www.soundcloud.com/charts/new")); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index 017efc50a..29bf4d177 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -51,12 +51,12 @@ public class SoundcloudPlaylistExtractorTest { } @Test - public void testCleanUrl() { - assertEquals("https://soundcloud.com/liluzivert/sets/the-perfect-luv-tape-r", extractor.getCleanUrl()); + public void testUrl() throws Exception { + assertEquals("https://soundcloud.com/liluzivert/sets/the-perfect-luv-tape-r", extractor.getUrl()); } @Test - public void testOriginalUrl() { + public void testOriginalUrl() throws Exception { assertEquals("https://soundcloud.com/liluzivert/sets/the-perfect-luv-tape-r?test=123", extractor.getOriginalUrl()); } @@ -125,7 +125,7 @@ public class SoundcloudPlaylistExtractorTest { public static void setUp() throws Exception { NewPipe.init(Downloader.getInstance()); extractor = (SoundcloudPlaylistExtractor) SoundCloud - .getPlaylistExtractor("http://soundcloud.com/finn-trapple/sets/random-house-dance-music-2"); + .getPlaylistExtractor("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2"); extractor.fetchPage(); } @@ -140,22 +140,22 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testName() { - assertEquals("Random House & Dance Music #2", extractor.getName()); + assertEquals("House, Electro , Dance Music 2", extractor.getName()); } @Test public void testId() { - assertEquals("436855608", extractor.getId()); + assertEquals("310980722", extractor.getId()); } @Test - public void testCleanUrl() { - assertEquals("https://soundcloud.com/finn-trapple/sets/random-house-dance-music-2", extractor.getCleanUrl()); + public void testUrl() throws Exception { + assertEquals("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2", extractor.getUrl()); } @Test - public void testOriginalUrl() { - assertEquals("http://soundcloud.com/finn-trapple/sets/random-house-dance-music-2", extractor.getOriginalUrl()); + public void testOriginalUrl() throws Exception { + assertEquals("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2", extractor.getOriginalUrl()); } /*////////////////////////////////////////////////////////////////////////// @@ -191,12 +191,12 @@ public class SoundcloudPlaylistExtractorTest { public void testUploaderUrl() { final String uploaderUrl = extractor.getUploaderUrl(); assertIsSecureUrl(uploaderUrl); - assertTrue(uploaderUrl, uploaderUrl.contains("finn-trapple")); + assertTrue(uploaderUrl, uploaderUrl.contains("hunter-leader")); } @Test public void testUploaderName() { - assertEquals("Finn TrApple", extractor.getUploaderName()); + assertEquals("Gosu", extractor.getUploaderName()); } @Test @@ -227,7 +227,7 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testGetPageInNewExtractor() throws Exception { - final PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getCleanUrl()); + final PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getUrl()); defaultTestGetPageInNewExtractor(extractor, newExtractor, SoundCloud.getServiceId()); } @@ -251,12 +251,12 @@ public class SoundcloudPlaylistExtractorTest { } @Test - public void testCleanUrl() { - assertEquals("https://soundcloud.com/user350509423/sets/edm-xxx", extractor.getCleanUrl()); + public void testUrl() throws Exception { + assertEquals("https://soundcloud.com/user350509423/sets/edm-xxx", extractor.getUrl()); } @Test - public void testOriginalUrl() { + public void testOriginalUrl() throws Exception { assertEquals("https://soundcloud.com/user350509423/sets/edm-xxx", extractor.getOriginalUrl()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandlerTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandlerTest.java index f2e5e2218..5c1d35cf4 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandlerTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandlerTest.java @@ -25,7 +25,7 @@ public class SoundcloudStreamUrlIdHandlerTest { @Test(expected = IllegalArgumentException.class) public void getIdWithNullAsUrl() throws ParsingException { - urlIdHandler.getId(null); + urlIdHandler.setUrl(null).getId(); } @Test @@ -37,7 +37,7 @@ public class SoundcloudStreamUrlIdHandlerTest { for (String invalidUrl : invalidUrls) { Throwable exception = null; try { - urlIdHandler.getId(invalidUrl); + urlIdHandler.setUrl(invalidUrl).getId(); } catch (ParsingException e) { exception = e; } @@ -49,21 +49,21 @@ public class SoundcloudStreamUrlIdHandlerTest { @Test public void getId() throws Exception { - assertEquals("309689103", urlIdHandler.getId("https://soundcloud.com/liluzivert/15-ysl")); - assertEquals("309689082", urlIdHandler.getId("https://www.soundcloud.com/liluzivert/15-luv-scars-ko")); - assertEquals("309689035", urlIdHandler.getId("http://soundcloud.com/liluzivert/15-boring-shit")); - assertEquals("294488599", urlIdHandler.getId("http://www.soundcloud.com/liluzivert/secure-the-bag-produced-by-glohan-beats")); - assertEquals("294488438", urlIdHandler.getId("HtTpS://sOuNdClOuD.cOm/LiLuZiVeRt/In-O4-pRoDuCeD-bY-dP-bEaTz")); - assertEquals("294488147", urlIdHandler.getId("https://soundcloud.com/liluzivert/fresh-produced-by-zaytoven#t=69")); - assertEquals("294487876", urlIdHandler.getId("https://soundcloud.com/liluzivert/threesome-produced-by-zaytoven#t=1:09")); - assertEquals("294487684", urlIdHandler.getId("https://soundcloud.com/liluzivert/blonde-brigitte-produced-manny-fresh#t=1:9")); - assertEquals("294487428", urlIdHandler.getId("https://soundcloud.com/liluzivert/today-produced-by-c-note#t=1m9s")); - assertEquals("294487157", urlIdHandler.getId("https://soundcloud.com/liluzivert/changed-my-phone-produced-by-c-note#t=1m09s")); + assertEquals("309689103", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/15-ysl").getId()); + assertEquals("309689082", urlIdHandler.setUrl("https://www.soundcloud.com/liluzivert/15-luv-scars-ko").getId()); + assertEquals("309689035", urlIdHandler.setUrl("http://soundcloud.com/liluzivert/15-boring-shit").getId()); + assertEquals("294488599", urlIdHandler.setUrl("http://www.soundcloud.com/liluzivert/secure-the-bag-produced-by-glohan-beats").getId()); + assertEquals("294488438", urlIdHandler.setUrl("HtTpS://sOuNdClOuD.cOm/LiLuZiVeRt/In-O4-pRoDuCeD-bY-dP-bEaTz").getId()); + assertEquals("294488147", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/fresh-produced-by-zaytoven#t=69").getId()); + assertEquals("294487876", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/threesome-produced-by-zaytoven#t=1:09").getId()); + assertEquals("294487684", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/blonde-brigitte-produced-manny-fresh#t=1:9").getId()); + assertEquals("294487428", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/today-produced-by-c-note#t=1m9s").getId()); + assertEquals("294487157", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/changed-my-phone-produced-by-c-note#t=1m09s").getId()); } @Test - public void testAcceptUrl() { + public void testAcceptUrl() throws ParsingException { assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/liluzivert/15-ysl")); assertTrue(urlIdHandler.acceptUrl("https://www.soundcloud.com/liluzivert/15-luv-scars-ko")); assertTrue(urlIdHandler.acceptUrl("http://soundcloud.com/liluzivert/15-boring-shit")); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java index cbe1948e1..e0e998030 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java @@ -6,6 +6,7 @@ import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionItem; @@ -64,7 +65,7 @@ public class SoundcloudSubscriptionExtractorTest { } } - private void testList(List subscriptionItems) { + private void testList(List subscriptionItems) throws ParsingException { for (SubscriptionItem item : subscriptionItems) { assertNotNull(item.getName()); assertNotNull(item.getUrl()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 510f210a9..2a905a288 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -7,6 +7,7 @@ import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import static org.junit.Assert.*; @@ -49,12 +50,12 @@ public class YoutubeChannelExtractorTest { } @Test - public void testCleanUrl() { - assertEquals("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw", extractor.getCleanUrl()); + public void testUrl() throws ParsingException { + assertEquals("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw", extractor.getUrl()); } @Test - public void testOriginalUrl() { + public void testOriginalUrl() throws ParsingException { assertEquals("http://www.youtube.com/user/Gronkh", extractor.getOriginalUrl()); } @@ -130,7 +131,7 @@ public class YoutubeChannelExtractorTest { @Test public void testGetPageInNewExtractor() throws Exception { - final ChannelExtractor newExtractor = YouTube.getChannelExtractor(extractor.getCleanUrl()); + final ChannelExtractor newExtractor = YouTube.getChannelExtractor(extractor.getUrl()); defaultTestGetPageInNewExtractor(extractor, newExtractor, YouTube.getServiceId()); } @@ -155,12 +156,12 @@ public class YoutubeChannelExtractorTest { } @Test - public void testCleanUrl() { - assertEquals("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q", extractor.getCleanUrl()); + public void testUrl() throws ParsingException { + assertEquals("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q", extractor.getUrl()); } @Test - public void testOriginalUrl() { + public void testOriginalUrl() throws ParsingException { assertEquals("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q", extractor.getOriginalUrl()); } @@ -251,12 +252,12 @@ public class YoutubeChannelExtractorTest { } @Test - public void testCleanUrl() { - assertEquals("https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg", extractor.getCleanUrl()); + public void testUrl() throws ParsingException { + assertEquals("https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg", extractor.getUrl()); } @Test - public void testOriginalUrl() { + public void testOriginalUrl() throws ParsingException { assertEquals("https://www.youtube.com/user/CaptainDisillusion/videos", extractor.getOriginalUrl()); } @@ -340,12 +341,12 @@ public class YoutubeChannelExtractorTest { } @Test - public void testCleanUrl() { - assertEquals("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w", extractor.getCleanUrl()); + public void testUrl() throws ParsingException { + assertEquals("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w", extractor.getUrl()); } @Test - public void testOriginalUrl() { + public void testOriginalUrl() throws ParsingException { assertEquals("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w", extractor.getOriginalUrl()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandlerTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandlerTest.java index 57bccdf90..59d7e75e1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandlerTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandlerTest.java @@ -39,17 +39,17 @@ public class YoutubeChannelUrlIdHandlerTest { @Test public void getIdFromUrl() throws ParsingException { - assertEquals("user/Gronkh", urlIdHandler.getId("https://www.youtube.com/user/Gronkh")); - assertEquals("user/Netzkino", urlIdHandler.getId("https://www.youtube.com/user/Netzkino/videos")); + assertEquals("user/Gronkh", urlIdHandler.setUrl("https://www.youtube.com/user/Gronkh").getId()); + assertEquals("user/Netzkino", urlIdHandler.setUrl("https://www.youtube.com/user/Netzkino/videos").getId()); - assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.getId("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA")); - assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.getId("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1")); + assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.setUrl("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA").getId()); + assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.setUrl("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId()); - assertEquals("user/Gronkh", urlIdHandler.getId("https://hooktube.com/user/Gronkh")); - assertEquals("user/Netzkino", urlIdHandler.getId("https://hooktube.com/user/Netzkino/videos")); + assertEquals("user/Gronkh", urlIdHandler.setUrl("https://hooktube.com/user/Gronkh").getId()); + assertEquals("user/Netzkino", urlIdHandler.setUrl("https://hooktube.com/user/Netzkino/videos").getId()); - assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.getId("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA")); - assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.getId("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1")); + assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.setUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA").getId()); + assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.setUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index 6b51d0d01..8e446aebc 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -7,6 +7,7 @@ import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; import org.schabi.newpipe.extractor.stream.StreamInfoItem; @@ -53,12 +54,12 @@ public class YoutubePlaylistExtractorTest { } @Test - public void testCleanUrl() { - assertEquals("https://www.youtube.com/playlist?list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj", extractor.getCleanUrl()); + public void testUrl() throws ParsingException { + assertEquals("https://www.youtube.com/playlist?list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj", extractor.getUrl()); } @Test - public void testOriginalUrl() { + public void testOriginalUrl() throws ParsingException { assertEquals("http://www.youtube.com/watch?v=lp-EO5I60KA&list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj", extractor.getOriginalUrl()); } @@ -135,7 +136,7 @@ public class YoutubePlaylistExtractorTest { @Test public void testGetPageInNewExtractor() throws Exception { - final PlaylistExtractor newExtractor = YouTube.getPlaylistExtractor(extractor.getCleanUrl()); + final PlaylistExtractor newExtractor = YouTube.getPlaylistExtractor(extractor.getUrl()); defaultTestGetPageInNewExtractor(extractor, newExtractor, YouTube.getServiceId()); } @@ -160,12 +161,12 @@ public class YoutubePlaylistExtractorTest { } @Test - public void testCleanUrl() { - assertEquals("https://www.youtube.com/playlist?list=PLOy0j9AvlVZPto6IkjKfpu0Scx--7PGTC", extractor.getCleanUrl()); + public void testUrl() throws ParsingException { + assertEquals("https://www.youtube.com/playlist?list=PLOy0j9AvlVZPto6IkjKfpu0Scx--7PGTC", extractor.getUrl()); } @Test - public void testOriginalUrl() { + public void testOriginalUrl() throws ParsingException { assertEquals("https://www.youtube.com/playlist?list=PLOy0j9AvlVZPto6IkjKfpu0Scx--7PGTC", extractor.getOriginalUrl()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java index 93b0677bb..e51e87f88 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java @@ -108,7 +108,7 @@ public class YoutubeStreamExtractorRestrictedTest { streams.addAll(extractor.getVideoStreams()); streams.addAll(extractor.getVideoOnlyStreams()); - assertTrue(streams.size() > 0); + assertTrue(Integer.toString(streams.size()),streams.size() > 0); for (VideoStream s : streams) { assertTrue(s.getUrl(), s.getUrl().contains(HTTPS)); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandlerTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandlerTest.java index 2711c7aed..89e525bd8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandlerTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandlerTest.java @@ -25,14 +25,14 @@ public class YoutubeStreamUrlIdHandlerTest { NewPipe.init(Downloader.getInstance()); } - @Test(expected = NullPointerException.class) + @Test(expected = IllegalArgumentException.class) public void getIdWithNullAsUrl() throws ParsingException { - urlIdHandler.getId(null); + urlIdHandler.setId(null); } @Test(expected = FoundAdException.class) public void getIdForAd() throws ParsingException { - urlIdHandler.getId(AD_URL); + urlIdHandler.setUrl(AD_URL).getId(); } @Test @@ -44,7 +44,7 @@ public class YoutubeStreamUrlIdHandlerTest { for (String invalidUrl : invalidUrls) { Throwable exception = null; try { - urlIdHandler.getId(invalidUrl); + urlIdHandler.setUrl(invalidUrl).getId(); } catch (ParsingException e) { exception = e; } @@ -56,42 +56,42 @@ public class YoutubeStreamUrlIdHandlerTest { @Test public void getIdfromYt() throws Exception { - assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/watch?v=jZViOEv90dI")); - assertEquals("W-fFHeTX70Q", urlIdHandler.getId("https://www.youtube.com/watch?v=W-fFHeTX70Q")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/watch?v=jZViOEv90dI?t=100")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("HTTPS://www.youtube.com/watch?v=jZViOEv90dI?t=100")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("https://youtu.be/jZViOEv90dI?t=9s")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("HTTPS://Youtu.be/jZViOEv90dI?t=9s")); - assertEquals("uEJuoEs1UxY", urlIdHandler.getId("http://www.youtube.com/watch_popup?v=uEJuoEs1UxY")); - assertEquals("uEJuoEs1UxY", urlIdHandler.getId("http://www.Youtube.com/watch_popup?v=uEJuoEs1UxY")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/embed/jZViOEv90dI")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube-nocookie.com/embed/jZViOEv90dI")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube.com/watch?v=jZViOEv90dI")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("http://youtube.com/watch?v=jZViOEv90dI")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("http://youtu.be/jZViOEv90dI?t=9s")); - assertEquals("7_WWz2DSnT8", urlIdHandler.getId("https://youtu.be/7_WWz2DSnT8")); - assertEquals("oy6NvWeVruY", urlIdHandler.getId("https://m.youtube.com/watch?v=oy6NvWeVruY")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube.com/embed/jZViOEv90dI")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.Youtube.com/embed/jZViOEv90dI")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube-nocookie.com/embed/jZViOEv90dI")); - assertEquals("EhxJLojIE_o", urlIdHandler.getId("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI")); - assertEquals("jZViOEv90dI", urlIdHandler.getId("vnd.youtube:jZViOEv90dI")); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://www.youtube.com/watch?v=jZViOEv90dI").getId()); + assertEquals("W-fFHeTX70Q", urlIdHandler.setUrl("https://www.youtube.com/watch?v=W-fFHeTX70Q").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://www.youtube.com/watch?v=jZViOEv90dI?t=100").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("HTTPS://www.youtube.com/watch?v=jZViOEv90dI?t=100").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://youtu.be/jZViOEv90dI?t=9s").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("HTTPS://Youtu.be/jZViOEv90dI?t=9s").getId()); + assertEquals("uEJuoEs1UxY", urlIdHandler.setUrl("http://www.youtube.com/watch_popup?v=uEJuoEs1UxY").getId()); + assertEquals("uEJuoEs1UxY", urlIdHandler.setUrl("http://www.Youtube.com/watch_popup?v=uEJuoEs1UxY").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://www.youtube.com/embed/jZViOEv90dI").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://www.youtube-nocookie.com/embed/jZViOEv90dI").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://www.youtube.com/watch?v=jZViOEv90dI").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://youtube.com/watch?v=jZViOEv90dI").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://youtu.be/jZViOEv90dI?t=9s").getId()); + assertEquals("7_WWz2DSnT8", urlIdHandler.setUrl("https://youtu.be/7_WWz2DSnT8").getId()); + assertEquals("oy6NvWeVruY", urlIdHandler.setUrl("https://m.youtube.com/watch?v=oy6NvWeVruY").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://www.youtube.com/embed/jZViOEv90dI").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://www.Youtube.com/embed/jZViOEv90dI").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://www.youtube-nocookie.com/embed/jZViOEv90dI").getId()); + assertEquals("EhxJLojIE_o", urlIdHandler.setUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI").getId()); + assertEquals("jZViOEv90dI", urlIdHandler.setUrl("vnd.youtube:jZViOEv90dI").getId()); } @Test public void getIdfromSharedLinksYt() throws Exception { String sharedId = "7JIArTByb3E"; String realId = "Q7JsK50NGaA"; - assertEquals(realId, urlIdHandler.getId("vnd.youtube://www.YouTube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link")); - assertEquals(realId, urlIdHandler.getId("vnd.youtube://www.youtube.com/shared?ci=" + sharedId)); - assertEquals(realId, urlIdHandler.getId("https://www.youtube.com/shared?ci=" + sharedId)); + assertEquals(realId, urlIdHandler.setUrl("vnd.youtube://www.YouTube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link").getId()); + assertEquals(realId, urlIdHandler.setUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId).getId()); + assertEquals(realId, urlIdHandler.setUrl("https://www.youtube.com/shared?ci=" + sharedId).getId()); } @Test - public void testAcceptYtUrl() { + public void testAcceptYtUrl() throws ParsingException { assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI")); assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI?t=100")); assertTrue(urlIdHandler.acceptUrl("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100")); @@ -111,7 +111,7 @@ public class YoutubeStreamUrlIdHandlerTest { } @Test - public void testAcceptSharedYtUrl() { + public void testAcceptSharedYtUrl() throws ParsingException { String sharedId = "8A940MXKFmQ"; assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link")); assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId)); @@ -119,7 +119,7 @@ public class YoutubeStreamUrlIdHandlerTest { } @Test - public void testAcceptHookUrl() { + public void testAcceptHookUrl() throws ParsingException { assertTrue(urlIdHandler.acceptUrl("https://hooktube.com/watch?v=TglNG-yjabU")); assertTrue(urlIdHandler.acceptUrl("hooktube.com/watch?v=3msbfr6pBNE")); assertTrue(urlIdHandler.acceptUrl("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2")); @@ -130,11 +130,11 @@ public class YoutubeStreamUrlIdHandlerTest { @Test public void testGetHookIdfromUrl() throws ParsingException { - assertEquals("TglNG-yjabU", urlIdHandler.getId("https://hooktube.com/watch?v=TglNG-yjabU")); - assertEquals("3msbfr6pBNE", urlIdHandler.getId("hooktube.com/watch?v=3msbfr6pBNE")); - assertEquals("ocH3oSnZG3c", urlIdHandler.getId("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2")); - assertEquals("3msbfr6pBNE", urlIdHandler.getId("hooktube.com/watch/3msbfr6pBNE")); - assertEquals("3msbfr6pBNE", urlIdHandler.getId("hooktube.com/v/3msbfr6pBNE")); - assertEquals("3msbfr6pBNE", urlIdHandler.getId("hooktube.com/embed/3msbfr6pBNE")); + assertEquals("TglNG-yjabU", urlIdHandler.setUrl("https://hooktube.com/watch?v=TglNG-yjabU").getId()); + assertEquals("3msbfr6pBNE", urlIdHandler.setUrl("hooktube.com/watch?v=3msbfr6pBNE").getId()); + assertEquals("ocH3oSnZG3c", urlIdHandler.setUrl("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2").getId()); + assertEquals("3msbfr6pBNE", urlIdHandler.setUrl("hooktube.com/watch/3msbfr6pBNE").getId()); + assertEquals("3msbfr6pBNE", urlIdHandler.setUrl("hooktube.com/v/3msbfr6pBNE").getId()); + assertEquals("3msbfr6pBNE", urlIdHandler.setUrl("hooktube.com/embed/3msbfr6pBNE").getId()); } } \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java index d4599e983..71c856821 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java @@ -92,7 +92,7 @@ public class YoutubeTrendingExtractorTest { } @Test - public void testGetCleanUrl() { - assertEquals(extractor.getCleanUrl(), extractor.getCleanUrl(), "https://www.youtube.com/feed/trending"); + public void testGetUrl() throws Exception { + assertEquals(extractor.getUrl(), extractor.getUrl(), "https://www.youtube.com/feed/trending"); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java index 3fff4419e..55f374a12 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java @@ -45,7 +45,7 @@ public class YoutubeTrendingKioskInfoTest { StreamingService service = YouTube; UrlIdHandler urlIdHandler = service.getKioskList().getUrlIdHandlerByType("Trending"); - kioskInfo = KioskInfo.getInfo(YouTube, urlIdHandler.getUrl("Trending"), null); + kioskInfo = KioskInfo.getInfo(YouTube, urlIdHandler.setId("Trending").getUrl(), null); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java index 7ee68a39a..434792a44 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java @@ -25,6 +25,7 @@ import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import static junit.framework.TestCase.assertFalse; import static org.junit.Assert.assertEquals; @@ -46,13 +47,13 @@ public class YoutubeTrendingUrlIdHandlerTest { @Test public void getUrl() throws Exception { - assertEquals(urlIdHandler.getUrl(""), "https://www.youtube.com/feed/trending"); + assertEquals(urlIdHandler.setId("").getUrl(), "https://www.youtube.com/feed/trending"); } @Test public void getId() throws Exception { - assertEquals(urlIdHandler.getId(""), "Trending"); + assertEquals(urlIdHandler.setUrl("").getId(), "Trending"); } @Test From 94d1eb6f3288a49bcea7e776e2208ae89fa30533 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Tue, 8 May 2018 21:19:03 +0200 Subject: [PATCH 5/7] restructure youtube service --- .../newpipe/extractor/services/youtube/YoutubeService.java | 5 +++++ .../youtube/{ => extractors}/YoutubeChannelExtractor.java | 3 +-- .../{ => extractors}/YoutubeChannelInfoItemExtractor.java | 2 +- .../youtube/{ => extractors}/YoutubePlaylistExtractor.java | 3 ++- .../{ => extractors}/YoutubePlaylistInfoItemExtractor.java | 2 +- .../youtube/{ => extractors}/YoutubeSearchEngine.java | 2 +- .../youtube/{ => extractors}/YoutubeStreamExtractor.java | 3 ++- .../{ => extractors}/YoutubeStreamInfoItemExtractor.java | 3 ++- .../{ => extractors}/YoutubeSubscriptionExtractor.java | 3 ++- .../youtube/{ => extractors}/YoutubeSuggestionExtractor.java | 2 +- .../youtube/{ => extractors}/YoutubeTrendingExtractor.java | 3 ++- .../{ => urlIdHandlers}/YoutubeChannelUrlIdHandler.java | 3 +-- .../youtube/{ => urlIdHandlers}/YoutubeParsingHelper.java | 2 +- .../{ => urlIdHandlers}/YoutubePlaylistUrlIdHandler.java | 3 +-- .../{ => urlIdHandlers}/YoutubeStreamUrlIdHandler.java | 3 +-- .../{ => urlIdHandlers}/YoutubeTrendingUrlIdHandler.java | 2 +- .../services/youtube/YoutubeChannelExtractorTest.java | 2 +- .../services/youtube/YoutubeChannelUrlIdHandlerTest.java | 1 + .../services/youtube/YoutubePlaylistExtractorTest.java | 1 + .../services/youtube/YoutubeSearchEngineAllTest.java | 1 + .../youtube/YoutubeStreamExtractorControversialTest.java | 2 ++ .../services/youtube/YoutubeStreamExtractorDefaultTest.java | 1 + .../services/youtube/YoutubeStreamExtractorDonationTest.java | 1 + .../youtube/YoutubeStreamExtractorRestrictedTest.java | 2 ++ .../services/youtube/YoutubeStreamUrlIdHandlerTest.java | 1 + .../services/youtube/YoutubeSubscriptionExtractorTest.java | 1 + .../services/youtube/YoutubeTrendingExtractorTest.java | 2 ++ .../services/youtube/YoutubeTrendingUrlIdHandlerTest.java | 2 +- 28 files changed, 40 insertions(+), 21 deletions(-) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => extractors}/YoutubeChannelExtractor.java (99%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => extractors}/YoutubeChannelInfoItemExtractor.java (97%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => extractors}/YoutubePlaylistExtractor.java (98%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => extractors}/YoutubePlaylistInfoItemExtractor.java (97%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => extractors}/YoutubeSearchEngine.java (98%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => extractors}/YoutubeStreamExtractor.java (99%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => extractors}/YoutubeStreamInfoItemExtractor.java (97%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => extractors}/YoutubeSubscriptionExtractor.java (97%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => extractors}/YoutubeSuggestionExtractor.java (97%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => extractors}/YoutubeTrendingExtractor.java (97%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => urlIdHandlers}/YoutubeChannelUrlIdHandler.java (94%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => urlIdHandlers}/YoutubeParsingHelper.java (96%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => urlIdHandlers}/YoutubePlaylistUrlIdHandler.java (92%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => urlIdHandlers}/YoutubeStreamUrlIdHandler.java (98%) rename extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/{ => urlIdHandlers}/YoutubeTrendingUrlIdHandler.java (95%) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 9fb585055..77929139b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -10,6 +10,11 @@ import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchEngine; +import org.schabi.newpipe.extractor.services.youtube.extractors.*; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeChannelUrlIdHandler; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubePlaylistUrlIdHandler; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeStreamUrlIdHandler; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeTrendingUrlIdHandler; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java similarity index 99% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 65c26dd18..a602f0ea1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -1,10 +1,9 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import com.sun.org.apache.xerces.internal.xs.StringList; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java similarity index 97% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index a04d85622..0c1e5254d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.extractors; import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java similarity index 98% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index a1b7eec58..601f6caeb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; @@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.*; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamType; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java similarity index 97% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java index fe6e7768f..1b5fa4492 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.extractors; import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.exceptions.ParsingException; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchEngine.java similarity index 98% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchEngine.java index d393c3600..cecfbd182 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchEngine.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.extractors; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java similarity index 99% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 9bbbe8b6c..87571e8b7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -15,6 +15,7 @@ import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.services.youtube.ItagItem; import org.schabi.newpipe.extractor.stream.*; import org.schabi.newpipe.extractor.utils.DonationLinkHelper; import org.schabi.newpipe.extractor.utils.Parser; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java similarity index 97% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index e4146313d..3b1bb7191 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -1,7 +1,8 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.extractors; import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.Utils; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java similarity index 97% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java index 32efe16be..f6b6b8bd9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java @@ -1,9 +1,10 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.extractors; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.services.youtube.YoutubeService; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionItem; import org.schabi.newpipe.extractor.utils.Parser; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java similarity index 97% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java index cd98f25b7..5d77ab0c3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonParser; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java similarity index 97% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index 59eddb508..9c4cb0802 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.extractors; /* * Created by Christian Schabesberger on 12.08.17. @@ -31,6 +31,7 @@ import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeTrendingUrlIdHandler; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeChannelUrlIdHandler.java similarity index 94% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandler.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeChannelUrlIdHandler.java index 4e7bdea03..e6dbf09b5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeChannelUrlIdHandler.java @@ -1,7 +1,6 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.urlIdHandlers; import org.schabi.newpipe.extractor.ListUrlIdHandler; -import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Parser; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeParsingHelper.java similarity index 96% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeParsingHelper.java index a0ae0c6c4..90f464ed9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeParsingHelper.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.urlIdHandlers; import org.schabi.newpipe.extractor.exceptions.ParsingException; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubePlaylistUrlIdHandler.java similarity index 92% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistUrlIdHandler.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubePlaylistUrlIdHandler.java index cf0acb4dc..262cbfc24 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubePlaylistUrlIdHandler.java @@ -1,8 +1,7 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.urlIdHandlers; import org.schabi.newpipe.extractor.ListUrlIdHandler; -import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.Parser; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeStreamUrlIdHandler.java similarity index 98% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeStreamUrlIdHandler.java index 7dd3e6e92..758706622 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeStreamUrlIdHandler.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.urlIdHandlers; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -6,7 +6,6 @@ import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.UrlIdHandler; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeTrendingUrlIdHandler.java similarity index 95% rename from extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeTrendingUrlIdHandler.java index 325c776c8..c3696cd72 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeTrendingUrlIdHandler.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.urlIdHandlers; /* * Created by Christian Schabesberger on 12.08.17. diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 2a905a288..c9ab333ff 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -1,7 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; @@ -9,6 +8,7 @@ import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandlerTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandlerTest.java index 59d7e75e1..345d8b95c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandlerTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandlerTest.java @@ -5,6 +5,7 @@ import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeChannelUrlIdHandler; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index 8e446aebc..957ceacc9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import static org.junit.Assert.assertEquals; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java index 57422ad70..5742f96c5 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.search.SearchEngine; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchEngine; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java index de8fdca6a..e36faeb20 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java @@ -7,6 +7,8 @@ import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeStreamUrlIdHandler; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.SubtitlesFormat; import org.schabi.newpipe.extractor.stream.VideoStream; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java index 88aa1b96e..982ceb986 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java @@ -6,6 +6,7 @@ import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.extractor.stream.*; import org.schabi.newpipe.extractor.utils.Utils; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java index 9f26a7bd2..60ee05460 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java @@ -25,6 +25,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; import static org.junit.Assert.assertTrue; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java index e51e87f88..35d7760d7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java @@ -7,6 +7,8 @@ import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeStreamUrlIdHandler; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.SubtitlesFormat; import org.schabi.newpipe.extractor.stream.VideoStream; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandlerTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandlerTest.java index 89e525bd8..57f000799 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandlerTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandlerTest.java @@ -6,6 +6,7 @@ import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeStreamUrlIdHandler; import java.util.ArrayList; import java.util.List; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java index 59c84636c..6b5f28759 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java @@ -6,6 +6,7 @@ import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.UrlIdHandler; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscriptionExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionItem; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java index 71c856821..ae8c9f611 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java @@ -25,6 +25,8 @@ import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeTrendingUrlIdHandler; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.utils.Utils; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java index 434792a44..315bdbfc1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java @@ -25,7 +25,7 @@ import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.UrlIdHandler; -import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeTrendingUrlIdHandler; import static junit.framework.TestCase.assertFalse; import static org.junit.Assert.assertEquals; From 0501a2f543bab38be4a3722e1567d55d72997bb3 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Tue, 8 May 2018 21:58:35 +0200 Subject: [PATCH 6/7] fix donation link parsing --- .../newpipe/extractor/utils/DonationLinkHelper.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java index 049132257..817595ad0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java @@ -15,9 +15,8 @@ public class DonationLinkHelper { AMAZON, } - public static DonationService getDonatoinServiceByLink(String link) throws MalformedURLException { - URL url = new URL(link); + URL url = new URL(fixLink(link)); switch (url.getHost()) { case "www.patreon.com": return DonationService.PATREON; @@ -33,11 +32,17 @@ public class DonationLinkHelper { } public static AffiliateService getAffiliateServiceByLink(String link) throws MalformedURLException { - URL url = new URL(link); + URL url = new URL(fixLink(link)); switch (url.getHost()) { case "amzn.to": return AffiliateService.AMAZON; default: return AffiliateService.NO_AFILIATE; } } + private static String fixLink(String link) { + return (link.startsWith("https://") || link.startsWith("http://")) + ? link + : "https://" + link; + } + } From 083f63c083065e39b046e544483a53827bb6cd19 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Tue, 15 May 2018 22:17:57 +0200 Subject: [PATCH 7/7] add parse linkspan --- .../extractor/stream/StreamExtractor.java | 4 ++ .../newpipe/extractor/stream/StreamInfo.java | 43 +++++++++++++----- .../newpipe/extractor/utils/Parser.java | 44 +++++++++++++------ .../schabi/newpipe/extractor/utils/Utils.java | 6 +-- 4 files changed, 71 insertions(+), 26 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index a03bb6131..1c2e20516 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -20,6 +20,7 @@ package org.schabi.newpipe.extractor.stream; * along with NewPipe. If not, see . */ +import org.nibor.autolink.LinkSpan; import org.schabi.newpipe.extractor.Extractor; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.Subtitles; @@ -143,6 +144,9 @@ public abstract class StreamExtractor extends Extractor { public abstract String[] getDonationLinks() throws ExtractionException; public abstract String[] getAffiliateLinks() throws ExtractionException; + public LinkSpan[] getLinksFromDescription() throws ExtractionException { + return Parser.getLinkSpanFromString(getDescription()); + } /** * Analyses the webpage's document and extracts any error message there might be. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 9443e1ff4..c718a8183 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -1,10 +1,14 @@ package org.schabi.newpipe.extractor.stream; +import org.nibor.autolink.LinkSpan; import org.schabi.newpipe.extractor.*; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.DashMpdParser; +import org.schabi.newpipe.extractor.utils.DonationLinkHelper; import org.schabi.newpipe.extractor.utils.ExtractorHelper; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.util.ArrayList; @@ -62,6 +66,7 @@ public class StreamInfo extends Info { streamInfo = extractImportantData(extractor); streamInfo = extractStreams(streamInfo, extractor); streamInfo = extractOptionalData(streamInfo, extractor); + streamInfo = extractLinks(streamInfo, extractor); } catch (ExtractionException e) { // Currently YouTube does not distinguish between age restricted videos and videos blocked // by country. This means that during the initialisation of the extractor, the extractor @@ -171,6 +176,25 @@ public class StreamInfo extends Info { return streamInfo; } + + private static StreamInfo extractLinks(StreamInfo streamInfo, StreamExtractor extractor) throws ParsingException { + try { + streamInfo.setLinksInDescription(extractor.getLinksFromDescription()); + } catch (Exception e) { + e.printStackTrace(); + } + try { + streamInfo.setAffiliateLinks(extractor.getAffiliateLinks()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setDonationLinks(extractor.getDonationLinks()); + } catch (Exception e) { + streamInfo.addError(e); + } + } + private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtractor extractor) { /* ---- optional data goes here: ---- */ // If one of these fails, the frontend needs to handle that they are not available. @@ -242,16 +266,6 @@ public class StreamInfo extends Info { } catch (Exception e) { streamInfo.addError(e); } - try { - streamInfo.setAffiliateLinks(extractor.getAffiliateLinks()); - } catch (Exception e) { - streamInfo.addError(e); - } - try { - streamInfo.setDonationLinks(extractor.getDonationLinks()); - } catch (Exception e) { - streamInfo.addError(e); - } streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor)); return streamInfo; @@ -284,6 +298,7 @@ public class StreamInfo extends Info { private long startPosition = 0; private List subtitles; + private LinkSpan[] linksInDescription; private String[] donationLinks; private String[] affiliateLinks; @@ -495,4 +510,12 @@ public class StreamInfo extends Info { public void setAffiliateLinks(String[] affiliateLinks) { this.affiliateLinks = affiliateLinks; } + + public LinkSpan[] getLinksInDescription() { + return linksInDescription; + } + + public void setLinksInDescription(LinkSpan[] linksInDescription) { + this.linksInDescription = linksInDescription; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java index 42f098dce..a1e2fb4fa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java @@ -88,20 +88,38 @@ public class Parser { public static String[] getLinksFromString(final String txt) throws ParsingException { try { - ArrayList links = new ArrayList<>(); - LinkExtractor linkExtractor = LinkExtractor.builder() - .linkTypes(EnumSet.of(LinkType.URL, LinkType.WWW)) - .build(); - Iterable linkss = linkExtractor.extractLinks(txt); - for(LinkSpan ls : linkss) { - links.add(txt.substring(ls.getBeginIndex(), ls.getEndIndex())); - } - - String[] linksarray = new String[links.size()]; - linksarray = links.toArray(linksarray); - return linksarray; + return getLinksFromLinkSpan(getLinkSpanFromString(txt), txt); } catch (Exception e) { throw new ParsingException("Could not get links from string", e); } } -} + + public static LinkSpan[] getLinkSpanFromString(final String txt) throws ParsingException { + try { + ArrayList linksSpans = new ArrayList<>(); + LinkExtractor linkExtractor = LinkExtractor.builder() + .linkTypes(EnumSet.of(LinkType.URL, LinkType.WWW)) + .build(); + for (LinkSpan ls : linkExtractor.extractLinks(txt)) { + linksSpans.add(ls); + } + LinkSpan[] linksarray = new LinkSpan[linksSpans.size()]; + return linksSpans.toArray(linksarray); + } catch (Exception e) { + throw new ParsingException("Could not get links from string", e); + } + } + + public static String[] getLinksFromLinkSpan(final LinkSpan[] links, final String txt) throws ParsingException { + try { + ArrayList linksout = new ArrayList<>(); + for (LinkSpan ls : links) { + linksout.add(txt.substring(ls.getBeginIndex(), ls.getEndIndex())); + } + String[] linksarray = new String[linksout.size()]; + return linksout.toArray(linksarray); + } catch (Exception e) { + throw new ParsingException("Could not get LinkSpans from string", e); + } + } +} \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 663fd093b..551c30c76 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -6,6 +6,9 @@ import java.util.List; public class Utils { + private static final String HTTP = "http://"; + private static final String HTTPS = "https://"; + private Utils() { //no instance } @@ -46,9 +49,6 @@ public class Utils { } } - private static final String HTTP = "http://"; - private static final String HTTPS = "https://"; - public static String replaceHttpWithHttps(final String url) { if (url == null) return null;