From 20f280cb5700d025059480d15d91f7b65c27cfec Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Fri, 12 Oct 2018 00:40:22 +0530 Subject: [PATCH] added peertube extractor --- .../schabi/newpipe/extractor/MediaFormat.java | 8 + .../schabi/newpipe/extractor/ServiceList.java | 6 +- .../newpipe/extractor/StreamingService.java | 9 +- .../services/peertube/PeertubeInstance.java | 76 +++++ .../peertube/PeertubeParsingHelper.java | 23 ++ .../services/peertube/PeertubeService.java | 175 ++++++++++++ .../extractors/PeertubeChannelExtractor.java | 181 ++++++++++++ .../extractors/PeertubeCommentsExtractor.java | 124 +++++++++ .../PeertubeCommentsInfoItemExtractor.java | 88 ++++++ .../extractors/PeertubePlaylistExtractor.java | 87 ++++++ .../extractors/PeertubeSearchExtractor.java | 131 +++++++++ .../extractors/PeertubeStreamExtractor.java | 260 ++++++++++++++++++ .../PeertubeStreamInfoItemExtractor.java | 78 ++++++ .../PeertubeSubscriptionExtractor.java | 21 ++ .../PeertubeSuggestionExtractor.java | 23 ++ .../extractors/PeertubeTrendingExtractor.java | 130 +++++++++ .../PeertubeChannelLinkHandlerFactory.java | 35 +++ .../PeertubeCommentsLinkHandlerFactory.java | 41 +++ .../PeertubePlaylistLinkHandlerFactory.java | 37 +++ .../PeertubeSearchQueryHandlerFactory.java | 38 +++ .../PeertubeStreamLinkHandlerFactory.java | 37 +++ .../PeertubeTrendingLinkHandlerFactory.java | 51 ++++ .../soundcloud/SoundcloudService.java | 5 + .../services/youtube/YoutubeService.java | 5 + .../newpipe/extractor/utils/JsonUtils.java | 20 ++ .../PeertubeChannelExtractorTest.java | 207 ++++++++++++++ ...PeertubeChannelLinkHandlerFactoryTest.java | 37 +++ .../PeertubeCommentsExtractorTest.java | 93 +++++++ ...eertubeCommentsLinkHandlerFactoryTest.java | 37 +++ ...eertubePlaylistLinkHandlerFactoryTest.java | 37 +++ .../PeertubeStreamExtractorDefaultTest.java | 113 ++++++++ .../PeertubeStreamLinkHandlerFactoryTest.java | 38 +++ .../PeertubeTrendingExtractorTest.java | 96 +++++++ ...eertubeTrendingLinkHandlerFactoryTest.java | 54 ++++ .../PeertubeSearchExtractorBaseTest.java | 28 ++ .../PeertubeSearchExtractorDefaultTest.java | 91 ++++++ .../peertube/search/PeertubeSearchQHTest.java | 18 ++ 37 files changed, 2536 insertions(+), 2 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java index e39e6977a..c0e44f758 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java @@ -117,6 +117,13 @@ public enum MediaFormat { return null; } + public static MediaFormat getFromSuffix(String suffix) { + for (MediaFormat vf: values()) { + if (vf.suffix.equals(suffix)) return vf; + } + return null; + } + /** * Get the name of the format * @return the name of the format @@ -140,4 +147,5 @@ public enum MediaFormat { public String getMimeType() { return mimeType; } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java index 846355ede..4d3dffc3d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor; +import org.schabi.newpipe.extractor.services.peertube.PeertubeService; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService; import org.schabi.newpipe.extractor.services.youtube.YoutubeService; @@ -18,11 +19,14 @@ public final class ServiceList { public static final YoutubeService YouTube; public static final SoundcloudService SoundCloud; + public static final PeertubeService PeerTube; private static final List SERVICES = unmodifiableList( asList( YouTube = new YoutubeService(0), - SoundCloud = new SoundcloudService(1) + SoundCloud = new SoundcloudService(1), + PeerTube = new PeertubeService(2) + )); /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index bab6c7f4f..418a61e15 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -22,7 +22,8 @@ import org.schabi.newpipe.extractor.utils.Localization; public abstract class StreamingService { public static class ServiceInfo { - private final String name; + private String name; + private final List mediaCapabilities; public ServiceInfo(String name, List mediaCapabilities) { @@ -33,6 +34,10 @@ public abstract class StreamingService { public String getName() { return name; } + + public void setName(String name) { + this.name = name; + } public List getMediaCapabilities() { return mediaCapabilities; @@ -191,6 +196,8 @@ public abstract class StreamingService { } public abstract boolean isCommentsSupported(); + + public abstract String getBaseUrl(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java new file mode 100644 index 000000000..e08477509 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java @@ -0,0 +1,76 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import java.io.IOException; + +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.DownloadResponse; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.utils.JsonUtils; + +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; + +public class PeertubeInstance { + + private final String url; + private String name; + public static final PeertubeInstance defaultInstance = new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"); + + public PeertubeInstance(String url) throws IOException { + this.url = url; + String response = validateInstance(url); + setInstanceMetaData(response); + } + + private PeertubeInstance(String url , String name) { + this.url = url; + this.name = name; + } + + public String getUrl() { + return url; + } + + private String validateInstance(String url) throws IOException { + Downloader downloader = NewPipe.getDownloader(); + DownloadResponse response = null; + + try { + response = downloader.get(url + "/api/v1/config"); + } catch (ReCaptchaException | IOException e) { + throw new IOException("unable to configure instance " + url, e); + } + + if(null == response || StringUtil.isBlank(response.getResponseBody())) { + throw new IOException("unable to configure instance " + url); + } + + return response.getResponseBody(); + } + + private void setInstanceMetaData(String responseBody) { + JsonObject json; + try { + json = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + return; + } + + if(null == json) return; + + try { + this.name = JsonUtils.getString(json, "instance.name"); + } catch (ParsingException e) { + return; + } + } + + public String getName() { + return name; + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java new file mode 100644 index 000000000..b50cede26 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -0,0 +1,23 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +public class PeertubeParsingHelper { + + private PeertubeParsingHelper() { + } + + public static String toDateString(String time) throws ParsingException { + try { + Date date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse(time); + SimpleDateFormat newDateFormat = new SimpleDateFormat("yyyy-MM-dd"); + return newDateFormat.format(date); + } catch (ParseException e) { + throw new ParsingException(e.getMessage(), e); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java new file mode 100644 index 000000000..66f5412cc --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -0,0 +1,175 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static java.util.Collections.singletonList; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; + +import java.io.IOException; +import java.util.List; + +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability; +import org.schabi.newpipe.extractor.SuggestionExtractor; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.comments.CommentsExtractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.kiosk.KioskList; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; +import org.schabi.newpipe.extractor.utils.Localization; + +public class PeertubeService extends StreamingService { + + private PeertubeInstance instance; + + public PeertubeService(int id) { + this(id, PeertubeInstance.defaultInstance); + } + + public PeertubeService(int id, PeertubeInstance instance) { + super(id, instance.getName(), singletonList(VIDEO)); + this.instance = instance; + } + + public PeertubeService(int id, String name, List capabilities) { + super(id, name, capabilities); + } + + @Override + public LinkHandlerFactory getStreamLHFactory() { + return PeertubeStreamLinkHandlerFactory.getInstance(); + } + + @Override + public ListLinkHandlerFactory getChannelLHFactory() { + return PeertubeChannelLinkHandlerFactory.getInstance(); + } + + @Override + public ListLinkHandlerFactory getPlaylistLHFactory() { + // TODO Auto-generated method stub + return null; + } + + @Override + public SearchQueryHandlerFactory getSearchQHFactory() { + return PeertubeSearchQueryHandlerFactory.getInstance(); + } + + @Override + public ListLinkHandlerFactory getCommentsLHFactory() { + return PeertubeCommentsLinkHandlerFactory.getInstance(); + } + + @Override + public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler, Localization localization) { + return new PeertubeSearchExtractor(this, queryHandler, localization); + } + + @Override + public SuggestionExtractor getSuggestionExtractor(Localization localization) { + // TODO Auto-generated method stub + return null; + } + + @Override + public SubscriptionExtractor getSubscriptionExtractor() { + // TODO Auto-generated method stub + return null; + } + + @Override + public KioskList getKioskList(Localization localization) throws ExtractionException { + KioskList.KioskExtractorFactory kioskFactory = new KioskList.KioskExtractorFactory() { + @Override + public KioskExtractor createNewKiosk(StreamingService streamingService, + String url, + String id, + Localization local) + throws ExtractionException { + return new PeertubeTrendingExtractor(PeertubeService.this, + new PeertubeTrendingLinkHandlerFactory().fromId(id), id, local); + } + }; + + KioskList list = new KioskList(getServiceId(), localization); + + // add kiosks here e.g.: + final PeertubeTrendingLinkHandlerFactory h = new PeertubeTrendingLinkHandlerFactory(); + try { + list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING); + list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_RECENT); + list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_LOCAL); + list.setDefaultKiosk(PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING); + } catch (Exception e) { + throw new ExtractionException(e); + } + + return list; + } + + @Override + public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler, Localization localization) + throws ExtractionException { + return new PeertubeChannelExtractor(this, linkHandler, localization); + } + + @Override + public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler, Localization localization) + throws ExtractionException { + // TODO Auto-generated method stub + return null; + } + + @Override + public StreamExtractor getStreamExtractor(LinkHandler linkHandler, Localization localization) + throws ExtractionException { + return new PeertubeStreamExtractor(this, linkHandler, localization); + } + + @Override + public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler, Localization localization) + throws ExtractionException { + return new PeertubeCommentsExtractor(this, linkHandler, localization); + } + + @Override + public boolean isCommentsSupported() { + return true; + } + + @Override + public String getBaseUrl() { + return instance.getUrl(); + } + + public void setInstance(String url) throws IOException { + this.instance = new PeertubeInstance(url); + if(!StringUtil.isBlank(instance.getName())) { + this.getServiceInfo().setName(instance.getName()); + }else { + this.getServiceInfo().setName("PeerTube"); + } + } + + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java new file mode 100644 index 000000000..efab4c63b --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -0,0 +1,181 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.io.IOException; + +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.DownloadResponse; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Localization; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Parser.RegexException; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; + +public class PeertubeChannelExtractor extends ChannelExtractor { + + private static final String START_KEY = "start"; + private static final String COUNT_KEY = "count"; + private static final int ITEMS_PER_PAGE = 12; + private static final String START_PATTERN = "start=(\\d*)"; + + private InfoItemsPage initPage; + private long total; + + private JsonObject json; + + public PeertubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) { + super(service, linkHandler, localization); + } + + @Override + public String getAvatarUrl() throws ParsingException { + String value = JsonUtils.getString(json, "avatar.path"); + return ServiceList.PeerTube.getBaseUrl() + value; + } + + @Override + public String getBannerUrl() throws ParsingException { + return null; + } + + @Override + public String getFeedUrl() throws ParsingException { + return null; + } + + @Override + public long getSubscriberCount() throws ParsingException { + Number number = JsonUtils.getNumber(json, "followersCount"); + return number.longValue(); + } + + @Override + public String getDescription() throws ParsingException { + try { + return JsonUtils.getString(json, "description"); + }catch(ParsingException e) { + return "No description"; + } + } + + @Override + public String[] getDonationLinks() throws ParsingException { + return new String[0]; + } + + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + super.fetchPage(); + return initPage; + } + + private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException { + JsonArray contents; + try { + contents = (JsonArray) JsonUtils.getValue(json, "data"); + }catch(Exception e) { + throw new ParsingException("unable to extract channel streams", e); + } + + for(Object c: contents) { + if(c instanceof JsonObject) { + final JsonObject item = (JsonObject) c; + PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item); + collector.commit(extractor); + } + } + + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + super.fetchPage(); + return initPage.getNextPageUrl(); + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + DownloadResponse response = getDownloader().get(pageUrl); + JsonObject json = null; + if(null != response && !StringUtil.isBlank(response.getResponseBody())) { + try { + json = JsonParser.object().from(response.getResponseBody()); + } catch (Exception e) { + throw new ParsingException("Could not parse json data for kiosk info", e); + } + } + + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + if(json != null) { + Number number = JsonUtils.getNumber(json, "total"); + if(number != null) this.total = number.longValue(); + collectStreamsFrom(collector, json, pageUrl); + } else { + throw new ExtractionException("Unable to get peertube kiosk info"); + } + return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl)); + } + + + private String getNextPageUrl(String prevPageUrl) { + String prevStart; + try { + prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl); + } catch (RegexException e) { + return ""; + } + if(StringUtil.isBlank(prevStart)) return ""; + long nextStart = 0; + try { + nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; + } catch (NumberFormatException e) { + return ""; + } + + if(nextStart >= total) { + return ""; + }else { + return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); + } + } + + @Override + public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { + DownloadResponse response = downloader.get(getUrl()); + if(null != response && null != response.getResponseBody()) { + setInitialData(response.getResponseBody()); + }else { + throw new ExtractionException("Unable to extract peertube channel data"); + } + + String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; + this.initPage = getPage(pageUrl); + } + + private void setInitialData(String responseBody) throws ExtractionException { + try { + json = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ExtractionException("Unable to extract peertube channel data", e); + } + if(null == json) throw new ExtractionException("Unable to extract peertube channel data"); + } + + @Override + public String getName() throws ParsingException { + return JsonUtils.getString(json, "displayName"); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java new file mode 100644 index 000000000..fde931fa4 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java @@ -0,0 +1,124 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.io.IOException; + +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.DownloadResponse; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.comments.CommentsExtractor; +import org.schabi.newpipe.extractor.comments.CommentsInfoItem; +import org.schabi.newpipe.extractor.comments.CommentsInfoItemsCollector; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Localization; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Parser.RegexException; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; + +public class PeertubeCommentsExtractor extends CommentsExtractor { + + private static final String START_KEY = "start"; + private static final String COUNT_KEY = "count"; + private static final int ITEMS_PER_PAGE = 12; + private static final String START_PATTERN = "start=(\\d*)"; + + private InfoItemsPage initPage; + private long total; + + public PeertubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler, Localization localization) { + super(service, uiHandler, localization); + } + + @Override + public String getName() throws ParsingException { + return "Comments"; + } + + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + super.fetchPage(); + return initPage; + } + + private void collectStreamsFrom(CommentsInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException { + JsonArray contents; + try { + contents = (JsonArray) JsonUtils.getValue(json, "data"); + }catch(Exception e) { + throw new ParsingException("unable to extract comments info", e); + } + + for(Object c: contents) { + if(c instanceof JsonObject) { + final JsonObject item = (JsonObject) c; + PeertubeCommentsInfoItemExtractor extractor = new PeertubeCommentsInfoItemExtractor(item, pageUrl); + collector.commit(extractor); + } + } + + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + super.fetchPage(); + return initPage.getNextPageUrl(); + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + DownloadResponse response = getDownloader().get(pageUrl); + JsonObject json = null; + if(null != response && !StringUtil.isBlank(response.getResponseBody())) { + try { + json = JsonParser.object().from(response.getResponseBody()); + } catch (Exception e) { + throw new ParsingException("Could not parse json data for comments info", e); + } + } + + CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); + if(json != null) { + Number number = JsonUtils.getNumber(json, "total"); + if(number != null) this.total = number.longValue(); + collectStreamsFrom(collector, json, pageUrl); + } else { + throw new ExtractionException("Unable to get peertube comments info"); + } + return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl)); + } + + @Override + public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { + String pageUrl = getUrl() + "?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; + this.initPage = getPage(pageUrl); + } + + private String getNextPageUrl(String prevPageUrl) { + String prevStart; + try { + prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl); + } catch (RegexException e) { + return ""; + } + if(StringUtil.isBlank(prevStart)) return ""; + long nextStart = 0; + try { + nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; + } catch (NumberFormatException e) { + return ""; + } + + if(nextStart >= total) { + return ""; + }else { + return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); + } + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java new file mode 100644 index 000000000..ea65aae3c --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -0,0 +1,88 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.JsonUtils; + +import com.grack.nanojson.JsonObject; + + +public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor { + + private final JsonObject item; + private final String url; + + public PeertubeCommentsInfoItemExtractor(JsonObject item, String url) { + this.item = item; + this.url = url; + } + + @Override + public String getUrl() throws ParsingException { + return url; + } + + @Override + public String getThumbnailUrl() throws ParsingException { + String value; + try { + value = JsonUtils.getString(item, "account.avatar.path"); + }catch(Exception e) { + value = "/client/assets/images/default-avatar.png"; + } + return ServiceList.PeerTube.getBaseUrl() + value; + } + + @Override + public String getName() throws ParsingException { + return JsonUtils.getString(item, "account.displayName"); + } + + @Override + public String getPublishedTime() throws ParsingException { + return JsonUtils.getString(item, "createdAt"); + } + + @Override + public Integer getLikeCount() throws ParsingException { + return 0; + } + + @Override + public String getCommentText() throws ParsingException { + String htmlText = JsonUtils.getString(item, "text"); + return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", ""); + } + + @Override + public String getCommentId() throws ParsingException { + Number value = JsonUtils.getNumber(item, "id"); + return value.toString(); + } + + @Override + public String getAuthorThumbnail() throws ParsingException { + String value; + try { + value = JsonUtils.getString(item, "account.avatar.path"); + }catch(Exception e) { + value = "/client/assets/images/default-avatar.png"; + } + return ServiceList.PeerTube.getBaseUrl() + value; + } + + @Override + public String getAuthorName() throws ParsingException { + return JsonUtils.getString(item, "account.displayName"); + } + + @Override + public String getAuthorEndpoint() throws ParsingException { + String name = JsonUtils.getString(item, "account.name"); + String host = JsonUtils.getString(item, "account.host"); + return PeertubeChannelLinkHandlerFactory.getInstance().fromId(name + "@" + host).getUrl(); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java new file mode 100644 index 000000000..a3ef309e4 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java @@ -0,0 +1,87 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.io.IOException; + +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.utils.Localization; + +public class PeertubePlaylistExtractor extends PlaylistExtractor{ + + public PeertubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) { + super(service, linkHandler, localization); + // TODO Auto-generated constructor stub + } + + @Override + public String getThumbnailUrl() throws ParsingException { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getBannerUrl() throws ParsingException { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getUploaderUrl() throws ParsingException { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getUploaderName() throws ParsingException { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getUploaderAvatarUrl() throws ParsingException { + // TODO Auto-generated method stub + return null; + } + + @Override + public long getStreamCount() throws ParsingException { + // TODO Auto-generated method stub + return 0; + } + + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + // TODO Auto-generated method stub + return null; + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + // TODO Auto-generated method stub + return null; + } + + @Override + public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { + // TODO Auto-generated method stub + + } + + @Override + public String getName() throws ParsingException { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java new file mode 100644 index 000000000..61c83a2b2 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -0,0 +1,131 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.io.IOException; + +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.DownloadResponse; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.InfoItemExtractor; +import org.schabi.newpipe.extractor.InfoItemsCollector; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Localization; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Parser.RegexException; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; + +public class PeertubeSearchExtractor extends SearchExtractor { + + private static final String START_KEY = "start"; + private static final String COUNT_KEY = "count"; + private static final int ITEMS_PER_PAGE = 12; + private static final String START_PATTERN = "start=(\\d*)"; + + private InfoItemsPage initPage; + private long total; + + public PeertubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler, + Localization localization) { + super(service, linkHandler, localization); + } + + @Override + public String getSearchSuggestion() throws ParsingException { + return null; + } + + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + super.fetchPage(); + return initPage; + } + + private InfoItemsCollector collectStreamsFrom(JsonObject json) throws ParsingException { + + final InfoItemsSearchCollector collector = getInfoItemSearchCollector(); + + JsonArray contents; + try { + contents = (JsonArray) JsonUtils.getValue(json, "data"); + }catch(Exception e) { + throw new ParsingException("unable to extract search info", e); + } + + for(Object c: contents) { + if(c instanceof JsonObject) { + final JsonObject item = (JsonObject) c; + PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item); + collector.commit(extractor); + } + } + + return collector; + + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + super.fetchPage(); + return initPage.getNextPageUrl(); + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + DownloadResponse response = getDownloader().get(pageUrl); + JsonObject json = null; + if(null != response && !StringUtil.isBlank(response.getResponseBody())) { + try { + json = JsonParser.object().from(response.getResponseBody()); + } catch (Exception e) { + throw new ParsingException("Could not parse json data for search info", e); + } + } + + if(json != null) { + Number number = JsonUtils.getNumber(json, "total"); + if(number != null) this.total = number.longValue(); + return new InfoItemsPage<>(collectStreamsFrom(json), getNextPageUrl(pageUrl)); + } else { + throw new ExtractionException("Unable to get peertube search info"); + } + } + + @Override + public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { + String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; + this.initPage = getPage(pageUrl); + } + + private String getNextPageUrl(String prevPageUrl) { + String prevStart; + try { + prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl); + } catch (RegexException e) { + return ""; + } + if(StringUtil.isBlank(prevStart)) return ""; + long nextStart = 0; + try { + nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; + } catch (NumberFormatException e) { + return ""; + } + + if(nextStart >= total) { + return ""; + }else { + return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); + } + } + + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java new file mode 100644 index 000000000..c35d7654e --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -0,0 +1,260 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.DownloadResponse; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.Subtitles; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.ItagItem; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.Stream; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.SubtitlesFormat; +import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Localization; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; + +public class PeertubeStreamExtractor extends StreamExtractor { + + + private JsonObject json; + + public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler, Localization localization) { + super(service, linkHandler, localization); + } + + @Override + public String getUploadDate() throws ParsingException { + String date = JsonUtils.getString(json, "publishedAt"); + return PeertubeParsingHelper.toDateString(date); + } + + @Override + public String getThumbnailUrl() throws ParsingException { + return ServiceList.PeerTube.getBaseUrl() + JsonUtils.getString(json, "thumbnailPath"); + } + + @Override + public String getDescription() throws ParsingException { + return JsonUtils.getString(json, "description"); + } + + @Override + public int getAgeLimit() throws ParsingException { + return NO_AGE_LIMIT; + } + + @Override + public long getLength() throws ParsingException { + Number value = JsonUtils.getNumber(json, "duration"); + return value.longValue(); + } + + @Override + public long getTimeStamp() throws ParsingException { + //TODO fetch timestamp from url if present; + return 0; + } + + @Override + public long getViewCount() throws ParsingException { + Number value = JsonUtils.getNumber(json, "views"); + return value.longValue(); + } + + @Override + public long getLikeCount() throws ParsingException { + Number value = JsonUtils.getNumber(json, "likes"); + return value.longValue(); + } + + @Override + public long getDislikeCount() throws ParsingException { + Number value = JsonUtils.getNumber(json, "dislikes"); + return value.longValue(); + } + + @Override + public String getUploaderUrl() throws ParsingException { + String name = JsonUtils.getString(json, "account.name"); + String host = JsonUtils.getString(json, "account.host"); + return PeertubeChannelLinkHandlerFactory.getInstance().fromId(name + "@" + host).getUrl(); + } + + @Override + public String getUploaderName() throws ParsingException { + return JsonUtils.getString(json, "account.displayName"); + } + + @Override + public String getUploaderAvatarUrl() throws ParsingException { + String avatarPath = JsonUtils.getString(json, "account.avatar.path"); + return ServiceList.PeerTube.getBaseUrl() + avatarPath; + } + + @Override + public String getDashMpdUrl() throws ParsingException { + return ""; + } + + @Override + public String getHlsUrl() throws ParsingException { + return ""; + } + + @Override + public List getAudioStreams() throws IOException, ExtractionException { + return null; + } + + @Override + public List getVideoStreams() throws IOException, ExtractionException { + assertPageFetched(); + List videoStreams = new ArrayList<>(); + try { + JsonArray streams = json.getArray("files", new JsonArray()); + for(Object s: streams) { + if(!(s instanceof JsonObject)) continue; + JsonObject stream = (JsonObject) s; + String url = JsonUtils.getString(stream, "fileUrl"); + String resolution = JsonUtils.getString(stream, "resolution.label"); + String extension = url.substring(url.lastIndexOf(".") + 1); + MediaFormat format = MediaFormat.getFromSuffix(extension); + VideoStream videoStream = new VideoStream(url, format, resolution); + if (!Stream.containSimilarStream(videoStream, videoStreams)) { + videoStreams.add(videoStream); + } + } + } catch (Exception e) { + throw new ParsingException("Could not get video streams", e); + } + + return videoStreams; + } + + + @Override + public List getVideoOnlyStreams() throws IOException, ExtractionException { + // TODO Auto-generated method stub + return null; + } + + @Override + public List getSubtitlesDefault() throws IOException, ExtractionException { + return Collections.emptyList(); + } + + @Override + public List getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException { + return Collections.emptyList(); + } + + @Override + public StreamType getStreamType() throws ParsingException { + return StreamType.VIDEO_STREAM; + } + + @Override + public StreamInfoItem getNextVideo() throws IOException, ExtractionException { + return null; + } + + @Override + public StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException { + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + + //TODO fetch related videos not trending + String apiUrl = new PeertubeTrendingLinkHandlerFactory().getUrl(PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING); + getStreamsFromApi(collector, apiUrl); + return collector; + } + + private void getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, IOException, ParsingException { + DownloadResponse response = getDownloader().get(apiUrl); + JsonObject relatedVideosJson = null; + if(null != response && !StringUtil.isBlank(response.getResponseBody())) { + try { + relatedVideosJson = JsonParser.object().from(response.getResponseBody()); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json data for related videos", e); + } + } + + if(relatedVideosJson != null) { + collectStreamsFrom(collector, relatedVideosJson); + } + } + + private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json) throws ParsingException { + JsonArray contents; + try { + contents = (JsonArray) JsonUtils.getValue(json, "data"); + }catch(Exception e) { + throw new ParsingException("unable to extract related videos", e); + } + + for(Object c: contents) { + if(c instanceof JsonObject) { + final JsonObject item = (JsonObject) c; + PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item); + collector.commit(extractor); + } + } + + } + + + @Override + public String getErrorMessage() { + return null; + } + + @Override + public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { + DownloadResponse response = downloader.get(getUrl()); + if(null != response && null != response.getResponseBody()) { + setInitialData(response.getResponseBody()); + }else { + throw new ExtractionException("Unable to extract peertube channel data"); + } + } + + private void setInitialData(String responseBody) throws ExtractionException { + try { + json = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ExtractionException("Unable to extract peertube stream data", e); + } + if(null == json) throw new ExtractionException("Unable to extract peertube stream data"); + } + + @Override + public String getName() throws ParsingException { + return JsonUtils.getString(json, "name"); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java new file mode 100644 index 000000000..b73fdc609 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java @@ -0,0 +1,78 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.utils.JsonUtils; + +import com.grack.nanojson.JsonObject; + +public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor { + + protected final JsonObject item; + + public PeertubeStreamInfoItemExtractor(JsonObject item) { + this.item = item; + } + + @Override + public String getUrl() throws ParsingException { + String uuid = JsonUtils.getString(item, "uuid"); + return PeertubeStreamLinkHandlerFactory.getInstance().fromId(uuid).getUrl(); + } + + @Override + public String getThumbnailUrl() throws ParsingException { + String value = JsonUtils.getString(item, "thumbnailPath"); + return ServiceList.PeerTube.getBaseUrl() + value; + } + + @Override + public String getName() throws ParsingException { + return JsonUtils.getString(item, "name"); + } + + @Override + public boolean isAd() throws ParsingException { + return false; + } + + @Override + public long getViewCount() throws ParsingException { + Number value = JsonUtils.getNumber(item, "views"); + return value.longValue(); + } + + @Override + public String getUploaderUrl() throws ParsingException { + String name = JsonUtils.getString(item, "account.name"); + String host = JsonUtils.getString(item, "account.host"); + return PeertubeChannelLinkHandlerFactory.getInstance().fromId(name + "@" + host).getUrl(); + } + + @Override + public String getUploaderName() throws ParsingException { + return JsonUtils.getString(item, "account.displayName"); + } + + @Override + public String getUploadDate() throws ParsingException { + String date = JsonUtils.getString(item, "publishedAt"); + return PeertubeParsingHelper.toDateString(date); + } + + @Override + public StreamType getStreamType() throws ParsingException { + return StreamType.VIDEO_STREAM; + } + + @Override + public long getDuration() throws ParsingException { + Number value = JsonUtils.getNumber(item, "duration"); + return value.longValue(); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java new file mode 100644 index 000000000..a2b246141 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java @@ -0,0 +1,21 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.util.List; + +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; + +public class PeertubeSubscriptionExtractor extends SubscriptionExtractor { + + public PeertubeSubscriptionExtractor(StreamingService service, List supportedSources) { + super(service, supportedSources); + // TODO Auto-generated constructor stub + } + + @Override + public String getRelatedUrl() { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java new file mode 100644 index 000000000..ec36fc83f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java @@ -0,0 +1,23 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.io.IOException; +import java.util.List; + +import org.schabi.newpipe.extractor.SuggestionExtractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.utils.Localization; + +public class PeertubeSuggestionExtractor extends SuggestionExtractor{ + + public PeertubeSuggestionExtractor(int serviceId, Localization localization) { + super(serviceId, localization); + // TODO Auto-generated constructor stub + } + + @Override + public List suggestionList(String query) throws IOException, ExtractionException { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java new file mode 100644 index 000000000..5aa5c0323 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java @@ -0,0 +1,130 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.io.IOException; + +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.DownloadResponse; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Localization; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Parser.RegexException; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; + +public class PeertubeTrendingExtractor extends KioskExtractor { + + private static final String START_KEY = "start"; + private static final String COUNT_KEY = "count"; + private static final int ITEMS_PER_PAGE = 12; + private static final String START_PATTERN = "start=(\\d*)"; + + private InfoItemsPage initPage; + private long total; + + public PeertubeTrendingExtractor(StreamingService streamingService, ListLinkHandler linkHandler, String kioskId, + Localization localization) { + super(streamingService, linkHandler, kioskId, localization); + } + + @Override + public String getName() throws ParsingException { + return getId(); + } + + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + super.fetchPage(); + return initPage; + } + + private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException { + JsonArray contents; + try { + contents = (JsonArray) JsonUtils.getValue(json, "data"); + }catch(Exception e) { + throw new ParsingException("unable to extract kiosk info", e); + } + + for(Object c: contents) { + if(c instanceof JsonObject) { + final JsonObject item = (JsonObject) c; + PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item); + collector.commit(extractor); + } + } + + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + super.fetchPage(); + return initPage.getNextPageUrl(); + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + DownloadResponse response = getDownloader().get(pageUrl); + JsonObject json = null; + if(null != response && !StringUtil.isBlank(response.getResponseBody())) { + try { + json = JsonParser.object().from(response.getResponseBody()); + } catch (Exception e) { + throw new ParsingException("Could not parse json data for kiosk info", e); + } + } + + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + if(json != null) { + Number number = JsonUtils.getNumber(json, "total"); + if(number != null) this.total = number.longValue(); + collectStreamsFrom(collector, json, pageUrl); + } else { + throw new ExtractionException("Unable to get peertube kiosk info"); + } + return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl)); + } + + @Override + public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { + String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; + this.initPage = getPage(pageUrl); + } + + private String getNextPageUrl(String prevPageUrl) { + String prevStart; + try { + prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl); + } catch (RegexException e) { + return ""; + } + if(StringUtil.isBlank(prevStart)) return ""; + long nextStart = 0; + try { + nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; + } catch (NumberFormatException e) { + return ""; + } + + if(nextStart >= total) { + return ""; + }else { + return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); + } + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java new file mode 100644 index 000000000..2d93b9f58 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java @@ -0,0 +1,35 @@ +package org.schabi.newpipe.extractor.services.peertube.linkHandler; + +import java.util.List; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Parser; + +public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { + + private static final PeertubeChannelLinkHandlerFactory instance = new PeertubeChannelLinkHandlerFactory(); + private static final String ID_PATTERN = "/accounts/([^/?&#]*)"; + private static final String ACCOUNTS_ENDPOINT = "/api/v1/accounts/"; + + public static PeertubeChannelLinkHandlerFactory getInstance() { + return instance; + } + + @Override + public String getId(String url) throws ParsingException { + return Parser.matchGroup1(ID_PATTERN, url); + } + + @Override + public String getUrl(String id, List contentFilters, String searchFilter) { + String baseUrl = ServiceList.PeerTube.getBaseUrl(); + return baseUrl + ACCOUNTS_ENDPOINT + id; + } + + @Override + public boolean onAcceptUrl(String url) { + return url.contains("/accounts/"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java new file mode 100644 index 000000000..dcd0c3256 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java @@ -0,0 +1,41 @@ +package org.schabi.newpipe.extractor.services.peertube.linkHandler; + +import java.util.List; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.FoundAdException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Parser; + +public class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory { + + private static final PeertubeCommentsLinkHandlerFactory instance = new PeertubeCommentsLinkHandlerFactory(); + private static final String ID_PATTERN = "/videos/(watch/)?([^/?&#]*)"; + private static final String COMMENTS_ENDPOINT = "/api/v1/videos/%s/comment-threads"; + + public static PeertubeCommentsLinkHandlerFactory getInstance() { + return instance; + } + + @Override + public String getUrl(String id) { + String baseUrl = ServiceList.PeerTube.getBaseUrl(); + return baseUrl + String.format(COMMENTS_ENDPOINT, id); + } + + @Override + public String getId(String url) throws ParsingException, IllegalArgumentException { + return Parser.matchGroup(ID_PATTERN, url, 2); + } + + @Override + public boolean onAcceptUrl(final String url) throws FoundAdException { + return url.contains("/videos/"); + } + + @Override + public String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException { + return getUrl(id); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java new file mode 100644 index 000000000..fa6413589 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java @@ -0,0 +1,37 @@ +package org.schabi.newpipe.extractor.services.peertube.linkHandler; + + +import java.util.List; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Parser; + +public class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { + + private static final PeertubePlaylistLinkHandlerFactory instance = new PeertubePlaylistLinkHandlerFactory(); + private static final String ID_PATTERN = "/video-channels/([^/?&#]*)"; + private static final String VIDEO_CHANNELS_ENDPOINT = "/api/v1/video-channels/"; + + public static PeertubePlaylistLinkHandlerFactory getInstance() { + return instance; + } + + @Override + public String getUrl(String id, List contentFilters, String sortFilter) { + String baseUrl = ServiceList.PeerTube.getBaseUrl(); + return baseUrl + VIDEO_CHANNELS_ENDPOINT + id; + } + + @Override + public String getId(String url) throws ParsingException { + return Parser.matchGroup1(ID_PATTERN, url); + } + + + @Override + public boolean onAcceptUrl(final String url) { + return url.contains("/video-channels/"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java new file mode 100644 index 000000000..536d1bf42 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java @@ -0,0 +1,38 @@ +package org.schabi.newpipe.extractor.services.peertube.linkHandler; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.List; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; + +public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory { + + public static final String CHARSET_UTF_8 = "UTF-8"; + public static final String VIDEOS = "videos"; + private static final String SEARCH_ENDPOINT = "/api/v1/search/videos"; + + public static PeertubeSearchQueryHandlerFactory getInstance() { + return new PeertubeSearchQueryHandlerFactory(); + } + + @Override + public String getUrl(String searchString, List contentFilters, String sortFilter) throws ParsingException { + String baseUrl = ServiceList.PeerTube.getBaseUrl(); + try { + final String url = baseUrl + SEARCH_ENDPOINT + + "?search=" + URLEncoder.encode(searchString, CHARSET_UTF_8); + + return url; + } catch (UnsupportedEncodingException e) { + throw new ParsingException("Could not encode query", e); + } + } + + @Override + public String[] getAvailableContentFilter() { + return new String[] { VIDEOS }; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java new file mode 100644 index 000000000..880c4d937 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java @@ -0,0 +1,37 @@ +package org.schabi.newpipe.extractor.services.peertube.linkHandler; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.FoundAdException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Parser; + +public class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory { + + private static final PeertubeStreamLinkHandlerFactory instance = new PeertubeStreamLinkHandlerFactory(); + private static final String ID_PATTERN = "/videos/(watch/)?([^/?&#]*)"; + private static final String VIDEO_ENDPOINT = "/api/v1/videos/"; + + private PeertubeStreamLinkHandlerFactory() { + } + + public static PeertubeStreamLinkHandlerFactory getInstance() { + return instance; + } + + @Override + public String getUrl(String id) { + String baseUrl = ServiceList.PeerTube.getBaseUrl(); + return baseUrl + VIDEO_ENDPOINT + id; + } + + @Override + public String getId(String url) throws ParsingException, IllegalArgumentException { + return Parser.matchGroup(ID_PATTERN, url, 2); + } + + @Override + public boolean onAcceptUrl(final String url) throws FoundAdException { + return url.contains("/videos/"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java new file mode 100644 index 000000000..c4358998c --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java @@ -0,0 +1,51 @@ +package org.schabi.newpipe.extractor.services.peertube.linkHandler; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; + +public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { + + + public static final Map KIOSK_MAP; + public static final String KIOSK_TRENDING = "Trending"; + public static final String KIOSK_RECENT = "Recently added"; + public static final String KIOSK_LOCAL = "Local"; + + static { + Map map = new HashMap<>(); + map.put(KIOSK_TRENDING, "%s/api/v1/videos?sort=-views"); + map.put(KIOSK_RECENT, "%s/api/v1/videos?sort=-publishedAt"); + map.put(KIOSK_LOCAL, "%s/api/v1/videos?filter=local"); + KIOSK_MAP = Collections.unmodifiableMap(map); + } + + public String getUrl(String id, List contentFilters, String sortFilter) { + String baseUrl = ServiceList.PeerTube.getBaseUrl(); + return String.format(KIOSK_MAP.get(id), baseUrl); + } + + @Override + public String getId(String url) throws ParsingException { + + if(url.contains("/videos/trending")) { + return KIOSK_TRENDING; + }else if(url.contains("/videos/recently-added")) { + return KIOSK_RECENT; + }else if(url.contains("/videos/local")){ + return KIOSK_LOCAL; + }else { + throw new ParsingException("no id found for this url"); + } + } + + @Override + public boolean onAcceptUrl(final String url) { + return url.contains("/videos/trending") || url.contains("/videos/recently-added") || url.contains("/videos/local"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index a0008f765..fe63a966a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -117,5 +117,10 @@ public class SoundcloudService extends StreamingService { public boolean isCommentsSupported() { return false; } + + @Override + public String getBaseUrl() { + return "https://soundcloud.com"; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index a505d60d2..64a4565f9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -155,4 +155,9 @@ public class YoutubeService extends StreamingService { return true; } + @Override + public String getBaseUrl() { + return "https://youtube.com"; + } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java index ddbf47c81..2844e6d2c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -28,6 +28,26 @@ public class JsonUtils { return result; } + @Nonnull + public static String getString(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + Object value = getValue(object, path); + if(value instanceof String) { + return (String) getValue(object, path); + }else { + throw new ParsingException("Unable to get " + path); + } + } + + @Nonnull + public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + Object value = getValue(object, path); + if(value instanceof Number) { + return (Number) getValue(object, path); + }else { + throw new ParsingException("Unable to get " + path); + } + } + @Nonnull public static List getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java new file mode 100644 index 000000000..573a51455 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -0,0 +1,207 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; + +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor; +import org.schabi.newpipe.extractor.utils.Localization; + +/** + * Test for {@link PeertubeChannelExtractor} + */ +public class PeertubeChannelExtractorTest { + public static class LilUzi implements BaseChannelExtractorTest { + private static PeertubeChannelExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + extractor = (PeertubeChannelExtractor) PeerTube + .getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/root@tube.openalgeria.org"); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(PeerTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() throws ParsingException { + assertEquals("Noureddine HADDAG", extractor.getName()); + } + + @Test + public void testId() throws ParsingException { + assertEquals("root@tube.openalgeria.org", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/api/v1/accounts/root@tube.openalgeria.org", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/api/v1/accounts/root@tube.openalgeria.org", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor, PeerTube.getServiceId()); + } + + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor, PeerTube.getServiceId()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ChannelExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testDescription() throws ParsingException { + assertNotNull(extractor.getDescription()); + } + + @Test + public void testAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getAvatarUrl()); + } + + @Ignore + @Test + public void testBannerUrl() throws ParsingException { + assertIsSecureUrl(extractor.getBannerUrl()); + } + + @Test + public void testFeedUrl() throws ParsingException { + assertEmpty(extractor.getFeedUrl()); + } + + @Test + public void testSubscriberCount() throws ParsingException { + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5); + } + } + + public static class DubMatix implements BaseChannelExtractorTest { + private static PeertubeChannelExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + extractor = (PeertubeChannelExtractor) PeerTube + .getChannelExtractor("https://peertube.mastodon.host/accounts/franceinter@tube.kdy.ch"); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Additional Testing + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testGetPageInNewExtractor() throws Exception { + final ChannelExtractor newExtractor = PeerTube.getChannelExtractor(extractor.getUrl()); + defaultTestGetPageInNewExtractor(extractor, newExtractor, PeerTube.getServiceId()); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(PeerTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() throws ParsingException { + assertEquals("France Inter", extractor.getName()); + } + + @Test + public void testId() throws ParsingException { + assertEquals("franceinter@tube.kdy.ch", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/api/v1/accounts/franceinter@tube.kdy.ch", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/accounts/franceinter@tube.kdy.ch", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor, PeerTube.getServiceId()); + } + + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor, PeerTube.getServiceId()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ChannelExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testDescription() throws ParsingException { + assertNotNull(extractor.getDescription()); + } + + @Test + public void testAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getAvatarUrl()); + } + + @Ignore + @Test + public void testBannerUrl() throws ParsingException { + assertIsSecureUrl(extractor.getBannerUrl()); + } + + @Test + public void testFeedUrl() throws ParsingException { + assertEmpty(extractor.getFeedUrl()); + } + + @Test + public void testSubscriberCount() throws ParsingException { + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 75); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java new file mode 100644 index 000000000..d50e5774c --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java @@ -0,0 +1,37 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Localization; + +/** + * Test for {@link PeertubeChannelLinkHandlerFactory} + */ +public class PeertubeChannelLinkHandlerFactoryTest { + + private static PeertubeChannelLinkHandlerFactory linkHandler; + + @BeforeClass + public static void setUp() { + linkHandler = PeertubeChannelLinkHandlerFactory.getInstance(); + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + } + + @Test + public void acceptrUrlTest() throws ParsingException { + assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net")); + } + + @Test + public void getIdFromUrl() throws ParsingException { + assertEquals("kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net").getId()); + assertEquals("kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java new file mode 100644 index 000000000..0074ea6e1 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java @@ -0,0 +1,93 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + +import java.io.IOException; +import java.util.List; + +import org.jsoup.helper.StringUtil; +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.comments.CommentsInfo; +import org.schabi.newpipe.extractor.comments.CommentsInfoItem; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor; +import org.schabi.newpipe.extractor.utils.Localization; + +public class PeertubeCommentsExtractorTest { + + private static PeertubeCommentsExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + extractor = (PeertubeCommentsExtractor) PeerTube + .getCommentsExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a"); + } + + @Test + public void testGetComments() throws IOException, ExtractionException { + boolean result = false; + InfoItemsPage comments = extractor.getInitialPage(); + result = findInComments(comments, "@root A great documentary on a great guy."); + + while (comments.hasNextPage() && !result) { + comments = extractor.getPage(comments.getNextPageUrl()); + result = findInComments(comments, "@root A great documentary on a great guy."); + } + + assertTrue(result); + } + + @Test + public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { + boolean result = false; + CommentsInfo commentsInfo = CommentsInfo.getInfo("https://peertube.mastodon.host/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828"); + assertTrue("Comments".equals(commentsInfo.getName())); + result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!"); + + String nextPage = commentsInfo.getNextPageUrl(); + while (!StringUtil.isBlank(nextPage) && !result) { + InfoItemsPage moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage); + result = findInComments(moreItems.getItems(), "Loved it!!!"); + nextPage = moreItems.getNextPageUrl(); + } + + assertTrue(result); + } + + @Test + public void testGetCommentsAllData() throws IOException, ExtractionException { + InfoItemsPage comments = extractor.getInitialPage(); + for(CommentsInfoItem c: comments.getItems()) { + assertFalse(StringUtil.isBlank(c.getAuthorEndpoint())); + assertFalse(StringUtil.isBlank(c.getAuthorName())); + assertFalse(StringUtil.isBlank(c.getAuthorThumbnail())); + assertFalse(StringUtil.isBlank(c.getCommentId())); + assertFalse(StringUtil.isBlank(c.getCommentText())); + assertFalse(StringUtil.isBlank(c.getName())); + assertFalse(StringUtil.isBlank(c.getPublishedTime())); + assertFalse(StringUtil.isBlank(c.getThumbnailUrl())); + assertFalse(StringUtil.isBlank(c.getUrl())); + assertFalse(c.getLikeCount() == null); + } + } + + private boolean findInComments(InfoItemsPage comments, String comment) { + return findInComments(comments.getItems(), comment); + } + + private boolean findInComments(List comments, String comment) { + for(CommentsInfoItem c: comments) { + if(c.getCommentText().contains(comment)) { + return true; + } + } + return false; + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java new file mode 100644 index 000000000..ce4fdd14a --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java @@ -0,0 +1,37 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Localization; + +/** + * Test for {@link PeertubeCommentsLinkHandlerFactory} + */ +public class PeertubeCommentsLinkHandlerFactoryTest { + + private static PeertubeCommentsLinkHandlerFactory linkHandler; + + @BeforeClass + public static void setUp() { + linkHandler = PeertubeCommentsLinkHandlerFactory.getInstance(); + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + } + + @Test + public void acceptrUrlTest() throws ParsingException { + assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/api/v1/videos/19319/comment-threads?start=0&count=10&sort=-createdAt")); + } + + @Test + public void getIdFromUrl() throws ParsingException { + assertEquals("19319", linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/videos/19319/comment-threads").getId()); + assertEquals("19319", linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/videos/19319/comment-threads?start=0&count=10&sort=-createdAt").getId()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java new file mode 100644 index 000000000..9705efc63 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java @@ -0,0 +1,37 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubePlaylistLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Localization; + +/** + * Test for {@link PeertubePlaylistLinkHandlerFactory} + */ +public class PeertubePlaylistLinkHandlerFactoryTest { + + private static PeertubePlaylistLinkHandlerFactory linkHandler; + + @BeforeClass + public static void setUp() { + linkHandler = PeertubePlaylistLinkHandlerFactory.getInstance(); + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + } + + @Test + public void acceptrUrlTest() throws ParsingException { + assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/video-channels/b45e84fb-c47f-475b-94f2-718126154d33/videos")); + } + + @Test + public void getIdFromUrl() throws ParsingException { + assertEquals("b45e84fb-c47f-475b-94f2-718126154d33", linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/b45e84fb-c47f-475b-94f2-718126154d33").getId()); + assertEquals("b45e84fb-c47f-475b-94f2-718126154d33", linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/b45e84fb-c47f-475b-94f2-718126154d33/videos").getId()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java new file mode 100644 index 000000000..d6e5494be --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -0,0 +1,113 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.utils.Localization; + +import java.io.IOException; + +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + +/** + * Test for {@link StreamExtractor} + */ +public class PeertubeStreamExtractorDefaultTest { + private static PeertubeStreamExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/86fe4f24-64c3-4ab4-9e7e-66177219ed21"); + extractor.fetchPage(); + } + + @Test + public void testGetInvalidTimeStamp() throws ParsingException { + assertTrue(extractor.getTimeStamp() + "", + extractor.getTimeStamp() <= 0); + } + + @Test + public void testGetTitle() throws ParsingException { + assertEquals(extractor.getName(), "Pažadėtoji 1 Sezonas 1050 Serija - Ziuri.me"); + } + + @Test + public void testGetDescription() throws ParsingException { + assertEquals(extractor.getDescription(), "Serialo veiksmas vyksta Radžastane. kur vis dar gyvos liaudies tradicijos. o jo centre - Anandi gimusi varguolių šeimoje. Mergaitė privalės ištekėti už turtingo paveldėtojo vos tik sulaukusi aštuonerių metų ir priprasti prie naujojo nuotakos vaidm..."); + } + + @Test + public void testGetUploaderName() throws ParsingException { + assertEquals(extractor.getUploaderName(), "djsets"); + } + + @Test + public void testGetLength() throws ParsingException { + assertEquals(extractor.getLength(), 1238); + } + + @Test + public void testGetViewCount() throws ParsingException { + assertTrue(Long.toString(extractor.getViewCount()), + extractor.getViewCount() > 700); + } + + @Test + public void testGetUploadDate() throws ParsingException { + assertEquals("2018-10-06", extractor.getUploadDate()); + } + + @Test + public void testGetUploaderUrl() throws ParsingException { + assertIsSecureUrl(extractor.getUploaderUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/accounts/djsets@hostyour.tv", extractor.getUploaderUrl()); + } + + @Test + public void testGetThumbnailUrl() throws ParsingException { + assertIsSecureUrl(extractor.getThumbnailUrl()); + } + + @Test + public void testGetUploaderAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + } + + @Test + public void testGetVideoStreams() throws IOException, ExtractionException { + assertFalse(extractor.getVideoStreams().isEmpty()); + } + + @Test + public void testStreamType() throws ParsingException { + assertTrue(extractor.getStreamType() == StreamType.VIDEO_STREAM); + } + + @Test + public void testGetRelatedVideos() throws ExtractionException, IOException { + StreamInfoItemsCollector relatedVideos = extractor.getRelatedVideos(); + assertFalse(relatedVideos.getItems().isEmpty()); + assertTrue(relatedVideos.getErrors().isEmpty()); + } + + @Test + public void testGetSubtitlesListDefault() throws IOException, ExtractionException { + assertTrue(extractor.getSubtitlesDefault().isEmpty()); + } + + @Test + public void testGetSubtitlesList() throws IOException, ExtractionException { + assertTrue(extractor.getSubtitlesDefault().isEmpty()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java new file mode 100644 index 000000000..b82632b2f --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java @@ -0,0 +1,38 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Localization; + +/** + * Test for {@link PeertubeStreamLinkHandlerFactory} + */ +public class PeertubeStreamLinkHandlerFactoryTest { + private static PeertubeStreamLinkHandlerFactory linkHandler; + + @BeforeClass + public static void setUp() throws Exception { + linkHandler = PeertubeStreamLinkHandlerFactory.getInstance(); + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + } + + @Test + public void getId() throws Exception { + assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60", linkHandler.fromUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60").getId()); + assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60", linkHandler.fromUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa").getId()); + } + + + @Test + public void testAcceptUrl() throws ParsingException { + assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60")); + assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa")); + } +} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java new file mode 100644 index 000000000..e11bb5ee0 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java @@ -0,0 +1,96 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + +import java.util.List; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.utils.Localization; + +/** + * Test for {@link PeertubeTrendingExtractor} + */ +public class PeertubeTrendingExtractorTest { + + static KioskExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + extractor = PeerTube + .getKioskList() + .getExtractorById("Trending", null); + extractor.fetchPage(); + } + + @Test + public void testGetDownloader() throws Exception { + assertNotNull(NewPipe.getDownloader()); + } + + @Test + public void testGetName() throws Exception { + assertEquals(extractor.getName(), "Trending"); + } + + @Test + public void testId() { + assertEquals(extractor.getId(), "Trending"); + } + + @Test + public void testGetStreams() throws Exception { + ListExtractor.InfoItemsPage page = extractor.getInitialPage(); + if(!page.getErrors().isEmpty()) { + System.err.println("----------"); + List errors = page.getErrors(); + for(Throwable e: errors) { + e.printStackTrace(); + System.err.println("----------"); + } + } + assertTrue("no streams are received", + !page.getItems().isEmpty() + && page.getErrors().isEmpty()); + } + + @Test + public void testGetStreamsErrors() throws Exception { + assertTrue("errors during stream list extraction", extractor.getInitialPage().getErrors().isEmpty()); + } + + @Test + public void testHasMoreStreams() throws Exception { + // Setup the streams + extractor.getInitialPage(); + assertTrue("has more streams", extractor.hasNextPage()); + } + + @Test + public void testGetNextPageUrl() throws Exception { + assertTrue(extractor.hasNextPage()); + } + + @Test + public void testGetNextPage() throws Exception { + extractor.getInitialPage().getItems(); + assertFalse("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null + || extractor.getPage(extractor.getNextPageUrl()).getItems().isEmpty()); + } + + @Test + public void testGetCleanUrl() throws Exception { + assertEquals(extractor.getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-views"); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java new file mode 100644 index 000000000..6febed8d0 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java @@ -0,0 +1,54 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Localization; + +/** + * Test for {@link PeertubeTrendingLinkHandlerFactory} + */ +public class PeertubeTrendingLinkHandlerFactoryTest { + private static LinkHandlerFactory LinkHandlerFactory; + + @BeforeClass + public static void setUp() throws Exception { + LinkHandlerFactory = new PeertubeTrendingLinkHandlerFactory(); + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + } + + @Test + public void getUrl() + throws Exception { + assertEquals(LinkHandlerFactory.fromId("Trending").getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-views"); + assertEquals(LinkHandlerFactory.fromId("Recently added").getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-publishedAt"); + assertEquals(LinkHandlerFactory.fromId("Local").getUrl(), "https://peertube.mastodon.host/api/v1/videos?filter=local"); + } + + @Test + public void getId() + throws Exception { + assertEquals(LinkHandlerFactory.fromUrl("https://peertube.mastodon.host/videos/trending").getId(), "Trending"); + assertEquals(LinkHandlerFactory.fromUrl("https://peertube.mastodon.host/videos/recently-added").getId(), "Recently added"); + assertEquals(LinkHandlerFactory.fromUrl("https://peertube.mastodon.host/videos/local").getId(), "Local"); + } + + @Test + public void acceptUrl() throws ParsingException { + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/trending")); + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/trending?adsf=fjaj#fhe")); + + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/recently-added")); + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/recently-added?adsf=fjaj#fhe")); + + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/local")); + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/local?adsf=fjaj#fhe")); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java new file mode 100644 index 000000000..08dd7a0a9 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java @@ -0,0 +1,28 @@ +package org.schabi.newpipe.extractor.services.peertube.search; + +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; + +/** + * Test for {@link PeertubeSearchExtractor} + */ +public abstract class PeertubeSearchExtractorBaseTest { + + protected static PeertubeSearchExtractor extractor; + protected static ListExtractor.InfoItemsPage itemsPage; + + @Test + public void testResultListElementsLength() { + assertTrue(Integer.toString(itemsPage.getItems().size()), + itemsPage.getItems().size() >= 3); + } + + @Test + public void testUrl() throws Exception { + assertTrue(extractor.getUrl(), extractor.getUrl().startsWith("https://peertube.mastodon.host/api/v1/search/videos")); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java new file mode 100644 index 000000000..439688257 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java @@ -0,0 +1,91 @@ +package org.schabi.newpipe.extractor.services.peertube.search; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.utils.Localization; + +/** + * Test for {@link PeertubeSearchExtractor} + */ +public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorBaseTest { + + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("internet's own boy"); + extractor.fetchPage(); + itemsPage = extractor.getInitialPage(); + } + + @Test + public void testGetSecondPageUrl() throws Exception { + assertEquals("", extractor.getNextPageUrl()); + } + + @Test + public void testResultList_FirstElement() { + InfoItem firstInfoItem = itemsPage.getItems().get(0); + + // THe channel should be the first item + assertEquals("name", "The Internet's Own Boy", firstInfoItem.getName()); + assertEquals("url","https://peertube.mastodon.host/api/v1/videos/04af977f-4201-4697-be67-a8d8cae6fa7a", firstInfoItem.getUrl()); + } + + @Test + public void testResultListCheckIfContainsStreamItems() { + boolean hasStreams = false; + for(InfoItem item : itemsPage.getItems()) { + if(item instanceof StreamInfoItem) { + hasStreams = true; + } + } + assertTrue("Has no InfoItemStreams", hasStreams); + } + + @Test + public void testGetSecondPage() throws Exception { + extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("internet"); + itemsPage = extractor.getInitialPage(); + PeertubeSearchExtractor secondExtractor = + (PeertubeSearchExtractor) PeerTube.getSearchExtractor("internet"); + ListExtractor.InfoItemsPage secondPage = secondExtractor.getPage(itemsPage.getNextPageUrl()); + assertTrue(Integer.toString(secondPage.getItems().size()), + secondPage.getItems().size() >= 10); + + // check if its the same result + boolean equals = true; + for (int i = 0; i < secondPage.getItems().size() + && i < itemsPage.getItems().size(); i++) { + if(!secondPage.getItems().get(i).getUrl().equals( + itemsPage.getItems().get(i).getUrl())) { + equals = false; + } + } + assertFalse("First and second page are equal", equals); + + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=internet&start=24&count=12", + secondPage.getNextPageUrl()); + } + + + @Test + public void testId() throws Exception { + assertEquals("internet's own boy", extractor.getId()); + } + + @Test + public void testName() { + assertEquals("internet's own boy", extractor.getName()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java new file mode 100644 index 000000000..89680d609 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java @@ -0,0 +1,18 @@ +package org.schabi.newpipe.extractor.services.peertube.search; + +import static org.junit.Assert.assertEquals; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + +import org.junit.Test; + +public class PeertubeSearchQHTest { + + @Test + public void testRegularValues() throws Exception { + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=asdf", PeerTube.getSearchQHFactory().fromQuery("asdf").getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=hans",PeerTube.getSearchQHFactory().fromQuery("hans").getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=Poifj%26jaijf", PeerTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=G%C3%BCl%C3%BCm", PeerTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); + } +}