diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java index e0fe4912c..e5c03d3f1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java @@ -1,5 +1,10 @@ package org.schabi.newpipe.extractor; +import java.io.IOException; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -8,11 +13,6 @@ import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import java.io.IOException; -import java.io.Serializable; - public abstract class Extractor{ /** * {@link StreamingService} currently related to this extractor.
@@ -93,6 +93,11 @@ public abstract class Extractor{ public String getUrl() throws ParsingException { return linkHandler.getUrl(); } + + @Nonnull + public String getBaseUrl() throws ParsingException { + return linkHandler.getBaseUrl(); + } @Nonnull public StreamingService getService() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java index 286a34cec..c65283a58 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java @@ -126,6 +126,13 @@ public enum MediaFormat { return null; } + public static MediaFormat getFromSuffix(String suffix) { + for (MediaFormat vf: values()) { + if (vf.suffix.equals(suffix)) return vf; + } + return null; + } + /** * Get the name of the format * @return the name of the format @@ -149,4 +156,5 @@ public enum MediaFormat { public String getMimeType() { return mimeType; } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java index a171db71d..6be1cea40 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java @@ -1,13 +1,13 @@ package org.schabi.newpipe.extractor; -import org.schabi.newpipe.extractor.services.media_ccc.MediaCCCService; -import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService; -import org.schabi.newpipe.extractor.services.youtube.YoutubeService; - +import java.util.Arrays; +import java.util.Collections; import java.util.List; -import static java.util.Arrays.asList; -import static java.util.Collections.unmodifiableList; +import org.schabi.newpipe.extractor.services.media_ccc.MediaCCCService; +import org.schabi.newpipe.extractor.services.peertube.PeertubeService; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService; +import org.schabi.newpipe.extractor.services.youtube.YoutubeService; /* * Copyright (C) Christian Schabesberger 2018 @@ -38,16 +38,18 @@ public final class ServiceList { public static final YoutubeService YouTube; public static final SoundcloudService SoundCloud; public static final MediaCCCService MediaCCC; + public static final PeertubeService PeerTube; /** * When creating a new service, put this service in the end of this list, * and give it the next free id. */ - private static final List SERVICES = unmodifiableList( - asList( + private static final List SERVICES = Collections.unmodifiableList( + Arrays.asList( YouTube = new YoutubeService(0), SoundCloud = new SoundcloudService(1), - MediaCCC = new MediaCCCService(2) + MediaCCC = new MediaCCCService(2), + PeerTube = new PeertubeService(3) )); /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index 4906de9ef..ad22642db 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -1,11 +1,19 @@ package org.schabi.newpipe.extractor; +import java.util.Collections; +import java.util.List; + import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskList; -import org.schabi.newpipe.extractor.linkhandler.*; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; @@ -16,9 +24,6 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; -import java.util.Collections; -import java.util.List; - /* * Copyright (C) Christian Schabesberger 2018 * StreamingService.java is part of NewPipe. @@ -44,6 +49,7 @@ public abstract class StreamingService { */ public static class ServiceInfo { private final String name; + private final List mediaCapabilities; /** @@ -59,7 +65,7 @@ public abstract class StreamingService { public String getName() { return name; } - + public List getMediaCapabilities() { return mediaCapabilities; } @@ -110,6 +116,8 @@ public abstract class StreamingService { public String toString() { return serviceId + ":" + serviceInfo.getName(); } + + public abstract String getBaseUrl(); /*////////////////////////////////////////////////////////////////////////// // Url Id handler @@ -250,7 +258,7 @@ public abstract class StreamingService { } return getCommentsExtractor(llhf.fromUrl(url)); } - + /*////////////////////////////////////////////////////////////////////////// // Utils //////////////////////////////////////////////////////////////////////////*/ diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java index 4b20cfa9f..5a3be0794 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java @@ -118,7 +118,7 @@ public class KioskList { for(Map.Entry e : kioskList.entrySet()) { KioskEntry ke = e.getValue(); if(ke.handlerFactory.acceptUrl(url)) { - return getExtractorById(e.getKey(), nextPageUrl, localization); + return getExtractorById(ke.handlerFactory.getId(url), nextPageUrl, localization); } } throw new ExtractionException("Could not find a kiosk that fits to the url: " + url); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java index c28bc5c83..525105134 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java @@ -2,6 +2,9 @@ package org.schabi.newpipe.extractor.linkhandler; import java.io.Serializable; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.utils.Utils; + public class LinkHandler implements Serializable { protected final String originalUrl; protected final String url; @@ -28,4 +31,8 @@ public class LinkHandler implements Serializable { public String getId() { return id; } + + public String getBaseUrl() throws ParsingException { + return Utils.getBaseUrl(url); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java index 60a65f5ef..b9e60ce0d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.linkhandler; import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.utils.Utils; /* * Created by Christian Schabesberger on 26.07.16. @@ -33,26 +34,41 @@ public abstract class LinkHandlerFactory { public abstract String getUrl(String id) throws ParsingException; public abstract boolean onAcceptUrl(final String url) throws ParsingException; + public String getUrl(String id, String baseUrl) throws ParsingException{ + return getUrl(id); + } + /////////////////////////////////// // Logic /////////////////////////////////// public LinkHandler fromUrl(String url) throws ParsingException { + final String baseUrl = Utils.getBaseUrl(url); + return fromUrl(url, baseUrl); + } + + public LinkHandler fromUrl(String url, String baseUrl) throws ParsingException { if(url == null) throw new IllegalArgumentException("url can not be null"); if(!acceptUrl(url)) { throw new ParsingException("Malformed unacceptable url: " + url); } final String id = getId(url); - return new LinkHandler(url, getUrl(id), id); + return new LinkHandler(url, getUrl(id,baseUrl), id); } - + public LinkHandler fromId(String id) throws ParsingException { if(id == null) throw new IllegalArgumentException("id can not be null"); final String url = getUrl(id); return new LinkHandler(url, url, id); } + public LinkHandler fromId(String id, String baseUrl) throws ParsingException { + if(id == null) throw new IllegalArgumentException("id can not be null"); + final String url = getUrl(id, baseUrl); + return new LinkHandler(url, url, id); + } + /** * When a VIEW_ACTION is caught this function will test if the url delivered within the calling * Intent was meant to be watched with this Service. @@ -65,4 +81,5 @@ public abstract class LinkHandlerFactory { throw fe; } } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java index a9c4e51aa..83955265c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java @@ -1,10 +1,11 @@ package org.schabi.newpipe.extractor.linkhandler; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - import java.util.ArrayList; import java.util.List; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.utils.Utils; + public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { /////////////////////////////////// @@ -14,23 +15,37 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { public List getContentFilter(String url) throws ParsingException { return new ArrayList<>(0);} public String getSortFilter(String url) throws ParsingException {return ""; } public abstract String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException; + + public String getUrl(String id, List contentFilter, String sortFilter, String baseUrl) throws ParsingException { + return getUrl(id, contentFilter, sortFilter); + } /////////////////////////////////// // Logic /////////////////////////////////// - - + @Override public ListLinkHandler fromUrl(String url) throws ParsingException { + String baseUrl = Utils.getBaseUrl(url); + return fromUrl(url, baseUrl); + } + + @Override + public ListLinkHandler fromUrl(String url, String baseUrl) throws ParsingException { if(url == null) throw new IllegalArgumentException("url may not be null"); - return new ListLinkHandler(super.fromUrl(url), getContentFilter(url), getSortFilter(url)); + return new ListLinkHandler(super.fromUrl(url, baseUrl), getContentFilter(url), getSortFilter(url)); } @Override public ListLinkHandler fromId(String id) throws ParsingException { return new ListLinkHandler(super.fromId(id), new ArrayList(0), ""); } + + @Override + public ListLinkHandler fromId(String id, String baseUrl) throws ParsingException { + return new ListLinkHandler(super.fromId(id, baseUrl), new ArrayList(0), ""); + } public ListLinkHandler fromQuery(String id, List contentFilters, @@ -38,8 +53,15 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { final String url = getUrl(id, contentFilters, sortFilter); return new ListLinkHandler(url, url, id, contentFilters, sortFilter); } + + public ListLinkHandler fromQuery(String id, + List contentFilters, + String sortFilter, String baseUrl) throws ParsingException { + final String url = getUrl(id, contentFilters, sortFilter, baseUrl); + return new ListLinkHandler(url, url, id, contentFilters, sortFilter); + } - + /** * For makeing ListLinkHandlerFactory compatible with LinkHandlerFactory we need to override this, * however it should not be overridden by the actual implementation. @@ -50,6 +72,11 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { return getUrl(id, new ArrayList(0), ""); } + @Override + public String getUrl(String id, String baseUrl) throws ParsingException { + return getUrl(id, new ArrayList(0), "", baseUrl); + } + /** * Will returns content filter the corresponding extractor can handle like "channels", "videos", "music", etc. * diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java index 7e7fa842d..f2d9b673a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java @@ -116,4 +116,9 @@ public class MediaCCCService extends StreamingService { return null; } + @Override + public String getBaseUrl() { + return "https://media.ccc.de"; + } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java new file mode 100644 index 000000000..992b8eb6d --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java @@ -0,0 +1,63 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import java.io.IOException; + +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.utils.JsonUtils; + +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; + +public class PeertubeInstance { + + private final String url; + private String name; + public static final PeertubeInstance defaultInstance = new PeertubeInstance("https://framatube.org", "FramaTube"); + + public PeertubeInstance(String url) { + this.url = url; + this.name = "PeerTube"; + } + + public PeertubeInstance(String url , String name) { + this.url = url; + this.name = name; + } + + public String getUrl() { + return url; + } + + public void fetchInstanceMetaData() throws Exception { + Downloader downloader = NewPipe.getDownloader(); + Response response = null; + + try { + response = downloader.get(url + "/api/v1/config"); + } catch (ReCaptchaException | IOException e) { + throw new Exception("unable to configure instance " + url, e); + } + + if(null == response || StringUtil.isBlank(response.responseBody())) { + throw new Exception("unable to configure instance " + url); + } + + try { + JsonObject json = JsonParser.object().from(response.responseBody()); + this.name = JsonUtils.getString(json, "instance.name"); + } catch (JsonParserException | ParsingException e) { + throw new Exception("unable to parse instance config", e); + } + } + + public String getName() { + return name; + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java new file mode 100644 index 000000000..f518faf9a --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -0,0 +1,39 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; + +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import com.grack.nanojson.JsonObject; + +public class PeertubeParsingHelper { + + private PeertubeParsingHelper() { + } + + public static void validate(JsonObject json) throws ContentNotAvailableException { + String error = json.getString("error"); + if(!StringUtil.isBlank(error)) { + throw new ContentNotAvailableException(error); + } + } + + public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException { + Date date; + try { + date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse(textualUploadDate); + } catch (ParseException e) { + throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); + } + + final Calendar uploadDate = Calendar.getInstance(); + uploadDate.setTime(date); + return uploadDate; + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java new file mode 100644 index 000000000..47271732f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -0,0 +1,160 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static java.util.Arrays.asList; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; + +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.comments.CommentsExtractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.kiosk.KioskList; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSuggestionExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; +import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; + +public class PeertubeService extends StreamingService { + + private PeertubeInstance instance; + + public PeertubeService(int id) { + this(id, PeertubeInstance.defaultInstance); + } + + public PeertubeService(int id, PeertubeInstance instance) { + super(id, "PeerTube", asList(VIDEO, COMMENTS)); + this.instance = instance; + } + + @Override + public LinkHandlerFactory getStreamLHFactory() { + return PeertubeStreamLinkHandlerFactory.getInstance(); + } + + @Override + public ListLinkHandlerFactory getChannelLHFactory() { + return PeertubeChannelLinkHandlerFactory.getInstance(); + } + + @Override + public ListLinkHandlerFactory getPlaylistLHFactory() { + // TODO Auto-generated method stub + return null; + } + + @Override + public SearchQueryHandlerFactory getSearchQHFactory() { + return PeertubeSearchQueryHandlerFactory.getInstance(); + } + + @Override + public ListLinkHandlerFactory getCommentsLHFactory() { + return PeertubeCommentsLinkHandlerFactory.getInstance(); + } + + @Override + public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) { + return new PeertubeSearchExtractor(this, queryHandler); + } + + @Override + public SuggestionExtractor getSuggestionExtractor() { + return new PeertubeSuggestionExtractor(this); + } + + @Override + public SubscriptionExtractor getSubscriptionExtractor() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) + throws ExtractionException { + return new PeertubeChannelExtractor(this, linkHandler); + } + + @Override + public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) + throws ExtractionException { + // TODO Auto-generated method stub + return null; + } + + @Override + public StreamExtractor getStreamExtractor(LinkHandler linkHandler) + throws ExtractionException { + return new PeertubeStreamExtractor(this, linkHandler); + } + + @Override + public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler) + throws ExtractionException { + return new PeertubeCommentsExtractor(this, linkHandler); + } + + @Override + public String getBaseUrl() { + return instance.getUrl(); + } + + public PeertubeInstance getInstance() { + return this.instance; + } + + public void setInstance(PeertubeInstance instance) { + this.instance = instance; + } + + @Override + public KioskList getKioskList() throws ExtractionException { + KioskList.KioskExtractorFactory kioskFactory = new KioskList.KioskExtractorFactory() { + @Override + public KioskExtractor createNewKiosk(StreamingService streamingService, + String url, + String id) + throws ExtractionException { + return new PeertubeTrendingExtractor(PeertubeService.this, + new PeertubeTrendingLinkHandlerFactory().fromId(id), id); + } + }; + + KioskList list = new KioskList(this); + + // add kiosks here e.g.: + final PeertubeTrendingLinkHandlerFactory h = new PeertubeTrendingLinkHandlerFactory(); + try { + list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING); + list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_MOST_LIKED); + list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_RECENT); + list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_LOCAL); + list.setDefaultKiosk(PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING); + } catch (Exception e) { + throw new ExtractionException(e); + } + + return list; + } + + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java new file mode 100644 index 000000000..8b214670c --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -0,0 +1,188 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.io.IOException; + +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Parser.RegexException; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; + +public class PeertubeChannelExtractor extends ChannelExtractor { + + private static final String START_KEY = "start"; + private static final String COUNT_KEY = "count"; + private static final int ITEMS_PER_PAGE = 12; + private static final String START_PATTERN = "start=(\\d*)"; + + private InfoItemsPage initPage; + private long total; + + private JsonObject json; + private final String baseUrl; + + public PeertubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) throws ParsingException { + super(service, linkHandler); + this.baseUrl = getBaseUrl(); + } + + @Override + public String getAvatarUrl() throws ParsingException { + String value; + try { + value = JsonUtils.getString(json, "avatar.path"); + }catch(Exception e) { + value = "/client/assets/images/default-avatar.png"; + } + return baseUrl + value; + } + + @Override + public String getBannerUrl() throws ParsingException { + return null; + } + + @Override + public String getFeedUrl() throws ParsingException { + return null; + } + + @Override + public long getSubscriberCount() throws ParsingException { + Number number = JsonUtils.getNumber(json, "followersCount"); + return number.longValue(); + } + + @Override + public String getDescription() throws ParsingException { + try { + return JsonUtils.getString(json, "description"); + }catch(ParsingException e) { + return "No description"; + } + } + + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + super.fetchPage(); + return initPage; + } + + private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException { + JsonArray contents; + try { + contents = (JsonArray) JsonUtils.getValue(json, "data"); + }catch(Exception e) { + throw new ParsingException("unable to extract channel streams", e); + } + + for(Object c: contents) { + if(c instanceof JsonObject) { + final JsonObject item = (JsonObject) c; + PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); + collector.commit(extractor); + } + } + + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + super.fetchPage(); + return initPage.getNextPageUrl(); + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + Response response = getDownloader().get(pageUrl); + JsonObject json = null; + if(null != response && !StringUtil.isBlank(response.responseBody())) { + try { + json = JsonParser.object().from(response.responseBody()); + } catch (Exception e) { + throw new ParsingException("Could not parse json data for kiosk info", e); + } + } + + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + if(json != null) { + PeertubeParsingHelper.validate(json); + Number number = JsonUtils.getNumber(json, "total"); + if(number != null) this.total = number.longValue(); + collectStreamsFrom(collector, json, pageUrl); + } else { + throw new ExtractionException("Unable to get peertube kiosk info"); + } + return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl)); + } + + + private String getNextPageUrl(String prevPageUrl) { + String prevStart; + try { + prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl); + } catch (RegexException e) { + return ""; + } + if(StringUtil.isBlank(prevStart)) return ""; + long nextStart = 0; + try { + nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; + } catch (NumberFormatException e) { + return ""; + } + + if(nextStart >= total) { + return ""; + }else { + return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); + } + } + + @Override + public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { + Response response = downloader.get(getUrl()); + if(null != response && null != response.responseBody()) { + setInitialData(response.responseBody()); + }else { + throw new ExtractionException("Unable to extract peertube channel data"); + } + + String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; + this.initPage = getPage(pageUrl); + } + + private void setInitialData(String responseBody) throws ExtractionException { + try { + json = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ExtractionException("Unable to extract peertube channel data", e); + } + if(null == json) throw new ExtractionException("Unable to extract peertube channel data"); + } + + @Override + public String getName() throws ParsingException { + return JsonUtils.getString(json, "displayName"); + } + + @Override + public String getOriginalUrl() throws ParsingException { + return baseUrl + "/accounts/" + getId(); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java new file mode 100644 index 000000000..b7aba4fac --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java @@ -0,0 +1,123 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.io.IOException; + +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.comments.CommentsExtractor; +import org.schabi.newpipe.extractor.comments.CommentsInfoItem; +import org.schabi.newpipe.extractor.comments.CommentsInfoItemsCollector; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Parser.RegexException; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; + +public class PeertubeCommentsExtractor extends CommentsExtractor { + + private static final String START_KEY = "start"; + private static final String COUNT_KEY = "count"; + private static final int ITEMS_PER_PAGE = 12; + private static final String START_PATTERN = "start=(\\d*)"; + + private InfoItemsPage initPage; + private long total; + + public PeertubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) { + super(service, uiHandler); + } + + @Override + public String getName() throws ParsingException { + return "Comments"; + } + + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + super.fetchPage(); + return initPage; + } + + private void collectStreamsFrom(CommentsInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException { + JsonArray contents; + try { + contents = (JsonArray) JsonUtils.getValue(json, "data"); + }catch(Exception e) { + throw new ParsingException("unable to extract comments info", e); + } + + for(Object c: contents) { + if(c instanceof JsonObject) { + final JsonObject item = (JsonObject) c; + PeertubeCommentsInfoItemExtractor extractor = new PeertubeCommentsInfoItemExtractor(item, this); + collector.commit(extractor); + } + } + + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + super.fetchPage(); + return initPage.getNextPageUrl(); + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + Response response = getDownloader().get(pageUrl); + JsonObject json = null; + if(null != response && !StringUtil.isBlank(response.responseBody())) { + try { + json = JsonParser.object().from(response.responseBody()); + } catch (Exception e) { + throw new ParsingException("Could not parse json data for comments info", e); + } + } + + CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); + if(json != null) { + Number number = JsonUtils.getNumber(json, "total"); + if(number != null) this.total = number.longValue(); + collectStreamsFrom(collector, json, pageUrl); + } else { + throw new ExtractionException("Unable to get peertube comments info"); + } + return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl)); + } + + @Override + public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { + String pageUrl = getUrl() + "?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; + this.initPage = getPage(pageUrl); + } + + private String getNextPageUrl(String prevPageUrl) { + String prevStart; + try { + prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl); + } catch (RegexException e) { + return ""; + } + if(StringUtil.isBlank(prevStart)) return ""; + long nextStart = 0; + try { + nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; + } catch (NumberFormatException e) { + return ""; + } + + if(nextStart >= total) { + return ""; + }else { + return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); + } + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java new file mode 100644 index 000000000..edb59e1fe --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -0,0 +1,104 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; +import org.schabi.newpipe.extractor.utils.JsonUtils; + +import com.grack.nanojson.JsonObject; + + +public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor { + + private final JsonObject item; + private final String url; + private final String baseUrl; + + public PeertubeCommentsInfoItemExtractor(JsonObject item, PeertubeCommentsExtractor extractor) throws ParsingException { + this.item = item; + this.url = extractor.getUrl(); + this.baseUrl = extractor.getBaseUrl(); + } + + @Override + public String getUrl() throws ParsingException { + return url; + } + + @Override + public String getThumbnailUrl() throws ParsingException { + String value; + try { + value = JsonUtils.getString(item, "account.avatar.path"); + }catch(Exception e) { + value = "/client/assets/images/default-avatar.png"; + } + return baseUrl + value; + } + + @Override + public String getName() throws ParsingException { + return JsonUtils.getString(item, "account.displayName"); + } + + @Override + public String getTextualPublishedTime() throws ParsingException { + return JsonUtils.getString(item, "createdAt"); + } + + @Override + public DateWrapper getPublishedTime() throws ParsingException { + String textualUploadDate = getTextualPublishedTime(); + return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); + } + + @Override + public int getLikeCount() throws ParsingException { + return -1; + } + + @Override + public String getCommentText() throws ParsingException { + String htmlText = JsonUtils.getString(item, "text"); + try { + Document doc = Jsoup.parse(htmlText); + return doc.body().text(); + }catch(Exception e) { + return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", ""); + } + } + + @Override + public String getCommentId() throws ParsingException { + Number value = JsonUtils.getNumber(item, "id"); + return value.toString(); + } + + @Override + public String getAuthorThumbnail() throws ParsingException { + String value; + try { + value = JsonUtils.getString(item, "account.avatar.path"); + }catch(Exception e) { + value = "/client/assets/images/default-avatar.png"; + } + return baseUrl + value; + } + + @Override + public String getAuthorName() throws ParsingException { + return JsonUtils.getString(item, "account.displayName"); + } + + @Override + public String getAuthorEndpoint() throws ParsingException { + String name = JsonUtils.getString(item, "account.name"); + String host = JsonUtils.getString(item, "account.host"); + return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java new file mode 100644 index 000000000..98d20b534 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java @@ -0,0 +1,86 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.io.IOException; + +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; + +public class PeertubePlaylistExtractor extends PlaylistExtractor{ + + public PeertubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { + super(service, linkHandler); + // TODO Auto-generated constructor stub + } + + @Override + public String getThumbnailUrl() throws ParsingException { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getBannerUrl() throws ParsingException { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getUploaderUrl() throws ParsingException { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getUploaderName() throws ParsingException { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getUploaderAvatarUrl() throws ParsingException { + // TODO Auto-generated method stub + return null; + } + + @Override + public long getStreamCount() throws ParsingException { + // TODO Auto-generated method stub + return 0; + } + + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + // TODO Auto-generated method stub + return null; + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + // TODO Auto-generated method stub + return null; + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + // TODO Auto-generated method stub + return null; + } + + @Override + public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { + // TODO Auto-generated method stub + + } + + @Override + public String getName() throws ParsingException { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java new file mode 100644 index 000000000..ff021f732 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -0,0 +1,130 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.io.IOException; + +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.InfoItemExtractor; +import org.schabi.newpipe.extractor.InfoItemsCollector; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Parser.RegexException; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; + +public class PeertubeSearchExtractor extends SearchExtractor { + + private static final String START_KEY = "start"; + private static final String COUNT_KEY = "count"; + private static final int ITEMS_PER_PAGE = 12; + private static final String START_PATTERN = "start=(\\d*)"; + + private InfoItemsPage initPage; + private long total; + + public PeertubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { + super(service, linkHandler); + } + + @Override + public String getSearchSuggestion() throws ParsingException { + return null; + } + + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + super.fetchPage(); + return initPage; + } + + private InfoItemsCollector collectStreamsFrom(JsonObject json) throws ParsingException { + + final InfoItemsSearchCollector collector = getInfoItemSearchCollector(); + + JsonArray contents; + try { + contents = (JsonArray) JsonUtils.getValue(json, "data"); + }catch(Exception e) { + throw new ParsingException("unable to extract search info", e); + } + + String baseUrl = getBaseUrl(); + for(Object c: contents) { + if(c instanceof JsonObject) { + final JsonObject item = (JsonObject) c; + PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); + collector.commit(extractor); + } + } + + return collector; + + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + super.fetchPage(); + return initPage.getNextPageUrl(); + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + Response response = getDownloader().get(pageUrl); + JsonObject json = null; + if(null != response && !StringUtil.isBlank(response.responseBody())) { + try { + json = JsonParser.object().from(response.responseBody()); + } catch (Exception e) { + throw new ParsingException("Could not parse json data for search info", e); + } + } + + if(json != null) { + Number number = JsonUtils.getNumber(json, "total"); + if(number != null) this.total = number.longValue(); + return new InfoItemsPage<>(collectStreamsFrom(json), getNextPageUrl(pageUrl)); + } else { + throw new ExtractionException("Unable to get peertube search info"); + } + } + + @Override + public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { + String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; + this.initPage = getPage(pageUrl); + } + + private String getNextPageUrl(String prevPageUrl) { + String prevStart; + try { + prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl); + } catch (RegexException e) { + return ""; + } + if(StringUtil.isBlank(prevStart)) return ""; + long nextStart = 0; + try { + nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; + } catch (NumberFormatException e) { + return ""; + } + + if(nextStart >= total) { + return ""; + }else { + return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); + } + } + + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java new file mode 100644 index 000000000..d8d0de005 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -0,0 +1,342 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.Stream; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.SubtitlesStream; +import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Utils; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; + +public class PeertubeStreamExtractor extends StreamExtractor { + + + private JsonObject json; + private List subtitles = new ArrayList<>(); + private final String baseUrl; + + public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler) throws ParsingException { + super(service, linkHandler); + this.baseUrl = getBaseUrl(); + } + + @Override + public String getTextualUploadDate() throws ParsingException { + return JsonUtils.getString(json, "publishedAt"); + } + + @Override + public DateWrapper getUploadDate() throws ParsingException { + final String textualUploadDate = getTextualUploadDate(); + + if (textualUploadDate == null) { + return null; + } + + return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); + } + + @Override + public String getThumbnailUrl() throws ParsingException { + return baseUrl + JsonUtils.getString(json, "thumbnailPath"); + } + + @Override + public String getDescription() throws ParsingException { + try { + return JsonUtils.getString(json, "description"); + }catch(ParsingException e) { + return "No description"; + } + } + + @Override + public int getAgeLimit() throws ParsingException { + return NO_AGE_LIMIT; + } + + @Override + public long getLength() throws ParsingException { + Number value = JsonUtils.getNumber(json, "duration"); + return value.longValue(); + } + + @Override + public long getTimeStamp() throws ParsingException { + //TODO fetch timestamp from url if present; + return 0; + } + + @Override + public long getViewCount() throws ParsingException { + Number value = JsonUtils.getNumber(json, "views"); + return value.longValue(); + } + + @Override + public long getLikeCount() throws ParsingException { + Number value = JsonUtils.getNumber(json, "likes"); + return value.longValue(); + } + + @Override + public long getDislikeCount() throws ParsingException { + Number value = JsonUtils.getNumber(json, "dislikes"); + return value.longValue(); + } + + @Override + public String getUploaderUrl() throws ParsingException { + String name = JsonUtils.getString(json, "account.name"); + String host = JsonUtils.getString(json, "account.host"); + return getService().getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); + } + + @Override + public String getUploaderName() throws ParsingException { + return JsonUtils.getString(json, "account.displayName"); + } + + @Override + public String getUploaderAvatarUrl() throws ParsingException { + String value; + try { + value = JsonUtils.getString(json, "account.avatar.path"); + }catch(Exception e) { + value = "/client/assets/images/default-avatar.png"; + } + return baseUrl + value; + } + + @Override + public String getDashMpdUrl() throws ParsingException { + return ""; + } + + @Override + public String getHlsUrl() throws ParsingException { + return ""; + } + + @Override + public List getAudioStreams() throws IOException, ExtractionException { + return null; + } + + @Override + public List getVideoStreams() throws IOException, ExtractionException { + assertPageFetched(); + List videoStreams = new ArrayList<>(); + try { + JsonArray streams = json.getArray("files", new JsonArray()); + for(Object s: streams) { + if(!(s instanceof JsonObject)) continue; + JsonObject stream = (JsonObject) s; + String url = JsonUtils.getString(stream, "fileUrl"); + String torrentUrl = JsonUtils.getString(stream, "torrentUrl"); + String resolution = JsonUtils.getString(stream, "resolution.label"); + String extension = url.substring(url.lastIndexOf(".") + 1); + MediaFormat format = MediaFormat.getFromSuffix(extension); + VideoStream videoStream = new VideoStream(url, torrentUrl, format, resolution); + if (!Stream.containSimilarStream(videoStream, videoStreams)) { + videoStreams.add(videoStream); + } + } + } catch (Exception e) { + throw new ParsingException("Could not get video streams", e); + } + + return videoStreams; + } + + + @Override + public List getVideoOnlyStreams() throws IOException, ExtractionException { + // TODO Auto-generated method stub + return null; + } + + @Override + public List getSubtitlesDefault() throws IOException, ExtractionException { + return subtitles; + } + + @Override + public List getSubtitles(final MediaFormat format) throws IOException, ExtractionException { + List filteredSubs = new ArrayList<>(); + for(SubtitlesStream sub: subtitles) { + if(sub.getFormat() == format) { + filteredSubs.add(sub); + } + } + return filteredSubs; + } + + @Override + public StreamType getStreamType() throws ParsingException { + return StreamType.VIDEO_STREAM; + } + + @Override + public StreamInfoItem getNextStream() throws IOException, ExtractionException { + return null; + } + + @Override + public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException { + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + List tags = getTags(); + String apiUrl = null; + if(!tags.isEmpty()) { + apiUrl = getRelatedStreamsUrl(tags); + + }else { + apiUrl = getUploaderUrl() + "/videos?start=0&count=8"; + } + if(!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl); + return collector; + } + + private List getTags(){ + try { + return (List) JsonUtils.getArray(json, "tags"); + } catch (Exception e) { + return Collections.emptyList(); + } + } + + private String getRelatedStreamsUrl(List tags) throws UnsupportedEncodingException { + String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT; + StringBuilder params = new StringBuilder(); + params.append("start=0&count=8&sort=-createdAt"); + for(String tag : tags) { + params.append("&tagsOneOf="); + params.append(URLEncoder.encode(tag, "UTF-8")); + } + return url + "?" + params.toString(); + } + + private void getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, IOException, ParsingException { + Response response = getDownloader().get(apiUrl); + JsonObject relatedVideosJson = null; + if(null != response && !StringUtil.isBlank(response.responseBody())) { + try { + relatedVideosJson = JsonParser.object().from(response.responseBody()); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json data for related videos", e); + } + } + + if(relatedVideosJson != null) { + collectStreamsFrom(collector, relatedVideosJson); + } + } + + private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json) throws ParsingException { + JsonArray contents; + try { + contents = (JsonArray) JsonUtils.getValue(json, "data"); + }catch(Exception e) { + throw new ParsingException("unable to extract related videos", e); + } + + for(Object c: contents) { + if(c instanceof JsonObject) { + final JsonObject item = (JsonObject) c; + PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); + //do not add the same stream in related streams + if(!extractor.getUrl().equals(getUrl())) collector.commit(extractor); + } + } + + } + + + @Override + public String getErrorMessage() { + return null; + } + + @Override + public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { + Response response = downloader.get(getUrl()); + if(null != response && null != response.responseBody()) { + setInitialData(response.responseBody()); + }else { + throw new ExtractionException("Unable to extract peertube channel data"); + } + + loadSubtitles(); + } + + private void setInitialData(String responseBody) throws ExtractionException { + try { + json = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ExtractionException("Unable to extract peertube stream data", e); + } + if(null == json) throw new ExtractionException("Unable to extract peertube stream data"); + PeertubeParsingHelper.validate(json); + } + + private void loadSubtitles() { + if (subtitles.isEmpty()) { + try { + Response response = getDownloader().get(getUrl() + "/captions"); + JsonObject captionsJson = JsonParser.object().from(response.responseBody()); + JsonArray captions = JsonUtils.getArray(captionsJson, "data"); + for(Object c: captions) { + if(c instanceof JsonObject) { + JsonObject caption = (JsonObject)c; + String url = baseUrl + JsonUtils.getString(caption, "captionPath"); + String languageCode = JsonUtils.getString(caption, "language.id"); + String ext = url.substring(url.lastIndexOf(".") + 1); + MediaFormat fmt = MediaFormat.getFromSuffix(ext); + if(fmt != null && languageCode != null) subtitles.add(new SubtitlesStream(fmt, languageCode, url, false)); + } + } + } catch (Exception e) { + // ignore all exceptions + } + } + } + + @Override + public String getName() throws ParsingException { + return JsonUtils.getString(json, "name"); + } + + @Override + public String getOriginalUrl() throws ParsingException { + return baseUrl + "/videos/watch/" + getId(); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java new file mode 100644 index 000000000..ca85270a9 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java @@ -0,0 +1,90 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; +import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.utils.JsonUtils; + +import com.grack.nanojson.JsonObject; + +public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor { + + protected final JsonObject item; + private final String baseUrl; + + public PeertubeStreamInfoItemExtractor(JsonObject item, String baseUrl) { + this.item = item; + this.baseUrl = baseUrl; + } + + @Override + public String getUrl() throws ParsingException { + String uuid = JsonUtils.getString(item, "uuid"); + return ServiceList.PeerTube.getStreamLHFactory().fromId(uuid, baseUrl).getUrl(); + } + + @Override + public String getThumbnailUrl() throws ParsingException { + String value = JsonUtils.getString(item, "thumbnailPath"); + return baseUrl + value; + } + + @Override + public String getName() throws ParsingException { + return JsonUtils.getString(item, "name"); + } + + @Override + public boolean isAd() throws ParsingException { + return false; + } + + @Override + public long getViewCount() throws ParsingException { + Number value = JsonUtils.getNumber(item, "views"); + return value.longValue(); + } + + @Override + public String getUploaderUrl() throws ParsingException { + String name = JsonUtils.getString(item, "account.name"); + String host = JsonUtils.getString(item, "account.host"); + return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl(); + } + + @Override + public String getUploaderName() throws ParsingException { + return JsonUtils.getString(item, "account.displayName"); + } + + @Override + public String getTextualUploadDate() throws ParsingException { + return JsonUtils.getString(item, "publishedAt"); + } + + @Override + public DateWrapper getUploadDate() throws ParsingException { + final String textualUploadDate = getTextualUploadDate(); + + if (textualUploadDate == null) { + return null; + } + + return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); + } + + @Override + public StreamType getStreamType() throws ParsingException { + return StreamType.VIDEO_STREAM; + } + + @Override + public long getDuration() throws ParsingException { + Number value = JsonUtils.getNumber(item, "duration"); + return value.longValue(); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java new file mode 100644 index 000000000..a2b246141 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java @@ -0,0 +1,21 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.util.List; + +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; + +public class PeertubeSubscriptionExtractor extends SubscriptionExtractor { + + public PeertubeSubscriptionExtractor(StreamingService service, List supportedSources) { + super(service, supportedSources); + // TODO Auto-generated constructor stub + } + + @Override + public String getRelatedUrl() { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java new file mode 100644 index 000000000..7e2ed4d9b --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java @@ -0,0 +1,22 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; + +public class PeertubeSuggestionExtractor extends SuggestionExtractor{ + + public PeertubeSuggestionExtractor(StreamingService service) { + super(service); + } + + @Override + public List suggestionList(String query) throws IOException, ExtractionException { + return Collections.emptyList(); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java new file mode 100644 index 000000000..1cb420460 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java @@ -0,0 +1,124 @@ +package org.schabi.newpipe.extractor.services.peertube.extractors; + +import java.io.IOException; + +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Parser.RegexException; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; + +public class PeertubeTrendingExtractor extends KioskExtractor { + + private static final String START_KEY = "start"; + private static final String COUNT_KEY = "count"; + private static final int ITEMS_PER_PAGE = 12; + private static final String START_PATTERN = "start=(\\d*)"; + + private InfoItemsPage initPage; + private long total; + + public PeertubeTrendingExtractor(StreamingService streamingService, ListLinkHandler linkHandler, String kioskId) { + super(streamingService, linkHandler, kioskId); + } + + @Override + public String getName() throws ParsingException { + return getId(); + } + + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + super.fetchPage(); + return initPage; + } + + private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException { + JsonArray contents; + try { + contents = (JsonArray) JsonUtils.getValue(json, "data"); + }catch(Exception e) { + throw new ParsingException("unable to extract kiosk info", e); + } + + String baseUrl = getBaseUrl(); + for(Object c: contents) { + if(c instanceof JsonObject) { + final JsonObject item = (JsonObject) c; + PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); + collector.commit(extractor); + } + } + + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + super.fetchPage(); + return initPage.getNextPageUrl(); + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + Response response = getDownloader().get(pageUrl); + JsonObject json = null; + if(null != response && !StringUtil.isBlank(response.responseBody())) { + try { + json = JsonParser.object().from(response.responseBody()); + } catch (Exception e) { + throw new ParsingException("Could not parse json data for kiosk info", e); + } + } + + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + if(json != null) { + Number number = JsonUtils.getNumber(json, "total"); + if(number != null) this.total = number.longValue(); + collectStreamsFrom(collector, json, pageUrl); + } else { + throw new ExtractionException("Unable to get peertube kiosk info"); + } + return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl)); + } + + @Override + public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { + String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; + this.initPage = getPage(pageUrl); + } + + private String getNextPageUrl(String prevPageUrl) { + String prevStart; + try { + prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl); + } catch (RegexException e) { + return ""; + } + if(StringUtil.isBlank(prevStart)) return ""; + long nextStart = 0; + try { + nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE; + } catch (NumberFormatException e) { + return ""; + } + + if(nextStart >= total) { + return ""; + }else { + return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart)); + } + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java new file mode 100644 index 000000000..640f99e59 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java @@ -0,0 +1,41 @@ +package org.schabi.newpipe.extractor.services.peertube.linkHandler; + +import java.util.List; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Parser; + +public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { + + private static final PeertubeChannelLinkHandlerFactory instance = new PeertubeChannelLinkHandlerFactory(); + private static final String ID_PATTERN = "/accounts/([^/?&#]*)"; + private static final String ACCOUNTS_ENDPOINT = "/api/v1/accounts/"; + + public static PeertubeChannelLinkHandlerFactory getInstance() { + return instance; + } + + @Override + public String getId(String url) throws ParsingException { + return Parser.matchGroup1(ID_PATTERN, url); + } + + @Override + public String getUrl(String id, List contentFilters, String searchFilter) throws ParsingException { + String baseUrl = ServiceList.PeerTube.getBaseUrl(); + return getUrl(id, contentFilters, searchFilter, baseUrl); + } + + @Override + public String getUrl(String id, List contentFilter, String sortFilter, String baseUrl) + throws ParsingException { + return baseUrl + ACCOUNTS_ENDPOINT + id; + } + + @Override + public boolean onAcceptUrl(String url) { + return url.contains("/accounts/"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java new file mode 100644 index 000000000..9faa3492b --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java @@ -0,0 +1,42 @@ +package org.schabi.newpipe.extractor.services.peertube.linkHandler; + +import java.util.List; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.FoundAdException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Parser; + +public class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory { + + private static final PeertubeCommentsLinkHandlerFactory instance = new PeertubeCommentsLinkHandlerFactory(); + private static final String ID_PATTERN = "/videos/(watch/)?([^/?&#]*)"; + private static final String COMMENTS_ENDPOINT = "/api/v1/videos/%s/comment-threads"; + + public static PeertubeCommentsLinkHandlerFactory getInstance() { + return instance; + } + + @Override + public String getId(String url) throws ParsingException, IllegalArgumentException { + return Parser.matchGroup(ID_PATTERN, url, 2); + } + + @Override + public boolean onAcceptUrl(final String url) throws FoundAdException { + return url.contains("/videos/"); + } + + @Override + public String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException { + String baseUrl = ServiceList.PeerTube.getBaseUrl(); + return getUrl(id, contentFilter, sortFilter, baseUrl); + } + + @Override + public String getUrl(String id, List contentFilter, String sortFilter, String baseUrl) throws ParsingException { + return baseUrl + String.format(COMMENTS_ENDPOINT, id); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java new file mode 100644 index 000000000..64e291a54 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java @@ -0,0 +1,42 @@ +package org.schabi.newpipe.extractor.services.peertube.linkHandler; + + +import java.util.List; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Parser; + +public class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { + + private static final PeertubePlaylistLinkHandlerFactory instance = new PeertubePlaylistLinkHandlerFactory(); + private static final String ID_PATTERN = "/video-channels/([^/?&#]*)"; + private static final String VIDEO_CHANNELS_ENDPOINT = "/api/v1/video-channels/"; + + public static PeertubePlaylistLinkHandlerFactory getInstance() { + return instance; + } + + @Override + public String getUrl(String id, List contentFilters, String sortFilter) { + String baseUrl = ServiceList.PeerTube.getBaseUrl(); + return getUrl(id, contentFilters, sortFilter, baseUrl); + } + + @Override + public String getUrl(String id, List contentFilters, String sortFilter, String baseUrl) { + return baseUrl + VIDEO_CHANNELS_ENDPOINT + id; + } + + @Override + public String getId(String url) throws ParsingException { + return Parser.matchGroup1(ID_PATTERN, url); + } + + + @Override + public boolean onAcceptUrl(final String url) { + return url.contains("/video-channels/"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java new file mode 100644 index 000000000..695911670 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java @@ -0,0 +1,43 @@ +package org.schabi.newpipe.extractor.services.peertube.linkHandler; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.List; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; + +public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory { + + public static final String CHARSET_UTF_8 = "UTF-8"; + public static final String VIDEOS = "videos"; + public static final String SEARCH_ENDPOINT = "/api/v1/search/videos"; + + public static PeertubeSearchQueryHandlerFactory getInstance() { + return new PeertubeSearchQueryHandlerFactory(); + } + + @Override + public String getUrl(String searchString, List contentFilters, String sortFilter) throws ParsingException { + String baseUrl = ServiceList.PeerTube.getBaseUrl(); + return getUrl(searchString, contentFilters, sortFilter, baseUrl); + } + + @Override + public String getUrl(String searchString, List contentFilters, String sortFilter, String baseUrl) throws ParsingException { + try { + final String url = baseUrl + SEARCH_ENDPOINT + + "?search=" + URLEncoder.encode(searchString, CHARSET_UTF_8); + + return url; + } catch (UnsupportedEncodingException e) { + throw new ParsingException("Could not encode query", e); + } + } + + @Override + public String[] getAvailableContentFilter() { + return new String[] { VIDEOS }; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java new file mode 100644 index 000000000..3ee7744cb --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java @@ -0,0 +1,42 @@ +package org.schabi.newpipe.extractor.services.peertube.linkHandler; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.FoundAdException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Parser; + +public class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory { + + private static final PeertubeStreamLinkHandlerFactory instance = new PeertubeStreamLinkHandlerFactory(); + private static final String ID_PATTERN = "/videos/(watch/)?([^/?&#]*)"; + private static final String VIDEO_ENDPOINT = "/api/v1/videos/"; + + private PeertubeStreamLinkHandlerFactory() { + } + + public static PeertubeStreamLinkHandlerFactory getInstance() { + return instance; + } + + @Override + public String getUrl(String id) { + String baseUrl = ServiceList.PeerTube.getBaseUrl(); + return getUrl(id, baseUrl); + } + + @Override + public String getUrl(String id, String baseUrl) { + return baseUrl + VIDEO_ENDPOINT + id; + } + + @Override + public String getId(String url) throws ParsingException, IllegalArgumentException { + return Parser.matchGroup(ID_PATTERN, url, 2); + } + + @Override + public boolean onAcceptUrl(final String url) throws FoundAdException { + return url.contains("/videos/"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java new file mode 100644 index 000000000..c0773c202 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java @@ -0,0 +1,77 @@ +package org.schabi.newpipe.extractor.services.peertube.linkHandler; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; + +public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { + + + private static final PeertubeTrendingLinkHandlerFactory instance = new PeertubeTrendingLinkHandlerFactory(); + + public static final Map KIOSK_MAP; + public static final Map REVERSE_KIOSK_MAP; + public static final String KIOSK_TRENDING = "Trending"; + public static final String KIOSK_MOST_LIKED = "Most liked"; + public static final String KIOSK_RECENT = "Recently added"; + public static final String KIOSK_LOCAL = "Local"; + + static { + Map map = new HashMap<>(); + map.put(KIOSK_TRENDING, "%s/api/v1/videos?sort=-trending"); + map.put(KIOSK_MOST_LIKED, "%s/api/v1/videos?sort=-likes"); + map.put(KIOSK_RECENT, "%s/api/v1/videos?sort=-publishedAt"); + map.put(KIOSK_LOCAL, "%s/api/v1/videos?sort=-publishedAt&filter=local"); + KIOSK_MAP = Collections.unmodifiableMap(map); + + Map reverseMap = new HashMap<>(); + for(Map.Entry entry : KIOSK_MAP.entrySet()){ + reverseMap.put(entry.getValue(), entry.getKey()); + } + REVERSE_KIOSK_MAP = Collections.unmodifiableMap(reverseMap); + } + + public static PeertubeTrendingLinkHandlerFactory getInstance() { + return instance; + } + + @Override + public String getUrl(String id, List contentFilters, String sortFilter) { + String baseUrl = ServiceList.PeerTube.getBaseUrl(); + return getUrl(id, contentFilters, sortFilter, baseUrl); + } + + @Override + public String getUrl(String id, List contentFilters, String sortFilter, String baseUrl) { + return String.format(KIOSK_MAP.get(id), baseUrl); + } + + @Override + public String getId(String url) throws ParsingException { + String baseUrl = ServiceList.PeerTube.getBaseUrl(); + url = url.replace(baseUrl, "%s"); + if (url.contains("/videos/trending")) { + return KIOSK_TRENDING; + } else if (url.contains("/videos/most-liked")) { + return KIOSK_MOST_LIKED; + } else if (url.contains("/videos/recently-added")) { + return KIOSK_RECENT; + } else if (url.contains("/videos/local")) { + return KIOSK_LOCAL; + } else if (REVERSE_KIOSK_MAP.containsKey(url)) { + return REVERSE_KIOSK_MAP.get(url); + } else { + throw new ParsingException("no id found for this url"); + } + } + + @Override + public boolean onAcceptUrl(final String url) { + return url.contains("/videos?") || url.contains("/videos/trending") || url.contains("/videos/most-liked") || url.contains("/videos/recently-added") || url.contains("/videos/local"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index b480baff8..def9c3860 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -1,25 +1,35 @@ package org.schabi.newpipe.extractor.services.soundcloud; +import static java.util.Collections.singletonList; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; -import org.schabi.newpipe.extractor.linkhandler.*; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; -import static java.util.Collections.singletonList; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; - public class SoundcloudService extends StreamingService { public SoundcloudService(int id) { super(id, "SoundCloud", singletonList(AUDIO)); } + + @Override + public String getBaseUrl() { + return "https://soundcloud.com"; + } @Override public SearchQueryHandlerFactory getSearchQHFactory() { @@ -110,5 +120,5 @@ public class SoundcloudService extends StreamingService { throws ExtractionException { return null; } - + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index ec728e53e..78c97cfbe 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -1,27 +1,47 @@ package org.schabi.newpipe.extractor.services.youtube; +import static java.util.Arrays.asList; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; + +import java.util.List; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; -import org.schabi.newpipe.extractor.linkhandler.*; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.*; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.*; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscriptionExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSuggestionExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; -import java.util.List; - -import static java.util.Arrays.asList; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.*; - /* * Created by Christian Schabesberger on 23.08.15. * @@ -48,6 +68,11 @@ public class YoutubeService extends StreamingService { super(id, "YouTube", asList(AUDIO, VIDEO, LIVE, COMMENTS)); } + @Override + public String getBaseUrl() { + return "https://youtube.com"; + } + @Override public LinkHandlerFactory getStreamLHFactory() { return YoutubeStreamLinkHandlerFactory.getInstance(); @@ -173,7 +198,6 @@ public class YoutubeService extends StreamingService { return SUPPORTED_LANGUAGES; } - @Override public List getSupportedCountries() { return SUPPORTED_COUNTRIES; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java index 80208be63..1d49e2535 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java @@ -1,13 +1,17 @@ package org.schabi.newpipe.extractor.stream; -import org.schabi.newpipe.extractor.MediaFormat; - import java.io.Serializable; import java.util.List; +import org.schabi.newpipe.extractor.MediaFormat; + +/** + * Creates a stream object from url, format and optional torrent url + */ public abstract class Stream implements Serializable { private final MediaFormat mediaFormat; public final String url; + public final String torrentUrl; /** * @deprecated Use {@link #getFormat()} or {@link #getFormatId()} @@ -15,8 +19,26 @@ public abstract class Stream implements Serializable { @Deprecated public final int format; + /** + * Instantiates a new stream object. + * + * @param url the url + * @param format the format + */ public Stream(String url, MediaFormat format) { + this(url, null, format); + } + + /** + * Instantiates a new stream object. + * + * @param url the url + * @param torrentUrl the url to torrent file, example https://webtorrent.io/torrents/big-buck-bunny.torrent + * @param format the format + */ + public Stream(String url, String torrentUrl, MediaFormat format) { this.url = url; + this.torrentUrl = torrentUrl; this.format = format.id; this.mediaFormat = format; } @@ -46,14 +68,38 @@ public abstract class Stream implements Serializable { return false; } + /** + * Gets the url. + * + * @return the url + */ public String getUrl() { return url; } + + /** + * Gets the torrent url. + * + * @return the torrent url, example https://webtorrent.io/torrents/big-buck-bunny.torrent + */ + public String getTorrentUrl() { + return torrentUrl; + } + /** + * Gets the format. + * + * @return the format + */ public MediaFormat getFormat() { return mediaFormat; } + /** + * Gets the format id. + * + * @return the format id + */ public int getFormatId() { return mediaFormat.id; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java index 1e877bee3..f4531d365 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java @@ -36,6 +36,16 @@ public class VideoStream extends Stream { this.resolution = resolution; this.isVideoOnly = isVideoOnly; } + + public VideoStream(String url, String torrentUrl, MediaFormat format, String resolution) { + this(url, torrentUrl, format, resolution, false); + } + + public VideoStream(String url, String torrentUrl, MediaFormat format, String resolution, boolean isVideoOnly) { + super(url, torrentUrl, format); + this.resolution = resolution; + this.isVideoOnly = isVideoOnly; + } @Override public boolean equalStats(Stream cmp) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 1f8da13b2..fd06d40f5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -144,9 +144,9 @@ public class Utils { try { return new URL(url); } catch (MalformedURLException e) { - // if no protocol is given try prepending "http://" + // if no protocol is given try prepending "https://" if (e.getMessage().equals("no protocol: " + url)) { - return new URL(HTTP + url); + return new URL(HTTPS + url); } throw e; @@ -175,4 +175,15 @@ public class Utils { } return s; } + + public static String getBaseUrl(String url) throws ParsingException { + URL uri; + try { + uri = stringToURL(url); + } catch (MalformedURLException e) { + throw new ParsingException("Malformed url: " + url, e); + } + return uri.getProtocol() + "://" + uri.getAuthority(); + } + } \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java new file mode 100644 index 000000000..3d8bdff33 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -0,0 +1,210 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; + +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor; + +/** + * Test for {@link PeertubeChannelExtractor} + */ +public class PeertubeChannelExtractorTest { + public static class KDE implements BaseChannelExtractorTest { + private static PeertubeChannelExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + // setting instance might break test when running in parallel + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); + extractor = (PeertubeChannelExtractor) PeerTube + .getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/kde"); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(PeerTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() throws ParsingException { + assertEquals("The KDE Community", extractor.getName()); + } + + @Test + public void testId() throws ParsingException { + assertEquals("kde", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/accounts/kde", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor, PeerTube.getServiceId()); + } + + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor, PeerTube.getServiceId()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ChannelExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testDescription() throws ParsingException { + assertNotNull(extractor.getDescription()); + } + + @Test + public void testAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getAvatarUrl()); + } + + @Ignore + @Test + public void testBannerUrl() throws ParsingException { + assertIsSecureUrl(extractor.getBannerUrl()); + } + + @Test + public void testFeedUrl() throws ParsingException { + assertEmpty(extractor.getFeedUrl()); + } + + @Test + public void testSubscriberCount() throws ParsingException { + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5); + } + } + + public static class Booteille implements BaseChannelExtractorTest { + private static PeertubeChannelExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + // setting instance might break test when running in parallel + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); + extractor = (PeertubeChannelExtractor) PeerTube + .getChannelExtractor("https://peertube.mastodon.host/accounts/booteille"); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Additional Testing + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testGetPageInNewExtractor() throws Exception { + final ChannelExtractor newExtractor = PeerTube.getChannelExtractor(extractor.getUrl()); + defaultTestGetPageInNewExtractor(extractor, newExtractor, PeerTube.getServiceId()); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(PeerTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() throws ParsingException { + assertEquals("booteille", extractor.getName()); + } + + @Test + public void testId() throws ParsingException { + assertEquals("booteille", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor, PeerTube.getServiceId()); + } + + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor, PeerTube.getServiceId()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ChannelExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testDescription() throws ParsingException { + assertNotNull(extractor.getDescription()); + } + + @Test + public void testAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getAvatarUrl()); + } + + @Ignore + @Test + public void testBannerUrl() throws ParsingException { + assertIsSecureUrl(extractor.getBannerUrl()); + } + + @Test + public void testFeedUrl() throws ParsingException { + assertEmpty(extractor.getFeedUrl()); + } + + @Test + public void testSubscriberCount() throws ParsingException { + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 2); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java new file mode 100644 index 000000000..d90e4321b --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java @@ -0,0 +1,36 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; + +/** + * Test for {@link PeertubeChannelLinkHandlerFactory} + */ +public class PeertubeChannelLinkHandlerFactoryTest { + + private static PeertubeChannelLinkHandlerFactory linkHandler; + + @BeforeClass + public static void setUp() { + linkHandler = PeertubeChannelLinkHandlerFactory.getInstance(); + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void acceptrUrlTest() throws ParsingException { + assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net")); + } + + @Test + public void getIdFromUrl() throws ParsingException { + assertEquals("kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net").getId()); + assertEquals("kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java new file mode 100644 index 000000000..fef1c3ee5 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java @@ -0,0 +1,92 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + +import java.io.IOException; +import java.util.List; + +import org.jsoup.helper.StringUtil; +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.comments.CommentsInfo; +import org.schabi.newpipe.extractor.comments.CommentsInfoItem; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor; + +public class PeertubeCommentsExtractorTest { + + private static PeertubeCommentsExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (PeertubeCommentsExtractor) PeerTube + .getCommentsExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a"); + } + + @Test + public void testGetComments() throws IOException, ExtractionException { + boolean result = false; + InfoItemsPage comments = extractor.getInitialPage(); + result = findInComments(comments, "@root A great documentary on a great guy."); + + while (comments.hasNextPage() && !result) { + comments = extractor.getPage(comments.getNextPageUrl()); + result = findInComments(comments, "@root A great documentary on a great guy."); + } + + assertTrue(result); + } + + @Test + public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { + boolean result = false; + CommentsInfo commentsInfo = CommentsInfo.getInfo("https://peertube.mastodon.host/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828"); + assertTrue("Comments".equals(commentsInfo.getName())); + result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!"); + + String nextPage = commentsInfo.getNextPageUrl(); + while (!StringUtil.isBlank(nextPage) && !result) { + InfoItemsPage moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage); + result = findInComments(moreItems.getItems(), "Loved it!!!"); + nextPage = moreItems.getNextPageUrl(); + } + + assertTrue(result); + } + + @Test + public void testGetCommentsAllData() throws IOException, ExtractionException { + InfoItemsPage comments = extractor.getInitialPage(); + for(CommentsInfoItem c: comments.getItems()) { + assertFalse(StringUtil.isBlank(c.getAuthorEndpoint())); + assertFalse(StringUtil.isBlank(c.getAuthorName())); + assertFalse(StringUtil.isBlank(c.getAuthorThumbnail())); + assertFalse(StringUtil.isBlank(c.getCommentId())); + assertFalse(StringUtil.isBlank(c.getCommentText())); + assertFalse(StringUtil.isBlank(c.getName())); + assertFalse(StringUtil.isBlank(c.getTextualPublishedTime())); + assertFalse(StringUtil.isBlank(c.getThumbnailUrl())); + assertFalse(StringUtil.isBlank(c.getUrl())); + assertFalse(c.getLikeCount() != -1); + } + } + + private boolean findInComments(InfoItemsPage comments, String comment) { + return findInComments(comments.getItems(), comment); + } + + private boolean findInComments(List comments, String comment) { + for(CommentsInfoItem c: comments) { + if(c.getCommentText().contains(comment)) { + return true; + } + } + return false; + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java new file mode 100644 index 000000000..c13150302 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java @@ -0,0 +1,36 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory; + +/** + * Test for {@link PeertubeCommentsLinkHandlerFactory} + */ +public class PeertubeCommentsLinkHandlerFactoryTest { + + private static PeertubeCommentsLinkHandlerFactory linkHandler; + + @BeforeClass + public static void setUp() { + linkHandler = PeertubeCommentsLinkHandlerFactory.getInstance(); + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void acceptrUrlTest() throws ParsingException { + assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/api/v1/videos/19319/comment-threads?start=0&count=10&sort=-createdAt")); + } + + @Test + public void getIdFromUrl() throws ParsingException { + assertEquals("19319", linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/videos/19319/comment-threads").getId()); + assertEquals("19319", linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/videos/19319/comment-threads?start=0&count=10&sort=-createdAt").getId()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java new file mode 100644 index 000000000..77e7d2e84 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java @@ -0,0 +1,36 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubePlaylistLinkHandlerFactory; + +/** + * Test for {@link PeertubePlaylistLinkHandlerFactory} + */ +public class PeertubePlaylistLinkHandlerFactoryTest { + + private static PeertubePlaylistLinkHandlerFactory linkHandler; + + @BeforeClass + public static void setUp() { + linkHandler = PeertubePlaylistLinkHandlerFactory.getInstance(); + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void acceptrUrlTest() throws ParsingException { + assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/video-channels/b45e84fb-c47f-475b-94f2-718126154d33/videos")); + } + + @Test + public void getIdFromUrl() throws ParsingException { + assertEquals("b45e84fb-c47f-475b-94f2-718126154d33", linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/b45e84fb-c47f-475b-94f2-718126154d33").getId()); + assertEquals("b45e84fb-c47f-475b-94f2-718126154d33", linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/b45e84fb-c47f-475b-94f2-718126154d33/videos").getId()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java new file mode 100644 index 000000000..75a692d42 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -0,0 +1,125 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static java.util.Objects.requireNonNull; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; + +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; + +/** + * Test for {@link StreamExtractor} + */ +public class PeertubeStreamExtractorDefaultTest { + private static PeertubeStreamExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + // setting instance might break test when running in parallel + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); + extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/afe5bf12-c58b-4efd-b56e-29c5a59e04bc"); + extractor.fetchPage(); + } + + @Test + public void testGetInvalidTimeStamp() throws ParsingException { + assertTrue(extractor.getTimeStamp() + "", + extractor.getTimeStamp() <= 0); + } + + @Test + public void testGetTitle() throws ParsingException { + assertEquals(extractor.getName(), "Power Corrupts the Best"); + } + + @Test + public void testGetDescription() throws ParsingException { + assertEquals(extractor.getDescription(), "A short reading from Bakunin, made for the group Audible Anarchist https://audibleanarchist.github.io/Webpage/"); + } + + @Test + public void testGetUploaderName() throws ParsingException { + assertEquals(extractor.getUploaderName(), "Rowsedower"); + } + + @Test + public void testGetLength() throws ParsingException { + assertEquals(extractor.getLength(), 269); + } + + @Test + public void testGetViewCount() throws ParsingException { + assertTrue(Long.toString(extractor.getViewCount()), + extractor.getViewCount() > 10); + } + + @Test + public void testGetUploadDate() throws ParsingException, ParseException { + final Calendar instance = Calendar.getInstance(); + instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-09-30T14:08:24.378Z")); + assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); + + } + + @Test + public void testGetUploaderUrl() throws ParsingException { + assertIsSecureUrl(extractor.getUploaderUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/accounts/reddebrek@peertube.mastodon.host", extractor.getUploaderUrl()); + } + + @Test + public void testGetThumbnailUrl() throws ParsingException { + assertIsSecureUrl(extractor.getThumbnailUrl()); + } + + @Test + public void testGetUploaderAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + } + + @Test + public void testGetVideoStreams() throws IOException, ExtractionException { + assertFalse(extractor.getVideoStreams().isEmpty()); + } + + @Test + public void testStreamType() throws ParsingException { + assertTrue(extractor.getStreamType() == StreamType.VIDEO_STREAM); + } + + @Ignore + @Test + public void testGetRelatedVideos() throws ExtractionException, IOException { + StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); + assertFalse(relatedVideos.getItems().isEmpty()); + assertTrue(relatedVideos.getErrors().isEmpty()); + } + + @Test + public void testGetSubtitlesListDefault() throws IOException, ExtractionException { + assertTrue(extractor.getSubtitlesDefault().isEmpty()); + } + + @Test + public void testGetSubtitlesList() throws IOException, ExtractionException { + assertTrue(extractor.getSubtitlesDefault().isEmpty()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java new file mode 100644 index 000000000..893b6eb8e --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java @@ -0,0 +1,37 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory; + +/** + * Test for {@link PeertubeStreamLinkHandlerFactory} + */ +public class PeertubeStreamLinkHandlerFactoryTest { + private static PeertubeStreamLinkHandlerFactory linkHandler; + + @BeforeClass + public static void setUp() throws Exception { + linkHandler = PeertubeStreamLinkHandlerFactory.getInstance(); + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void getId() throws Exception { + assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60", linkHandler.fromUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60").getId()); + assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60", linkHandler.fromUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa").getId()); + } + + + @Test + public void testAcceptUrl() throws ParsingException { + assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60")); + assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa")); + } +} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java new file mode 100644 index 000000000..bb4e1529b --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java @@ -0,0 +1,97 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + +import java.util.List; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; + +/** + * Test for {@link PeertubeTrendingExtractor} + */ +public class PeertubeTrendingExtractorTest { + + static KioskExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + // setting instance might break test when running in parallel + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); + extractor = PeerTube + .getKioskList() + .getExtractorById("Trending", null); + extractor.fetchPage(); + } + + @Test + public void testGetDownloader() throws Exception { + assertNotNull(NewPipe.getDownloader()); + } + + @Test + public void testGetName() throws Exception { + assertEquals(extractor.getName(), "Trending"); + } + + @Test + public void testId() { + assertEquals(extractor.getId(), "Trending"); + } + + @Test + public void testGetStreams() throws Exception { + ListExtractor.InfoItemsPage page = extractor.getInitialPage(); + if(!page.getErrors().isEmpty()) { + System.err.println("----------"); + List errors = page.getErrors(); + for(Throwable e: errors) { + e.printStackTrace(); + System.err.println("----------"); + } + } + assertTrue("no streams are received", + !page.getItems().isEmpty() + && page.getErrors().isEmpty()); + } + + @Test + public void testGetStreamsErrors() throws Exception { + assertTrue("errors during stream list extraction", extractor.getInitialPage().getErrors().isEmpty()); + } + + @Test + public void testHasMoreStreams() throws Exception { + // Setup the streams + extractor.getInitialPage(); + assertTrue("has more streams", extractor.hasNextPage()); + } + + @Test + public void testGetNextPageUrl() throws Exception { + assertTrue(extractor.hasNextPage()); + } + + @Test + public void testGetNextPage() throws Exception { + extractor.getInitialPage().getItems(); + assertFalse("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null + || extractor.getPage(extractor.getNextPageUrl()).getItems().isEmpty()); + } + + @Test + public void testGetCleanUrl() throws Exception { + assertEquals(extractor.getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-trending"); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java new file mode 100644 index 000000000..d192e8be3 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java @@ -0,0 +1,61 @@ +package org.schabi.newpipe.extractor.services.peertube; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory; + +/** + * Test for {@link PeertubeTrendingLinkHandlerFactory} + */ +public class PeertubeTrendingLinkHandlerFactoryTest { + private static LinkHandlerFactory LinkHandlerFactory; + + @BeforeClass + public static void setUp() throws Exception { + // setting instance might break test when running in parallel + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); + LinkHandlerFactory = new PeertubeTrendingLinkHandlerFactory(); + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void getUrl() + throws Exception { + assertEquals(LinkHandlerFactory.fromId("Trending").getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-trending"); + assertEquals(LinkHandlerFactory.fromId("Most liked").getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-likes"); + assertEquals(LinkHandlerFactory.fromId("Recently added").getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-publishedAt"); + assertEquals(LinkHandlerFactory.fromId("Local").getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-publishedAt&filter=local"); + } + + @Test + public void getId() + throws Exception { + assertEquals(LinkHandlerFactory.fromUrl("https://peertube.mastodon.host/videos/trending").getId(), "Trending"); + assertEquals(LinkHandlerFactory.fromUrl("https://peertube.mastodon.host/videos/most-liked").getId(), "Most liked"); + assertEquals(LinkHandlerFactory.fromUrl("https://peertube.mastodon.host/videos/recently-added").getId(), "Recently added"); + assertEquals(LinkHandlerFactory.fromUrl("https://peertube.mastodon.host/videos/local").getId(), "Local"); + } + + @Test + public void acceptUrl() throws ParsingException { + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/trending")); + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/trending?adsf=fjaj#fhe")); + + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/most-liked")); + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/most-liked?adsf=fjaj#fhe")); + + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/recently-added")); + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/recently-added?adsf=fjaj#fhe")); + + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/local")); + assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/local?adsf=fjaj#fhe")); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java new file mode 100644 index 000000000..08dd7a0a9 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java @@ -0,0 +1,28 @@ +package org.schabi.newpipe.extractor.services.peertube.search; + +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; + +/** + * Test for {@link PeertubeSearchExtractor} + */ +public abstract class PeertubeSearchExtractorBaseTest { + + protected static PeertubeSearchExtractor extractor; + protected static ListExtractor.InfoItemsPage itemsPage; + + @Test + public void testResultListElementsLength() { + assertTrue(Integer.toString(itemsPage.getItems().size()), + itemsPage.getItems().size() >= 3); + } + + @Test + public void testUrl() throws Exception { + assertTrue(extractor.getUrl(), extractor.getUrl().startsWith("https://peertube.mastodon.host/api/v1/search/videos")); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java new file mode 100644 index 000000000..dd9c6deb3 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java @@ -0,0 +1,91 @@ +package org.schabi.newpipe.extractor.services.peertube.search; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; + +/** + * Test for {@link PeertubeSearchExtractor} + */ +public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorBaseTest { + + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + // setting instance might break test when running in parallel + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); + extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("kde"); + extractor.fetchPage(); + itemsPage = extractor.getInitialPage(); + } + + @Test + public void testGetSecondPageUrl() throws Exception { + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=kde&start=12&count=12", extractor.getNextPageUrl()); + } + + @Test + public void testResultList_FirstElement() { + InfoItem firstInfoItem = itemsPage.getItems().get(0); + + assertTrue("search does not match", firstInfoItem.getName().toLowerCase().contains("kde")); + } + + @Test + public void testResultListCheckIfContainsStreamItems() { + boolean hasStreams = false; + for(InfoItem item : itemsPage.getItems()) { + if(item instanceof StreamInfoItem) { + hasStreams = true; + } + } + assertTrue("Has no InfoItemStreams", hasStreams); + } + + @Test + public void testGetSecondPage() throws Exception { + extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("internet"); + itemsPage = extractor.getInitialPage(); + PeertubeSearchExtractor secondExtractor = + (PeertubeSearchExtractor) PeerTube.getSearchExtractor("internet"); + ListExtractor.InfoItemsPage secondPage = secondExtractor.getPage(itemsPage.getNextPageUrl()); + assertTrue(Integer.toString(secondPage.getItems().size()), + secondPage.getItems().size() >= 10); + + // check if its the same result + boolean equals = true; + for (int i = 0; i < secondPage.getItems().size() + && i < itemsPage.getItems().size(); i++) { + if(!secondPage.getItems().get(i).getUrl().equals( + itemsPage.getItems().get(i).getUrl())) { + equals = false; + } + } + assertFalse("First and second page are equal", equals); + + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=internet&start=24&count=12", + secondPage.getNextPageUrl()); + } + + + @Test + public void testId() throws Exception { + assertEquals("kde", extractor.getId()); + } + + @Test + public void testName() { + assertEquals("kde", extractor.getName()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java new file mode 100644 index 000000000..23e3100ce --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java @@ -0,0 +1,26 @@ +package org.schabi.newpipe.extractor.services.peertube.search; + +import static org.junit.Assert.assertEquals; +import static org.schabi.newpipe.extractor.ServiceList.PeerTube; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance; + +public class PeertubeSearchQHTest { + + @BeforeClass + public static void setUpClass() throws Exception { + // setting instance might break test when running in parallel + PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); + } + + @Test + public void testRegularValues() throws Exception { + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=asdf", PeerTube.getSearchQHFactory().fromQuery("asdf").getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=hans",PeerTube.getSearchQHFactory().fromQuery("hans").getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=Poifj%26jaijf", PeerTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=G%C3%BCl%C3%BCm", PeerTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); + assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); + } +} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 36c96c264..251ae246e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -1,5 +1,14 @@ package org.schabi.newpipe.extractor.services.youtube; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems; +import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems; + import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -11,11 +20,6 @@ import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; -import static org.junit.Assert.*; -import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; -import static org.schabi.newpipe.extractor.services.DefaultTests.*; - /** * Test for {@link ChannelExtractor} */