lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #145 from yausername/peertube 

Basic PeerTube support (without P2P)
This commit is contained in:
Tobias Groza 2019-12-02 23:08:52 +01:00 committed by GitHub
parent b6d3252290 00c23683b1
commit 43b54ccc3c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
46 changed files with 2890 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java
View File

@ -1,5 +1,10 @@
package org.schabi.newpipe.extractor;
import java.io.IOException;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -8,11 +13,6 @@ import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.Serializable;
public abstract class Extractor{
/**
* {@link StreamingService} currently related to this extractor.<br>
@ -93,6 +93,11 @@ public abstract class Extractor{
public String getUrl() throws ParsingException {
return linkHandler.getUrl();
}
@Nonnull
public String getBaseUrl() throws ParsingException {
return linkHandler.getBaseUrl();
}
@Nonnull
public StreamingService getService() {

8
extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java
View File

@ -126,6 +126,13 @@ public enum MediaFormat {
return null;
}
public static MediaFormat getFromSuffix(String suffix) {
for (MediaFormat vf: values()) {
if (vf.suffix.equals(suffix)) return vf;
}
return null;
}
/**
* Get the name of the format
* @return the name of the format
@ -149,4 +156,5 @@ public enum MediaFormat {
public String getMimeType() {
return mimeType;
}
}

20
extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java
View File

@ -1,13 +1,13 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.services.media_ccc.MediaCCCService;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService;
import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static java.util.Arrays.asList;
import static java.util.Collections.unmodifiableList;
import org.schabi.newpipe.extractor.services.media_ccc.MediaCCCService;
import org.schabi.newpipe.extractor.services.peertube.PeertubeService;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService;
import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
/*
* Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
@ -38,16 +38,18 @@ public final class ServiceList {
public static final YoutubeService YouTube;
public static final SoundcloudService SoundCloud;
public static final MediaCCCService MediaCCC;
public static final PeertubeService PeerTube;
/**
* When creating a new service, put this service in the end of this list,
* and give it the next free id.
*/
private static final List<StreamingService> SERVICES = unmodifiableList(
asList(
private static final List<StreamingService> SERVICES = Collections.unmodifiableList(
Arrays.asList(
YouTube = new YoutubeService(0),
SoundCloud = new SoundcloudService(1),
MediaCCC = new MediaCCCService(2)
MediaCCC = new MediaCCCService(2),
PeerTube = new PeertubeService(3)
));
/**

20
extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java
View File

@ -1,11 +1,19 @@
package org.schabi.newpipe.extractor;
import java.util.Collections;
import java.util.List;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.*;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
@ -16,9 +24,6 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.util.Collections;
import java.util.List;
/*
* Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
* StreamingService.java is part of NewPipe.
@ -44,6 +49,7 @@ public abstract class StreamingService {
*/
public static class ServiceInfo {
private final String name;
private final List<MediaCapability> mediaCapabilities;
/**
@ -59,7 +65,7 @@ public abstract class StreamingService {
public String getName() {
return name;
}
public List<MediaCapability> getMediaCapabilities() {
return mediaCapabilities;
}
@ -110,6 +116,8 @@ public abstract class StreamingService {
public String toString() {
return serviceId + ":" + serviceInfo.getName();
}
public abstract String getBaseUrl();
/*//////////////////////////////////////////////////////////////////////////
// Url Id handler
@ -250,7 +258,7 @@ public abstract class StreamingService {
}
return getCommentsExtractor(llhf.fromUrl(url));
}
/*//////////////////////////////////////////////////////////////////////////
// Utils
//////////////////////////////////////////////////////////////////////////*/

2
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java
View File

@ -118,7 +118,7 @@ public class KioskList {
for(Map.Entry<String, KioskEntry> e : kioskList.entrySet()) {
KioskEntry ke = e.getValue();
if(ke.handlerFactory.acceptUrl(url)) {
return getExtractorById(e.getKey(), nextPageUrl, localization);
return getExtractorById(ke.handlerFactory.getId(url), nextPageUrl, localization);
}
}
throw new ExtractionException("Could not find a kiosk that fits to the url: " + url);

7
extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java
View File

@ -2,6 +2,9 @@ package org.schabi.newpipe.extractor.linkhandler;
import java.io.Serializable;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Utils;
public class LinkHandler implements Serializable {
protected final String originalUrl;
protected final String url;
@ -28,4 +31,8 @@ public class LinkHandler implements Serializable {
public String getId() {
return id;
}
public String getBaseUrl() throws ParsingException {
return Utils.getBaseUrl(url);
}
}

21
extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java
View File

@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.linkhandler;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Utils;
/*
* Created by Christian Schabesberger on 26.07.16.
@ -33,26 +34,41 @@ public abstract class LinkHandlerFactory {
public abstract String getUrl(String id) throws ParsingException;
public abstract boolean onAcceptUrl(final String url) throws ParsingException;
public String getUrl(String id, String baseUrl) throws ParsingException{
return getUrl(id);
}
///////////////////////////////////
// Logic
///////////////////////////////////
public LinkHandler fromUrl(String url) throws ParsingException {
final String baseUrl = Utils.getBaseUrl(url);
return fromUrl(url, baseUrl);
}
public LinkHandler fromUrl(String url, String baseUrl) throws ParsingException {
if(url == null) throw new IllegalArgumentException("url can not be null");
if(!acceptUrl(url)) {
throw new ParsingException("Malformed unacceptable url: " + url);
}
final String id = getId(url);
return new LinkHandler(url, getUrl(id), id);
return new LinkHandler(url, getUrl(id,baseUrl), id);
}
public LinkHandler fromId(String id) throws ParsingException {
if(id == null) throw new IllegalArgumentException("id can not be null");
final String url = getUrl(id);
return new LinkHandler(url, url, id);
}
public LinkHandler fromId(String id, String baseUrl) throws ParsingException {
if(id == null) throw new IllegalArgumentException("id can not be null");
final String url = getUrl(id, baseUrl);
return new LinkHandler(url, url, id);
}
/**
* When a VIEW_ACTION is caught this function will test if the url delivered within the calling
* Intent was meant to be watched with this Service.
@ -65,4 +81,5 @@ public abstract class LinkHandlerFactory {
throw fe;
}
}
}

39
extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java
View File

@ -1,10 +1,11 @@
package org.schabi.newpipe.extractor.linkhandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.util.ArrayList;
import java.util.List;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Utils;
public abstract class ListLinkHandlerFactory extends LinkHandlerFactory {
///////////////////////////////////
@ -14,23 +15,37 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory {
public List<String> getContentFilter(String url) throws ParsingException { return new ArrayList<>(0);}
public String getSortFilter(String url) throws ParsingException {return ""; }
public abstract String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException;
public String getUrl(String id, List<String> contentFilter, String sortFilter, String baseUrl) throws ParsingException {
return getUrl(id, contentFilter, sortFilter);
}
///////////////////////////////////
// Logic
///////////////////////////////////
@Override
public ListLinkHandler fromUrl(String url) throws ParsingException {
String baseUrl = Utils.getBaseUrl(url);
return fromUrl(url, baseUrl);
}
@Override
public ListLinkHandler fromUrl(String url, String baseUrl) throws ParsingException {
if(url == null) throw new IllegalArgumentException("url may not be null");
return new ListLinkHandler(super.fromUrl(url), getContentFilter(url), getSortFilter(url));
return new ListLinkHandler(super.fromUrl(url, baseUrl), getContentFilter(url), getSortFilter(url));
}
@Override
public ListLinkHandler fromId(String id) throws ParsingException {
return new ListLinkHandler(super.fromId(id), new ArrayList<String>(0), "");
}
@Override
public ListLinkHandler fromId(String id, String baseUrl) throws ParsingException {
return new ListLinkHandler(super.fromId(id, baseUrl), new ArrayList<String>(0), "");
}
public ListLinkHandler fromQuery(String id,
List<String> contentFilters,
@ -38,8 +53,15 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory {
final String url = getUrl(id, contentFilters, sortFilter);
return new ListLinkHandler(url, url, id, contentFilters, sortFilter);
}
public ListLinkHandler fromQuery(String id,
List<String> contentFilters,
String sortFilter, String baseUrl) throws ParsingException {
final String url = getUrl(id, contentFilters, sortFilter, baseUrl);
return new ListLinkHandler(url, url, id, contentFilters, sortFilter);
}
/**
* For makeing ListLinkHandlerFactory compatible with LinkHandlerFactory we need to override this,
* however it should not be overridden by the actual implementation.
@ -50,6 +72,11 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory {
return getUrl(id, new ArrayList<String>(0), "");
}
@Override
public String getUrl(String id, String baseUrl) throws ParsingException {
return getUrl(id, new ArrayList<String>(0), "", baseUrl);
}
/**
* Will returns content filter the corresponding extractor can handle like "channels", "videos", "music", etc.
*

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java
View File

@ -116,4 +116,9 @@ public class MediaCCCService extends StreamingService {
return null;
}
@Override
public String getBaseUrl() {
return "https://media.ccc.de";
}
}

63
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java Normal file
View File

@ -0,0 +1,63 @@
package org.schabi.newpipe.extractor.services.peertube;
import java.io.IOException;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
public class PeertubeInstance {
private final String url;
private String name;
public static final PeertubeInstance defaultInstance = new PeertubeInstance("https://framatube.org", "FramaTube");
public PeertubeInstance(String url) {
this.url = url;
this.name = "PeerTube";
}
public PeertubeInstance(String url , String name) {
this.url = url;
this.name = name;
}
public String getUrl() {
return url;
}
public void fetchInstanceMetaData() throws Exception {
Downloader downloader = NewPipe.getDownloader();
Response response = null;
try {
response = downloader.get(url + "/api/v1/config");
} catch (ReCaptchaException | IOException e) {
throw new Exception("unable to configure instance " + url, e);
}
if(null == response || StringUtil.isBlank(response.responseBody())) {
throw new Exception("unable to configure instance " + url);
}
try {
JsonObject json = JsonParser.object().from(response.responseBody());
this.name = JsonUtils.getString(json, "instance.name");
} catch (JsonParserException | ParsingException e) {
throw new Exception("unable to parse instance config", e);
}
}
public String getName() {
return name;
}
}

39
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java Normal file
View File

@ -0,0 +1,39 @@
package org.schabi.newpipe.extractor.services.peertube;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import com.grack.nanojson.JsonObject;
public class PeertubeParsingHelper {
private PeertubeParsingHelper() {
}
public static void validate(JsonObject json) throws ContentNotAvailableException {
String error = json.getString("error");
if(!StringUtil.isBlank(error)) {
throw new ContentNotAvailableException(error);
}
}
public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
Date date;
try {
date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse(textualUploadDate);
} catch (ParseException e) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
}
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
}
}

160
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java Normal file
View File

@ -0,0 +1,160 @@
package org.schabi.newpipe.extractor.services.peertube;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSuggestionExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
public class PeertubeService extends StreamingService {
private PeertubeInstance instance;
public PeertubeService(int id) {
this(id, PeertubeInstance.defaultInstance);
}
public PeertubeService(int id, PeertubeInstance instance) {
super(id, "PeerTube", asList(VIDEO, COMMENTS));
this.instance = instance;
}
@Override
public LinkHandlerFactory getStreamLHFactory() {
return PeertubeStreamLinkHandlerFactory.getInstance();
}
@Override
public ListLinkHandlerFactory getChannelLHFactory() {
return PeertubeChannelLinkHandlerFactory.getInstance();
}
@Override
public ListLinkHandlerFactory getPlaylistLHFactory() {
// TODO Auto-generated method stub
return null;
}
@Override
public SearchQueryHandlerFactory getSearchQHFactory() {
return PeertubeSearchQueryHandlerFactory.getInstance();
}
@Override
public ListLinkHandlerFactory getCommentsLHFactory() {
return PeertubeCommentsLinkHandlerFactory.getInstance();
}
@Override
public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) {
return new PeertubeSearchExtractor(this, queryHandler);
}
@Override
public SuggestionExtractor getSuggestionExtractor() {
return new PeertubeSuggestionExtractor(this);
}
@Override
public SubscriptionExtractor getSubscriptionExtractor() {
// TODO Auto-generated method stub
return null;
}
@Override
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler)
throws ExtractionException {
return new PeertubeChannelExtractor(this, linkHandler);
}
@Override
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler)
throws ExtractionException {
// TODO Auto-generated method stub
return null;
}
@Override
public StreamExtractor getStreamExtractor(LinkHandler linkHandler)
throws ExtractionException {
return new PeertubeStreamExtractor(this, linkHandler);
}
@Override
public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler)
throws ExtractionException {
return new PeertubeCommentsExtractor(this, linkHandler);
}
@Override
public String getBaseUrl() {
return instance.getUrl();
}
public PeertubeInstance getInstance() {
return this.instance;
}
public void setInstance(PeertubeInstance instance) {
this.instance = instance;
}
@Override
public KioskList getKioskList() throws ExtractionException {
KioskList.KioskExtractorFactory kioskFactory = new KioskList.KioskExtractorFactory() {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService,
String url,
String id)
throws ExtractionException {
return new PeertubeTrendingExtractor(PeertubeService.this,
new PeertubeTrendingLinkHandlerFactory().fromId(id), id);
}
};
KioskList list = new KioskList(this);
// add kiosks here e.g.:
final PeertubeTrendingLinkHandlerFactory h = new PeertubeTrendingLinkHandlerFactory();
try {
list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING);
list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_MOST_LIKED);
list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_RECENT);
list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_LOCAL);
list.setDefaultKiosk(PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING);
} catch (Exception e) {
throw new ExtractionException(e);
}
return list;
}
}

188
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java Normal file
View File

@ -0,0 +1,188 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import java.io.IOException;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
public class PeertubeChannelExtractor extends ChannelExtractor {
private static final String START_KEY = "start";
private static final String COUNT_KEY = "count";
private static final int ITEMS_PER_PAGE = 12;
private static final String START_PATTERN = "start=(\\d*)";
private InfoItemsPage<StreamInfoItem> initPage;
private long total;
private JsonObject json;
private final String baseUrl;
public PeertubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) throws ParsingException {
super(service, linkHandler);
this.baseUrl = getBaseUrl();
}
@Override
public String getAvatarUrl() throws ParsingException {
String value;
try {
value = JsonUtils.getString(json, "avatar.path");
}catch(Exception e) {
value = "/client/assets/images/default-avatar.png";
}
return baseUrl + value;
}
@Override
public String getBannerUrl() throws ParsingException {
return null;
}
@Override
public String getFeedUrl() throws ParsingException {
return null;
}
@Override
public long getSubscriberCount() throws ParsingException {
Number number = JsonUtils.getNumber(json, "followersCount");
return number.longValue();
}
@Override
public String getDescription() throws ParsingException {
try {
return JsonUtils.getString(json, "description");
}catch(ParsingException e) {
return "No description";
}
}
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
super.fetchPage();
return initPage;
}
private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException {
JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
}catch(Exception e) {
throw new ParsingException("unable to extract channel streams", e);
}
for(Object c: contents) {
if(c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
collector.commit(extractor);
}
}
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
super.fetchPage();
return initPage.getNextPageUrl();
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if(null != response && !StringUtil.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
throw new ParsingException("Could not parse json data for kiosk info", e);
}
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
if(json != null) {
PeertubeParsingHelper.validate(json);
Number number = JsonUtils.getNumber(json, "total");
if(number != null) this.total = number.longValue();
collectStreamsFrom(collector, json, pageUrl);
} else {
throw new ExtractionException("Unable to get peertube kiosk info");
}
return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl));
}
private String getNextPageUrl(String prevPageUrl) {
String prevStart;
try {
prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl);
} catch (RegexException e) {
return "";
}
if(StringUtil.isBlank(prevStart)) return "";
long nextStart = 0;
try {
nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE;
} catch (NumberFormatException e) {
return "";
}
if(nextStart >= total) {
return "";
}else {
return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart));
}
}
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
Response response = downloader.get(getUrl());
if(null != response && null != response.responseBody()) {
setInitialData(response.responseBody());
}else {
throw new ExtractionException("Unable to extract peertube channel data");
}
String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
this.initPage = getPage(pageUrl);
}
private void setInitialData(String responseBody) throws ExtractionException {
try {
json = JsonParser.object().from(responseBody);
} catch (JsonParserException e) {
throw new ExtractionException("Unable to extract peertube channel data", e);
}
if(null == json) throw new ExtractionException("Unable to extract peertube channel data");
}
@Override
public String getName() throws ParsingException {
return JsonUtils.getString(json, "displayName");
}
@Override
public String getOriginalUrl() throws ParsingException {
return baseUrl + "/accounts/" + getId();
}
}

123
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java Normal file
View File

@ -0,0 +1,123 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import java.io.IOException;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemsCollector;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
public class PeertubeCommentsExtractor extends CommentsExtractor {
private static final String START_KEY = "start";
private static final String COUNT_KEY = "count";
private static final int ITEMS_PER_PAGE = 12;
private static final String START_PATTERN = "start=(\\d*)";
private InfoItemsPage<CommentsInfoItem> initPage;
private long total;
public PeertubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) {
super(service, uiHandler);
}
@Override
public String getName() throws ParsingException {
return "Comments";
}
@Override
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws IOException, ExtractionException {
super.fetchPage();
return initPage;
}
private void collectStreamsFrom(CommentsInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException {
JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
}catch(Exception e) {
throw new ParsingException("unable to extract comments info", e);
}
for(Object c: contents) {
if(c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
PeertubeCommentsInfoItemExtractor extractor = new PeertubeCommentsInfoItemExtractor(item, this);
collector.commit(extractor);
}
}
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
super.fetchPage();
return initPage.getNextPageUrl();
}
@Override
public InfoItemsPage<CommentsInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if(null != response && !StringUtil.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
throw new ParsingException("Could not parse json data for comments info", e);
}
}
CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
if(json != null) {
Number number = JsonUtils.getNumber(json, "total");
if(number != null) this.total = number.longValue();
collectStreamsFrom(collector, json, pageUrl);
} else {
throw new ExtractionException("Unable to get peertube comments info");
}
return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl));
}
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
String pageUrl = getUrl() + "?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
this.initPage = getPage(pageUrl);
}
private String getNextPageUrl(String prevPageUrl) {
String prevStart;
try {
prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl);
} catch (RegexException e) {
return "";
}
if(StringUtil.isBlank(prevStart)) return "";
long nextStart = 0;
try {
nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE;
} catch (NumberFormatException e) {
return "";
}
if(nextStart >= total) {
return "";
}else {
return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart));
}
}
}

104
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java Normal file
View File

@ -0,0 +1,104 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import com.grack.nanojson.JsonObject;
public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
private final JsonObject item;
private final String url;
private final String baseUrl;
public PeertubeCommentsInfoItemExtractor(JsonObject item, PeertubeCommentsExtractor extractor) throws ParsingException {
this.item = item;
this.url = extractor.getUrl();
this.baseUrl = extractor.getBaseUrl();
}
@Override
public String getUrl() throws ParsingException {
return url;
}
@Override
public String getThumbnailUrl() throws ParsingException {
String value;
try {
value = JsonUtils.getString(item, "account.avatar.path");
}catch(Exception e) {
value = "/client/assets/images/default-avatar.png";
}
return baseUrl + value;
}
@Override
public String getName() throws ParsingException {
return JsonUtils.getString(item, "account.displayName");
}
@Override
public String getTextualPublishedTime() throws ParsingException {
return JsonUtils.getString(item, "createdAt");
}
@Override
public DateWrapper getPublishedTime() throws ParsingException {
String textualUploadDate = getTextualPublishedTime();
return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate));
}
@Override
public int getLikeCount() throws ParsingException {
return -1;
}
@Override
public String getCommentText() throws ParsingException {
String htmlText = JsonUtils.getString(item, "text");
try {
Document doc = Jsoup.parse(htmlText);
return doc.body().text();
}catch(Exception e) {
return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", "");
}
}
@Override
public String getCommentId() throws ParsingException {
Number value = JsonUtils.getNumber(item, "id");
return value.toString();
}
@Override
public String getAuthorThumbnail() throws ParsingException {
String value;
try {
value = JsonUtils.getString(item, "account.avatar.path");
}catch(Exception e) {
value = "/client/assets/images/default-avatar.png";
}
return baseUrl + value;
}
@Override
public String getAuthorName() throws ParsingException {
return JsonUtils.getString(item, "account.displayName");
}
@Override
public String getAuthorEndpoint() throws ParsingException {
String name = JsonUtils.getString(item, "account.name");
String host = JsonUtils.getString(item, "account.host");
return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl();
}
}

86
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java Normal file
View File

@ -0,0 +1,86 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import java.io.IOException;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
public class PeertubePlaylistExtractor extends PlaylistExtractor{
public PeertubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
// TODO Auto-generated constructor stub
}
@Override
public String getThumbnailUrl() throws ParsingException {
// TODO Auto-generated method stub
return null;
}
@Override
public String getBannerUrl() throws ParsingException {
// TODO Auto-generated method stub
return null;
}
@Override
public String getUploaderUrl() throws ParsingException {
// TODO Auto-generated method stub
return null;
}
@Override
public String getUploaderName() throws ParsingException {
// TODO Auto-generated method stub
return null;
}
@Override
public String getUploaderAvatarUrl() throws ParsingException {
// TODO Auto-generated method stub
return null;
}
@Override
public long getStreamCount() throws ParsingException {
// TODO Auto-generated method stub
return 0;
}
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
// TODO Auto-generated method stub
return null;
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
// TODO Auto-generated method stub
return null;
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
// TODO Auto-generated method stub
return null;
}
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
// TODO Auto-generated method stub
}
@Override
public String getName() throws ParsingException {
// TODO Auto-generated method stub
return null;
}
}

130
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java Normal file
View File

@ -0,0 +1,130 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import java.io.IOException;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
public class PeertubeSearchExtractor extends SearchExtractor {
private static final String START_KEY = "start";
private static final String COUNT_KEY = "count";
private static final int ITEMS_PER_PAGE = 12;
private static final String START_PATTERN = "start=(\\d*)";
private InfoItemsPage<InfoItem> initPage;
private long total;
public PeertubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
super(service, linkHandler);
}
@Override
public String getSearchSuggestion() throws ParsingException {
return null;
}
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
super.fetchPage();
return initPage;
}
private InfoItemsCollector<InfoItem, InfoItemExtractor> collectStreamsFrom(JsonObject json) throws ParsingException {
final InfoItemsSearchCollector collector = getInfoItemSearchCollector();
JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
}catch(Exception e) {
throw new ParsingException("unable to extract search info", e);
}
String baseUrl = getBaseUrl();
for(Object c: contents) {
if(c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
collector.commit(extractor);
}
}
return collector;
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
super.fetchPage();
return initPage.getNextPageUrl();
}
@Override
public InfoItemsPage<InfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if(null != response && !StringUtil.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
throw new ParsingException("Could not parse json data for search info", e);
}
}
if(json != null) {
Number number = JsonUtils.getNumber(json, "total");
if(number != null) this.total = number.longValue();
return new InfoItemsPage<>(collectStreamsFrom(json), getNextPageUrl(pageUrl));
} else {
throw new ExtractionException("Unable to get peertube search info");
}
}
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
this.initPage = getPage(pageUrl);
}
private String getNextPageUrl(String prevPageUrl) {
String prevStart;
try {
prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl);
} catch (RegexException e) {
return "";
}
if(StringUtil.isBlank(prevStart)) return "";
long nextStart = 0;
try {
nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE;
} catch (NumberFormatException e) {
return "";
}
if(nextStart >= total) {
return "";
}else {
return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart));
}
}
}

342
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java Normal file
View File

@ -0,0 +1,342 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
public class PeertubeStreamExtractor extends StreamExtractor {
private JsonObject json;
private List<SubtitlesStream> subtitles = new ArrayList<>();
private final String baseUrl;
public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler) throws ParsingException {
super(service, linkHandler);
this.baseUrl = getBaseUrl();
}
@Override
public String getTextualUploadDate() throws ParsingException {
return JsonUtils.getString(json, "publishedAt");
}
@Override
public DateWrapper getUploadDate() throws ParsingException {
final String textualUploadDate = getTextualUploadDate();
if (textualUploadDate == null) {
return null;
}
return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate));
}
@Override
public String getThumbnailUrl() throws ParsingException {
return baseUrl + JsonUtils.getString(json, "thumbnailPath");
}
@Override
public String getDescription() throws ParsingException {
try {
return JsonUtils.getString(json, "description");
}catch(ParsingException e) {
return "No description";
}
}
@Override
public int getAgeLimit() throws ParsingException {
return NO_AGE_LIMIT;
}
@Override
public long getLength() throws ParsingException {
Number value = JsonUtils.getNumber(json, "duration");
return value.longValue();
}
@Override
public long getTimeStamp() throws ParsingException {
//TODO fetch timestamp from url if present;
return 0;
}
@Override
public long getViewCount() throws ParsingException {
Number value = JsonUtils.getNumber(json, "views");
return value.longValue();
}
@Override
public long getLikeCount() throws ParsingException {
Number value = JsonUtils.getNumber(json, "likes");
return value.longValue();
}
@Override
public long getDislikeCount() throws ParsingException {
Number value = JsonUtils.getNumber(json, "dislikes");
return value.longValue();
}
@Override
public String getUploaderUrl() throws ParsingException {
String name = JsonUtils.getString(json, "account.name");
String host = JsonUtils.getString(json, "account.host");
return getService().getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl();
}
@Override
public String getUploaderName() throws ParsingException {
return JsonUtils.getString(json, "account.displayName");
}
@Override
public String getUploaderAvatarUrl() throws ParsingException {
String value;
try {
value = JsonUtils.getString(json, "account.avatar.path");
}catch(Exception e) {
value = "/client/assets/images/default-avatar.png";
}
return baseUrl + value;
}
@Override
public String getDashMpdUrl() throws ParsingException {
return "";
}
@Override
public String getHlsUrl() throws ParsingException {
return "";
}
@Override
public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
return null;
}
@Override
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
assertPageFetched();
List<VideoStream> videoStreams = new ArrayList<>();
try {
JsonArray streams = json.getArray("files", new JsonArray());
for(Object s: streams) {
if(!(s instanceof JsonObject)) continue;
JsonObject stream = (JsonObject) s;
String url = JsonUtils.getString(stream, "fileUrl");
String torrentUrl = JsonUtils.getString(stream, "torrentUrl");
String resolution = JsonUtils.getString(stream, "resolution.label");
String extension = url.substring(url.lastIndexOf(".") + 1);
MediaFormat format = MediaFormat.getFromSuffix(extension);
VideoStream videoStream = new VideoStream(url, torrentUrl, format, resolution);
if (!Stream.containSimilarStream(videoStream, videoStreams)) {
videoStreams.add(videoStream);
}
}
} catch (Exception e) {
throw new ParsingException("Could not get video streams", e);
}
return videoStreams;
}
@Override
public List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException {
// TODO Auto-generated method stub
return null;
}
@Override
public List<SubtitlesStream> getSubtitlesDefault() throws IOException, ExtractionException {
return subtitles;
}
@Override
public List<SubtitlesStream> getSubtitles(final MediaFormat format) throws IOException, ExtractionException {
List<SubtitlesStream> filteredSubs = new ArrayList<>();
for(SubtitlesStream sub: subtitles) {
if(sub.getFormat() == format) {
filteredSubs.add(sub);
}
}
return filteredSubs;
}
@Override
public StreamType getStreamType() throws ParsingException {
return StreamType.VIDEO_STREAM;
}
@Override
public StreamInfoItem getNextStream() throws IOException, ExtractionException {
return null;
}
@Override
public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
List<String> tags = getTags();
String apiUrl = null;
if(!tags.isEmpty()) {
apiUrl = getRelatedStreamsUrl(tags);
}else {
apiUrl = getUploaderUrl() + "/videos?start=0&count=8";
}
if(!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl);
return collector;
}
private List<String> getTags(){
try {
return (List) JsonUtils.getArray(json, "tags");
} catch (Exception e) {
return Collections.emptyList();
}
}
private String getRelatedStreamsUrl(List<String> tags) throws UnsupportedEncodingException {
String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT;
StringBuilder params = new StringBuilder();
params.append("start=0&count=8&sort=-createdAt");
for(String tag : tags) {
params.append("&tagsOneOf=");
params.append(URLEncoder.encode(tag, "UTF-8"));
}
return url + "?" + params.toString();
}
private void getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, IOException, ParsingException {
Response response = getDownloader().get(apiUrl);
JsonObject relatedVideosJson = null;
if(null != response && !StringUtil.isBlank(response.responseBody())) {
try {
relatedVideosJson = JsonParser.object().from(response.responseBody());
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json data for related videos", e);
}
}
if(relatedVideosJson != null) {
collectStreamsFrom(collector, relatedVideosJson);
}
}
private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json) throws ParsingException {
JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
}catch(Exception e) {
throw new ParsingException("unable to extract related videos", e);
}
for(Object c: contents) {
if(c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
//do not add the same stream in related streams
if(!extractor.getUrl().equals(getUrl())) collector.commit(extractor);
}
}
}
@Override
public String getErrorMessage() {
return null;
}
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
Response response = downloader.get(getUrl());
if(null != response && null != response.responseBody()) {
setInitialData(response.responseBody());
}else {
throw new ExtractionException("Unable to extract peertube channel data");
}
loadSubtitles();
}
private void setInitialData(String responseBody) throws ExtractionException {
try {
json = JsonParser.object().from(responseBody);
} catch (JsonParserException e) {
throw new ExtractionException("Unable to extract peertube stream data", e);
}
if(null == json) throw new ExtractionException("Unable to extract peertube stream data");
PeertubeParsingHelper.validate(json);
}
private void loadSubtitles() {
if (subtitles.isEmpty()) {
try {
Response response = getDownloader().get(getUrl() + "/captions");
JsonObject captionsJson = JsonParser.object().from(response.responseBody());
JsonArray captions = JsonUtils.getArray(captionsJson, "data");
for(Object c: captions) {
if(c instanceof JsonObject) {
JsonObject caption = (JsonObject)c;
String url = baseUrl + JsonUtils.getString(caption, "captionPath");
String languageCode = JsonUtils.getString(caption, "language.id");
String ext = url.substring(url.lastIndexOf(".") + 1);
MediaFormat fmt = MediaFormat.getFromSuffix(ext);
if(fmt != null && languageCode != null) subtitles.add(new SubtitlesStream(fmt, languageCode, url, false));
}
}
} catch (Exception e) {
// ignore all exceptions
}
}
}
@Override
public String getName() throws ParsingException {
return JsonUtils.getString(json, "name");
}
@Override
public String getOriginalUrl() throws ParsingException {
return baseUrl + "/videos/watch/" + getId();
}
}

90
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java Normal file
View File

@ -0,0 +1,90 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import com.grack.nanojson.JsonObject;
public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
protected final JsonObject item;
private final String baseUrl;
public PeertubeStreamInfoItemExtractor(JsonObject item, String baseUrl) {
this.item = item;
this.baseUrl = baseUrl;
}
@Override
public String getUrl() throws ParsingException {
String uuid = JsonUtils.getString(item, "uuid");
return ServiceList.PeerTube.getStreamLHFactory().fromId(uuid, baseUrl).getUrl();
}
@Override
public String getThumbnailUrl() throws ParsingException {
String value = JsonUtils.getString(item, "thumbnailPath");
return baseUrl + value;
}
@Override
public String getName() throws ParsingException {
return JsonUtils.getString(item, "name");
}
@Override
public boolean isAd() throws ParsingException {
return false;
}
@Override
public long getViewCount() throws ParsingException {
Number value = JsonUtils.getNumber(item, "views");
return value.longValue();
}
@Override
public String getUploaderUrl() throws ParsingException {
String name = JsonUtils.getString(item, "account.name");
String host = JsonUtils.getString(item, "account.host");
return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl();
}
@Override
public String getUploaderName() throws ParsingException {
return JsonUtils.getString(item, "account.displayName");
}
@Override
public String getTextualUploadDate() throws ParsingException {
return JsonUtils.getString(item, "publishedAt");
}
@Override
public DateWrapper getUploadDate() throws ParsingException {
final String textualUploadDate = getTextualUploadDate();
if (textualUploadDate == null) {
return null;
}
return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate));
}
@Override
public StreamType getStreamType() throws ParsingException {
return StreamType.VIDEO_STREAM;
}
@Override
public long getDuration() throws ParsingException {
Number value = JsonUtils.getNumber(item, "duration");
return value.longValue();
}
}

21
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java Normal file
View File

@ -0,0 +1,21 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import java.util.List;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
public class PeertubeSubscriptionExtractor extends SubscriptionExtractor {
public PeertubeSubscriptionExtractor(StreamingService service, List<ContentSource> supportedSources) {
super(service, supportedSources);
// TODO Auto-generated constructor stub
}
@Override
public String getRelatedUrl() {
// TODO Auto-generated method stub
return null;
}
}

22
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java Normal file
View File

@ -0,0 +1,22 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
public class PeertubeSuggestionExtractor extends SuggestionExtractor{
public PeertubeSuggestionExtractor(StreamingService service) {
super(service);
}
@Override
public List<String> suggestionList(String query) throws IOException, ExtractionException {
return Collections.emptyList();
}
}

124
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java Normal file
View File

@ -0,0 +1,124 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import java.io.IOException;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
public class PeertubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
private static final String START_KEY = "start";
private static final String COUNT_KEY = "count";
private static final int ITEMS_PER_PAGE = 12;
private static final String START_PATTERN = "start=(\\d*)";
private InfoItemsPage<StreamInfoItem> initPage;
private long total;
public PeertubeTrendingExtractor(StreamingService streamingService, ListLinkHandler linkHandler, String kioskId) {
super(streamingService, linkHandler, kioskId);
}
@Override
public String getName() throws ParsingException {
return getId();
}
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
super.fetchPage();
return initPage;
}
private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException {
JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
}catch(Exception e) {
throw new ParsingException("unable to extract kiosk info", e);
}
String baseUrl = getBaseUrl();
for(Object c: contents) {
if(c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
collector.commit(extractor);
}
}
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
super.fetchPage();
return initPage.getNextPageUrl();
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if(null != response && !StringUtil.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
throw new ParsingException("Could not parse json data for kiosk info", e);
}
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
if(json != null) {
Number number = JsonUtils.getNumber(json, "total");
if(number != null) this.total = number.longValue();
collectStreamsFrom(collector, json, pageUrl);
} else {
throw new ExtractionException("Unable to get peertube kiosk info");
}
return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl));
}
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
this.initPage = getPage(pageUrl);
}
private String getNextPageUrl(String prevPageUrl) {
String prevStart;
try {
prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl);
} catch (RegexException e) {
return "";
}
if(StringUtil.isBlank(prevStart)) return "";
long nextStart = 0;
try {
nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE;
} catch (NumberFormatException e) {
return "";
}
if(nextStart >= total) {
return "";
}else {
return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart));
}
}
}

41
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java Normal file
View File

@ -0,0 +1,41 @@
package org.schabi.newpipe.extractor.services.peertube.linkHandler;
import java.util.List;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Parser;
public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
private static final PeertubeChannelLinkHandlerFactory instance = new PeertubeChannelLinkHandlerFactory();
private static final String ID_PATTERN = "/accounts/([^/?&#]*)";
private static final String ACCOUNTS_ENDPOINT = "/api/v1/accounts/";
public static PeertubeChannelLinkHandlerFactory getInstance() {
return instance;
}
@Override
public String getId(String url) throws ParsingException {
return Parser.matchGroup1(ID_PATTERN, url);
}
@Override
public String getUrl(String id, List<String> contentFilters, String searchFilter) throws ParsingException {
String baseUrl = ServiceList.PeerTube.getBaseUrl();
return getUrl(id, contentFilters, searchFilter, baseUrl);
}
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter, String baseUrl)
throws ParsingException {
return baseUrl + ACCOUNTS_ENDPOINT + id;
}
@Override
public boolean onAcceptUrl(String url) {
return url.contains("/accounts/");
}
}

42
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java Normal file
View File

@ -0,0 +1,42 @@
package org.schabi.newpipe.extractor.services.peertube.linkHandler;
import java.util.List;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Parser;
public class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
private static final PeertubeCommentsLinkHandlerFactory instance = new PeertubeCommentsLinkHandlerFactory();
private static final String ID_PATTERN = "/videos/(watch/)?([^/?&#]*)";
private static final String COMMENTS_ENDPOINT = "/api/v1/videos/%s/comment-threads";
public static PeertubeCommentsLinkHandlerFactory getInstance() {
return instance;
}
@Override
public String getId(String url) throws ParsingException, IllegalArgumentException {
return Parser.matchGroup(ID_PATTERN, url, 2);
}
@Override
public boolean onAcceptUrl(final String url) throws FoundAdException {
return url.contains("/videos/");
}
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException {
String baseUrl = ServiceList.PeerTube.getBaseUrl();
return getUrl(id, contentFilter, sortFilter, baseUrl);
}
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter, String baseUrl) throws ParsingException {
return baseUrl + String.format(COMMENTS_ENDPOINT, id);
}
}

42
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java Normal file
View File

@ -0,0 +1,42 @@
package org.schabi.newpipe.extractor.services.peertube.linkHandler;
import java.util.List;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Parser;
public class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
private static final PeertubePlaylistLinkHandlerFactory instance = new PeertubePlaylistLinkHandlerFactory();
private static final String ID_PATTERN = "/video-channels/([^/?&#]*)";
private static final String VIDEO_CHANNELS_ENDPOINT = "/api/v1/video-channels/";
public static PeertubePlaylistLinkHandlerFactory getInstance() {
return instance;
}
@Override
public String getUrl(String id, List<String> contentFilters, String sortFilter) {
String baseUrl = ServiceList.PeerTube.getBaseUrl();
return getUrl(id, contentFilters, sortFilter, baseUrl);
}
@Override
public String getUrl(String id, List<String> contentFilters, String sortFilter, String baseUrl) {
return baseUrl + VIDEO_CHANNELS_ENDPOINT + id;
}
@Override
public String getId(String url) throws ParsingException {
return Parser.matchGroup1(ID_PATTERN, url);
}
@Override
public boolean onAcceptUrl(final String url) {
return url.contains("/video-channels/");
}
}

43
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java Normal file
View File

@ -0,0 +1,43 @@
package org.schabi.newpipe.extractor.services.peertube.linkHandler;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.List;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String CHARSET_UTF_8 = "UTF-8";
public static final String VIDEOS = "videos";
public static final String SEARCH_ENDPOINT = "/api/v1/search/videos";
public static PeertubeSearchQueryHandlerFactory getInstance() {
return new PeertubeSearchQueryHandlerFactory();
}
@Override
public String getUrl(String searchString, List<String> contentFilters, String sortFilter) throws ParsingException {
String baseUrl = ServiceList.PeerTube.getBaseUrl();
return getUrl(searchString, contentFilters, sortFilter, baseUrl);
}
@Override
public String getUrl(String searchString, List<String> contentFilters, String sortFilter, String baseUrl) throws ParsingException {
try {
final String url = baseUrl + SEARCH_ENDPOINT
+ "?search=" + URLEncoder.encode(searchString, CHARSET_UTF_8);
return url;
} catch (UnsupportedEncodingException e) {
throw new ParsingException("Could not encode query", e);
}
}
@Override
public String[] getAvailableContentFilter() {
return new String[] { VIDEOS };
}
}

42
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java Normal file
View File

@ -0,0 +1,42 @@
package org.schabi.newpipe.extractor.services.peertube.linkHandler;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Parser;
public class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory {
private static final PeertubeStreamLinkHandlerFactory instance = new PeertubeStreamLinkHandlerFactory();
private static final String ID_PATTERN = "/videos/(watch/)?([^/?&#]*)";
private static final String VIDEO_ENDPOINT = "/api/v1/videos/";
private PeertubeStreamLinkHandlerFactory() {
}
public static PeertubeStreamLinkHandlerFactory getInstance() {
return instance;
}
@Override
public String getUrl(String id) {
String baseUrl = ServiceList.PeerTube.getBaseUrl();
return getUrl(id, baseUrl);
}
@Override
public String getUrl(String id, String baseUrl) {
return baseUrl + VIDEO_ENDPOINT + id;
}
@Override
public String getId(String url) throws ParsingException, IllegalArgumentException {
return Parser.matchGroup(ID_PATTERN, url, 2);
}
@Override
public boolean onAcceptUrl(final String url) throws FoundAdException {
return url.contains("/videos/");
}
}

77
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java Normal file
View File

@ -0,0 +1,77 @@
package org.schabi.newpipe.extractor.services.peertube.linkHandler;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
private static final PeertubeTrendingLinkHandlerFactory instance = new PeertubeTrendingLinkHandlerFactory();
public static final Map<String, String> KIOSK_MAP;
public static final Map<String, String> REVERSE_KIOSK_MAP;
public static final String KIOSK_TRENDING = "Trending";
public static final String KIOSK_MOST_LIKED = "Most liked";
public static final String KIOSK_RECENT = "Recently added";
public static final String KIOSK_LOCAL = "Local";
static {
Map<String, String> map = new HashMap<>();
map.put(KIOSK_TRENDING, "%s/api/v1/videos?sort=-trending");
map.put(KIOSK_MOST_LIKED, "%s/api/v1/videos?sort=-likes");
map.put(KIOSK_RECENT, "%s/api/v1/videos?sort=-publishedAt");
map.put(KIOSK_LOCAL, "%s/api/v1/videos?sort=-publishedAt&filter=local");
KIOSK_MAP = Collections.unmodifiableMap(map);
Map<String, String> reverseMap = new HashMap<>();
for(Map.Entry<String, String> entry : KIOSK_MAP.entrySet()){
reverseMap.put(entry.getValue(), entry.getKey());
}
REVERSE_KIOSK_MAP = Collections.unmodifiableMap(reverseMap);
}
public static PeertubeTrendingLinkHandlerFactory getInstance() {
return instance;
}
@Override
public String getUrl(String id, List<String> contentFilters, String sortFilter) {
String baseUrl = ServiceList.PeerTube.getBaseUrl();
return getUrl(id, contentFilters, sortFilter, baseUrl);
}
@Override
public String getUrl(String id, List<String> contentFilters, String sortFilter, String baseUrl) {
return String.format(KIOSK_MAP.get(id), baseUrl);
}
@Override
public String getId(String url) throws ParsingException {
String baseUrl = ServiceList.PeerTube.getBaseUrl();
url = url.replace(baseUrl, "%s");
if (url.contains("/videos/trending")) {
return KIOSK_TRENDING;
} else if (url.contains("/videos/most-liked")) {
return KIOSK_MOST_LIKED;
} else if (url.contains("/videos/recently-added")) {
return KIOSK_RECENT;
} else if (url.contains("/videos/local")) {
return KIOSK_LOCAL;
} else if (REVERSE_KIOSK_MAP.containsKey(url)) {
return REVERSE_KIOSK_MAP.get(url);
} else {
throw new ParsingException("no id found for this url");
}
}
@Override
public boolean onAcceptUrl(final String url) {
return url.contains("/videos?") || url.contains("/videos/trending") || url.contains("/videos/most-liked") || url.contains("/videos/recently-added") || url.contains("/videos/local");
}
}

20
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java
View File

@ -1,25 +1,35 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.*;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
public class SoundcloudService extends StreamingService {
public SoundcloudService(int id) {
super(id, "SoundCloud", singletonList(AUDIO));
}
@Override
public String getBaseUrl() {
return "https://soundcloud.com";
}
@Override
public SearchQueryHandlerFactory getSearchQHFactory() {
@ -110,5 +120,5 @@ public class SoundcloudService extends StreamingService {
throws ExtractionException {
return null;
}
}

42
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View File

@ -1,27 +1,47 @@
package org.schabi.newpipe.extractor.services.youtube;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
import java.util.List;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.*;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.*;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.*;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscriptionExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSuggestionExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.util.List;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.*;
/*
* Created by Christian Schabesberger on 23.08.15.
*
@ -48,6 +68,11 @@ public class YoutubeService extends StreamingService {
super(id, "YouTube", asList(AUDIO, VIDEO, LIVE, COMMENTS));
}
@Override
public String getBaseUrl() {
return "https://youtube.com";
}
@Override
public LinkHandlerFactory getStreamLHFactory() {
return YoutubeStreamLinkHandlerFactory.getInstance();
@ -173,7 +198,6 @@ public class YoutubeService extends StreamingService {
return SUPPORTED_LANGUAGES;
}
@Override
public List<ContentCountry> getSupportedCountries() {
return SUPPORTED_COUNTRIES;
}

50
extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java
View File

@ -1,13 +1,17 @@
package org.schabi.newpipe.extractor.stream;
import org.schabi.newpipe.extractor.MediaFormat;
import java.io.Serializable;
import java.util.List;
import org.schabi.newpipe.extractor.MediaFormat;
/**
* Creates a stream object from url, format and optional torrent url
*/
public abstract class Stream implements Serializable {
private final MediaFormat mediaFormat;
public final String url;
public final String torrentUrl;
/**
* @deprecated Use {@link #getFormat()} or {@link #getFormatId()}
@ -15,8 +19,26 @@ public abstract class Stream implements Serializable {
@Deprecated
public final int format;
/**
* Instantiates a new stream object.
*
* @param url the url
* @param format the format
*/
public Stream(String url, MediaFormat format) {
this(url, null, format);
}
/**
* Instantiates a new stream object.
*
* @param url the url
* @param torrentUrl the url to torrent file, example https://webtorrent.io/torrents/big-buck-bunny.torrent
* @param format the format
*/
public Stream(String url, String torrentUrl, MediaFormat format) {
this.url = url;
this.torrentUrl = torrentUrl;
this.format = format.id;
this.mediaFormat = format;
}
@ -46,14 +68,38 @@ public abstract class Stream implements Serializable {
return false;
}
/**
* Gets the url.
*
* @return the url
*/
public String getUrl() {
return url;
}
/**
* Gets the torrent url.
*
* @return the torrent url, example https://webtorrent.io/torrents/big-buck-bunny.torrent
*/
public String getTorrentUrl() {
return torrentUrl;
}
/**
* Gets the format.
*
* @return the format
*/
public MediaFormat getFormat() {
return mediaFormat;
}
/**
* Gets the format id.
*
* @return the format id
*/
public int getFormatId() {
return mediaFormat.id;
}

10
extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java
View File

@ -36,6 +36,16 @@ public class VideoStream extends Stream {
this.resolution = resolution;
this.isVideoOnly = isVideoOnly;
}
public VideoStream(String url, String torrentUrl, MediaFormat format, String resolution) {
this(url, torrentUrl, format, resolution, false);
}
public VideoStream(String url, String torrentUrl, MediaFormat format, String resolution, boolean isVideoOnly) {
super(url, torrentUrl, format);
this.resolution = resolution;
this.isVideoOnly = isVideoOnly;
}
@Override
public boolean equalStats(Stream cmp) {

15
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java
View File

@ -144,9 +144,9 @@ public class Utils {
try {
return new URL(url);
} catch (MalformedURLException e) {
// if no protocol is given try prepending "http://"
// if no protocol is given try prepending "https://"
if (e.getMessage().equals("no protocol: " + url)) {
return new URL(HTTP + url);
return new URL(HTTPS + url);
}
throw e;
@ -175,4 +175,15 @@ public class Utils {
}
return s;
}
public static String getBaseUrl(String url) throws ParsingException {
URL uri;
try {
uri = stringToURL(url);
} catch (MalformedURLException e) {
throw new ParsingException("Malformed url: " + url, e);
}
return uri.getProtocol() + "://" + uri.getAuthority();
}
}

210
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java Normal file
View File

@ -0,0 +1,210 @@
package org.schabi.newpipe.extractor.services.peertube;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor;
/**
* Test for {@link PeertubeChannelExtractor}
*/
public class PeertubeChannelExtractorTest {
public static class KDE implements BaseChannelExtractorTest {
private static PeertubeChannelExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
extractor = (PeertubeChannelExtractor) PeerTube
.getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/kde");
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testServiceId() {
assertEquals(PeerTube.getServiceId(), extractor.getServiceId());
}
@Test
public void testName() throws ParsingException {
assertEquals("The KDE Community", extractor.getName());
}
@Test
public void testId() throws ParsingException {
assertEquals("kde", extractor.getId());
}
@Test
public void testUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/accounts/kde", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor, PeerTube.getServiceId());
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor, PeerTube.getServiceId());
}
/*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testDescription() throws ParsingException {
assertNotNull(extractor.getDescription());
}
@Test
public void testAvatarUrl() throws ParsingException {
assertIsSecureUrl(extractor.getAvatarUrl());
}
@Ignore
@Test
public void testBannerUrl() throws ParsingException {
assertIsSecureUrl(extractor.getBannerUrl());
}
@Test
public void testFeedUrl() throws ParsingException {
assertEmpty(extractor.getFeedUrl());
}
@Test
public void testSubscriberCount() throws ParsingException {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5);
}
}
public static class Booteille implements BaseChannelExtractorTest {
private static PeertubeChannelExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
extractor = (PeertubeChannelExtractor) PeerTube
.getChannelExtractor("https://peertube.mastodon.host/accounts/booteille");
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Additional Testing
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testGetPageInNewExtractor() throws Exception {
final ChannelExtractor newExtractor = PeerTube.getChannelExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor, PeerTube.getServiceId());
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testServiceId() {
assertEquals(PeerTube.getServiceId(), extractor.getServiceId());
}
@Test
public void testName() throws ParsingException {
assertEquals("booteille", extractor.getName());
}
@Test
public void testId() throws ParsingException {
assertEquals("booteille", extractor.getId());
}
@Test
public void testUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor, PeerTube.getServiceId());
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor, PeerTube.getServiceId());
}
/*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testDescription() throws ParsingException {
assertNotNull(extractor.getDescription());
}
@Test
public void testAvatarUrl() throws ParsingException {
assertIsSecureUrl(extractor.getAvatarUrl());
}
@Ignore
@Test
public void testBannerUrl() throws ParsingException {
assertIsSecureUrl(extractor.getBannerUrl());
}
@Test
public void testFeedUrl() throws ParsingException {
assertEmpty(extractor.getFeedUrl());
}
@Test
public void testSubscriberCount() throws ParsingException {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 2);
}
}
}

36
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java Normal file
View File

@ -0,0 +1,36 @@
package org.schabi.newpipe.extractor.services.peertube;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
/**
* Test for {@link PeertubeChannelLinkHandlerFactory}
*/
public class PeertubeChannelLinkHandlerFactoryTest {
private static PeertubeChannelLinkHandlerFactory linkHandler;
@BeforeClass
public static void setUp() {
linkHandler = PeertubeChannelLinkHandlerFactory.getInstance();
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test
public void acceptrUrlTest() throws ParsingException {
assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net"));
}
@Test
public void getIdFromUrl() throws ParsingException {
assertEquals("kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net").getId());
assertEquals("kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId());
}
}

92
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java Normal file
View File

@ -0,0 +1,92 @@
package org.schabi.newpipe.extractor.services.peertube;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import java.io.IOException;
import java.util.List;
import org.jsoup.helper.StringUtil;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.comments.CommentsInfo;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor;
public class PeertubeCommentsExtractorTest {
private static PeertubeCommentsExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (PeertubeCommentsExtractor) PeerTube
.getCommentsExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
}
@Test
public void testGetComments() throws IOException, ExtractionException {
boolean result = false;
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
result = findInComments(comments, "@root A great documentary on a great guy.");
while (comments.hasNextPage() && !result) {
comments = extractor.getPage(comments.getNextPageUrl());
result = findInComments(comments, "@root A great documentary on a great guy.");
}
assertTrue(result);
}
@Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
boolean result = false;
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://peertube.mastodon.host/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
assertTrue("Comments".equals(commentsInfo.getName()));
result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");
String nextPage = commentsInfo.getNextPageUrl();
while (!StringUtil.isBlank(nextPage) && !result) {
InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage);
result = findInComments(moreItems.getItems(), "Loved it!!!");
nextPage = moreItems.getNextPageUrl();
}
assertTrue(result);
}
@Test
public void testGetCommentsAllData() throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
for(CommentsInfoItem c: comments.getItems()) {
assertFalse(StringUtil.isBlank(c.getAuthorEndpoint()));
assertFalse(StringUtil.isBlank(c.getAuthorName()));
assertFalse(StringUtil.isBlank(c.getAuthorThumbnail()));
assertFalse(StringUtil.isBlank(c.getCommentId()));
assertFalse(StringUtil.isBlank(c.getCommentText()));
assertFalse(StringUtil.isBlank(c.getName()));
assertFalse(StringUtil.isBlank(c.getTextualPublishedTime()));
assertFalse(StringUtil.isBlank(c.getThumbnailUrl()));
assertFalse(StringUtil.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() != -1);
}
}
private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) {
return findInComments(comments.getItems(), comment);
}
private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
for(CommentsInfoItem c: comments) {
if(c.getCommentText().contains(comment)) {
return true;
}
}
return false;
}
}

36
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java Normal file
View File

@ -0,0 +1,36 @@
package org.schabi.newpipe.extractor.services.peertube;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory;
/**
* Test for {@link PeertubeCommentsLinkHandlerFactory}
*/
public class PeertubeCommentsLinkHandlerFactoryTest {
private static PeertubeCommentsLinkHandlerFactory linkHandler;
@BeforeClass
public static void setUp() {
linkHandler = PeertubeCommentsLinkHandlerFactory.getInstance();
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test
public void acceptrUrlTest() throws ParsingException {
assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/api/v1/videos/19319/comment-threads?start=0&count=10&sort=-createdAt"));
}
@Test
public void getIdFromUrl() throws ParsingException {
assertEquals("19319", linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/videos/19319/comment-threads").getId());
assertEquals("19319", linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/videos/19319/comment-threads?start=0&count=10&sort=-createdAt").getId());
}
}

36
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java Normal file
View File

@ -0,0 +1,36 @@
package org.schabi.newpipe.extractor.services.peertube;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubePlaylistLinkHandlerFactory;
/**
* Test for {@link PeertubePlaylistLinkHandlerFactory}
*/
public class PeertubePlaylistLinkHandlerFactoryTest {
private static PeertubePlaylistLinkHandlerFactory linkHandler;
@BeforeClass
public static void setUp() {
linkHandler = PeertubePlaylistLinkHandlerFactory.getInstance();
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test
public void acceptrUrlTest() throws ParsingException {
assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/video-channels/b45e84fb-c47f-475b-94f2-718126154d33/videos"));
}
@Test
public void getIdFromUrl() throws ParsingException {
assertEquals("b45e84fb-c47f-475b-94f2-718126154d33", linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/b45e84fb-c47f-475b-94f2-718126154d33").getId());
assertEquals("b45e84fb-c47f-475b-94f2-718126154d33", linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/b45e84fb-c47f-475b-94f2-718126154d33/videos").getId());
}
}

125
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java Normal file
View File

@ -0,0 +1,125 @@
package org.schabi.newpipe.extractor.services.peertube;
import static java.util.Objects.requireNonNull;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
/**
* Test for {@link StreamExtractor}
*/
public class PeertubeStreamExtractorDefaultTest {
private static PeertubeStreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/afe5bf12-c58b-4efd-b56e-29c5a59e04bc");
extractor.fetchPage();
}
@Test
public void testGetInvalidTimeStamp() throws ParsingException {
assertTrue(extractor.getTimeStamp() + "",
extractor.getTimeStamp() <= 0);
}
@Test
public void testGetTitle() throws ParsingException {
assertEquals(extractor.getName(), "Power Corrupts the Best");
}
@Test
public void testGetDescription() throws ParsingException {
assertEquals(extractor.getDescription(), "A short reading from Bakunin, made for the group Audible Anarchist https://audibleanarchist.github.io/Webpage/");
}
@Test
public void testGetUploaderName() throws ParsingException {
assertEquals(extractor.getUploaderName(), "Rowsedower");
}
@Test
public void testGetLength() throws ParsingException {
assertEquals(extractor.getLength(), 269);
}
@Test
public void testGetViewCount() throws ParsingException {
assertTrue(Long.toString(extractor.getViewCount()),
extractor.getViewCount() > 10);
}
@Test
public void testGetUploadDate() throws ParsingException, ParseException {
final Calendar instance = Calendar.getInstance();
instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-09-30T14:08:24.378Z"));
assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
}
@Test
public void testGetUploaderUrl() throws ParsingException {
assertIsSecureUrl(extractor.getUploaderUrl());
assertEquals("https://peertube.mastodon.host/api/v1/accounts/reddebrek@peertube.mastodon.host", extractor.getUploaderUrl());
}
@Test
public void testGetThumbnailUrl() throws ParsingException {
assertIsSecureUrl(extractor.getThumbnailUrl());
}
@Test
public void testGetUploaderAvatarUrl() throws ParsingException {
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
}
@Test
public void testGetVideoStreams() throws IOException, ExtractionException {
assertFalse(extractor.getVideoStreams().isEmpty());
}
@Test
public void testStreamType() throws ParsingException {
assertTrue(extractor.getStreamType() == StreamType.VIDEO_STREAM);
}
@Ignore
@Test
public void testGetRelatedVideos() throws ExtractionException, IOException {
StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();
assertFalse(relatedVideos.getItems().isEmpty());
assertTrue(relatedVideos.getErrors().isEmpty());
}
@Test
public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
assertTrue(extractor.getSubtitlesDefault().isEmpty());
}
@Test
public void testGetSubtitlesList() throws IOException, ExtractionException {
assertTrue(extractor.getSubtitlesDefault().isEmpty());
}
}

37
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java Normal file
View File

@ -0,0 +1,37 @@
package org.schabi.newpipe.extractor.services.peertube;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory;
/**
* Test for {@link PeertubeStreamLinkHandlerFactory}
*/
public class PeertubeStreamLinkHandlerFactoryTest {
private static PeertubeStreamLinkHandlerFactory linkHandler;
@BeforeClass
public static void setUp() throws Exception {
linkHandler = PeertubeStreamLinkHandlerFactory.getInstance();
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test
public void getId() throws Exception {
assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60", linkHandler.fromUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60").getId());
assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60", linkHandler.fromUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa").getId());
}
@Test
public void testAcceptUrl() throws ParsingException {
assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60"));
assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa"));
}
}

97
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java Normal file
View File

@ -0,0 +1,97 @@
package org.schabi.newpipe.extractor.services.peertube;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import java.util.List;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
/**
* Test for {@link PeertubeTrendingExtractor}
*/
public class PeertubeTrendingExtractorTest {
static KioskExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
extractor = PeerTube
.getKioskList()
.getExtractorById("Trending", null);
extractor.fetchPage();
}
@Test
public void testGetDownloader() throws Exception {
assertNotNull(NewPipe.getDownloader());
}
@Test
public void testGetName() throws Exception {
assertEquals(extractor.getName(), "Trending");
}
@Test
public void testId() {
assertEquals(extractor.getId(), "Trending");
}
@Test
public void testGetStreams() throws Exception {
ListExtractor.InfoItemsPage<StreamInfoItem> page = extractor.getInitialPage();
if(!page.getErrors().isEmpty()) {
System.err.println("----------");
List<Throwable> errors = page.getErrors();
for(Throwable e: errors) {
e.printStackTrace();
System.err.println("----------");
}
}
assertTrue("no streams are received",
!page.getItems().isEmpty()
&& page.getErrors().isEmpty());
}
@Test
public void testGetStreamsErrors() throws Exception {
assertTrue("errors during stream list extraction", extractor.getInitialPage().getErrors().isEmpty());
}
@Test
public void testHasMoreStreams() throws Exception {
// Setup the streams
extractor.getInitialPage();
assertTrue("has more streams", extractor.hasNextPage());
}
@Test
public void testGetNextPageUrl() throws Exception {
assertTrue(extractor.hasNextPage());
}
@Test
public void testGetNextPage() throws Exception {
extractor.getInitialPage().getItems();
assertFalse("extractor has next streams", extractor.getPage(extractor.getNextPageUrl()) == null
|| extractor.getPage(extractor.getNextPageUrl()).getItems().isEmpty());
}
@Test
public void testGetCleanUrl() throws Exception {
assertEquals(extractor.getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-trending");
}
}

61
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java Normal file
View File

@ -0,0 +1,61 @@
package org.schabi.newpipe.extractor.services.peertube;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory;
/**
* Test for {@link PeertubeTrendingLinkHandlerFactory}
*/
public class PeertubeTrendingLinkHandlerFactoryTest {
private static LinkHandlerFactory LinkHandlerFactory;
@BeforeClass
public static void setUp() throws Exception {
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
LinkHandlerFactory = new PeertubeTrendingLinkHandlerFactory();
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test
public void getUrl()
throws Exception {
assertEquals(LinkHandlerFactory.fromId("Trending").getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-trending");
assertEquals(LinkHandlerFactory.fromId("Most liked").getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-likes");
assertEquals(LinkHandlerFactory.fromId("Recently added").getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-publishedAt");
assertEquals(LinkHandlerFactory.fromId("Local").getUrl(), "https://peertube.mastodon.host/api/v1/videos?sort=-publishedAt&filter=local");
}
@Test
public void getId()
throws Exception {
assertEquals(LinkHandlerFactory.fromUrl("https://peertube.mastodon.host/videos/trending").getId(), "Trending");
assertEquals(LinkHandlerFactory.fromUrl("https://peertube.mastodon.host/videos/most-liked").getId(), "Most liked");
assertEquals(LinkHandlerFactory.fromUrl("https://peertube.mastodon.host/videos/recently-added").getId(), "Recently added");
assertEquals(LinkHandlerFactory.fromUrl("https://peertube.mastodon.host/videos/local").getId(), "Local");
}
@Test
public void acceptUrl() throws ParsingException {
assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/trending"));
assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/trending?adsf=fjaj#fhe"));
assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/most-liked"));
assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/most-liked?adsf=fjaj#fhe"));
assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/recently-added"));
assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/recently-added?adsf=fjaj#fhe"));
assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/local"));
assertTrue(LinkHandlerFactory.acceptUrl("https://peertube.mastodon.host/videos/local?adsf=fjaj#fhe"));
}
}

28
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorBaseTest.java Normal file
View File

@ -0,0 +1,28 @@
package org.schabi.newpipe.extractor.services.peertube.search;
import static org.junit.Assert.assertTrue;
import org.junit.Test;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor;
/**
* Test for {@link PeertubeSearchExtractor}
*/
public abstract class PeertubeSearchExtractorBaseTest {
protected static PeertubeSearchExtractor extractor;
protected static ListExtractor.InfoItemsPage<InfoItem> itemsPage;
@Test
public void testResultListElementsLength() {
assertTrue(Integer.toString(itemsPage.getItems().size()),
itemsPage.getItems().size() >= 3);
}
@Test
public void testUrl() throws Exception {
assertTrue(extractor.getUrl(), extractor.getUrl().startsWith("https://peertube.mastodon.host/api/v1/search/videos"));
}
}

91
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java Normal file
View File

@ -0,0 +1,91 @@
package org.schabi.newpipe.extractor.services.peertube.search;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
/**
* Test for {@link PeertubeSearchExtractor}
*/
public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorBaseTest {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("kde");
extractor.fetchPage();
itemsPage = extractor.getInitialPage();
}
@Test
public void testGetSecondPageUrl() throws Exception {
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=kde&start=12&count=12", extractor.getNextPageUrl());
}
@Test
public void testResultList_FirstElement() {
InfoItem firstInfoItem = itemsPage.getItems().get(0);
assertTrue("search does not match", firstInfoItem.getName().toLowerCase().contains("kde"));
}
@Test
public void testResultListCheckIfContainsStreamItems() {
boolean hasStreams = false;
for(InfoItem item : itemsPage.getItems()) {
if(item instanceof StreamInfoItem) {
hasStreams = true;
}
}
assertTrue("Has no InfoItemStreams", hasStreams);
}
@Test
public void testGetSecondPage() throws Exception {
extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("internet");
itemsPage = extractor.getInitialPage();
PeertubeSearchExtractor secondExtractor =
(PeertubeSearchExtractor) PeerTube.getSearchExtractor("internet");
ListExtractor.InfoItemsPage<InfoItem> secondPage = secondExtractor.getPage(itemsPage.getNextPageUrl());
assertTrue(Integer.toString(secondPage.getItems().size()),
secondPage.getItems().size() >= 10);
// check if its the same result
boolean equals = true;
for (int i = 0; i < secondPage.getItems().size()
&& i < itemsPage.getItems().size(); i++) {
if(!secondPage.getItems().get(i).getUrl().equals(
itemsPage.getItems().get(i).getUrl())) {
equals = false;
}
}
assertFalse("First and second page are equal", equals);
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=internet&start=24&count=12",
secondPage.getNextPageUrl());
}
@Test
public void testId() throws Exception {
assertEquals("kde", extractor.getId());
}
@Test
public void testName() {
assertEquals("kde", extractor.getName());
}
}

26
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java Normal file
View File

@ -0,0 +1,26 @@
package org.schabi.newpipe.extractor.services.peertube.search;
import static org.junit.Assert.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance;
public class PeertubeSearchQHTest {
@BeforeClass
public static void setUpClass() throws Exception {
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
}
@Test
public void testRegularValues() throws Exception {
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=asdf", PeerTube.getSearchQHFactory().fromQuery("asdf").getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=hans",PeerTube.getSearchQHFactory().fromQuery("hans").getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=Poifj%26jaijf", PeerTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=G%C3%BCl%C3%BCm", PeerTube.getSearchQHFactory().fromQuery("Gülüm").getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl());
}
}

14
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java
View File

@ -1,5 +1,14 @@
package org.schabi.newpipe.extractor.services.youtube;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
@ -11,11 +20,6 @@ import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
/**
* Test for {@link ChannelExtractor}
*/