lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'dev'

This commit is contained in:
TobiGr 2020-02-21 22:27:54 +01:00
parent 33961b2092 4bc5b8d33f
commit 468092894f
130 changed files with 1896 additions and 1102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -21,7 +21,7 @@ To test changes quickly you can build the library locally. Using the local Maven
2. It's _recommended_ that you change the `version` of this library (e.g. `LOCAL_SNAPSHOT`).
3. Run gradle's `ìnstall` task to deploy this library to your local repository (using the wrapper, present in the root of this project: `./gradlew install`)
4. Change the dependency version used in your project to match the one you chose in step 2 (`implementation 'com.github.TeamNewPipe:NewPipeExtractor:LOCAL_SNAPSHOT'`)
> Tip for Android Studio users: After you make changes and run the `install` task, use the menu option `File → "Sync with File System"` to refresh the library in your project.
## Supported sites

36
extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java
View File

@ -1,10 +1,5 @@
package org.schabi.newpipe.extractor;
import java.io.IOException;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -13,7 +8,11 @@ import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
public abstract class Extractor{
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
public abstract class Extractor {
/**
* {@link StreamingService} currently related to this extractor.<br>
* Useful for getting other things from a service (like the url handlers for cleaning/accepting/get id from urls).
@ -21,19 +20,21 @@ public abstract class Extractor{
private final StreamingService service;
private final LinkHandler linkHandler;
@Nullable private Localization forcedLocalization = null;
@Nullable private ContentCountry forcedContentCountry = null;
@Nullable
private Localization forcedLocalization = null;
@Nullable
private ContentCountry forcedContentCountry = null;
private boolean pageFetched = false;
private final Downloader downloader;
public Extractor(final StreamingService service, final LinkHandler linkHandler) {
if(service == null) throw new NullPointerException("service is null");
if(linkHandler == null) throw new NullPointerException("LinkHandler is null");
if (service == null) throw new NullPointerException("service is null");
if (linkHandler == null) throw new NullPointerException("LinkHandler is null");
this.service = service;
this.linkHandler = linkHandler;
this.downloader = NewPipe.getDownloader();
if(downloader == null) throw new NullPointerException("downloader is null");
if (downloader == null) throw new NullPointerException("downloader is null");
}
/**
@ -46,11 +47,12 @@ public abstract class Extractor{
/**
* Fetch the current page.
* @throws IOException if the page can not be loaded
*
* @throws IOException if the page can not be loaded
* @throws ExtractionException if the pages content is not understood
*/
public void fetchPage() throws IOException, ExtractionException {
if(pageFetched) return;
if (pageFetched) return;
onFetchPage(downloader);
pageFetched = true;
}
@ -65,8 +67,9 @@ public abstract class Extractor{
/**
* Fetch the current page.
*
* @param downloader the download to use
* @throws IOException if the page can not be loaded
* @throws IOException if the page can not be loaded
* @throws ExtractionException if the pages content is not understood
*/
public abstract void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException;
@ -78,6 +81,7 @@ public abstract class Extractor{
/**
* Get the name
*
* @return the name
* @throws ParsingException if the name cannot be extracted
*/
@ -93,10 +97,10 @@ public abstract class Extractor{
public String getUrl() throws ParsingException {
return linkHandler.getUrl();
}
@Nonnull
public String getBaseUrl() throws ParsingException {
return linkHandler.getBaseUrl();
return linkHandler.getBaseUrl();
}
@Nonnull

2
extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java
View File

@ -27,7 +27,7 @@ import java.util.List;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public abstract class InfoItemsCollector<I extends InfoItem, E extends InfoItemExtractor> implements Collector<I,E> {
public abstract class InfoItemsCollector<I extends InfoItem, E extends InfoItemExtractor> implements Collector<I, E> {
private final List<I> itemList = new ArrayList<>();
private final List<Throwable> errors = new ArrayList<>();

14
extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java
View File

@ -115,26 +115,28 @@ public enum MediaFormat {
}
/**
* Get the media format by it's id.
* Get the media format by its id.
*
* @param id the id
* @return the id of the media format or null.
*/
public static MediaFormat getFormatById(int id) {
for (MediaFormat vf: values()) {
for (MediaFormat vf : values()) {
if (vf.id == id) return vf;
}
return null;
}
public static MediaFormat getFromSuffix(String suffix) {
for (MediaFormat vf: values()) {
for (MediaFormat vf : values()) {
if (vf.suffix.equals(suffix)) return vf;
}
return null;
}
/**
* Get the name of the format
*
* @return the name of the format
*/
public String getName() {
@ -143,6 +145,7 @@ public enum MediaFormat {
/**
* Get the filename extension
*
* @return the filename extension
*/
public String getSuffix() {
@ -151,10 +154,11 @@ public enum MediaFormat {
/**
* Get the mime type
*
* @return the mime type
*/
public String getMimeType() {
return mimeType;
}
}

8
extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java
View File

@ -1,14 +1,14 @@
package org.schabi.newpipe.extractor;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import org.schabi.newpipe.extractor.services.media_ccc.MediaCCCService;
import org.schabi.newpipe.extractor.services.peertube.PeertubeService;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService;
import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
/*
* Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
* ServiceList.java is part of NewPipe.

36
extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java
View File

@ -1,19 +1,12 @@
package org.schabi.newpipe.extractor;
import java.util.Collections;
import java.util.List;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.feed.FeedExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.*;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
@ -24,6 +17,10 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import javax.annotation.Nullable;
import java.util.Collections;
import java.util.List;
/*
* Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
* StreamingService.java is part of NewPipe.
@ -65,7 +62,7 @@ public abstract class StreamingService {
public String getName() {
return name;
}
public List<MediaCapability> getMediaCapabilities() {
return mediaCapabilities;
}
@ -116,7 +113,7 @@ public abstract class StreamingService {
public String toString() {
return serviceId + ":" + serviceInfo.getName();
}
public abstract String getBaseUrl();
/*//////////////////////////////////////////////////////////////////////////
@ -173,6 +170,19 @@ public abstract class StreamingService {
*/
public abstract SubscriptionExtractor getSubscriptionExtractor();
/**
* This method decides which strategy will be chosen to fetch the feed. In YouTube, for example, a separate feed
* exists which is lightweight and made specifically to be used like this.
* <p>
* In services which there's no other way to retrieve them, null should be returned.
*
* @return a {@link FeedExtractor} instance or null.
*/
@Nullable
public FeedExtractor getFeedExtractor(String url) throws ExtractionException {
return null;
}
/**
* Must create a new instance of a KioskList implementation.
* @return a new KioskList instance
@ -253,12 +263,12 @@ public abstract class StreamingService {
public CommentsExtractor getCommentsExtractor(String url) throws ExtractionException {
ListLinkHandlerFactory llhf = getCommentsLHFactory();
if(null == llhf) {
if (llhf == null) {
return null;
}
return getCommentsExtractor(llhf.fromUrl(url));
}
/*//////////////////////////////////////////////////////////////////////////
// Utils
//////////////////////////////////////////////////////////////////////////*/

21
extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java
View File

@ -5,9 +5,7 @@ import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
@ -35,8 +33,8 @@ import java.io.IOException;
public class ChannelInfo extends ListInfo<StreamInfoItem> {
public ChannelInfo(int serviceId, ListLinkHandler linkHandler, String name) throws ParsingException {
super(serviceId, linkHandler, name);
public ChannelInfo(int serviceId, String id, String url, String originalUrl, String name, ListLinkHandler listLinkHandler) {
super(serviceId, id, url, originalUrl, name, listLinkHandler.getContentFilters(), listLinkHandler.getSortFilter());
}
public static ChannelInfo getInfo(String url) throws IOException, ExtractionException {
@ -57,15 +55,14 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
public static ChannelInfo getInfo(ChannelExtractor extractor) throws IOException, ExtractionException {
ChannelInfo info = new ChannelInfo(extractor.getServiceId(),
extractor.getLinkHandler(),
extractor.getName());
final int serviceId = extractor.getServiceId();
final String id = extractor.getId();
final String url = extractor.getUrl();
final String originalUrl = extractor.getOriginalUrl();
final String name = extractor.getName();
final ChannelInfo info = new ChannelInfo(serviceId, id, url, originalUrl, name, extractor.getLinkHandler());
try {
info.setOriginalUrl(extractor.getOriginalUrl());
} catch (Exception e) {
info.addError(e);
}
try {
info.setAvatarUrl(extractor.getAvatarUrl());
} catch (Exception e) {

2
extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemsCollector.java
View File

@ -33,7 +33,7 @@ public class ChannelInfoItemsCollector extends InfoItemsCollector<ChannelInfoIte
// important information
int serviceId = getServiceId();
String name = extractor.getName();
String url = extractor.getUrl();
String url = extractor.getUrl();
ChannelInfoItem resultItem = new ChannelInfoItem(serviceId, url, name);

31
extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java
View File

@ -1,7 +1,5 @@
package org.schabi.newpipe.extractor.comments;
import java.io.IOException;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.NewPipe;
@ -10,20 +8,21 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
public class CommentsInfo extends ListInfo<CommentsInfoItem>{
import java.io.IOException;
private CommentsInfo(int serviceId, ListLinkHandler listUrlIdHandler, String name) {
super(serviceId, listUrlIdHandler, name);
// TODO Auto-generated constructor stub
}
public static CommentsInfo getInfo(String url) throws IOException, ExtractionException {
public class CommentsInfo extends ListInfo<CommentsInfoItem> {
private CommentsInfo(int serviceId, ListLinkHandler listUrlIdHandler, String name) {
super(serviceId, listUrlIdHandler, name);
}
public static CommentsInfo getInfo(String url) throws IOException, ExtractionException {
return getInfo(NewPipe.getServiceByUrl(url), url);
}
public static CommentsInfo getInfo(StreamingService serviceByUrl, String url) throws ExtractionException, IOException {
return getInfo(serviceByUrl.getCommentsExtractor(url));
}
public static CommentsInfo getInfo(StreamingService serviceByUrl, String url) throws ExtractionException, IOException {
return getInfo(serviceByUrl.getCommentsExtractor(url));
}
private static CommentsInfo getInfo(CommentsExtractor commentsExtractor) throws IOException, ExtractionException {
// for services which do not have a comments extractor
@ -44,21 +43,21 @@ public class CommentsInfo extends ListInfo<CommentsInfoItem>{
return commentsInfo;
}
public static InfoItemsPage<CommentsInfoItem> getMoreItems(CommentsInfo commentsInfo, String pageUrl)
throws ExtractionException, IOException {
return getMoreItems(NewPipe.getService(commentsInfo.getServiceId()), commentsInfo, pageUrl);
}
public static InfoItemsPage<CommentsInfoItem> getMoreItems(StreamingService service, CommentsInfo commentsInfo,
String pageUrl) throws IOException, ExtractionException {
String pageUrl) throws IOException, ExtractionException {
if (null == commentsInfo.getCommentsExtractor()) {
commentsInfo.setCommentsExtractor(service.getCommentsExtractor(commentsInfo.getUrl()));
commentsInfo.getCommentsExtractor().fetchPage();
}
return commentsInfo.getCommentsExtractor().getPage(pageUrl);
}
private transient CommentsExtractor commentsExtractor;
public CommentsExtractor getCommentsExtractor() {

17
extractor/src/main/java/org/schabi/newpipe/extractor/feed/FeedExtractor.java Normal file
View File

@ -0,0 +1,17 @@
package org.schabi.newpipe.extractor.feed;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
/**
* This class helps to extract items from lightweight feeds that the services may provide.
* <p>
* YouTube is an example of a service that has this alternative available.
*/
public abstract class FeedExtractor extends ListExtractor<StreamInfoItem> {
public FeedExtractor(StreamingService service, ListLinkHandler listLinkHandler) {
super(service, listLinkHandler);
}
}

52
extractor/src/main/java/org/schabi/newpipe/extractor/feed/FeedInfo.java Normal file
View File

@ -0,0 +1,52 @@
package org.schabi.newpipe.extractor.feed;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import java.io.IOException;
import java.util.List;
public class FeedInfo extends ListInfo<StreamInfoItem> {
public FeedInfo(int serviceId, String id, String url, String originalUrl, String name, List<String> contentFilter, String sortFilter) {
super(serviceId, id, url, originalUrl, name, contentFilter, sortFilter);
}
public static FeedInfo getInfo(String url) throws IOException, ExtractionException {
return getInfo(NewPipe.getServiceByUrl(url), url);
}
public static FeedInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
final FeedExtractor extractor = service.getFeedExtractor(url);
if (extractor == null) {
throw new IllegalArgumentException("Service \"" + service.getServiceInfo().getName() + "\" doesn't support FeedExtractor.");
}
extractor.fetchPage();
return getInfo(extractor);
}
public static FeedInfo getInfo(FeedExtractor extractor) throws IOException, ExtractionException {
extractor.fetchPage();
final int serviceId = extractor.getServiceId();
final String id = extractor.getId();
final String url = extractor.getUrl();
final String originalUrl = extractor.getOriginalUrl();
final String name = extractor.getName();
final FeedInfo info = new FeedInfo(serviceId, id, url, originalUrl, name, null, null);
final InfoItemsPage<StreamInfoItem> itemsPage = ExtractorHelper.getItemsPageOrLogError(info, extractor);
info.setRelatedItems(itemsPage.getItems());
info.setNextPageUrl(itemsPage.getNextPageUrl());
return info;
}
}

1
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java
View File

@ -25,7 +25,6 @@ import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import javax.annotation.Nonnull;

27
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java
View File

@ -13,27 +13,30 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
public class KioskList {
public class KioskList {
public interface KioskExtractorFactory {
KioskExtractor createNewKiosk(final StreamingService streamingService,
final String url,
final String kioskId)
throws ExtractionException, IOException;
throws ExtractionException, IOException;
}
private final StreamingService service;
private final HashMap<String, KioskEntry> kioskList = new HashMap<>();
private String defaultKiosk = null;
@Nullable private Localization forcedLocalization;
@Nullable private ContentCountry forcedContentCountry;
@Nullable
private Localization forcedLocalization;
@Nullable
private ContentCountry forcedContentCountry;
private class KioskEntry {
public KioskEntry(KioskExtractorFactory ef, ListLinkHandlerFactory h) {
extractorFactory = ef;
handlerFactory = h;
}
final KioskExtractorFactory extractorFactory;
final ListLinkHandlerFactory handlerFactory;
}
@ -43,8 +46,8 @@ public class KioskList {
}
public void addKioskEntry(KioskExtractorFactory extractorFactory, ListLinkHandlerFactory handlerFactory, String id)
throws Exception {
if(kioskList.get(id) != null) {
throws Exception {
if (kioskList.get(id) != null) {
throw new Exception("Kiosk with type " + id + " already exists.");
}
kioskList.put(id, new KioskEntry(extractorFactory, handlerFactory));
@ -66,10 +69,10 @@ public class KioskList {
public KioskExtractor getDefaultKioskExtractor(String nextPageUrl, Localization localization)
throws ExtractionException, IOException {
if(defaultKiosk != null && !defaultKiosk.equals("")) {
if (defaultKiosk != null && !defaultKiosk.equals("")) {
return getExtractorById(defaultKiosk, nextPageUrl, localization);
} else {
if(!kioskList.isEmpty()) {
if (!kioskList.isEmpty()) {
// if not set get any entry
Object[] keySet = kioskList.keySet().toArray();
return getExtractorById(keySet[0].toString(), nextPageUrl, localization);
@ -91,7 +94,7 @@ public class KioskList {
public KioskExtractor getExtractorById(String kioskId, String nextPageUrl, Localization localization)
throws ExtractionException, IOException {
KioskEntry ke = kioskList.get(kioskId);
if(ke == null) {
if (ke == null) {
throw new ExtractionException("No kiosk found with the type: " + kioskId);
} else {
final KioskExtractor kioskExtractor = ke.extractorFactory.createNewKiosk(service,
@ -109,15 +112,15 @@ public class KioskList {
}
public KioskExtractor getExtractorByUrl(String url, String nextPageUrl)
throws ExtractionException, IOException{
throws ExtractionException, IOException {
return getExtractorByUrl(url, nextPageUrl, NewPipe.getPreferredLocalization());
}
public KioskExtractor getExtractorByUrl(String url, String nextPageUrl, Localization localization)
throws ExtractionException, IOException {
for(Map.Entry<String, KioskEntry> e : kioskList.entrySet()) {
for (Map.Entry<String, KioskEntry> e : kioskList.entrySet()) {
KioskEntry ke = e.getValue();
if(ke.handlerFactory.acceptUrl(url)) {
if (ke.handlerFactory.acceptUrl(url)) {
return getExtractorById(ke.handlerFactory.getId(url), nextPageUrl, localization);
}
}

8
extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandler.java
View File

@ -1,10 +1,10 @@
package org.schabi.newpipe.extractor.linkhandler;
import java.io.Serializable;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.Serializable;
public class LinkHandler implements Serializable {
protected final String originalUrl;
protected final String url;
@ -31,8 +31,8 @@ public class LinkHandler implements Serializable {
public String getId() {
return id;
}
public String getBaseUrl() throws ParsingException {
return Utils.getBaseUrl(url);
return Utils.getBaseUrl(url);
}
}

23
extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java
View File

@ -34,37 +34,38 @@ public abstract class LinkHandlerFactory {
public abstract String getUrl(String id) throws ParsingException;
public abstract boolean onAcceptUrl(final String url) throws ParsingException;
public String getUrl(String id, String baseUrl) throws ParsingException{
return getUrl(id);
public String getUrl(String id, String baseUrl) throws ParsingException {
return getUrl(id);
}
///////////////////////////////////
// Logic
///////////////////////////////////
public LinkHandler fromUrl(String url) throws ParsingException {
if (url == null) throw new IllegalArgumentException("url can not be null");
final String baseUrl = Utils.getBaseUrl(url);
return fromUrl(url, baseUrl);
}
public LinkHandler fromUrl(String url, String baseUrl) throws ParsingException {
if(url == null) throw new IllegalArgumentException("url can not be null");
if(!acceptUrl(url)) {
if (url == null) throw new IllegalArgumentException("url can not be null");
if (!acceptUrl(url)) {
throw new ParsingException("Malformed unacceptable url: " + url);
}
final String id = getId(url);
return new LinkHandler(url, getUrl(id,baseUrl), id);
return new LinkHandler(url, getUrl(id, baseUrl), id);
}
public LinkHandler fromId(String id) throws ParsingException {
if(id == null) throw new IllegalArgumentException("id can not be null");
if (id == null) throw new IllegalArgumentException("id can not be null");
final String url = getUrl(id);
return new LinkHandler(url, url, id);
}
public LinkHandler fromId(String id, String baseUrl) throws ParsingException {
if(id == null) throw new IllegalArgumentException("id can not be null");
if (id == null) throw new IllegalArgumentException("id can not be null");
final String url = getUrl(id, baseUrl);
return new LinkHandler(url, url, id);
}
@ -81,5 +82,5 @@ public abstract class LinkHandlerFactory {
throw fe;
}
}
}

35
extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java
View File

@ -1,29 +1,35 @@
package org.schabi.newpipe.extractor.linkhandler;
import java.util.ArrayList;
import java.util.List;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Utils;
import java.util.ArrayList;
import java.util.List;
public abstract class ListLinkHandlerFactory extends LinkHandlerFactory {
///////////////////////////////////
// To Override
///////////////////////////////////
public List<String> getContentFilter(String url) throws ParsingException { return new ArrayList<>(0);}
public String getSortFilter(String url) throws ParsingException {return ""; }
public List<String> getContentFilter(String url) throws ParsingException {
return new ArrayList<>(0);
}
public String getSortFilter(String url) throws ParsingException {
return "";
}
public abstract String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException;
public String getUrl(String id, List<String> contentFilter, String sortFilter, String baseUrl) throws ParsingException {
return getUrl(id, contentFilter, sortFilter);
return getUrl(id, contentFilter, sortFilter);
}
///////////////////////////////////
// Logic
///////////////////////////////////
@Override
public ListLinkHandler fromUrl(String url) throws ParsingException {
String baseUrl = Utils.getBaseUrl(url);
@ -32,7 +38,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory {
@Override
public ListLinkHandler fromUrl(String url, String baseUrl) throws ParsingException {
if(url == null) throw new IllegalArgumentException("url may not be null");
if (url == null) throw new IllegalArgumentException("url may not be null");
return new ListLinkHandler(super.fromUrl(url, baseUrl), getContentFilter(url), getSortFilter(url));
}
@ -41,7 +47,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory {
public ListLinkHandler fromId(String id) throws ParsingException {
return new ListLinkHandler(super.fromId(id), new ArrayList<String>(0), "");
}
@Override
public ListLinkHandler fromId(String id, String baseUrl) throws ParsingException {
return new ListLinkHandler(super.fromId(id, baseUrl), new ArrayList<String>(0), "");
@ -53,7 +59,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory {
final String url = getUrl(id, contentFilters, sortFilter);
return new ListLinkHandler(url, url, id, contentFilters, sortFilter);
}
public ListLinkHandler fromQuery(String id,
List<String> contentFilters,
String sortFilter, String baseUrl) throws ParsingException {
@ -61,10 +67,11 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory {
return new ListLinkHandler(url, url, id, contentFilters, sortFilter);
}
/**
* For makeing ListLinkHandlerFactory compatible with LinkHandlerFactory we need to override this,
* For making ListLinkHandlerFactory compatible with LinkHandlerFactory we need to override this,
* however it should not be overridden by the actual implementation.
*
* @param id
* @return the url coresponding to id without any filters applied
*/
@ -76,7 +83,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory {
public String getUrl(String id, String baseUrl) throws ParsingException {
return getUrl(id, new ArrayList<String>(0), "", baseUrl);
}
/**
* Will returns content filter the corresponding extractor can handle like "channels", "videos", "music", etc.
*

1
extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandler.java
View File

@ -24,6 +24,7 @@ public class SearchQueryHandler extends ListLinkHandler {
/**
* Returns the search string. Since ListQIHandler is based on ListLinkHandler
* getSearchString() is equivalent to calling getId().
*
* @return the search string
*/
public String getSearchString() {

16
extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java
View File

@ -13,14 +13,19 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
@Override
public abstract String getUrl(String querry, List<String> contentFilter, String sortFilter) throws ParsingException;
public String getSearchString(String url) { return "";}
public String getSearchString(String url) {
return "";
}
///////////////////////////////////
// Logic
///////////////////////////////////
@Override
public String getId(String url) { return getSearchString(url); }
public String getId(String url) {
return getSearchString(url);
}
@Override
public SearchQueryHandler fromQuery(String querry,
@ -34,10 +39,13 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
}
/**
* It's not mandatorry for NewPipe to handle the Url
* It's not mandatory for NewPipe to handle the Url
*
* @param url
* @return
*/
@Override
public boolean onAcceptUrl(String url) { return false; }
public boolean onAcceptUrl(String url) {
return false;
}
}

5
extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java
View File

@ -3,7 +3,10 @@ package org.schabi.newpipe.extractor.localization;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.Serializable;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
public class Localization implements Serializable {
public static final Localization DEFAULT = new Localization("en", "GB");

4
extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java
View File

@ -21,8 +21,9 @@ public class TimeAgoParser {
/**
* Creates a helper to parse upload dates in the format '2 days ago'.
* <p>
* Instantiate a new {@link TimeAgoParser} every time you extract a new batch of items.
* Instantiate a new {@link TimeAgoParser} every time you extract a new batch of items.
* </p>
*
* @param patternsHolder An object that holds the "time ago" patterns, special cases, and the language word separator.
*/
public TimeAgoParser(PatternsHolder patternsHolder) {
@ -164,6 +165,7 @@ public class TimeAgoParser {
/**
* Marks the time as approximated by setting minutes, seconds and milliseconds to 0.
*
* @param calendarTime Time to be marked as approximated
*/
private void markApproximatedTime(Calendar calendarTime) {

9
extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java
View File

@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector;
@ -34,7 +33,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
/**
* Collector for search results
*
* <p>
* This collector can handle the following extractor types:
* <ul>
* <li>{@link StreamInfoItemExtractor}</li>
@ -59,11 +58,11 @@ public class InfoItemsSearchCollector extends InfoItemsCollector<InfoItem, InfoI
@Override
public InfoItem extract(InfoItemExtractor extractor) throws ParsingException {
// Use the corresponding collector for each item extractor type
if(extractor instanceof StreamInfoItemExtractor) {
if (extractor instanceof StreamInfoItemExtractor) {
return streamCollector.extract((StreamInfoItemExtractor) extractor);
} else if(extractor instanceof ChannelInfoItemExtractor) {
} else if (extractor instanceof ChannelInfoItemExtractor) {
return userCollector.extract((ChannelInfoItemExtractor) extractor);
} else if(extractor instanceof PlaylistInfoItemExtractor) {
} else if (extractor instanceof PlaylistInfoItemExtractor) {
return playlistCollector.extract((PlaylistInfoItemExtractor) extractor);
} else {
throw new IllegalArgumentException("Invalid extractor type: " + extractor);

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java
View File

@ -55,7 +55,7 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
JsonArray events = conferenceData.getArray("events");
for(int i = 0; i < events.size(); i++) {
for (int i = 0; i < events.size(); i++) {
collector.commit(new MediaCCCStreamInfoItemExtractor(events.getObject(i)));
}
return new InfoItemsPage<>(collector, null);

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java
View File

@ -32,7 +32,7 @@ public class MediaCCCConferenceKiosk extends KioskExtractor<ChannelInfoItem> {
public InfoItemsPage<ChannelInfoItem> getInitialPage() throws IOException, ExtractionException {
JsonArray conferences = doc.getArray("conferences");
ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(getServiceId());
for(int i = 0; i < conferences.size(); i++) {
for (int i = 0; i < conferences.size(); i++) {
collector.commit(new MediaCCCConferenceInfoItemExtractor(conferences.getObject(i)));
}

20
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java
View File

@ -50,7 +50,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
InfoItemsSearchCollector searchItems = getInfoItemSearchCollector();
searchItems.reset();
if(getLinkHandler().getContentFilters().contains(CONFERENCES)
if (getLinkHandler().getContentFilters().contains(CONFERENCES)
|| getLinkHandler().getContentFilters().contains(ALL)
|| getLinkHandler().getContentFilters().isEmpty()) {
searchConferences(getSearchString(),
@ -58,7 +58,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
searchItems);
}
if(getLinkHandler().getContentFilters().contains(EVENTS)
if (getLinkHandler().getContentFilters().contains(EVENTS)
|| getLinkHandler().getContentFilters().contains(ALL)
|| getLinkHandler().getContentFilters().isEmpty()) {
JsonArray events = doc.getArray("events");
@ -82,8 +82,8 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
if(getLinkHandler().getContentFilters().contains(EVENTS)
|| getLinkHandler().getContentFilters().contains(ALL)
if (getLinkHandler().getContentFilters().contains(EVENTS)
|| getLinkHandler().getContentFilters().contains(ALL)
|| getLinkHandler().getContentFilters().isEmpty()) {
final String site;
final String url = getUrl();
@ -94,17 +94,17 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
throw new ExtractionException("Could not parse json.", jpe);
}
}
if(getLinkHandler().getContentFilters().contains(CONFERENCES)
if (getLinkHandler().getContentFilters().contains(CONFERENCES)
|| getLinkHandler().getContentFilters().contains(ALL)
|| getLinkHandler().getContentFilters().isEmpty())
conferenceKiosk.fetchPage();
conferenceKiosk.fetchPage();
}
private void searchConferences(String searchString,
List<ChannelInfoItem> channelItems,
InfoItemsSearchCollector collector) {
for(final ChannelInfoItem item : channelItems) {
if(item.getName().toUpperCase().contains(
List<ChannelInfoItem> channelItems,
InfoItemsSearchCollector collector) {
for (final ChannelInfoItem item : channelItems) {
if (item.getName().toUpperCase().contains(
searchString.toUpperCase())) {
collector.commit(new ChannelInfoItemExtractor() {
@Override

60
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java
View File

@ -17,6 +17,7 @@ import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
public class MediaCCCStreamExtractor extends StreamExtractor {
@ -47,8 +48,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getDescription() throws ParsingException {
return data.getString("description");
public Description getDescription() throws ParsingException {
return new Description(data.getString("description"), Description.PLAIN_TEXT);
}
@Override
@ -116,17 +117,17 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
final JsonArray recordings = data.getArray("recordings");
final List<AudioStream> audioStreams = new ArrayList<>();
for(int i = 0; i < recordings.size(); i++) {
for (int i = 0; i < recordings.size(); i++) {
final JsonObject recording = recordings.getObject(i);
final String mimeType = recording.getString("mime_type");
if(mimeType.startsWith("audio")) {
if (mimeType.startsWith("audio")) {
//first we need to resolve the actual video data from CDN
final MediaFormat mediaFormat;
if(mimeType.endsWith("opus")) {
if (mimeType.endsWith("opus")) {
mediaFormat = MediaFormat.OPUS;
} else if(mimeType.endsWith("mpeg")) {
} else if (mimeType.endsWith("mpeg")) {
mediaFormat = MediaFormat.MP3;
} else if(mimeType.endsWith("ogg")){
} else if (mimeType.endsWith("ogg")) {
mediaFormat = MediaFormat.OGG;
} else {
throw new ExtractionException("Unknown media format: " + mimeType);
@ -142,16 +143,16 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
final JsonArray recordings = data.getArray("recordings");
final List<VideoStream> videoStreams = new ArrayList<>();
for(int i = 0; i < recordings.size(); i++) {
for (int i = 0; i < recordings.size(); i++) {
final JsonObject recording = recordings.getObject(i);
final String mimeType = recording.getString("mime_type");
if(mimeType.startsWith("video")) {
if (mimeType.startsWith("video")) {
//first we need to resolve the actual video data from CDN
final MediaFormat mediaFormat;
if(mimeType.endsWith("webm")) {
if (mimeType.endsWith("webm")) {
mediaFormat = MediaFormat.WEBM;
} else if(mimeType.endsWith("mp4")) {
} else if (mimeType.endsWith("mp4")) {
mediaFormat = MediaFormat.MPEG_4;
} else {
throw new ExtractionException("Unknown media format: " + mimeType);
@ -225,4 +226,41 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
public String getOriginalUrl() throws ParsingException {
return data.getString("frontend_link");
}
@Override
public String getHost() throws ParsingException {
return "";
}
@Override
public String getPrivacy() throws ParsingException {
return "";
}
@Override
public String getCategory() throws ParsingException {
return "";
}
@Override
public String getLicence() throws ParsingException {
return "";
}
@Override
public Locale getLanguageInfo() throws ParsingException {
return null;
}
@Nonnull
@Override
public List<String> getTags() throws ParsingException {
return new ArrayList<>();
}
@Nonnull
@Override
public String getSupportInfo() throws ParsingException {
return "";
}
}

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java
View File

@ -15,9 +15,9 @@ public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory
@Override
public String getId(String url) throws ParsingException {
if(url.startsWith("https://api.media.ccc.de/public/conferences/")) {
if (url.startsWith("https://api.media.ccc.de/public/conferences/")) {
return url.replace("https://api.media.ccc.de/public/conferences/", "");
} else if(url.startsWith("https://media.ccc.de/c/")) {
} else if (url.startsWith("https://media.ccc.de/c/")) {
return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", url);
} else {
throw new ParsingException("Could not get id from url: " + url);

1
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java
View File

@ -1,7 +1,6 @@
package org.schabi.newpipe.extractor.services.media_ccc.linkHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import java.util.List;

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java
View File

@ -15,7 +15,7 @@ public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory
@Override
public String[] getAvailableContentFilter() {
return new String[] {
return new String[]{
ALL,
CONFERENCES,
EVENTS

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java
View File

@ -7,8 +7,8 @@ public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory {
@Override
public String getId(String url) throws ParsingException {
if(url.startsWith("https://api.media.ccc.de/public/events/") &&
!url.contains("?q=")) {
if (url.startsWith("https://api.media.ccc.de/public/events/") &&
!url.contains("?q=")) {
return url.replace("https://api.media.ccc.de/public/events/", "");
}
throw new ParsingException("Could not get id from url: " + url);

31
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java
View File

@ -1,7 +1,8 @@
package org.schabi.newpipe.extractor.services.peertube;
import java.io.IOException;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -10,22 +11,20 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import java.io.IOException;
public class PeertubeInstance {
private final String url;
private String name;
public static final PeertubeInstance defaultInstance = new PeertubeInstance("https://framatube.org", "FramaTube");
public PeertubeInstance(String url) {
this.url = url;
this.name = "PeerTube";
}
public PeertubeInstance(String url , String name) {
public PeertubeInstance(String url, String name) {
this.url = url;
this.name = name;
}
@ -33,22 +32,22 @@ public class PeertubeInstance {
public String getUrl() {
return url;
}
public void fetchInstanceMetaData() throws Exception {
Downloader downloader = NewPipe.getDownloader();
Response response = null;
try {
response = downloader.get(url + "/api/v1/config");
} catch (ReCaptchaException | IOException e) {
throw new Exception("unable to configure instance " + url, e);
}
if(null == response || StringUtil.isBlank(response.responseBody())) {
if (response == null || StringUtil.isBlank(response.responseBody())) {
throw new Exception("unable to configure instance " + url);
}
try {
try {
JsonObject json = JsonParser.object().from(response.responseBody());
this.name = JsonUtils.getString(json, "instance.name");
} catch (JsonParserException | ParsingException e) {
@ -59,5 +58,5 @@ public class PeertubeInstance {
public String getName() {
return name;
}
}

22
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java
View File

@ -1,32 +1,34 @@
package org.schabi.newpipe.extractor.services.peertube;
import com.grack.nanojson.JsonObject;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import com.grack.nanojson.JsonObject;
import java.util.TimeZone;
public class PeertubeParsingHelper {
private PeertubeParsingHelper() {
}
public static void validate(JsonObject json) throws ContentNotAvailableException {
String error = json.getString("error");
if(!StringUtil.isBlank(error)) {
if (!StringUtil.isBlank(error)) {
throw new ContentNotAvailableException(error);
}
}
public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
Date date;
try {
date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse(textualUploadDate);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
date = sdf.parse(textualUploadDate);
} catch (ParseException e) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
}

44
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java
View File

@ -1,49 +1,35 @@
package org.schabi.newpipe.extractor.services.peertube;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.*;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSuggestionExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.extractors.*;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.*;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
public class PeertubeService extends StreamingService {
private PeertubeInstance instance;
public PeertubeService(int id) {
this(id, PeertubeInstance.defaultInstance);
}
public PeertubeService(int id, PeertubeInstance instance) {
super(id, "PeerTube", asList(VIDEO, COMMENTS));
this.instance = instance;
this.instance = instance;
}
@Override
@ -117,15 +103,15 @@ public class PeertubeService extends StreamingService {
public String getBaseUrl() {
return instance.getUrl();
}
public PeertubeInstance getInstance() {
return this.instance;
}
public void setInstance(PeertubeInstance instance) {
this.instance = instance;
}
@Override
public KioskList getKioskList() throws ExtractionException {
KioskList.KioskExtractorFactory kioskFactory = new KioskList.KioskExtractorFactory() {
@ -155,6 +141,6 @@ public class PeertubeService extends StreamingService {
return list;
}
}

65
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java
View File

@ -1,7 +1,9 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import java.io.IOException;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
@ -17,21 +19,18 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import java.io.IOException;
public class PeertubeChannelExtractor extends ChannelExtractor {
private static final String START_KEY = "start";
private static final String COUNT_KEY = "count";
private static final int ITEMS_PER_PAGE = 12;
private static final String START_PATTERN = "start=(\\d*)";
private InfoItemsPage<StreamInfoItem> initPage;
private long total;
private JsonObject json;
private final String baseUrl;
@ -45,7 +44,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
String value;
try {
value = JsonUtils.getString(json, "avatar.path");
}catch(Exception e) {
} catch (Exception e) {
value = "/client/assets/images/default-avatar.png";
}
return baseUrl + value;
@ -71,7 +70,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
public String getDescription() throws ParsingException {
try {
return JsonUtils.getString(json, "description");
}catch(ParsingException e) {
} catch (ParsingException e) {
return "No description";
}
}
@ -86,18 +85,18 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
}catch(Exception e) {
} catch (Exception e) {
throw new ParsingException("unable to extract channel streams", e);
}
for(Object c: contents) {
if(c instanceof JsonObject) {
for (Object c : contents) {
if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
collector.commit(extractor);
}
}
}
@Override
@ -110,26 +109,26 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if(null != response && !StringUtil.isBlank(response.responseBody())) {
if (null != response && !StringUtil.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
throw new ParsingException("Could not parse json data for kiosk info", e);
}
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
if(json != null) {
if (json != null) {
PeertubeParsingHelper.validate(json);
Number number = JsonUtils.getNumber(json, "total");
if(number != null) this.total = number.longValue();
if (number != null) this.total = number.longValue();
collectStreamsFrom(collector, json, pageUrl);
} else {
throw new ExtractionException("Unable to get peertube kiosk info");
throw new ExtractionException("Unable to get PeerTube kiosk info");
}
return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl));
}
private String getNextPageUrl(String prevPageUrl) {
String prevStart;
@ -138,30 +137,30 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
} catch (RegexException e) {
return "";
}
if(StringUtil.isBlank(prevStart)) return "";
if (StringUtil.isBlank(prevStart)) return "";
long nextStart = 0;
try {
nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE;
} catch (NumberFormatException e) {
return "";
}
if(nextStart >= total) {
if (nextStart >= total) {
return "";
}else {
} else {
return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart));
}
}
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
Response response = downloader.get(getUrl());
if(null != response && null != response.responseBody()) {
if (null != response && null != response.responseBody()) {
setInitialData(response.responseBody());
}else {
throw new ExtractionException("Unable to extract peertube channel data");
} else {
throw new ExtractionException("Unable to extract PeerTube channel data");
}
String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
this.initPage = getPage(pageUrl);
}
@ -172,14 +171,14 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
} catch (JsonParserException e) {
throw new ExtractionException("Unable to extract peertube channel data", e);
}
if(null == json) throw new ExtractionException("Unable to extract peertube channel data");
if (json == null) throw new ExtractionException("Unable to extract PeerTube channel data");
}
@Override
public String getName() throws ParsingException {
return JsonUtils.getString(json, "displayName");
}
@Override
public String getOriginalUrl() throws ParsingException {
return baseUrl + "/accounts/" + getId();

43
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java
View File

@ -1,7 +1,8 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import java.io.IOException;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
@ -16,9 +17,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import java.io.IOException;
public class PeertubeCommentsExtractor extends CommentsExtractor {
@ -26,10 +25,10 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
private static final String COUNT_KEY = "count";
private static final int ITEMS_PER_PAGE = 12;
private static final String START_PATTERN = "start=(\\d*)";
private InfoItemsPage<CommentsInfoItem> initPage;
private long total;
public PeertubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) {
super(service, uiHandler);
}
@ -38,7 +37,7 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
public String getName() throws ParsingException {
return "Comments";
}
@Override
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws IOException, ExtractionException {
super.fetchPage();
@ -49,18 +48,18 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
}catch(Exception e) {
} catch (Exception e) {
throw new ParsingException("unable to extract comments info", e);
}
for(Object c: contents) {
if(c instanceof JsonObject) {
for (Object c : contents) {
if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
PeertubeCommentsInfoItemExtractor extractor = new PeertubeCommentsInfoItemExtractor(item, this);
collector.commit(extractor);
}
}
}
@Override
@ -73,18 +72,18 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
public InfoItemsPage<CommentsInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if(null != response && !StringUtil.isBlank(response.responseBody())) {
if (null != response && !StringUtil.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
throw new ParsingException("Could not parse json data for comments info", e);
}
}
CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
if(json != null) {
if (json != null) {
Number number = JsonUtils.getNumber(json, "total");
if(number != null) this.total = number.longValue();
if (number != null) this.total = number.longValue();
collectStreamsFrom(collector, json, pageUrl);
} else {
throw new ExtractionException("Unable to get peertube comments info");
@ -97,7 +96,7 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
String pageUrl = getUrl() + "?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
this.initPage = getPage(pageUrl);
}
private String getNextPageUrl(String prevPageUrl) {
String prevStart;
try {
@ -105,17 +104,17 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
} catch (RegexException e) {
return "";
}
if(StringUtil.isBlank(prevStart)) return "";
if (StringUtil.isBlank(prevStart)) return "";
long nextStart = 0;
try {
nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE;
} catch (NumberFormatException e) {
return "";
}
if(nextStart >= total) {
if (nextStart >= total) {
return "";
}else {
} else {
return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart));
}
}

17
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java
View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.ServiceList;
@ -9,8 +10,6 @@ import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import com.grack.nanojson.JsonObject;
public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
@ -34,7 +33,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
String value;
try {
value = JsonUtils.getString(item, "account.avatar.path");
}catch(Exception e) {
} catch (Exception e) {
value = "/client/assets/images/default-avatar.png";
}
return baseUrl + value;
@ -49,13 +48,13 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
public String getTextualPublishedTime() throws ParsingException {
return JsonUtils.getString(item, "createdAt");
}
@Override
public DateWrapper getPublishedTime() throws ParsingException {
String textualUploadDate = getTextualPublishedTime();
return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate));
}
@Override
public int getLikeCount() throws ParsingException {
return -1;
@ -67,7 +66,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
try {
Document doc = Jsoup.parse(htmlText);
return doc.body().text();
}catch(Exception e) {
} catch (Exception e) {
return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", "");
}
}
@ -83,7 +82,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
String value;
try {
value = JsonUtils.getString(item, "account.avatar.path");
}catch(Exception e) {
} catch (Exception e) {
value = "/client/assets/images/default-avatar.png";
}
return baseUrl + value;
@ -91,7 +90,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
@Override
public String getAuthorName() throws ParsingException {
return JsonUtils.getString(item, "account.displayName");
return JsonUtils.getString(item, "account.name") + "@" + JsonUtils.getString(item, "account.host");
}
@Override
@ -100,5 +99,5 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
String host = JsonUtils.getString(item, "account.host");
return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl();
}
}

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java
View File

@ -1,7 +1,5 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import java.io.IOException;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -10,7 +8,9 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
public class PeertubePlaylistExtractor extends PlaylistExtractor{
import java.io.IOException;
public class PeertubePlaylistExtractor extends PlaylistExtractor {
public PeertubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
@ -73,8 +73,7 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor{
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
// TODO Auto-generated method stub
}
@Override

51
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java
View File

@ -1,7 +1,8 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import java.io.IOException;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.InfoItemExtractor;
@ -18,9 +19,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import java.io.IOException;
public class PeertubeSearchExtractor extends SearchExtractor {
@ -28,10 +27,10 @@ public class PeertubeSearchExtractor extends SearchExtractor {
private static final String COUNT_KEY = "count";
private static final int ITEMS_PER_PAGE = 12;
private static final String START_PATTERN = "start=(\\d*)";
private InfoItemsPage<InfoItem> initPage;
private long total;
public PeertubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
super(service, linkHandler);
}
@ -40,7 +39,7 @@ public class PeertubeSearchExtractor extends SearchExtractor {
public String getSearchSuggestion() throws ParsingException {
return null;
}
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
super.fetchPage();
@ -48,27 +47,27 @@ public class PeertubeSearchExtractor extends SearchExtractor {
}
private InfoItemsCollector<InfoItem, InfoItemExtractor> collectStreamsFrom(JsonObject json) throws ParsingException {
final InfoItemsSearchCollector collector = getInfoItemSearchCollector();
JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
}catch(Exception e) {
} catch (Exception e) {
throw new ParsingException("unable to extract search info", e);
}
String baseUrl = getBaseUrl();
for(Object c: contents) {
if(c instanceof JsonObject) {
for (Object c : contents) {
if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
collector.commit(extractor);
}
}
return collector;
}
@Override
@ -81,17 +80,17 @@ public class PeertubeSearchExtractor extends SearchExtractor {
public InfoItemsPage<InfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if(null != response && !StringUtil.isBlank(response.responseBody())) {
if (null != response && !StringUtil.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
throw new ParsingException("Could not parse json data for search info", e);
}
}
if(json != null) {
if (json != null) {
Number number = JsonUtils.getNumber(json, "total");
if(number != null) this.total = number.longValue();
if (number != null) this.total = number.longValue();
return new InfoItemsPage<>(collectStreamsFrom(json), getNextPageUrl(pageUrl));
} else {
throw new ExtractionException("Unable to get peertube search info");
@ -103,7 +102,7 @@ public class PeertubeSearchExtractor extends SearchExtractor {
String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
this.initPage = getPage(pageUrl);
}
private String getNextPageUrl(String prevPageUrl) {
String prevStart;
try {
@ -111,20 +110,20 @@ public class PeertubeSearchExtractor extends SearchExtractor {
} catch (RegexException e) {
return "";
}
if(StringUtil.isBlank(prevStart)) return "";
if (StringUtil.isBlank(prevStart)) return "";
long nextStart = 0;
try {
nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE;
} catch (NumberFormatException e) {
return "";
}
if(nextStart >= total) {
if (nextStart >= total) {
return "";
}else {
} else {
return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart));
}
}
}

183
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java
View File

@ -1,15 +1,12 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
@ -20,34 +17,30 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
public class PeertubeStreamExtractor extends StreamExtractor {
private final String baseUrl;
private JsonObject json;
private List<SubtitlesStream> subtitles = new ArrayList<>();
private final String baseUrl;
public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler) throws ParsingException {
super(service, linkHandler);
this.baseUrl = getBaseUrl();
}
@Override
public String getTextualUploadDate() throws ParsingException {
return JsonUtils.getString(json, "publishedAt");
@ -63,24 +56,42 @@ public class PeertubeStreamExtractor extends StreamExtractor {
return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate));
}
@Override
public String getThumbnailUrl() throws ParsingException {
return baseUrl + JsonUtils.getString(json, "thumbnailPath");
return baseUrl + JsonUtils.getString(json, "previewPath");
}
@Override
public String getDescription() throws ParsingException {
public Description getDescription() throws ParsingException {
String text;
try {
return JsonUtils.getString(json, "description");
}catch(ParsingException e) {
return "No description";
text = JsonUtils.getString(json, "description");
} catch (ParsingException e) {
return Description.emptyDescription;
}
if (text.length() == 250 && text.substring(247).equals("...")) {
//if description is shortened, get full description
Downloader dl = NewPipe.getDownloader();
try {
Response response = dl.get(getUrl() + "/description");
JsonObject jsonObject = JsonParser.object().from(response.responseBody());
text = JsonUtils.getString(jsonObject, "description");
} catch (ReCaptchaException | IOException | JsonParserException e) {
e.printStackTrace();
}
}
return new Description(text, Description.MARKDOWN);
}
@Override
public int getAgeLimit() throws ParsingException {
return NO_AGE_LIMIT;
boolean isNSFW = JsonUtils.getBoolean(json, "nsfw");
if (isNSFW) {
return 18;
} else {
return NO_AGE_LIMIT;
}
}
@Override
@ -130,7 +141,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
String value;
try {
value = JsonUtils.getString(json, "account.avatar.path");
}catch(Exception e) {
} catch (Exception e) {
value = "/client/assets/images/default-avatar.png";
}
return baseUrl + value;
@ -157,8 +168,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
List<VideoStream> videoStreams = new ArrayList<>();
try {
JsonArray streams = json.getArray("files", new JsonArray());
for(Object s: streams) {
if(!(s instanceof JsonObject)) continue;
for (Object s : streams) {
if (!(s instanceof JsonObject)) continue;
JsonObject stream = (JsonObject) s;
String url = JsonUtils.getString(stream, "fileUrl");
String torrentUrl = JsonUtils.getString(stream, "torrentUrl");
@ -192,8 +203,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
@Override
public List<SubtitlesStream> getSubtitles(final MediaFormat format) throws IOException, ExtractionException {
List<SubtitlesStream> filteredSubs = new ArrayList<>();
for(SubtitlesStream sub: subtitles) {
if(sub.getFormat() == format) {
for (SubtitlesStream sub : subtitles) {
if (sub.getFormat() == format) {
filteredSubs.add(sub);
}
}
@ -215,29 +226,40 @@ public class PeertubeStreamExtractor extends StreamExtractor {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
List<String> tags = getTags();
String apiUrl = null;
if(!tags.isEmpty()) {
if (!tags.isEmpty()) {
apiUrl = getRelatedStreamsUrl(tags);
}else {
} else {
apiUrl = getUploaderUrl() + "/videos?start=0&count=8";
}
if(!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl);
if (!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl);
return collector;
}
private List<String> getTags(){
@Override
public List<String> getTags() {
try {
return (List) JsonUtils.getArray(json, "tags");
} catch (Exception e) {
return Collections.emptyList();
}
}
@Nonnull
@Override
public String getSupportInfo() throws ParsingException {
try {
return JsonUtils.getString(json, "support");
} catch (ParsingException e) {
return "";
}
}
private String getRelatedStreamsUrl(List<String> tags) throws UnsupportedEncodingException {
String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT;
StringBuilder params = new StringBuilder();
params.append("start=0&count=8&sort=-createdAt");
for(String tag : tags) {
for (String tag : tags) {
params.append("&tagsOneOf=");
params.append(URLEncoder.encode(tag, "UTF-8"));
}
@ -247,38 +269,38 @@ public class PeertubeStreamExtractor extends StreamExtractor {
private void getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, IOException, ParsingException {
Response response = getDownloader().get(apiUrl);
JsonObject relatedVideosJson = null;
if(null != response && !StringUtil.isBlank(response.responseBody())) {
if (null != response && !StringUtil.isBlank(response.responseBody())) {
try {
relatedVideosJson = JsonParser.object().from(response.responseBody());
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json data for related videos", e);
}
}
if(relatedVideosJson != null) {
if (relatedVideosJson != null) {
collectStreamsFrom(collector, relatedVideosJson);
}
}
private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json) throws ParsingException {
JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
}catch(Exception e) {
} catch (Exception e) {
throw new ParsingException("unable to extract related videos", e);
}
for(Object c: contents) {
if(c instanceof JsonObject) {
for (Object c : contents) {
if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
//do not add the same stream in related streams
if(!extractor.getUrl().equals(getUrl())) collector.commit(extractor);
if (!extractor.getUrl().equals(getUrl())) collector.commit(extractor);
}
}
}
@Override
public String getErrorMessage() {
@ -288,12 +310,12 @@ public class PeertubeStreamExtractor extends StreamExtractor {
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
Response response = downloader.get(getUrl());
if(null != response && null != response.responseBody()) {
if (null != response && null != response.responseBody()) {
setInitialData(response.responseBody());
}else {
} else {
throw new ExtractionException("Unable to extract peertube channel data");
}
loadSubtitles();
}
@ -303,24 +325,25 @@ public class PeertubeStreamExtractor extends StreamExtractor {
} catch (JsonParserException e) {
throw new ExtractionException("Unable to extract peertube stream data", e);
}
if(null == json) throw new ExtractionException("Unable to extract peertube stream data");
if (null == json) throw new ExtractionException("Unable to extract peertube stream data");
PeertubeParsingHelper.validate(json);
}
private void loadSubtitles() {
if (subtitles.isEmpty()) {
try {
Response response = getDownloader().get(getUrl() + "/captions");
Response response = getDownloader().get(getUrl() + "/captions");
JsonObject captionsJson = JsonParser.object().from(response.responseBody());
JsonArray captions = JsonUtils.getArray(captionsJson, "data");
for(Object c: captions) {
if(c instanceof JsonObject) {
JsonObject caption = (JsonObject)c;
for (Object c : captions) {
if (c instanceof JsonObject) {
JsonObject caption = (JsonObject) c;
String url = baseUrl + JsonUtils.getString(caption, "captionPath");
String languageCode = JsonUtils.getString(caption, "language.id");
String ext = url.substring(url.lastIndexOf(".") + 1);
MediaFormat fmt = MediaFormat.getFromSuffix(ext);
if(fmt != null && languageCode != null) subtitles.add(new SubtitlesStream(fmt, languageCode, url, false));
if (fmt != null && languageCode != null)
subtitles.add(new SubtitlesStream(fmt, languageCode, url, false));
}
}
} catch (Exception e) {
@ -339,4 +362,32 @@ public class PeertubeStreamExtractor extends StreamExtractor {
return baseUrl + "/videos/watch/" + getId();
}
@Override
public String getHost() throws ParsingException {
return JsonUtils.getString(json, "account.host");
}
@Override
public String getPrivacy() throws ParsingException {
return JsonUtils.getString(json, "privacy.label");
}
@Override
public String getCategory() throws ParsingException {
return JsonUtils.getString(json, "category.label");
}
@Override
public String getLicence() throws ParsingException {
return JsonUtils.getString(json, "licence.label");
}
@Override
public Locale getLanguageInfo() throws ParsingException {
try {
return new Locale(JsonUtils.getString(json, "language.id"));
} catch (ParsingException e) {
return null;
}
}
}

27
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java
View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
@ -8,58 +9,56 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import com.grack.nanojson.JsonObject;
public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
protected final JsonObject item;
private final String baseUrl;
public PeertubeStreamInfoItemExtractor(JsonObject item, String baseUrl) {
this.item = item;
this.baseUrl = baseUrl;
}
@Override
public String getUrl() throws ParsingException {
String uuid = JsonUtils.getString(item, "uuid");
return ServiceList.PeerTube.getStreamLHFactory().fromId(uuid, baseUrl).getUrl();
}
@Override
public String getThumbnailUrl() throws ParsingException {
String value = JsonUtils.getString(item, "thumbnailPath");
return baseUrl + value;
}
@Override
public String getName() throws ParsingException {
return JsonUtils.getString(item, "name");
}
@Override
public boolean isAd() throws ParsingException {
return false;
}
@Override
public long getViewCount() throws ParsingException {
Number value = JsonUtils.getNumber(item, "views");
return value.longValue();
}
@Override
public String getUploaderUrl() throws ParsingException {
String name = JsonUtils.getString(item, "account.name");
String host = JsonUtils.getString(item, "account.host");
return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl();
}
@Override
public String getUploaderName() throws ParsingException {
return JsonUtils.getString(item, "account.displayName");
}
@Override
public String getTextualUploadDate() throws ParsingException {
return JsonUtils.getString(item, "publishedAt");
@ -75,12 +74,12 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor
return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate));
}
@Override
public StreamType getStreamType() throws ParsingException {
return StreamType.VIDEO_STREAM;
}
@Override
public long getDuration() throws ParsingException {
Number value = JsonUtils.getNumber(item, "duration");

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java
View File

@ -1,10 +1,10 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import java.util.List;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import java.util.List;
public class PeertubeSubscriptionExtractor extends SubscriptionExtractor {
public PeertubeSubscriptionExtractor(StreamingService service, List<ContentSource> supportedSources) {

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java
View File

@ -1,14 +1,14 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
public class PeertubeSuggestionExtractor extends SuggestionExtractor{
import java.io.IOException;
import java.util.Collections;
import java.util.List;
public class PeertubeSuggestionExtractor extends SuggestionExtractor {
public PeertubeSuggestionExtractor(StreamingService service) {
super(service);

43
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java
View File

@ -1,7 +1,8 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import java.io.IOException;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -16,17 +17,15 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import java.io.IOException;
public class PeertubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
private static final String START_KEY = "start";
private static final String COUNT_KEY = "count";
private static final int ITEMS_PER_PAGE = 12;
private static final String START_PATTERN = "start=(\\d*)";
private InfoItemsPage<StreamInfoItem> initPage;
private long total;
@ -49,19 +48,19 @@ public class PeertubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
}catch(Exception e) {
throw new ParsingException("unable to extract kiosk info", e);
} catch (Exception e) {
throw new ParsingException("Unable to extract kiosk info", e);
}
String baseUrl = getBaseUrl();
for(Object c: contents) {
if(c instanceof JsonObject) {
for (Object c : contents) {
if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
collector.commit(extractor);
}
}
}
@Override
@ -74,18 +73,18 @@ public class PeertubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if(null != response && !StringUtil.isBlank(response.responseBody())) {
if (null != response && !StringUtil.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
throw new ParsingException("Could not parse json data for kiosk info", e);
}
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
if(json != null) {
if (json != null) {
Number number = JsonUtils.getNumber(json, "total");
if(number != null) this.total = number.longValue();
if (number != null) this.total = number.longValue();
collectStreamsFrom(collector, json, pageUrl);
} else {
throw new ExtractionException("Unable to get peertube kiosk info");
@ -98,7 +97,7 @@ public class PeertubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
this.initPage = getPage(pageUrl);
}
private String getNextPageUrl(String prevPageUrl) {
String prevStart;
try {
@ -106,17 +105,17 @@ public class PeertubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
} catch (RegexException e) {
return "";
}
if(StringUtil.isBlank(prevStart)) return "";
if (StringUtil.isBlank(prevStart)) return "";
long nextStart = 0;
try {
nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE;
} catch (NumberFormatException e) {
return "";
}
if(nextStart >= total) {
if (nextStart >= total) {
return "";
}else {
} else {
return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart));
}
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java
View File

@ -1,12 +1,12 @@
package org.schabi.newpipe.extractor.services.peertube.linkHandler;
import java.util.List;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Parser;
import java.util.List;
public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
private static final PeertubeChannelLinkHandlerFactory instance = new PeertubeChannelLinkHandlerFactory();
@ -27,7 +27,7 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
String baseUrl = ServiceList.PeerTube.getBaseUrl();
return getUrl(id, contentFilters, searchFilter, baseUrl);
}
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter, String baseUrl)
throws ParsingException {

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeCommentsLinkHandlerFactory.java
View File

@ -1,13 +1,13 @@
package org.schabi.newpipe.extractor.services.peertube.linkHandler;
import java.util.List;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Parser;
import java.util.List;
public class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
private static final PeertubeCommentsLinkHandlerFactory instance = new PeertubeCommentsLinkHandlerFactory();
@ -33,10 +33,10 @@ public class PeertubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
String baseUrl = ServiceList.PeerTube.getBaseUrl();
return getUrl(id, contentFilter, sortFilter, baseUrl);
}
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter, String baseUrl) throws ParsingException {
return baseUrl + String.format(COMMENTS_ENDPOINT, id);
}
}

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java
View File

@ -1,13 +1,13 @@
package org.schabi.newpipe.extractor.services.peertube.linkHandler;
import java.util.List;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Parser;
import java.util.List;
public class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
private static final PeertubePlaylistLinkHandlerFactory instance = new PeertubePlaylistLinkHandlerFactory();
@ -23,12 +23,12 @@ public class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
String baseUrl = ServiceList.PeerTube.getBaseUrl();
return getUrl(id, contentFilters, sortFilter, baseUrl);
}
@Override
public String getUrl(String id, List<String> contentFilters, String sortFilter, String baseUrl) {
return baseUrl + VIDEO_CHANNELS_ENDPOINT + id;
}
@Override
public String getId(String url) throws ParsingException {
return Parser.matchGroup1(ID_PATTERN, url);

14
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java
View File

@ -1,13 +1,13 @@
package org.schabi.newpipe.extractor.services.peertube.linkHandler;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.List;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.List;
public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String CHARSET_UTF_8 = "UTF-8";
@ -21,9 +21,9 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
@Override
public String getUrl(String searchString, List<String> contentFilters, String sortFilter) throws ParsingException {
String baseUrl = ServiceList.PeerTube.getBaseUrl();
return getUrl(searchString, contentFilters, sortFilter, baseUrl);
return getUrl(searchString, contentFilters, sortFilter, baseUrl);
}
@Override
public String getUrl(String searchString, List<String> contentFilters, String sortFilter, String baseUrl) throws ParsingException {
try {
@ -38,6 +38,6 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
@Override
public String[] getAvailableContentFilter() {
return new String[] { VIDEOS };
return new String[]{VIDEOS};
}
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java
View File

@ -24,7 +24,7 @@ public class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory {
String baseUrl = ServiceList.PeerTube.getBaseUrl();
return getUrl(id, baseUrl);
}
@Override
public String getUrl(String id, String baseUrl) {
return baseUrl + VIDEO_ENDPOINT + id;

26
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java
View File

@ -1,26 +1,26 @@
package org.schabi.newpipe.extractor.services.peertube.linkHandler;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
private static final PeertubeTrendingLinkHandlerFactory instance = new PeertubeTrendingLinkHandlerFactory();
public static final Map<String, String> KIOSK_MAP;
public static final Map<String, String> REVERSE_KIOSK_MAP;
public static final String KIOSK_TRENDING = "Trending";
public static final String KIOSK_MOST_LIKED = "Most liked";
public static final String KIOSK_RECENT = "Recently added";
public static final String KIOSK_LOCAL = "Local";
static {
Map<String, String> map = new HashMap<>();
map.put(KIOSK_TRENDING, "%s/api/v1/videos?sort=-trending");
@ -28,24 +28,24 @@ public class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
map.put(KIOSK_RECENT, "%s/api/v1/videos?sort=-publishedAt");
map.put(KIOSK_LOCAL, "%s/api/v1/videos?sort=-publishedAt&filter=local");
KIOSK_MAP = Collections.unmodifiableMap(map);
Map<String, String> reverseMap = new HashMap<>();
for(Map.Entry<String, String> entry : KIOSK_MAP.entrySet()){
for (Map.Entry<String, String> entry : KIOSK_MAP.entrySet()) {
reverseMap.put(entry.getValue(), entry.getKey());
}
REVERSE_KIOSK_MAP = Collections.unmodifiableMap(reverseMap);
}
public static PeertubeTrendingLinkHandlerFactory getInstance() {
return instance;
}
@Override
public String getUrl(String id, List<String> contentFilters, String sortFilter) {
String baseUrl = ServiceList.PeerTube.getBaseUrl();
return getUrl(id, contentFilters, sortFilter, baseUrl);
}
@Override
public String getUrl(String id, List<String> contentFilters, String sortFilter, String baseUrl) {
return String.format(KIOSK_MAP.get(id), baseUrl);

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java
View File

@ -4,9 +4,9 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@ -86,7 +86,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
if(streamInfoItemsCollector == null) {
if (streamInfoItemsCollector == null) {
computeNextPageAndGetStreams();
}
return new InfoItemsPage<>(streamInfoItemsCollector, getNextPageUrl());
@ -94,7 +94,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Override
public String getNextPageUrl() throws ExtractionException {
if(nextPageUrl == null) {
if (nextPageUrl == null) {
computeNextPageAndGetStreams();
}
return nextPageUrl;

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelLinkHandlerFactory.java
View File

@ -1,7 +1,7 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java
View File

@ -1,7 +1,7 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@ -12,8 +12,8 @@ import javax.annotation.Nonnull;
import java.io.IOException;
public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
private StreamInfoItemsCollector collector = null;
private String nextPageUrl = null;
private StreamInfoItemsCollector collector = null;
private String nextPageUrl = null;
public SoundcloudChartsExtractor(StreamingService service,
ListLinkHandler linkHandler,
@ -68,7 +68,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
if(nextPageUrl == null) {
if (nextPageUrl == null) {
computNextPageAndStreams();
}
return nextPageUrl;
@ -77,7 +77,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
if(collector == null) {
if (collector == null) {
computNextPageAndStreams();
}
return new InfoItemsPage<>(collector, getNextPageUrl());

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View File

@ -33,7 +33,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudParsingHelper {
private static final String HARDCODED_CLIENT_ID = "cZQKaMjH39KNADF4y2aeFtVqNSpgoKVj"; // Updated on 08/02/20
private static String clientId;
private SoundcloudParsingHelper() {
}
@ -103,7 +103,7 @@ public class SoundcloudParsingHelper {
/**
* Call the endpoint "/resolve" of the api.<p>
*
* <p>
* See https://developers.soundcloud.com/docs/api/reference#resolve
*/
public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException {

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java
View File

@ -3,8 +3,8 @@ package org.schabi.newpipe.extractor.services.soundcloud;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistLinkHandlerFactory.java
View File

@ -1,7 +1,7 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java
View File

@ -4,7 +4,10 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java
View File

@ -25,7 +25,7 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto
try {
String url = "https://api-v2.soundcloud.com/search";
if(contentFilter.size() > 0) {
if (contentFilter.size() > 0) {
switch (contentFilter.get(0)) {
case TRACKS:
url += "/tracks";
@ -58,7 +58,7 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto
@Override
public String[] getAvailableContentFilter() {
return new String[] {
return new String[]{
ALL,
TRACKS,
USERS,

27
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java
View File

@ -1,31 +1,26 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.*;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
public class SoundcloudService extends StreamingService {
public SoundcloudService(int id) {
super(id, "SoundCloud", singletonList(AUDIO));
}
@Override
public String getBaseUrl() {
return "https://soundcloud.com";
@ -110,15 +105,15 @@ public class SoundcloudService extends StreamingService {
return new SoundcloudSubscriptionExtractor(this);
}
@Override
public ListLinkHandlerFactory getCommentsLHFactory() {
return null;
}
@Override
public ListLinkHandlerFactory getCommentsLHFactory() {
return null;
}
@Override
@Override
public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler)
throws ExtractionException {
return null;
}
}

49
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java
View File

@ -4,7 +4,9 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -20,6 +22,7 @@ import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
public class SoundcloudStreamExtractor extends StreamExtractor {
private JsonObject track;
@ -73,10 +76,9 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
return artworkUrlBetterResolution;
}
@Nonnull
@Override
public String getDescription() {
return track.getString("description");
public Description getDescription() {
return new Description(track.getString("description"), Description.PLAIN_TEXT);
}
@Override
@ -183,7 +185,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
try {
JsonObject mp3UrlObject = JsonParser.object().from(res);
// Links in this file are also only valid for a short period.
audioStreams.add(new AudioStream(mp3UrlObject.getString("url"),
audioStreams.add(new AudioStream(mp3UrlObject.getString("url"),
MediaFormat.MP3, 128));
} catch (JsonParserException e) {
throw new ParsingException("Could not parse streamable url", e);
@ -254,4 +256,41 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
public String getErrorMessage() {
return null;
}
@Override
public String getHost() throws ParsingException {
return "";
}
@Override
public String getPrivacy() throws ParsingException {
return "";
}
@Override
public String getCategory() throws ParsingException {
return "";
}
@Override
public String getLicence() throws ParsingException {
return "";
}
@Override
public Locale getLanguageInfo() throws ParsingException {
return null;
}
@Nonnull
@Override
public List<String> getTags() throws ParsingException {
return new ArrayList<>();
}
@Nonnull
@Override
public String getSupportInfo() throws ParsingException {
return "";
}
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactory.java
View File

@ -1,7 +1,7 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java
View File

@ -4,9 +4,9 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;

48
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View File

@ -1,47 +1,29 @@
package org.schabi.newpipe.extractor.services.youtube;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
import java.util.List;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.feed.FeedExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.*;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscriptionExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSuggestionExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.extractors.*;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.*;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import javax.annotation.Nonnull;
import java.util.List;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.*;
/*
* Created by Christian Schabesberger on 23.08.15.
*
@ -72,7 +54,7 @@ public class YoutubeService extends StreamingService {
public String getBaseUrl() {
return "https://youtube.com";
}
@Override
public LinkHandlerFactory getStreamLHFactory() {
return YoutubeStreamLinkHandlerFactory.getInstance();
@ -129,7 +111,7 @@ public class YoutubeService extends StreamingService {
public KioskExtractor createNewKiosk(StreamingService streamingService,
String url,
String id)
throws ExtractionException {
throws ExtractionException {
return new YoutubeTrendingExtractor(YoutubeService.this,
new YoutubeTrendingLinkHandlerFactory().fromUrl(url), id);
}
@ -147,6 +129,12 @@ public class YoutubeService extends StreamingService {
return new YoutubeSubscriptionExtractor(this);
}
@Nonnull
@Override
public FeedExtractor getFeedExtractor(final String channelUrl) throws ExtractionException {
return new YoutubeFeedExtractor(this, getChannelLHFactory().fromUrl(channelUrl));
}
@Override
public ListLinkHandlerFactory getCommentsLHFactory() {
return YoutubeCommentsLinkHandlerFactory.getInstance();

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -46,7 +46,6 @@ import java.io.IOException;
@SuppressWarnings("WeakerAccess")
public class YoutubeChannelExtractor extends ChannelExtractor {
/*package-private*/ static final String CHANNEL_URL_BASE = "https://www.youtube.com/channel/";
private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000";
private Document doc;
@ -130,7 +129,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getFeedUrl() throws ParsingException {
try {
return CHANNEL_FEED_BASE + getId();
return YoutubeParsingHelper.getFeedUrlFrom(getId());
} catch (Exception e) {
throw new ParsingException("Could not get feed url", e);
}
@ -197,7 +196,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl)
throws ParsingException {
throws ParsingException {
String loadMoreHtmlDataRaw = ajaxJson.getString("load_more_widget_html");
if (!loadMoreHtmlDataRaw.isEmpty()) {
return getNextPageUrlFrom(Jsoup.parse(loadMoreHtmlDataRaw, pageUrl));

29
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java
View File

@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
@ -22,7 +21,9 @@ import javax.annotation.Nonnull;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.*;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import static java.util.Collections.singletonList;
@ -58,14 +59,14 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
}
private String getNextPageUrl(JsonObject ajaxJson) throws IOException, ParsingException {
JsonArray arr;
try {
arr = JsonUtils.getArray(ajaxJson, "response.continuationContents.commentSectionContinuation.continuations");
} catch (Exception e) {
return "";
}
if(arr.isEmpty()) {
if (arr.isEmpty()) {
return "";
}
String continuation;
@ -107,11 +108,11 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
}
private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonObject ajaxJson) throws ParsingException {
JsonArray contents;
try {
contents = JsonUtils.getArray(ajaxJson, "response.continuationContents.commentSectionContinuation.items");
}catch(Exception e) {
} catch (Exception e) {
//no comments
return;
}
@ -119,12 +120,12 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
List<Object> comments;
try {
comments = JsonUtils.getValues(contents, "commentThreadRenderer.comment.commentRenderer");
}catch(Exception e) {
} catch (Exception e) {
throw new ParsingException("unable to get parse youtube comments", e);
}
for(Object c: comments) {
if(c instanceof JsonObject) {
for (Object c : comments) {
if (c instanceof JsonObject) {
CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor((JsonObject) c, getUrl(), getTimeAgoParser());
collector.commit(extractor);
}
@ -132,7 +133,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
}
private void fetchTitle(JsonArray contents) {
if(null == title) {
if (title == null) {
try {
title = getYoutubeText(JsonUtils.getObject(contents.getObject(0), "commentThreadRenderer.commentTargetTitle"));
} catch (Exception e) {
@ -190,7 +191,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
int endIndex = doc.indexOf(end, beginIndex);
return doc.substring(beginIndex, endIndex);
}
public static String getYoutubeText(@Nonnull JsonObject object) throws ParsingException {
try {
return JsonUtils.getString(object, "simpleText");
@ -198,7 +199,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
try {
JsonArray arr = JsonUtils.getArray(object, "runs");
String result = "";
for(int i=0; i<arr.size();i++) {
for (int i = 0; i < arr.size(); i++) {
result = result + JsonUtils.getString(arr.getObject(i), "text");
}
return result;
@ -207,5 +208,5 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
}
}
}
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java
View File

@ -4,8 +4,8 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;

82
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java Normal file
View File

@ -0,0 +1,82 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.feed.FeedExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
public class YoutubeFeedExtractor extends FeedExtractor {
public YoutubeFeedExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
}
private Document document;
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String channelIdOrUser = getLinkHandler().getId();
final String feedUrl = YoutubeParsingHelper.getFeedUrlFrom(channelIdOrUser);
final Response response = downloader.get(feedUrl);
document = Jsoup.parse(response.responseBody());
}
@Nonnull
@Override
public ListExtractor.InfoItemsPage<StreamInfoItem> getInitialPage() {
final Elements entries = document.select("feed > entry");
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
for (Element entryElement : entries) {
collector.commit(new YoutubeFeedInfoItemExtractor(entryElement));
}
return new InfoItemsPage<>(collector, null);
}
@Nonnull
@Override
public String getId() {
return document.getElementsByTag("yt:channelId").first().text();
}
@Nonnull
@Override
public String getUrl() {
return document.select("feed > author > uri").first().text();
}
@Nonnull
@Override
public String getName() {
return document.select("feed > author > name").first().text();
}
@Override
public String getNextPageUrl() {
return null;
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) {
return null;
}
@Override
public boolean hasNextPage() {
return false;
}
}

94
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java Normal file
View File

@ -0,0 +1,94 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import javax.annotation.Nullable;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.TimeZone;
public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor {
private final Element entryElement;
public YoutubeFeedInfoItemExtractor(Element entryElement) {
this.entryElement = entryElement;
}
@Override
public StreamType getStreamType() {
// It is not possible to determine the stream type using the feed endpoint.
// All entries are considered a video stream.
return StreamType.VIDEO_STREAM;
}
@Override
public boolean isAd() {
return false;
}
@Override
public long getDuration() {
// Not available when fetching through the feed endpoint.
return -1;
}
@Override
public long getViewCount() {
return Long.parseLong(entryElement.getElementsByTag("media:statistics").first().attr("views"));
}
@Override
public String getUploaderName() {
return entryElement.select("author > name").first().text();
}
@Override
public String getUploaderUrl() {
return entryElement.select("author > uri").first().text();
}
@Nullable
@Override
public String getTextualUploadDate() {
return entryElement.getElementsByTag("published").first().text();
}
@Nullable
@Override
public DateWrapper getUploadDate() throws ParsingException {
final Date date;
try {
final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss+00:00");
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
date = dateFormat.parse(getTextualUploadDate());
} catch (ParseException e) {
throw new ParsingException("Could not parse date (\"" + getTextualUploadDate() + "\")", e);
}
final Calendar calendar = Calendar.getInstance();
calendar.setTime(date);
return new DateWrapper(calendar);
}
@Override
public String getName() {
return entryElement.getElementsByTag("title").first().text();
}
@Override
public String getUrl() {
return entryElement.getElementsByTag("link").first().attr("href");
}
@Override
public String getThumbnailUrl() {
return entryElement.getElementsByTag("media:thumbnail").first().attr("url");
}
}

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -68,7 +68,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getBannerUrl() {
return ""; // Banner can't be handled by frontend right now.
// Whoever is willing to implement this should also implement this in the fornt end
// Whoever is willing to implement this should also implement it in the frontend.
}
@Override
@ -157,7 +157,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
private String getNextPageUrlFromAjax(final JsonObject pageJson, final String pageUrl)
throws ParsingException{
throws ParsingException {
String nextPageHtml = pageJson.getString("load_more_widget_html");
if (!nextPageHtml.isEmpty()) {
return getNextPageUrlFrom(Jsoup.parse(nextPageHtml, pageUrl));
@ -191,7 +191,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (final Element li : element.children()) {
if(isDeletedItem(li)) {
if (isDeletedItem(li)) {
continue;
}
@ -242,7 +242,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
private Element getUploaderLink() {
// should always be present since we filter deleted items
if(uploaderLink == null) {
if (uploaderLink == null) {
uploaderLink = li.select("div[class=pl-video-owner] a").first();
}
return uploaderLink;
@ -284,6 +284,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
/**
* Check if the playlist item is deleted
*
* @param li the list item
* @return true if the item is deleted
*/

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java
View File

@ -53,7 +53,7 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
.select("ul[class=\"yt-lockup-meta-info\"]")
.select("li").select("a").first();
if(a != null) {
if (a != null) {
return a.attr("abs:href");
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -104,7 +104,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
"&page=" + Integer.toString(pageNr + 1));
}
private InfoItemsSearchCollector collectItems(Document doc) throws NothingFoundException {
private InfoItemsSearchCollector collectItems(Document doc) throws NothingFoundException {
InfoItemsSearchCollector collector = getInfoItemSearchCollector();
collector.reset();

351
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -22,8 +22,8 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.*;
@ -106,19 +106,22 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public String getName() throws ParsingException {
assertPageFetched();
String name = getStringFromMetaData("title");
if(name == null) {
// Fallback to HTML method
try {
return playerResponse.getObject("videoDetails").getString("title");
} catch (Exception e) {
// fallback HTML method
String name = null;
try {
name = doc.select("meta[name=title]").attr(CONTENT);
} catch (Exception e) {
throw new ParsingException("Could not get the title", e);
} catch (Exception ignored) {
}
if (name == null) {
throw new ParsingException("Could not get name", e);
}
return name;
}
if(name == null || name.isEmpty()) {
throw new ParsingException("Could not get the title");
}
return name;
}
@Override
@ -128,9 +131,18 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
try {
return doc.select("meta[itemprop=datePublished]").attr(CONTENT);
} catch (Exception e) {//todo: add fallback method
throw new ParsingException("Could not get upload date", e);
return playerResponse.getObject("microformat").getObject("playerMicroformatRenderer").getString("publishDate");
} catch (Exception e) {
String uploadDate = null;
try {
uploadDate = doc.select("meta[itemprop=datePublished]").attr(CONTENT);
} catch (Exception ignored) {
}
if (uploadDate == null) {
throw new ParsingException("Could not get upload date", e);
}
return uploadDate;
}
}
@ -142,41 +154,46 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return null;
}
return new DateWrapper(YoutubeParsingHelper.parseDateFrom(textualUploadDate));
return new DateWrapper(YoutubeParsingHelper.parseDateFrom(textualUploadDate), true);
}
@Nonnull
@Override
public String getThumbnailUrl() throws ParsingException {
assertPageFetched();
// Try to get high resolution thumbnail first, if it fails, use low res from the player instead
try {
return doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href");
} catch (Exception ignored) {
// Try other method...
}
JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails");
// the last thumbnail is the one with the highest resolution
return thumbnails.getObject(thumbnails.size() - 1).getString("url");
try {
if (playerArgs != null && playerArgs.isString("thumbnail_url")) return playerArgs.getString("thumbnail_url");
} catch (Exception ignored) {
// Try other method...
}
try {
return videoInfoPage.get("thumbnail_url");
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
String url = null;
try {
url = doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href");
} catch (Exception ignored) {}
if (url == null) {
throw new ParsingException("Could not get thumbnail url", e);
}
return url;
}
}
@Nonnull
@Override
public String getDescription() throws ParsingException {
public Description getDescription() throws ParsingException {
assertPageFetched();
try {
return parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html());
// first try to get html-formatted description
return new Description(parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html()), Description.HTML);
} catch (Exception e) {
throw new ParsingException("Could not get the description", e);
try {
// fallback to raw non-html description
return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT);
} catch (Exception ignored) {
throw new ParsingException("Could not get the description", e);
}
}
}
@ -202,7 +219,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private String parseHtmlAndGetFullLinks(String descriptionHtml)
throws MalformedURLException, UnsupportedEncodingException, ParsingException {
final Document description = Jsoup.parse(descriptionHtml, getUrl());
for(Element a : description.select("a")) {
for (Element a : description.select("a")) {
final String rawUrl = a.attr("abs:href");
final URL redirectLink = new URL(rawUrl);
@ -227,22 +244,22 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// getUrl() is https://www.youtube.com/watch?v=..., never youtu.be, never &t=.
a.attr("href", getUrl() + setTimestamp);
} else if((queryString = redirectLink.getQuery()) != null) {
} else if ((queryString = redirectLink.getQuery()) != null) {
// if the query string is null we are not dealing with a redirect link,
// so we don't need to override it.
final String link =
Parser.compatParseMap(queryString).get("q");
if(link != null) {
if (link != null) {
// if link is null the a tag is a hashtag.
// They refer to the youtube search. We do not handle them.
a.text(link);
a.attr("href", link);
} else if(redirectLink.toString().contains("https://www.youtube.com/")) {
} else if (redirectLink.toString().contains("https://www.youtube.com/")) {
a.text(redirectLink.toString());
a.attr("href", redirectLink.toString());
}
} else if(redirectLink.toString().contains("https://www.youtube.com/")) {
} else if (redirectLink.toString().contains("https://www.youtube.com/")) {
descriptionHtml = descriptionHtml.replace(rawUrl, redirectLink.toString());
a.text(redirectLink.toString());
a.attr("href", redirectLink.toString());
@ -269,25 +286,22 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public long getLength() throws ParsingException {
assertPageFetched();
// try getting duration from playerargs
try {
String durationMs = playerResponse
.getObject("streamingData")
.getArray("formats")
.getObject(0)
.getString("approxDurationMs");
return Long.parseLong(durationMs)/1000;
} catch (Exception e) {
}
//try getting value from age gated video
try {
String duration = playerResponse
.getObject("videoDetails")
.getString("lengthSeconds");
return Long.parseLong(duration);
} catch (Exception e) {
throw new ParsingException("Every methode to get the duration has failed: ", e);
try {
String durationMs = playerResponse
.getObject("streamingData")
.getArray("formats")
.getObject(0)
.getString("approxDurationMs");
return Math.round(Long.parseLong(durationMs) / 1000f);
} catch (Exception ignored) {
throw new ParsingException("Could not get duration", e);
}
}
}
@ -307,11 +321,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try {
if (getStreamType().equals(StreamType.LIVE_STREAM)) {
return getLiveStreamWatchingCount();
} else {
return Long.parseLong(playerResponse.getObject("videoDetails").getString("viewCount"));
}
} catch (Exception e) {
try {
return Long.parseLong(doc.select("meta[itemprop=interactionCount]").attr(CONTENT));
} catch (Exception ignored) {
throw new ParsingException("Could not get view count", e);
}
return Long.parseLong(doc.select("meta[itemprop=interactionCount]").attr(CONTENT));
} catch (Exception e) {//todo: find fallback method
throw new ParsingException("Could not get number of views", e);
}
}
@ -373,7 +391,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try {
likesString = button.select("span.yt-uix-button-content").first().text();
} catch (NullPointerException e) {
//if this kicks in our button has no content and therefore likes/dislikes are disabled
//if this kicks in our button has no content and therefore ratings must be disabled
if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
throw new ParsingException("Ratings are enabled even though the like button is missing", e);
}
return -1;
}
return Integer.parseInt(Utils.removeNonDigitCharacters(likesString));
@ -393,7 +414,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try {
dislikesString = button.select("span.yt-uix-button-content").first().text();
} catch (NullPointerException e) {
//if this kicks in our button has no content and therefore likes/dislikes are disabled
//if this kicks in our button has no content and therefore ratings must be disabled
if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
throw new ParsingException("Ratings are enabled even though the dislike button is missing", e);
}
return -1;
}
return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString));
@ -409,60 +433,59 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public String getUploaderUrl() throws ParsingException {
assertPageFetched();
try {
return doc.select("div[class=\"yt-user-info\"]").first().children()
.select("a").first().attr("abs:href");
return "https://www.youtube.com/channel/" +
playerResponse.getObject("videoDetails").getString("channelId");
} catch (Exception e) {
throw new ParsingException("Could not get channel link", e);
}
}
String uploaderUrl = null;
try {
uploaderUrl = doc.select("div[class=\"yt-user-info\"]").first().children()
.select("a").first().attr("abs:href");
} catch (Exception ignored) {}
@Nullable
private String getStringFromMetaData(String field) {
assertPageFetched();
String value = null;
if(playerArgs != null) {
// This can not fail
value = playerArgs.getString(field);
if (uploaderUrl == null) {
throw new ParsingException("Could not get channel link", e);
}
return uploaderUrl;
}
if(value == null) {
// This can not fail too
value = videoInfoPage.get(field);
}
return value;
}
@Nonnull
@Override
public String getUploaderName() throws ParsingException {
assertPageFetched();
String name = getStringFromMetaData("author");
if(name == null) {
try {
return playerResponse.getObject("videoDetails").getString("author");
} catch (Exception e) {
String name = null;
try {
// Fallback to HTML method
name = doc.select("div.yt-user-info").first().text();
} catch (Exception e) {
throw new ParsingException("Could not get uploader name", e);
} catch (Exception ignored) {}
if (name == null) {
throw new ParsingException("Could not get uploader name");
}
return name;
}
if(name == null || name.isEmpty()) {
throw new ParsingException("Could not get uploader name");
}
return name;
}
@Nonnull
@Override
public String getUploaderAvatarUrl() throws ParsingException {
assertPageFetched();
String uploaderAvatarUrl = null;
try {
return doc.select("a[class*=\"yt-user-photo\"]").first()
uploaderAvatarUrl = doc.select("a[class*=\"yt-user-photo\"]").first()
.select("img").first()
.attr("abs:data-thumb");
} catch (Exception e) {//todo: add fallback method
throw new ParsingException("Could not get uploader thumbnail URL.", e);
throw new ParsingException("Could not get uploader avatar url", e);
}
if (uploaderAvatarUrl == null) {
throw new ParsingException("Could not get uploader avatar url");
}
return uploaderAvatarUrl;
}
@Nonnull
@ -590,12 +613,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public StreamType getStreamType() throws ParsingException {
assertPageFetched();
try {
if (playerArgs != null && (playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live") ||
(!playerResponse.getObject("streamingData").has(FORMATS)))) {
if (!playerResponse.getObject("streamingData").has(FORMATS) ||
(playerArgs != null && playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))) {
return StreamType.LIVE_STREAM;
}
} catch (Exception e) {
throw new ParsingException("Could not get hls manifest url", e);
throw new ParsingException("Could not get stream type", e);
}
return StreamType.VIDEO_STREAM;
}
@ -699,8 +722,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final String playerUrl;
// Check if the video is age restricted
Elements e = doc.select("meta[property=\"og:restrictions:age\"]");
if (!e.isEmpty()) {
if (!doc.select("meta[property=\"og:restrictions:age\"]").isEmpty()) {
final EmbeddedInfo info = getEmbeddedInfo();
final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts);
final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody();
@ -1058,60 +1080,97 @@ public class YoutubeStreamExtractor extends StreamExtractor {
};
}
@Nonnull
@Override
public List<Frameset> getFrames() throws ExtractionException {
try {
final String script = doc.select("#player-api").first().siblingElements().select("script").html();
int p = script.indexOf("ytplayer.config");
if (p == -1) {
return Collections.emptyList();
}
p = script.indexOf('{', p);
int e = script.indexOf("ytplayer.load", p);
if (e == -1) {
return Collections.emptyList();
}
JsonObject jo = JsonParser.object().from(script.substring(p, e - 1));
final String resp = jo.getObject("args").getString("player_response");
jo = JsonParser.object().from(resp);
final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|");
final String url = spec[0];
final ArrayList<Frameset> result = new ArrayList<>(spec.length - 1);
for (int i = 1; i < spec.length; ++i) {
final String[] parts = spec[i].split("#");
if (parts.length != 8) {
continue;
}
final int frameWidth = Integer.parseInt(parts[0]);
final int frameHeight = Integer.parseInt(parts[1]);
final int totalCount = Integer.parseInt(parts[2]);
final int framesPerPageX = Integer.parseInt(parts[3]);
final int framesPerPageY = Integer.parseInt(parts[4]);
final String baseUrl = url.replace("$L", String.valueOf(i - 1)).replace("$N", parts[6]) + "&sigh=" + parts[7];
final List<String> urls;
if (baseUrl.contains("$M")) {
final int totalPages = (int) Math.ceil(totalCount / (double) (framesPerPageX * framesPerPageY));
urls = new ArrayList<>(totalPages);
for (int j = 0; j < totalPages; j++) {
urls.add(baseUrl.replace("$M", String.valueOf(j)));
}
} else {
urls = Collections.singletonList(baseUrl);
}
result.add(new Frameset(
urls,
frameWidth,
frameHeight,
totalCount,
framesPerPageX,
framesPerPageY
));
}
result.trimToSize();
return result;
} catch (Exception e) {
throw new ExtractionException(e);
}
}
@Nonnull
@Override
public List<Frameset> getFrames() throws ExtractionException {
try {
final String script = doc.select("#player-api").first().siblingElements().select("script").html();
int p = script.indexOf("ytplayer.config");
if (p == -1) {
return Collections.emptyList();
}
p = script.indexOf('{', p);
int e = script.indexOf("ytplayer.load", p);
if (e == -1) {
return Collections.emptyList();
}
JsonObject jo = JsonParser.object().from(script.substring(p, e - 1));
final String resp = jo.getObject("args").getString("player_response");
jo = JsonParser.object().from(resp);
final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|");
final String url = spec[0];
final ArrayList<Frameset> result = new ArrayList<>(spec.length - 1);
for (int i = 1; i < spec.length; ++i) {
final String[] parts = spec[i].split("#");
if (parts.length != 8) {
continue;
}
final int frameWidth = Integer.parseInt(parts[0]);
final int frameHeight = Integer.parseInt(parts[1]);
final int totalCount = Integer.parseInt(parts[2]);
final int framesPerPageX = Integer.parseInt(parts[3]);
final int framesPerPageY = Integer.parseInt(parts[4]);
final String baseUrl = url.replace("$L", String.valueOf(i - 1)).replace("$N", parts[6]) + "&sigh=" + parts[7];
final List<String> urls;
if (baseUrl.contains("$M")) {
final int totalPages = (int) Math.ceil(totalCount / (double) (framesPerPageX * framesPerPageY));
urls = new ArrayList<>(totalPages);
for (int j = 0; j < totalPages; j++) {
urls.add(baseUrl.replace("$M", String.valueOf(j)));
}
} else {
urls = Collections.singletonList(baseUrl);
}
result.add(new Frameset(
urls,
frameWidth,
frameHeight,
totalCount,
framesPerPageX,
framesPerPageY
));
}
result.trimToSize();
return result;
} catch (Exception e) {
throw new ExtractionException(e);
}
}
@Override
public String getHost() throws ParsingException {
return "";
}
@Override
public String getPrivacy() throws ParsingException {
return "";
}
@Override
public String getCategory() throws ParsingException {
return "";
}
@Override
public String getLicence() throws ParsingException {
return "";
}
@Override
public Locale getLanguageInfo() throws ParsingException {
return null;
}
@Nonnull
@Override
public List<String> getTags() throws ParsingException {
return new ArrayList<>();
}
@Nonnull
@Override
public String getSupportInfo() throws ParsingException {
return "";
}
}

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java
View File

@ -3,8 +3,8 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
@ -42,6 +42,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
/**
* Creates an extractor of StreamInfoItems from a YouTube page.
*
* @param item The page element
* @param timeAgoParser A parser of the textual dates or {@code null}.
*/
@ -68,10 +69,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
private boolean isPremiumVideo() {
Element premiumSpan = item.select("span[class=\"standalone-collection-badge-renderer-red-text\"]").first();
if(premiumSpan == null) return false;
if (premiumSpan == null) return false;
// if this span has text it most likely says ("Free Video") so we can play this
if(premiumSpan.hasText()) return false;
if (premiumSpan.hasText()) return false;
return true;
}
@ -233,7 +234,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
return Long.parseLong(Utils.removeNonDigitCharacters(input));
} catch (NumberFormatException e) {
// if this happens the video probably has no views
if (!input.isEmpty()){
if (!input.isEmpty()) {
return 0;
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java
View File

@ -3,9 +3,9 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
@ -57,12 +57,12 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
String response = dl.get(url, getExtractorLocalization()).responseBody();
// trim JSONP part "JP(...)"
response = response.substring(3, response.length()-1);
response = response.substring(3, response.length() - 1);
try {
JsonArray collection = JsonParser.array().from(response).getArray(1, new JsonArray());
for (Object suggestion : collection) {
if (!(suggestion instanceof JsonArray)) continue;
String suggestionStr = ((JsonArray)suggestion).getString(0);
String suggestionStr = ((JsonArray) suggestion).getString(0);
if (suggestionStr == null) continue;
suggestions.add(suggestionStr);
}

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View File

@ -88,8 +88,8 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for(Element ul : uls) {
for(final Element li : ul.children()) {
for (Element ul : uls) {
for (final Element li : ul.children()) {
final Element el = li.select("div[class*=\"yt-lockup-dismissable\"]").first();
collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) {
@Override

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java
View File

@ -38,6 +38,9 @@ public class YoutubeParsingHelper {
private YoutubeParsingHelper() {
}
private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
private static final String[] RECAPTCHA_DETECTION_SELECTORS = {
"form[action*=\"/das_captcha\"]",
"input[name*=\"action_recaptcha_verify\"]"
@ -118,6 +121,16 @@ public class YoutubeParsingHelper {
+ Long.parseLong(seconds);
}
public static String getFeedUrlFrom(final String channelIdOrUser) {
if (channelIdOrUser.startsWith("user/")) {
return FEED_BASE_USER + channelIdOrUser.replace("user/", "");
} else if (channelIdOrUser.startsWith("channel/")) {
return FEED_BASE_CHANNEL_ID + channelIdOrUser.replace("channel/", "");
} else {
return FEED_BASE_CHANNEL_ID + channelIdOrUser;
}
}
public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
Date date;
try {

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java
View File

@ -31,7 +31,7 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
}
String path = urlObj.getPath();
if (!path.equals("/watch" ) && !path.equals("/playlist")) {
if (!path.equals("/watch") && !path.equals("/playlist")) {
throw new ParsingException("the url given is neither a video nor a playlist URL");
}

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java
View File

@ -26,7 +26,7 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
final String url = "https://www.youtube.com/results"
+ "?q=" + URLEncoder.encode(searchString, CHARSET_UTF_8);
if(contentFilters.size() > 0) {
if (contentFilters.size() > 0) {
switch (contentFilters.get(0)) {
case VIDEOS: return url + "&sp=EgIQAVAU";
case CHANNELS: return url + "&sp=EgIQAlAU";
@ -44,7 +44,7 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
@Override
public String[] getAvailableContentFilter() {
return new String[] {
return new String[]{
ALL,
VIDEOS,
CHANNELS,

31
extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java Normal file
View File

@ -0,0 +1,31 @@
package org.schabi.newpipe.extractor.stream;
import java.io.Serializable;
public class Description implements Serializable {
public static final int HTML = 1;
public static final int MARKDOWN = 2;
public static final int PLAIN_TEXT = 3;
public static final Description emptyDescription = new Description("", PLAIN_TEXT);
private String content;
private int type;
public Description(String content, int type) {
this.type = type;
if (content == null) {
this.content = "";
} else {
this.content = content;
}
}
public String getContent() {
return content;
}
public int getType() {
return type;
}
}

102
extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java
View File

@ -1,66 +1,64 @@
package org.schabi.newpipe.extractor.stream;
import javax.annotation.Nullable;
import java.util.Collection;
import java.util.List;
public final class Frameset {
private List<String> urls;
private int frameWidth;
private int frameHeight;
private int totalCount;
private int framesPerPageX;
private int framesPerPageY;
private List<String> urls;
private int frameWidth;
private int frameHeight;
private int totalCount;
private int framesPerPageX;
private int framesPerPageY;
public Frameset(List<String> urls, int frameWidth, int frameHeight, int totalCount, int framesPerPageX, int framesPerPageY) {
this.urls = urls;
this.totalCount = totalCount;
this.frameWidth = frameWidth;
this.frameHeight = frameHeight;
this.framesPerPageX = framesPerPageX;
this.framesPerPageY = framesPerPageY;
}
public Frameset(List<String> urls, int frameWidth, int frameHeight, int totalCount, int framesPerPageX, int framesPerPageY) {
this.urls = urls;
this.totalCount = totalCount;
this.frameWidth = frameWidth;
this.frameHeight = frameHeight;
this.framesPerPageX = framesPerPageX;
this.framesPerPageY = framesPerPageY;
}
/**
* @return list of urls to images with frames
*/
public List<String> getUrls() {
return urls;
}
/**
* @return list of urls to images with frames
*/
public List<String> getUrls() {
return urls;
}
/**
* @return total count of frames
*/
public int getTotalCount() {
return totalCount;
}
/**
* @return total count of frames
*/
public int getTotalCount() {
return totalCount;
}
/**
* @return maximum frames count by x
*/
public int getFramesPerPageX() {
return framesPerPageX;
}
/**
* @return maximum frames count by x
*/
public int getFramesPerPageX() {
return framesPerPageX;
}
/**
* @return maximum frames count by y
*/
public int getFramesPerPageY() {
return framesPerPageY;
}
/**
* @return maximum frames count by y
*/
public int getFramesPerPageY() {
return framesPerPageY;
}
/**
* @return width of a one frame, in pixels
*/
public int getFrameWidth() {
return frameWidth;
}
/**
* @return width of a one frame, in pixels
*/
public int getFrameWidth() {
return frameWidth;
}
/**
* @return height of a one frame, in pixels
*/
public int getFrameHeight() {
return frameHeight;
}
/**
* @return height of a one frame, in pixels
*/
public int getFrameHeight() {
return frameHeight;
}
}

14
extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java
View File

@ -1,10 +1,10 @@
package org.schabi.newpipe.extractor.stream;
import org.schabi.newpipe.extractor.MediaFormat;
import java.io.Serializable;
import java.util.List;
import org.schabi.newpipe.extractor.MediaFormat;
/**
* Creates a stream object from url, format and optional torrent url
*/
@ -22,19 +22,19 @@ public abstract class Stream implements Serializable {
/**
* Instantiates a new stream object.
*
* @param url the url
* @param url the url
* @param format the format
*/
public Stream(String url, MediaFormat format) {
this(url, null, format);
}
/**
* Instantiates a new stream object.
*
* @param url the url
* @param url the url
* @param torrentUrl the url to torrent file, example https://webtorrent.io/torrents/big-buck-bunny.torrent
* @param format the format
* @param format the format
*/
public Stream(String url, String torrentUrl, MediaFormat format) {
this.url = url;
@ -76,7 +76,7 @@ public abstract class Stream implements Serializable {
public String getUrl() {
return url;
}
/**
* Gets the torrent url.
*

131
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java
View File

@ -34,6 +34,7 @@ import javax.annotation.Nullable;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
/**
* Scrapes information from a video/audio streaming service (eg, YouTube).
@ -75,6 +76,7 @@ public abstract class StreamExtractor extends Extractor {
/**
* This will return the url to the thumbnail of the stream. Try to return the medium resolution here.
*
* @return The url of the thumbnail.
* @throws ParsingException
*/
@ -82,15 +84,17 @@ public abstract class StreamExtractor extends Extractor {
public abstract String getThumbnailUrl() throws ParsingException;
/**
* This is the stream description. On YouTube this is the video description. You can return simple HTML here.
* @return The description of the stream/video.
* This is the stream description.
*
* @return The description of the stream/video or Description.emptyDescription if the description is empty.
* @throws ParsingException
*/
@Nonnull
public abstract String getDescription() throws ParsingException;
public abstract Description getDescription() throws ParsingException;
/**
* Get the age limit.
*
* @return The age which limits the content or {@value NO_AGE_LIMIT} if there is no limit
* @throws ParsingException if an error occurs while parsing
*/
@ -98,6 +102,7 @@ public abstract class StreamExtractor extends Extractor {
/**
* This should return the length of a video in seconds.
*
* @return The length of the stream in seconds.
* @throws ParsingException
*/
@ -107,6 +112,7 @@ public abstract class StreamExtractor extends Extractor {
* If the url you are currently handling contains a time stamp/seek, you can return the
* position it represents here.
* If the url has no time stamp simply return zero.
*
* @return the timestamp in seconds
* @throws ParsingException
*/
@ -115,22 +121,25 @@ public abstract class StreamExtractor extends Extractor {
/**
* The count of how many people have watched the video/listened to the audio stream.
* If the current stream has no view count or its not available simply return -1
*
* @return amount of views.
* @throws ParsingException
*/
public abstract long getViewCount() throws ParsingException;
/**
* The Amount of likes a video/audio stream got.
* The amount of likes a video/audio stream got.
* If the current stream has no likes or its not available simply return -1
*
* @return the amount of likes the stream got
* @throws ParsingException
*/
public abstract long getLikeCount() throws ParsingException;
/**
* The Amount of dislikes a video/audio stream got.
* The amount of dislikes a video/audio stream got.
* If the current stream has no dislikes or its not available simply return -1
*
* @return the amount of likes the stream got
* @throws ParsingException
*/
@ -142,6 +151,7 @@ public abstract class StreamExtractor extends Extractor {
* <a href="https://teamnewpipe.github.io/documentation/03_Implement_a_service/#channel">ChannelExtractor</a>,
* so be sure to implement that one before you return a value here, otherwise NewPipe will crash if one selects
* this url.
*
* @return the url to the page of the creator/uploader of the stream or an empty String
* @throws ParsingException
*/
@ -151,6 +161,7 @@ public abstract class StreamExtractor extends Extractor {
/**
* The name of the creator/uploader of the stream.
* If the name is not available you can simply return an empty string.
*
* @return the name of the creator/uploader of the stream or an empty String
* @throws ParsingException
*/
@ -160,6 +171,7 @@ public abstract class StreamExtractor extends Extractor {
/**
* The url to the image file/profile picture/avatar of the creator/uploader of the stream.
* If the url is not available you can return an empty String.
*
* @return The url of the image file of the uploader or an empty String
* @throws ParsingException
*/
@ -169,20 +181,24 @@ public abstract class StreamExtractor extends Extractor {
/**
* Get the dash mpd url. If you don't know what a dash MPD is you can read about it
* <a href="https://www.brendanlong.com/the-structure-of-an-mpeg-dash-mpd.html">here</a>.
*
* @return the url as a string or an empty string
* @throws ParsingException if an error occurs while reading
*/
@Nonnull public abstract String getDashMpdUrl() throws ParsingException;
@Nonnull
public abstract String getDashMpdUrl() throws ParsingException;
/**
* I am not sure if this is in use, and how this is used. However the frontend is missing support
* for HLS streams. Prove me if I am wrong. Please open an
* <a href="https://github.com/teamnewpipe/newpipe/issues">issue</a>,
* or fix this description if you know whats up with this.
*
* @return The Url to the hls stream.
* @throws ParsingException
*/
@Nonnull public abstract String getHlsUrl() throws ParsingException;
@Nonnull
public abstract String getHlsUrl() throws ParsingException;
/**
* This should return a list of available
@ -190,6 +206,7 @@ public abstract class StreamExtractor extends Extractor {
* You can also return null or an empty list, however be aware that if you don't return anything
* in getVideoStreams(), getVideoOnlyStreams() and getDashMpdUrl() either the Collector will handle this as
* a failed extraction procedure.
*
* @return a list of audio only streams in the format of AudioStream
* @throws IOException
* @throws ExtractionException
@ -203,6 +220,7 @@ public abstract class StreamExtractor extends Extractor {
* You can also return null or an empty list, however be aware that if you don't return anything
* in getAudioStreams(), getVideoOnlyStreams() and getDashMpdUrl() either the Collector will handle this as
* a failed extraction procedure.
*
* @return a list of combined video and streams in the format of AudioStream
* @throws IOException
* @throws ExtractionException
@ -216,6 +234,7 @@ public abstract class StreamExtractor extends Extractor {
* You can also return null or an empty list, however be aware that if you don't return anything
* in getAudioStreams(), getVideoStreams() and getDashMpdUrl() either the Collector will handle this as
* a failed extraction procedure.
*
* @return a list of video and streams in the format of AudioStream
* @throws IOException
* @throws ExtractionException
@ -226,6 +245,7 @@ public abstract class StreamExtractor extends Extractor {
* This will return a list of available
* <a href="https://teamnewpipe.github.io/NewPipeExtractor/javadoc/org/schabi/newpipe/extractor/stream/Subtitles.html">Subtitles</a>s.
* If no subtitles are available an empty list can returned.
*
* @return a list of available subtitles or an empty list
* @throws IOException
* @throws ExtractionException
@ -238,6 +258,7 @@ public abstract class StreamExtractor extends Extractor {
* <a href="https://teamnewpipe.github.io/NewPipeExtractor/javadoc/org/schabi/newpipe/extractor/stream/Subtitles.html">Subtitles</a>s.
* given by a specific type.
* If no subtitles in that specific format are available an empty list can returned.
*
* @param format the media format by which the subtitles should be filtered
* @return a list of available subtitles or an empty list
* @throws IOException
@ -248,15 +269,17 @@ public abstract class StreamExtractor extends Extractor {
/**
* Get the <a href="https://teamnewpipe.github.io/NewPipeExtractor/javadoc/">StreamType</a>.
*
* @return the type of the stream
* @throws ParsingException
*/
public abstract StreamType getStreamType() throws ParsingException;
/**
* should return the url of the next stream. NewPipe will automatically play
* Should return the url of the next stream. NewPipe will automatically play
* the next stream if the user wants that.
* If the next stream is is not available simply return null
*
* @return the InfoItem of the next stream
* @throws IOException
* @throws ExtractionException
@ -268,7 +291,8 @@ public abstract class StreamExtractor extends Extractor {
* streams. If you don't like suggested streams you should implement them anyway since they can
* be disabled by the user later in the frontend.
* This list MUST NOT contain the next available video as this should be return through getNextStream()
* If is is not available simply return null
* If it is not available simply return null
*
* @return a list of InfoItems showing the related videos/streams
* @throws IOException
* @throws ExtractionException
@ -277,6 +301,7 @@ public abstract class StreamExtractor extends Extractor {
/**
* Should return a list of Frameset object that contains preview of stream frames
*
* @return list of preview frames or empty list if frames preview is not supported or not found for specified stream
* @throws IOException
* @throws ExtractionException
@ -299,9 +324,10 @@ public abstract class StreamExtractor extends Extractor {
/**
* Override this function if the format of time stamp in the url is not the same format as that form youtube.
* Honestly I don't even know the time stamp fromat of youtube.
* Honestly I don't even know the time stamp format of YouTube.
*
* @param regexPattern
* @return the sime stamp/seek for the video in seconds
* @return the time stamp/seek for the video in seconds
* @throws ParsingException
*/
protected long getTimestampSeconds(String regexPattern) throws ParsingException {
@ -309,10 +335,10 @@ public abstract class StreamExtractor extends Extractor {
try {
timeStamp = Parser.matchGroup1(regexPattern, getOriginalUrl());
} catch (Parser.RegexException e) {
// catch this instantly since an url does not necessarily have to have a time stamp
// catch this instantly since a url does not necessarily have a timestamp
// -2 because well the testing system will then know its the regex that failed :/
// not good i know
// -2 because the testing system will consequently know that the regex failed
// not good, I know
return -2;
}
@ -349,4 +375,81 @@ public abstract class StreamExtractor extends Extractor {
return 0;
}
}
/**
* The host of the stream (Eg. peertube.cpy.re).
* If the host is not available, or if the service doesn't use
* a federated system, but a centralised system,
* you can simply return an empty string.
*
* @return the host of the stream or an empty String.
* @throws ParsingException
*/
@Nonnull
public abstract String getHost() throws ParsingException;
/**
* The privacy of the stream (Eg. Public, Private, Unlisted…).
* If the privacy is not available you can simply return an empty string.
*
* @return the privacy of the stream or an empty String.
* @throws ParsingException
*/
@Nonnull
public abstract String getPrivacy() throws ParsingException;
/**
* The name of the category of the stream.
* If the category is not available you can simply return an empty string.
*
* @return the category of the stream or an empty String.
* @throws ParsingException
*/
@Nonnull
public abstract String getCategory() throws ParsingException;
/**
* The name of the licence of the stream.
* If the licence is not available you can simply return an empty string.
*
* @return the licence of the stream or an empty String.
* @throws ParsingException
*/
@Nonnull
public abstract String getLicence() throws ParsingException;
/**
* The locale language of the stream.
* If the language is not available you can simply return null.
* If the language is provided by a language code, you can return
* new Locale(language_code);
*
* @return the locale language of the stream or null.
* @throws ParsingException
*/
@Nullable
public abstract Locale getLanguageInfo() throws ParsingException;
/**
* The list of tags of the stream.
* If the tag list is not available you can simply return an empty list.
*
* @return the list of tags of the stream or an empty list.
* @throws ParsingException
*/
@Nonnull
public abstract List<String> getTags() throws ParsingException;
/**
* The support information of the stream.
* see: https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37
* (support button).
* If the support information are not available,
* you can simply return an empty String.
*
* @return the support information of the stream or an empty String.
* @throws ParsingException
*/
@Nonnull
public abstract String getSupportInfo() throws ParsingException;
}

107
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java
View File

@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
/*
* Created by Christian Schabesberger on 26.08.15.
@ -270,6 +271,43 @@ public class StreamInfo extends Info {
streamInfo.addError(e);
}
//additional info
try {
streamInfo.setHost(extractor.getHost());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setPrivacy(extractor.getPrivacy());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setCategory(extractor.getCategory());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setLicence(extractor.getLicence());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setLanguageInfo(extractor.getLanguageInfo());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setTags(extractor.getTags());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setSupportInfo(extractor.getSupportInfo());
} catch (Exception e) {
streamInfo.addError(e);
}
streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor));
return streamInfo;
@ -281,7 +319,7 @@ public class StreamInfo extends Info {
private DateWrapper uploadDate;
private long duration = -1;
private int ageLimit = -1;
private String description;
private Description description;
private long viewCount = -1;
private long likeCount = -1;
@ -308,6 +346,14 @@ public class StreamInfo extends Info {
private long startPosition = 0;
private List<SubtitlesStream> subtitles = new ArrayList<>();
private String host = "";
private String privacy = "";
private String category = "";
private String licence = "";
private String support = "";
private Locale language = null;
private List<String> tags = new ArrayList<>();
/**
* Get the stream type
*
@ -371,11 +417,11 @@ public class StreamInfo extends Info {
this.ageLimit = ageLimit;
}
public String getDescription() {
public Description getDescription() {
return description;
}
public void setDescription(String description) {
public void setDescription(Description description) {
this.description = description;
}
@ -533,4 +579,59 @@ public class StreamInfo extends Info {
this.subtitles = subtitles;
}
public String getHost() {
return this.host;
}
public void setHost(String str) {
this.host = str;
}
public String getPrivacy() {
return this.privacy;
}
public void setPrivacy(String str) {
this.privacy = str;
}
public String getCategory() {
return this.category;
}
public void setCategory(String cat) {
this.category = cat;
}
public String getLicence() {
return this.licence;
}
public void setLicence(String str) {
this.licence = str;
}
public Locale getLanguageInfo() {
return this.language;
}
public void setLanguageInfo(Locale lang) {
this.language = lang;
}
public List<String> getTags() {
return this.tags;
}
public void setTags(List<String> tags) {
this.tags = tags;
}
public void setSupportInfo(String support) {
this.support = support;
}
public String getSupportInfo() {
return this.support;
}
}

11
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java
View File

@ -31,6 +31,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor {
/**
* Get the stream type
*
* @return the stream type
* @throws ParsingException thrown if there is an error in the extraction
*/
@ -38,6 +39,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor {
/**
* Check if the stream is an ad.
*
* @return {@code true} if the stream is an ad.
* @throws ParsingException thrown if there is an error in the extraction
*/
@ -45,6 +47,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor {
/**
* Get the stream duration in seconds
*
* @return the stream duration in seconds
* @throws ParsingException thrown if there is an error in the extraction
*/
@ -52,6 +55,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor {
/**
* Parses the number of views
*
* @return the number of views or -1 for live streams
* @throws ParsingException thrown if there is an error in the extraction
*/
@ -59,6 +63,7 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor {
/**
* Get the uploader name
*
* @return the uploader name
* @throws ParsingException if parsing fails
*/
@ -80,9 +85,9 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor {
/**
* Extracts the upload date and time of this item and parses it.
* <p>
* If the service doesn't provide an exact time, an approximation can be returned.
* <br>
* If the service doesn't provide any date at all, then {@code null} should be returned.
* If the service doesn't provide an exact time, an approximation can be returned.
* <br>
* If the service doesn't provide any date at all, then {@code null} should be returned.
* </p>
*
* @return The date and time (can be approximated) this item was uploaded or {@code null}.

4
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java
View File

@ -101,8 +101,8 @@ public class StreamInfoItemsCollector extends InfoItemsCollector<StreamInfoItem,
public List<StreamInfoItem> getStreamInfoItemList() {
List<StreamInfoItem> siiList = new Vector<>();
for(InfoItem ii : super.getItems()) {
if(ii instanceof StreamInfoItem) {
for (InfoItem ii : super.getItems()) {
if (ii instanceof StreamInfoItem) {
siiList.add((StreamInfoItem) ii);
}
}

4
extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java
View File

@ -52,7 +52,7 @@ public class SubtitlesStream extends Stream implements Serializable {
@Override
public boolean equalStats(Stream cmp) {
return super.equalStats(cmp)&&
return super.equalStats(cmp) &&
cmp instanceof SubtitlesStream &&
code.equals(((SubtitlesStream) cmp).code) &&
autoGenerated == ((SubtitlesStream) cmp).autoGenerated;
@ -67,7 +67,7 @@ public class SubtitlesStream extends Stream implements Serializable {
}
public Locale getLocale() {
return locale;
return locale;
}
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java
View File

@ -36,7 +36,7 @@ public class VideoStream extends Stream {
this.resolution = resolution;
this.isVideoOnly = isVideoOnly;
}
public VideoStream(String url, String torrentUrl, MediaFormat format, String resolution) {
this(url, torrentUrl, format, resolution, false);
}
@ -56,6 +56,7 @@ public class VideoStream extends Stream {
/**
* Get the video resolution
*
* @return the video resolution
*/
public String getResolution() {
@ -64,8 +65,9 @@ public class VideoStream extends Stream {
/**
* Check if the video is video only.
*
* <p>
* Video only streams have no audio
*
* @return {@code true} if this stream is vid
*/
public boolean isVideoOnly() {

12
extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java
View File

@ -1,8 +1,8 @@
package org.schabi.newpipe.extractor.utils;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
@ -109,11 +109,11 @@ public class DashMpdParser {
* <p>
* It has video, video only and audio streams and will only add to the list if it don't
* find a similar stream in the respective lists (calling {@link Stream#equalStats}).
*
* <p>
* Info about dash MPD can be found here
* @see <a href="https://www.brendanlong.com/the-structure-of-an-mpeg-dash-mpd.html">www.brendanlog.com</a>
*
* @param streamInfo where the parsed streams will be added
* @see <a href="https://www.brendanlong.com/the-structure-of-an-mpeg-dash-mpd.html">www.brendanlog.com</a>
*/
public static ParserResult getStreams(final StreamInfo streamInfo)
throws DashMpdParsingException, ReCaptchaException {
@ -160,7 +160,7 @@ public class DashMpdParser {
final MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType);
if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) {
if(segmentationList == null) {
if (segmentationList == null) {
final AudioStream audioStream = new AudioStream(url, mediaFormat, itag.avgBitrate);
if (!Stream.containSimilarStream(audioStream, streamInfo.getAudioStreams())) {
audioStreams.add(audioStream);
@ -172,7 +172,7 @@ public class DashMpdParser {
} else {
boolean isVideoOnly = itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY);
if(segmentationList == null) {
if (segmentationList == null) {
final VideoStream videoStream = new VideoStream(url,
mediaFormat,
itag.resolutionString,
@ -191,7 +191,7 @@ public class DashMpdParser {
itag.resolutionString,
isVideoOnly);
if(isVideoOnly) {
if (isVideoOnly) {
segmentedVideoOnlyStreams.add(videoStream);
} else {
segmentedVideoStreams.add(videoStream);

5
extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java
View File

@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.comments.CommentsInfo;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfo;
@ -31,7 +30,7 @@ public class ExtractorHelper {
public static List<InfoItem> getRelatedVideosOrLogError(StreamInfo info, StreamExtractor extractor) {
try {
InfoItemsCollector<? extends InfoItem, ?> collector = extractor.getRelatedStreams();
if(collector == null) return Collections.emptyList();
if (collector == null) return Collections.emptyList();
info.addAllErrors(collector.getErrors());
//noinspection unchecked
@ -41,5 +40,5 @@ public class ExtractorHelper {
return Collections.emptyList();
}
}
}

66
extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java
View File

@ -1,76 +1,84 @@
package org.schabi.newpipe.extractor.utils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
public class JsonUtils {
private JsonUtils() {
}
@Nonnull
public static Object getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
public static Object getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
List<String> keys = Arrays.asList(path.split("\\."));
object = getObject(object, keys.subList(0, keys.size() - 1));
if (null == object) throw new ParsingException("Unable to get " + path);
Object result = object.get(keys.get(keys.size() - 1));
if(null == result) throw new ParsingException("Unable to get " + path);
if (null == result) throw new ParsingException("Unable to get " + path);
return result;
}
@Nonnull
public static String getString(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
public static String getString(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
Object value = getValue(object, path);
if(value instanceof String) {
if (value instanceof String) {
return (String) value;
}else {
} else {
throw new ParsingException("Unable to get " + path);
}
}
@Nonnull
public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
public static Boolean getBoolean(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
Object value = getValue(object, path);
if(value instanceof Number) {
if (value instanceof Boolean) {
return (Boolean) value;
} else {
throw new ParsingException("Unable to get " + path);
}
}
@Nonnull
public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
Object value = getValue(object, path);
if (value instanceof Number) {
return (Number) value;
}else {
} else {
throw new ParsingException("Unable to get " + path);
}
}
@Nonnull
public static JsonObject getObject(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
public static JsonObject getObject(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
Object value = getValue(object, path);
if(value instanceof JsonObject) {
if (value instanceof JsonObject) {
return (JsonObject) value;
}else {
} else {
throw new ParsingException("Unable to get " + path);
}
}
@Nonnull
public static JsonArray getArray(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
public static JsonArray getArray(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
Object value = getValue(object, path);
if(value instanceof JsonArray) {
if (value instanceof JsonArray) {
return (JsonArray) value;
}else {
} else {
throw new ParsingException("Unable to get " + path);
}
}
@Nonnull
public static List<Object> getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException {
List<Object> result = new ArrayList<>();
for (int i = 0; i < array.size(); i++) {
JsonObject obj = array.getObject(i);

16
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java
View File

@ -1,5 +1,10 @@
package org.schabi.newpipe.extractor.utils;
import org.nibor.autolink.LinkExtractor;
import org.nibor.autolink.LinkSpan;
import org.nibor.autolink.LinkType;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.util.ArrayList;
@ -9,11 +14,6 @@ import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.nibor.autolink.LinkExtractor;
import org.nibor.autolink.LinkSpan;
import org.nibor.autolink.LinkType;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
/*
* Created by Christian Schabesberger on 02.02.16.
*
@ -51,7 +51,7 @@ public class Parser {
public static String matchGroup1(String pattern, String input) throws RegexException {
return matchGroup(pattern, input, 1);
}
public static String matchGroup1(Pattern pattern, String input) throws RegexException {
return matchGroup(pattern, input, 1);
}
@ -60,7 +60,7 @@ public class Parser {
Pattern pat = Pattern.compile(pattern);
return matchGroup(pat, input, group);
}
public static String matchGroup(Pattern pat, String input, int group) throws RegexException {
Matcher mat = pat.matcher(input);
boolean foundMatch = mat.find();
@ -102,7 +102,7 @@ public class Parser {
.linkTypes(EnumSet.of(LinkType.URL, LinkType.WWW))
.build();
Iterable<LinkSpan> linkss = linkExtractor.extractLinks(txt);
for(LinkSpan ls : linkss) {
for (LinkSpan ls : linkss) {
links.add(txt.substring(ls.getBeginIndex(), ls.getEndIndex()));
}

12
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java
View File

@ -1,13 +1,13 @@
package org.schabi.newpipe.extractor.utils;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.List;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
public class Utils {
private Utils() {
@ -35,6 +35,7 @@ public class Utils {
* <li>1.23K -&gt; 1230</li>
* <li>1.23M -&gt; 1230000</li>
* </ul>
*
* @param numberWord string to be converted to a long
* @return a long
* @throws NumberFormatException
@ -165,17 +166,17 @@ public class Utils {
return setsNoPort || usesDefaultPort;
}
public static String removeUTF8BOM(String s) {
if (s.startsWith("\uFEFF")) {
s = s.substring(1);
}
if (s.endsWith("\uFEFF")) {
s = s.substring(0, s.length()-1);
s = s.substring(0, s.length() - 1);
}
return s;
}
public static String getBaseUrl(String url) throws ParsingException {
URL uri;
try {
@ -185,5 +186,4 @@ public class Utils {
}
return uri.getProtocol() + "://" + uri.getAuthority();
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java
View File

@ -45,6 +45,6 @@ public class MediaCCCConferenceExtractorTest {
@Test
public void testGetInitalPage() throws Exception {
assertEquals(97,extractor.getInitialPage().getItems().size());
assertEquals(97, extractor.getInitialPage().getItems().size());
}
}

6
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java
View File

@ -24,7 +24,7 @@ public class MediaCCCConferenceListExtractorTest {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getKioskList().getDefaultKioskExtractor();
extractor = MediaCCC.getKioskList().getDefaultKioskExtractor();
extractor.fetchPage();
}
@ -49,8 +49,8 @@ public class MediaCCCConferenceListExtractorTest {
}
private boolean contains(List<InfoItem> itemList, String name) {
for(InfoItem item : itemList) {
if(item.getName().equals(name))
for (InfoItem item : itemList) {
if (item.getName().equals(name))
return true;
}
return false;

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java
View File

@ -22,7 +22,7 @@ public class MediaCCCOggTest {
public static void setUpClass() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/1317");
extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/1317");
extractor.fetchPage();
}
@ -33,7 +33,7 @@ public class MediaCCCOggTest {
@Test
public void getAudioStreamsContainOgg() throws Exception {
for(AudioStream stream : extractor.getAudioStreams()) {
for (AudioStream stream : extractor.getAudioStreams()) {
assertEquals("OGG", stream.getFormat().toString());
}
}

10
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java
View File

@ -28,7 +28,7 @@ public class MediaCCCSearchExtractorAllTest {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory()
extractor = MediaCCC.getSearchExtractor(new MediaCCCSearchQueryHandlerFactory()
.fromQuery("c3", Arrays.asList(new String[0]), ""));
extractor.fetchPage();
itemsPage = extractor.getInitialPage();
@ -37,8 +37,8 @@ public class MediaCCCSearchExtractorAllTest {
@Test
public void testIfChannelInfoItemsAvailable() {
boolean isAvialable = false;
for(InfoItem item : itemsPage.getItems()) {
if(item instanceof ChannelInfoItem) {
for (InfoItem item : itemsPage.getItems()) {
if (item instanceof ChannelInfoItem) {
isAvialable = true;
}
}
@ -48,8 +48,8 @@ public class MediaCCCSearchExtractorAllTest {
@Test
public void testIfStreamInfoitemsAvailable() {
boolean isAvialable = false;
for(InfoItem item : itemsPage.getItems()) {
if(item instanceof StreamInfoItem) {
for (InfoItem item : itemsPage.getItems()) {
if (item instanceof StreamInfoItem) {
isAvialable = true;
}
}

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java
View File

@ -27,7 +27,7 @@ public class MediaCCCSearchExtractorConferencesTest {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory()
extractor = MediaCCC.getSearchExtractor(new MediaCCCSearchQueryHandlerFactory()
.fromQuery("c3", Arrays.asList(new String[]{"conferences"}), ""));
extractor.fetchPage();
itemsPage = extractor.getInitialPage();
@ -35,7 +35,7 @@ public class MediaCCCSearchExtractorConferencesTest {
@Test
public void testReturnTypeChannel() {
for(InfoItem item : itemsPage.getItems()) {
for (InfoItem item : itemsPage.getItems()) {
assertTrue("Item is not of type channel", item instanceof ChannelInfoItem);
}
}

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java
View File

@ -28,7 +28,7 @@ public class MediaCCCSearchExtractorEventsTest {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory()
extractor = MediaCCC.getSearchExtractor(new MediaCCCSearchQueryHandlerFactory()
.fromQuery("linux", Arrays.asList(new String[]{"events"}), ""));
extractor.fetchPage();
itemsPage = extractor.getInitialPage();
@ -65,7 +65,7 @@ public class MediaCCCSearchExtractorEventsTest {
@Test
public void testReturnTypeStream() throws Exception {
for(InfoItem item : itemsPage.getItems()) {
for (InfoItem item : itemsPage.getItems()) {
assertTrue("Item is not of type StreamInfoItem", item instanceof StreamInfoItem);
}
}

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java
View File

@ -28,7 +28,7 @@ public class MediaCCCStreamExtractorTest implements BaseExtractorTest {
public static void setUpClass() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/8afc16c2-d76a-53f6-85e4-90494665835d");
extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/8afc16c2-d76a-53f6-85e4-90494665835d");
extractor.fetchPage();
}
@ -89,7 +89,7 @@ public class MediaCCCStreamExtractorTest implements BaseExtractorTest {
@Test
public void testGetTextualUploadDate() throws ParsingException {
Assert.assertEquals("2018-05-11", extractor.getTextualUploadDate());
Assert.assertEquals("2018-05-11T02:00:00.000+02:00", extractor.getTextualUploadDate());
}
@Test

18
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java
View File

@ -1,15 +1,5 @@
package org.schabi.newpipe.extractor.services.peertube;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
@ -20,6 +10,12 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
/**
* Test for {@link PeertubeChannelExtractor}
*/
@ -204,7 +200,7 @@ public class PeertubeChannelExtractorTest {
@Test
public void testSubscriberCount() throws ParsingException {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 2);
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 1);
}
}
}

6
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java
View File

@ -1,8 +1,5 @@
package org.schabi.newpipe.extractor.services.peertube;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
@ -10,6 +7,9 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/**
* Test for {@link PeertubeChannelLinkHandlerFactory}
*/

26
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java
View File

@ -1,12 +1,5 @@
package org.schabi.newpipe.extractor.services.peertube;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import java.io.IOException;
import java.util.List;
import org.jsoup.helper.StringUtil;
import org.junit.BeforeClass;
import org.junit.Test;
@ -18,6 +11,13 @@ import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor;
import java.io.IOException;
import java.util.List;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
public class PeertubeCommentsExtractorTest {
private static PeertubeCommentsExtractor extractor;
@ -26,7 +26,7 @@ public class PeertubeCommentsExtractorTest {
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (PeertubeCommentsExtractor) PeerTube
.getCommentsExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
.getCommentsExtractor("https://framatube.org/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
}
@Test
@ -46,7 +46,7 @@ public class PeertubeCommentsExtractorTest {
@Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
boolean result = false;
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://peertube.mastodon.host/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
assertTrue("Comments".equals(commentsInfo.getName()));
result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");
@ -59,11 +59,11 @@ public class PeertubeCommentsExtractorTest {
assertTrue(result);
}
@Test
public void testGetCommentsAllData() throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
for(CommentsInfoItem c: comments.getItems()) {
for (CommentsInfoItem c : comments.getItems()) {
assertFalse(StringUtil.isBlank(c.getAuthorEndpoint()));
assertFalse(StringUtil.isBlank(c.getAuthorName()));
assertFalse(StringUtil.isBlank(c.getAuthorThumbnail()));
@ -82,8 +82,8 @@ public class PeertubeCommentsExtractorTest {
}
private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
for(CommentsInfoItem c: comments) {
if(c.getCommentText().contains(comment)) {
for (CommentsInfoItem c : comments) {
if (c.getCommentText().contains(comment)) {
return true;
}
}

Some files were not shown because too many files have changed in this diff Show More