lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge 083f63c083 into e5125c15f1

This commit is contained in:
Christian Schabesberger 2018-05-15 20:20:01 +00:00 committed by GitHub
parent e5125c15f1 083f63c083
commit 4ba9e6e5ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
65 changed files with 592 additions and 461 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java
View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor;
import edu.umd.cs.findbugs.annotations.NonNull;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -14,42 +15,28 @@ public abstract class Extractor {
*/
private final StreamingService service;
/**
* Dirty/original url that was passed in the constructor.
* <p>
* What makes a url "dirty" or not is, for example, the additional parameters
* (not important as—in this case—the id):
* <pre>
* https://www.youtube.com/watch?v=a9Zf_258aTI<i>&amp;t=4s</i> → <i><b>&amp;t=4s</b></i>
* </pre>
* But as you can imagine, the time parameter is very important when calling {@link org.schabi.newpipe.extractor.stream.StreamExtractor#getTimeStamp()}.
*/
private final String originalUrl;
private final UrlIdHandler urlIdHandler;
/**
* The cleaned url, result of passing the {@link #originalUrl} to the associated urlIdHandler ({@link #getUrlIdHandler()}).
* <p>
* Is lazily-cleaned by calling {@link #getCleanUrl()}
*/
@Nullable
private String cleanUrl;
private boolean pageFetched = false;
private final Downloader downloader;
public Extractor(final StreamingService service, final String url) {
public Extractor(final StreamingService service, final UrlIdHandler urlIdHandler) {
if(service == null) throw new NullPointerException("service is null");
if(url == null) throw new NullPointerException("url is null");
if(urlIdHandler == null) throw new NullPointerException("UrlIdHandler is null");
this.service = service;
this.originalUrl = url;
this.urlIdHandler = urlIdHandler;
this.downloader = NewPipe.getDownloader();
if(downloader == null) throw new NullPointerException("downloader is null");
}
/**
* @return a {@link UrlIdHandler} of the current extractor type (e.g. a ChannelExtractor should return a channel url handler).
* @return The {@link UrlIdHandler} of the current extractor object (e.g. a ChannelExtractor should return a channel url handler).
*/
@Nonnull
protected abstract UrlIdHandler getUrlIdHandler() throws ParsingException;
protected UrlIdHandler getUrlIdHandler() {
return urlIdHandler;
}
/**
* Fetch the current page.
@ -79,7 +66,9 @@ public abstract class Extractor {
public abstract void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException;
@Nonnull
public abstract String getId() throws ParsingException;
public String getId() throws ParsingException {
return urlIdHandler.getId();
}
/**
* Get the name
@ -90,26 +79,13 @@ public abstract class Extractor {
public abstract String getName() throws ParsingException;
@Nonnull
public String getOriginalUrl() {
return originalUrl;
public String getOriginalUrl() throws ParsingException {
return urlIdHandler.getOriginalUrl();
}
/**
* Get a clean url and as a fallback the original url.
* @return the clean url or the original url
*/
@Nonnull
public String getCleanUrl() {
if (cleanUrl != null && !cleanUrl.isEmpty()) return cleanUrl;
try {
cleanUrl = getUrlIdHandler().cleanUrl(originalUrl);
} catch (Exception e) {
cleanUrl = null;
// Fallback to the original url
return originalUrl;
}
return cleanUrl;
public String getUrl() throws ParsingException {
return urlIdHandler.getUrl();
}
@Nonnull

4
extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java
View File

@ -12,8 +12,8 @@ import java.util.List;
*/
public abstract class ListExtractor<R extends InfoItem> extends Extractor {
public ListExtractor(StreamingService service, String url) {
super(service, url);
public ListExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) {
super(service, urlIdHandler);
}
/**

42
extractor/src/main/java/org/schabi/newpipe/extractor/ListUrlIdHandler.java Normal file
View File

@ -0,0 +1,42 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
public abstract class ListUrlIdHandler extends UrlIdHandler {
protected String[] contentFilter;
protected String sortFilter;
public ListUrlIdHandler setQuery(String id, String[] contentFilter, String softFilter) throws ParsingException {
setId(id);
this.contentFilter = contentFilter;
this.sortFilter = softFilter;
return this;
}
public ListUrlIdHandler setUrl(String url) throws ParsingException {
return (ListUrlIdHandler) super.setUrl(url);
}
public ListUrlIdHandler setId(String id) throws ParsingException {
return (ListUrlIdHandler) super.setId(id);
}
/**
* Will returns content filter the corresponding extractor can handle like "channels", "videos", "music", etc.
*
* @return filter that can be applied when building a query for getting a list
*/
public String[] getAvailableContentFilter() {
return new String[0];
}
/**
* Will returns sort filter the corresponding extractor can handle like "A-Z", "oldest first", "size", etc.
*
* @return filter that can be applied when building a query for getting a list
*/
public String[] getAvailableSortFilter() {
return new String[0];
}
}

45
extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java
View File

@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchEngine;
@ -62,22 +63,52 @@ public abstract class StreamingService {
return serviceId + ":" + serviceInfo.getName();
}
////////////////////////////////////////////
// Url Id handler
////////////////////////////////////////////
public abstract UrlIdHandler getStreamUrlIdHandler();
public abstract UrlIdHandler getChannelUrlIdHandler();
public abstract UrlIdHandler getPlaylistUrlIdHandler();
public abstract ListUrlIdHandler getChannelUrlIdHandler();
public abstract ListUrlIdHandler getPlaylistUrlIdHandler();
////////////////////////////////////////////
// Extractor
////////////////////////////////////////////
public abstract SearchEngine getSearchEngine();
public abstract SuggestionExtractor getSuggestionExtractor();
public abstract StreamExtractor getStreamExtractor(String url);
public abstract KioskList getKioskList() throws ExtractionException;
public abstract ChannelExtractor getChannelExtractor(String url);
public abstract PlaylistExtractor getPlaylistExtractor(String url);
public abstract SubscriptionExtractor getSubscriptionExtractor();
public abstract KioskList getKioskList() throws ExtractionException;
public abstract ChannelExtractor getChannelExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException;
public abstract PlaylistExtractor getPlaylistExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException;
public abstract StreamExtractor getStreamExtractor(UrlIdHandler urlIdHandler) throws ExtractionException;
public ChannelExtractor getChannelExtractor(String id, String[] contentFilter, String sortFilter) throws ExtractionException {
return getChannelExtractor(getChannelUrlIdHandler().setQuery(id, contentFilter, sortFilter));
}
public PlaylistExtractor getPlaylistExtractor(String id, String[] contentFilter, String sortFilter) throws ExtractionException {
return getPlaylistExtractor(getPlaylistUrlIdHandler().setQuery(id, contentFilter, sortFilter));
}
public ChannelExtractor getChannelExtractor(String url) throws ExtractionException {
return getChannelExtractor(getChannelUrlIdHandler().setUrl(url));
}
public PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException {
return getPlaylistExtractor(getPlaylistUrlIdHandler().setUrl(url));
}
public StreamExtractor getStreamExtractor(String url) throws ExtractionException {
return getStreamExtractor(getStreamUrlIdHandler().setUrl(url));
}
/**
* figure out where the link is pointing to (a channel, video, playlist, etc.)
*/
public final LinkType getLinkTypeByUrl(String url) {
public final LinkType getLinkTypeByUrl(String url) throws ParsingException {
UrlIdHandler sH = getStreamUrlIdHandler();
UrlIdHandler cH = getChannelUrlIdHandler();
UrlIdHandler pH = getPlaylistUrlIdHandler();

47
extractor/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java
View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
/*
@ -22,16 +23,52 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public interface UrlIdHandler {
public abstract class UrlIdHandler {
String getUrl(String id) throws ParsingException;
String getId(String url) throws ParsingException;
String cleanUrl(String complexUrl) throws ParsingException;
protected String id = "";
protected String originalUrl = "";
public abstract String onGetIdFromUrl(String url) throws ParsingException;
public abstract String getUrl() throws ParsingException;
public abstract boolean onAcceptUrl(final String url) throws ParsingException;
public UrlIdHandler setUrl(String url) throws ParsingException {
if(url == null) throw new IllegalArgumentException("url can not be null");
originalUrl = url;
id = onGetIdFromUrl(url);
return this;
}
public UrlIdHandler setId(String id) throws ParsingException {
if(id == null) throw new IllegalArgumentException("id can not be null");
this.id = id;
if(!acceptUrl(getUrl())) {
throw new ParsingException("Malformed unacceptable url: " + getUrl());
}
return this;
}
public String getId() {
return id;
}
public String getOriginalUrl() throws ParsingException {
return (originalUrl == null || originalUrl.isEmpty())
? getUrl()
: originalUrl;
}
/**
* When a VIEW_ACTION is caught this function will test if the url delivered within the calling
* Intent was meant to be watched with this Service.
* Return false if this service shall not allow to be called through ACTIONs.
*/
boolean acceptUrl(String url);
public boolean acceptUrl(final String url) {
try {
return onAcceptUrl(url);
} catch (Exception e) {
return false;
}
}
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java
View File

@ -1,8 +1,10 @@
package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.ListUrlIdHandler;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
@ -30,8 +32,8 @@ import javax.annotation.Nonnull;
public abstract class ChannelExtractor extends ListExtractor<StreamInfoItem> {
public ChannelExtractor(StreamingService service, String url) {
super(service, url);
public ChannelExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) {
super(service, urlIdHandler);
}
@Nonnull

2
extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java
View File

@ -54,7 +54,7 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
// important data
int serviceId = extractor.getServiceId();
String url = extractor.getCleanUrl();
String url = extractor.getUrl();
String originalUrl = extractor.getOriginalUrl();
String id = extractor.getId();
String name = extractor.getName();

8
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java
View File

@ -21,6 +21,7 @@ package org.schabi.newpipe.extractor.kiosk;
*/
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.ListUrlIdHandler;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -33,10 +34,9 @@ public abstract class KioskExtractor extends ListExtractor<StreamInfoItem> {
private final String id;
public KioskExtractor(StreamingService streamingService,
String url,
String kioskId)
throws ExtractionException {
super(streamingService, url);
ListUrlIdHandler urlIdHandler,
String kioskId) {
super(streamingService, urlIdHandler);
this.id = kioskId;
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java
View File

@ -71,7 +71,7 @@ public class KioskInfo extends ListInfo<StreamInfoItem> {
int serviceId = extractor.getServiceId();
String name = extractor.getName();
String id = extractor.getId();
String url = extractor.getCleanUrl();
String url = extractor.getUrl();
String originalUrl = extractor.getOriginalUrl();
KioskInfo info = new KioskInfo(serviceId, id, url, originalUrl, name);

2
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java
View File

@ -73,7 +73,7 @@ public class KioskList {
throw new ExtractionException("No kiosk found with the type: " + kioskId);
} else {
return ke.extractorFactory.createNewKiosk(NewPipe.getService(service_id),
ke.handler.getUrl(kioskId), kioskId);
ke.handler.setId(kioskId).getUrl(), kioskId);
}
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java
View File

@ -1,8 +1,10 @@
package org.schabi.newpipe.extractor.playlist;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.ListUrlIdHandler;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
@ -10,8 +12,8 @@ import javax.annotation.Nonnull;
public abstract class PlaylistExtractor extends ListExtractor<StreamInfoItem> {
public PlaylistExtractor(StreamingService service, String url) {
super(service, url);
public PlaylistExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) {
super(service, urlIdHandler);
}
@Nonnull

2
extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java
View File

@ -38,7 +38,7 @@ public class PlaylistInfo extends ListInfo<StreamInfoItem> {
public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws IOException, ExtractionException {
int serviceId = extractor.getServiceId();
String url = extractor.getCleanUrl();
String url = extractor.getUrl();
String originalUrl = extractor.getOriginalUrl();
String id = extractor.getId();
String name = extractor.getName();

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java
View File

@ -5,6 +5,7 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.ListUrlIdHandler;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -25,14 +26,14 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
private StreamInfoItemsCollector streamInfoItemsCollector = null;
private String nextPageUrl = null;
public SoundcloudChannelExtractor(StreamingService service, String url) {
super(service, url);
public SoundcloudChannelExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) {
super(service, urlIdHandler);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
userId = getUrlIdHandler().getId(getOriginalUrl());
userId = getUrlIdHandler().getId();
String apiUrl = "https://api-v2.soundcloud.com/users/" + userId +
"?client_id=" + SoundcloudParsingHelper.clientId();
@ -44,12 +45,6 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
}
}
@Nonnull
@Override
public String getCleanUrl() {
return user.isString("permalink_url") ? replaceHttpWithHttps(user.getString("permalink_url")) : getOriginalUrl();
}
@Nonnull
@Override
public String getId() {

24
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java
View File

@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.soundcloud;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.ListUrlIdHandler;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -10,7 +11,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
public class SoundcloudChannelUrlIdHandler extends ListUrlIdHandler {
private static final SoundcloudChannelUrlIdHandler instance = new SoundcloudChannelUrlIdHandler();
private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
"(/((tracks|albums|sets|reposts|followers|following)/?)?)?([#?].*)?$";
@ -19,17 +20,9 @@ public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
return instance;
}
@Override
public String getUrl(String id) throws ParsingException {
try {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/" + id);
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}
@Override
public String getId(String url) throws ParsingException {
public String onGetIdFromUrl(String url) throws ParsingException {
Utils.checkUrl(URL_PATTERN, url);
try {
@ -40,21 +33,16 @@ public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
}
@Override
public String cleanUrl(String complexUrl) throws ParsingException {
Utils.checkUrl(URL_PATTERN, complexUrl);
public String getUrl() throws ParsingException {
try {
Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
.select("meta[property=og:url]").first();
return replaceHttpWithHttps(ogElement.attr("content"));
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/" + id);
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}
@Override
public boolean acceptUrl(String url) {
public boolean onAcceptUrl(final String url) {
return Parser.isMatch(URL_PATTERN, url.toLowerCase());
}
}

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java
View File

@ -1,6 +1,7 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.ListUrlIdHandler;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -14,15 +15,11 @@ import java.util.Arrays;
import java.util.List;
public class SoundcloudChartsExtractor extends KioskExtractor {
private String url;
private StreamInfoItemsCollector collector = null;
private String nextPageUrl = null;
public SoundcloudChartsExtractor(StreamingService service, String url, String kioskId)
throws ExtractionException {
super(service, url, kioskId);
this.url = url;
public SoundcloudChartsExtractor(StreamingService service, ListUrlIdHandler urlIdHandler, String kioskId) {
super(service, urlIdHandler, kioskId);
}
@Override

36
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java
View File

@ -1,13 +1,25 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.schabi.newpipe.extractor.ListUrlIdHandler;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Parser;
public class SoundcloudChartsUrlIdHandler implements UrlIdHandler {
public class SoundcloudChartsUrlIdHandler extends ListUrlIdHandler {
private final String TOP_URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$";
private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$";
public String getUrl(String id) {
@Override
public String onGetIdFromUrl(String url) {
if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) {
return "Top 50";
} else {
return "New & hot";
}
}
public String getUrl() {
if (id.equals("Top 50")) {
return "https://soundcloud.com/charts/top";
} else {
@ -16,25 +28,7 @@ public class SoundcloudChartsUrlIdHandler implements UrlIdHandler {
}
@Override
public String getId(String url) {
if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) {
return "Top 50";
} else {
return "New & hot";
}
}
@Override
public String cleanUrl(String url) {
if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) {
return "https://soundcloud.com/charts/top";
} else {
return "https://soundcloud.com/charts/new";
}
}
@Override
public boolean acceptUrl(String url) {
public boolean onAcceptUrl(final String url) {
return Parser.isMatch(URL_PATTERN, url.toLowerCase());
}
}

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java
View File

@ -4,6 +4,7 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.ListUrlIdHandler;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -24,14 +25,14 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
private StreamInfoItemsCollector streamInfoItemsCollector = null;
private String nextPageUrl = null;
public SoundcloudPlaylistExtractor(StreamingService service, String url) {
super(service, url);
public SoundcloudPlaylistExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) {
super(service, urlIdHandler);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
playlistId = getUrlIdHandler().getId(getOriginalUrl());
playlistId = getUrlIdHandler().getId();
String apiUrl = "https://api.soundcloud.com/playlists/" + playlistId +
"?client_id=" + SoundcloudParsingHelper.clientId() +
"&representation=compact";
@ -44,12 +45,6 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
}
}
@Nonnull
@Override
public String getCleanUrl() {
return playlist.isString("permalink_url") ? replaceHttpWithHttps(playlist.getString("permalink_url")) : getOriginalUrl();
}
@Nonnull
@Override
public String getId() {

49
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java
View File

@ -1,16 +1,11 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.ListUrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
public class SoundcloudPlaylistUrlIdHandler extends ListUrlIdHandler {
private static final SoundcloudPlaylistUrlIdHandler instance = new SoundcloudPlaylistUrlIdHandler();
private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
"/sets/[0-9a-z_-]+/?([#?].*)?$";
@ -20,7 +15,18 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
}
@Override
public String getUrl(String id) throws ParsingException {
public String onGetIdFromUrl(String url) throws ParsingException {
Utils.checkUrl(URL_PATTERN, url);
try {
return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
} catch (Exception e) {
throw new ParsingException("Could not get id of url: " + url + " " + e.getMessage(), e);
}
}
@Override
public String getUrl() throws ParsingException {
try {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/playlists/" + id);
} catch (Exception e) {
@ -29,32 +35,7 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
}
@Override
public String getId(String url) throws ParsingException {
Utils.checkUrl(URL_PATTERN, url);
try {
return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}
@Override
public String cleanUrl(String complexUrl) throws ParsingException {
Utils.checkUrl(URL_PATTERN, complexUrl);
try {
Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
.select("meta[property=og:url]").first();
return replaceHttpWithHttps(ogElement.attr("content"));
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}
@Override
public boolean acceptUrl(String url) {
public boolean onAcceptUrl(final String url) throws ParsingException {
return Parser.isMatch(URL_PATTERN, url.toLowerCase());
}
}

20
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java
View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.schabi.newpipe.extractor.ListUrlIdHandler;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.UrlIdHandler;
@ -32,29 +33,29 @@ public class SoundcloudService extends StreamingService {
}
@Override
public UrlIdHandler getChannelUrlIdHandler() {
public ListUrlIdHandler getChannelUrlIdHandler() {
return SoundcloudChannelUrlIdHandler.getInstance();
}
@Override
public UrlIdHandler getPlaylistUrlIdHandler() {
public ListUrlIdHandler getPlaylistUrlIdHandler() {
return SoundcloudPlaylistUrlIdHandler.getInstance();
}
@Override
public StreamExtractor getStreamExtractor(String url) {
return new SoundcloudStreamExtractor(this, url);
public StreamExtractor getStreamExtractor(UrlIdHandler urlIdHandler) throws ExtractionException {
return new SoundcloudStreamExtractor(this, urlIdHandler);
}
@Override
public ChannelExtractor getChannelExtractor(String url) {
return new SoundcloudChannelExtractor(this, url);
public ChannelExtractor getChannelExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException {
return new SoundcloudChannelExtractor(this, urlIdHandler);
}
@Override
public PlaylistExtractor getPlaylistExtractor(String url) {
return new SoundcloudPlaylistExtractor(this, url);
public PlaylistExtractor getPlaylistExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException {
return new SoundcloudPlaylistExtractor(this, urlIdHandler);
}
@Override
@ -71,8 +72,7 @@ public class SoundcloudService extends StreamingService {
String id)
throws ExtractionException {
return new SoundcloudChartsExtractor(SoundcloudService.this,
url,
id);
new SoundcloudChartsUrlIdHandler().setUrl(url), id);
}
};

15
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java
View File

@ -22,8 +22,8 @@ import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudStreamExtractor extends StreamExtractor {
private JsonObject track;
public SoundcloudStreamExtractor(StreamingService service, String url) {
super(service, url);
public SoundcloudStreamExtractor(StreamingService service, UrlIdHandler urlIdHandler) {
super(service, urlIdHandler);
}
@Override
@ -36,12 +36,6 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
}
}
@Nonnull
@Override
public String getCleanUrl() {
return track.isString("permalink_url") ? replaceHttpWithHttps(track.getString("permalink_url")) : getOriginalUrl();
}
@Nonnull
@Override
public String getId() {
@ -214,6 +208,11 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
return new String[0];
}
@Override
public String[] getAffiliateLinks() {
return new String[0];
}
@Override
public String getErrorMessage() {
return null;

22
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java
View File

@ -10,7 +10,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
public class SoundcloudStreamUrlIdHandler extends UrlIdHandler {
private static final SoundcloudStreamUrlIdHandler instance = new SoundcloudStreamUrlIdHandler();
private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
"/(?!(tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?([#?].*)?$";
@ -23,7 +23,7 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
}
@Override
public String getUrl(String id) throws ParsingException {
public String getUrl() throws ParsingException {
try {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/tracks/" + id);
} catch (Exception e) {
@ -32,7 +32,7 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
}
@Override
public String getId(String url) throws ParsingException {
public String onGetIdFromUrl(String url) throws ParsingException {
Utils.checkUrl(URL_PATTERN, url);
try {
@ -43,21 +43,7 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
}
@Override
public String cleanUrl(String complexUrl) throws ParsingException {
Utils.checkUrl(URL_PATTERN, complexUrl);
try {
Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
.select("meta[property=og:url]").first();
return replaceHttpWithHttps(ogElement.attr("content"));
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}
@Override
public boolean acceptUrl(String url) {
public boolean onAcceptUrl(final String url) throws ParsingException {
return Parser.isMatch(URL_PATTERN, url.toLowerCase());
}
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java
View File

@ -31,7 +31,7 @@ public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor {
String id;
try {
id = service.getChannelUrlIdHandler().getId(getUrlFrom(channelUrl));
id = service.getChannelUrlIdHandler().setUrl(getUrlFrom(channelUrl)).getId();
} catch (ExtractionException e) {
throw new InvalidSourceException(e);
}

3
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java
View File

@ -43,6 +43,9 @@ public class ItagItem {
new ItagItem(139, AUDIO, M4A, 48),
new ItagItem(140, AUDIO, M4A, 128),
new ItagItem(141, AUDIO, M4A, 256),
new ItagItem(249, AUDIO, OPUS, 50),
new ItagItem(250, AUDIO, OPUS, 70),
new ItagItem(160, AUDIO, OPUS, 160),
/// VIDEO ONLY ////////////////////////////////////////////
// ID Type Format Resolution FPS ///

25
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.schabi.newpipe.extractor.ListUrlIdHandler;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.UrlIdHandler;
@ -9,6 +10,11 @@ import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.services.youtube.extractors.*;
import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeChannelUrlIdHandler;
import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubePlaylistUrlIdHandler;
import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeStreamUrlIdHandler;
import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeTrendingUrlIdHandler;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
@ -53,28 +59,28 @@ public class YoutubeService extends StreamingService {
}
@Override
public UrlIdHandler getChannelUrlIdHandler() {
public ListUrlIdHandler getChannelUrlIdHandler() {
return YoutubeChannelUrlIdHandler.getInstance();
}
@Override
public UrlIdHandler getPlaylistUrlIdHandler() {
public ListUrlIdHandler getPlaylistUrlIdHandler() {
return YoutubePlaylistUrlIdHandler.getInstance();
}
@Override
public StreamExtractor getStreamExtractor(String url) {
return new YoutubeStreamExtractor(this, url);
public StreamExtractor getStreamExtractor(UrlIdHandler urlIdHandler) throws ExtractionException {
return new YoutubeStreamExtractor(this, urlIdHandler);
}
@Override
public ChannelExtractor getChannelExtractor(String url) {
return new YoutubeChannelExtractor(this, url);
public ChannelExtractor getChannelExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException {
return new YoutubeChannelExtractor(this, urlIdHandler);
}
@Override
public PlaylistExtractor getPlaylistExtractor(String url) {
return new YoutubePlaylistExtractor(this, url);
public PlaylistExtractor getPlaylistExtractor(ListUrlIdHandler urlIdHandler) throws ExtractionException {
return new YoutubePlaylistExtractor(this, urlIdHandler);
}
@Override
@ -92,7 +98,8 @@ public class YoutubeService extends StreamingService {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String id)
throws ExtractionException {
return new YoutubeTrendingExtractor(YoutubeService.this, url, id);
return new YoutubeTrendingExtractor(YoutubeService.this,
new YoutubeTrendingUrlIdHandler().setUrl(url), id);
}
}, new YoutubeTrendingUrlIdHandler(), "Trending");
list.setDefaultKiosk("Trending");

23
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java → extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -1,17 +1,13 @@
package org.schabi.newpipe.extractor.services.youtube;
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.sun.org.apache.xerces.internal.xs.StringList;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -52,16 +48,15 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
private Document doc;
public YoutubeChannelExtractor(StreamingService service, String url) {
super(service, url);
public YoutubeChannelExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) {
super(service, urlIdHandler);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
String channelUrl = super.getCleanUrl() + CHANNEL_URL_PARAMETERS;
String channelUrl = super.getUrl() + CHANNEL_URL_PARAMETERS;
String pageContent = downloader.download(channelUrl);
doc = Jsoup.parse(pageContent, channelUrl);
}
@Override
@ -71,11 +66,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Nonnull
@Override
public String getCleanUrl() {
public String getUrl() throws ParsingException {
try {
return "https://www.youtube.com/channel/" + getId();
} catch (ParsingException e) {
return super.getCleanUrl();
return super.getUrl();
}
}
@ -196,7 +191,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
for(Element a : linkHolder.select("a")) {
String link = a.attr("abs:href");
if(DonationLinkHelper.getServiceByLink(link) != DonationLinkHelper.DonationService.NO_DONATION) {
if(DonationLinkHelper.getDonatoinServiceByLink(link) != DonationLinkHelper.DonationService.NO_DONATION) {
links.add(link);
}
}
@ -236,7 +231,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
collector.reset();
final String uploaderName = getName();
final String uploaderUrl = getCleanUrl();
final String uploaderUrl = getUrl();
for (final Element li : element.children()) {
if (li.select("div[class=\"feed-item-dismissable\"]").first() != null) {
collector.commit(new YoutubeStreamInfoItemExtractor(li) {

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java → extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java
View File

@ -1,4 +1,4 @@
package org.schabi.newpipe.extractor.services.youtube;
package org.schabi.newpipe.extractor.services.youtube.extractors;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;

30
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java → extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -1,4 +1,4 @@
package org.schabi.newpipe.extractor.services.youtube;
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
@ -6,13 +6,11 @@ import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
@ -26,14 +24,14 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
private Document doc;
public YoutubePlaylistExtractor(StreamingService service, String url) {
super(service, url);
public YoutubePlaylistExtractor(StreamingService service, ListUrlIdHandler urlIdHandler) throws ExtractionException {
super(service, urlIdHandler);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
String pageContent = downloader.download(getCleanUrl());
doc = Jsoup.parse(pageContent, getCleanUrl());
String pageContent = downloader.download(getUrl());
doc = Jsoup.parse(pageContent, getUrl());
}
@Override
@ -41,16 +39,6 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
return getNextPageUrlFrom(doc);
}
@Nonnull
@Override
public String getId() throws ParsingException {
try {
return getUrlIdHandler().getId(getCleanUrl());
} catch (Exception e) {
throw new ParsingException("Could not get playlist id");
}
}
@Nonnull
@Override
public String getName() throws ParsingException {
@ -202,7 +190,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getUrl() throws ParsingException {
try {
return streamUrlIdHandler.getUrl(li.attr("data-video-id"));
return streamUrlIdHandler.setId(li.attr("data-video-id")).getUrl();
} catch (Exception e) {
throw new ParsingException("Could not get web page url for the video", e);
}
@ -267,7 +255,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getThumbnailUrl() throws ParsingException {
try {
return "https://i.ytimg.com/vi/" + streamUrlIdHandler.getId(getUrl()) + "/hqdefault.jpg";
return "https://i.ytimg.com/vi/" + streamUrlIdHandler.setUrl(getUrl()).getId() + "/hqdefault.jpg";
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java → extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java
View File

@ -1,4 +1,4 @@
package org.schabi.newpipe.extractor.services.youtube;
package org.schabi.newpipe.extractor.services.youtube.extractors;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.exceptions.ParsingException;

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java → extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchEngine.java
View File

@ -1,4 +1,4 @@
package org.schabi.newpipe.extractor.services.youtube;
package org.schabi.newpipe.extractor.services.youtube.extractors;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

45
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java → extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -1,4 +1,4 @@
package org.schabi.newpipe.extractor.services.youtube;
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
@ -10,14 +10,12 @@ import org.jsoup.nodes.Element;
import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.Subtitles;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.utils.DonationLinkHelper;
import org.schabi.newpipe.extractor.utils.Parser;
@ -86,24 +84,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private boolean isAgeRestricted;
public YoutubeStreamExtractor(StreamingService service, String url) {
super(service, url);
public YoutubeStreamExtractor(StreamingService service, UrlIdHandler urlIdHandler) throws ExtractionException {
super(service, urlIdHandler);
}
/*//////////////////////////////////////////////////////////////////////////
// Impl
//////////////////////////////////////////////////////////////////////////*/
@Nonnull
@Override
public String getId() throws ParsingException {
try {
return getUrlIdHandler().getId(getCleanUrl());
} catch (Exception e) {
throw new ParsingException("Could not get stream id");
}
}
@Nonnull
@Override
public String getName() throws ParsingException {
@ -534,7 +522,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try {
ArrayList<String> donationLinks = new ArrayList<>();
for (String s : Parser.getLinksFromString(getDescription())) {
if (DonationLinkHelper.getServiceByLink(s) != DonationLinkHelper.DonationService.NO_DONATION) {
if (DonationLinkHelper.getDonatoinServiceByLink(s) != DonationLinkHelper.DonationService.NO_DONATION) {
donationLinks.add(s);
}
}
@ -546,6 +534,23 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
@Override
public String[] getAffiliateLinks() throws ParsingException {
try {
ArrayList<String> donationLinks = new ArrayList<>();
for (String s : Parser.getLinksFromString(getDescription())) {
if (DonationLinkHelper.getAffiliateServiceByLink(s) != DonationLinkHelper.AffiliateService.NO_AFILIATE) {
donationLinks.add(s);
}
}
String[] donlret = new String[donationLinks.size()];
donlret = donationLinks.toArray(donlret);
return donlret;
} catch (Exception e) {
throw new ParsingException("Could not get afiliate links", e);
}
}
/*//////////////////////////////////////////////////////////////////////////
// Fetch page
@ -564,7 +569,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private String pageHtml = null;
private String getPageHtml(Downloader downloader) throws IOException, ExtractionException {
final String verifiedUrl = getCleanUrl() + VERIFIED_URL_PARAMS;
final String verifiedUrl = getUrl() + VERIFIED_URL_PARAMS;
if (pageHtml == null) {
pageHtml = downloader.download(verifiedUrl);
}
@ -574,7 +579,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String pageContent = getPageHtml(downloader);
doc = Jsoup.parse(pageContent, getCleanUrl());
doc = Jsoup.parse(pageContent, getUrl());
final String playerUrl;
// Check if the video is age restricted

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java → extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java
View File

@ -1,7 +1,8 @@
package org.schabi.newpipe.extractor.services.youtube;
package org.schabi.newpipe.extractor.services.youtube.extractors;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Utils;
@ -135,7 +136,11 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
Element meta = item.select("div[class=\"yt-lockup-meta\"]").first();
if (meta == null) return -1;
// This case can happen if google releases a special video
if(meta.select("li").size() < 2) return -1;
input = meta.select("li").get(1).text();
} catch (IndexOutOfBoundsException e) {
throw new ParsingException("Could not parse yt-lockup-meta although available: " + getUrl(), e);
}

3
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractor.java → extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java
View File

@ -1,9 +1,10 @@
package org.schabi.newpipe.extractor.services.youtube;
package org.schabi.newpipe.extractor.services.youtube.extractors;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
import org.schabi.newpipe.extractor.utils.Parser;

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java → extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java
View File

@ -1,4 +1,4 @@
package org.schabi.newpipe.extractor.services.youtube;
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonParser;

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java → extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View File

@ -1,4 +1,4 @@
package org.schabi.newpipe.extractor.services.youtube;
package org.schabi.newpipe.extractor.services.youtube.extractors;
/*
* Created by Christian Schabesberger on 12.08.17.
@ -25,11 +25,13 @@ import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.ListUrlIdHandler;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeTrendingUrlIdHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
@ -40,15 +42,15 @@ public class YoutubeTrendingExtractor extends KioskExtractor {
private Document doc;
public YoutubeTrendingExtractor(StreamingService service, String url, String kioskId)
public YoutubeTrendingExtractor(StreamingService service, ListUrlIdHandler urlIdHandler, String kioskId)
throws ExtractionException {
super(service, url, kioskId);
super(service, urlIdHandler, kioskId);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String contentCountry = getContentCountry();
String url = getCleanUrl();
String url = getUrl();
if(contentCountry != null && !contentCountry.isEmpty()) {
url += "?gl=" + contentCountry;
}

19
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandler.java → extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeChannelUrlIdHandler.java
View File

@ -1,6 +1,6 @@
package org.schabi.newpipe.extractor.services.youtube;
package org.schabi.newpipe.extractor.services.youtube.urlIdHandlers;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.ListUrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Parser;
@ -24,7 +24,7 @@ import org.schabi.newpipe.extractor.utils.Parser;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeChannelUrlIdHandler implements UrlIdHandler {
public class YoutubeChannelUrlIdHandler extends ListUrlIdHandler {
private static final YoutubeChannelUrlIdHandler instance = new YoutubeChannelUrlIdHandler();
private static final String ID_PATTERN = "/(user/[A-Za-z0-9_-]*|channel/[A-Za-z0-9_-]*)";
@ -34,22 +34,17 @@ public class YoutubeChannelUrlIdHandler implements UrlIdHandler {
}
@Override
public String getUrl(String id) {
return "https://www.youtube.com/" + id;
}
@Override
public String getId(String url) throws ParsingException {
public String onGetIdFromUrl(String url) throws ParsingException {
return Parser.matchGroup1(ID_PATTERN, url);
}
@Override
public String cleanUrl(String complexUrl) throws ParsingException {
return getUrl(getId(complexUrl));
public String getUrl() {
return "https://www.youtube.com/" + id;
}
@Override
public boolean acceptUrl(String url) {
public boolean onAcceptUrl(String url) {
return (url.contains("youtube") || url.contains("youtu.be") || url.contains("hooktube.com"))
&& (url.contains("/user/") || url.contains("/channel/"));
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java → extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeParsingHelper.java
View File

@ -1,4 +1,4 @@
package org.schabi.newpipe.extractor.services.youtube;
package org.schabi.newpipe.extractor.services.youtube.urlIdHandlers;
import org.schabi.newpipe.extractor.exceptions.ParsingException;

16
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistUrlIdHandler.java → extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubePlaylistUrlIdHandler.java
View File

@ -1,11 +1,11 @@
package org.schabi.newpipe.extractor.services.youtube;
package org.schabi.newpipe.extractor.services.youtube.urlIdHandlers;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.ListUrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Parser;
public class YoutubePlaylistUrlIdHandler implements UrlIdHandler {
public class YoutubePlaylistUrlIdHandler extends ListUrlIdHandler {
private static final YoutubePlaylistUrlIdHandler instance = new YoutubePlaylistUrlIdHandler();
private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{10,})";
@ -15,12 +15,12 @@ public class YoutubePlaylistUrlIdHandler implements UrlIdHandler {
}
@Override
public String getUrl(String id) {
public String getUrl() {
return "https://www.youtube.com/playlist?list=" + id;
}
@Override
public String getId(String url) throws ParsingException {
public String onGetIdFromUrl(String url) throws ParsingException {
try {
return Parser.matchGroup1("list=" + ID_PATTERN, url);
} catch (final Exception exception) {
@ -28,13 +28,9 @@ public class YoutubePlaylistUrlIdHandler implements UrlIdHandler {
}
}
@Override
public String cleanUrl(String complexUrl) throws ParsingException {
return getUrl(getId(complexUrl));
}
@Override
public boolean acceptUrl(String url) {
public boolean onAcceptUrl(final String url) {
final boolean hasNotEmptyUrl = url != null && !url.isEmpty();
final boolean isYoutubeDomain = hasNotEmptyUrl && (url.contains("youtube") || url.contains("youtu.be"));
return isYoutubeDomain && url.contains("list=");

19
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java → extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeStreamUrlIdHandler.java
View File

@ -1,4 +1,4 @@
package org.schabi.newpipe.extractor.services.youtube;
package org.schabi.newpipe.extractor.services.youtube.urlIdHandlers;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
@ -37,7 +37,7 @@ import java.net.URLDecoder;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeStreamUrlIdHandler implements UrlIdHandler {
public class YoutubeStreamUrlIdHandler extends UrlIdHandler {
private static final YoutubeStreamUrlIdHandler instance = new YoutubeStreamUrlIdHandler();
private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{11})";
@ -50,12 +50,12 @@ public class YoutubeStreamUrlIdHandler implements UrlIdHandler {
}
@Override
public String getUrl(String id) {
public String getUrl() {
return "https://www.youtube.com/watch?v=" + id;
}
@Override
public String getId(String url) throws ParsingException, IllegalArgumentException {
public String onGetIdFromUrl(String url) throws ParsingException, IllegalArgumentException {
if (url.isEmpty()) {
throw new IllegalArgumentException("The url parameter should not be empty");
}
@ -167,19 +167,14 @@ public class YoutubeStreamUrlIdHandler implements UrlIdHandler {
}
@Override
public String cleanUrl(String complexUrl) throws ParsingException {
return getUrl(getId(complexUrl));
}
@Override
public boolean acceptUrl(String url) {
String lowercaseUrl = url.toLowerCase();
public boolean onAcceptUrl(final String url) {
final String lowercaseUrl = url.toLowerCase();
if (lowercaseUrl.contains("youtube")
|| lowercaseUrl.contains("youtu.be")
|| lowercaseUrl.contains("hooktube")) {
// bad programming I know
try {
getId(url);
onGetIdFromUrl(url);
return true;
} catch (Exception e) {
return false;

17
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandler.java → extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/urlIdHandlers/YoutubeTrendingUrlIdHandler.java
View File

@ -1,4 +1,4 @@
package org.schabi.newpipe.extractor.services.youtube;
package org.schabi.newpipe.extractor.services.youtube.urlIdHandlers;
/*
* Created by Christian Schabesberger on 12.08.17.
@ -20,27 +20,24 @@ package org.schabi.newpipe.extractor.services.youtube;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import org.schabi.newpipe.extractor.ListUrlIdHandler;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Parser;
public class YoutubeTrendingUrlIdHandler implements UrlIdHandler {
public class YoutubeTrendingUrlIdHandler extends ListUrlIdHandler {
public String getUrl(String id) {
public String getUrl() {
return "https://www.youtube.com/feed/trending";
}
@Override
public String getId(String url) {
public String onGetIdFromUrl(String url) {
return "Trending";
}
@Override
public String cleanUrl(String url) {
return getUrl("");
}
@Override
public boolean acceptUrl(String url) {
public boolean onAcceptUrl(final String url) {
return Parser.isMatch("^(https://|http://|)(www.|m.|)youtube.com/feed/trending(|\\?.*)$", url);
}
}

9
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java
View File

@ -20,6 +20,7 @@ package org.schabi.newpipe.extractor.stream;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import org.nibor.autolink.LinkSpan;
import org.schabi.newpipe.extractor.Extractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.Subtitles;
@ -39,8 +40,8 @@ public abstract class StreamExtractor extends Extractor {
public static final int NO_AGE_LIMIT = 0;
public StreamExtractor(StreamingService service, String url) {
super(service, url);
public StreamExtractor(StreamingService service, UrlIdHandler urlIdHandler) {
super(service, urlIdHandler);
}
@Nonnull
@ -142,6 +143,10 @@ public abstract class StreamExtractor extends Extractor {
public abstract StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException;
public abstract String[] getDonationLinks() throws ExtractionException;
public abstract String[] getAffiliateLinks() throws ExtractionException;
public LinkSpan[] getLinksFromDescription() throws ExtractionException {
return Parser.getLinkSpanFromString(getDescription());
}
/**
* Analyses the webpage's document and extracts any error message there might be.

54
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java
View File

@ -1,10 +1,14 @@
package org.schabi.newpipe.extractor.stream;
import org.nibor.autolink.LinkSpan;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.DashMpdParser;
import org.schabi.newpipe.extractor.utils.DonationLinkHelper;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.util.ArrayList;
@ -62,6 +66,7 @@ public class StreamInfo extends Info {
streamInfo = extractImportantData(extractor);
streamInfo = extractStreams(streamInfo, extractor);
streamInfo = extractOptionalData(streamInfo, extractor);
streamInfo = extractLinks(streamInfo, extractor);
} catch (ExtractionException e) {
// Currently YouTube does not distinguish between age restricted videos and videos blocked
// by country. This means that during the initialisation of the extractor, the extractor
@ -85,7 +90,7 @@ public class StreamInfo extends Info {
// if one of these is not available an exception is meant to be thrown directly into the frontend.
int serviceId = extractor.getServiceId();
String url = extractor.getCleanUrl();
String url = extractor.getUrl();
String originalUrl = extractor.getOriginalUrl();
StreamType streamType = extractor.getStreamType();
String id = extractor.getId();
@ -171,6 +176,25 @@ public class StreamInfo extends Info {
return streamInfo;
}
private static StreamInfo extractLinks(StreamInfo streamInfo, StreamExtractor extractor) throws ParsingException {
try {
streamInfo.setLinksInDescription(extractor.getLinksFromDescription());
} catch (Exception e) {
e.printStackTrace();
}
try {
streamInfo.setAffiliateLinks(extractor.getAffiliateLinks());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setDonationLinks(extractor.getDonationLinks());
} catch (Exception e) {
streamInfo.addError(e);
}
}
private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtractor extractor) {
/* ---- optional data goes here: ---- */
// If one of these fails, the frontend needs to handle that they are not available.
@ -274,6 +298,10 @@ public class StreamInfo extends Info {
private long startPosition = 0;
private List<Subtitles> subtitles;
private LinkSpan[] linksInDescription;
private String[] donationLinks;
private String[] affiliateLinks;
/**
* Get the stream type
*
@ -466,4 +494,28 @@ public class StreamInfo extends Info {
public void setSubtitles(List<Subtitles> subtitles) {
this.subtitles = subtitles;
}
public String[] getDonationLinks() {
return donationLinks;
}
public void setDonationLinks(String[] donationLinks) {
this.donationLinks = donationLinks;
}
public String[] getAffiliateLinks() {
return affiliateLinks;
}
public void setAffiliateLinks(String[] affiliateLinks) {
this.affiliateLinks = affiliateLinks;
}
public LinkSpan[] getLinksInDescription() {
return linksInDescription;
}
public void setLinksInDescription(LinkSpan[] linksInDescription) {
this.linksInDescription = linksInDescription;
}
}

23
extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java
View File

@ -7,12 +7,16 @@ public class DonationLinkHelper {
public enum DonationService {
NO_DONATION,
PATREON,
PAYPAL
PAYPAL,
}
public enum AffiliateService {
NO_AFILIATE,
AMAZON,
}
public static DonationService getServiceByLink(String link) throws MalformedURLException {
URL url = new URL(link);
public static DonationService getDonatoinServiceByLink(String link) throws MalformedURLException {
URL url = new URL(fixLink(link));
switch (url.getHost()) {
case "www.patreon.com":
return DonationService.PATREON;
@ -27,5 +31,18 @@ public class DonationLinkHelper {
}
}
public static AffiliateService getAffiliateServiceByLink(String link) throws MalformedURLException {
URL url = new URL(fixLink(link));
switch (url.getHost()) {
case "amzn.to": return AffiliateService.AMAZON;
default: return AffiliateService.NO_AFILIATE;
}
}
private static String fixLink(String link) {
return (link.startsWith("https://") || link.startsWith("http://"))
? link
: "https://" + link;
}
}

44
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java
View File

@ -88,20 +88,38 @@ public class Parser {
public static String[] getLinksFromString(final String txt) throws ParsingException {
try {
ArrayList<String> links = new ArrayList<>();
LinkExtractor linkExtractor = LinkExtractor.builder()
.linkTypes(EnumSet.of(LinkType.URL, LinkType.WWW))
.build();
Iterable<LinkSpan> linkss = linkExtractor.extractLinks(txt);
for(LinkSpan ls : linkss) {
links.add(txt.substring(ls.getBeginIndex(), ls.getEndIndex()));
}
String[] linksarray = new String[links.size()];
linksarray = links.toArray(linksarray);
return linksarray;
return getLinksFromLinkSpan(getLinkSpanFromString(txt), txt);
} catch (Exception e) {
throw new ParsingException("Could not get links from string", e);
}
}
}
public static LinkSpan[] getLinkSpanFromString(final String txt) throws ParsingException {
try {
ArrayList<LinkSpan> linksSpans = new ArrayList<>();
LinkExtractor linkExtractor = LinkExtractor.builder()
.linkTypes(EnumSet.of(LinkType.URL, LinkType.WWW))
.build();
for (LinkSpan ls : linkExtractor.extractLinks(txt)) {
linksSpans.add(ls);
}
LinkSpan[] linksarray = new LinkSpan[linksSpans.size()];
return linksSpans.toArray(linksarray);
} catch (Exception e) {
throw new ParsingException("Could not get links from string", e);
}
}
public static String[] getLinksFromLinkSpan(final LinkSpan[] links, final String txt) throws ParsingException {
try {
ArrayList<String> linksout = new ArrayList<>();
for (LinkSpan ls : links) {
linksout.add(txt.substring(ls.getBeginIndex(), ls.getEndIndex()));
}
String[] linksarray = new String[linksout.size()];
return linksout.toArray(linksarray);
} catch (Exception e) {
throw new ParsingException("Could not get LinkSpans from string", e);
}
}
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java
View File

@ -6,6 +6,9 @@ import java.util.List;
public class Utils {
private static final String HTTP = "http://";
private static final String HTTPS = "https://";
private Utils() {
//no instance
}
@ -46,9 +49,6 @@ public class Utils {
}
}
private static final String HTTP = "http://";
private static final String HTTPS = "https://";
public static String replaceHttpWithHttps(final String url) {
if (url == null) return null;

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseExtractorTest.java
View File

@ -5,6 +5,6 @@ public interface BaseExtractorTest {
void testServiceId() throws Exception;
void testName() throws Exception;
void testId() throws Exception;
void testCleanUrl() throws Exception;
void testUrl() throws Exception;
void testOriginalUrl() throws Exception;
}

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/BasePlaylistExtractorTest.java
View File

@ -4,7 +4,6 @@ package org.schabi.newpipe.extractor.services;
public interface BasePlaylistExtractorTest extends BaseListExtractorTest {
void testThumbnailUrl() throws Exception;
void testBannerUrl() throws Exception;
void testUploaderUrl() throws Exception;
void testUploaderName() throws Exception;
void testUploaderAvatarUrl() throws Exception;
void testStreamCount() throws Exception;

15
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java
View File

@ -5,6 +5,7 @@ import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import static org.junit.Assert.*;
@ -48,12 +49,12 @@ public class SoundcloudChannelExtractorTest {
}
@Test
public void testCleanUrl() {
assertEquals("https://soundcloud.com/liluzivert", extractor.getCleanUrl());
public void testUrl() throws ParsingException {
assertEquals("https://soundcloud.com/liluzivert", extractor.getUrl());
}
@Test
public void testOriginalUrl() {
public void testOriginalUrl() throws ParsingException {
assertEquals("http://soundcloud.com/liluzivert/sets", extractor.getOriginalUrl());
}
@ -118,7 +119,7 @@ public class SoundcloudChannelExtractorTest {
@Test
public void testGetPageInNewExtractor() throws Exception {
final ChannelExtractor newExtractor = SoundCloud.getChannelExtractor(extractor.getCleanUrl());
final ChannelExtractor newExtractor = SoundCloud.getChannelExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor, SoundCloud.getServiceId());
}
@ -142,12 +143,12 @@ public class SoundcloudChannelExtractorTest {
}
@Test
public void testCleanUrl() {
assertEquals("https://soundcloud.com/dubmatix", extractor.getCleanUrl());
public void testUrl() throws ParsingException {
assertEquals("https://soundcloud.com/dubmatix", extractor.getUrl());
}
@Test
public void testOriginalUrl() {
public void testOriginalUrl() throws ParsingException {
assertEquals("https://soundcloud.com/dubmatix", extractor.getOriginalUrl());
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java
View File

@ -88,6 +88,6 @@ public class SoundcloudChartsExtractorTest {
@Test
public void testGetCleanUrl() throws Exception {
assertEquals(extractor.getCleanUrl(), "https://soundcloud.com/charts/top");
assertEquals(extractor.getUrl(), "https://soundcloud.com/charts/top");
}
}

15
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandlerTest.java
View File

@ -4,6 +4,7 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.assertEquals;
@ -22,19 +23,19 @@ public class SoundcloudChartsUrlIdHandlerTest {
}
@Test
public void getUrl() {
assertEquals(urlIdHandler.getUrl("Top 50"), "https://soundcloud.com/charts/top");
assertEquals(urlIdHandler.getUrl("New & hot"), "https://soundcloud.com/charts/new");
public void getUrl() throws Exception {
assertEquals(urlIdHandler.setId("Top 50").getUrl(), "https://soundcloud.com/charts/top");
assertEquals(urlIdHandler.setId("New & hot").getUrl(), "https://soundcloud.com/charts/new");
}
@Test
public void getId() {
assertEquals(urlIdHandler.getId("http://soundcloud.com/charts/top?genre=all-music"), "Top 50");
assertEquals(urlIdHandler.getId("HTTP://www.soundcloud.com/charts/new/?genre=all-music&country=all-countries"), "New & hot");
public void getId() throws ParsingException {
assertEquals(urlIdHandler.setUrl("http://soundcloud.com/charts/top?genre=all-music").getId(), "Top 50");
assertEquals(urlIdHandler.setUrl("HTTP://www.soundcloud.com/charts/new/?genre=all-music&country=all-countries").getId(), "New & hot");
}
@Test
public void acceptUrl() {
public void acceptUrl() throws ParsingException {
assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/charts"));
assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/charts/"));
assertTrue(urlIdHandler.acceptUrl("https://www.soundcloud.com/charts/new"));

32
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java
View File

@ -51,12 +51,12 @@ public class SoundcloudPlaylistExtractorTest {
}
@Test
public void testCleanUrl() {
assertEquals("https://soundcloud.com/liluzivert/sets/the-perfect-luv-tape-r", extractor.getCleanUrl());
public void testUrl() throws Exception {
assertEquals("https://soundcloud.com/liluzivert/sets/the-perfect-luv-tape-r", extractor.getUrl());
}
@Test
public void testOriginalUrl() {
public void testOriginalUrl() throws Exception {
assertEquals("https://soundcloud.com/liluzivert/sets/the-perfect-luv-tape-r?test=123", extractor.getOriginalUrl());
}
@ -125,7 +125,7 @@ public class SoundcloudPlaylistExtractorTest {
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
extractor = (SoundcloudPlaylistExtractor) SoundCloud
.getPlaylistExtractor("http://soundcloud.com/finn-trapple/sets/random-house-dance-music-2");
.getPlaylistExtractor("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2");
extractor.fetchPage();
}
@ -140,22 +140,22 @@ public class SoundcloudPlaylistExtractorTest {
@Test
public void testName() {
assertEquals("Random House & Dance Music #2", extractor.getName());
assertEquals("House, Electro , Dance Music 2", extractor.getName());
}
@Test
public void testId() {
assertEquals("436855608", extractor.getId());
assertEquals("310980722", extractor.getId());
}
@Test
public void testCleanUrl() {
assertEquals("https://soundcloud.com/finn-trapple/sets/random-house-dance-music-2", extractor.getCleanUrl());
public void testUrl() throws Exception {
assertEquals("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2", extractor.getUrl());
}
@Test
public void testOriginalUrl() {
assertEquals("http://soundcloud.com/finn-trapple/sets/random-house-dance-music-2", extractor.getOriginalUrl());
public void testOriginalUrl() throws Exception {
assertEquals("https://soundcloud.com/hunter-leader/sets/house-electro-dance-music-2", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
@ -191,12 +191,12 @@ public class SoundcloudPlaylistExtractorTest {
public void testUploaderUrl() {
final String uploaderUrl = extractor.getUploaderUrl();
assertIsSecureUrl(uploaderUrl);
assertTrue(uploaderUrl, uploaderUrl.contains("finn-trapple"));
assertTrue(uploaderUrl, uploaderUrl.contains("hunter-leader"));
}
@Test
public void testUploaderName() {
assertEquals("Finn TrApple", extractor.getUploaderName());
assertEquals("Gosu", extractor.getUploaderName());
}
@Test
@ -227,7 +227,7 @@ public class SoundcloudPlaylistExtractorTest {
@Test
public void testGetPageInNewExtractor() throws Exception {
final PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getCleanUrl());
final PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor, SoundCloud.getServiceId());
}
@ -251,12 +251,12 @@ public class SoundcloudPlaylistExtractorTest {
}
@Test
public void testCleanUrl() {
assertEquals("https://soundcloud.com/user350509423/sets/edm-xxx", extractor.getCleanUrl());
public void testUrl() throws Exception {
assertEquals("https://soundcloud.com/user350509423/sets/edm-xxx", extractor.getUrl());
}
@Test
public void testOriginalUrl() {
public void testOriginalUrl() throws Exception {
assertEquals("https://soundcloud.com/user350509423/sets/edm-xxx", extractor.getOriginalUrl());
}

26
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandlerTest.java
View File

@ -25,7 +25,7 @@ public class SoundcloudStreamUrlIdHandlerTest {
@Test(expected = IllegalArgumentException.class)
public void getIdWithNullAsUrl() throws ParsingException {
urlIdHandler.getId(null);
urlIdHandler.setUrl(null).getId();
}
@Test
@ -37,7 +37,7 @@ public class SoundcloudStreamUrlIdHandlerTest {
for (String invalidUrl : invalidUrls) {
Throwable exception = null;
try {
urlIdHandler.getId(invalidUrl);
urlIdHandler.setUrl(invalidUrl).getId();
} catch (ParsingException e) {
exception = e;
}
@ -49,21 +49,21 @@ public class SoundcloudStreamUrlIdHandlerTest {
@Test
public void getId() throws Exception {
assertEquals("309689103", urlIdHandler.getId("https://soundcloud.com/liluzivert/15-ysl"));
assertEquals("309689082", urlIdHandler.getId("https://www.soundcloud.com/liluzivert/15-luv-scars-ko"));
assertEquals("309689035", urlIdHandler.getId("http://soundcloud.com/liluzivert/15-boring-shit"));
assertEquals("294488599", urlIdHandler.getId("http://www.soundcloud.com/liluzivert/secure-the-bag-produced-by-glohan-beats"));
assertEquals("294488438", urlIdHandler.getId("HtTpS://sOuNdClOuD.cOm/LiLuZiVeRt/In-O4-pRoDuCeD-bY-dP-bEaTz"));
assertEquals("294488147", urlIdHandler.getId("https://soundcloud.com/liluzivert/fresh-produced-by-zaytoven#t=69"));
assertEquals("294487876", urlIdHandler.getId("https://soundcloud.com/liluzivert/threesome-produced-by-zaytoven#t=1:09"));
assertEquals("294487684", urlIdHandler.getId("https://soundcloud.com/liluzivert/blonde-brigitte-produced-manny-fresh#t=1:9"));
assertEquals("294487428", urlIdHandler.getId("https://soundcloud.com/liluzivert/today-produced-by-c-note#t=1m9s"));
assertEquals("294487157", urlIdHandler.getId("https://soundcloud.com/liluzivert/changed-my-phone-produced-by-c-note#t=1m09s"));
assertEquals("309689103", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/15-ysl").getId());
assertEquals("309689082", urlIdHandler.setUrl("https://www.soundcloud.com/liluzivert/15-luv-scars-ko").getId());
assertEquals("309689035", urlIdHandler.setUrl("http://soundcloud.com/liluzivert/15-boring-shit").getId());
assertEquals("294488599", urlIdHandler.setUrl("http://www.soundcloud.com/liluzivert/secure-the-bag-produced-by-glohan-beats").getId());
assertEquals("294488438", urlIdHandler.setUrl("HtTpS://sOuNdClOuD.cOm/LiLuZiVeRt/In-O4-pRoDuCeD-bY-dP-bEaTz").getId());
assertEquals("294488147", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/fresh-produced-by-zaytoven#t=69").getId());
assertEquals("294487876", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/threesome-produced-by-zaytoven#t=1:09").getId());
assertEquals("294487684", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/blonde-brigitte-produced-manny-fresh#t=1:9").getId());
assertEquals("294487428", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/today-produced-by-c-note#t=1m9s").getId());
assertEquals("294487157", urlIdHandler.setUrl("https://soundcloud.com/liluzivert/changed-my-phone-produced-by-c-note#t=1m09s").getId());
}
@Test
public void testAcceptUrl() {
public void testAcceptUrl() throws ParsingException {
assertTrue(urlIdHandler.acceptUrl("https://soundcloud.com/liluzivert/15-ysl"));
assertTrue(urlIdHandler.acceptUrl("https://www.soundcloud.com/liluzivert/15-luv-scars-ko"));
assertTrue(urlIdHandler.acceptUrl("http://soundcloud.com/liluzivert/15-boring-shit"));

3
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java
View File

@ -6,6 +6,7 @@ import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
@ -64,7 +65,7 @@ public class SoundcloudSubscriptionExtractorTest {
}
}
private void testList(List<SubscriptionItem> subscriptionItems) {
private void testList(List<SubscriptionItem> subscriptionItems) throws ParsingException {
for (SubscriptionItem item : subscriptionItems) {
assertNotNull(item.getName());
assertNotNull(item.getUrl());

29
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java
View File

@ -1,13 +1,14 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
@ -49,12 +50,12 @@ public class YoutubeChannelExtractorTest {
}
@Test
public void testCleanUrl() {
assertEquals("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw", extractor.getCleanUrl());
public void testUrl() throws ParsingException {
assertEquals("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw", extractor.getUrl());
}
@Test
public void testOriginalUrl() {
public void testOriginalUrl() throws ParsingException {
assertEquals("http://www.youtube.com/user/Gronkh", extractor.getOriginalUrl());
}
@ -130,7 +131,7 @@ public class YoutubeChannelExtractorTest {
@Test
public void testGetPageInNewExtractor() throws Exception {
final ChannelExtractor newExtractor = YouTube.getChannelExtractor(extractor.getCleanUrl());
final ChannelExtractor newExtractor = YouTube.getChannelExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor, YouTube.getServiceId());
}
@ -155,12 +156,12 @@ public class YoutubeChannelExtractorTest {
}
@Test
public void testCleanUrl() {
assertEquals("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q", extractor.getCleanUrl());
public void testUrl() throws ParsingException {
assertEquals("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q", extractor.getUrl());
}
@Test
public void testOriginalUrl() {
public void testOriginalUrl() throws ParsingException {
assertEquals("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q", extractor.getOriginalUrl());
}
@ -251,12 +252,12 @@ public class YoutubeChannelExtractorTest {
}
@Test
public void testCleanUrl() {
assertEquals("https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg", extractor.getCleanUrl());
public void testUrl() throws ParsingException {
assertEquals("https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg", extractor.getUrl());
}
@Test
public void testOriginalUrl() {
public void testOriginalUrl() throws ParsingException {
assertEquals("https://www.youtube.com/user/CaptainDisillusion/videos", extractor.getOriginalUrl());
}
@ -340,12 +341,12 @@ public class YoutubeChannelExtractorTest {
}
@Test
public void testCleanUrl() {
assertEquals("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w", extractor.getCleanUrl());
public void testUrl() throws ParsingException {
assertEquals("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w", extractor.getUrl());
}
@Test
public void testOriginalUrl() {
public void testOriginalUrl() throws ParsingException {
assertEquals("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w", extractor.getOriginalUrl());
}

17
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandlerTest.java
View File

@ -5,6 +5,7 @@ import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeChannelUrlIdHandler;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@ -39,17 +40,17 @@ public class YoutubeChannelUrlIdHandlerTest {
@Test
public void getIdFromUrl() throws ParsingException {
assertEquals("user/Gronkh", urlIdHandler.getId("https://www.youtube.com/user/Gronkh"));
assertEquals("user/Netzkino", urlIdHandler.getId("https://www.youtube.com/user/Netzkino/videos"));
assertEquals("user/Gronkh", urlIdHandler.setUrl("https://www.youtube.com/user/Gronkh").getId());
assertEquals("user/Netzkino", urlIdHandler.setUrl("https://www.youtube.com/user/Netzkino/videos").getId());
assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.getId("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA"));
assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.getId("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1"));
assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.setUrl("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA").getId());
assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.setUrl("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId());
assertEquals("user/Gronkh", urlIdHandler.getId("https://hooktube.com/user/Gronkh"));
assertEquals("user/Netzkino", urlIdHandler.getId("https://hooktube.com/user/Netzkino/videos"));
assertEquals("user/Gronkh", urlIdHandler.setUrl("https://hooktube.com/user/Gronkh").getId());
assertEquals("user/Netzkino", urlIdHandler.setUrl("https://hooktube.com/user/Netzkino/videos").getId());
assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.getId("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA"));
assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.getId("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1"));
assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.setUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA").getId());
assertEquals("channel/UClq42foiSgl7sSpLupnugGA", urlIdHandler.setUrl("https://hooktube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1").getId());
}
}

16
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View File

@ -7,8 +7,10 @@ import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import static org.junit.Assert.assertEquals;
@ -53,12 +55,12 @@ public class YoutubePlaylistExtractorTest {
}
@Test
public void testCleanUrl() {
assertEquals("https://www.youtube.com/playlist?list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj", extractor.getCleanUrl());
public void testUrl() throws ParsingException {
assertEquals("https://www.youtube.com/playlist?list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj", extractor.getUrl());
}
@Test
public void testOriginalUrl() {
public void testOriginalUrl() throws ParsingException {
assertEquals("http://www.youtube.com/watch?v=lp-EO5I60KA&list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj", extractor.getOriginalUrl());
}
@ -135,7 +137,7 @@ public class YoutubePlaylistExtractorTest {
@Test
public void testGetPageInNewExtractor() throws Exception {
final PlaylistExtractor newExtractor = YouTube.getPlaylistExtractor(extractor.getCleanUrl());
final PlaylistExtractor newExtractor = YouTube.getPlaylistExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor, YouTube.getServiceId());
}
@ -160,12 +162,12 @@ public class YoutubePlaylistExtractorTest {
}
@Test
public void testCleanUrl() {
assertEquals("https://www.youtube.com/playlist?list=PLOy0j9AvlVZPto6IkjKfpu0Scx--7PGTC", extractor.getCleanUrl());
public void testUrl() throws ParsingException {
assertEquals("https://www.youtube.com/playlist?list=PLOy0j9AvlVZPto6IkjKfpu0Scx--7PGTC", extractor.getUrl());
}
@Test
public void testOriginalUrl() {
public void testOriginalUrl() throws ParsingException {
assertEquals("https://www.youtube.com/playlist?list=PLOy0j9AvlVZPto6IkjKfpu0Scx--7PGTC", extractor.getOriginalUrl());
}

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java
View File

@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchEngine;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java
View File

@ -7,6 +7,8 @@ import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeStreamUrlIdHandler;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.SubtitlesFormat;
import org.schabi.newpipe.extractor.stream.VideoStream;

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java
View File

@ -6,6 +6,7 @@ import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.utils.Utils;

7
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java
View File

@ -25,6 +25,7 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import static org.junit.Assert.assertTrue;
@ -49,4 +50,10 @@ public class YoutubeStreamExtractorDonationTest {
assertTrue(String.valueOf(extractor.getDonationLinks().length),
extractor.getDonationLinks().length == 2);
}
@Test
public void getAffiliateLinksTest() throws Exception {
assertTrue(String.valueOf(extractor.getAffiliateLinks().length),
extractor.getAffiliateLinks().length == 1);
}
}

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java
View File

@ -7,6 +7,8 @@ import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeStreamUrlIdHandler;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.SubtitlesFormat;
import org.schabi.newpipe.extractor.stream.VideoStream;
@ -108,7 +110,7 @@ public class YoutubeStreamExtractorRestrictedTest {
streams.addAll(extractor.getVideoStreams());
streams.addAll(extractor.getVideoOnlyStreams());
assertTrue(streams.size() > 0);
assertTrue(Integer.toString(streams.size()),streams.size() > 0);
for (VideoStream s : streams) {
assertTrue(s.getUrl(),
s.getUrl().contains(HTTPS));

77
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandlerTest.java
View File

@ -6,6 +6,7 @@ import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeStreamUrlIdHandler;
import java.util.ArrayList;
import java.util.List;
@ -25,14 +26,14 @@ public class YoutubeStreamUrlIdHandlerTest {
NewPipe.init(Downloader.getInstance());
}
@Test(expected = NullPointerException.class)
@Test(expected = IllegalArgumentException.class)
public void getIdWithNullAsUrl() throws ParsingException {
urlIdHandler.getId(null);
urlIdHandler.setId(null);
}
@Test(expected = FoundAdException.class)
public void getIdForAd() throws ParsingException {
urlIdHandler.getId(AD_URL);
urlIdHandler.setUrl(AD_URL).getId();
}
@Test
@ -44,7 +45,7 @@ public class YoutubeStreamUrlIdHandlerTest {
for (String invalidUrl : invalidUrls) {
Throwable exception = null;
try {
urlIdHandler.getId(invalidUrl);
urlIdHandler.setUrl(invalidUrl).getId();
} catch (ParsingException e) {
exception = e;
}
@ -56,42 +57,42 @@ public class YoutubeStreamUrlIdHandlerTest {
@Test
public void getIdfromYt() throws Exception {
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/watch?v=jZViOEv90dI"));
assertEquals("W-fFHeTX70Q", urlIdHandler.getId("https://www.youtube.com/watch?v=W-fFHeTX70Q"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/watch?v=jZViOEv90dI?t=100"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("HTTPS://www.youtube.com/watch?v=jZViOEv90dI?t=100"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://youtu.be/jZViOEv90dI?t=9s"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("HTTPS://Youtu.be/jZViOEv90dI?t=9s"));
assertEquals("uEJuoEs1UxY", urlIdHandler.getId("http://www.youtube.com/watch_popup?v=uEJuoEs1UxY"));
assertEquals("uEJuoEs1UxY", urlIdHandler.getId("http://www.Youtube.com/watch_popup?v=uEJuoEs1UxY"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/embed/jZViOEv90dI"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube-nocookie.com/embed/jZViOEv90dI"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube.com/watch?v=jZViOEv90dI"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://youtube.com/watch?v=jZViOEv90dI"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://youtu.be/jZViOEv90dI?t=9s"));
assertEquals("7_WWz2DSnT8", urlIdHandler.getId("https://youtu.be/7_WWz2DSnT8"));
assertEquals("oy6NvWeVruY", urlIdHandler.getId("https://m.youtube.com/watch?v=oy6NvWeVruY"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube.com/embed/jZViOEv90dI"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.Youtube.com/embed/jZViOEv90dI"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube-nocookie.com/embed/jZViOEv90dI"));
assertEquals("EhxJLojIE_o", urlIdHandler.getId("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI"));
assertEquals("jZViOEv90dI", urlIdHandler.getId("vnd.youtube:jZViOEv90dI"));
assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://www.youtube.com/watch?v=jZViOEv90dI").getId());
assertEquals("W-fFHeTX70Q", urlIdHandler.setUrl("https://www.youtube.com/watch?v=W-fFHeTX70Q").getId());
assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://www.youtube.com/watch?v=jZViOEv90dI?t=100").getId());
assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100").getId());
assertEquals("jZViOEv90dI", urlIdHandler.setUrl("HTTPS://www.youtube.com/watch?v=jZViOEv90dI?t=100").getId());
assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://youtu.be/jZViOEv90dI?t=9s").getId());
assertEquals("jZViOEv90dI", urlIdHandler.setUrl("HTTPS://Youtu.be/jZViOEv90dI?t=9s").getId());
assertEquals("uEJuoEs1UxY", urlIdHandler.setUrl("http://www.youtube.com/watch_popup?v=uEJuoEs1UxY").getId());
assertEquals("uEJuoEs1UxY", urlIdHandler.setUrl("http://www.Youtube.com/watch_popup?v=uEJuoEs1UxY").getId());
assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://www.youtube.com/embed/jZViOEv90dI").getId());
assertEquals("jZViOEv90dI", urlIdHandler.setUrl("https://www.youtube-nocookie.com/embed/jZViOEv90dI").getId());
assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://www.youtube.com/watch?v=jZViOEv90dI").getId());
assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://youtube.com/watch?v=jZViOEv90dI").getId());
assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://youtu.be/jZViOEv90dI?t=9s").getId());
assertEquals("7_WWz2DSnT8", urlIdHandler.setUrl("https://youtu.be/7_WWz2DSnT8").getId());
assertEquals("oy6NvWeVruY", urlIdHandler.setUrl("https://m.youtube.com/watch?v=oy6NvWeVruY").getId());
assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://www.youtube.com/embed/jZViOEv90dI").getId());
assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://www.Youtube.com/embed/jZViOEv90dI").getId());
assertEquals("jZViOEv90dI", urlIdHandler.setUrl("http://www.youtube-nocookie.com/embed/jZViOEv90dI").getId());
assertEquals("EhxJLojIE_o", urlIdHandler.setUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare").getId());
assertEquals("jZViOEv90dI", urlIdHandler.setUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI").getId());
assertEquals("jZViOEv90dI", urlIdHandler.setUrl("vnd.youtube:jZViOEv90dI").getId());
}
@Test
public void getIdfromSharedLinksYt() throws Exception {
String sharedId = "7JIArTByb3E";
String realId = "Q7JsK50NGaA";
assertEquals(realId, urlIdHandler.getId("vnd.youtube://www.YouTube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link"));
assertEquals(realId, urlIdHandler.getId("vnd.youtube://www.youtube.com/shared?ci=" + sharedId));
assertEquals(realId, urlIdHandler.getId("https://www.youtube.com/shared?ci=" + sharedId));
assertEquals(realId, urlIdHandler.setUrl("vnd.youtube://www.YouTube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link").getId());
assertEquals(realId, urlIdHandler.setUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId).getId());
assertEquals(realId, urlIdHandler.setUrl("https://www.youtube.com/shared?ci=" + sharedId).getId());
}
@Test
public void testAcceptYtUrl() {
public void testAcceptYtUrl() throws ParsingException {
assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI"));
assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI?t=100"));
assertTrue(urlIdHandler.acceptUrl("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100"));
@ -111,7 +112,7 @@ public class YoutubeStreamUrlIdHandlerTest {
}
@Test
public void testAcceptSharedYtUrl() {
public void testAcceptSharedYtUrl() throws ParsingException {
String sharedId = "8A940MXKFmQ";
assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link"));
assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId));
@ -119,7 +120,7 @@ public class YoutubeStreamUrlIdHandlerTest {
}
@Test
public void testAcceptHookUrl() {
public void testAcceptHookUrl() throws ParsingException {
assertTrue(urlIdHandler.acceptUrl("https://hooktube.com/watch?v=TglNG-yjabU"));
assertTrue(urlIdHandler.acceptUrl("hooktube.com/watch?v=3msbfr6pBNE"));
assertTrue(urlIdHandler.acceptUrl("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2"));
@ -130,11 +131,11 @@ public class YoutubeStreamUrlIdHandlerTest {
@Test
public void testGetHookIdfromUrl() throws ParsingException {
assertEquals("TglNG-yjabU", urlIdHandler.getId("https://hooktube.com/watch?v=TglNG-yjabU"));
assertEquals("3msbfr6pBNE", urlIdHandler.getId("hooktube.com/watch?v=3msbfr6pBNE"));
assertEquals("ocH3oSnZG3c", urlIdHandler.getId("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2"));
assertEquals("3msbfr6pBNE", urlIdHandler.getId("hooktube.com/watch/3msbfr6pBNE"));
assertEquals("3msbfr6pBNE", urlIdHandler.getId("hooktube.com/v/3msbfr6pBNE"));
assertEquals("3msbfr6pBNE", urlIdHandler.getId("hooktube.com/embed/3msbfr6pBNE"));
assertEquals("TglNG-yjabU", urlIdHandler.setUrl("https://hooktube.com/watch?v=TglNG-yjabU").getId());
assertEquals("3msbfr6pBNE", urlIdHandler.setUrl("hooktube.com/watch?v=3msbfr6pBNE").getId());
assertEquals("ocH3oSnZG3c", urlIdHandler.setUrl("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2").getId());
assertEquals("3msbfr6pBNE", urlIdHandler.setUrl("hooktube.com/watch/3msbfr6pBNE").getId());
assertEquals("3msbfr6pBNE", urlIdHandler.setUrl("hooktube.com/v/3msbfr6pBNE").getId());
assertEquals("3msbfr6pBNE", urlIdHandler.setUrl("hooktube.com/embed/3msbfr6pBNE").getId());
}
}

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java
View File

@ -6,6 +6,7 @@ import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscriptionExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionItem;

7
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java
View File

@ -25,6 +25,8 @@ import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor;
import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeTrendingUrlIdHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.Utils;
@ -48,6 +50,7 @@ public class YoutubeTrendingExtractorTest {
.getKioskList()
.getExtractorById("Trending", null);
extractor.fetchPage();
extractor.setContentCountry("de");
}
@Test
@ -91,7 +94,7 @@ public class YoutubeTrendingExtractorTest {
}
@Test
public void testGetCleanUrl() {
assertEquals(extractor.getCleanUrl(), extractor.getCleanUrl(), "https://www.youtube.com/feed/trending");
public void testGetUrl() throws Exception {
assertEquals(extractor.getUrl(), extractor.getUrl(), "https://www.youtube.com/feed/trending");
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java
View File

@ -45,7 +45,7 @@ public class YoutubeTrendingKioskInfoTest {
StreamingService service = YouTube;
UrlIdHandler urlIdHandler = service.getKioskList().getUrlIdHandlerByType("Trending");
kioskInfo = KioskInfo.getInfo(YouTube, urlIdHandler.getUrl("Trending"), null);
kioskInfo = KioskInfo.getInfo(YouTube, urlIdHandler.setId("Trending").getUrl(), null);
}
@Test

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java
View File

@ -25,6 +25,7 @@ import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.services.youtube.urlIdHandlers.YoutubeTrendingUrlIdHandler;
import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.assertEquals;
@ -46,13 +47,13 @@ public class YoutubeTrendingUrlIdHandlerTest {
@Test
public void getUrl()
throws Exception {
assertEquals(urlIdHandler.getUrl(""), "https://www.youtube.com/feed/trending");
assertEquals(urlIdHandler.setId("").getUrl(), "https://www.youtube.com/feed/trending");
}
@Test
public void getId()
throws Exception {
assertEquals(urlIdHandler.getId(""), "Trending");
assertEquals(urlIdHandler.setUrl("").getId(), "Trending");
}
@Test