lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master'

This commit is contained in:
Guilherme Calé 2018-02-04 11:39:08 +00:00
parent 36b681ac39 7fd21ec085
commit 9641c60c8a
39 changed files with 474 additions and 347 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/org/schabi/newpipe/extractor/Extractor.java
View File

@ -66,6 +66,10 @@ public abstract class Extractor {
if(!pageFetched) throw new IllegalStateException("Page is not fetched. Make sure you call fetchPage()");
}
protected boolean isPageFetched() {
return pageFetched;
}
/**
* Fetch the current page.
* @param downloader the download to use

21
src/main/java/org/schabi/newpipe/extractor/ListExtractor.java
View File

@ -15,29 +15,10 @@ public abstract class ListExtractor extends Extractor {
/**
* Get a new ListExtractor with the given nextStreamsUrl set.
* <p>
* The extractor <b>WILL</b> fetch the page if {@link #fetchPageUponCreation()} return true, otherwise, it will <b>NOT</b>.
* <p>
* You can call {@link #fetchPage()} later, but this is mainly used just to get more items, so we don't waste bandwidth
* downloading the whole page, but if the service that is being implemented need it, just do its own logic in {@link #fetchPageUponCreation()}.
*/
public ListExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
public ListExtractor(StreamingService service, String url, String nextStreamsUrl) throws ExtractionException {
super(service, url);
setNextStreamsUrl(nextStreamsUrl);
if (fetchPageUponCreation()) {
fetchPage();
}
}
/**
* Decide if the page will be fetched upon creation.
* <p>
* The default implementation checks if the nextStreamsUrl is null or empty (indication that the caller
* don't need or know what is the next page, thus, fetch the page).
*/
protected boolean fetchPageUponCreation() {
return nextStreamsUrl == null || nextStreamsUrl.isEmpty();
}
@Nonnull

15
src/main/java/org/schabi/newpipe/extractor/Subtitles.java
View File

@ -2,23 +2,26 @@ package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.stream.SubtitlesFormat;
public class Subtitles {
import java.io.Serializable;
import java.util.Locale;
public class Subtitles implements Serializable {
private final SubtitlesFormat format;
private final String languageCode;
private final Locale locale;
private final String URL;
private final boolean autoGenerated;
public Subtitles(SubtitlesFormat format, String languageCode, String URL, boolean autoGenerated) {
public Subtitles(SubtitlesFormat format, Locale locale, String URL, boolean autoGenerated) {
this.format = format;
this.languageCode = languageCode;
this.locale = locale;
this.URL = URL;
this.autoGenerated = autoGenerated;
}
public SubtitlesFormat getFileType() { return format; }
public String getLanguageCode() {
return languageCode;
public Locale getLocale() {
return locale;
}
public String getURL() {

4
src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java
View File

@ -55,7 +55,9 @@ public class ChannelInfo extends ListInfo {
}
public static ChannelInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
return getInfo(service.getChannelExtractor(url));
ChannelExtractor extractor = service.getChannelExtractor(url);
extractor.fetchPage();
return getInfo(extractor);
}
public static ChannelInfo getInfo(ChannelExtractor extractor) throws ParsingException {

25
src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java
View File

@ -28,21 +28,24 @@ import java.io.IOException;
public class KioskInfo extends ListInfo {
public KioskInfo(int serviceId, String id, String url, String name) {
private KioskInfo(int serviceId, String id, String url, String name) {
super(serviceId, id, url, name);
}
public static ListExtractor.NextItemsResult getMoreItems(ServiceList serviceItem,
String url,
String nextStreamsUrl) throws IOException, ExtractionException {
return getMoreItems(serviceItem.getService(), url, nextStreamsUrl);
String nextStreamsUrl,
String contentCountry) throws IOException, ExtractionException {
return getMoreItems(serviceItem.getService(), url, nextStreamsUrl, contentCountry);
}
public static ListExtractor.NextItemsResult getMoreItems(StreamingService service,
String url,
String nextStreamsUrl) throws IOException, ExtractionException {
String nextStreamsUrl,
String contentCountry) throws IOException, ExtractionException {
KioskList kl = service.getKioskList();
KioskExtractor extractor = kl.getExtractorByUrl(url, nextStreamsUrl);
extractor.setContentCountry(contentCountry);
return extractor.getNextStreams();
}
@ -62,13 +65,17 @@ public class KioskInfo extends ListInfo {
String contentCountry) throws IOException, ExtractionException {
KioskList kl = service.getKioskList();
KioskExtractor extractor = kl.getExtractorByUrl(url, null);
return getInfo(extractor, contentCountry);
}
public static KioskInfo getInfo(KioskExtractor extractor,
String contentCountry) throws IOException, ExtractionException {
extractor.setContentCountry(contentCountry);
extractor.fetchPage();
return getInfo(extractor);
}
/**
* Get KioskInfo from KioskExtractor
*
* @param extractor an extractor where fetchPage() was already got called on.
*/
public static KioskInfo getInfo(KioskExtractor extractor) throws ExtractionException {
int serviceId = extractor.getServiceId();
String name = extractor.getName();

11
src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java
View File

@ -32,16 +32,23 @@ public class PlaylistInfo extends ListInfo {
}
public static PlaylistInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
return getInfo(service.getPlaylistExtractor(url));
PlaylistExtractor extractor = service.getPlaylistExtractor(url);
extractor.fetchPage();
return getInfo(extractor);
}
/**
* Get PlaylistInfo from PlaylistExtractor
*
* @param extractor an extractor where fetchPage() was already got called on.
*/
public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws ParsingException {
int serviceId = extractor.getServiceId();
String url = extractor.getCleanUrl();
String id = extractor.getId();
String name = extractor.getName();
PlaylistInfo info = new PlaylistInfo(serviceId, url, id, name);
PlaylistInfo info = new PlaylistInfo(serviceId, id, url, name);
try {
info.setStreamCount(extractor.getStreamCount());

3
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java
View File

@ -9,9 +9,8 @@ import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
private static final SoundcloudChannelUrlIdHandler instance = new SoundcloudChannelUrlIdHandler();
private final String URL_PATTERN = "^https?://(www\\.)?soundcloud.com/[0-9a-z_-]+" +
private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
"(/((tracks|albums|sets|reposts|followers|following)/?)?)?([#?].*)?$";
public static SoundcloudChannelUrlIdHandler getInstance() {

9
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsUrlIdHandler.java
View File

@ -4,6 +4,9 @@ import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.utils.Parser;
public class SoundcloudChartsUrlIdHandler implements UrlIdHandler {
private final String TOP_URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$";
private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$";
public String getUrl(String id) {
if (id.equals("Top 50")) {
return "https://soundcloud.com/charts/top";
@ -14,7 +17,7 @@ public class SoundcloudChartsUrlIdHandler implements UrlIdHandler {
@Override
public String getId(String url) {
if (Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$", url.toLowerCase())) {
if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) {
return "Top 50";
} else {
return "New & hot";
@ -23,7 +26,7 @@ public class SoundcloudChartsUrlIdHandler implements UrlIdHandler {
@Override
public String cleanUrl(String url) {
if (Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$", url.toLowerCase())) {
if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) {
return "https://soundcloud.com/charts/top";
} else {
return "https://soundcloud.com/charts/new";
@ -32,6 +35,6 @@ public class SoundcloudChartsUrlIdHandler implements UrlIdHandler {
@Override
public boolean acceptUrl(String url) {
return Parser.isMatch("^https?://(www\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$", url.toLowerCase());
return Parser.isMatch(URL_PATTERN, url.toLowerCase());
}
}

35
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudExtractorHelper.java Normal file
View File

@ -0,0 +1,35 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import com.grack.nanojson.JsonObject;
import javax.annotation.Nonnull;
public class SoundcloudExtractorHelper {
private static final String HTTP = "http://";
private static final String HTTPS = "https://";
private static String replaceHttpWithHttps(final String url) {
if(!url.isEmpty() && url.startsWith(HTTP)) {
return HTTPS + url.substring(HTTP.length());
}
return url;
}
@Nonnull
static String getUploaderUrl(JsonObject object) {
String url = object.getObject("user").getString("permalink_url", "");
return replaceHttpWithHttps(url);
}
@Nonnull
static String getAvatarUrl(JsonObject object) {
String url = object.getObject("user", new JsonObject()).getString("avatar_url", "");
return replaceHttpWithHttps(url);
}
public static String getUploaderName(JsonObject object) {
return object.getObject("user").getString("username", "");
}
}

7
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java
View File

@ -4,7 +4,6 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -69,17 +68,17 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
@Override
public String getUploaderUrl() {
return playlist.getObject("user").getString("permalink_url", "");
return SoundcloudExtractorHelper.getUploaderUrl(playlist);
}
@Override
public String getUploaderName() {
return playlist.getObject("user").getString("username", "");
return SoundcloudExtractorHelper.getUploaderName(playlist);
}
@Override
public String getUploaderAvatarUrl() {
return playlist.getObject("user", new JsonObject()).getString("avatar_url", "");
return SoundcloudExtractorHelper.getAvatarUrl(playlist);
}
@Override

3
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java
View File

@ -9,9 +9,8 @@ import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
private static final SoundcloudPlaylistUrlIdHandler instance = new SoundcloudPlaylistUrlIdHandler();
private final String URL_PATTERN = "^https?://(www\\.)?soundcloud.com/[0-9a-z_-]+" +
private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
"/sets/[0-9a-z_-]+/?([#?].*)?$";
public static SoundcloudPlaylistUrlIdHandler getInstance() {

14
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java
View File

@ -102,19 +102,19 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getUploaderUrl() {
return track.getObject("user").getString("permalink_url", "");
return SoundcloudExtractorHelper.getUploaderUrl(track);
}
@Nonnull
@Override
public String getUploaderName() {
return track.getObject("user").getString("username", "");
return SoundcloudExtractorHelper.getUploaderName(track);
}
@Nonnull
@Override
public String getUploaderAvatarUrl() {
return track.getObject("user", new JsonObject()).getString("avatar_url", "");
return SoundcloudExtractorHelper.getAvatarUrl(track);
}
@Override
@ -167,15 +167,15 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
}
@Override
@Nullable
@Nonnull
public List<Subtitles> getSubtitlesDefault() throws IOException, ExtractionException {
return null;
return Collections.emptyList();
}
@Override
@Nullable
@Nonnull
public List<Subtitles> getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException {
return null;
return Collections.emptyList();
}
@Override

3
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java
View File

@ -9,9 +9,8 @@ import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
private static final SoundcloudStreamUrlIdHandler instance = new SoundcloudStreamUrlIdHandler();
private final String URL_PATTERN = "^https?://(www\\.)?soundcloud.com/[0-9a-z_-]+" +
private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
"/(?!(tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?([#?].*)?$";
private SoundcloudStreamUrlIdHandler() {

19
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java
View File

@ -52,10 +52,18 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
*/
private Document nextStreamsAjax;
/**
* Unfortunately, we have to fetch the page even if we are only getting next streams,
* as they don't deliver enough information on their own (the channel name, for example).
* <br/>
* This help us to keep track on what are we fetching.
*/
private boolean fetchingNextStreams;
public YoutubeChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
super(service, url, nextStreamsUrl);
fetchingNextStreams = nextStreamsUrl != null && !nextStreamsUrl.isEmpty();
}
@Override
@ -70,14 +78,6 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
nextStreamsAjax = null;
}
@Override
protected boolean fetchPageUponCreation() {
// Unfortunately, we have to fetch the page even if we are getting only next streams,
// as they don't deliver enough information on their own (the channel name, for example).
fetchingNextStreams = nextStreamsUrl != null && !nextStreamsUrl.isEmpty();
return true;
}
@Nonnull
@Override
public String getCleanUrl() {
@ -176,7 +176,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
throw new ExtractionException("Channel doesn't have more streams");
}
fetchPage();
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
setupNextStreamsAjax(NewPipe.getDownloader());
collectStreamsFrom(collector, nextStreamsAjax.select("body").first());

2
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java
View File

@ -265,7 +265,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getUploaderUrl() throws ParsingException {
return getUploaderLink().attr("href");
return getUploaderLink().attr("abs:href");
}
@Override

144
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java
View File

@ -26,8 +26,6 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/*
* Created by Christian Schabesberger on 06.08.15.
@ -74,6 +72,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
public class SubtitlesException extends ContentNotAvailableException {
SubtitlesException(String message, Throwable cause) {
super(message, cause);
}
}
/*//////////////////////////////////////////////////////////////////////////*/
private Document doc;
@ -82,6 +86,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull
private final Map<String, String> videoInfoPage = new HashMap<>();
@Nonnull
private List<SubtitlesInfo> subtitlesInfos = new ArrayList<>();
private boolean isAgeRestricted;
public YoutubeStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
@ -419,54 +426,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
@Override
@Nullable
@Nonnull
public List<Subtitles> getSubtitlesDefault() throws IOException, ExtractionException {
return getSubtitles(SubtitlesFormat.TTML);
return getSubtitles(SubtitlesFormat.VTT);
}
@Override
@Nullable
public List<Subtitles> getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException {
@Nonnull
public List<Subtitles> getSubtitles(final SubtitlesFormat format) throws IOException, ExtractionException {
assertPageFetched();
if(isAgeRestricted) {
// If the video is age restricted getPlayerConfig will fail
return null;
List<Subtitles> subtitles = new ArrayList<>();
for (final SubtitlesInfo subtitlesInfo : subtitlesInfos) {
subtitles.add(subtitlesInfo.getSubtitle(format));
}
// TODO: This should be done in onFetchPage()
JsonObject playerConfig = getPlayerConfig(getPageHtml(NewPipe.getDownloader()));
String playerResponse = playerConfig.getObject("args").getString("player_response");
JsonObject captions;
try {
// Captions does not exist, return null
if (!JsonParser.object().from(playerResponse).has("captions")) return null;
captions = JsonParser.object().from(playerResponse).getObject("captions");
} catch (JsonParserException e) {
// Failed to parse subtitles
return null;
}
JsonArray captionsArray = captions.getObject("playerCaptionsTracklistRenderer").getArray("captionTracks");
int captionsSize = captionsArray.size();
// Should not happen, if there is the "captions" object, it should always has some captions in it
if(captionsSize == 0) return null;
List<Subtitles> result = new ArrayList<>();
for (int x = 0; x < captionsSize; x++) {
String baseUrl = captionsArray.getObject(x).getString("baseUrl");
String extension = format.getExtension();
String URL = baseUrl.replaceAll("&fmt=[^&]*", "&fmt=" + extension);
String captionsLangCode = captionsArray.getObject(x).getString("vssId");
boolean isAutoGenerated = captionsLangCode.startsWith("a.");
String languageCode = captionsLangCode.replaceFirst((isAutoGenerated) ? "a." : ".", "");
result.add(new Subtitles(format, languageCode, URL, isAutoGenerated));
}
return result;
return subtitles;
}
@Override
@ -483,7 +456,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
collector.commit(extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]")
.first().select("li").first()));
return ((StreamInfoItem) collector.getItemList().get(0));
return collector.getItemList().get(0);
} catch (Exception e) {
throw new ParsingException("Could not get next video", e);
}
@ -580,6 +553,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (decryptionCode.isEmpty()) {
decryptionCode = loadDecryptionCode(playerUrl);
}
if (subtitlesInfos.isEmpty()) {
subtitlesInfos.addAll(getAvailableSubtitlesInfo());
}
}
private JsonObject getPlayerConfig(String pageContent) throws ParsingException {
@ -732,6 +709,53 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return result == null ? "" : result.toString();
}
@Nonnull
private List<SubtitlesInfo> getAvailableSubtitlesInfo() throws SubtitlesException {
// If the video is age restricted getPlayerConfig will fail
if(isAgeRestricted) return Collections.emptyList();
final JsonObject playerConfig;
try {
playerConfig = getPlayerConfig(getPageHtml(NewPipe.getDownloader()));
} catch (IOException | ExtractionException e) {
throw new SubtitlesException("Unable to download player configs", e);
}
final String playerResponse = playerConfig.getObject("args").getString("player_response");
final JsonObject captions;
try {
if (!JsonParser.object().from(playerResponse).has("captions")) {
// Captions does not exist
return Collections.emptyList();
}
captions = JsonParser.object().from(playerResponse).getObject("captions");
} catch (JsonParserException e) {
// Failed to parse subtitles
throw new SubtitlesException("Unable to parse subtitles listing", e);
}
final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer");
final JsonArray captionsArray = renderer.getArray("captionTracks");
// todo: use this to apply auto translation to different language from a source language
final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages");
final int captionsSize = captionsArray.size();
// Should not happen, if there is the "captions" object, it should always has some captions in it
if(captionsSize == 0) return Collections.emptyList();
// Obtain the base url, this only needs to be done once
List<SubtitlesInfo> result = new ArrayList<>();
for (int i = 0; i < captionsSize; i++) {
final String languageCode = captionsArray.getObject(i).getString("languageCode");
final String baseUrl = captionsArray.getObject(i).getString("baseUrl");
final boolean isAutoGenerated = captionsArray.getObject(i).getString("vssId").startsWith("a.");
result.add(new SubtitlesInfo(baseUrl, languageCode, isAutoGenerated));
}
return result;
}
/*//////////////////////////////////////////////////////////////////////////
// Data Class
//////////////////////////////////////////////////////////////////////////*/
@ -746,17 +770,45 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
private class SubtitlesInfo {
final String cleanUrl;
final String languageCode;
final boolean isGenerated;
final Locale locale;
public SubtitlesInfo(final String baseUrl, final String languageCode, final boolean isGenerated) {
this.cleanUrl = baseUrl
.replaceAll("&fmt=[^&]*", "") // Remove preexisting format if exists
.replaceAll("&tlang=[^&]*", ""); // Remove translation language
this.languageCode = languageCode;
this.isGenerated = isGenerated;
final String[] splits = languageCode.split("-");
this.locale = splits.length == 2 ? new Locale(splits[0], splits[1]) : new Locale(languageCode);
}
public Subtitles getSubtitle(final SubtitlesFormat format) {
return new Subtitles(format, locale, cleanUrl + "&fmt=" + format.getExtension(), isGenerated);
}
}
/*//////////////////////////////////////////////////////////////////////////
// Utils
//////////////////////////////////////////////////////////////////////////*/
@Nonnull
private String getVideoInfoUrl(final String id, final String sts) {
private static String getVideoInfoUrl(final String id, final String sts) {
return "https://www.youtube.com/get_video_info?" + "video_id=" + id +
"&eurl=https://youtube.googleapis.com/v/" + id +
"&sts=" + sts + "&ps=default&gl=US&hl=en";
}
@Nonnull
private static String getSubtitleFormatUrl(final String baseUrl, final SubtitlesFormat format) {
return baseUrl.replaceAll("&fmt=[^&]*", "") + "&fmt=" + format.getExtension();
}
private Map<String, ItagItem> getItags(String encodedUrlMapKey, ItagItem.ItagType itagTypeWanted) throws ParsingException {
Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();

2
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java
View File

@ -100,7 +100,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
try {
return item.select("div[class=\"yt-lockup-byline\"]").first()
.select("a").first()
.attr("href");
.attr("abs:href");
} catch (Exception e){}
// try this if the first didn't work

135
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java
View File

@ -23,6 +23,7 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -37,7 +38,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor {
private Document doc;
public YoutubeTrendingExtractor(StreamingService service, String url, String nextStreamsUrl, String kioskId)
throws IOException, ExtractionException {
throws IOException, ExtractionException {
super(service, url, nextStreamsUrl, kioskId);
}
@ -81,76 +82,78 @@ public class YoutubeTrendingExtractor extends KioskExtractor {
@Override
public StreamInfoItemCollector getStreams() throws ParsingException {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
Element ul = doc.select("ul[class*=\"expanded-shelf-content-list\"]").first();
for(final Element li : ul.children()) {
final Element el = li.select("div[class*=\"yt-lockup-dismissable\"]").first();
collector.commit(new YoutubeStreamInfoItemExtractor(li) {
@Override
public String getUrl() throws ParsingException {
try {
Element dl = el.select("h3").first().select("a").first();
return dl.attr("abs:href");
} catch (Exception e) {
throw new ParsingException("Could not get web page url for the video", e);
}
}
@Override
public String getName() throws ParsingException {
try {
Element dl = el.select("h3").first().select("a").first();
return dl.text();
} catch (Exception e) {
throw new ParsingException("Could not get web page url for the video", e);
}
}
@Override
public String getUploaderUrl() throws ParsingException {
try {
String link = getUploaderLink().attr("href");
if(link.isEmpty()) {
throw new IllegalArgumentException("is empty");
Elements uls = doc.select("ul[class*=\"expanded-shelf-content-list\"]");
for(Element ul : uls) {
for(final Element li : ul.children()) {
final Element el = li.select("div[class*=\"yt-lockup-dismissable\"]").first();
collector.commit(new YoutubeStreamInfoItemExtractor(li) {
@Override
public String getUrl() throws ParsingException {
try {
Element dl = el.select("h3").first().select("a").first();
return dl.attr("abs:href");
} catch (Exception e) {
throw new ParsingException("Could not get web page url for the video", e);
}
return link;
} catch (Exception e) {
throw new ParsingException("Could not get Uploader name");
}
}
private Element getUploaderLink() {
Element uploaderEl = el.select("div[class*=\"yt-lockup-byline \"]").first();
return uploaderEl.select("a").first();
}
@Override
public String getUploaderName() throws ParsingException {
try {
return getUploaderLink().text();
} catch (Exception e) {
throw new ParsingException("Could not get Uploader name");
}
}
@Override
public String getThumbnailUrl() throws ParsingException {
try {
String url;
Element te = li.select("span[class=\"yt-thumb-simple\"]").first()
.select("img").first();
url = te.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we've caught such an item.
if (url.contains(".gif")) {
url = te.attr("abs:data-thumb");
@Override
public String getName() throws ParsingException {
try {
Element dl = el.select("h3").first().select("a").first();
return dl.text();
} catch (Exception e) {
throw new ParsingException("Could not get web page url for the video", e);
}
return url;
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
}
});
@Override
public String getUploaderUrl() throws ParsingException {
try {
String link = getUploaderLink().attr("abs:href");
if (link.isEmpty()) {
throw new IllegalArgumentException("is empty");
}
return link;
} catch (Exception e) {
throw new ParsingException("Could not get Uploader name");
}
}
private Element getUploaderLink() {
Element uploaderEl = el.select("div[class*=\"yt-lockup-byline \"]").first();
return uploaderEl.select("a").first();
}
@Override
public String getUploaderName() throws ParsingException {
try {
return getUploaderLink().text();
} catch (Exception e) {
throw new ParsingException("Could not get Uploader name");
}
}
@Override
public String getThumbnailUrl() throws ParsingException {
try {
String url;
Element te = li.select("span[class=\"yt-thumb-simple\"]").first()
.select("img").first();
url = te.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we've caught such an item.
if (url.contains(".gif")) {
url = te.attr("abs:data-thumb");
}
return url;
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
}
});
}
}
return collector;

5
src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java
View File

@ -132,10 +132,9 @@ public abstract class StreamExtractor extends Extractor {
public abstract List<VideoStream> getVideoStreams() throws IOException, ExtractionException;
public abstract List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException;
@Nullable
@Nonnull
public abstract List<Subtitles> getSubtitlesDefault() throws IOException, ExtractionException;
@Nullable
@Nonnull
public abstract List<Subtitles> getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException;
public abstract StreamType getStreamType() throws ParsingException;

16
src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java
View File

@ -138,6 +138,10 @@ public class StreamInfo extends Info {
return start_position;
}
public List<Subtitles> getSubtitles() {
return subtitles;
}
public void setStreamType(StreamType stream_type) {
this.stream_type = stream_type;
}
@ -214,6 +218,10 @@ public class StreamInfo extends Info {
this.start_position = start_position;
}
public void setSubtitles(List<Subtitles> subtitles) {
this.subtitles = subtitles;
}
public static class StreamExtractException extends ExtractionException {
StreamExtractException(String message) {
super(message);
@ -313,6 +321,12 @@ public class StreamInfo extends Info {
streamInfo.addError(new ExtractionException("Couldn't get video only streams", e));
}
try {
streamInfo.setSubtitles(extractor.getSubtitlesDefault());
} catch (Exception e) {
streamInfo.addError(new ExtractionException("Couldn't get subtitles", e));
}
// Lists can be null if a exception was thrown during extraction
if (streamInfo.getVideoStreams() == null) streamInfo.setVideoStreams(Collections.<VideoStream>emptyList());
if (streamInfo.getVideoOnlyStreams()== null) streamInfo.setVideoOnlyStreams(Collections.<VideoStream>emptyList());
@ -444,4 +458,6 @@ public class StreamInfo extends Info {
public List<InfoItem> related_streams;
//in seconds. some metadata is not passed using a StreamInfo object!
public long start_position = 0;
public List<Subtitles> subtitles;
}

17
src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java
View File

@ -1,9 +1,12 @@
package org.schabi.newpipe.extractor;
import javax.annotation.Nonnull;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.List;
import static org.junit.Assert.assertEquals;
public class ExtractorAsserts {
public static void assertEmptyErrors(String message, List<Throwable> errors) {
if(!errors.isEmpty()) {
@ -14,11 +17,21 @@ public class ExtractorAsserts {
}
}
public static void assertIsValidUrl(String url) {
@Nonnull
private static URL urlFromString(String url) {
try {
new URL(url);
return new URL(url);
} catch (MalformedURLException e) {
throw new AssertionError("Invalid url: " + url, e);
}
}
public static void assertIsValidUrl(String url) {
urlFromString(url);
}
public static void assertIsSecureUrl(String urlToCheck) {
URL url = urlFromString(urlToCheck);
assertEquals("Protocol of URL is not secure", "https", url.getProtocol());
}
}

57
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/BaseSoundcloudSearchTest.java Normal file
View File

@ -0,0 +1,57 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.Test;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
import org.schabi.newpipe.extractor.search.SearchResult;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamType;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
public abstract class BaseSoundcloudSearchTest {
protected static SearchResult result;
@Test
public void testResultList() {
assertFalse("Got empty result list", result.resultList.isEmpty());
for(InfoItem infoItem: result.resultList) {
assertIsSecureUrl(infoItem.getUrl());
assertIsSecureUrl(infoItem.getThumbnailUrl());
assertFalse(infoItem.getName().isEmpty());
assertFalse("Name is probably a URI: " + infoItem.getName(),
infoItem.getName().contains("://"));
if(infoItem instanceof StreamInfoItem) {
// test stream item
StreamInfoItem streamInfoItem = (StreamInfoItem) infoItem;
assertIsSecureUrl(streamInfoItem.getUploaderUrl());
assertFalse(streamInfoItem.getUploadDate().isEmpty());
assertFalse(streamInfoItem.getUploaderName().isEmpty());
assertEquals(StreamType.AUDIO_STREAM, streamInfoItem.getStreamType());
} else if(infoItem instanceof ChannelInfoItem) {
// Nothing special to check?
} else if(infoItem instanceof PlaylistInfoItem) {
// test playlist item
assertTrue(infoItem.getUrl().contains("/sets/"));
long streamCount = ((PlaylistInfoItem) infoItem).getStreamCount();
assertTrue(streamCount > 0);
} else {
fail("Unknown infoItem type: " + infoItem);
}
}
}
@Test
public void testResultErrors() {
assertNotNull(result.errors);
if (!result.errors.isEmpty()) {
for (Throwable error : result.errors) {
error.printStackTrace();
}
}
assertTrue(result.errors.isEmpty());
}
}

4
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java
View File

@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
/**
@ -23,6 +24,7 @@ public class SoundcloudChannelExtractorTest {
NewPipe.init(Downloader.getInstance());
extractor = SoundCloud.getService()
.getChannelExtractor("https://soundcloud.com/liluzivert");
extractor.fetchPage();
}
@Test
@ -42,7 +44,7 @@ public class SoundcloudChannelExtractorTest {
@Test
public void testGetAvatarUrl() throws Exception {
assertTrue(extractor.getAvatarUrl().contains("https://"));
assertIsSecureUrl(extractor.getAvatarUrl());
}
@Test

9
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java
View File

@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
/**
@ -22,6 +23,7 @@ public class SoundcloudPlaylistExtractorTest {
NewPipe.init(Downloader.getInstance());
extractor = SoundCloud.getService()
.getPlaylistExtractor("https://soundcloud.com/liluzivert/sets/the-perfect-luv-tape-r");
extractor.fetchPage();
}
@Test
@ -41,12 +43,13 @@ public class SoundcloudPlaylistExtractorTest {
@Test
public void testGetThumbnailUrl() throws Exception {
assertTrue(extractor.getThumbnailUrl(), extractor.getThumbnailUrl().contains("https://"));
assertIsSecureUrl(extractor.getThumbnailUrl());
}
@Test
public void testGetUploaderUrl() throws Exception {
assertEquals(extractor.getUploaderUrl(), "http://soundcloud.com/liluzivert");
assertIsSecureUrl(extractor.getUploaderUrl());
assertEquals(extractor.getUploaderUrl(), "https://soundcloud.com/liluzivert");
}
@Test
@ -56,7 +59,7 @@ public class SoundcloudPlaylistExtractorTest {
@Test
public void testGetUploaderAvatarUrl() throws Exception {
assertTrue(extractor.getUploaderAvatarUrl(), extractor.getUploaderAvatarUrl().contains("https://"));
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
}
@Test

17
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineAllTest.java
View File

@ -6,9 +6,7 @@ import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchResult;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
@ -16,8 +14,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
/**
* Test for {@link SearchEngine}
*/
public class SoundcloudSearchEngineAllTest {
private static SearchResult result;
public class SoundcloudSearchEngineAllTest extends BaseSoundcloudSearchTest {
@BeforeClass
public static void setUp() throws Exception {
@ -30,18 +27,6 @@ public class SoundcloudSearchEngineAllTest {
.getSearchResult();
}
@Test
public void testResultList() {
assertFalse(result.resultList.isEmpty());
}
@Test
public void testResultErrors() {
assertNotNull(result.errors);
if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace();
assertTrue(result.errors.isEmpty());
}
@Ignore
@Test
public void testSuggestion() {

15
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineChannelTest.java
View File

@ -15,8 +15,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
/**
* Test for {@link SearchEngine}
*/
public class SoundcloudSearchEngineChannelTest {
private static SearchResult result;
public class SoundcloudSearchEngineChannelTest extends BaseSoundcloudSearchTest {
@BeforeClass
public static void setUp() throws Exception {
@ -29,11 +28,6 @@ public class SoundcloudSearchEngineChannelTest {
.getSearchResult();
}
@Test
public void testResultList() {
assertFalse(result.resultList.isEmpty());
}
@Test
public void testResultsItemType() {
for (InfoItem infoItem : result.resultList) {
@ -41,13 +35,6 @@ public class SoundcloudSearchEngineChannelTest {
}
}
@Test
public void testResultErrors() {
assertNotNull(result.errors);
if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace();
assertTrue(result.errors.isEmpty());
}
@Ignore
@Test
public void testSuggestion() {

15
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEnginePlaylistTest.java
View File

@ -36,8 +36,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
/**
* Test for {@link SearchEngine}
*/
public class SoundcloudSearchEnginePlaylistTest {
private static SearchResult result;
public class SoundcloudSearchEnginePlaylistTest extends BaseSoundcloudSearchTest {
@BeforeClass
public static void setUp() throws Exception {
@ -49,11 +48,6 @@ public class SoundcloudSearchEnginePlaylistTest {
.getSearchResult();
}
@Test
public void testResultList() {
assertFalse(result.resultList.isEmpty());
}
@Test
public void testUserItemType() {
for (InfoItem infoItem : result.resultList) {
@ -61,13 +55,6 @@ public class SoundcloudSearchEnginePlaylistTest {
}
}
@Test
public void testResultErrors() {
assertNotNull(result.errors);
if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace();
assertTrue(result.errors.isEmpty());
}
@Ignore
@Test
public void testSuggestion() {

17
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineStreamTest.java
View File

@ -15,25 +15,19 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
/**
* Test for {@link SearchEngine}
*/
public class SoundcloudSearchEngineStreamTest {
private static SearchResult result;
public class SoundcloudSearchEngineStreamTest extends BaseSoundcloudSearchTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
SearchEngine engine = SoundCloud.getService().getSearchEngine();
// SoundCloud will suggest "lil uzi vert" instead of "lil uzi vert",
// SoundCloud will suggest "lil uzi vert" instead of "lill uzi vert",
// keep in mind that the suggestions can NOT change by country (the parameter "de")
result = engine.search("lill uzi vert", 0, "de", SearchEngine.Filter.STREAM)
.getSearchResult();
}
@Test
public void testResultList() {
assertFalse(result.resultList.isEmpty());
}
@Test
public void testResultsItemType() {
for (InfoItem infoItem : result.resultList) {
@ -41,13 +35,6 @@ public class SoundcloudSearchEngineStreamTest {
}
}
@Test
public void testResultErrors() {
assertNotNull(result.errors);
if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace();
assertTrue(result.errors.isEmpty());
}
@Ignore
@Test
public void testSuggestion() {

14
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java
View File

@ -14,6 +14,7 @@ import org.schabi.newpipe.extractor.stream.SubtitlesFormat;
import java.io.IOException;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
/**
@ -69,22 +70,23 @@ public class SoundcloudStreamExtractorDefaultTest {
@Test
public void testGetUploadDate() throws ParsingException {
assertEquals(extractor.getUploadDate(), "2016-07-31");
assertEquals("2016-07-31", extractor.getUploadDate());
}
@Test
public void testGetUploaderUrl() throws ParsingException {
assertEquals(extractor.getUploaderUrl(), "http://soundcloud.com/liluzivert");
assertIsSecureUrl(extractor.getUploaderUrl());
assertEquals("https://soundcloud.com/liluzivert", extractor.getUploaderUrl());
}
@Test
public void testGetThumbnailUrl() throws ParsingException {
assertTrue(extractor.getThumbnailUrl(), extractor.getThumbnailUrl().contains("https://"));
assertIsSecureUrl(extractor.getThumbnailUrl());
}
@Test
public void testGetUploaderAvatarUrl() throws ParsingException {
assertTrue(extractor.getUploaderAvatarUrl(), extractor.getUploaderAvatarUrl().contains("https://"));
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
}
@Test
@ -107,12 +109,12 @@ public class SoundcloudStreamExtractorDefaultTest {
@Test
public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertTrue(extractor.getSubtitlesDefault() == null);
assertTrue(extractor.getSubtitlesDefault().isEmpty());
}
@Test
public void testGetSubtitlesList() throws IOException, ExtractionException {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertTrue(extractor.getSubtitles(SubtitlesFormat.VTT) == null);
assertTrue(extractor.getSubtitlesDefault().isEmpty());
}
}

55
src/test/java/org/schabi/newpipe/extractor/services/youtube/BaseYoutubeSearchTest.java Normal file
View File

@ -0,0 +1,55 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Test;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
import org.schabi.newpipe.extractor.search.SearchResult;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import static org.junit.Assert.*;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
public abstract class BaseYoutubeSearchTest {
protected static SearchResult result;
@Test
public void testResultList() {
assertFalse("Got empty result list", result.resultList.isEmpty());
for(InfoItem infoItem: result.resultList) {
assertIsSecureUrl(infoItem.getUrl());
assertIsSecureUrl(infoItem.getThumbnailUrl());
assertFalse(infoItem.getName().isEmpty());
assertFalse("Name is probably a URI: " + infoItem.getName(),
infoItem.getName().contains("://"));
if(infoItem instanceof StreamInfoItem) {
// test stream item
StreamInfoItem streamInfoItem = (StreamInfoItem) infoItem;
assertIsSecureUrl(streamInfoItem.getUploaderUrl());
assertFalse(streamInfoItem.getUploadDate().isEmpty());
assertFalse(streamInfoItem.getUploaderName().isEmpty());
} else if(infoItem instanceof ChannelInfoItem) {
// Nothing special to check?
} else if(infoItem instanceof PlaylistInfoItem) {
// test playlist item
long streamCount = ((PlaylistInfoItem) infoItem).getStreamCount();
assertTrue(streamCount > 0);
} else {
fail("Unknown infoItem type: " + infoItem);
}
}
}
@Test
public void testResultErrors() {
assertNotNull(result.errors);
if (!result.errors.isEmpty()) {
for (Throwable error : result.errors) {
error.printStackTrace();
}
}
assertTrue(result.errors.isEmpty());
}
}

1
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java
View File

@ -44,6 +44,7 @@ public class YoutubeChannelExtractorTest {
NewPipe.init(Downloader.getInstance());
extractor = (YoutubeChannelExtractor) YouTube.getService()
.getChannelExtractor("https://www.youtube.com/user/Gronkh");
extractor.fetchPage();
}
@Test

11
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View File

@ -1,6 +1,7 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.ListExtractor;
@ -28,7 +29,8 @@ public class YoutubePlaylistExtractorTest {
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
extractor = (YoutubePlaylistExtractor) YouTube.getService()
.getPlaylistExtractor("https://www.youtube.com/playlist?list=PL7XlqX4npddfrdpMCxBnNZXg2GFll7t5y");
.getPlaylistExtractor("https://www.youtube.com/watch?v=lp-EO5I60KA&list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj");
extractor.fetchPage();
}
@Test
@ -38,12 +40,12 @@ public class YoutubePlaylistExtractorTest {
@Test
public void testGetId() throws Exception {
assertEquals(extractor.getId(), "PL7XlqX4npddfrdpMCxBnNZXg2GFll7t5y");
assertEquals(extractor.getId(), "PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj");
}
@Test
public void testGetName() throws Exception {
assertEquals(extractor.getName(), "important videos");
assertEquals(extractor.getName(), "Pop Music Playlist: Timeless Pop Hits (Updated Weekly 2018)");
}
@Test
@ -103,7 +105,8 @@ public class YoutubePlaylistExtractorTest {
assertTrue("extractor didn't have more streams", extractor.hasMoreStreams());
}
@Test
@Test @Ignore
public void testGetNextStreams() throws Exception {
// Setup the streams
extractor.getStreams();

23
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java
View File

@ -40,8 +40,7 @@ import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsValidUrl;
/**
* Test for {@link SearchEngine}
*/
public class YoutubeSearchEngineAllTest {
private static SearchResult result;
public class YoutubeSearchEngineAllTest extends BaseYoutubeSearchTest {
@BeforeClass
public static void setUpClass() throws Exception {
@ -53,29 +52,13 @@ public class YoutubeSearchEngineAllTest {
}
@Test
public void testResultList() {
final List<InfoItem> results = result.getResults();
assertFalse("Results are empty: " + results, results.isEmpty());
InfoItem firstInfoItem = results.get(0);
public void testResultList_FirstElement() {
InfoItem firstInfoItem = result.getResults().get(0);
// THe channel should be the first item
assertTrue(firstInfoItem instanceof ChannelInfoItem);
assertEquals("name", "PewDiePie", firstInfoItem.name);
assertEquals("url","https://www.youtube.com/user/PewDiePie", firstInfoItem.url);
for(InfoItem item: results) {
assertIsValidUrl(item.url);
}
}
@Test
public void testResultErrors() {
for (Throwable error : result.getErrors()) {
error.printStackTrace();
}
assertTrue(result.getErrors().isEmpty());
}
@Ignore

18
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineChannelTest.java
View File

@ -37,8 +37,7 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
/**
* Test for {@link SearchEngine}
*/
public class YoutubeSearchEngineChannelTest {
private static SearchResult result;
public class YoutubeSearchEngineChannelTest extends BaseYoutubeSearchTest {
@BeforeClass
public static void setUp() throws Exception {
@ -51,14 +50,6 @@ public class YoutubeSearchEngineChannelTest {
.getSearchResult();
}
@Test
public void testResultList() {
assertFalse(result.resultList.isEmpty());
for(InfoItem item: result.getResults()) {
assertIsValidUrl(item.url);
}
}
@Test
public void testResultsItemType() {
for (InfoItem infoItem : result.resultList) {
@ -66,13 +57,6 @@ public class YoutubeSearchEngineChannelTest {
}
}
@Test
public void testResultErrors() {
assertNotNull(result.errors);
if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace();
assertTrue(result.errors.isEmpty());
}
@Ignore
@Test
public void testSuggestion() {

22
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEnginePlaylistTest.java
View File

@ -6,6 +6,7 @@ import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchResult;
@ -37,8 +38,7 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
/**
* Test for {@link SearchEngine}
*/
public class YoutubeSearchEnginePlaylistTest {
private static SearchResult result;
public class YoutubeSearchEnginePlaylistTest extends BaseYoutubeSearchTest {
@BeforeClass
public static void setUp() throws Exception {
@ -52,27 +52,13 @@ public class YoutubeSearchEnginePlaylistTest {
}
@Test
public void testResultList() {
assertFalse(result.resultList.isEmpty());
for(InfoItem item: result.getResults()) {
assertIsValidUrl(item.url);
}
}
@Test
public void testUserItemType() {
public void testInfoItemType() {
for (InfoItem infoItem : result.resultList) {
assertTrue(infoItem instanceof PlaylistInfoItem);
assertEquals(InfoItem.InfoType.PLAYLIST, infoItem.info_type);
}
}
@Test
public void testResultErrors() {
assertNotNull(result.errors);
if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace();
assertTrue(result.errors.isEmpty());
}
@Ignore
@Test
public void testSuggestion() {

18
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineStreamTest.java
View File

@ -37,8 +37,7 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
/**
* Test for {@link SearchEngine}
*/
public class YoutubeSearchEngineStreamTest {
private static SearchResult result;
public class YoutubeSearchEngineStreamTest extends BaseYoutubeSearchTest {
@BeforeClass
public static void setUp() throws Exception {
@ -51,14 +50,6 @@ public class YoutubeSearchEngineStreamTest {
.getSearchResult();
}
@Test
public void testResultList() {
assertFalse(result.resultList.isEmpty());
for(InfoItem item: result.getResults()) {
assertIsValidUrl(item.url);
}
}
@Test
public void testResultsItemType() {
for (InfoItem infoItem : result.resultList) {
@ -66,13 +57,6 @@ public class YoutubeSearchEngineStreamTest {
}
}
@Test
public void testResultErrors() {
assertNotNull(result.errors);
if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace();
assertTrue(result.errors.isEmpty());
}
@Ignore
@Test
public void testSuggestion() {

15
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java
View File

@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
/*
@ -38,7 +39,6 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
* Test for {@link StreamExtractor}
*/
public class YoutubeStreamExtractorDefaultTest {
public static final String HTTPS = "https://";
private static YoutubeStreamExtractor extractor;
@BeforeClass
@ -102,14 +102,12 @@ public class YoutubeStreamExtractorDefaultTest {
@Test
public void testGetThumbnailUrl() throws ParsingException {
assertTrue(extractor.getThumbnailUrl(),
extractor.getThumbnailUrl().contains(HTTPS));
assertIsSecureUrl(extractor.getThumbnailUrl());
}
@Test
public void testGetUploaderAvatarUrl() throws ParsingException {
assertTrue(extractor.getUploaderAvatarUrl(),
extractor.getUploaderAvatarUrl().contains(HTTPS));
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
}
@Test
@ -120,8 +118,7 @@ public class YoutubeStreamExtractorDefaultTest {
@Test
public void testGetVideoStreams() throws IOException, ExtractionException {
for (VideoStream s : extractor.getVideoStreams()) {
assertTrue(s.url,
s.url.contains(HTTPS));
assertIsSecureUrl(s.url);
assertTrue(s.resolution.length() > 0);
assertTrue(Integer.toString(s.getFormatId()),
0 <= s.getFormatId() && s.getFormatId() <= 4);
@ -150,12 +147,12 @@ public class YoutubeStreamExtractorDefaultTest {
@Test
public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertTrue(extractor.getSubtitlesDefault() == null);
assertTrue(extractor.getSubtitlesDefault().isEmpty());
}
@Test
public void testGetSubtitlesList() throws IOException, ExtractionException {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertTrue(extractor.getSubtitles(SubtitlesFormat.VTT) == null);
assertTrue(extractor.getSubtitles(SubtitlesFormat.TTML).isEmpty());
}
}

11
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java
View File

@ -16,6 +16,7 @@ import java.util.ArrayList;
import java.util.List;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
/**
@ -85,14 +86,12 @@ public class YoutubeStreamExtractorRestrictedTest {
@Test
public void testGetThumbnailUrl() throws ParsingException {
assertTrue(extractor.getThumbnailUrl(),
extractor.getThumbnailUrl().contains(HTTPS));
assertIsSecureUrl(extractor.getThumbnailUrl());
}
@Test
public void testGetUploaderAvatarUrl() throws ParsingException {
assertTrue(extractor.getUploaderAvatarUrl(),
extractor.getUploaderAvatarUrl().contains(HTTPS));
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
}
// FIXME: 25.11.17 Are there no streams or are they not listed?
@ -123,12 +122,12 @@ public class YoutubeStreamExtractorRestrictedTest {
@Test
public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertNull(extractor.getSubtitlesDefault());
assertTrue(extractor.getSubtitlesDefault().isEmpty());
}
@Test
public void testGetSubtitlesList() throws IOException, ExtractionException {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertNull(extractor.getSubtitles(SubtitlesFormat.VTT));
assertTrue(extractor.getSubtitles(SubtitlesFormat.TTML).isEmpty());
}
}

5
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java
View File

@ -48,6 +48,7 @@ public class YoutubeTrendingExtractorTest {
extractor = (YoutubeTrendingExtractor) YouTube.getService()
.getKioskList()
.getExtractorById("Trending", null);
extractor.fetchPage();
}
@Test
@ -66,10 +67,10 @@ public class YoutubeTrendingExtractorTest {
}
@Test
public void testGetStreams() throws Exception {
public void testGetStreamsQuantity() throws Exception {
StreamInfoItemCollector collector = extractor.getStreams();
Utils.printErrors(collector);
assertFalse("no streams are received", collector.getItemList().isEmpty());
assertTrue("no streams are received", collector.getItemList().size() >= 20);
}
@Test