Merge branch 'dev' of https://github.com/TeamNewPipe/NewPipeExtractor into feature/frames
This commit is contained in:
Vasiliy
commit
d8279f91f6
|
|
@ -49,7 +49,7 @@ import java.util.ArrayList;
|
|||
public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
/*package-private*/ static final String CHANNEL_URL_BASE = "https://www.youtube.com/channel/";
|
||||
private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id=";
|
||||
private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000";
|
||||
private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000&gl=US&hl=en";
|
||||
|
||||
private Document doc;
|
||||
|
||||
|
|
@ -82,6 +82,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
@Nonnull
|
||||
@Override
|
||||
public String getId() throws ParsingException {
|
||||
try {
|
||||
return doc.select("meta[itemprop=\"channelId\"]").first().attr("content");
|
||||
} catch (Exception ignored) {}
|
||||
|
||||
// fallback method; does not work with channels that have no "Subscribe" button (e.g. EminemVEVO)
|
||||
try {
|
||||
Element element = doc.getElementsByClass("yt-uix-subscription-button").first();
|
||||
if (element == null) element = doc.getElementsByClass("yt-uix-subscription-preferences-button").first();
|
||||
|
|
@ -135,10 +140,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
|
||||
@Override
|
||||
public long getSubscriberCount() throws ParsingException {
|
||||
|
||||
final Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first();
|
||||
if (el != null) {
|
||||
String elTitle = el.attr("title");
|
||||
try {
|
||||
return Long.parseLong(Utils.removeNonDigitCharacters(el.text()));
|
||||
return Utils.mixedNumberWordToLong(elTitle);
|
||||
} catch (NumberFormatException e) {
|
||||
throw new ParsingException("Could not get subscriber count", e);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -56,19 +56,25 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
|
|||
|
||||
@Override
|
||||
public String getUrl() throws ParsingException {
|
||||
String buttonTrackingUrl = el.select("button[class*=\"yt-uix-button\"]").first()
|
||||
.attr("abs:data-href");
|
||||
try {
|
||||
String buttonTrackingUrl = el.select("button[class*=\"yt-uix-button\"]").first()
|
||||
.attr("abs:data-href");
|
||||
|
||||
Pattern channelIdPattern = Pattern.compile("(?:.*?)\\%252Fchannel\\%252F([A-Za-z0-9\\-\\_]+)(?:.*)");
|
||||
Matcher match = channelIdPattern.matcher(buttonTrackingUrl);
|
||||
Pattern channelIdPattern = Pattern.compile("(?:.*?)\\%252Fchannel\\%252F([A-Za-z0-9\\-\\_]+)(?:.*)");
|
||||
Matcher match = channelIdPattern.matcher(buttonTrackingUrl);
|
||||
|
||||
if (match.matches()) {
|
||||
return YoutubeChannelExtractor.CHANNEL_URL_BASE + match.group(1);
|
||||
} else {
|
||||
// fallback method just in case youtube changes things; it should never run and tests will fail
|
||||
// provides an url with "/user/NAME", that is inconsistent with stream and channel extractor
|
||||
if (match.matches()) {
|
||||
return YoutubeChannelExtractor.CHANNEL_URL_BASE + match.group(1);
|
||||
}
|
||||
} catch(Exception ignored) {}
|
||||
|
||||
// fallback method for channels without "Subscribe" button (or just in case yt changes things)
|
||||
// provides an url with "/user/NAME", inconsistent with stream and channel extractor: tests will fail
|
||||
try {
|
||||
return el.select("a[class*=\"yt-uix-tile-link\"]").first()
|
||||
.attr("abs:href");
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get channel url", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
private JsonObject playerArgs;
|
||||
@Nonnull
|
||||
private final Map<String, String> videoInfoPage = new HashMap<>();
|
||||
private JsonObject playerResponse;
|
||||
|
||||
@Nonnull
|
||||
private List<SubtitlesInfo> subtitlesInfos = new ArrayList<>();
|
||||
|
|
@ -253,20 +254,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
public long getLength() throws ParsingException {
|
||||
assertPageFetched();
|
||||
|
||||
final JsonObject playerResponse;
|
||||
try {
|
||||
final String pr;
|
||||
if(playerArgs != null) {
|
||||
pr = playerArgs.getString("player_response");
|
||||
} else {
|
||||
pr = videoInfoPage.get("player_response");
|
||||
}
|
||||
playerResponse = JsonParser.object()
|
||||
.from(pr);
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get playerResponse", e);
|
||||
}
|
||||
|
||||
// try getting duration from playerargs
|
||||
try {
|
||||
String durationMs = playerResponse
|
||||
|
|
@ -442,31 +429,24 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
@Override
|
||||
public String getHlsUrl() throws ParsingException {
|
||||
assertPageFetched();
|
||||
try {
|
||||
String hlsvp = "";
|
||||
if (playerArgs != null) {
|
||||
if( playerArgs.isString("hlsvp") ) {
|
||||
hlsvp = playerArgs.getString("hlsvp", "");
|
||||
}else {
|
||||
hlsvp = JsonParser.object()
|
||||
.from(playerArgs.getString("player_response", "{}"))
|
||||
.getObject("streamingData", new JsonObject())
|
||||
.getString("hlsManifestUrl", "");
|
||||
}
|
||||
}
|
||||
|
||||
return hlsvp;
|
||||
try {
|
||||
return playerResponse.getObject("streamingData").getString("hlsManifestUrl");
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get hls manifest url", e);
|
||||
if (playerArgs != null && playerArgs.isString("hlsvp")) {
|
||||
return playerArgs.getString("hlsvp");
|
||||
} else {
|
||||
throw new ParsingException("Could not get hls manifest url", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
|
||||
public List<AudioStream> getAudioStreams() throws ExtractionException {
|
||||
assertPageFetched();
|
||||
List<AudioStream> audioStreams = new ArrayList<>();
|
||||
try {
|
||||
for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.AUDIO).entrySet()) {
|
||||
for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO).entrySet()) {
|
||||
ItagItem itag = entry.getValue();
|
||||
|
||||
AudioStream audioStream = new AudioStream(entry.getKey(), itag.getMediaFormat(), itag.avgBitrate);
|
||||
|
|
@ -482,11 +462,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
|
||||
public List<VideoStream> getVideoStreams() throws ExtractionException {
|
||||
assertPageFetched();
|
||||
List<VideoStream> videoStreams = new ArrayList<>();
|
||||
try {
|
||||
for (Map.Entry<String, ItagItem> entry : getItags(URL_ENCODED_FMT_STREAM_MAP, ItagItem.ItagType.VIDEO).entrySet()) {
|
||||
for (Map.Entry<String, ItagItem> entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) {
|
||||
ItagItem itag = entry.getValue();
|
||||
|
||||
VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString);
|
||||
|
|
@ -506,7 +486,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
assertPageFetched();
|
||||
List<VideoStream> videoOnlyStreams = new ArrayList<>();
|
||||
try {
|
||||
for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) {
|
||||
for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) {
|
||||
ItagItem itag = entry.getValue();
|
||||
|
||||
VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString, true);
|
||||
|
|
@ -543,7 +523,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
assertPageFetched();
|
||||
try {
|
||||
if (playerArgs != null && (playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live") ||
|
||||
playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
|
||||
(!playerResponse.getObject("streamingData").has(FORMATS)))) {
|
||||
return StreamType.LIVE_STREAM;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
|
|
@ -595,21 +575,26 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
*/
|
||||
@Override
|
||||
public String getErrorMessage() {
|
||||
String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text();
|
||||
StringBuilder errorReason;
|
||||
Element errorElement = doc.select("h1[id=\"unavailable-message\"]").first();
|
||||
|
||||
if (errorMessage == null || errorMessage.isEmpty()) {
|
||||
if (errorElement == null) {
|
||||
errorReason = null;
|
||||
} else if (errorMessage.contains("GEMA")) {
|
||||
// Gema sometimes blocks youtube music content in germany:
|
||||
// https://www.gema.de/en/
|
||||
// Detailed description:
|
||||
// https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
|
||||
errorReason = new StringBuilder("GEMA");
|
||||
} else {
|
||||
errorReason = new StringBuilder(errorMessage);
|
||||
errorReason.append(" ");
|
||||
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
|
||||
String errorMessage = errorElement.text();
|
||||
if (errorMessage == null || errorMessage.isEmpty()) {
|
||||
errorReason = null;
|
||||
} else if (errorMessage.contains("GEMA")) {
|
||||
// Gema sometimes blocks youtube music content in germany:
|
||||
// https://www.gema.de/en/
|
||||
// Detailed description:
|
||||
// https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
|
||||
errorReason = new StringBuilder("GEMA");
|
||||
} else {
|
||||
errorReason = new StringBuilder(errorMessage);
|
||||
errorReason.append(" ");
|
||||
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
|
||||
}
|
||||
}
|
||||
|
||||
return errorReason != null ? errorReason.toString() : null;
|
||||
|
|
@ -619,8 +604,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
// Fetch page
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
private static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map";
|
||||
private static final String ADAPTIVE_FMTS = "adaptive_fmts";
|
||||
private static final String FORMATS = "formats";
|
||||
private static final String ADAPTIVE_FORMATS = "adaptiveFormats";
|
||||
private static final String HTTPS = "https:";
|
||||
private static final String CONTENT = "content";
|
||||
private static final String DECRYPTION_FUNC_NAME = "decrypt";
|
||||
|
|
@ -667,6 +652,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
playerUrl = getPlayerUrl(ytPlayerConfig);
|
||||
isAgeRestricted = false;
|
||||
}
|
||||
playerResponse = getPlayerResponse();
|
||||
|
||||
if (decryptionCode.isEmpty()) {
|
||||
decryptionCode = loadDecryptionCode(playerUrl);
|
||||
|
|
@ -728,6 +714,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
}
|
||||
}
|
||||
|
||||
private JsonObject getPlayerResponse() throws ParsingException {
|
||||
try {
|
||||
String playerResponseStr;
|
||||
if(playerArgs != null) {
|
||||
playerResponseStr = playerArgs.getString("player_response");
|
||||
} else {
|
||||
playerResponseStr = videoInfoPage.get("player_response");
|
||||
}
|
||||
return JsonParser.object().from(playerResponseStr);
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not parse yt player response", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private EmbeddedInfo getEmbeddedInfo() throws ParsingException, ReCaptchaException {
|
||||
try {
|
||||
|
|
@ -843,19 +843,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
} catch (IOException | ExtractionException e) {
|
||||
throw new SubtitlesException("Unable to download player configs", e);
|
||||
}
|
||||
final String playerResponse = playerConfig.getObject("args", new JsonObject())
|
||||
.getString("player_response");
|
||||
|
||||
final JsonObject captions;
|
||||
try {
|
||||
if (playerResponse == null || !JsonParser.object().from(playerResponse).has("captions")) {
|
||||
// Captions does not exist
|
||||
return Collections.emptyList();
|
||||
}
|
||||
captions = JsonParser.object().from(playerResponse).getObject("captions");
|
||||
} catch (JsonParserException e) {
|
||||
throw new SubtitlesException("Unable to parse subtitles listing", e);
|
||||
if (!playerResponse.has("captions")) {
|
||||
// Captions does not exist
|
||||
return Collections.emptyList();
|
||||
}
|
||||
captions = playerResponse.getObject("captions");
|
||||
|
||||
final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer", new JsonObject());
|
||||
final JsonArray captionsArray = renderer.getArray("captionTracks", new JsonArray());
|
||||
|
|
@ -924,45 +918,36 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
"&sts=" + sts + "&ps=default&gl=US&hl=en";
|
||||
}
|
||||
|
||||
private Map<String, ItagItem> getItags(String encodedUrlMapKey, ItagItem.ItagType itagTypeWanted) throws ParsingException {
|
||||
private Map<String, ItagItem> getItags(String streamingDataKey, ItagItem.ItagType itagTypeWanted) throws ParsingException {
|
||||
Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();
|
||||
|
||||
String encodedUrlMap = "";
|
||||
if (playerArgs != null && playerArgs.isString(encodedUrlMapKey)) {
|
||||
encodedUrlMap = playerArgs.getString(encodedUrlMapKey, "");
|
||||
} else if (videoInfoPage.containsKey(encodedUrlMapKey)) {
|
||||
encodedUrlMap = videoInfoPage.get(encodedUrlMapKey);
|
||||
JsonObject streamingData = playerResponse.getObject("streamingData");
|
||||
if (!streamingData.has(streamingDataKey)) {
|
||||
return urlAndItags;
|
||||
}
|
||||
|
||||
for (String url_data_str : encodedUrlMap.split(",")) {
|
||||
try {
|
||||
// This loop iterates through multiple streams, therefore tags
|
||||
// is related to one and the same stream at a time.
|
||||
Map<String, String> tags = Parser.compatParseMap(
|
||||
org.jsoup.parser.Parser.unescapeEntities(url_data_str, true));
|
||||
JsonArray formats = streamingData.getArray(streamingDataKey);
|
||||
for (int i = 0; i != formats.size(); ++i) {
|
||||
JsonObject formatData = formats.getObject(i);
|
||||
int itag = formatData.getInt("itag");
|
||||
|
||||
int itag = Integer.parseInt(tags.get("itag"));
|
||||
|
||||
if (ItagItem.isSupported(itag)) {
|
||||
if (ItagItem.isSupported(itag)) {
|
||||
try {
|
||||
ItagItem itagItem = ItagItem.getItag(itag);
|
||||
if (itagItem.itagType == itagTypeWanted) {
|
||||
String streamUrl = tags.get("url");
|
||||
// if video has a signature: decrypt it and add it to the url
|
||||
if (tags.get("s") != null) {
|
||||
if (tags.get("sp") == null) {
|
||||
// fallback for urls not conaining the "sp" tag
|
||||
streamUrl = streamUrl + "&signature=" + decryptSignature(tags.get("s"), decryptionCode);
|
||||
}
|
||||
else {
|
||||
streamUrl = streamUrl + "&" + tags.get("sp") + "=" + decryptSignature(tags.get("s"), decryptionCode);
|
||||
}
|
||||
String streamUrl;
|
||||
if (formatData.has("url")) {
|
||||
streamUrl = formatData.getString("url");
|
||||
} else {
|
||||
// this url has an encrypted signature
|
||||
Map<String, String> cipher = Parser.compatParseMap(formatData.getString("cipher"));
|
||||
streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + decryptSignature(cipher.get("s"), decryptionCode);
|
||||
}
|
||||
|
||||
urlAndItags.put(streamUrl, itagItem);
|
||||
}
|
||||
} catch (UnsupportedEncodingException ignored) {
|
||||
|
||||
}
|
||||
} catch (DecryptException e) {
|
||||
throw e;
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ public class YoutubeParsingHelper {
|
|||
public static boolean isYoutubeURL(URL url) {
|
||||
String host = url.getHost();
|
||||
return host.equalsIgnoreCase("youtube.com") || host.equalsIgnoreCase("www.youtube.com")
|
||||
|| host.equalsIgnoreCase("m.youtube.com");
|
||||
|| host.equalsIgnoreCase("m.youtube.com") || host.equalsIgnoreCase("music.youtube.com");
|
||||
}
|
||||
|
||||
public static boolean isYoutubeServiceURL(URL url) {
|
||||
|
|
@ -48,7 +48,7 @@ public class YoutubeParsingHelper {
|
|||
|
||||
public static boolean isInvidioURL(URL url) {
|
||||
String host = url.getHost();
|
||||
return host.equalsIgnoreCase("invidio.us") || host.equalsIgnoreCase("www.invidio.us");
|
||||
return host.equalsIgnoreCase("invidio.us") || host.equalsIgnoreCase("dev.invidio.us") || host.equalsIgnoreCase("www.invidio.us") || host.equalsIgnoreCase("invidious.snopyta.org") || host.equalsIgnoreCase("de.invidious.snopyta.org") || host.equalsIgnoreCase("fi.invidious.snopyta.org") || host.equalsIgnoreCase("vid.wxzm.sx") || host.equalsIgnoreCase("invidious.kabi.tk") || host.equalsIgnoreCase("invidiou.sh") || host.equalsIgnoreCase("www.invidiou.sh") || host.equalsIgnoreCase("no.invidiou.sh") || host.equalsIgnoreCase("invidious.enkirton.net") || host.equalsIgnoreCase("tube.poal.co") || host.equalsIgnoreCase("invidious.13ad.de") || host.equalsIgnoreCase("yt.elukerio.org");
|
||||
}
|
||||
|
||||
public static long parseDurationString(String input)
|
||||
|
|
|
|||
|
|
@ -114,7 +114,8 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
|
|||
|
||||
case "YOUTUBE.COM":
|
||||
case "WWW.YOUTUBE.COM":
|
||||
case "M.YOUTUBE.COM": {
|
||||
case "M.YOUTUBE.COM":
|
||||
case "MUSIC.YOUTUBE.COM": {
|
||||
if (path.equals("attribution_link")) {
|
||||
String uQueryValue = Utils.getQueryValue(url, "u");
|
||||
|
||||
|
|
@ -163,7 +164,20 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
|
|||
}
|
||||
|
||||
case "WWW.INVIDIO.US":
|
||||
case "INVIDIO.US": { // code-block for hooktube.com and invidio.us
|
||||
case "DEV.INVIDIO.US":
|
||||
case "INVIDIO.US":
|
||||
case "INVIDIOUS.SNOPYTA.ORG":
|
||||
case "DE.INVIDIOUS.SNOPYTA.ORG":
|
||||
case "FI.INVIDIOUS.SNOPYTA.ORG":
|
||||
case "VID.WXZM.SX":
|
||||
case "INVIDIOUS.KABI.TK":
|
||||
case "INVIDIOU.SH":
|
||||
case "WWW.INVIDIOU.SH":
|
||||
case "NO.INVIDIOU.SH":
|
||||
case "INVIDIOUS.ENKIRTON.NET":
|
||||
case "TUBE.POAL.CO":
|
||||
case "INVIDIOUS.13AD.DE":
|
||||
case "YT.ELUKERIO.ORG": { // code-block for hooktube.com and Invidious instances
|
||||
if (path.equals("watch")) {
|
||||
String viewQueryValue = Utils.getQueryValue(url, "v");
|
||||
if (viewQueryValue != null) {
|
||||
|
|
|
|||
|
|
@ -27,6 +27,38 @@ public class Utils {
|
|||
return toRemove.replaceAll("\\D+", "");
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>Convert a mixed number word to a long.</p>
|
||||
* <p>Examples:</p>
|
||||
* <ul>
|
||||
* <li>123 -> 123</li>
|
||||
* <li>1.23K -> 1230</li>
|
||||
* <li>1.23M -> 1230000</li>
|
||||
* </ul>
|
||||
* @param numberWord string to be converted to a long
|
||||
* @return a long
|
||||
* @throws NumberFormatException
|
||||
* @throws ParsingException
|
||||
*/
|
||||
public static long mixedNumberWordToLong(String numberWord) throws NumberFormatException, ParsingException {
|
||||
String multiplier = "";
|
||||
try {
|
||||
multiplier = Parser.matchGroup("[\\d]+([\\.,][\\d]+)?([KMBkmb])+", numberWord, 2);
|
||||
} catch(ParsingException ignored) {}
|
||||
double count = Double.parseDouble(Parser.matchGroup1("([\\d]+([\\.,][\\d]+)?)", numberWord)
|
||||
.replace(",", "."));
|
||||
switch (multiplier.toUpperCase()) {
|
||||
case "K":
|
||||
return (long) (count * 1e3);
|
||||
case "M":
|
||||
return (long) (count * 1e6);
|
||||
case "B":
|
||||
return (long) (count * 1e9);
|
||||
default:
|
||||
return (long) (count);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the url matches the pattern.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -105,6 +105,7 @@ public class YoutubeChannelExtractorTest {
|
|||
@Test
|
||||
public void testSubscriberCount() throws Exception {
|
||||
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0);
|
||||
assertTrue("Subscriber count too small", extractor.getSubscriberCount() >= 4e6);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -195,6 +196,7 @@ public class YoutubeChannelExtractorTest {
|
|||
@Test
|
||||
public void testSubscriberCount() throws Exception {
|
||||
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0);
|
||||
assertTrue("Subscriber count too small", extractor.getSubscriberCount() >= 10e6);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -390,6 +392,100 @@ public class YoutubeChannelExtractorTest {
|
|||
}
|
||||
}
|
||||
|
||||
// this channel has no "Subscribe" button
|
||||
public static class EminemVEVO implements BaseChannelExtractorTest {
|
||||
private static YoutubeChannelExtractor extractor;
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
|
||||
extractor = (YoutubeChannelExtractor) YouTube
|
||||
.getChannelExtractor("https://www.youtube.com/user/EminemVEVO/");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// Extractor
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
@Test
|
||||
public void testServiceId() {
|
||||
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testName() throws Exception {
|
||||
assertEquals("EminemVEVO", extractor.getName());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testId() throws Exception {
|
||||
assertEquals("UC20vb-R_px4CguHzzBPhoyQ", extractor.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUrl() throws ParsingException {
|
||||
assertEquals("https://www.youtube.com/channel/UC20vb-R_px4CguHzzBPhoyQ", extractor.getUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOriginalUrl() throws ParsingException {
|
||||
assertEquals("https://www.youtube.com/user/EminemVEVO/", extractor.getOriginalUrl());
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// ListExtractor
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
@Test
|
||||
public void testRelatedItems() throws Exception {
|
||||
defaultTestRelatedItems(extractor, YouTube.getServiceId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMoreRelatedItems() throws Exception {
|
||||
defaultTestMoreItems(extractor, YouTube.getServiceId());
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// ChannelExtractor
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
@Test
|
||||
public void testDescription() throws Exception {
|
||||
final String description = extractor.getDescription();
|
||||
assertTrue(description, description.contains("Eminem on Vevo"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAvatarUrl() throws Exception {
|
||||
String avatarUrl = extractor.getAvatarUrl();
|
||||
assertIsSecureUrl(avatarUrl);
|
||||
assertTrue(avatarUrl, avatarUrl.contains("yt3"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBannerUrl() throws Exception {
|
||||
String bannerUrl = extractor.getBannerUrl();
|
||||
assertIsSecureUrl(bannerUrl);
|
||||
assertTrue(bannerUrl, bannerUrl.contains("yt3"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFeedUrl() throws Exception {
|
||||
assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UC20vb-R_px4CguHzzBPhoyQ", extractor.getFeedUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSubscriberCount() throws Exception {
|
||||
// there is no "Subscribe" button
|
||||
long subscribers = extractor.getSubscriberCount();
|
||||
assertEquals("Wrong subscriber count", -1, subscribers);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static class RandomChannel implements BaseChannelExtractorTest {
|
||||
private static YoutubeChannelExtractor extractor;
|
||||
|
||||
|
|
@ -481,8 +577,9 @@ public class YoutubeChannelExtractorTest {
|
|||
|
||||
@Test
|
||||
public void testSubscriberCount() throws Exception {
|
||||
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 50);
|
||||
long subscribers = extractor.getSubscriberCount();
|
||||
assertTrue("Wrong subscriber count: " + subscribers, subscribers >= 50);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ public class YoutubePlaylistLinkHandlerFactoryTest {
|
|||
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
|
||||
assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId());
|
||||
assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId());
|
||||
assertEquals("OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM", linkHandler.fromUrl("https://music.youtube.com/playlist?list=OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM").getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -54,6 +55,7 @@ public class YoutubePlaylistLinkHandlerFactoryTest {
|
|||
assertTrue(linkHandler.acceptUrl("https://youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
|
||||
assertTrue(linkHandler.acceptUrl("www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC"));
|
||||
assertTrue(linkHandler.acceptUrl("www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
|
||||
assertTrue(linkHandler.acceptUrl("https://music.youtube.com/playlist?list=OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM"));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
|
|
@ -80,6 +80,7 @@ public class YoutubeStreamLinkHandlerFactoryTest {
|
|||
assertEquals("EhxJLojIE_o", linkHandler.fromUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare").getId());
|
||||
assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI").getId());
|
||||
assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI").getId());
|
||||
assertEquals("O0EDx9WAelc", linkHandler.fromUrl("https://music.youtube.com/watch?v=O0EDx9WAelc").getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -98,8 +99,8 @@ public class YoutubeStreamLinkHandlerFactoryTest {
|
|||
assertTrue(linkHandler.acceptUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare"));
|
||||
assertTrue(linkHandler.acceptUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI"));
|
||||
assertTrue(linkHandler.acceptUrl("vnd.youtube:jZViOEv90dI"));
|
||||
|
||||
assertTrue(linkHandler.acceptUrl("vnd.youtube.launch:jZViOEv90dI"));
|
||||
assertTrue(linkHandler.acceptUrl("https://music.youtube.com/watch?v=O0EDx9WAelc"));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
package org.schabi.newpipe.extractor.services.youtube.stream;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Ignore;
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
package org.schabi.newpipe.extractor.services.youtube.stream;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Ignore;
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
package org.schabi.newpipe.extractor.services.youtube.stream;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
|
@ -0,0 +1,138 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube.stream;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.schabi.newpipe.Downloader;
|
||||
import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
import org.schabi.newpipe.extractor.stream.VideoStream;
|
||||
import org.schabi.newpipe.extractor.utils.Localization;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
|
||||
|
||||
public class YoutubeStreamExtractorLivestreamTest {
|
||||
private static YoutubeStreamExtractor extractor;
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
|
||||
extractor = (YoutubeStreamExtractor) YouTube
|
||||
.getStreamExtractor("https://www.youtube.com/watch?v=EcEMX-63PKY");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetInvalidTimeStamp() throws ParsingException {
|
||||
assertTrue(extractor.getTimeStamp() + "",
|
||||
extractor.getTimeStamp() <= 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetTitle() throws ParsingException {
|
||||
assertFalse(extractor.getName().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetDescription() throws ParsingException {
|
||||
assertNotNull(extractor.getDescription());
|
||||
assertFalse(extractor.getDescription().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetFullLinksInDescription() throws ParsingException {
|
||||
assertTrue(extractor.getDescription().contains("https://www.instagram.com/nathalie.baraton/"));
|
||||
assertFalse(extractor.getDescription().contains("https://www.instagram.com/nathalie.ba..."));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetUploaderName() throws ParsingException {
|
||||
assertNotNull(extractor.getUploaderName());
|
||||
assertFalse(extractor.getUploaderName().isEmpty());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testGetLength() throws ParsingException {
|
||||
assertEquals(0, extractor.getLength());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetViewCount() throws ParsingException {
|
||||
long count = extractor.getViewCount();
|
||||
assertTrue(Long.toString(count), count >= 7148995);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetUploadDate() throws ParsingException {
|
||||
assertTrue(extractor.getUploadDate().length() > 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetUploaderUrl() throws ParsingException {
|
||||
assertEquals("https://www.youtube.com/channel/UCSJ4gkVC6NrvII8umztf0Ow", extractor.getUploaderUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetThumbnailUrl() throws ParsingException {
|
||||
assertIsSecureUrl(extractor.getThumbnailUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetUploaderAvatarUrl() throws ParsingException {
|
||||
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetAudioStreams() throws ExtractionException {
|
||||
assertFalse(extractor.getAudioStreams().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetVideoStreams() throws ExtractionException {
|
||||
for (VideoStream s : extractor.getVideoStreams()) {
|
||||
assertIsSecureUrl(s.url);
|
||||
assertTrue(s.resolution.length() > 0);
|
||||
assertTrue(Integer.toString(s.getFormatId()),
|
||||
0 <= s.getFormatId() && s.getFormatId() <= 0x100);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testStreamType() throws ParsingException {
|
||||
assertSame(extractor.getStreamType(), StreamType.LIVE_STREAM);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetDashMpd() throws ParsingException {
|
||||
// we dont expect this particular video to have a DASH file. For this purpouse we use a different test class.
|
||||
assertTrue(extractor.getDashMpdUrl(), extractor.getDashMpdUrl().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetRelatedVideos() throws ExtractionException, IOException {
|
||||
StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();
|
||||
Utils.printErrors(relatedVideos.getErrors());
|
||||
assertFalse(relatedVideos.getItems().isEmpty());
|
||||
assertTrue(relatedVideos.getErrors().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
|
||||
assertTrue(extractor.getSubtitlesDefault().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetSubtitlesList() throws IOException, ExtractionException {
|
||||
assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
package org.schabi.newpipe.extractor.utils;
|
||||
|
||||
import com.grack.nanojson.JsonParserException;
|
||||
import org.junit.Test;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
public class UtilsTest {
|
||||
@Test
|
||||
public void testMixedNumberWordToLong() throws JsonParserException, ParsingException {
|
||||
assertEquals(10, Utils.mixedNumberWordToLong("10"));
|
||||
assertEquals(10.5e3, Utils.mixedNumberWordToLong("10.5K"), 0.0);
|
||||
assertEquals(10.5e6, Utils.mixedNumberWordToLong("10.5M"), 0.0);
|
||||
assertEquals(10.5e6, Utils.mixedNumberWordToLong("10,5M"), 0.0);
|
||||
assertEquals(1.5e9, Utils.mixedNumberWordToLong("1,5B"), 0.0);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue