lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #14 from mauriciocolli/refactor-extractor 

Fix dash parser and more refactor
This commit is contained in:
Mauricio Colli 2017-07-11 22:26:48 -03:00 committed by GitHub
parent bda65e83d6 b1989c0a83
commit b5b25a4188
41 changed files with 802 additions and 751 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Extractor.java
View File

@ -14,7 +14,7 @@ public abstract class Extractor implements Serializable {
this.urlIdHandler = urlIdHandler;
this.serviceId = serviceId;
this.url = url;
this.previewInfoCollector = new StreamInfoItemCollector(urlIdHandler, serviceId);
this.previewInfoCollector = new StreamInfoItemCollector(serviceId);
}
public String getUrl() {

6
Info.java
View File

@ -11,9 +11,9 @@ public abstract class Info implements Serializable {
* Id of this Info object <br>
* e.g. Youtube: https://www.youtube.com/watch?v=RER5qCTzZ7 > RER5qCTzZ7
*/
public String id = "";
public String url = "";
public String name = "";
public String id;
public String url;
public String name;
public List<Throwable> errors = new Vector<>();
}

19
InfoItem.java
View File

@ -1,7 +1,5 @@
package org.schabi.newpipe.extractor;
import java.io.Serializable;
/*
* Created by the-scrabi on 11.02.17.
*
@ -22,14 +20,21 @@ import java.io.Serializable;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public interface InfoItem extends Serializable {
enum InfoType {
import java.io.Serializable;
public abstract class InfoItem implements Serializable {
public enum InfoType {
STREAM,
PLAYLIST,
CHANNEL
}
InfoType infoType();
String getTitle();
String getLink();
public InfoItem(InfoType infoType) {
this.info_type = infoType;
}
public final InfoType info_type;
public int service_id = -1;
public String url;
public String name;
}

2
InfoItemCollector.java
View File

@ -25,7 +25,7 @@ import java.util.Vector;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class InfoItemCollector {
public abstract class InfoItemCollector {
private List<InfoItem> itemList = new Vector<>();
private List<Throwable> errors = new Vector<>();
private int serviceId = -1;

13
MediaFormat.java
View File

@ -91,4 +91,17 @@ public enum MediaFormat {
}
return "";
}
/**
* Return the MediaFormat with the supplied mime type
*
* @return MediaFormat associated with this mime type,
* or null if none match it.
*/
public static MediaFormat getFromMimeType(String mimeType) {
for (MediaFormat vf : MediaFormat.values()) {
if (vf.mimeType.equals(mimeType)) return vf;
}
return null;
}
}

1
channel/ChannelExtractor.java
View File

@ -1,6 +1,5 @@
package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.Extractor;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;

21
channel/ChannelInfo.java
View File

@ -29,21 +29,16 @@ import java.util.List;
public class ChannelInfo extends Info {
public static ChannelInfo getInfo(ChannelExtractor extractor)
throws ParsingException {
public static ChannelInfo getInfo(ChannelExtractor extractor) throws ParsingException {
ChannelInfo info = new ChannelInfo();
// important data
info.service_id = extractor.getServiceId();
info.url = extractor.getUrl();
info.id = extractor.getChannelId();
info.name = extractor.getChannelName();
info.hasMoreStreams = extractor.hasMoreStreams();
info.has_more_streams = extractor.hasMoreStreams();
try {
info.id = extractor.getChannelId();
} catch (Exception e) {
info.errors.add(e);
}
try {
info.avatar_url = extractor.getAvatarUrl();
} catch (Exception e) {
@ -75,10 +70,10 @@ public class ChannelInfo extends Info {
return info;
}
public String avatar_url = "";
public String banner_url = "";
public String feed_url = "";
public List<InfoItem> related_streams = null;
public String avatar_url;
public String banner_url;
public String feed_url;
public List<InfoItem> related_streams;
public long subscriber_count = -1;
public boolean hasMoreStreams = false;
public boolean has_more_streams = false;
}

25
channel/ChannelInfoItem.java
View File

@ -22,25 +22,14 @@ import org.schabi.newpipe.extractor.InfoItem;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class ChannelInfoItem implements InfoItem {
public class ChannelInfoItem extends InfoItem {
public int serviceId = -1;
public String channelName = "";
public String thumbnailUrl = "";
public String webPageUrl = "";
public String description = "";
public long subscriberCount = -1;
public long viewCount = -1;
public String thumbnail_url;
public String description;
public long subscriber_count = -1;
public long view_count = -1;
public InfoType infoType() {
return InfoType.CHANNEL;
}
public String getTitle() {
return channelName;
}
public String getLink() {
return webPageUrl;
public ChannelInfoItem() {
super(InfoType.CHANNEL);
}
}

12
channel/ChannelInfoItemCollector.java
View File

@ -31,24 +31,24 @@ public class ChannelInfoItemCollector extends InfoItemCollector {
public ChannelInfoItem extract(ChannelInfoItemExtractor extractor) throws ParsingException {
ChannelInfoItem resultItem = new ChannelInfoItem();
// important information
resultItem.channelName = extractor.getChannelName();
resultItem.name = extractor.getChannelName();
resultItem.serviceId = getServiceId();
resultItem.webPageUrl = extractor.getWebPageUrl();
resultItem.service_id = getServiceId();
resultItem.url = extractor.getWebPageUrl();
// optional information
try {
resultItem.subscriberCount = extractor.getSubscriberCount();
resultItem.subscriber_count = extractor.getSubscriberCount();
} catch (Exception e) {
addError(e);
}
try {
resultItem.viewCount = extractor.getViewCount();
resultItem.view_count = extractor.getViewCount();
} catch (Exception e) {
addError(e);
}
try {
resultItem.thumbnailUrl = extractor.getThumbnailUrl();
resultItem.thumbnail_url = extractor.getThumbnailUrl();
} catch (Exception e) {
addError(e);
}

11
exceptions/ContentNotAvailableException.java Normal file
View File

@ -0,0 +1,11 @@
package org.schabi.newpipe.extractor.exceptions;
public class ContentNotAvailableException extends ParsingException {
public ContentNotAvailableException(String message) {
super(message);
}
public ContentNotAvailableException(String message, Throwable cause) {
super(message, cause);
}
}

1
playlist/PlaylistExtractor.java
View File

@ -1,6 +1,5 @@
package org.schabi.newpipe.extractor.playlist;
import org.schabi.newpipe.extractor.Extractor;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;

22
playlist/PlaylistInfo.java
View File

@ -14,14 +14,10 @@ public class PlaylistInfo extends Info {
info.service_id = extractor.getServiceId();
info.url = extractor.getUrl();
info.id = extractor.getPlaylistId();
info.name = extractor.getPlaylistName();
info.hasMoreStreams = extractor.hasMoreStreams();
info.has_more_streams = extractor.hasMoreStreams();
try {
info.id = extractor.getPlaylistId();
} catch (Exception e) {
info.errors.add(e);
}
try {
info.streams_count = extractor.getStreamsCount();
} catch (Exception e) {
@ -63,12 +59,12 @@ public class PlaylistInfo extends Info {
return info;
}
public String avatar_url = "";
public String banner_url = "";
public String uploader_url = "";
public String uploader_name = "";
public String uploader_avatar_url = "";
public String avatar_url;
public String banner_url;
public String uploader_url;
public String uploader_name;
public String uploader_avatar_url;
public long streams_count = 0;
public List<InfoItem> related_streams = null;
public boolean hasMoreStreams = false;
public List<InfoItem> related_streams;
public boolean has_more_streams;
}

23
playlist/PlaylistInfoItem.java
View File

@ -2,22 +2,15 @@ package org.schabi.newpipe.extractor.playlist;
import org.schabi.newpipe.extractor.InfoItem;
public class PlaylistInfoItem implements InfoItem {
public class PlaylistInfoItem extends InfoItem {
public int serviceId = -1;
public String name = "";
public String thumbnailUrl = "";
public String webPageUrl = "";
public String thumbnail_url;
/**
* How many streams this playlist have
*/
public long streams_count = 0;
public InfoType infoType() {
return InfoType.PLAYLIST;
}
public String getTitle() {
return name;
}
public String getLink() {
return webPageUrl;
public PlaylistInfoItem() {
super(InfoType.PLAYLIST);
}
}

13
playlist/PlaylistInfoItemCollector.java
View File

@ -1,6 +1,7 @@
package org.schabi.newpipe.extractor.playlist;
import org.schabi.newpipe.extractor.InfoItemCollector;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
public class PlaylistInfoItemCollector extends InfoItemCollector {
@ -12,10 +13,16 @@ public class PlaylistInfoItemCollector extends InfoItemCollector {
final PlaylistInfoItem resultItem = new PlaylistInfoItem();
resultItem.name = extractor.getPlaylistName();
resultItem.serviceId = getServiceId();
resultItem.webPageUrl = extractor.getWebPageUrl();
resultItem.service_id = getServiceId();
resultItem.url = extractor.getWebPageUrl();
try {
resultItem.thumbnailUrl = extractor.getThumbnailUrl();
resultItem.thumbnail_url = extractor.getThumbnailUrl();
} catch (Exception e) {
addError(e);
}
try {
resultItem.streams_count = extractor.getStreamsCount();
} catch (Exception e) {
addError(e);
}

1
playlist/PlaylistInfoItemExtractor.java
View File

@ -6,4 +6,5 @@ public interface PlaylistInfoItemExtractor {
String getThumbnailUrl() throws ParsingException;
String getPlaylistName() throws ParsingException;
String getWebPageUrl() throws ParsingException;
long getStreamsCount() throws ParsingException;
}

9
search/InfoItemSearchCollector.java
View File

@ -1,7 +1,6 @@
package org.schabi.newpipe.extractor.search;
import org.schabi.newpipe.extractor.InfoItemCollector;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -30,15 +29,15 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
*/
public class InfoItemSearchCollector extends InfoItemCollector {
private String suggestion = "";
private String suggestion;
private StreamInfoItemCollector streamCollector;
private ChannelInfoItemCollector channelCollector;
SearchResult result = new SearchResult();
private SearchResult result = new SearchResult();
InfoItemSearchCollector(UrlIdHandler handler, int serviceId) {
InfoItemSearchCollector(int serviceId) {
super(serviceId);
streamCollector = new StreamInfoItemCollector(handler, serviceId);
streamCollector = new StreamInfoItemCollector(serviceId);
channelCollector = new ChannelInfoItemCollector(serviceId);
}

7
search/SearchEngine.java
View File

@ -1,6 +1,5 @@
package org.schabi.newpipe.extractor.search;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.io.IOException;
@ -28,7 +27,7 @@ import java.util.EnumSet;
public abstract class SearchEngine {
public enum Filter {
STREAM, CHANNEL, PLAY_LIST
STREAM, CHANNEL, PLAYLIST
}
public static class NothingFoundException extends ExtractionException {
@ -39,8 +38,8 @@ public abstract class SearchEngine {
private InfoItemSearchCollector collector;
public SearchEngine(UrlIdHandler urlIdHandler, int serviceId) {
collector = new InfoItemSearchCollector(urlIdHandler, serviceId);
public SearchEngine(int serviceId) {
collector = new InfoItemSearchCollector(serviceId);
}
protected InfoItemSearchCollector getInfoItemSearchCollector() {

2
search/SearchResult.java
View File

@ -49,7 +49,7 @@ public class SearchResult {
return result;
}
public String suggestion = "";
public String suggestion;
public List<InfoItem> resultList = new Vector<>();
public List<Throwable> errors = new Vector<>();
}

160
services/youtube/ItagItem.java Normal file
View File

@ -0,0 +1,160 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import static org.schabi.newpipe.extractor.MediaFormat.M4A;
import static org.schabi.newpipe.extractor.MediaFormat.MPEG_4;
import static org.schabi.newpipe.extractor.MediaFormat.WEBM;
import static org.schabi.newpipe.extractor.MediaFormat.WEBMA;
import static org.schabi.newpipe.extractor.MediaFormat.v3GPP;
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.AUDIO;
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.VIDEO;
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.VIDEO_ONLY;
public class ItagItem {
/**
* List can be found here https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L360
*/
private static final ItagItem[] ITAG_LIST = {
/////////////////////////////////////////////////////
// VIDEO ID Type Format Resolution FPS ///
///////////////////////////////////////////////////
new ItagItem(17, VIDEO, v3GPP, "144p"),
new ItagItem(36, VIDEO, v3GPP, "240p"),
new ItagItem(18, VIDEO, MPEG_4, "360p"),
new ItagItem(34, VIDEO, MPEG_4, "360p"),
new ItagItem(35, VIDEO, MPEG_4, "480p"),
new ItagItem(59, VIDEO, MPEG_4, "480p"),
new ItagItem(78, VIDEO, MPEG_4, "480p"),
new ItagItem(22, VIDEO, MPEG_4, "720p"),
new ItagItem(37, VIDEO, MPEG_4, "1080p"),
new ItagItem(38, VIDEO, MPEG_4, "1080p"),
new ItagItem(43, VIDEO, WEBM, "360p"),
new ItagItem(44, VIDEO, WEBM, "480p"),
new ItagItem(45, VIDEO, WEBM, "720p"),
new ItagItem(46, VIDEO, WEBM, "1080p"),
////////////////////////////////////////////////////////////////////
// AUDIO ID ItagType Format Bitrate ///
//////////////////////////////////////////////////////////////////
// Disable Opus codec as it's not well supported in older devices
// new ItagItem(249, AUDIO, WEBMA, 50),
// new ItagItem(250, AUDIO, WEBMA, 70),
// new ItagItem(251, AUDIO, WEBMA, 16),
new ItagItem(171, AUDIO, WEBMA, 128),
new ItagItem(172, AUDIO, WEBMA, 256),
new ItagItem(139, AUDIO, M4A, 48),
new ItagItem(140, AUDIO, M4A, 128),
new ItagItem(141, AUDIO, M4A, 256),
/// VIDEO ONLY ////////////////////////////////////////////
// ID Type Format Resolution FPS ///
/////////////////////////////////////////////////////////
// Don't add VideoOnly streams that have normal variants
new ItagItem(160, VIDEO_ONLY, MPEG_4, "144p"),
new ItagItem(133, VIDEO_ONLY, MPEG_4, "240p"),
// new ItagItem(134, VIDEO_ONLY, MPEG_4, "360p"),
new ItagItem(135, VIDEO_ONLY, MPEG_4, "480p"),
new ItagItem(212, VIDEO_ONLY, MPEG_4, "480p"),
// new ItagItem(136, VIDEO_ONLY, MPEG_4, "720p"),
new ItagItem(298, VIDEO_ONLY, MPEG_4, "720p60", 60),
new ItagItem(137, VIDEO_ONLY, MPEG_4, "1080p"),
new ItagItem(299, VIDEO_ONLY, MPEG_4, "1080p60", 60),
new ItagItem(266, VIDEO_ONLY, MPEG_4, "2160p"),
new ItagItem(278, VIDEO_ONLY, WEBM, "144p"),
new ItagItem(242, VIDEO_ONLY, WEBM, "240p"),
// new ItagItem(243, VIDEO_ONLY, WEBM, "360p"),
new ItagItem(244, VIDEO_ONLY, WEBM, "480p"),
new ItagItem(245, VIDEO_ONLY, WEBM, "480p"),
new ItagItem(246, VIDEO_ONLY, WEBM, "480p"),
new ItagItem(247, VIDEO_ONLY, WEBM, "720p"),
new ItagItem(248, VIDEO_ONLY, WEBM, "1080p"),
new ItagItem(271, VIDEO_ONLY, WEBM, "1440p"),
// #272 is either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
new ItagItem(272, VIDEO_ONLY, WEBM, "2160p"),
new ItagItem(302, VIDEO_ONLY, WEBM, "720p60", 60),
new ItagItem(303, VIDEO_ONLY, WEBM, "1080p60", 60),
new ItagItem(308, VIDEO_ONLY, WEBM, "1440p60", 60),
new ItagItem(313, VIDEO_ONLY, WEBM, "2160p"),
new ItagItem(315, VIDEO_ONLY, WEBM, "2160p60", 60)
};
/*//////////////////////////////////////////////////////////////////////////
// Utils
//////////////////////////////////////////////////////////////////////////*/
public static boolean isSupported(int itag) {
for (ItagItem item : ITAG_LIST) {
if (itag == item.id) {
return true;
}
}
return false;
}
public static ItagItem getItag(int itagId) throws ParsingException {
for (ItagItem item : ITAG_LIST) {
if (itagId == item.id) {
return item;
}
}
throw new ParsingException("itag=" + Integer.toString(itagId) + " not supported");
}
/*//////////////////////////////////////////////////////////////////////////
// Contructors and misc
//////////////////////////////////////////////////////////////////////////*/
public enum ItagType {
AUDIO,
VIDEO,
VIDEO_ONLY
}
/**
* Call {@link #ItagItem(int, ItagType, MediaFormat, String, int)} with the fps set to 30.
*/
public ItagItem(int id, ItagType type, MediaFormat format, String resolution) {
this.id = id;
this.itagType = type;
this.mediaFormatId = format.id;
this.resolutionString = resolution;
this.fps = 30;
}
/**
* Constructor for videos.
*
* @param resolution string that will be used in the frontend
*/
public ItagItem(int id, ItagType type, MediaFormat format, String resolution, int fps) {
this.id = id;
this.itagType = type;
this.mediaFormatId = format.id;
this.resolutionString = resolution;
this.fps = fps;
}
public ItagItem(int id, ItagType type, MediaFormat format, int avgBitrate) {
this.id = id;
this.itagType = type;
this.mediaFormatId = format.id;
this.avgBitrate = avgBitrate;
}
public int id;
public ItagType itagType;
public int mediaFormatId;
// Audio fields
public int avgBitrate = -1;
// Video fields
public String resolutionString;
public int fps = -1;
}

13
services/youtube/YoutubeChannelExtractor.java
View File

@ -13,10 +13,11 @@ import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.AbstractStreamInfo;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
@ -135,7 +136,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
if (subscriberCount == -1) {
Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first();
if (el != null) {
subscriberCount = Long.parseLong(el.text().replaceAll("\\D+", ""));
subscriberCount = Long.parseLong(Utils.removeNonDigitCharacters(el.text()));
} else {
throw new ParsingException("Could not get subscriber count");
}
@ -164,7 +165,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
throw new ExtractionException("Channel doesn't have more streams");
}
StreamInfoItemCollector collector = new StreamInfoItemCollector(getUrlIdHandler(), getServiceId());
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
setupNextStreamsAjax(NewPipe.getDownloader());
collectStreamsFrom(collector, nextStreamsAjax.select("body").first());
@ -223,8 +224,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
if (li.select("div[class=\"feed-item-dismissable\"]").first() != null) {
collector.commit(new StreamInfoItemExtractor() {
@Override
public AbstractStreamInfo.StreamType getStreamType() throws ParsingException {
return AbstractStreamInfo.StreamType.VIDEO_STREAM;
public StreamType getStreamType() throws ParsingException {
return StreamType.VIDEO_STREAM;
}
@Override
@ -302,7 +303,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return -1;
}
output = input.replaceAll("\\D+", "");
output = Utils.removeNonDigitCharacters(input);
try {
return Long.parseLong(output);

5
services/youtube/YoutubeChannelInfoItemExtractor.java
View File

@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Utils;
/*
* Created by Christian Schabesberger on 12.02.17.
@ -62,7 +63,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
if (subsEl == null) {
return 0;
} else {
return Long.parseLong(subsEl.text().replaceAll("\\D+", ""));
return Long.parseLong(Utils.removeNonDigitCharacters(subsEl.text()));
}
}
@ -72,7 +73,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
if (metaEl == null) {
return 0;
} else {
return Long.parseLong(metaEl.text().replaceAll("\\D+", ""));
return Long.parseLong(Utils.removeNonDigitCharacters(metaEl.text()));
}
}

13
services/youtube/YoutubeChannelUrlIdHandler.java
View File

@ -26,18 +26,29 @@ import org.schabi.newpipe.extractor.utils.Parser;
public class YoutubeChannelUrlIdHandler implements UrlIdHandler {
private static final YoutubeChannelUrlIdHandler instance = new YoutubeChannelUrlIdHandler();
private static final String ID_PATTERN = "/(user/[A-Za-z0-9_-]*|channel/[A-Za-z0-9_-]*)";
public static YoutubeChannelUrlIdHandler getInstance() {
return instance;
}
@Override
public String getUrl(String channelId) {
return "https://www.youtube.com/" + channelId;
}
@Override
public String getId(String siteUrl) throws ParsingException {
return Parser.matchGroup1("/(user/[A-Za-z0-9_-]*|channel/[A-Za-z0-9_-]*)", siteUrl);
return Parser.matchGroup1(ID_PATTERN, siteUrl);
}
@Override
public String cleanUrl(String siteUrl) throws ParsingException {
return getUrl(getId(siteUrl));
}
@Override
public boolean acceptUrl(String videoUrl) {
return (videoUrl.contains("youtube") ||
videoUrl.contains("youtu.be")) &&

11
services/youtube/YoutubePlaylistExtractor.java
View File

@ -12,10 +12,11 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.stream.AbstractStreamInfo;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
@ -157,7 +158,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
try {
streamsCount = Long.parseLong(input.replaceAll("\\D+", ""));
streamsCount = Long.parseLong(Utils.removeNonDigitCharacters(input));
} catch (NumberFormatException e) {
// When there's no videos in a playlist, there's no number in the "innerHtml",
// all characters that is not a number is removed, so we try to parse a empty string
@ -186,7 +187,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
throw new ExtractionException("Playlist doesn't have more streams");
}
StreamInfoItemCollector collector = new StreamInfoItemCollector(getUrlIdHandler(), getServiceId());
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
setupNextStreamsAjax(NewPipe.getDownloader());
collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first());
@ -244,8 +245,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
for (final Element li : element.children()) {
collector.commit(new StreamInfoItemExtractor() {
@Override
public AbstractStreamInfo.StreamType getStreamType() throws ParsingException {
return AbstractStreamInfo.StreamType.VIDEO_STREAM;
public StreamType getStreamType() throws ParsingException {
return StreamType.VIDEO_STREAM;
}
@Override

5
services/youtube/YoutubePlaylistUrlIdHandler.java
View File

@ -7,8 +7,13 @@ import org.schabi.newpipe.extractor.utils.Parser;
public class YoutubePlaylistUrlIdHandler implements UrlIdHandler {
private static final YoutubePlaylistUrlIdHandler instance = new YoutubePlaylistUrlIdHandler();
private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{34})";
public static YoutubePlaylistUrlIdHandler getInstance() {
return instance;
}
@Override
public String getUrl(String listId) {
return "https://www.youtube.com/playlist?list=" + listId;

5
services/youtube/YoutubeSearchEngine.java
View File

@ -5,7 +5,6 @@ import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.search.InfoItemSearchCollector;
import org.schabi.newpipe.extractor.search.SearchEngine;
@ -40,8 +39,8 @@ public class YoutubeSearchEngine extends SearchEngine {
private static final String TAG = YoutubeSearchEngine.class.toString();
public static final String CHARSET_UTF_8 = "UTF-8";
public YoutubeSearchEngine(UrlIdHandler urlIdHandler, int serviceId) {
super(urlIdHandler, serviceId);
public YoutubeSearchEngine(int serviceId) {
super(serviceId);
}
@Override

6
services/youtube/YoutubeService.java
View File

@ -58,7 +58,7 @@ public class YoutubeService extends StreamingService {
@Override
public SearchEngine getSearchEngineInstance() {
return new YoutubeSearchEngine(getStreamUrlIdHandlerInstance(), getServiceId());
return new YoutubeSearchEngine(getServiceId());
}
@Override
@ -68,13 +68,13 @@ public class YoutubeService extends StreamingService {
@Override
public UrlIdHandler getChannelUrlIdHandlerInstance() {
return new YoutubeChannelUrlIdHandler();
return YoutubeChannelUrlIdHandler.getInstance();
}
@Override
public UrlIdHandler getPlaylistUrlIdHandlerInstance() {
return new YoutubePlaylistUrlIdHandler();
return YoutubePlaylistUrlIdHandler.getInstance();
}
@Override

692
services/youtube/YoutubeStreamExtractor.java
View File

@ -9,20 +9,21 @@ import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.AbstractStreamInfo;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfo;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.util.List;
@ -52,12 +53,11 @@ import java.util.regex.Pattern;
*/
public class YoutubeStreamExtractor extends StreamExtractor {
public static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map";
public static final String HTTPS = "https:";
public static final String CONTENT = "content";
public static final String REGEX_INT = "[^\\d]";
private static final String TAG = YoutubeStreamExtractor.class.getSimpleName();
// exceptions
/*//////////////////////////////////////////////////////////////////////////
// Exceptions
//////////////////////////////////////////////////////////////////////////*/
public class DecryptException extends ParsingException {
DecryptException(String message, Throwable cause) {
@ -65,8 +65,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
// special content not available exceptions
public class GemaException extends ContentNotAvailableException {
GemaException(String message) {
super(message);
@ -79,267 +77,27 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
// ----------------
/*//////////////////////////////////////////////////////////////////////////*/
// Sometimes if the html page of youtube is already downloaded, youtube web page will internally
// download the /get_video_info page. Since a certain date dashmpd url is only available over
// this /get_video_info page, so we always need to download this one to.
// %%video_id%% will be replaced by the actual video id
// $$el_type$$ will be replaced by the actual el_type (se the declarations below)
private static final String GET_VIDEO_INFO_URL =
"https://www.youtube.com/get_video_info?video_id=%%video_id%%$$el_type$$&ps=default&eurl=&gl=US&hl=en";
// eltype is necessary for the url above
private static final String EL_INFO = "el=info";
private Document doc;
private final String dirtyUrl;
public enum ItagType {
AUDIO,
VIDEO,
VIDEO_ONLY
public YoutubeStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId) throws ExtractionException, IOException {
super(urlIdHandler, urlIdHandler.cleanUrl(pageUrl), serviceId);
dirtyUrl = pageUrl;
fetchDocument();
}
private static class ItagItem {
public ItagItem(int id, ItagType type, MediaFormat format, String res, int fps) {
this.id = id;
this.itagType = type;
this.mediaFormatId = format.id;
this.resolutionString = res;
this.fps = fps;
}
/*//////////////////////////////////////////////////////////////////////////
// Impl
//////////////////////////////////////////////////////////////////////////*/
public ItagItem(int id, ItagType type, MediaFormat format, int samplingRate, int bandWidth) {
this(id, type, format, 0, samplingRate, bandWidth);
}
public ItagItem(int id, ItagType type, MediaFormat format, int avgBitrate, int samplingRate, int bandWidth) {
this.id = id;
this.itagType = type;
this.mediaFormatId = format.id;
this.avgBitrate = avgBitrate;
this.samplingRate = samplingRate;
this.bandWidth = bandWidth;
}
public int id;
public ItagType itagType;
public int mediaFormatId;
public String resolutionString;
public int fps = -1;
public int avgBitrate = -1;
public int samplingRate = -1;
public int bandWidth = -1;
}
/**
* List can be found here https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L360
*/
private static final ItagItem[] itagList = {
//////////////////////////////////////////////////////////////////////////
// VIDEO ID ItagType Format Resolution FPS ///
////////////////////////////////////////////////////////////////////////
new ItagItem(17, ItagType.VIDEO, MediaFormat.v3GPP, "144p" , 12),
new ItagItem(18, ItagType.VIDEO, MediaFormat.MPEG_4, "360p" , 24),
new ItagItem(22, ItagType.VIDEO, MediaFormat.MPEG_4, "720p" , 24),
new ItagItem(36, ItagType.VIDEO, MediaFormat.v3GPP, "240p" , 24),
new ItagItem(37, ItagType.VIDEO, MediaFormat.MPEG_4, "1080p" , 24),
new ItagItem(38, ItagType.VIDEO, MediaFormat.MPEG_4, "1080p" , 24),
new ItagItem(43, ItagType.VIDEO, MediaFormat.WEBM, "360p" , 24),
new ItagItem(44, ItagType.VIDEO, MediaFormat.WEBM, "480p" , 24),
new ItagItem(45, ItagType.VIDEO, MediaFormat.WEBM, "720p" , 24),
new ItagItem(46, ItagType.VIDEO, MediaFormat.WEBM, "1080p" , 24),
//////////////////////////////////////////////////////////////////////////////////////////
// AUDIO ID ItagType Format Bitrate SamplingR Bandwidth ///
////////////////////////////////////////////////////////////////////////////////////////
// Disable Opus codec as it's not well supported in older devices
// new ItagItem(249, ItagType.AUDIO, MediaFormat.WEBMA, 50, 0, 0),
// new ItagItem(250, ItagType.AUDIO, MediaFormat.WEBMA, 70, 0, 0),
// new ItagItem(251, ItagType.AUDIO, MediaFormat.WEBMA, 160, 0, 0),
new ItagItem(171, ItagType.AUDIO, MediaFormat.WEBMA, 128, 0, 0),
new ItagItem(172, ItagType.AUDIO, MediaFormat.WEBMA, 256, 0, 0),
new ItagItem(140, ItagType.AUDIO, MediaFormat.M4A, 128, 0, 0),
new ItagItem(141, ItagType.AUDIO, MediaFormat.M4A, 256, 0, 0),
/// VIDEO ONLY ///////////////////////////////////////////////////////////////////
// ID ItagType Format Resolution FPS ///
////////////////////////////////////////////////////////////////////////////////
// Don't add VideoOnly streams that have normal variants
// new ItagItem(160, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "144p" , 24),
// new ItagItem(133, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "240p" , 24),
// new ItagItem(134, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "360p" , 24),
new ItagItem(135, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "480p" , 30),
// new ItagItem(136, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "720p" , 30),
new ItagItem(298, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "720p60" , 60),
new ItagItem(137, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "1080p" , 30),
new ItagItem(299, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "1080p60" , 60),
new ItagItem(266, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "2160p" , 30),
// new ItagItem(243, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "360p" , 30),
new ItagItem(244, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "480p" , 30),
new ItagItem(245, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "480p" , 30),
new ItagItem(246, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "480p" , 30),
new ItagItem(247, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "720p" , 30),
new ItagItem(248, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1080p" , 30),
new ItagItem(271, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1440p" , 30),
// #272 is either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
new ItagItem(272, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "2160p" , 30),
new ItagItem(302, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "720p60" , 60),
new ItagItem(303, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1080p60" , 60),
new ItagItem(308, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1440p60" , 60),
new ItagItem(313, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "2160p" , 30),
new ItagItem(315, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "2160p60" , 60)
};
public static boolean itagIsSupported(int itag) {
for (ItagItem item : itagList) {
if (itag == item.id) {
return true;
}
}
return false;
}
public static ItagItem getItagItem(int itag) throws ParsingException {
for (ItagItem item : itagList) {
if (itag == item.id) {
return item;
}
}
throw new ParsingException("itag=" + Integer.toString(itag) + " not supported");
}
private static final String TAG = YoutubeStreamExtractor.class.toString();
private final Document doc;
private JSONObject playerArgs;
private boolean isAgeRestricted;
private Map<String, String> videoInfoPage;
// static values
private static final String DECRYPTION_FUNC_NAME = "decrypt";
// cached values
private static volatile String decryptionCode = "";
UrlIdHandler urlidhandler = YoutubeStreamUrlIdHandler.getInstance();
String pageUrl = "";
public YoutubeStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId)
throws ExtractionException, IOException {
super(urlIdHandler, pageUrl, serviceId);
//most common videoInfo fields are now set in our superclass, for all services
this.pageUrl = pageUrl;
Downloader downloader = NewPipe.getDownloader();
String pageContent = downloader.download(urlidhandler.cleanUrl(pageUrl));
doc = Jsoup.parse(pageContent, pageUrl);
JSONObject ytPlayerConfig;
String playerUrl;
// Check if the video is age restricted
if (pageContent.contains("<meta property=\"og:restrictions:age")) {
String videoInfoUrl = GET_VIDEO_INFO_URL.replace("%%video_id%%",
urlidhandler.getId(pageUrl)).replace("$$el_type$$", "&" + EL_INFO);
String videoInfoPageString = downloader.download(videoInfoUrl);
videoInfoPage = Parser.compatParseMap(videoInfoPageString);
playerUrl = getPlayerUrlFromRestrictedVideo(pageUrl);
isAgeRestricted = true;
} else {
ytPlayerConfig = getPlayerConfig(pageContent);
playerArgs = getPlayerArgs(ytPlayerConfig);
playerUrl = getPlayerUrl(ytPlayerConfig);
isAgeRestricted = false;
}
if (decryptionCode.isEmpty()) {
decryptionCode = loadDecryptionCode(playerUrl);
}
}
private JSONObject getPlayerConfig(String pageContent) throws ParsingException {
@Override
public String getId() throws ParsingException {
try {
String ytPlayerConfigRaw =
Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
return new JSONObject(ytPlayerConfigRaw);
} catch (Parser.RegexException e) {
String errorReason = getErrorMessage();
switch(errorReason) {
case "GEMA":
throw new GemaException(errorReason);
case "":
throw new ContentNotAvailableException("Content not available: player config empty", e);
default:
throw new ContentNotAvailableException("Content not available", e);
}
} catch (JSONException e) {
throw new ParsingException("Could not parse yt player config", e);
}
}
private JSONObject getPlayerArgs(JSONObject playerConfig) throws ParsingException {
JSONObject playerArgs;
//attempt to load the youtube js player JSON arguments
boolean isLiveStream = false; //used to determine if this is a livestream or not
try {
playerArgs = playerConfig.getJSONObject("args");
// check if we have a live stream. We need to filter it, since its not yet supported.
if ((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))
|| (playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
isLiveStream = true;
}
} catch (JSONException e) {
throw new ParsingException("Could not parse yt player config", e);
}
if (isLiveStream) {
throw new LiveStreamException("This is a Life stream. Can't use those right now.");
}
return playerArgs;
}
private String getPlayerUrl(JSONObject playerConfig) throws ParsingException {
try {
// The Youtube service needs to be initialized by downloading the
// js-Youtube-player. This is done in order to get the algorithm
// for decrypting cryptic signatures inside certain stream urls.
String playerUrl = "";
JSONObject ytAssets = playerConfig.getJSONObject("assets");
playerUrl = ytAssets.getString("js");
if (playerUrl.startsWith("//")) {
playerUrl = HTTPS + playerUrl;
}
return playerUrl;
} catch (JSONException e) {
throw new ParsingException(
"Could not load decryption code for the Youtube service.", e);
}
}
private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException, ReCaptchaException {
try {
Downloader downloader = NewPipe.getDownloader();
String playerUrl = "";
String videoId = urlidhandler.getId(pageUrl);
String embedUrl = "https://www.youtube.com/embed/" + videoId;
String embedPageContent = downloader.download(embedUrl);
//todo: find out if this can be reapaced by Parser.matchGroup1()
Pattern assetsPattern = Pattern.compile("\"assets\":.+?\"js\":\\s*(\"[^\"]+\")");
Matcher patternMatcher = assetsPattern.matcher(embedPageContent);
while (patternMatcher.find()) {
playerUrl = patternMatcher.group(1);
}
playerUrl = playerUrl.replace("\\", "").replace("\"", "");
if (playerUrl.startsWith("//")) {
playerUrl = HTTPS + playerUrl;
}
return playerUrl;
} catch (IOException e) {
throw new ParsingException(
"Could load decryption code form restricted video for the Youtube service.", e);
} catch (ReCaptchaException e) {
throw new ReCaptchaException("reCaptcha Challenge requested");
return getUrlIdHandler().getId(getUrl());
} catch (Exception e) {
throw new ParsingException("Could not get stream id");
}
}
@ -457,7 +215,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public String getDashMpdUrl() throws ParsingException {
try {
String dashManifestUrl = "";
String dashManifestUrl;
if (videoInfoPage != null && videoInfoPage.containsKey("dashmpd")) {
dashManifestUrl = videoInfoPage.get("dashmpd");
} else if (playerArgs.has("dashmpd")) {
@ -479,7 +237,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
@Override
public List<AudioStream> getAudioStreams() throws ParsingException {
Vector<AudioStream> audioStreams = new Vector<>();
@ -507,9 +264,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
int itag = Integer.parseInt(tags.get("itag"));
if (itagIsSupported(itag)) {
ItagItem itagItem = getItagItem(itag);
if (itagItem.itagType == ItagType.AUDIO) {
if (ItagItem.isSupported(itag)) {
ItagItem itagItem = ItagItem.getItag(itag);
if (itagItem.itagType == ItagItem.ItagType.AUDIO) {
String streamUrl = tags.get("url");
// if video has a signature: decrypt it and add it to the url
if (tags.get("s") != null) {
@ -517,11 +274,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
+ decryptSignature(tags.get("s"), decryptionCode);
}
audioStreams.add(new AudioStream(streamUrl,
itagItem.mediaFormatId,
itagItem.avgBitrate,
itagItem.bandWidth,
itagItem.samplingRate));
AudioStream audioStream = new AudioStream(streamUrl, itagItem.mediaFormatId, itagItem.avgBitrate);
if (!Stream.containSimilarStream(audioStream, audioStreams)) {
audioStreams.add(audioStream);
}
}
}
}
@ -552,19 +308,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
int itag = Integer.parseInt(tags.get("itag"));
if (itagIsSupported(itag)) {
ItagItem itagItem = getItagItem(itag);
if (itagItem.itagType == ItagType.VIDEO) {
if (ItagItem.isSupported(itag)) {
ItagItem itagItem = ItagItem.getItag(itag);
if (itagItem.itagType == ItagItem.ItagType.VIDEO) {
String streamUrl = tags.get("url");
// if video has a signature: decrypt it and add it to the url
if (tags.get("s") != null) {
streamUrl = streamUrl + "&signature="
+ decryptSignature(tags.get("s"), decryptionCode);
}
videoStreams.add(new VideoStream(
streamUrl,
itagItem.mediaFormatId,
itagItem.resolutionString));
VideoStream videoStream = new VideoStream(streamUrl, itagItem.mediaFormatId, itagItem.resolutionString);
if (!Stream.containSimilarStream(videoStream, videoStreams)) {
videoStreams.add(videoStream);
}
}
}
} catch (Exception e) {
@ -612,9 +369,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
int itag = Integer.parseInt(tags.get("itag"));
if (itagIsSupported(itag)) {
ItagItem itagItem = getItagItem(itag);
if (itagItem.itagType == ItagType.VIDEO_ONLY) {
if (ItagItem.isSupported(itag)) {
ItagItem itagItem = ItagItem.getItag(itag);
if (itagItem.itagType == ItagItem.ItagType.VIDEO_ONLY) {
String streamUrl = tags.get("url");
// if video has a signature: decrypt it and add it to the url
if (tags.get("s") != null) {
@ -622,11 +379,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
+ decryptSignature(tags.get("s"), decryptionCode);
}
videoOnlyStreams.add(new VideoStream(
true, //isVideoOnly
streamUrl,
itagItem.mediaFormatId,
itagItem.resolutionString));
VideoStream videoStream = new VideoStream(streamUrl, itagItem.mediaFormatId, itagItem.resolutionString, true);
if (!Stream.containSimilarStream(videoStream, videoOnlyStreams)) {
videoOnlyStreams.add(videoStream);
}
}
}
}
@ -649,7 +405,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public int getTimeStamp() throws ParsingException {
String timeStamp;
try {
timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl);
timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", dirtyUrl);
} catch (Parser.RegexException e) {
// catch this instantly since an url does not necessarily have to have a time stamp
@ -730,7 +486,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
//if this ckicks in our button has no content and thefore likes/dislikes are disabled
return -1;
}
return Integer.parseInt(likesString.replaceAll(REGEX_INT, ""));
return Integer.parseInt(Utils.removeNonDigitCharacters(likesString));
} catch (NumberFormatException nfe) {
throw new ParsingException(
"failed to parse likesString \"" + likesString + "\" as integers", nfe);
@ -750,7 +506,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
//if this kicks in our button has no content and therefore likes/dislikes are disabled
return -1;
}
return Integer.parseInt(dislikesString.replaceAll(REGEX_INT, ""));
return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString));
} catch (NumberFormatException nfe) {
throw new ParsingException(
"failed to parse dislikesString \"" + dislikesString + "\" as integers", nfe);
@ -788,11 +544,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
@Override
public String getPageUrl() {
return pageUrl;
}
@Override
public String getChannelUrl() throws ParsingException {
try {
@ -804,92 +555,186 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
@Override
public StreamInfo.StreamType getStreamType() throws ParsingException {
public StreamType getStreamType() throws ParsingException {
//todo: if implementing livestream support this value should be generated dynamically
return StreamInfo.StreamType.VIDEO_STREAM;
return StreamType.VIDEO_STREAM;
}
/**
* Provides information about links to other videos on the video page, such as related videos.
* This is encapsulated in a StreamInfoItem object,
* which is a subset of the fields in a full StreamInfo.
* {@inheritDoc}
*/
private StreamInfoItemExtractor extractVideoPreviewInfo(final Element li) {
return new StreamInfoItemExtractor() {
@Override
public AbstractStreamInfo.StreamType getStreamType() throws ParsingException {
return AbstractStreamInfo.StreamType.VIDEO_STREAM;
}
@Override
public String getErrorMessage() {
String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text();
StringBuilder errorReason;
@Override
public boolean isAd() throws ParsingException {
return !li.select("span[class*=\"icon-not-available\"]").isEmpty();
}
if (errorMessage == null || errorMessage.isEmpty()) {
errorReason = null;
} else if (errorMessage.contains("GEMA")) {
// Gema sometimes blocks youtube music content in germany:
// https://www.gema.de/en/
// Detailed description:
// https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
errorReason = new StringBuilder("GEMA");
} else {
errorReason = new StringBuilder(errorMessage);
errorReason.append(" ");
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
}
@Override
public String getWebPageUrl() throws ParsingException {
return li.select("a.content-link").first().attr("abs:href");
}
@Override
public String getTitle() throws ParsingException {
//todo: check NullPointerException causing
return li.select("span.title").first().text();
//this page causes the NullPointerException, after finding it by searching for "tjvg":
//https://www.youtube.com/watch?v=Uqg0aEhLFAg
}
@Override
public int getDuration() throws ParsingException {
return YoutubeParsingHelper.parseDurationString(
li.select("span.video-time").first().text());
}
@Override
public String getUploader() throws ParsingException {
return li.select("span.g-hovercard").first().text();
}
@Override
public String getUploadDate() throws ParsingException {
return null;
}
@Override
public long getViewCount() throws ParsingException {
//this line is unused
//String views = li.select("span.view-count").first().text();
//Log.i(TAG, "title:"+info.title);
//Log.i(TAG, "view count:"+views);
try {
return Long.parseLong(li.select("span.view-count")
.first().text().replaceAll(REGEX_INT, ""));
} catch (Exception e) {
//related videos sometimes have no view count
return 0;
}
}
@Override
public String getThumbnailUrl() throws ParsingException {
Element img = li.select("img").first();
String thumbnailUrl = img.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we caught such an item.
if (thumbnailUrl.contains(".gif")) {
thumbnailUrl = img.attr("data-thumb");
}
if (thumbnailUrl.startsWith("//")) {
thumbnailUrl = HTTPS + thumbnailUrl;
}
return thumbnailUrl;
}
};
return errorReason != null ? errorReason.toString() : null;
}
/*//////////////////////////////////////////////////////////////////////////
// Utils
//////////////////////////////////////////////////////////////////////////*/
private JSONObject playerArgs;
private boolean isAgeRestricted;
private Map<String, String> videoInfoPage;
private static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map";
private static final String HTTPS = "https:";
private static final String CONTENT = "content";
/**
* Sometimes if the html page of youtube is already downloaded, youtube web page will internally
* download the /get_video_info page. Since a certain date dashmpd url is only available over
* this /get_video_info page, so we always need to download this one to.
* <p>
* %%video_id%% will be replaced by the actual video id
* $$el_type$$ will be replaced by the actual el_type (se the declarations below)
*/
private static final String GET_VIDEO_INFO_URL =
"https://www.youtube.com/get_video_info?video_id=%%video_id%%$$el_type$$&ps=default&eurl=&gl=US&hl=en";
// eltype is necessary for the url above
private static final String EL_INFO = "el=info";
// static values
private static final String DECRYPTION_FUNC_NAME = "decrypt";
// cached values
private static volatile String decryptionCode = "";
private void fetchDocument() throws IOException, ReCaptchaException, ParsingException {
Downloader downloader = NewPipe.getDownloader();
String pageContent = downloader.download(getUrl());
doc = Jsoup.parse(pageContent, getUrl());
JSONObject ytPlayerConfig;
String playerUrl;
String videoInfoUrl = GET_VIDEO_INFO_URL.replace("%%video_id%%", getId()).replace("$$el_type$$", "&" + EL_INFO);
String videoInfoPageString = downloader.download(videoInfoUrl);
videoInfoPage = Parser.compatParseMap(videoInfoPageString);
// Check if the video is age restricted
if (pageContent.contains("<meta property=\"og:restrictions:age")) {
playerUrl = getPlayerUrlFromRestrictedVideo(getUrl());
isAgeRestricted = true;
} else {
ytPlayerConfig = getPlayerConfig(pageContent);
playerArgs = getPlayerArgs(ytPlayerConfig);
playerUrl = getPlayerUrl(ytPlayerConfig);
isAgeRestricted = false;
}
if (decryptionCode.isEmpty()) {
decryptionCode = loadDecryptionCode(playerUrl);
}
}
private JSONObject getPlayerConfig(String pageContent) throws ParsingException {
try {
String ytPlayerConfigRaw =
Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
return new JSONObject(ytPlayerConfigRaw);
} catch (Parser.RegexException e) {
String errorReason = getErrorMessage();
switch (errorReason) {
case "GEMA":
throw new GemaException(errorReason);
case "":
throw new ContentNotAvailableException("Content not available: player config empty", e);
default:
throw new ContentNotAvailableException("Content not available", e);
}
} catch (JSONException e) {
throw new ParsingException("Could not parse yt player config", e);
}
}
private JSONObject getPlayerArgs(JSONObject playerConfig) throws ParsingException {
JSONObject playerArgs;
//attempt to load the youtube js player JSON arguments
boolean isLiveStream = false; //used to determine if this is a livestream or not
try {
playerArgs = playerConfig.getJSONObject("args");
// check if we have a live stream. We need to filter it, since its not yet supported.
if ((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))
|| (playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
isLiveStream = true;
}
} catch (JSONException e) {
throw new ParsingException("Could not parse yt player config", e);
}
if (isLiveStream) {
throw new LiveStreamException("This is a Life stream. Can't use those right now.");
}
return playerArgs;
}
private String getPlayerUrl(JSONObject playerConfig) throws ParsingException {
try {
// The Youtube service needs to be initialized by downloading the
// js-Youtube-player. This is done in order to get the algorithm
// for decrypting cryptic signatures inside certain stream urls.
String playerUrl;
JSONObject ytAssets = playerConfig.getJSONObject("assets");
playerUrl = ytAssets.getString("js");
if (playerUrl.startsWith("//")) {
playerUrl = HTTPS + playerUrl;
}
return playerUrl;
} catch (JSONException e) {
throw new ParsingException(
"Could not load decryption code for the Youtube service.", e);
}
}
private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException, ReCaptchaException {
try {
Downloader downloader = NewPipe.getDownloader();
String playerUrl = "";
String videoId = getUrlIdHandler().getId(pageUrl);
String embedUrl = "https://www.youtube.com/embed/" + videoId;
String embedPageContent = downloader.download(embedUrl);
//todo: find out if this can be reapaced by Parser.matchGroup1()
Pattern assetsPattern = Pattern.compile("\"assets\":.+?\"js\":\\s*(\"[^\"]+\")");
Matcher patternMatcher = assetsPattern.matcher(embedPageContent);
while (patternMatcher.find()) {
playerUrl = patternMatcher.group(1);
}
playerUrl = playerUrl.replace("\\", "").replace("\"", "");
if (playerUrl.startsWith("//")) {
playerUrl = HTTPS + playerUrl;
}
return playerUrl;
} catch (IOException e) {
throw new ParsingException(
"Could load decryption code form restricted video for the Youtube service.", e);
} catch (ReCaptchaException e) {
throw new ReCaptchaException("reCaptcha Challenge requested");
}
}
private String loadDecryptionCode(String playerUrl) throws DecryptException {
String decryptionFuncName;
@ -935,8 +780,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return decryptionCode;
}
private String decryptSignature(String encryptedSig, String decryptionCode)
throws DecryptException {
private String decryptSignature(String encryptedSig, String decryptionCode) throws DecryptException {
Context context = Context.enter();
context.setOptimizationLevel(-1);
Object result = null;
@ -953,28 +797,84 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return result == null ? "" : result.toString();
}
/**
* {@inheritDoc}
* Provides information about links to other videos on the video page, such as related videos.
* This is encapsulated in a StreamInfoItem object,
* which is a subset of the fields in a full StreamInfo.
*/
public String getErrorMessage() {
String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text();
StringBuilder errorReason;
private StreamInfoItemExtractor extractVideoPreviewInfo(final Element li) {
return new StreamInfoItemExtractor() {
@Override
public StreamType getStreamType() throws ParsingException {
return StreamType.VIDEO_STREAM;
}
if (errorMessage == null || errorMessage.isEmpty()) {
errorReason = null;
} else if(errorMessage.contains("GEMA")) {
// Gema sometimes blocks youtube music content in germany:
// https://www.gema.de/en/
// Detailed description:
// https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
errorReason = new StringBuilder("GEMA");
} else {
errorReason = new StringBuilder(errorMessage);
errorReason.append(" ");
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
}
@Override
public boolean isAd() throws ParsingException {
return !li.select("span[class*=\"icon-not-available\"]").isEmpty();
}
return errorReason != null ? errorReason.toString() : null;
@Override
public String getWebPageUrl() throws ParsingException {
return li.select("a.content-link").first().attr("abs:href");
}
@Override
public String getTitle() throws ParsingException {
//todo: check NullPointerException causing
return li.select("span.title").first().text();
//this page causes the NullPointerException, after finding it by searching for "tjvg":
//https://www.youtube.com/watch?v=Uqg0aEhLFAg
}
@Override
public int getDuration() throws ParsingException {
return YoutubeParsingHelper.parseDurationString(
li.select("span.video-time").first().text());
}
@Override
public String getUploader() throws ParsingException {
return li.select("span.g-hovercard").first().text();
}
@Override
public String getUploadDate() throws ParsingException {
return null;
}
@Override
public long getViewCount() throws ParsingException {
//this line is unused
//String views = li.select("span.view-count").first().text();
//Log.i(TAG, "title:"+info.title);
//Log.i(TAG, "view count:"+views);
try {
return Long.parseLong(Utils.removeNonDigitCharacters(
li.select("span.view-count").first().text()));
} catch (Exception e) {
//related videos sometimes have no view count
return 0;
}
}
@Override
public String getThumbnailUrl() throws ParsingException {
Element img = li.select("img").first();
String thumbnailUrl = img.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we caught such an item.
if (thumbnailUrl.contains(".gif")) {
thumbnailUrl = img.attr("data-thumb");
}
if (thumbnailUrl.startsWith("//")) {
thumbnailUrl = HTTPS + thumbnailUrl;
}
return thumbnailUrl;
}
};
}
}

11
services/youtube/YoutubeStreamInfoItemExtractor.java
View File

@ -3,8 +3,9 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.AbstractStreamInfo;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Utils;
/*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
@ -116,7 +117,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
}
}
output = input.replaceAll("[^0-9]+", "");
output = Utils.removeNonDigitCharacters(input);
try {
return Long.parseLong(output);
@ -150,11 +151,11 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
}
@Override
public AbstractStreamInfo.StreamType getStreamType() {
public StreamType getStreamType() {
if (isLiveStream(item)) {
return AbstractStreamInfo.StreamType.LIVE_STREAM;
return StreamType.LIVE_STREAM;
} else {
return AbstractStreamInfo.StreamType.VIDEO_STREAM;
return StreamType.VIDEO_STREAM;
}
}

1
services/youtube/YoutubeStreamUrlIdHandler.java
View File

@ -140,6 +140,7 @@ public class YoutubeStreamUrlIdHandler implements UrlIdHandler {
return Parser.matchGroup1("ci=" + ID_PATTERN, uri.getQuery());
}
@Override
public String cleanUrl(String complexUrl) throws ParsingException {
return getUrl(getId(complexUrl));
}

41
stream/AbstractStreamInfo.java
View File

@ -1,41 +0,0 @@
package org.schabi.newpipe.extractor.stream;
/*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* AbstractStreamInfo.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import org.schabi.newpipe.extractor.Info;
/**
* Common properties between StreamInfo and StreamInfoItem.
*/
public abstract class AbstractStreamInfo extends Info {
public enum StreamType {
NONE, // placeholder to check if stream type was checked or not
VIDEO_STREAM,
AUDIO_STREAM,
LIVE_STREAM,
AUDIO_LIVE_STREAM,
FILE
}
public StreamType stream_type;
public String uploader = "";
public String thumbnail_url = "";
public String upload_date = "";
public long view_count = -1;
}

34
stream/AudioStream.java
View File

@ -1,7 +1,5 @@
package org.schabi.newpipe.extractor.stream;
import java.io.Serializable;
/*
* Created by Christian Schabesberger on 04.03.16.
*
@ -22,31 +20,17 @@ import java.io.Serializable;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class AudioStream implements Serializable {
public String url = "";
public int format = -1;
public int bandwidth = -1;
public int sampling_rate = -1;
public int avgBitrate = -1;
public class AudioStream extends Stream {
public int average_bitrate = -1;
public AudioStream(String url, int format, int avgBitrate, int bandwidth, int samplingRate) {
this.url = url;
this.format = format;
this.avgBitrate = avgBitrate;
this.bandwidth = bandwidth;
this.sampling_rate = samplingRate;
public AudioStream(String url, int format, int averageBitrate) {
super(url, format);
this.average_bitrate = averageBitrate;
}
// reveals whether two streams are the same, but have different urls
public boolean equalStats(AudioStream cmp) {
return format == cmp.format
&& bandwidth == cmp.bandwidth
&& sampling_rate == cmp.sampling_rate
&& avgBitrate == cmp.avgBitrate;
}
// reveals whether two streams are equal
public boolean equals(AudioStream cmp) {
return cmp != null && equalStats(cmp) && url.equals(cmp.url);
@Override
public boolean equalStats(Stream cmp) {
return super.equalStats(cmp) && cmp instanceof AudioStream &&
average_bitrate == ((AudioStream) cmp).average_bitrate;
}
}

39
stream/Stream.java Normal file
View File

@ -0,0 +1,39 @@
package org.schabi.newpipe.extractor.stream;
import java.io.Serializable;
import java.util.List;
public abstract class Stream implements Serializable {
public String url;
public int format = -1;
public Stream(String url, int format) {
this.url = url;
this.format = format;
}
/**
* Reveals whether two streams are the same, but have different urls
*/
public boolean equalStats(Stream cmp) {
return cmp != null && format == cmp.format;
}
/**
* Reveals whether two Streams are equal
*/
public boolean equals(Stream cmp) {
return equalStats(cmp) && url.equals(cmp.url);
}
/**
* Check if the list already contains one stream with equals stats
*/
public static boolean containSimilarStream(Stream stream, List<? extends Stream> streamList) {
if (stream == null || streamList == null) return false;
for (Stream cmpStream : streamList) {
if (stream.equalStats(cmpStream)) return true;
}
return false;
}
}

14
stream/StreamExtractor.java
View File

@ -31,20 +31,11 @@ import java.util.List;
*/
public abstract class StreamExtractor extends Extractor {
public static class ContentNotAvailableException extends ParsingException {
public ContentNotAvailableException(String message) {
super(message);
}
public ContentNotAvailableException(String message, Throwable cause) {
super(message, cause);
}
}
public StreamExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) {
super(urlIdHandler, serviceId, url);
}
public abstract String getId() throws ParsingException;
public abstract int getTimeStamp() throws ParsingException;
public abstract String getTitle() throws ParsingException;
public abstract String getDescription() throws ParsingException;
@ -65,8 +56,7 @@ public abstract class StreamExtractor extends Extractor {
public abstract int getDislikeCount() throws ParsingException;
public abstract StreamInfoItemExtractor getNextVideo() throws ParsingException;
public abstract StreamInfoItemCollector getRelatedVideos() throws ParsingException;
public abstract String getPageUrl();
public abstract StreamInfo.StreamType getStreamType() throws ParsingException;
public abstract StreamType getStreamType() throws ParsingException;
/**
* Analyses the webpage's document and extracts any error message there might be.

146
stream/StreamInfo.java
View File

@ -1,7 +1,8 @@
package org.schabi.newpipe.extractor.stream;
import org.schabi.newpipe.extractor.Info;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.utils.DashMpdParser;
@ -31,11 +32,11 @@ import java.util.Vector;
/**
* Info object for opened videos, ie the video ready to play.
*/
@SuppressWarnings("ALL")
public class StreamInfo extends AbstractStreamInfo {
@SuppressWarnings("WeakerAccess")
public class StreamInfo extends Info {
public static class StreamExctractException extends ExtractionException {
StreamExctractException(String message) {
public static class StreamExtractException extends ExtractionException {
StreamExtractException(String message) {
super(message);
}
}
@ -43,43 +44,11 @@ public class StreamInfo extends AbstractStreamInfo {
public StreamInfo() {
}
/**
* Creates a new StreamInfo object from an existing AbstractVideoInfo.
* All the shared properties are copied to the new StreamInfo.
*/
@SuppressWarnings("WeakerAccess")
public StreamInfo(AbstractStreamInfo avi) {
this.id = avi.id;
this.url = avi.url;
this.name = avi.name;
this.uploader = avi.uploader;
this.thumbnail_url = avi.thumbnail_url;
this.upload_date = avi.upload_date;
this.upload_date = avi.upload_date;
this.view_count = avi.view_count;
//todo: better than this
if (avi instanceof StreamInfoItem) {
//shitty String to convert code
/*
String dur = ((StreamInfoItem)avi).duration;
int minutes = Integer.parseInt(dur.substring(0, dur.indexOf(":")));
int seconds = Integer.parseInt(dur.substring(dur.indexOf(":")+1, dur.length()));
*/
this.duration = ((StreamInfoItem) avi).duration;
}
}
public void addException(Exception e) {
errors.add(e);
}
/**
* Fills out the video info fields which are common to all services.
* Probably needs to be overridden by subclasses
*/
public static StreamInfo getVideoInfo(StreamExtractor extractor)
throws ExtractionException, StreamExtractor.ContentNotAvailableException {
public static StreamInfo getVideoInfo(StreamExtractor extractor) throws ExtractionException {
StreamInfo streamInfo = new StreamInfo();
try {
@ -87,15 +56,15 @@ public class StreamInfo extends AbstractStreamInfo {
streamInfo = extractStreams(streamInfo, extractor);
streamInfo = extractOptionalData(streamInfo, extractor);
} catch (ExtractionException e) {
// Currently YouTube does not distinguish between age restricted videos and videos blocked
// by country. This means that during the initialisation of the extractor, the extractor
// will assume that a video is age restricted while in reality it it blocked by country.
//
// We will now detect whether the video is blocked by country or not.
// Currently YouTube does not distinguish between age restricted videos and videos blocked
// by country. This means that during the initialisation of the extractor, the extractor
// will assume that a video is age restricted while in reality it it blocked by country.
//
// We will now detect whether the video is blocked by country or not.
String errorMsg = extractor.getErrorMessage();
if (errorMsg != null) {
throw new StreamExtractor.ContentNotAvailableException(errorMsg);
throw new ContentNotAvailableException(errorMsg);
} else {
throw e;
}
@ -104,18 +73,14 @@ public class StreamInfo extends AbstractStreamInfo {
return streamInfo;
}
private static StreamInfo extractImportantData(
StreamInfo streamInfo, StreamExtractor extractor)
throws ExtractionException {
private static StreamInfo extractImportantData(StreamInfo streamInfo, StreamExtractor extractor) throws ExtractionException {
/* ---- important data, withoug the video can't be displayed goes here: ---- */
// if one of these is not available an exception is meant to be thrown directly into the frontend.
UrlIdHandler uiconv = extractor.getUrlIdHandler();
streamInfo.service_id = extractor.getServiceId();
streamInfo.url = extractor.getPageUrl();
streamInfo.url = extractor.getUrl();
streamInfo.stream_type = extractor.getStreamType();
streamInfo.id = uiconv.getId(extractor.getPageUrl());
streamInfo.id = extractor.getId();
streamInfo.name = extractor.getTitle();
streamInfo.age_limit = extractor.getAgeLimit();
@ -130,9 +95,7 @@ public class StreamInfo extends AbstractStreamInfo {
return streamInfo;
}
private static StreamInfo extractStreams(
StreamInfo streamInfo, StreamExtractor extractor)
throws ExtractionException {
private static StreamInfo extractStreams(StreamInfo streamInfo, StreamExtractor extractor) throws ExtractionException {
/* ---- stream extraction goes here ---- */
// At least one type of stream has to be available,
// otherwise an exception will be thrown directly into the frontend.
@ -149,34 +112,33 @@ public class StreamInfo extends AbstractStreamInfo {
} catch (Exception e) {
streamInfo.addException(new ExtractionException("Couldn't get audio streams", e));
}
// also try to get streams from the dashMpd
if (streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) {
if (streamInfo.audio_streams == null) {
streamInfo.audio_streams = new Vector<>();
}
//todo: make this quick and dirty solution a real fallback
// same as the quick and dirty above
try {
streamInfo.audio_streams.addAll(
DashMpdParser.getAudioStreams(streamInfo.dashMpdUrl));
} catch (Exception e) {
streamInfo.addException(
new ExtractionException("Couldn't get audio streams from dash mpd", e));
}
}
/* Extract video stream url*/
try {
streamInfo.video_streams = extractor.getVideoStreams();
} catch (Exception e) {
streamInfo.addException(
new ExtractionException("Couldn't get video streams", e));
streamInfo.addException(new ExtractionException("Couldn't get video streams", e));
}
/* Extract video only stream url*/
try {
streamInfo.video_only_streams = extractor.getVideoOnlyStreams();
} catch (Exception e) {
streamInfo.addException(
new ExtractionException("Couldn't get video only streams", e));
streamInfo.addException(new ExtractionException("Couldn't get video only streams", e));
}
// Lists can be null if a exception was thrown during extraction
if (streamInfo.video_streams == null) streamInfo.video_streams = new Vector<>();
if (streamInfo.video_only_streams == null) streamInfo.video_only_streams = new Vector<>();
if (streamInfo.audio_streams == null) streamInfo.audio_streams = new Vector<>();
if (streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) {
try {
// Will try to find in the dash manifest for any stream that the ItagItem has (by the id),
// it has video, video only and audio streams and will only add to the list if it don't
// find a similar stream in the respective lists (calling Stream#equalStats).
DashMpdParser.getStreams(streamInfo);
} catch (Exception e) {
streamInfo.addException(new ExtractionException("Couldn't get streams from dash mpd", e));
}
}
// either dash_mpd audio_only or video has to be available, otherwise we didn't get a stream,
@ -184,15 +146,14 @@ public class StreamInfo extends AbstractStreamInfo {
if ((streamInfo.video_streams == null || streamInfo.video_streams.isEmpty())
&& (streamInfo.audio_streams == null || streamInfo.audio_streams.isEmpty())
&& (streamInfo.dashMpdUrl == null || streamInfo.dashMpdUrl.isEmpty())) {
throw new StreamExctractException(
throw new StreamExtractException(
"Could not get any stream. See error variable to get further details.");
}
return streamInfo;
}
private static StreamInfo extractOptionalData(
StreamInfo streamInfo, StreamExtractor extractor) {
private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtractor extractor) {
/* ---- optional data goes here: ---- */
// If one of these fails, the frontend needs to handle that they are not available.
// Exceptions are therefore not thrown into the frontend, but stored into the error List,
@ -259,8 +220,7 @@ public class StreamInfo extends AbstractStreamInfo {
streamInfo.addException(e);
}
try {
StreamInfoItemCollector c = new StreamInfoItemCollector(
extractor.getUrlIdHandler(), extractor.getServiceId());
StreamInfoItemCollector c = new StreamInfoItemCollector(extractor.getServiceId());
StreamInfoItemExtractor nextVideo = extractor.getNextVideo();
c.commit(nextVideo);
if (c.getItemList().size() != 0) {
@ -282,26 +242,36 @@ public class StreamInfo extends AbstractStreamInfo {
return streamInfo;
}
public String uploader_thumbnail_url = "";
public String channel_url = "";
public String description = "";
public void addException(Exception e) {
errors.add(e);
}
public List<VideoStream> video_streams = null;
public List<AudioStream> audio_streams = null;
public List<VideoStream> video_only_streams = null;
public StreamType stream_type;
public String uploader;
public String thumbnail_url;
public String upload_date;
public long view_count = -1;
public String uploader_thumbnail_url;
public String channel_url;
public String description;
public List<VideoStream> video_streams;
public List<AudioStream> audio_streams;
public List<VideoStream> video_only_streams;
// video streams provided by the dash mpd do not need to be provided as VideoStream.
// Later on this will also aplly to audio streams. Since dash mpd is standarized,
// crawling such a file is not service dependent. Therefore getting audio only streams by yust
// providing the dash mpd fille will be possible in the future.
public String dashMpdUrl = "";
public String dashMpdUrl;
public int duration = -1;
public int age_limit = -1;
public int like_count = -1;
public int dislike_count = -1;
public String average_rating = "";
public StreamInfoItem next_video = null;
public List<InfoItem> related_streams = null;
public String average_rating;
public StreamInfoItem next_video;
public List<InfoItem> related_streams = new Vector<>();
//in seconds. some metadata is not passed using a StreamInfo object!
public int start_position = 0;
}

20
stream/StreamInfoItem.java
View File

@ -25,18 +25,16 @@ import org.schabi.newpipe.extractor.InfoItem;
/**
* Info object for previews of unopened videos, eg search results, related videos
*/
public class StreamInfoItem extends AbstractStreamInfo implements InfoItem {
public int duration;
public class StreamInfoItem extends InfoItem {
public StreamType stream_type;
public InfoType infoType() {
return InfoType.STREAM;
}
public String uploader;
public String thumbnail_url;
public String upload_date;
public long view_count = -1;
public int duration = -1;
public String getTitle() {
return name;
}
public String getLink() {
return url;
public StreamInfoItem() {
super(InfoType.STREAM);
}
}

19
stream/StreamInfoItemCollector.java
View File

@ -1,8 +1,6 @@
package org.schabi.newpipe.extractor.stream;
import org.schabi.newpipe.extractor.InfoItemCollector;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -28,15 +26,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
public class StreamInfoItemCollector extends InfoItemCollector {
private UrlIdHandler urlIdHandler;
public StreamInfoItemCollector(UrlIdHandler handler, int serviceId) {
public StreamInfoItemCollector(int serviceId) {
super(serviceId);
urlIdHandler = handler;
}
private UrlIdHandler getUrlIdHandler() {
return urlIdHandler;
}
public StreamInfoItem extract(StreamInfoItemExtractor extractor) throws Exception {
@ -48,13 +39,7 @@ public class StreamInfoItemCollector extends InfoItemCollector {
// important information
resultItem.service_id = getServiceId();
resultItem.url = extractor.getWebPageUrl();
if (getUrlIdHandler() == null) {
throw new ParsingException("Error: UrlIdHandler not set");
} else if (!resultItem.url.isEmpty()) {
resultItem.id = NewPipe.getService(getServiceId())
.getStreamUrlIdHandlerInstance()
.getId(resultItem.url);
}
resultItem.name = extractor.getTitle();
resultItem.stream_type = extractor.getStreamType();

2
stream/StreamInfoItemExtractor.java
View File

@ -23,7 +23,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
*/
public interface StreamInfoItemExtractor {
AbstractStreamInfo.StreamType getStreamType() throws ParsingException;
StreamType getStreamType() throws ParsingException;
String getWebPageUrl() throws ParsingException;
String getTitle() throws ParsingException;
int getDuration() throws ParsingException;

10
stream/StreamType.java Normal file
View File

@ -0,0 +1,10 @@
package org.schabi.newpipe.extractor.stream;
public enum StreamType {
NONE, // placeholder to check if stream type was checked or not
VIDEO_STREAM,
AUDIO_STREAM,
LIVE_STREAM,
AUDIO_LIVE_STREAM,
FILE
}

31
stream/VideoStream.java
View File

@ -1,7 +1,5 @@
package org.schabi.newpipe.extractor.stream;
import java.io.Serializable;
/*
* Created by Christian Schabesberger on 04.03.16.
*
@ -22,31 +20,24 @@ import java.io.Serializable;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class VideoStream implements Serializable {
//url of the stream
public String url = "";
public int format = -1;
public String resolution = "";
public boolean isVideoOnly = false;
public class VideoStream extends Stream {
public String resolution;
public boolean isVideoOnly;
public VideoStream(String url, int format, String res) {
this(false, url, format, res);
this(url, format, res, false);
}
public VideoStream(boolean isVideoOnly, String url, int format, String res) {
this.url = url;
this.format = format;
public VideoStream(String url, int format, String res, boolean isVideoOnly) {
super(url, format);
this.resolution = res;
this.isVideoOnly = isVideoOnly;
}
// reveals whether two streams are the same, but have different urls
public boolean equalStats(VideoStream cmp) {
return format == cmp.format && resolution.equals(cmp.resolution);
}
// reveals whether two streams are equal
public boolean equals(VideoStream cmp) {
return cmp != null && equalStats(cmp) && url.equals(cmp.url);
@Override
public boolean equalStats(Stream cmp) {
return super.equalStats(cmp) && cmp instanceof VideoStream &&
resolution.equals(((VideoStream) cmp).resolution) &&
isVideoOnly == ((VideoStream) cmp).isVideoOnly;
}
}

65
utils/DashMpdParser.java
View File

@ -5,7 +5,11 @@ import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamInfo;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
@ -13,8 +17,6 @@ import org.w3c.dom.NodeList;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.Vector;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@ -44,24 +46,25 @@ public class DashMpdParser {
private DashMpdParser() {
}
static class DashMpdParsingException extends ParsingException {
public static class DashMpdParsingException extends ParsingException {
DashMpdParsingException(String message, Exception e) {
super(message, e);
}
}
public static List<AudioStream> getAudioStreams(String dashManifestUrl)
throws DashMpdParsingException, ReCaptchaException {
/**
* Download manifest and return nodelist with elements of tag "AdaptationSet"
*/
public static void getStreams(StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException {
String dashDoc;
Downloader downloader = NewPipe.getDownloader();
try {
dashDoc = downloader.download(dashManifestUrl);
dashDoc = downloader.download(streamInfo.dashMpdUrl);
} catch (IOException ioe) {
throw new DashMpdParsingException("Could not get dash mpd: " + dashManifestUrl, ioe);
throw new DashMpdParsingException("Could not get dash mpd: " + streamInfo.dashMpdUrl, ioe);
} catch (ReCaptchaException e) {
throw new ReCaptchaException("reCaptcha Challenge needed");
}
Vector<AudioStream> audioStreams = new Vector<>();
try {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
@ -69,27 +72,43 @@ public class DashMpdParser {
InputStream stream = new ByteArrayInputStream(dashDoc.getBytes());
Document doc = builder.parse(stream);
NodeList adaptationSetList = doc.getElementsByTagName("AdaptationSet");
for (int i = 0; i < adaptationSetList.getLength(); i++) {
Element adaptationSet = (Element) adaptationSetList.item(i);
String memeType = adaptationSet.getAttribute("mimeType");
if (memeType.contains("audio")) {
Element representation = (Element) adaptationSet.getElementsByTagName("Representation").item(0);
NodeList representationList = doc.getElementsByTagName("Representation");
for (int i = 0; i < representationList.getLength(); i++) {
Element representation = ((Element) representationList.item(i));
try {
String mimeType = ((Element) representation.getParentNode()).getAttribute("mimeType");
String id = representation.getAttribute("id");
String url = representation.getElementsByTagName("BaseURL").item(0).getTextContent();
int bandwidth = Integer.parseInt(representation.getAttribute("bandwidth"));
int samplingRate = Integer.parseInt(representation.getAttribute("audioSamplingRate"));
int format = -1;
if (memeType.equals(MediaFormat.WEBMA.mimeType)) {
format = MediaFormat.WEBMA.id;
} else if (memeType.equals(MediaFormat.M4A.mimeType)) {
format = MediaFormat.M4A.id;
ItagItem itag = ItagItem.getItag(Integer.parseInt(id));
if (itag != null) {
MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType);
int format = mediaFormat != null ? mediaFormat.id : -1;
if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) {
AudioStream audioStream = new AudioStream(url, format, itag.avgBitrate);
if (!Stream.containSimilarStream(audioStream, streamInfo.audio_streams)) {
streamInfo.audio_streams.add(audioStream);
}
} else {
boolean isVideoOnly = itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY);
VideoStream videoStream = new VideoStream(url, format, itag.resolutionString, isVideoOnly);
if (isVideoOnly) {
if (!Stream.containSimilarStream(videoStream, streamInfo.video_only_streams)) {
streamInfo.video_only_streams.add(videoStream);
}
} else if (!Stream.containSimilarStream(videoStream, streamInfo.video_streams)) {
streamInfo.video_streams.add(videoStream);
}
}
}
audioStreams.add(new AudioStream(url, format, 0, bandwidth, samplingRate));
} catch (Exception ignored) {
}
}
} catch (Exception e) {
throw new DashMpdParsingException("Could not parse Dash mpd", e);
}
return audioStreams;
}
}

20
utils/Utils.java Normal file
View File

@ -0,0 +1,20 @@
package org.schabi.newpipe.extractor.utils;
public class Utils {
private Utils() {
//no instance
}
/**
* Remove all non-digit characters from a string.<p>
* Examples:<br/>
* <ul><li>1 234 567 views -> 1234567</li>
* <li>$ 31,133.124 -> 31133124</li></ul>
*
* @param toRemove string to remove non-digit chars
* @return a string that contains only digits
*/
public static String removeNonDigitCharacters(String toRemove) {
return toRemove.replaceAll("\\D+", "");
}
}