videoInfoPage;
+
+ private static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map";
+ private static final String HTTPS = "https:";
+ private static final String CONTENT = "content";
+
+ /**
+ * Sometimes if the html page of youtube is already downloaded, youtube web page will internally
+ * download the /get_video_info page. Since a certain date dashmpd url is only available over
+ * this /get_video_info page, so we always need to download this one to.
+ *
+ * %%video_id%% will be replaced by the actual video id
+ * $$el_type$$ will be replaced by the actual el_type (se the declarations below)
+ */
+ private static final String GET_VIDEO_INFO_URL =
+ "https://www.youtube.com/get_video_info?video_id=%%video_id%%$$el_type$$&ps=default&eurl=&gl=US&hl=en";
+ // eltype is necessary for the url above
+ private static final String EL_INFO = "el=info";
+
+
+ // static values
+ private static final String DECRYPTION_FUNC_NAME = "decrypt";
+
+ // cached values
+ private static volatile String decryptionCode = "";
+
+ private void fetchDocument() throws IOException, ReCaptchaException, ParsingException {
+ Downloader downloader = NewPipe.getDownloader();
+
+ String pageContent = downloader.download(getUrl());
+ doc = Jsoup.parse(pageContent, getUrl());
+
+ JSONObject ytPlayerConfig;
+ String playerUrl;
+
+ String videoInfoUrl = GET_VIDEO_INFO_URL.replace("%%video_id%%", getId()).replace("$$el_type$$", "&" + EL_INFO);
+ String videoInfoPageString = downloader.download(videoInfoUrl);
+ videoInfoPage = Parser.compatParseMap(videoInfoPageString);
+
+ // Check if the video is age restricted
+ if (pageContent.contains("
@@ -116,7 +117,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
}
}
- output = input.replaceAll("[^0-9]+", "");
+ output = Utils.removeNonDigitCharacters(input);
try {
return Long.parseLong(output);
@@ -150,11 +151,11 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
}
@Override
- public AbstractStreamInfo.StreamType getStreamType() {
+ public StreamType getStreamType() {
if (isLiveStream(item)) {
- return AbstractStreamInfo.StreamType.LIVE_STREAM;
+ return StreamType.LIVE_STREAM;
} else {
- return AbstractStreamInfo.StreamType.VIDEO_STREAM;
+ return StreamType.VIDEO_STREAM;
}
}
diff --git a/services/youtube/YoutubeStreamUrlIdHandler.java b/services/youtube/YoutubeStreamUrlIdHandler.java
index a7f55edc5..a6f629043 100644
--- a/services/youtube/YoutubeStreamUrlIdHandler.java
+++ b/services/youtube/YoutubeStreamUrlIdHandler.java
@@ -140,6 +140,7 @@ public class YoutubeStreamUrlIdHandler implements UrlIdHandler {
return Parser.matchGroup1("ci=" + ID_PATTERN, uri.getQuery());
}
+ @Override
public String cleanUrl(String complexUrl) throws ParsingException {
return getUrl(getId(complexUrl));
}
diff --git a/stream/AbstractStreamInfo.java b/stream/AbstractStreamInfo.java
deleted file mode 100644
index d9de9b3b6..000000000
--- a/stream/AbstractStreamInfo.java
+++ /dev/null
@@ -1,41 +0,0 @@
-package org.schabi.newpipe.extractor.stream;
-
-/*
- * Copyright (C) Christian Schabesberger 2016
- * AbstractStreamInfo.java is part of NewPipe.
- *
- * NewPipe is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * NewPipe is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with NewPipe. If not, see .
- */
-
-import org.schabi.newpipe.extractor.Info;
-
-/**
- * Common properties between StreamInfo and StreamInfoItem.
- */
-public abstract class AbstractStreamInfo extends Info {
- public enum StreamType {
- NONE, // placeholder to check if stream type was checked or not
- VIDEO_STREAM,
- AUDIO_STREAM,
- LIVE_STREAM,
- AUDIO_LIVE_STREAM,
- FILE
- }
-
- public StreamType stream_type;
- public String uploader = "";
- public String thumbnail_url = "";
- public String upload_date = "";
- public long view_count = -1;
-}
diff --git a/stream/AudioStream.java b/stream/AudioStream.java
index 5a4224dfe..4633f3b73 100644
--- a/stream/AudioStream.java
+++ b/stream/AudioStream.java
@@ -1,7 +1,5 @@
package org.schabi.newpipe.extractor.stream;
-import java.io.Serializable;
-
/*
* Created by Christian Schabesberger on 04.03.16.
*
@@ -22,31 +20,17 @@ import java.io.Serializable;
* along with NewPipe. If not, see .
*/
-public class AudioStream implements Serializable {
- public String url = "";
- public int format = -1;
- public int bandwidth = -1;
- public int sampling_rate = -1;
- public int avgBitrate = -1;
+public class AudioStream extends Stream {
+ public int average_bitrate = -1;
- public AudioStream(String url, int format, int avgBitrate, int bandwidth, int samplingRate) {
- this.url = url;
- this.format = format;
- this.avgBitrate = avgBitrate;
- this.bandwidth = bandwidth;
- this.sampling_rate = samplingRate;
+ public AudioStream(String url, int format, int averageBitrate) {
+ super(url, format);
+ this.average_bitrate = averageBitrate;
}
- // reveals whether two streams are the same, but have different urls
- public boolean equalStats(AudioStream cmp) {
- return format == cmp.format
- && bandwidth == cmp.bandwidth
- && sampling_rate == cmp.sampling_rate
- && avgBitrate == cmp.avgBitrate;
- }
-
- // reveals whether two streams are equal
- public boolean equals(AudioStream cmp) {
- return cmp != null && equalStats(cmp) && url.equals(cmp.url);
+ @Override
+ public boolean equalStats(Stream cmp) {
+ return super.equalStats(cmp) && cmp instanceof AudioStream &&
+ average_bitrate == ((AudioStream) cmp).average_bitrate;
}
}
diff --git a/stream/Stream.java b/stream/Stream.java
new file mode 100644
index 000000000..c7606cee0
--- /dev/null
+++ b/stream/Stream.java
@@ -0,0 +1,39 @@
+package org.schabi.newpipe.extractor.stream;
+
+import java.io.Serializable;
+import java.util.List;
+
+public abstract class Stream implements Serializable {
+ public String url;
+ public int format = -1;
+
+ public Stream(String url, int format) {
+ this.url = url;
+ this.format = format;
+ }
+
+ /**
+ * Reveals whether two streams are the same, but have different urls
+ */
+ public boolean equalStats(Stream cmp) {
+ return cmp != null && format == cmp.format;
+ }
+
+ /**
+ * Reveals whether two Streams are equal
+ */
+ public boolean equals(Stream cmp) {
+ return equalStats(cmp) && url.equals(cmp.url);
+ }
+
+ /**
+ * Check if the list already contains one stream with equals stats
+ */
+ public static boolean containSimilarStream(Stream stream, List streamList) {
+ if (stream == null || streamList == null) return false;
+ for (Stream cmpStream : streamList) {
+ if (stream.equalStats(cmpStream)) return true;
+ }
+ return false;
+ }
+}
diff --git a/stream/StreamExtractor.java b/stream/StreamExtractor.java
index a35e8110c..308816bf2 100644
--- a/stream/StreamExtractor.java
+++ b/stream/StreamExtractor.java
@@ -31,20 +31,11 @@ import java.util.List;
*/
public abstract class StreamExtractor extends Extractor {
- public static class ContentNotAvailableException extends ParsingException {
- public ContentNotAvailableException(String message) {
- super(message);
- }
-
- public ContentNotAvailableException(String message, Throwable cause) {
- super(message, cause);
- }
- }
-
public StreamExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) {
super(urlIdHandler, serviceId, url);
}
+ public abstract String getId() throws ParsingException;
public abstract int getTimeStamp() throws ParsingException;
public abstract String getTitle() throws ParsingException;
public abstract String getDescription() throws ParsingException;
@@ -65,8 +56,7 @@ public abstract class StreamExtractor extends Extractor {
public abstract int getDislikeCount() throws ParsingException;
public abstract StreamInfoItemExtractor getNextVideo() throws ParsingException;
public abstract StreamInfoItemCollector getRelatedVideos() throws ParsingException;
- public abstract String getPageUrl();
- public abstract StreamInfo.StreamType getStreamType() throws ParsingException;
+ public abstract StreamType getStreamType() throws ParsingException;
/**
* Analyses the webpage's document and extracts any error message there might be.
diff --git a/stream/StreamInfo.java b/stream/StreamInfo.java
index 72d7eb94b..9719d9359 100644
--- a/stream/StreamInfo.java
+++ b/stream/StreamInfo.java
@@ -1,7 +1,8 @@
package org.schabi.newpipe.extractor.stream;
+import org.schabi.newpipe.extractor.Info;
import org.schabi.newpipe.extractor.InfoItem;
-import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.utils.DashMpdParser;
@@ -31,11 +32,11 @@ import java.util.Vector;
/**
* Info object for opened videos, ie the video ready to play.
*/
-@SuppressWarnings("ALL")
-public class StreamInfo extends AbstractStreamInfo {
+@SuppressWarnings("WeakerAccess")
+public class StreamInfo extends Info {
- public static class StreamExctractException extends ExtractionException {
- StreamExctractException(String message) {
+ public static class StreamExtractException extends ExtractionException {
+ StreamExtractException(String message) {
super(message);
}
}
@@ -43,43 +44,11 @@ public class StreamInfo extends AbstractStreamInfo {
public StreamInfo() {
}
- /**
- * Creates a new StreamInfo object from an existing AbstractVideoInfo.
- * All the shared properties are copied to the new StreamInfo.
- */
- @SuppressWarnings("WeakerAccess")
- public StreamInfo(AbstractStreamInfo avi) {
- this.id = avi.id;
- this.url = avi.url;
- this.name = avi.name;
- this.uploader = avi.uploader;
- this.thumbnail_url = avi.thumbnail_url;
- this.upload_date = avi.upload_date;
- this.upload_date = avi.upload_date;
- this.view_count = avi.view_count;
-
- //todo: better than this
- if (avi instanceof StreamInfoItem) {
- //shitty String to convert code
- /*
- String dur = ((StreamInfoItem)avi).duration;
- int minutes = Integer.parseInt(dur.substring(0, dur.indexOf(":")));
- int seconds = Integer.parseInt(dur.substring(dur.indexOf(":")+1, dur.length()));
- */
- this.duration = ((StreamInfoItem) avi).duration;
- }
- }
-
- public void addException(Exception e) {
- errors.add(e);
- }
-
/**
* Fills out the video info fields which are common to all services.
* Probably needs to be overridden by subclasses
*/
- public static StreamInfo getVideoInfo(StreamExtractor extractor)
- throws ExtractionException, StreamExtractor.ContentNotAvailableException {
+ public static StreamInfo getVideoInfo(StreamExtractor extractor) throws ExtractionException {
StreamInfo streamInfo = new StreamInfo();
try {
@@ -87,15 +56,15 @@ public class StreamInfo extends AbstractStreamInfo {
streamInfo = extractStreams(streamInfo, extractor);
streamInfo = extractOptionalData(streamInfo, extractor);
} catch (ExtractionException e) {
- // Currently YouTube does not distinguish between age restricted videos and videos blocked
- // by country. This means that during the initialisation of the extractor, the extractor
- // will assume that a video is age restricted while in reality it it blocked by country.
- //
- // We will now detect whether the video is blocked by country or not.
+ // Currently YouTube does not distinguish between age restricted videos and videos blocked
+ // by country. This means that during the initialisation of the extractor, the extractor
+ // will assume that a video is age restricted while in reality it it blocked by country.
+ //
+ // We will now detect whether the video is blocked by country or not.
String errorMsg = extractor.getErrorMessage();
if (errorMsg != null) {
- throw new StreamExtractor.ContentNotAvailableException(errorMsg);
+ throw new ContentNotAvailableException(errorMsg);
} else {
throw e;
}
@@ -104,18 +73,14 @@ public class StreamInfo extends AbstractStreamInfo {
return streamInfo;
}
- private static StreamInfo extractImportantData(
- StreamInfo streamInfo, StreamExtractor extractor)
- throws ExtractionException {
+ private static StreamInfo extractImportantData(StreamInfo streamInfo, StreamExtractor extractor) throws ExtractionException {
/* ---- important data, withoug the video can't be displayed goes here: ---- */
// if one of these is not available an exception is meant to be thrown directly into the frontend.
- UrlIdHandler uiconv = extractor.getUrlIdHandler();
-
streamInfo.service_id = extractor.getServiceId();
- streamInfo.url = extractor.getPageUrl();
+ streamInfo.url = extractor.getUrl();
streamInfo.stream_type = extractor.getStreamType();
- streamInfo.id = uiconv.getId(extractor.getPageUrl());
+ streamInfo.id = extractor.getId();
streamInfo.name = extractor.getTitle();
streamInfo.age_limit = extractor.getAgeLimit();
@@ -130,9 +95,7 @@ public class StreamInfo extends AbstractStreamInfo {
return streamInfo;
}
- private static StreamInfo extractStreams(
- StreamInfo streamInfo, StreamExtractor extractor)
- throws ExtractionException {
+ private static StreamInfo extractStreams(StreamInfo streamInfo, StreamExtractor extractor) throws ExtractionException {
/* ---- stream extraction goes here ---- */
// At least one type of stream has to be available,
// otherwise an exception will be thrown directly into the frontend.
@@ -149,34 +112,33 @@ public class StreamInfo extends AbstractStreamInfo {
} catch (Exception e) {
streamInfo.addException(new ExtractionException("Couldn't get audio streams", e));
}
- // also try to get streams from the dashMpd
- if (streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) {
- if (streamInfo.audio_streams == null) {
- streamInfo.audio_streams = new Vector<>();
- }
- //todo: make this quick and dirty solution a real fallback
- // same as the quick and dirty above
- try {
- streamInfo.audio_streams.addAll(
- DashMpdParser.getAudioStreams(streamInfo.dashMpdUrl));
- } catch (Exception e) {
- streamInfo.addException(
- new ExtractionException("Couldn't get audio streams from dash mpd", e));
- }
- }
/* Extract video stream url*/
try {
streamInfo.video_streams = extractor.getVideoStreams();
} catch (Exception e) {
- streamInfo.addException(
- new ExtractionException("Couldn't get video streams", e));
+ streamInfo.addException(new ExtractionException("Couldn't get video streams", e));
}
/* Extract video only stream url*/
try {
streamInfo.video_only_streams = extractor.getVideoOnlyStreams();
} catch (Exception e) {
- streamInfo.addException(
- new ExtractionException("Couldn't get video only streams", e));
+ streamInfo.addException(new ExtractionException("Couldn't get video only streams", e));
+ }
+
+ // Lists can be null if a exception was thrown during extraction
+ if (streamInfo.video_streams == null) streamInfo.video_streams = new Vector<>();
+ if (streamInfo.video_only_streams == null) streamInfo.video_only_streams = new Vector<>();
+ if (streamInfo.audio_streams == null) streamInfo.audio_streams = new Vector<>();
+
+ if (streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) {
+ try {
+ // Will try to find in the dash manifest for any stream that the ItagItem has (by the id),
+ // it has video, video only and audio streams and will only add to the list if it don't
+ // find a similar stream in the respective lists (calling Stream#equalStats).
+ DashMpdParser.getStreams(streamInfo);
+ } catch (Exception e) {
+ streamInfo.addException(new ExtractionException("Couldn't get streams from dash mpd", e));
+ }
}
// either dash_mpd audio_only or video has to be available, otherwise we didn't get a stream,
@@ -184,15 +146,14 @@ public class StreamInfo extends AbstractStreamInfo {
if ((streamInfo.video_streams == null || streamInfo.video_streams.isEmpty())
&& (streamInfo.audio_streams == null || streamInfo.audio_streams.isEmpty())
&& (streamInfo.dashMpdUrl == null || streamInfo.dashMpdUrl.isEmpty())) {
- throw new StreamExctractException(
+ throw new StreamExtractException(
"Could not get any stream. See error variable to get further details.");
}
return streamInfo;
}
- private static StreamInfo extractOptionalData(
- StreamInfo streamInfo, StreamExtractor extractor) {
+ private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtractor extractor) {
/* ---- optional data goes here: ---- */
// If one of these fails, the frontend needs to handle that they are not available.
// Exceptions are therefore not thrown into the frontend, but stored into the error List,
@@ -259,8 +220,7 @@ public class StreamInfo extends AbstractStreamInfo {
streamInfo.addException(e);
}
try {
- StreamInfoItemCollector c = new StreamInfoItemCollector(
- extractor.getUrlIdHandler(), extractor.getServiceId());
+ StreamInfoItemCollector c = new StreamInfoItemCollector(extractor.getServiceId());
StreamInfoItemExtractor nextVideo = extractor.getNextVideo();
c.commit(nextVideo);
if (c.getItemList().size() != 0) {
@@ -282,26 +242,36 @@ public class StreamInfo extends AbstractStreamInfo {
return streamInfo;
}
- public String uploader_thumbnail_url = "";
- public String channel_url = "";
- public String description = "";
+ public void addException(Exception e) {
+ errors.add(e);
+ }
- public List video_streams = null;
- public List audio_streams = null;
- public List video_only_streams = null;
+ public StreamType stream_type;
+ public String uploader;
+ public String thumbnail_url;
+ public String upload_date;
+ public long view_count = -1;
+
+ public String uploader_thumbnail_url;
+ public String channel_url;
+ public String description;
+
+ public List video_streams;
+ public List audio_streams;
+ public List video_only_streams;
// video streams provided by the dash mpd do not need to be provided as VideoStream.
// Later on this will also aplly to audio streams. Since dash mpd is standarized,
// crawling such a file is not service dependent. Therefore getting audio only streams by yust
// providing the dash mpd fille will be possible in the future.
- public String dashMpdUrl = "";
+ public String dashMpdUrl;
public int duration = -1;
public int age_limit = -1;
public int like_count = -1;
public int dislike_count = -1;
- public String average_rating = "";
- public StreamInfoItem next_video = null;
- public List related_streams = null;
+ public String average_rating;
+ public StreamInfoItem next_video;
+ public List related_streams = new Vector<>();
//in seconds. some metadata is not passed using a StreamInfo object!
public int start_position = 0;
}
diff --git a/stream/StreamInfoItem.java b/stream/StreamInfoItem.java
index 31318556a..9568cad89 100644
--- a/stream/StreamInfoItem.java
+++ b/stream/StreamInfoItem.java
@@ -25,18 +25,16 @@ import org.schabi.newpipe.extractor.InfoItem;
/**
* Info object for previews of unopened videos, eg search results, related videos
*/
-public class StreamInfoItem extends AbstractStreamInfo implements InfoItem {
- public int duration;
+public class StreamInfoItem extends InfoItem {
+ public StreamType stream_type;
- public InfoType infoType() {
- return InfoType.STREAM;
- }
+ public String uploader;
+ public String thumbnail_url;
+ public String upload_date;
+ public long view_count = -1;
+ public int duration = -1;
- public String getTitle() {
- return name;
- }
-
- public String getLink() {
- return url;
+ public StreamInfoItem() {
+ super(InfoType.STREAM);
}
}
\ No newline at end of file
diff --git a/stream/StreamInfoItemCollector.java b/stream/StreamInfoItemCollector.java
index ced5a1591..12f0f4071 100644
--- a/stream/StreamInfoItemCollector.java
+++ b/stream/StreamInfoItemCollector.java
@@ -1,8 +1,6 @@
package org.schabi.newpipe.extractor.stream;
import org.schabi.newpipe.extractor.InfoItemCollector;
-import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -28,15 +26,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
public class StreamInfoItemCollector extends InfoItemCollector {
- private UrlIdHandler urlIdHandler;
-
- public StreamInfoItemCollector(UrlIdHandler handler, int serviceId) {
+ public StreamInfoItemCollector(int serviceId) {
super(serviceId);
- urlIdHandler = handler;
- }
-
- private UrlIdHandler getUrlIdHandler() {
- return urlIdHandler;
}
public StreamInfoItem extract(StreamInfoItemExtractor extractor) throws Exception {
@@ -48,13 +39,7 @@ public class StreamInfoItemCollector extends InfoItemCollector {
// important information
resultItem.service_id = getServiceId();
resultItem.url = extractor.getWebPageUrl();
- if (getUrlIdHandler() == null) {
- throw new ParsingException("Error: UrlIdHandler not set");
- } else if (!resultItem.url.isEmpty()) {
- resultItem.id = NewPipe.getService(getServiceId())
- .getStreamUrlIdHandlerInstance()
- .getId(resultItem.url);
- }
+
resultItem.name = extractor.getTitle();
resultItem.stream_type = extractor.getStreamType();
diff --git a/stream/StreamInfoItemExtractor.java b/stream/StreamInfoItemExtractor.java
index e6c9519f6..a635d907e 100644
--- a/stream/StreamInfoItemExtractor.java
+++ b/stream/StreamInfoItemExtractor.java
@@ -23,7 +23,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
*/
public interface StreamInfoItemExtractor {
- AbstractStreamInfo.StreamType getStreamType() throws ParsingException;
+ StreamType getStreamType() throws ParsingException;
String getWebPageUrl() throws ParsingException;
String getTitle() throws ParsingException;
int getDuration() throws ParsingException;
diff --git a/stream/StreamType.java b/stream/StreamType.java
new file mode 100644
index 000000000..2d6b9a571
--- /dev/null
+++ b/stream/StreamType.java
@@ -0,0 +1,10 @@
+package org.schabi.newpipe.extractor.stream;
+
+public enum StreamType {
+ NONE, // placeholder to check if stream type was checked or not
+ VIDEO_STREAM,
+ AUDIO_STREAM,
+ LIVE_STREAM,
+ AUDIO_LIVE_STREAM,
+ FILE
+}
diff --git a/stream/VideoStream.java b/stream/VideoStream.java
index c97e7cde0..a696468aa 100644
--- a/stream/VideoStream.java
+++ b/stream/VideoStream.java
@@ -1,7 +1,5 @@
package org.schabi.newpipe.extractor.stream;
-import java.io.Serializable;
-
/*
* Created by Christian Schabesberger on 04.03.16.
*
@@ -22,31 +20,24 @@ import java.io.Serializable;
* along with NewPipe. If not, see .
*/
-public class VideoStream implements Serializable {
- //url of the stream
- public String url = "";
- public int format = -1;
- public String resolution = "";
- public boolean isVideoOnly = false;
+public class VideoStream extends Stream {
+ public String resolution;
+ public boolean isVideoOnly;
public VideoStream(String url, int format, String res) {
- this(false, url, format, res);
+ this(url, format, res, false);
}
- public VideoStream(boolean isVideoOnly, String url, int format, String res) {
- this.url = url;
- this.format = format;
+ public VideoStream(String url, int format, String res, boolean isVideoOnly) {
+ super(url, format);
this.resolution = res;
this.isVideoOnly = isVideoOnly;
}
- // reveals whether two streams are the same, but have different urls
- public boolean equalStats(VideoStream cmp) {
- return format == cmp.format && resolution.equals(cmp.resolution);
- }
-
- // reveals whether two streams are equal
- public boolean equals(VideoStream cmp) {
- return cmp != null && equalStats(cmp) && url.equals(cmp.url);
+ @Override
+ public boolean equalStats(Stream cmp) {
+ return super.equalStats(cmp) && cmp instanceof VideoStream &&
+ resolution.equals(((VideoStream) cmp).resolution) &&
+ isVideoOnly == ((VideoStream) cmp).isVideoOnly;
}
}
diff --git a/utils/DashMpdParser.java b/utils/DashMpdParser.java
index 185ac7271..2fa6b249a 100644
--- a/utils/DashMpdParser.java
+++ b/utils/DashMpdParser.java
@@ -5,7 +5,11 @@ import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
+import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.stream.AudioStream;
+import org.schabi.newpipe.extractor.stream.Stream;
+import org.schabi.newpipe.extractor.stream.StreamInfo;
+import org.schabi.newpipe.extractor.stream.VideoStream;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
@@ -13,8 +17,6 @@ import org.w3c.dom.NodeList;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.util.List;
-import java.util.Vector;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -44,24 +46,25 @@ public class DashMpdParser {
private DashMpdParser() {
}
- static class DashMpdParsingException extends ParsingException {
+ public static class DashMpdParsingException extends ParsingException {
DashMpdParsingException(String message, Exception e) {
super(message, e);
}
}
- public static List getAudioStreams(String dashManifestUrl)
- throws DashMpdParsingException, ReCaptchaException {
+ /**
+ * Download manifest and return nodelist with elements of tag "AdaptationSet"
+ */
+ public static void getStreams(StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException {
String dashDoc;
Downloader downloader = NewPipe.getDownloader();
try {
- dashDoc = downloader.download(dashManifestUrl);
+ dashDoc = downloader.download(streamInfo.dashMpdUrl);
} catch (IOException ioe) {
- throw new DashMpdParsingException("Could not get dash mpd: " + dashManifestUrl, ioe);
+ throw new DashMpdParsingException("Could not get dash mpd: " + streamInfo.dashMpdUrl, ioe);
} catch (ReCaptchaException e) {
throw new ReCaptchaException("reCaptcha Challenge needed");
}
- Vector audioStreams = new Vector<>();
try {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
@@ -69,27 +72,43 @@ public class DashMpdParser {
InputStream stream = new ByteArrayInputStream(dashDoc.getBytes());
Document doc = builder.parse(stream);
- NodeList adaptationSetList = doc.getElementsByTagName("AdaptationSet");
- for (int i = 0; i < adaptationSetList.getLength(); i++) {
- Element adaptationSet = (Element) adaptationSetList.item(i);
- String memeType = adaptationSet.getAttribute("mimeType");
- if (memeType.contains("audio")) {
- Element representation = (Element) adaptationSet.getElementsByTagName("Representation").item(0);
+ NodeList representationList = doc.getElementsByTagName("Representation");
+
+ for (int i = 0; i < representationList.getLength(); i++) {
+ Element representation = ((Element) representationList.item(i));
+ try {
+ String mimeType = ((Element) representation.getParentNode()).getAttribute("mimeType");
+ String id = representation.getAttribute("id");
String url = representation.getElementsByTagName("BaseURL").item(0).getTextContent();
- int bandwidth = Integer.parseInt(representation.getAttribute("bandwidth"));
- int samplingRate = Integer.parseInt(representation.getAttribute("audioSamplingRate"));
- int format = -1;
- if (memeType.equals(MediaFormat.WEBMA.mimeType)) {
- format = MediaFormat.WEBMA.id;
- } else if (memeType.equals(MediaFormat.M4A.mimeType)) {
- format = MediaFormat.M4A.id;
+ ItagItem itag = ItagItem.getItag(Integer.parseInt(id));
+ if (itag != null) {
+ MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType);
+ int format = mediaFormat != null ? mediaFormat.id : -1;
+
+ if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) {
+ AudioStream audioStream = new AudioStream(url, format, itag.avgBitrate);
+
+ if (!Stream.containSimilarStream(audioStream, streamInfo.audio_streams)) {
+ streamInfo.audio_streams.add(audioStream);
+ }
+ } else {
+ boolean isVideoOnly = itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY);
+ VideoStream videoStream = new VideoStream(url, format, itag.resolutionString, isVideoOnly);
+
+ if (isVideoOnly) {
+ if (!Stream.containSimilarStream(videoStream, streamInfo.video_only_streams)) {
+ streamInfo.video_only_streams.add(videoStream);
+ }
+ } else if (!Stream.containSimilarStream(videoStream, streamInfo.video_streams)) {
+ streamInfo.video_streams.add(videoStream);
+ }
+ }
}
- audioStreams.add(new AudioStream(url, format, 0, bandwidth, samplingRate));
+ } catch (Exception ignored) {
}
}
} catch (Exception e) {
throw new DashMpdParsingException("Could not parse Dash mpd", e);
}
- return audioStreams;
}
}
diff --git a/utils/Utils.java b/utils/Utils.java
new file mode 100644
index 000000000..8c4449b11
--- /dev/null
+++ b/utils/Utils.java
@@ -0,0 +1,20 @@
+package org.schabi.newpipe.extractor.utils;
+
+public class Utils {
+ private Utils() {
+ //no instance
+ }
+
+ /**
+ * Remove all non-digit characters from a string.
+ * Examples:
+ *
- 1 234 567 views -> 1234567
+ * - $ 31,133.124 -> 31133124
+ *
+ * @param toRemove string to remove non-digit chars
+ * @return a string that contains only digits
+ */
+ public static String removeNonDigitCharacters(String toRemove) {
+ return toRemove.replaceAll("\\D+", "");
+ }
+}