Abstracted YT Dash manifest creators and made them instantiable
This commit is contained in:
litetex
parent
862ed8762d
commit
1d0a27cd60
|
|
@ -1,56 +0,0 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
/**
|
||||
* Streaming format types used by YouTube in their streams.
|
||||
*
|
||||
* <p>
|
||||
* It is different from {@link org.schabi.newpipe.extractor.stream.DeliveryMethod delivery methods}!
|
||||
* </p>
|
||||
*/
|
||||
// TODO: Kill
|
||||
public enum DeliveryType {
|
||||
|
||||
/**
|
||||
* YouTube's progressive delivery method, which works with HTTP range headers.
|
||||
* (Note that official clients use the corresponding parameter instead.)
|
||||
*
|
||||
* <p>
|
||||
* Initialization and index ranges are available to get metadata (the corresponding values
|
||||
* are returned in the player response).
|
||||
* </p>
|
||||
*/
|
||||
PROGRESSIVE,
|
||||
|
||||
/**
|
||||
* YouTube's OTF delivery method which uses a sequence parameter to get segments of
|
||||
* streams.
|
||||
*
|
||||
* <p>
|
||||
* The first sequence (which can be fetched with the {@code &sq=0} parameter) contains all the
|
||||
* metadata needed to build the stream source (sidx boxes, segment length, segment count,
|
||||
* duration, ...).
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* Only used for videos; mostly those with a small amount of views, or ended livestreams
|
||||
* which have just been re-encoded as normal videos.
|
||||
* </p>
|
||||
*/
|
||||
OTF,
|
||||
|
||||
/**
|
||||
* YouTube's delivery method for livestreams which uses a sequence parameter to get
|
||||
* segments of streams.
|
||||
*
|
||||
* <p>
|
||||
* Each sequence (which can be fetched with the {@code &sq=0} parameter) contains its own
|
||||
* metadata (sidx boxes, segment length, ...), which make no need of an initialization
|
||||
* segment.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* Only used for livestreams (ended or running).
|
||||
* </p>
|
||||
*/
|
||||
LIVE
|
||||
}
|
||||
|
|
@ -0,0 +1,650 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isIosStreamingUrl;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5SimplyEmbeddedPlayerStreamingUrl;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebStreamingUrl;
|
||||
import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.ADAPTATION_SET;
|
||||
import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.AUDIO_CHANNEL_CONFIGURATION;
|
||||
import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.MPD;
|
||||
import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.PERIOD;
|
||||
import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.REPRESENTATION;
|
||||
import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.ROLE;
|
||||
import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.SEGMENT_TEMPLATE;
|
||||
import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.SEGMENT_TIMELINE;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.downloader.Downloader;
|
||||
import org.schabi.newpipe.extractor.downloader.Response;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.services.youtube.itag.format.AudioItagFormat;
|
||||
import org.schabi.newpipe.extractor.services.youtube.itag.format.VideoItagFormat;
|
||||
import org.schabi.newpipe.extractor.services.youtube.itag.info.ItagInfo;
|
||||
import org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreationException;
|
||||
import org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreator;
|
||||
import org.schabi.newpipe.extractor.streamdata.stream.quality.VideoQualityData;
|
||||
import org.w3c.dom.Attr;
|
||||
import org.w3c.dom.DOMException;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.StringWriter;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.xml.XMLConstants;
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
import javax.xml.transform.OutputKeys;
|
||||
import javax.xml.transform.Transformer;
|
||||
import javax.xml.transform.TransformerException;
|
||||
import javax.xml.transform.TransformerFactory;
|
||||
import javax.xml.transform.dom.DOMSource;
|
||||
import javax.xml.transform.stream.StreamResult;
|
||||
|
||||
// TODO: Doc
|
||||
public abstract class AbstractYoutubeDashManifestCreator implements DashManifestCreator {
|
||||
|
||||
/**
|
||||
* The redirect count limit that this class uses, which is the same limit as OkHttp.
|
||||
*/
|
||||
public static final int MAXIMUM_REDIRECT_COUNT = 20;
|
||||
|
||||
/**
|
||||
* URL parameter of the first sequence for live, post-live-DVR and OTF streams.
|
||||
*/
|
||||
public static final String SQ_0 = "&sq=0";
|
||||
|
||||
/**
|
||||
* URL parameter of the first stream request made by official clients.
|
||||
*/
|
||||
public static final String RN_0 = "&rn=0";
|
||||
|
||||
/**
|
||||
* URL parameter specific to web clients. When this param is added, if a redirection occurs,
|
||||
* the server will not redirect clients to the redirect URL. Instead, it will provide this URL
|
||||
* as the response body.
|
||||
*/
|
||||
public static final String ALR_YES = "&alr=yes";
|
||||
|
||||
protected final ItagInfo<?> itagInfo;
|
||||
protected final long durationSecondsFallback;
|
||||
|
||||
protected Document document;
|
||||
|
||||
protected AbstractYoutubeDashManifestCreator(
|
||||
@Nonnull final ItagInfo<?> itagInfo,
|
||||
final long durationSecondsFallback) {
|
||||
this.itagInfo = Objects.requireNonNull(itagInfo);
|
||||
this.durationSecondsFallback = durationSecondsFallback;
|
||||
}
|
||||
|
||||
protected boolean isLiveDelivery() {
|
||||
return false;
|
||||
}
|
||||
|
||||
// region generate manifest elements
|
||||
|
||||
/**
|
||||
* Generate a {@link Document} with common manifest creator elements added to it.
|
||||
*
|
||||
* <p>
|
||||
* Those are:
|
||||
* <ul>
|
||||
* <li>{@code MPD} (using {@link #generateDocumentAndMpdElement(long)});</li>
|
||||
* <li>{@code Period} (using {@link #generatePeriodElement()});</li>
|
||||
* <li>{@code AdaptationSet} (using {@link #generateAdaptationSetElement()});</li>
|
||||
* <li>{@code Role} (using {@link #generateRoleElement()});</li>
|
||||
* <li>{@code Representation} (using {@link #generateRepresentationElement()});</li>
|
||||
* <li>and, for audio streams, {@code AudioChannelConfiguration} (using
|
||||
* {@link #generateAudioChannelConfigurationElement()}).</li>
|
||||
* </ul>
|
||||
* </p>
|
||||
*
|
||||
* @param streamDurationMs the duration of the stream, in milliseconds
|
||||
* @throws DashManifestCreationException May throw a CreationException
|
||||
*/
|
||||
protected void generateDocumentAndCommonElements(final long streamDurationMs) {
|
||||
generateDocumentAndMpdElement(streamDurationMs);
|
||||
|
||||
generatePeriodElement();
|
||||
generateAdaptationSetElement();
|
||||
generateRoleElement();
|
||||
generateRepresentationElement();
|
||||
if (itagInfo.getItagFormat() instanceof AudioItagFormat) {
|
||||
generateAudioChannelConfigurationElement();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a {@link Document} instance and generate the {@code <MPD>} element of the manifest.
|
||||
*
|
||||
* <p>
|
||||
* The generated {@code <MPD>} element looks like the manifest returned into the player
|
||||
* response of videos:
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* {@code <MPD xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
* xmlns="urn:mpeg:DASH:schema:MPD:2011"
|
||||
* xsi:schemaLocation="urn:mpeg:DASH:schema:MPD:2011 DASH-MPD.xsd" minBufferTime="PT1.500S"
|
||||
* profiles="urn:mpeg:dash:profile:isoff-main:2011" type="static"
|
||||
* mediaPresentationDuration="PT$duration$S">}
|
||||
* (where {@code $duration$} represents the duration in seconds (a number with 3 digits after
|
||||
* the decimal point)).
|
||||
* </p>
|
||||
*
|
||||
* @param durationMs the duration of the stream, in milliseconds
|
||||
* @throws DashManifestCreationException May throw a CreationException
|
||||
*/
|
||||
protected void generateDocumentAndMpdElement(final long durationMs) {
|
||||
try {
|
||||
newDocument();
|
||||
|
||||
final Element mpdElement = createElement(MPD);
|
||||
document.appendChild(mpdElement);
|
||||
|
||||
appendNewAttrWithValue(
|
||||
mpdElement, "xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
|
||||
|
||||
appendNewAttrWithValue(mpdElement, "xmlns", "urn:mpeg:DASH:schema:MPD:2011");
|
||||
|
||||
appendNewAttrWithValue(
|
||||
mpdElement,
|
||||
"xsi:schemaLocation",
|
||||
"urn:mpeg:DASH:schema:MPD:2011 DASH-MPD.xsd");
|
||||
|
||||
appendNewAttrWithValue(mpdElement, "minBufferTime", "PT1.500S");
|
||||
|
||||
appendNewAttrWithValue(
|
||||
mpdElement, "profiles", "urn:mpeg:dash:profile:full:2011");
|
||||
|
||||
appendNewAttrWithValue(mpdElement, "type", "static");
|
||||
|
||||
final String durationSeconds =
|
||||
String.format(Locale.ENGLISH, "%.3f", durationMs / 1000.0);
|
||||
appendNewAttrWithValue(
|
||||
mpdElement, "mediaPresentationDuration", "PT" + durationSeconds + "S");
|
||||
} catch (final Exception e) {
|
||||
throw new DashManifestCreationException(
|
||||
"Could not generate the DASH manifest or append the MPD document to it", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <Period>} element, appended as a child of the {@code <MPD>} element.
|
||||
*
|
||||
* <p>
|
||||
* The {@code <MPD>} element needs to be generated before this element with
|
||||
* {@link #generateDocumentAndMpdElement(long)}.
|
||||
* </p>
|
||||
*
|
||||
* @throws DashManifestCreationException May throw a CreationException
|
||||
*/
|
||||
protected void generatePeriodElement() {
|
||||
try {
|
||||
getFirstElementByName(MPD).appendChild(createElement(PERIOD));
|
||||
} catch (final DOMException e) {
|
||||
throw DashManifestCreationException.couldNotAddElement(PERIOD, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <AdaptationSet>} element, appended as a child of the {@code <Period>}
|
||||
* element.
|
||||
*
|
||||
* <p>
|
||||
* The {@code <Period>} element needs to be generated before this element with
|
||||
* {@link #generatePeriodElement()}.
|
||||
* </p>
|
||||
*
|
||||
* @throws DashManifestCreationException May throw a CreationException
|
||||
*/
|
||||
protected void generateAdaptationSetElement() {
|
||||
try {
|
||||
final Element adaptationSetElement = createElement(ADAPTATION_SET);
|
||||
|
||||
appendNewAttrWithValue(adaptationSetElement, "id", "0");
|
||||
|
||||
appendNewAttrWithValue(
|
||||
adaptationSetElement,
|
||||
"mimeType",
|
||||
itagInfo.getItagFormat().mediaFormat().mimeType());
|
||||
|
||||
appendNewAttrWithValue(adaptationSetElement, "subsegmentAlignment", "true");
|
||||
|
||||
getFirstElementByName(PERIOD).appendChild(adaptationSetElement);
|
||||
} catch (final DOMException e) {
|
||||
throw DashManifestCreationException.couldNotAddElement(ADAPTATION_SET, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <Role>} element, appended as a child of the {@code <AdaptationSet>}
|
||||
* element.
|
||||
*
|
||||
* <p>
|
||||
* This element, with its attributes and values, is:
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* {@code <Role schemeIdUri="urn:mpeg:DASH:role:2011" value="main"/>}
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The {@code <AdaptationSet>} element needs to be generated before this element with
|
||||
* {@link #generateAdaptationSetElement(Document)}).
|
||||
* </p>
|
||||
*
|
||||
* @throws DashManifestCreationException May throw a CreationException
|
||||
*/
|
||||
protected void generateRoleElement() {
|
||||
try {
|
||||
final Element roleElement = createElement(ROLE);
|
||||
|
||||
appendNewAttrWithValue(roleElement, "schemeIdUri", "urn:mpeg:DASH:role:2011");
|
||||
|
||||
appendNewAttrWithValue(roleElement, "value", "main");
|
||||
|
||||
getFirstElementByName(ADAPTATION_SET).appendChild(roleElement);
|
||||
} catch (final DOMException e) {
|
||||
throw DashManifestCreationException.couldNotAddElement(ROLE, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <Representation>} element, appended as a child of the
|
||||
* {@code <AdaptationSet>} element.
|
||||
*
|
||||
* <p>
|
||||
* The {@code <AdaptationSet>} element needs to be generated before this element with
|
||||
* {@link #generateAdaptationSetElement()}).
|
||||
* </p>
|
||||
* @throws DashManifestCreationException May throw a CreationException
|
||||
*/
|
||||
protected void generateRepresentationElement() {
|
||||
try {
|
||||
final Element representationElement = createElement(REPRESENTATION);
|
||||
|
||||
appendNewAttrWithValue(
|
||||
representationElement, "id", itagInfo.getItagFormat().id());
|
||||
|
||||
final String codec = itagInfo.getCodec();
|
||||
if (isNullOrEmpty(codec)) {
|
||||
throw DashManifestCreationException.couldNotAddElement(ADAPTATION_SET,
|
||||
"the codec value of the ItagItem is null or empty");
|
||||
}
|
||||
|
||||
appendNewAttrWithValue(
|
||||
representationElement, "codecs", codec);
|
||||
|
||||
appendNewAttrWithValue(
|
||||
representationElement, "startWithSAP", "1");
|
||||
|
||||
appendNewAttrWithValue(
|
||||
representationElement, "maxPlayoutRate", "1");
|
||||
|
||||
final Integer bitrate = itagInfo.getBitRate();
|
||||
if (bitrate == null || bitrate <= 0) {
|
||||
throw DashManifestCreationException.couldNotAddElement(
|
||||
REPRESENTATION,
|
||||
"invalid bitrate=" + bitrate);
|
||||
}
|
||||
|
||||
appendNewAttrWithValue(
|
||||
representationElement, "bandwidth", bitrate);
|
||||
|
||||
if (itagInfo.getItagFormat() instanceof VideoItagFormat) {
|
||||
|
||||
final VideoQualityData videoQualityData = itagInfo.getCombinedVideoQualityData();
|
||||
|
||||
if (videoQualityData.height() <= 0 && videoQualityData.width() <= 0) {
|
||||
throw DashManifestCreationException.couldNotAddElement(
|
||||
REPRESENTATION,
|
||||
"both width and height are <= 0");
|
||||
}
|
||||
|
||||
if (videoQualityData.width() > 0) {
|
||||
appendNewAttrWithValue(
|
||||
representationElement, "width", videoQualityData.width());
|
||||
}
|
||||
|
||||
appendNewAttrWithValue(
|
||||
representationElement, "height", videoQualityData.height());
|
||||
|
||||
if (videoQualityData.fps() > 0) {
|
||||
appendNewAttrWithValue(
|
||||
representationElement, "height", videoQualityData.fps());
|
||||
}
|
||||
}
|
||||
|
||||
if (itagInfo.getItagFormat() instanceof AudioItagFormat
|
||||
&& itagInfo.getAudioSampleRate() != null
|
||||
&& itagInfo.getAudioSampleRate() > 0) {
|
||||
|
||||
appendNewAttrWithValue(
|
||||
representationElement,
|
||||
"audioSamplingRate",
|
||||
itagInfo.getAudioSampleRate());
|
||||
}
|
||||
|
||||
getFirstElementByName(ADAPTATION_SET).appendChild(representationElement);
|
||||
} catch (final DOMException e) {
|
||||
throw DashManifestCreationException.couldNotAddElement(REPRESENTATION, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <AudioChannelConfiguration>} element, appended as a child of the
|
||||
* {@code <Representation>} element.
|
||||
*
|
||||
* <p>
|
||||
* This method is only used when generating DASH manifests of audio streams.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* It will produce the following element:
|
||||
* <br>
|
||||
* {@code <AudioChannelConfiguration
|
||||
* schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011"
|
||||
* value="audioChannelsValue"}
|
||||
* <br>
|
||||
* (where {@code audioChannelsValue} is get from the {@link ItagInfo} passed as the second
|
||||
* parameter of this method)
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The {@code <Representation>} element needs to be generated before this element with
|
||||
* {@link #generateRepresentationElement()}).
|
||||
* </p>
|
||||
*
|
||||
* @throws DashManifestCreationException May throw a CreationException
|
||||
*/
|
||||
protected void generateAudioChannelConfigurationElement() {
|
||||
try {
|
||||
final Element audioChannelConfigElement = createElement(AUDIO_CHANNEL_CONFIGURATION);
|
||||
|
||||
appendNewAttrWithValue(
|
||||
audioChannelConfigElement,
|
||||
"schemeIdUri",
|
||||
"urn:mpeg:dash:23003:3:audio_channel_configuration:2011");
|
||||
|
||||
final Integer audioChannels = itagInfo.getAudioChannels();
|
||||
if (audioChannels != null && audioChannels <= 0) {
|
||||
throw new DashManifestCreationException(
|
||||
"Invalid value for 'audioChannels'=" + audioChannels);
|
||||
}
|
||||
|
||||
appendNewAttrWithValue(
|
||||
audioChannelConfigElement, "value", itagInfo.getAudioChannels());
|
||||
|
||||
getFirstElementByName(REPRESENTATION).appendChild(audioChannelConfigElement);
|
||||
} catch (final DOMException e) {
|
||||
throw DashManifestCreationException.couldNotAddElement(AUDIO_CHANNEL_CONFIGURATION, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <SegmentTemplate>} element, appended as a child of the
|
||||
* {@code <Representation>} element.
|
||||
*
|
||||
* <p>
|
||||
* This method is only used when generating DASH manifests from OTF and post-live-DVR streams.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* It will produce a {@code <SegmentTemplate>} element with the following attributes:
|
||||
* <ul>
|
||||
* <li>{@code startNumber}, which takes the value {@code 0} for post-live-DVR streams and
|
||||
* {@code 1} for OTF streams;</li>
|
||||
* <li>{@code timescale}, which is always {@code 1000};</li>
|
||||
* <li>{@code media}, which is the base URL of the stream on which is appended
|
||||
* {@code &sq=$Number$};</li>
|
||||
* <li>{@code initialization} (only for OTF streams), which is the base URL of the stream
|
||||
* on which is appended {@link #SQ_0}.</li>
|
||||
* </ul>
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The {@code <Representation>} element needs to be generated before this element with
|
||||
* {@link #generateRepresentationElement()}).
|
||||
* </p>
|
||||
*
|
||||
* @param baseUrl the base URL of the OTF/post-live-DVR stream
|
||||
* @throws DashManifestCreationException May throw a CreationException
|
||||
*/
|
||||
protected void generateSegmentTemplateElement(@Nonnull final String baseUrl) {
|
||||
try {
|
||||
final Element segmentTemplateElement = createElement(SEGMENT_TEMPLATE);
|
||||
|
||||
// The first sequence of post DVR streams is the beginning of the video stream and not
|
||||
// an initialization segment
|
||||
appendNewAttrWithValue(
|
||||
segmentTemplateElement, "startNumber", isLiveDelivery() ? "0" : "1");
|
||||
|
||||
appendNewAttrWithValue(
|
||||
segmentTemplateElement, "timescale", "1000");
|
||||
|
||||
// Post-live-DVR/ended livestreams streams don't require an initialization sequence
|
||||
if (!isLiveDelivery()) {
|
||||
appendNewAttrWithValue(
|
||||
segmentTemplateElement, "initialization", baseUrl + SQ_0);
|
||||
}
|
||||
|
||||
appendNewAttrWithValue(
|
||||
segmentTemplateElement, "media", baseUrl + "&sq=$Number$");
|
||||
|
||||
getFirstElementByName(REPRESENTATION).appendChild(segmentTemplateElement);
|
||||
} catch (final DOMException e) {
|
||||
throw DashManifestCreationException.couldNotAddElement(SEGMENT_TEMPLATE, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <SegmentTimeline>} element, appended as a child of the
|
||||
* {@code <SegmentTemplate>} element.
|
||||
*
|
||||
* <p>
|
||||
* The {@code <SegmentTemplate>} element needs to be generated before this element with
|
||||
* {@link #generateSegmentTemplateElement(String)}.
|
||||
* </p>
|
||||
*
|
||||
* @throws DashManifestCreationException May throw a CreationException
|
||||
*/
|
||||
protected void generateSegmentTimelineElement()
|
||||
throws DashManifestCreationException {
|
||||
try {
|
||||
final Element segmentTemplateElement = getFirstElementByName(SEGMENT_TEMPLATE);
|
||||
final Element segmentTimelineElement = createElement(SEGMENT_TIMELINE);
|
||||
|
||||
segmentTemplateElement.appendChild(segmentTimelineElement);
|
||||
} catch (final DOMException e) {
|
||||
throw DashManifestCreationException.couldNotAddElement(SEGMENT_TIMELINE, e);
|
||||
}
|
||||
}
|
||||
// endregion
|
||||
|
||||
// region initResponse
|
||||
|
||||
@Nonnull
|
||||
protected Response getInitializationResponse(@Nonnull String baseStreamingUrl) {
|
||||
final boolean isHtml5StreamingUrl = isWebStreamingUrl(baseStreamingUrl)
|
||||
|| isTvHtml5SimplyEmbeddedPlayerStreamingUrl(baseStreamingUrl);
|
||||
final boolean isAndroidStreamingUrl = isAndroidStreamingUrl(baseStreamingUrl);
|
||||
final boolean isIosStreamingUrl = isIosStreamingUrl(baseStreamingUrl);
|
||||
if (isHtml5StreamingUrl) {
|
||||
baseStreamingUrl += ALR_YES;
|
||||
}
|
||||
baseStreamingUrl = appendBaseStreamingUrlParams(baseStreamingUrl);
|
||||
|
||||
final Downloader downloader = NewPipe.getDownloader();
|
||||
if (isHtml5StreamingUrl) {
|
||||
return getStreamingWebUrlWithoutRedirects(downloader, baseStreamingUrl);
|
||||
} else if (isAndroidStreamingUrl || isIosStreamingUrl) {
|
||||
try {
|
||||
final Map<String, List<String>> headers = new HashMap<>();
|
||||
headers.put("User-Agent", Collections.singletonList(
|
||||
isAndroidStreamingUrl
|
||||
? getAndroidUserAgent(null)
|
||||
: getIosUserAgent(null)));
|
||||
final byte[] emptyBody = "".getBytes(StandardCharsets.UTF_8);
|
||||
return downloader.post(baseStreamingUrl, headers, emptyBody);
|
||||
} catch (final IOException | ExtractionException e) {
|
||||
throw new DashManifestCreationException("Could not get the "
|
||||
+ (isIosStreamingUrl ? "IOS" : "ANDROID") + " streaming URL response", e);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
return downloader.get(baseStreamingUrl);
|
||||
} catch (final IOException | ExtractionException e) {
|
||||
throw new DashManifestCreationException("Could not get the streaming URL response", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
protected Response getStreamingWebUrlWithoutRedirects(
|
||||
@Nonnull final Downloader downloader,
|
||||
@Nonnull String streamingUrl) {
|
||||
try {
|
||||
final Map<String, List<String>> headers = new HashMap<>();
|
||||
addClientInfoHeaders(headers);
|
||||
|
||||
for (int r = 0; r < MAXIMUM_REDIRECT_COUNT; r++) {
|
||||
final Response response = downloader.get(streamingUrl, headers);
|
||||
|
||||
final int responseCode = response.responseCode();
|
||||
if (responseCode != 200) {
|
||||
throw new DashManifestCreationException(
|
||||
"Could not get the initialization URL: HTTP response code "
|
||||
+ responseCode);
|
||||
}
|
||||
|
||||
// A valid HTTP 1.0+ response should include a Content-Type header, so we can
|
||||
// require that the response from video servers has this header.
|
||||
final String responseMimeType =
|
||||
Objects.requireNonNull(
|
||||
response.getHeader("Content-Type"),
|
||||
"Could not get the Content-Type header from the response headers");
|
||||
|
||||
// The response body is not the redirection URL
|
||||
if (!responseMimeType.equals("text/plain")) {
|
||||
return response;
|
||||
}
|
||||
|
||||
streamingUrl = response.responseBody();
|
||||
}
|
||||
|
||||
throw new DashManifestCreationException("Too many redirects");
|
||||
|
||||
} catch (final IOException | ExtractionException e) {
|
||||
throw new DashManifestCreationException(
|
||||
"Could not get the streaming URL response of a HTML5 client", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
protected String appendBaseStreamingUrlParams(@Nonnull final String baseStreamingUrl) {
|
||||
return baseStreamingUrl + SQ_0 + RN_0;
|
||||
}
|
||||
|
||||
// endregion
|
||||
|
||||
// region document util
|
||||
|
||||
/**
|
||||
* Generate a new {@link DocumentBuilder} secured from XXE attacks, on platforms which
|
||||
* support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and
|
||||
* {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link DocumentBuilderFactory} instances.
|
||||
*/
|
||||
protected void newDocument() throws ParserConfigurationException {
|
||||
final DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
|
||||
try {
|
||||
documentBuilderFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
|
||||
documentBuilderFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
|
||||
} catch (final Exception ignored) {
|
||||
// Ignore exceptions as setting these attributes to secure XML generation is not
|
||||
// supported by all platforms (like the Android implementation)
|
||||
}
|
||||
|
||||
final DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
|
||||
document = documentBuilder.newDocument();
|
||||
}
|
||||
|
||||
protected Attr appendNewAttrWithValue(
|
||||
final Element baseElement,
|
||||
final String name,
|
||||
final Object value
|
||||
) {
|
||||
return appendNewAttrWithValue(baseElement, name, String.valueOf(value));
|
||||
}
|
||||
|
||||
protected Attr appendNewAttrWithValue(
|
||||
final Element baseElement,
|
||||
final String name,
|
||||
final String value
|
||||
) {
|
||||
final Attr attr = document.createAttribute(name);
|
||||
attr.setValue(value);
|
||||
baseElement.setAttributeNode(attr);
|
||||
|
||||
return attr;
|
||||
}
|
||||
|
||||
protected Element getFirstElementByName(final String name) {
|
||||
return (Element) document.getElementsByTagName(name).item(0);
|
||||
}
|
||||
|
||||
protected Element createElement(final String name) {
|
||||
return document.createElement(name);
|
||||
}
|
||||
|
||||
@SuppressWarnings("squid:S2755") // warning is still shown despite applied solution
|
||||
protected String documentToXml() throws TransformerException {
|
||||
/*
|
||||
* Generate a new {@link TransformerFactory} secured from XXE attacks, on platforms which
|
||||
* support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and
|
||||
* {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link TransformerFactory} instances.
|
||||
*/
|
||||
final TransformerFactory transformerFactory = TransformerFactory.newInstance();
|
||||
try {
|
||||
transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
|
||||
transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
|
||||
} catch (final Exception ignored) {
|
||||
// Ignore exceptions as setting these attributes to secure XML generation is not
|
||||
// supported by all platforms (like the Android implementation)
|
||||
}
|
||||
|
||||
final Transformer transformer = transformerFactory.newTransformer();
|
||||
transformer.setOutputProperty(OutputKeys.VERSION, "1.0");
|
||||
transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");
|
||||
transformer.setOutputProperty(OutputKeys.STANDALONE, "no");
|
||||
|
||||
final StringWriter result = new StringWriter();
|
||||
transformer.transform(new DOMSource(document), new StreamResult(result));
|
||||
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
protected String documentToXmlSafe() {
|
||||
try {
|
||||
return documentToXml();
|
||||
} catch (final TransformerException e) {
|
||||
throw new DashManifestCreationException("Failed to convert XML-document to string", e);
|
||||
}
|
||||
}
|
||||
|
||||
// endregion
|
||||
}
|
||||
|
|
@ -0,0 +1,162 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator;
|
||||
|
||||
import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.SEGMENT_TIMELINE;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isBlank;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.removeNonDigitCharacters;
|
||||
|
||||
import org.schabi.newpipe.extractor.downloader.Response;
|
||||
import org.schabi.newpipe.extractor.services.youtube.itag.info.ItagInfo;
|
||||
import org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreationException;
|
||||
import org.w3c.dom.Element;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
public class YoutubeOtfDashManifestCreator extends AbstractYoutubeDashManifestCreator {
|
||||
|
||||
public YoutubeOtfDashManifestCreator(@Nonnull final ItagInfo<?> itagInfo,
|
||||
final long durationSecondsFallback) {
|
||||
super(itagInfo, durationSecondsFallback);
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String generateManifest() {
|
||||
// Try to avoid redirects when streaming the content by saving the last URL we get
|
||||
// from video servers.
|
||||
final Response response = getInitializationResponse(itagInfo.getStreamUrl());
|
||||
final String otfBaseStreamingUrl = response.latestUrl()
|
||||
.replace(SQ_0, EMPTY_STRING)
|
||||
.replace(RN_0, EMPTY_STRING)
|
||||
.replace(ALR_YES, EMPTY_STRING);
|
||||
|
||||
final int responseCode = response.responseCode();
|
||||
if (responseCode != 200) {
|
||||
throw new DashManifestCreationException("Could not get the initialization URL: " +
|
||||
"response code "
|
||||
+ responseCode);
|
||||
}
|
||||
|
||||
final String[] segmentDurations;
|
||||
|
||||
try {
|
||||
final String[] segmentsAndDurationsResponseSplit = response.responseBody()
|
||||
// Get the lines with the durations and the following
|
||||
.split("Segment-Durations-Ms: ")[1]
|
||||
// Remove the other lines
|
||||
.split("\n")[0]
|
||||
// Get all durations and repetitions which are separated by a comma
|
||||
.split(",");
|
||||
final int lastIndex = segmentsAndDurationsResponseSplit.length - 1;
|
||||
if (isBlank(segmentsAndDurationsResponseSplit[lastIndex])) {
|
||||
segmentDurations = Arrays.copyOf(segmentsAndDurationsResponseSplit, lastIndex);
|
||||
} else {
|
||||
segmentDurations = segmentsAndDurationsResponseSplit;
|
||||
}
|
||||
} catch (final Exception e) {
|
||||
throw new DashManifestCreationException("Could not get segment durations", e);
|
||||
}
|
||||
|
||||
long streamDurationMs;
|
||||
try {
|
||||
streamDurationMs = getStreamDuration(segmentDurations);
|
||||
} catch (final DashManifestCreationException e) {
|
||||
streamDurationMs = durationSecondsFallback * 1000;
|
||||
}
|
||||
|
||||
generateDocumentAndCommonElements(streamDurationMs);
|
||||
generateSegmentTemplateElement(otfBaseStreamingUrl);
|
||||
generateSegmentTimelineElement();
|
||||
generateSegmentElementsForOtfStreams(segmentDurations);
|
||||
|
||||
return documentToXmlSafe();
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate segment elements for OTF streams.
|
||||
*
|
||||
* <p>
|
||||
* By parsing by the first media sequence, we know how many durations and repetitions there are
|
||||
* so we just have to loop into segment durations to generate the following elements for each
|
||||
* duration repeated X times:
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* {@code <S d="segmentDuration" r="durationRepetition" />}
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* If there is no repetition of the duration between two segments, the {@code r} attribute is
|
||||
* not added to the {@code S} element, as it is not needed.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* These elements will be appended as children of the {@code <SegmentTimeline>} element, which
|
||||
* needs to be generated before these elements with
|
||||
* {@link #generateSegmentTimelineElement()}.
|
||||
* </p>
|
||||
*
|
||||
* @param segmentDurations the sequences "length" or "length(r=repeat_count" extracted with the
|
||||
* regular expressions
|
||||
* @throws DashManifestCreationException May throw a CreationException
|
||||
*/
|
||||
protected void generateSegmentElementsForOtfStreams(@Nonnull final String[] segmentDurations) {
|
||||
try {
|
||||
final Element segmentTimelineElement = getFirstElementByName(SEGMENT_TIMELINE);
|
||||
streamAndSplitSegmentDurations(segmentDurations)
|
||||
.map(segmentLengthRepeat -> {
|
||||
final Element sElement = createElement("S");
|
||||
// There are repetitions of a segment duration in other segments
|
||||
if (segmentLengthRepeat.length > 1) {
|
||||
appendNewAttrWithValue(sElement, "r", Integer.parseInt(
|
||||
removeNonDigitCharacters(segmentLengthRepeat[1])));
|
||||
}
|
||||
|
||||
appendNewAttrWithValue(
|
||||
sElement, "d", Integer.parseInt(segmentLengthRepeat[0]));
|
||||
return sElement;
|
||||
})
|
||||
.forEach(segmentTimelineElement::appendChild);
|
||||
} catch (final Exception e) {
|
||||
throw DashManifestCreationException.couldNotAddElement("segment (S)", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the duration of an OTF stream.
|
||||
*
|
||||
* <p>
|
||||
* The duration of OTF streams is not returned into the player response and needs to be
|
||||
* calculated by adding the duration of each segment.
|
||||
* </p>
|
||||
*
|
||||
* @param segmentDurations the segment duration object extracted from the initialization
|
||||
* sequence of the stream
|
||||
* @return the duration of the OTF stream, in milliseconds
|
||||
* @throws DashManifestCreationException May throw a CreationException
|
||||
*/
|
||||
protected long getStreamDuration(@Nonnull final String[] segmentDurations) {
|
||||
try {
|
||||
return streamAndSplitSegmentDurations(segmentDurations)
|
||||
.mapToLong(segmentLengthRepeat -> {
|
||||
final long segmentLength = Integer.parseInt(segmentLengthRepeat[0]);
|
||||
final long segmentRepeatCount = segmentLengthRepeat.length > 1
|
||||
? Long.parseLong(removeNonDigitCharacters(segmentLengthRepeat[1]))
|
||||
: 0;
|
||||
return segmentLength + segmentRepeatCount * segmentLength;
|
||||
})
|
||||
.sum();
|
||||
} catch (final NumberFormatException e) {
|
||||
throw new DashManifestCreationException("Could not get stream length from sequences " +
|
||||
"list", e);
|
||||
}
|
||||
}
|
||||
|
||||
protected Stream<String[]> streamAndSplitSegmentDurations(@Nonnull final String[] segmentDurations) {
|
||||
return Stream.of(segmentDurations)
|
||||
.map(segDuration -> segDuration.split("\\(r="));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,121 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator;
|
||||
|
||||
import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.SEGMENT_TIMELINE;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
import org.schabi.newpipe.extractor.downloader.Response;
|
||||
import org.schabi.newpipe.extractor.services.youtube.itag.info.ItagInfo;
|
||||
import org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreationException;
|
||||
import org.w3c.dom.Attr;
|
||||
import org.w3c.dom.DOMException;
|
||||
import org.w3c.dom.Element;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
public class YoutubePostLiveStreamDvrDashManifestCreator extends AbstractYoutubeDashManifestCreator {
|
||||
|
||||
protected YoutubePostLiveStreamDvrDashManifestCreator(@Nonnull final ItagInfo<?> itagInfo,
|
||||
final long durationSecondsFallback) {
|
||||
super(itagInfo, durationSecondsFallback);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean isLiveDelivery() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String generateManifest() {
|
||||
final Integer targetDurationSec = itagInfo.getTargetDurationSec();
|
||||
if (targetDurationSec == null || targetDurationSec <= 0) {
|
||||
throw new DashManifestCreationException(
|
||||
"Invalid value for 'targetDurationSec'=" + targetDurationSec);
|
||||
}
|
||||
|
||||
// Try to avoid redirects when streaming the content by saving the latest URL we get
|
||||
// from video servers.
|
||||
final Response response = getInitializationResponse(itagInfo.getStreamUrl());
|
||||
final String realPostLiveStreamDvrStreamingUrl = response.latestUrl()
|
||||
.replace(SQ_0, "")
|
||||
.replace(RN_0, "")
|
||||
.replace(ALR_YES, "");
|
||||
|
||||
final int responseCode = response.responseCode();
|
||||
if (responseCode != 200) {
|
||||
throw new DashManifestCreationException(
|
||||
"Could not get the initialization sequence: response code " + responseCode);
|
||||
}
|
||||
|
||||
final String streamDurationMsString;
|
||||
final String segmentCount;
|
||||
try {
|
||||
final Map<String, List<String>> responseHeaders = response.responseHeaders();
|
||||
streamDurationMsString = responseHeaders.get("X-Head-Time-Millis").get(0);
|
||||
segmentCount = responseHeaders.get("X-Head-Seqnum").get(0);
|
||||
} catch (final IndexOutOfBoundsException e) {
|
||||
throw new DashManifestCreationException(
|
||||
"Could not get the value of the X-Head-Time-Millis or the X-Head-Seqnum header",
|
||||
e);
|
||||
}
|
||||
|
||||
if (isNullOrEmpty(segmentCount)) {
|
||||
throw new DashManifestCreationException("Could not get the number of segments");
|
||||
}
|
||||
|
||||
long streamDurationMs;
|
||||
try {
|
||||
streamDurationMs = Long.parseLong(streamDurationMsString);
|
||||
} catch (final NumberFormatException e) {
|
||||
streamDurationMs = durationSecondsFallback * 1000;
|
||||
}
|
||||
|
||||
generateDocumentAndCommonElements(streamDurationMs);
|
||||
|
||||
generateSegmentTemplateElement(realPostLiveStreamDvrStreamingUrl);
|
||||
generateSegmentTimelineElement();
|
||||
generateSegmentElementForPostLiveDvrStreams(targetDurationSec, segmentCount);
|
||||
|
||||
return documentToXmlSafe();
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the segment ({@code <S>}) element.
|
||||
*
|
||||
* <p>
|
||||
* We don't know the exact duration of segments for post-live-DVR streams but an
|
||||
* average instead (which is the {@code targetDurationSec} value), so we can use the following
|
||||
* structure to generate the segment timeline for DASH manifests of ended livestreams:
|
||||
* <br>
|
||||
* {@code <S d="targetDurationSecValue" r="segmentCount" />}
|
||||
* </p>
|
||||
*
|
||||
* @param targetDurationSeconds the {@code targetDurationSec} value from YouTube player
|
||||
* response's stream
|
||||
* @param segmentCount the number of segments
|
||||
* @throws DashManifestCreationException May throw a CreationException
|
||||
*/
|
||||
private void generateSegmentElementForPostLiveDvrStreams(
|
||||
final int targetDurationSeconds,
|
||||
@Nonnull final String segmentCount
|
||||
) {
|
||||
try {
|
||||
final Element sElement = document.createElement("S");
|
||||
|
||||
appendNewAttrWithValue(sElement, "d", targetDurationSeconds * 1000);
|
||||
|
||||
final Attr rAttribute = document.createAttribute("r");
|
||||
rAttribute.setValue(segmentCount);
|
||||
sElement.setAttributeNode(rAttribute);
|
||||
|
||||
appendNewAttrWithValue(sElement, "r", segmentCount);
|
||||
|
||||
getFirstElementByName(SEGMENT_TIMELINE).appendChild(sElement);
|
||||
} catch (final DOMException e) {
|
||||
throw DashManifestCreationException.couldNotAddElement("segment (S)", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,159 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator;
|
||||
|
||||
import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.BASE_URL;
|
||||
import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.INITIALIZATION;
|
||||
import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.MPD;
|
||||
import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.REPRESENTATION;
|
||||
import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.SEGMENT_BASE;
|
||||
|
||||
import org.schabi.newpipe.extractor.services.youtube.itag.info.ItagInfo;
|
||||
import org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreationException;
|
||||
import org.w3c.dom.DOMException;
|
||||
import org.w3c.dom.Element;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
public class YoutubeProgressiveDashManifestCreator extends AbstractYoutubeDashManifestCreator {
|
||||
|
||||
public YoutubeProgressiveDashManifestCreator(@Nonnull final ItagInfo<?> itagInfo,
|
||||
final long durationSecondsFallback) {
|
||||
super(itagInfo, durationSecondsFallback);
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String generateManifest() {
|
||||
final long streamDurationMs;
|
||||
if (itagInfo.getApproxDurationMs() != null) {
|
||||
streamDurationMs = itagInfo.getApproxDurationMs();
|
||||
} else if (durationSecondsFallback > 0) {
|
||||
streamDurationMs = durationSecondsFallback * 1000;
|
||||
} else {
|
||||
throw DashManifestCreationException.couldNotAddElement(MPD, "the duration of the " +
|
||||
"stream could not be determined and durationSecondsFallback is <= 0");
|
||||
}
|
||||
|
||||
generateDocumentAndCommonElements(streamDurationMs);
|
||||
generateSegmentBaseElement();
|
||||
generateInitializationElement();
|
||||
|
||||
return documentToXmlSafe();
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <BaseURL>} element, appended as a child of the
|
||||
* {@code <Representation>} element.
|
||||
*
|
||||
* <p>
|
||||
* The {@code <Representation>} element needs to be generated before this element with
|
||||
* {@link #generateRepresentationElement()}).
|
||||
* </p>
|
||||
*
|
||||
* @param baseUrl the base URL of the stream, which must not be null and will be set as the
|
||||
* content of the {@code <BaseURL>} element
|
||||
* @throws DashManifestCreationException May throw a CreationException
|
||||
*/
|
||||
protected void generateBaseUrlElement(@Nonnull final String baseUrl)
|
||||
throws DashManifestCreationException {
|
||||
try {
|
||||
final Element representationElement = getFirstElementByName(REPRESENTATION);
|
||||
|
||||
appendNewAttrWithValue(representationElement, BASE_URL, baseUrl);
|
||||
} catch (final DOMException e) {
|
||||
throw DashManifestCreationException.couldNotAddElement(BASE_URL, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <SegmentBase>} element, appended as a child of the
|
||||
* {@code <Representation>} element.
|
||||
*
|
||||
* <p>
|
||||
* It generates the following element:
|
||||
* <br>
|
||||
* {@code <SegmentBase indexRange="indexStart-indexEnd" />}
|
||||
* <br>
|
||||
* (where {@code indexStart} and {@code indexEnd} are gotten from the {@link ItagInfo} passed
|
||||
* as the second parameter)
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The {@code <Representation>} element needs to be generated before this element with
|
||||
* {@link #generateRepresentationElement()}),
|
||||
* and the {@code BaseURL} element with {@link #generateBaseUrlElement(String)}
|
||||
* should be generated too.
|
||||
* </p>
|
||||
*
|
||||
* @throws DashManifestCreationException May throw a CreationException
|
||||
*/
|
||||
protected void generateSegmentBaseElement() {
|
||||
try {
|
||||
final Element segmentBaseElement = createElement(SEGMENT_BASE);
|
||||
|
||||
if (itagInfo.getIndexRange() == null
|
||||
|| itagInfo.getIndexRange().start() < 0
|
||||
|| itagInfo.getIndexRange().end() < 0) {
|
||||
throw DashManifestCreationException.couldNotAddElement(SEGMENT_BASE,
|
||||
"invalid index-range: " + itagInfo.getIndexRange());
|
||||
}
|
||||
|
||||
appendNewAttrWithValue(
|
||||
segmentBaseElement,
|
||||
"indexRange",
|
||||
itagInfo.getIndexRange().start() + "-" + itagInfo.getIndexRange().end());
|
||||
|
||||
getFirstElementByName(REPRESENTATION).appendChild(segmentBaseElement);
|
||||
} catch (final DOMException e) {
|
||||
throw DashManifestCreationException.couldNotAddElement(SEGMENT_BASE, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <Initialization>} element, appended as a child of the
|
||||
* {@code <SegmentBase>} element.
|
||||
*
|
||||
* <p>
|
||||
* It generates the following element:
|
||||
* <br>
|
||||
* {@code <Initialization range="initStart-initEnd"/>}
|
||||
* <br>
|
||||
* (where {@code indexStart} and {@code indexEnd} are gotten from the {@link ItagInfo} passed
|
||||
* as the second parameter)
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The {@code <SegmentBase>} element needs to be generated before this element with
|
||||
* {@link #generateSegmentBaseElement()}).
|
||||
* </p>
|
||||
*
|
||||
* @throws DashManifestCreationException May throw a CreationException
|
||||
*/
|
||||
protected void generateInitializationElement() {
|
||||
try {
|
||||
final Element initializationElement = createElement(INITIALIZATION);
|
||||
|
||||
if (itagInfo.getInitRange() == null
|
||||
|| itagInfo.getInitRange().start() < 0
|
||||
|| itagInfo.getInitRange().end() < 0) {
|
||||
throw DashManifestCreationException.couldNotAddElement(SEGMENT_BASE,
|
||||
"invalid (init)-range: " + itagInfo.getInitRange());
|
||||
}
|
||||
|
||||
appendNewAttrWithValue(
|
||||
initializationElement,
|
||||
"range",
|
||||
itagInfo.getInitRange().start() + "-" + itagInfo.getInitRange().end());
|
||||
|
||||
getFirstElementByName(SEGMENT_BASE).appendChild(initializationElement);
|
||||
} catch (final DOMException e) {
|
||||
throw DashManifestCreationException.couldNotAddElement(INITIALIZATION, e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
protected String appendBaseStreamingUrlParams(@Nonnull final String baseStreamingUrl) {
|
||||
return baseStreamingUrl + RN_0;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
/**
|
||||
* Exception that is thrown when a YouTube DASH manifest creator encounters a problem
|
||||
* while creating a manifest.
|
||||
*/
|
||||
public final class CreationException extends RuntimeException {
|
||||
|
||||
/**
|
||||
* Create a new {@link CreationException} with a detail message.
|
||||
*
|
||||
* @param message the detail message to add in the exception
|
||||
*/
|
||||
public CreationException(final String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new {@link CreationException} with a detail message and a cause.
|
||||
* @param message the detail message to add in the exception
|
||||
* @param cause the exception cause of this {@link CreationException}
|
||||
*/
|
||||
public CreationException(final String message, final Exception cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
|
||||
// Methods to create exceptions easily without having to use big exception messages and to
|
||||
// reduce duplication
|
||||
|
||||
/**
|
||||
* Create a new {@link CreationException} with a cause and the following detail message format:
|
||||
* <br>
|
||||
* {@code "Could not add " + element + " element", cause}, where {@code element} is an element
|
||||
* of a DASH manifest.
|
||||
*
|
||||
* @param element the element which was not added to the DASH document
|
||||
* @param cause the exception which prevented addition of the element to the DASH document
|
||||
* @return a new {@link CreationException}
|
||||
*/
|
||||
@Nonnull
|
||||
public static CreationException couldNotAddElement(final String element,
|
||||
final Exception cause) {
|
||||
return new CreationException("Could not add " + element + " element", cause);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new {@link CreationException} with a cause and the following detail message format:
|
||||
* <br>
|
||||
* {@code "Could not add " + element + " element: " + reason}, where {@code element} is an
|
||||
* element of a DASH manifest and {@code reason} the reason why this element cannot be added to
|
||||
* the DASH document.
|
||||
*
|
||||
* @param element the element which was not added to the DASH document
|
||||
* @param reason the reason message of why the element has been not added to the DASH document
|
||||
* @return a new {@link CreationException}
|
||||
*/
|
||||
@Nonnull
|
||||
public static CreationException couldNotAddElement(final String element, final String reason) {
|
||||
return new CreationException("Could not add " + element + " element: " + reason);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,756 +0,0 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
|
||||
|
||||
import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.downloader.Downloader;
|
||||
import org.schabi.newpipe.extractor.downloader.Response;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
|
||||
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
|
||||
import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
|
||||
import org.w3c.dom.Attr;
|
||||
import org.w3c.dom.DOMException;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.xml.XMLConstants;
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
import javax.xml.transform.OutputKeys;
|
||||
import javax.xml.transform.Transformer;
|
||||
import javax.xml.transform.TransformerException;
|
||||
import javax.xml.transform.TransformerFactory;
|
||||
import javax.xml.transform.dom.DOMSource;
|
||||
import javax.xml.transform.stream.StreamResult;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.StringWriter;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isIosStreamingUrl;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5SimplyEmbeddedPlayerStreamingUrl;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebStreamingUrl;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
/**
|
||||
* Utilities and constants for YouTube DASH manifest creators.
|
||||
*
|
||||
* <p>
|
||||
* This class includes common methods of manifest creators and useful constants.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* Generation of DASH documents and their conversion as a string is done using external classes
|
||||
* from {@link org.w3c.dom} and {@link javax.xml} packages.
|
||||
* </p>
|
||||
*/
|
||||
public final class YoutubeDashManifestCreatorsUtils {
|
||||
|
||||
private YoutubeDashManifestCreatorsUtils() {
|
||||
}
|
||||
|
||||
/**
|
||||
* The redirect count limit that this class uses, which is the same limit as OkHttp.
|
||||
*/
|
||||
public static final int MAXIMUM_REDIRECT_COUNT = 20;
|
||||
|
||||
/**
|
||||
* URL parameter of the first sequence for live, post-live-DVR and OTF streams.
|
||||
*/
|
||||
public static final String SQ_0 = "&sq=0";
|
||||
|
||||
/**
|
||||
* URL parameter of the first stream request made by official clients.
|
||||
*/
|
||||
public static final String RN_0 = "&rn=0";
|
||||
|
||||
/**
|
||||
* URL parameter specific to web clients. When this param is added, if a redirection occurs,
|
||||
* the server will not redirect clients to the redirect URL. Instead, it will provide this URL
|
||||
* as the response body.
|
||||
*/
|
||||
public static final String ALR_YES = "&alr=yes";
|
||||
|
||||
// XML elements of DASH MPD manifests
|
||||
// see https://www.brendanlong.com/the-structure-of-an-mpeg-dash-mpd.html
|
||||
public static final String MPD = "MPD";
|
||||
public static final String PERIOD = "Period";
|
||||
public static final String ADAPTATION_SET = "AdaptationSet";
|
||||
public static final String ROLE = "Role";
|
||||
public static final String REPRESENTATION = "Representation";
|
||||
public static final String AUDIO_CHANNEL_CONFIGURATION = "AudioChannelConfiguration";
|
||||
public static final String SEGMENT_TEMPLATE = "SegmentTemplate";
|
||||
public static final String SEGMENT_TIMELINE = "SegmentTimeline";
|
||||
public static final String BASE_URL = "BaseURL";
|
||||
public static final String SEGMENT_BASE = "SegmentBase";
|
||||
public static final String INITIALIZATION = "Initialization";
|
||||
|
||||
/**
|
||||
* Create an attribute with {@link Document#createAttribute(String)}, assign to it the provided
|
||||
* name and value, then add it to the provided element using {@link
|
||||
* Element#setAttributeNode(Attr)}.
|
||||
*
|
||||
* @param element element to which to add the created node
|
||||
* @param doc document to use to create the attribute
|
||||
* @param name name of the attribute
|
||||
* @param value value of the attribute, will be set using {@link Attr#setValue(String)}
|
||||
*/
|
||||
public static void setAttribute(final Element element,
|
||||
final Document doc,
|
||||
final String name,
|
||||
final String value) {
|
||||
final Attr attr = doc.createAttribute(name);
|
||||
attr.setValue(value);
|
||||
element.setAttributeNode(attr);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a {@link Document} with common manifest creator elements added to it.
|
||||
*
|
||||
* <p>
|
||||
* Those are:
|
||||
* <ul>
|
||||
* <li>{@code MPD} (using {@link #generateDocumentAndMpdElement(long)});</li>
|
||||
* <li>{@code Period} (using {@link #generatePeriodElement(Document)});</li>
|
||||
* <li>{@code AdaptationSet} (using {@link #generateAdaptationSetElement(Document,
|
||||
* ItagItem)});</li>
|
||||
* <li>{@code Role} (using {@link #generateRoleElement(Document)});</li>
|
||||
* <li>{@code Representation} (using {@link #generateRepresentationElement(Document,
|
||||
* ItagItem)});</li>
|
||||
* <li>and, for audio streams, {@code AudioChannelConfiguration} (using
|
||||
* {@link #generateAudioChannelConfigurationElement(Document, ItagItem)}).</li>
|
||||
* </ul>
|
||||
* </p>
|
||||
*
|
||||
* @param itagItem the {@link ItagItem} associated to the stream, which must not be null
|
||||
* @param streamDuration the duration of the stream, in milliseconds
|
||||
* @return a {@link Document} with the common elements added in it
|
||||
*/
|
||||
@Nonnull
|
||||
public static Document generateDocumentAndDoCommonElementsGeneration(
|
||||
@Nonnull final ItagItem itagItem,
|
||||
final long streamDuration) throws CreationException {
|
||||
final Document doc = generateDocumentAndMpdElement(streamDuration);
|
||||
|
||||
generatePeriodElement(doc);
|
||||
generateAdaptationSetElement(doc, itagItem);
|
||||
generateRoleElement(doc);
|
||||
generateRepresentationElement(doc, itagItem);
|
||||
if (itagItem.itagType == ItagItem.ItagType.AUDIO) {
|
||||
generateAudioChannelConfigurationElement(doc, itagItem);
|
||||
}
|
||||
|
||||
return doc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a {@link Document} instance and generate the {@code <MPD>} element of the manifest.
|
||||
*
|
||||
* <p>
|
||||
* The generated {@code <MPD>} element looks like the manifest returned into the player
|
||||
* response of videos:
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* {@code <MPD xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
* xmlns="urn:mpeg:DASH:schema:MPD:2011"
|
||||
* xsi:schemaLocation="urn:mpeg:DASH:schema:MPD:2011 DASH-MPD.xsd" minBufferTime="PT1.500S"
|
||||
* profiles="urn:mpeg:dash:profile:isoff-main:2011" type="static"
|
||||
* mediaPresentationDuration="PT$duration$S">}
|
||||
* (where {@code $duration$} represents the duration in seconds (a number with 3 digits after
|
||||
* the decimal point)).
|
||||
* </p>
|
||||
*
|
||||
* @param duration the duration of the stream, in milliseconds
|
||||
* @return a {@link Document} instance which contains a {@code <MPD>} element
|
||||
*/
|
||||
@Nonnull
|
||||
public static Document generateDocumentAndMpdElement(final long duration)
|
||||
throws CreationException {
|
||||
try {
|
||||
final Document doc = newDocument();
|
||||
|
||||
final Element mpdElement = doc.createElement(MPD);
|
||||
doc.appendChild(mpdElement);
|
||||
|
||||
setAttribute(mpdElement, doc, "xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
|
||||
setAttribute(mpdElement, doc, "xmlns", "urn:mpeg:DASH:schema:MPD:2011");
|
||||
setAttribute(mpdElement, doc, "xsi:schemaLocation",
|
||||
"urn:mpeg:DASH:schema:MPD:2011 DASH-MPD.xsd");
|
||||
setAttribute(mpdElement, doc, "minBufferTime", "PT1.500S");
|
||||
setAttribute(mpdElement, doc, "profiles", "urn:mpeg:dash:profile:full:2011");
|
||||
setAttribute(mpdElement, doc, "type", "static");
|
||||
setAttribute(mpdElement, doc, "mediaPresentationDuration",
|
||||
String.format(Locale.ENGLISH, "PT%.3fS", duration / 1000.0));
|
||||
|
||||
return doc;
|
||||
} catch (final Exception e) {
|
||||
throw new CreationException(
|
||||
"Could not generate the DASH manifest or append the MPD doc to it", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <Period>} element, appended as a child of the {@code <MPD>} element.
|
||||
*
|
||||
* <p>
|
||||
* The {@code <MPD>} element needs to be generated before this element with
|
||||
* {@link #generateDocumentAndMpdElement(long)}.
|
||||
* </p>
|
||||
*
|
||||
* @param doc the {@link Document} on which the the {@code <Period>} element will be appended
|
||||
*/
|
||||
public static void generatePeriodElement(@Nonnull final Document doc)
|
||||
throws CreationException {
|
||||
try {
|
||||
final Element mpdElement = (Element) doc.getElementsByTagName(MPD).item(0);
|
||||
final Element periodElement = doc.createElement(PERIOD);
|
||||
mpdElement.appendChild(periodElement);
|
||||
} catch (final DOMException e) {
|
||||
throw CreationException.couldNotAddElement(PERIOD, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <AdaptationSet>} element, appended as a child of the {@code <Period>}
|
||||
* element.
|
||||
*
|
||||
* <p>
|
||||
* The {@code <Period>} element needs to be generated before this element with
|
||||
* {@link #generatePeriodElement(Document)}.
|
||||
* </p>
|
||||
*
|
||||
* @param doc the {@link Document} on which the {@code <Period>} element will be appended
|
||||
* @param itagItem the {@link ItagItem} corresponding to the stream, which must not be null
|
||||
*/
|
||||
public static void generateAdaptationSetElement(@Nonnull final Document doc,
|
||||
@Nonnull final ItagItem itagItem)
|
||||
throws CreationException {
|
||||
try {
|
||||
final Element periodElement = (Element) doc.getElementsByTagName(PERIOD)
|
||||
.item(0);
|
||||
final Element adaptationSetElement = doc.createElement(ADAPTATION_SET);
|
||||
|
||||
setAttribute(adaptationSetElement, doc, "id", "0");
|
||||
|
||||
final MediaFormat mediaFormat = itagItem.getMediaFormat();
|
||||
if (mediaFormat == null || isNullOrEmpty(mediaFormat.getMimeType())) {
|
||||
throw CreationException.couldNotAddElement(ADAPTATION_SET,
|
||||
"the MediaFormat or its mime type is null or empty");
|
||||
}
|
||||
|
||||
setAttribute(adaptationSetElement, doc, "mimeType", mediaFormat.getMimeType());
|
||||
setAttribute(adaptationSetElement, doc, "subsegmentAlignment", "true");
|
||||
|
||||
periodElement.appendChild(adaptationSetElement);
|
||||
} catch (final DOMException e) {
|
||||
throw CreationException.couldNotAddElement(ADAPTATION_SET, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <Role>} element, appended as a child of the {@code <AdaptationSet>}
|
||||
* element.
|
||||
*
|
||||
* <p>
|
||||
* This element, with its attributes and values, is:
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* {@code <Role schemeIdUri="urn:mpeg:DASH:role:2011" value="main"/>}
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The {@code <AdaptationSet>} element needs to be generated before this element with
|
||||
* {@link #generateAdaptationSetElement(Document, ItagItem)}).
|
||||
* </p>
|
||||
*
|
||||
* @param doc the {@link Document} on which the the {@code <Role>} element will be appended
|
||||
*/
|
||||
public static void generateRoleElement(@Nonnull final Document doc)
|
||||
throws CreationException {
|
||||
try {
|
||||
final Element adaptationSetElement = (Element) doc.getElementsByTagName(
|
||||
ADAPTATION_SET).item(0);
|
||||
final Element roleElement = doc.createElement(ROLE);
|
||||
|
||||
setAttribute(roleElement, doc, "schemeIdUri", "urn:mpeg:DASH:role:2011");
|
||||
setAttribute(roleElement, doc, "value", "main");
|
||||
|
||||
adaptationSetElement.appendChild(roleElement);
|
||||
} catch (final DOMException e) {
|
||||
throw CreationException.couldNotAddElement(ROLE, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <Representation>} element, appended as a child of the
|
||||
* {@code <AdaptationSet>} element.
|
||||
*
|
||||
* <p>
|
||||
* The {@code <AdaptationSet>} element needs to be generated before this element with
|
||||
* {@link #generateAdaptationSetElement(Document, ItagItem)}).
|
||||
* </p>
|
||||
*
|
||||
* @param doc the {@link Document} on which the the {@code <SegmentTimeline>} element will be
|
||||
* appended
|
||||
* @param itagItem the {@link ItagItem} to use, which must not be null
|
||||
*/
|
||||
public static void generateRepresentationElement(@Nonnull final Document doc,
|
||||
@Nonnull final ItagItem itagItem)
|
||||
throws CreationException {
|
||||
try {
|
||||
final Element adaptationSetElement = (Element) doc.getElementsByTagName(
|
||||
ADAPTATION_SET).item(0);
|
||||
final Element representationElement = doc.createElement(REPRESENTATION);
|
||||
|
||||
final int id = itagItem.id;
|
||||
if (id <= 0) {
|
||||
throw CreationException.couldNotAddElement(REPRESENTATION,
|
||||
"the id of the ItagItem is <= 0");
|
||||
}
|
||||
setAttribute(representationElement, doc, "id", String.valueOf(id));
|
||||
|
||||
final String codec = itagItem.getCodec();
|
||||
if (isNullOrEmpty(codec)) {
|
||||
throw CreationException.couldNotAddElement(ADAPTATION_SET,
|
||||
"the codec value of the ItagItem is null or empty");
|
||||
}
|
||||
setAttribute(representationElement, doc, "codecs", codec);
|
||||
setAttribute(representationElement, doc, "startWithSAP", "1");
|
||||
setAttribute(representationElement, doc, "maxPlayoutRate", "1");
|
||||
|
||||
final int bitrate = itagItem.getBitrate();
|
||||
if (bitrate <= 0) {
|
||||
throw CreationException.couldNotAddElement(REPRESENTATION,
|
||||
"the bitrate of the ItagItem is <= 0");
|
||||
}
|
||||
setAttribute(representationElement, doc, "bandwidth", String.valueOf(bitrate));
|
||||
|
||||
if (itagItem.itagType == ItagItem.ItagType.VIDEO
|
||||
|| itagItem.itagType == ItagItem.ItagType.VIDEO_ONLY) {
|
||||
final int height = itagItem.getHeight();
|
||||
final int width = itagItem.getWidth();
|
||||
if (height <= 0 && width <= 0) {
|
||||
throw CreationException.couldNotAddElement(REPRESENTATION,
|
||||
"both width and height of the ItagItem are <= 0");
|
||||
}
|
||||
|
||||
if (width > 0) {
|
||||
setAttribute(representationElement, doc, "width", String.valueOf(width));
|
||||
}
|
||||
setAttribute(representationElement, doc, "height",
|
||||
String.valueOf(itagItem.getHeight()));
|
||||
|
||||
final int fps = itagItem.getFps();
|
||||
if (fps > 0) {
|
||||
setAttribute(representationElement, doc, "frameRate", String.valueOf(fps));
|
||||
}
|
||||
}
|
||||
|
||||
if (itagItem.itagType == ItagItem.ItagType.AUDIO && itagItem.getSampleRate() > 0) {
|
||||
final Attr audioSamplingRateAttribute = doc.createAttribute(
|
||||
"audioSamplingRate");
|
||||
audioSamplingRateAttribute.setValue(String.valueOf(itagItem.getSampleRate()));
|
||||
}
|
||||
|
||||
adaptationSetElement.appendChild(representationElement);
|
||||
} catch (final DOMException e) {
|
||||
throw CreationException.couldNotAddElement(REPRESENTATION, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <AudioChannelConfiguration>} element, appended as a child of the
|
||||
* {@code <Representation>} element.
|
||||
*
|
||||
* <p>
|
||||
* This method is only used when generating DASH manifests of audio streams.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* It will produce the following element:
|
||||
* <br>
|
||||
* {@code <AudioChannelConfiguration
|
||||
* schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011"
|
||||
* value="audioChannelsValue"}
|
||||
* <br>
|
||||
* (where {@code audioChannelsValue} is get from the {@link ItagItem} passed as the second
|
||||
* parameter of this method)
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The {@code <Representation>} element needs to be generated before this element with
|
||||
* {@link #generateRepresentationElement(Document, ItagItem)}).
|
||||
* </p>
|
||||
*
|
||||
* @param doc the {@link Document} on which the {@code <AudioChannelConfiguration>} element will
|
||||
* be appended
|
||||
* @param itagItem the {@link ItagItem} to use, which must not be null
|
||||
*/
|
||||
public static void generateAudioChannelConfigurationElement(
|
||||
@Nonnull final Document doc,
|
||||
@Nonnull final ItagItem itagItem) throws CreationException {
|
||||
try {
|
||||
final Element representationElement = (Element) doc.getElementsByTagName(
|
||||
REPRESENTATION).item(0);
|
||||
final Element audioChannelConfigurationElement = doc.createElement(
|
||||
AUDIO_CHANNEL_CONFIGURATION);
|
||||
|
||||
setAttribute(audioChannelConfigurationElement, doc, "schemeIdUri",
|
||||
"urn:mpeg:dash:23003:3:audio_channel_configuration:2011");
|
||||
|
||||
if (itagItem.getAudioChannels() <= 0) {
|
||||
throw new CreationException("the number of audioChannels in the ItagItem is <= 0: "
|
||||
+ itagItem.getAudioChannels());
|
||||
}
|
||||
setAttribute(audioChannelConfigurationElement, doc, "value",
|
||||
String.valueOf(itagItem.getAudioChannels()));
|
||||
|
||||
representationElement.appendChild(audioChannelConfigurationElement);
|
||||
} catch (final DOMException e) {
|
||||
throw CreationException.couldNotAddElement(AUDIO_CHANNEL_CONFIGURATION, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a DASH manifest {@link Document doc} to a string and cache it.
|
||||
*
|
||||
* @param originalBaseStreamingUrl the original base URL of the stream
|
||||
* @param doc the doc to be converted
|
||||
* @param manifestCreatorCache the {@link ManifestCreatorCache} on which store the string
|
||||
* generated
|
||||
* @return the DASH manifest {@link Document doc} converted to a string
|
||||
*/
|
||||
public static String buildAndCacheResult(
|
||||
@Nonnull final String originalBaseStreamingUrl,
|
||||
@Nonnull final Document doc,
|
||||
@Nonnull final ManifestCreatorCache<String, String> manifestCreatorCache)
|
||||
throws CreationException {
|
||||
|
||||
try {
|
||||
final String documentXml = documentToXml(doc);
|
||||
manifestCreatorCache.put(originalBaseStreamingUrl, documentXml);
|
||||
return documentXml;
|
||||
} catch (final Exception e) {
|
||||
throw new CreationException(
|
||||
"Could not convert the DASH manifest generated to a string", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <SegmentTemplate>} element, appended as a child of the
|
||||
* {@code <Representation>} element.
|
||||
*
|
||||
* <p>
|
||||
* This method is only used when generating DASH manifests from OTF and post-live-DVR streams.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* It will produce a {@code <SegmentTemplate>} element with the following attributes:
|
||||
* <ul>
|
||||
* <li>{@code startNumber}, which takes the value {@code 0} for post-live-DVR streams and
|
||||
* {@code 1} for OTF streams;</li>
|
||||
* <li>{@code timescale}, which is always {@code 1000};</li>
|
||||
* <li>{@code media}, which is the base URL of the stream on which is appended
|
||||
* {@code &sq=$Number$};</li>
|
||||
* <li>{@code initialization} (only for OTF streams), which is the base URL of the stream
|
||||
* on which is appended {@link #SQ_0}.</li>
|
||||
* </ul>
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The {@code <Representation>} element needs to be generated before this element with
|
||||
* {@link #generateRepresentationElement(Document, ItagItem)}).
|
||||
* </p>
|
||||
*
|
||||
* @param doc the {@link Document} on which the {@code <SegmentTemplate>} element will
|
||||
* be appended
|
||||
* @param baseUrl the base URL of the OTF/post-live-DVR stream
|
||||
* @param deliveryType the stream {@link DeliveryType delivery type}, which must be either
|
||||
* {@link DeliveryType#OTF OTF} or {@link DeliveryType#LIVE LIVE}
|
||||
*/
|
||||
public static void generateSegmentTemplateElement(@Nonnull final Document doc,
|
||||
@Nonnull final String baseUrl,
|
||||
final DeliveryType deliveryType)
|
||||
throws CreationException {
|
||||
if (deliveryType != DeliveryType.OTF && deliveryType != DeliveryType.LIVE) {
|
||||
throw CreationException.couldNotAddElement(SEGMENT_TEMPLATE, "invalid delivery type: "
|
||||
+ deliveryType);
|
||||
}
|
||||
|
||||
try {
|
||||
final Element representationElement = (Element) doc.getElementsByTagName(
|
||||
REPRESENTATION).item(0);
|
||||
final Element segmentTemplateElement = doc.createElement(SEGMENT_TEMPLATE);
|
||||
|
||||
// The first sequence of post DVR streams is the beginning of the video stream and not
|
||||
// an initialization segment
|
||||
setAttribute(segmentTemplateElement, doc, "startNumber",
|
||||
deliveryType == DeliveryType.LIVE ? "0" : "1");
|
||||
setAttribute(segmentTemplateElement, doc, "timescale", "1000");
|
||||
|
||||
// Post-live-DVR/ended livestreams streams don't require an initialization sequence
|
||||
if (deliveryType != DeliveryType.LIVE) {
|
||||
setAttribute(segmentTemplateElement, doc, "initialization", baseUrl + SQ_0);
|
||||
}
|
||||
|
||||
setAttribute(segmentTemplateElement, doc, "media", baseUrl + "&sq=$Number$");
|
||||
|
||||
representationElement.appendChild(segmentTemplateElement);
|
||||
} catch (final DOMException e) {
|
||||
throw CreationException.couldNotAddElement(SEGMENT_TEMPLATE, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <SegmentTimeline>} element, appended as a child of the
|
||||
* {@code <SegmentTemplate>} element.
|
||||
*
|
||||
* <p>
|
||||
* The {@code <SegmentTemplate>} element needs to be generated before this element with
|
||||
* {@link #generateSegmentTemplateElement(Document, String, DeliveryType)}.
|
||||
* </p>
|
||||
*
|
||||
* @param doc the {@link Document} on which the the {@code <SegmentTimeline>} element will be
|
||||
* appended
|
||||
*/
|
||||
public static void generateSegmentTimelineElement(@Nonnull final Document doc)
|
||||
throws CreationException {
|
||||
try {
|
||||
final Element segmentTemplateElement = (Element) doc.getElementsByTagName(
|
||||
SEGMENT_TEMPLATE).item(0);
|
||||
final Element segmentTimelineElement = doc.createElement(SEGMENT_TIMELINE);
|
||||
|
||||
segmentTemplateElement.appendChild(segmentTimelineElement);
|
||||
} catch (final DOMException e) {
|
||||
throw CreationException.couldNotAddElement(SEGMENT_TIMELINE, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the "initialization" {@link Response response} of a stream.
|
||||
*
|
||||
* <p>This method fetches, for OTF streams and for post-live-DVR streams:
|
||||
* <ul>
|
||||
* <li>the base URL of the stream, to which are appended {@link #SQ_0} and
|
||||
* {@link #RN_0} parameters, with a {@code GET} request for streaming URLs from HTML5
|
||||
* clients and a {@code POST} request for the ones from the {@code ANDROID} and the
|
||||
* {@code IOS} clients;</li>
|
||||
* <li>for streaming URLs from HTML5 clients, the {@link #ALR_YES} param is also added.
|
||||
* </li>
|
||||
* </ul>
|
||||
* </p>
|
||||
*
|
||||
* @param baseStreamingUrl the base URL of the stream, which must not be null
|
||||
* @param itagItem the {@link ItagItem} of stream, which must not be null
|
||||
* @param deliveryType the {@link DeliveryType} of the stream
|
||||
* @return the "initialization" response, without redirections on the network on which the
|
||||
* request(s) is/are made
|
||||
*/
|
||||
@SuppressWarnings("checkstyle:FinalParameters")
|
||||
@Nonnull
|
||||
public static Response getInitializationResponse(@Nonnull String baseStreamingUrl,
|
||||
@Nonnull final ItagItem itagItem,
|
||||
final DeliveryType deliveryType)
|
||||
throws CreationException {
|
||||
final boolean isHtml5StreamingUrl = isWebStreamingUrl(baseStreamingUrl)
|
||||
|| isTvHtml5SimplyEmbeddedPlayerStreamingUrl(baseStreamingUrl);
|
||||
final boolean isAndroidStreamingUrl = isAndroidStreamingUrl(baseStreamingUrl);
|
||||
final boolean isIosStreamingUrl = isIosStreamingUrl(baseStreamingUrl);
|
||||
if (isHtml5StreamingUrl) {
|
||||
baseStreamingUrl += ALR_YES;
|
||||
}
|
||||
baseStreamingUrl = appendRnSqParamsIfNeeded(baseStreamingUrl, deliveryType);
|
||||
|
||||
final Downloader downloader = NewPipe.getDownloader();
|
||||
if (isHtml5StreamingUrl) {
|
||||
final String mimeTypeExpected = itagItem.getMediaFormat().getMimeType();
|
||||
if (!isNullOrEmpty(mimeTypeExpected)) {
|
||||
return getStreamingWebUrlWithoutRedirects(downloader, baseStreamingUrl,
|
||||
mimeTypeExpected);
|
||||
}
|
||||
} else if (isAndroidStreamingUrl || isIosStreamingUrl) {
|
||||
try {
|
||||
final Map<String, List<String>> headers = Collections.singletonMap("User-Agent",
|
||||
Collections.singletonList(isAndroidStreamingUrl
|
||||
? getAndroidUserAgent(null) : getIosUserAgent(null)));
|
||||
final byte[] emptyBody = "".getBytes(StandardCharsets.UTF_8);
|
||||
return downloader.post(baseStreamingUrl, headers, emptyBody);
|
||||
} catch (final IOException | ExtractionException e) {
|
||||
throw new CreationException("Could not get the "
|
||||
+ (isIosStreamingUrl ? "ANDROID" : "IOS") + " streaming URL response", e);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
return downloader.get(baseStreamingUrl);
|
||||
} catch (final IOException | ExtractionException e) {
|
||||
throw new CreationException("Could not get the streaming URL response", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a new {@link DocumentBuilder} secured from XXE attacks, on platforms which
|
||||
* support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and
|
||||
* {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link DocumentBuilderFactory} instances.
|
||||
*
|
||||
* @return an instance of {@link Document} secured against XXE attacks on supported platforms,
|
||||
* that should then be convertible to an XML string without security problems
|
||||
*/
|
||||
private static Document newDocument() throws ParserConfigurationException {
|
||||
final DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
|
||||
try {
|
||||
documentBuilderFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
|
||||
documentBuilderFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
|
||||
} catch (final Exception ignored) {
|
||||
// Ignore exceptions as setting these attributes to secure XML generation is not
|
||||
// supported by all platforms (like the Android implementation)
|
||||
}
|
||||
|
||||
return documentBuilderFactory.newDocumentBuilder().newDocument();
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a new {@link TransformerFactory} secured from XXE attacks, on platforms which
|
||||
* support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and
|
||||
* {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link TransformerFactory} instances.
|
||||
*
|
||||
* @param doc the doc to convert, which must have been created using {@link #newDocument()} to
|
||||
* properly prevent XXE attacks
|
||||
* @return the doc converted to an XML string, making sure there can't be XXE attacks
|
||||
*/
|
||||
// Sonar warning is suppressed because it is still shown even if we apply its solution
|
||||
@SuppressWarnings("squid:S2755")
|
||||
private static String documentToXml(@Nonnull final Document doc)
|
||||
throws TransformerException {
|
||||
|
||||
final TransformerFactory transformerFactory = TransformerFactory.newInstance();
|
||||
try {
|
||||
transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
|
||||
transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
|
||||
} catch (final Exception ignored) {
|
||||
// Ignore exceptions as setting these attributes to secure XML generation is not
|
||||
// supported by all platforms (like the Android implementation)
|
||||
}
|
||||
|
||||
final Transformer transformer = transformerFactory.newTransformer();
|
||||
transformer.setOutputProperty(OutputKeys.VERSION, "1.0");
|
||||
transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");
|
||||
transformer.setOutputProperty(OutputKeys.STANDALONE, "no");
|
||||
|
||||
final StringWriter result = new StringWriter();
|
||||
transformer.transform(new DOMSource(doc), new StreamResult(result));
|
||||
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Append {@link #SQ_0} for post-live-DVR and OTF streams and {@link #RN_0} to all streams.
|
||||
*
|
||||
* @param baseStreamingUrl the base streaming URL to which the parameter(s) are being appended
|
||||
* @param deliveryType the {@link DeliveryType} of the stream
|
||||
* @return the base streaming URL to which the param(s) are appended, depending on the
|
||||
* {@link DeliveryType} of the stream
|
||||
*/
|
||||
@Nonnull
|
||||
private static String appendRnSqParamsIfNeeded(@Nonnull final String baseStreamingUrl,
|
||||
@Nonnull final DeliveryType deliveryType) {
|
||||
return baseStreamingUrl + (deliveryType == DeliveryType.PROGRESSIVE ? "" : SQ_0) + RN_0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a URL on which no redirection between playback hosts should be present on the network
|
||||
* and/or IP used to fetch the streaming URL, for HTML5 clients.
|
||||
*
|
||||
* <p>This method will follow redirects which works in the following way:
|
||||
* <ol>
|
||||
* <li>the {@link #ALR_YES} param is appended to all streaming URLs</li>
|
||||
* <li>if no redirection occurs, the video server will return the streaming data;</li>
|
||||
* <li>if a redirection occurs, the server will respond with HTTP status code 200 and a
|
||||
* {@code text/plain} mime type. The redirection URL is the response body;</li>
|
||||
* <li>the redirection URL is requested and the steps above from step 2 are repeated,
|
||||
* until too many redirects are reached of course (the maximum number of redirects is
|
||||
* {@link #MAXIMUM_REDIRECT_COUNT the same as OkHttp}).</li>
|
||||
* </ol>
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* For non-HTML5 clients, redirections are managed in the standard way in
|
||||
* {@link #getInitializationResponse(String, ItagItem, DeliveryType)}.
|
||||
* </p>
|
||||
*
|
||||
* @param downloader the {@link Downloader} instance to be used
|
||||
* @param streamingUrl the streaming URL which we are trying to get a streaming URL
|
||||
* without any redirection on the network and/or IP used
|
||||
* @param responseMimeTypeExpected the response mime type expected from Google video servers
|
||||
* @return the {@link Response} of the stream, which should have no redirections
|
||||
*/
|
||||
@SuppressWarnings("checkstyle:FinalParameters")
|
||||
@Nonnull
|
||||
private static Response getStreamingWebUrlWithoutRedirects(
|
||||
@Nonnull final Downloader downloader,
|
||||
@Nonnull String streamingUrl,
|
||||
@Nonnull final String responseMimeTypeExpected)
|
||||
throws CreationException {
|
||||
try {
|
||||
final Map<String, List<String>> headers = new HashMap<>();
|
||||
addClientInfoHeaders(headers);
|
||||
|
||||
String responseMimeType = "";
|
||||
|
||||
int redirectsCount = 0;
|
||||
while (!responseMimeType.equals(responseMimeTypeExpected)
|
||||
&& redirectsCount < MAXIMUM_REDIRECT_COUNT) {
|
||||
final Response response = downloader.get(streamingUrl, headers);
|
||||
|
||||
final int responseCode = response.responseCode();
|
||||
if (responseCode != 200) {
|
||||
throw new CreationException(
|
||||
"Could not get the initialization URL: HTTP response code "
|
||||
+ responseCode);
|
||||
}
|
||||
|
||||
// A valid HTTP 1.0+ response should include a Content-Type header, so we can
|
||||
// require that the response from video servers has this header.
|
||||
responseMimeType = Objects.requireNonNull(response.getHeader("Content-Type"),
|
||||
"Could not get the Content-Type header from the response headers");
|
||||
|
||||
// The response body is the redirection URL
|
||||
if (responseMimeType.equals("text/plain")) {
|
||||
streamingUrl = response.responseBody();
|
||||
redirectsCount++;
|
||||
} else {
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
if (redirectsCount >= MAXIMUM_REDIRECT_COUNT) {
|
||||
throw new CreationException(
|
||||
"Too many redirects when trying to get the the streaming URL response of a "
|
||||
+ "HTML5 client");
|
||||
}
|
||||
|
||||
// This should never be reached, but is required because we don't want to return null
|
||||
// here
|
||||
throw new CreationException(
|
||||
"Could not get the streaming URL response of a HTML5 client: unreachable code "
|
||||
+ "reached!");
|
||||
} catch (final IOException | ExtractionException e) {
|
||||
throw new CreationException(
|
||||
"Could not get the streaming URL response of a HTML5 client", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,265 +0,0 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ALR_YES;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.RN_0;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TIMELINE;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SQ_0;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.buildAndCacheResult;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateDocumentAndDoCommonElementsGeneration;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTemplateElement;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTimelineElement;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.getInitializationResponse;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.setAttribute;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isBlank;
|
||||
|
||||
import org.schabi.newpipe.extractor.downloader.Response;
|
||||
import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
|
||||
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
|
||||
import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
import org.w3c.dom.DOMException;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Objects;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
/**
|
||||
* Class which generates DASH manifests of YouTube {@link DeliveryType#OTF OTF streams}.
|
||||
*/
|
||||
public final class YoutubeOtfDashManifestCreator {
|
||||
|
||||
/**
|
||||
* Cache of DASH manifests generated for OTF streams.
|
||||
*/
|
||||
private static final ManifestCreatorCache<String, String> OTF_STREAMS_CACHE
|
||||
= new ManifestCreatorCache<>();
|
||||
|
||||
private YoutubeOtfDashManifestCreator() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Create DASH manifests from a YouTube OTF stream.
|
||||
*
|
||||
* <p>
|
||||
* OTF streams are YouTube-DASH specific streams which work with sequences and without the need
|
||||
* to get a manifest (even if one is provided, it is not used by official clients).
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* They can be found only on videos; mostly those with a small amount of views, or ended
|
||||
* livestreams which have just been re-encoded as normal videos.
|
||||
* </p>
|
||||
*
|
||||
* <p>This method needs:
|
||||
* <ul>
|
||||
* <li>the base URL of the stream (which, if you try to access to it, returns HTTP
|
||||
* status code 404 after redirects, and if the URL is valid);</li>
|
||||
* <li>an {@link ItagItem}, which needs to contain the following information:
|
||||
* <ul>
|
||||
* <li>its type (see {@link ItagItem.ItagType}), to identify if the content is
|
||||
* an audio or a video stream;</li>
|
||||
* <li>its bitrate;</li>
|
||||
* <li>its mime type;</li>
|
||||
* <li>its codec(s);</li>
|
||||
* <li>for an audio stream: its audio channels;</li>
|
||||
* <li>for a video stream: its width and height.</li>
|
||||
* </ul>
|
||||
* </li>
|
||||
* <li>the duration of the video, which will be used if the duration could not be
|
||||
* parsed from the first sequence of the stream.</li>
|
||||
* </ul>
|
||||
* </p>
|
||||
*
|
||||
* <p>In order to generate the DASH manifest, this method will:
|
||||
* <ul>
|
||||
* <li>request the first sequence of the stream (the base URL on which the first
|
||||
* sequence parameter is appended (see {@link YoutubeDashManifestCreatorsUtils#SQ_0}))
|
||||
* with a {@code POST} or {@code GET} request (depending of the client on which the
|
||||
* streaming URL comes from is a mobile one ({@code POST}) or not ({@code GET}));</li>
|
||||
* <li>follow its redirection(s), if any;</li>
|
||||
* <li>save the last URL, remove the first sequence parameter;</li>
|
||||
* <li>use the information provided in the {@link ItagItem} to generate all
|
||||
* elements of the DASH manifest.</li>
|
||||
* </ul>
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* If the duration cannot be extracted, the {@code durationSecondsFallback} value will be used
|
||||
* as the stream duration.
|
||||
* </p>
|
||||
*
|
||||
* @param otfBaseStreamingUrl the base URL of the OTF stream, which must not be null
|
||||
* @param itagItem the {@link ItagItem} corresponding to the stream, which
|
||||
* must not be null
|
||||
* @param durationSecondsFallback the duration of the video, which will be used if the duration
|
||||
* could not be extracted from the first sequence
|
||||
* @return the manifest generated into a string
|
||||
*/
|
||||
@Nonnull
|
||||
public static String fromOtfStreamingUrl(
|
||||
@Nonnull final String otfBaseStreamingUrl,
|
||||
@Nonnull final ItagItem itagItem,
|
||||
final long durationSecondsFallback) throws CreationException {
|
||||
if (OTF_STREAMS_CACHE.containsKey(otfBaseStreamingUrl)) {
|
||||
return Objects.requireNonNull(OTF_STREAMS_CACHE.get(otfBaseStreamingUrl)).getSecond();
|
||||
}
|
||||
|
||||
String realOtfBaseStreamingUrl = otfBaseStreamingUrl;
|
||||
// Try to avoid redirects when streaming the content by saving the last URL we get
|
||||
// from video servers.
|
||||
final Response response = getInitializationResponse(realOtfBaseStreamingUrl,
|
||||
itagItem, DeliveryType.OTF);
|
||||
realOtfBaseStreamingUrl = response.latestUrl().replace(SQ_0, "")
|
||||
.replace(RN_0, "").replace(ALR_YES, "");
|
||||
|
||||
final int responseCode = response.responseCode();
|
||||
if (responseCode != 200) {
|
||||
throw new CreationException("Could not get the initialization URL: response code "
|
||||
+ responseCode);
|
||||
}
|
||||
|
||||
final String[] segmentDuration;
|
||||
|
||||
try {
|
||||
final String[] segmentsAndDurationsResponseSplit = response.responseBody()
|
||||
// Get the lines with the durations and the following
|
||||
.split("Segment-Durations-Ms: ")[1]
|
||||
// Remove the other lines
|
||||
.split("\n")[0]
|
||||
// Get all durations and repetitions which are separated by a comma
|
||||
.split(",");
|
||||
final int lastIndex = segmentsAndDurationsResponseSplit.length - 1;
|
||||
if (isBlank(segmentsAndDurationsResponseSplit[lastIndex])) {
|
||||
segmentDuration = Arrays.copyOf(segmentsAndDurationsResponseSplit, lastIndex);
|
||||
} else {
|
||||
segmentDuration = segmentsAndDurationsResponseSplit;
|
||||
}
|
||||
} catch (final Exception e) {
|
||||
throw new CreationException("Could not get segment durations", e);
|
||||
}
|
||||
|
||||
long streamDuration;
|
||||
try {
|
||||
streamDuration = getStreamDuration(segmentDuration);
|
||||
} catch (final CreationException e) {
|
||||
streamDuration = durationSecondsFallback * 1000;
|
||||
}
|
||||
|
||||
final Document doc = generateDocumentAndDoCommonElementsGeneration(itagItem,
|
||||
streamDuration);
|
||||
|
||||
generateSegmentTemplateElement(doc, realOtfBaseStreamingUrl, DeliveryType.OTF);
|
||||
generateSegmentTimelineElement(doc);
|
||||
generateSegmentElementsForOtfStreams(segmentDuration, doc);
|
||||
|
||||
return buildAndCacheResult(otfBaseStreamingUrl, doc, OTF_STREAMS_CACHE);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the cache of DASH manifests generated for OTF streams
|
||||
*/
|
||||
@Nonnull
|
||||
public static ManifestCreatorCache<String, String> getCache() {
|
||||
return OTF_STREAMS_CACHE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate segment elements for OTF streams.
|
||||
*
|
||||
* <p>
|
||||
* By parsing by the first media sequence, we know how many durations and repetitions there are
|
||||
* so we just have to loop into segment durations to generate the following elements for each
|
||||
* duration repeated X times:
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* {@code <S d="segmentDuration" r="durationRepetition" />}
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* If there is no repetition of the duration between two segments, the {@code r} attribute is
|
||||
* not added to the {@code S} element, as it is not needed.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* These elements will be appended as children of the {@code <SegmentTimeline>} element, which
|
||||
* needs to be generated before these elements with
|
||||
* {@link YoutubeDashManifestCreatorsUtils#generateSegmentTimelineElement(Document)}.
|
||||
* </p>
|
||||
*
|
||||
* @param segmentDurations the sequences "length" or "length(r=repeat_count" extracted with the
|
||||
* regular expressions
|
||||
* @param doc the {@link Document} on which the {@code <S>} elements will be
|
||||
* appended
|
||||
*/
|
||||
private static void generateSegmentElementsForOtfStreams(
|
||||
@Nonnull final String[] segmentDurations,
|
||||
@Nonnull final Document doc) throws CreationException {
|
||||
try {
|
||||
final Element segmentTimelineElement = (Element) doc.getElementsByTagName(
|
||||
SEGMENT_TIMELINE).item(0);
|
||||
|
||||
for (final String segmentDuration : segmentDurations) {
|
||||
final Element sElement = doc.createElement("S");
|
||||
|
||||
final String[] segmentLengthRepeat = segmentDuration.split("\\(r=");
|
||||
// make sure segmentLengthRepeat[0], which is the length, is convertible to int
|
||||
Integer.parseInt(segmentLengthRepeat[0]);
|
||||
|
||||
// There are repetitions of a segment duration in other segments
|
||||
if (segmentLengthRepeat.length > 1) {
|
||||
final int segmentRepeatCount = Integer.parseInt(
|
||||
Utils.removeNonDigitCharacters(segmentLengthRepeat[1]));
|
||||
setAttribute(sElement, doc, "r", String.valueOf(segmentRepeatCount));
|
||||
}
|
||||
setAttribute(sElement, doc, "d", segmentLengthRepeat[0]);
|
||||
|
||||
segmentTimelineElement.appendChild(sElement);
|
||||
}
|
||||
|
||||
} catch (final DOMException | IllegalStateException | IndexOutOfBoundsException
|
||||
| NumberFormatException e) {
|
||||
throw CreationException.couldNotAddElement("segment (S)", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the duration of an OTF stream.
|
||||
*
|
||||
* <p>
|
||||
* The duration of OTF streams is not returned into the player response and needs to be
|
||||
* calculated by adding the duration of each segment.
|
||||
* </p>
|
||||
*
|
||||
* @param segmentDuration the segment duration object extracted from the initialization
|
||||
* sequence of the stream
|
||||
* @return the duration of the OTF stream, in milliseconds
|
||||
*/
|
||||
private static long getStreamDuration(@Nonnull final String[] segmentDuration)
|
||||
throws CreationException {
|
||||
try {
|
||||
long streamLengthMs = 0;
|
||||
|
||||
for (final String segDuration : segmentDuration) {
|
||||
final String[] segmentLengthRepeat = segDuration.split("\\(r=");
|
||||
long segmentRepeatCount = 0;
|
||||
|
||||
// There are repetitions of a segment duration in other segments
|
||||
if (segmentLengthRepeat.length > 1) {
|
||||
segmentRepeatCount = Long.parseLong(Utils.removeNonDigitCharacters(
|
||||
segmentLengthRepeat[1]));
|
||||
}
|
||||
|
||||
final long segmentLength = Integer.parseInt(segmentLengthRepeat[0]);
|
||||
streamLengthMs += segmentLength + segmentRepeatCount * segmentLength;
|
||||
}
|
||||
|
||||
return streamLengthMs;
|
||||
} catch (final NumberFormatException e) {
|
||||
throw new CreationException("Could not get stream length from sequences list", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,217 +0,0 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ALR_YES;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.RN_0;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TIMELINE;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SQ_0;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.buildAndCacheResult;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateDocumentAndDoCommonElementsGeneration;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTemplateElement;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTimelineElement;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.getInitializationResponse;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.setAttribute;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
import org.schabi.newpipe.extractor.downloader.Response;
|
||||
import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
|
||||
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
|
||||
import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
|
||||
import org.w3c.dom.DOMException;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
/**
|
||||
* Class which generates DASH manifests of YouTube post-live DVR streams (which use the
|
||||
* {@link DeliveryType#LIVE LIVE delivery type}).
|
||||
*/
|
||||
public final class YoutubePostLiveStreamDvrDashManifestCreator {
|
||||
|
||||
/**
|
||||
* Cache of DASH manifests generated for post-live-DVR streams.
|
||||
*/
|
||||
private static final ManifestCreatorCache<String, String> POST_LIVE_DVR_STREAMS_CACHE
|
||||
= new ManifestCreatorCache<>();
|
||||
|
||||
private YoutubePostLiveStreamDvrDashManifestCreator() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Create DASH manifests from a YouTube post-live-DVR stream/ended livestream.
|
||||
*
|
||||
* <p>
|
||||
* Post-live-DVR streams/ended livestreams are one of the YouTube DASH specific streams which
|
||||
* works with sequences and without the need to get a manifest (even if one is provided but not
|
||||
* used by main clients (and is not complete for big ended livestreams because it doesn't
|
||||
* return the full stream)).
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* They can be found only on livestreams which have ended very recently (a few hours, most of
|
||||
* the time)
|
||||
* </p>
|
||||
*
|
||||
* <p>This method needs:
|
||||
* <ul>
|
||||
* <li>the base URL of the stream (which, if you try to access to it, returns HTTP
|
||||
* status code 404 after redirects, and if the URL is valid);</li>
|
||||
* <li>an {@link ItagItem}, which needs to contain the following information:
|
||||
* <ul>
|
||||
* <li>its type (see {@link ItagItem.ItagType}), to identify if the content is
|
||||
* an audio or a video stream;</li>
|
||||
* <li>its bitrate;</li>
|
||||
* <li>its mime type;</li>
|
||||
* <li>its codec(s);</li>
|
||||
* <li>for an audio stream: its audio channels;</li>
|
||||
* <li>for a video stream: its width and height.</li>
|
||||
* </ul>
|
||||
* </li>
|
||||
* <li>the duration of the video, which will be used if the duration could not be
|
||||
* parsed from the first sequence of the stream.</li>
|
||||
* </ul>
|
||||
* </p>
|
||||
*
|
||||
* <p>In order to generate the DASH manifest, this method will:
|
||||
* <ul>
|
||||
* <li>request the first sequence of the stream (the base URL on which the first
|
||||
* sequence parameter is appended (see {@link YoutubeDashManifestCreatorsUtils#SQ_0}))
|
||||
* with a {@code POST} or {@code GET} request (depending of the client on which the
|
||||
* streaming URL comes from is a mobile one ({@code POST}) or not ({@code GET}));</li>
|
||||
* <li>follow its redirection(s), if any;</li>
|
||||
* <li>save the last URL, remove the first sequence parameters;</li>
|
||||
* <li>use the information provided in the {@link ItagItem} to generate all elements
|
||||
* of the DASH manifest.</li>
|
||||
* </ul>
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* If the duration cannot be extracted, the {@code durationSecondsFallback} value will be used
|
||||
* as the stream duration.
|
||||
* </p>
|
||||
*
|
||||
* @param postLiveStreamDvrStreamingUrl the base URL of the post-live-DVR stream/ended
|
||||
* livestream, which must not be null
|
||||
* @param itagItem the {@link ItagItem} corresponding to the stream, which
|
||||
* must not be null
|
||||
* @param targetDurationSec the target duration of each sequence, in seconds (this
|
||||
* value is returned with the {@code targetDurationSec}
|
||||
* field for each stream in YouTube's player response)
|
||||
* @param durationSecondsFallback the duration of the ended livestream, which will be
|
||||
* used if the duration could not be extracted from the
|
||||
* first sequence
|
||||
* @return the manifest generated into a string
|
||||
*/
|
||||
@Nonnull
|
||||
public static String fromPostLiveStreamDvrStreamingUrl(
|
||||
@Nonnull final String postLiveStreamDvrStreamingUrl,
|
||||
@Nonnull final ItagItem itagItem,
|
||||
final int targetDurationSec,
|
||||
final long durationSecondsFallback) throws CreationException {
|
||||
if (POST_LIVE_DVR_STREAMS_CACHE.containsKey(postLiveStreamDvrStreamingUrl)) {
|
||||
return Objects.requireNonNull(
|
||||
POST_LIVE_DVR_STREAMS_CACHE.get(postLiveStreamDvrStreamingUrl)).getSecond();
|
||||
}
|
||||
|
||||
String realPostLiveStreamDvrStreamingUrl = postLiveStreamDvrStreamingUrl;
|
||||
final String streamDurationString;
|
||||
final String segmentCount;
|
||||
|
||||
if (targetDurationSec <= 0) {
|
||||
throw new CreationException("targetDurationSec value is <= 0: " + targetDurationSec);
|
||||
}
|
||||
|
||||
try {
|
||||
// Try to avoid redirects when streaming the content by saving the latest URL we get
|
||||
// from video servers.
|
||||
final Response response = getInitializationResponse(realPostLiveStreamDvrStreamingUrl,
|
||||
itagItem, DeliveryType.LIVE);
|
||||
realPostLiveStreamDvrStreamingUrl = response.latestUrl().replace(SQ_0, "")
|
||||
.replace(RN_0, "").replace(ALR_YES, "");
|
||||
|
||||
final int responseCode = response.responseCode();
|
||||
if (responseCode != 200) {
|
||||
throw new CreationException(
|
||||
"Could not get the initialization sequence: response code " + responseCode);
|
||||
}
|
||||
|
||||
final Map<String, List<String>> responseHeaders = response.responseHeaders();
|
||||
streamDurationString = responseHeaders.get("X-Head-Time-Millis").get(0);
|
||||
segmentCount = responseHeaders.get("X-Head-Seqnum").get(0);
|
||||
} catch (final IndexOutOfBoundsException e) {
|
||||
throw new CreationException(
|
||||
"Could not get the value of the X-Head-Time-Millis or the X-Head-Seqnum header",
|
||||
e);
|
||||
}
|
||||
|
||||
if (isNullOrEmpty(segmentCount)) {
|
||||
throw new CreationException("Could not get the number of segments");
|
||||
}
|
||||
|
||||
long streamDuration;
|
||||
try {
|
||||
streamDuration = Long.parseLong(streamDurationString);
|
||||
} catch (final NumberFormatException e) {
|
||||
streamDuration = durationSecondsFallback;
|
||||
}
|
||||
|
||||
final Document doc = generateDocumentAndDoCommonElementsGeneration(itagItem,
|
||||
streamDuration);
|
||||
|
||||
generateSegmentTemplateElement(doc, realPostLiveStreamDvrStreamingUrl,
|
||||
DeliveryType.LIVE);
|
||||
generateSegmentTimelineElement(doc);
|
||||
generateSegmentElementForPostLiveDvrStreams(doc, targetDurationSec, segmentCount);
|
||||
|
||||
return buildAndCacheResult(postLiveStreamDvrStreamingUrl, doc,
|
||||
POST_LIVE_DVR_STREAMS_CACHE);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the cache of DASH manifests generated for post-live-DVR streams
|
||||
*/
|
||||
@Nonnull
|
||||
public static ManifestCreatorCache<String, String> getCache() {
|
||||
return POST_LIVE_DVR_STREAMS_CACHE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the segment ({@code <S>}) element.
|
||||
*
|
||||
* <p>
|
||||
* We don't know the exact duration of segments for post-live-DVR streams but an
|
||||
* average instead (which is the {@code targetDurationSec} value), so we can use the following
|
||||
* structure to generate the segment timeline for DASH manifests of ended livestreams:
|
||||
* <br>
|
||||
* {@code <S d="targetDurationSecValue" r="segmentCount" />}
|
||||
* </p>
|
||||
*
|
||||
* @param doc the {@link Document} on which the {@code <S>} element will
|
||||
* be appended
|
||||
* @param targetDurationSeconds the {@code targetDurationSec} value from YouTube player
|
||||
* response's stream
|
||||
* @param segmentCount the number of segments, extracted by {@link
|
||||
* #fromPostLiveStreamDvrStreamingUrl(String, ItagItem, int, long)}
|
||||
*/
|
||||
private static void generateSegmentElementForPostLiveDvrStreams(
|
||||
@Nonnull final Document doc,
|
||||
final int targetDurationSeconds,
|
||||
@Nonnull final String segmentCount) throws CreationException {
|
||||
try {
|
||||
final Element segmentTimelineElement = (Element) doc.getElementsByTagName(
|
||||
SEGMENT_TIMELINE).item(0);
|
||||
final Element sElement = doc.createElement("S");
|
||||
|
||||
setAttribute(sElement, doc, "d", String.valueOf(targetDurationSeconds * 1000));
|
||||
setAttribute(sElement, doc, "r", segmentCount);
|
||||
|
||||
segmentTimelineElement.appendChild(sElement);
|
||||
} catch (final DOMException e) {
|
||||
throw CreationException.couldNotAddElement("segment (S)", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,235 +0,0 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
|
||||
|
||||
import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
|
||||
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
|
||||
import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
|
||||
import org.w3c.dom.DOMException;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import java.util.Objects;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.BASE_URL;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.INITIALIZATION;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.MPD;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.REPRESENTATION;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_BASE;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.buildAndCacheResult;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateDocumentAndDoCommonElementsGeneration;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.setAttribute;
|
||||
|
||||
/**
|
||||
* Class which generates DASH manifests of {@link DeliveryType#PROGRESSIVE YouTube progressive}
|
||||
* streams.
|
||||
*/
|
||||
public final class YoutubeProgressiveDashManifestCreator {
|
||||
|
||||
/**
|
||||
* Cache of DASH manifests generated for progressive streams.
|
||||
*/
|
||||
private static final ManifestCreatorCache<String, String> PROGRESSIVE_STREAMS_CACHE
|
||||
= new ManifestCreatorCache<>();
|
||||
|
||||
private YoutubeProgressiveDashManifestCreator() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Create DASH manifests from a YouTube progressive stream.
|
||||
*
|
||||
* <p>
|
||||
* Progressive streams are YouTube DASH streams which work with range requests and without the
|
||||
* need to get a manifest.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* They can be found on all videos, and for all streams for most of videos which come from a
|
||||
* YouTube partner, and on videos with a large number of views.
|
||||
* </p>
|
||||
*
|
||||
* <p>This method needs:
|
||||
* <ul>
|
||||
* <li>the base URL of the stream (which, if you try to access to it, returns the whole
|
||||
* stream, after redirects, and if the URL is valid);</li>
|
||||
* <li>an {@link ItagItem}, which needs to contain the following information:
|
||||
* <ul>
|
||||
* <li>its type (see {@link ItagItem.ItagType}), to identify if the content is
|
||||
* an audio or a video stream;</li>
|
||||
* <li>its bitrate;</li>
|
||||
* <li>its mime type;</li>
|
||||
* <li>its codec(s);</li>
|
||||
* <li>for an audio stream: its audio channels;</li>
|
||||
* <li>for a video stream: its width and height.</li>
|
||||
* </ul>
|
||||
* </li>
|
||||
* <li>the duration of the video (parameter {@code durationSecondsFallback}), which
|
||||
* will be used as the stream duration if the duration could not be parsed from the
|
||||
* {@link ItagItem}.</li>
|
||||
* </ul>
|
||||
* </p>
|
||||
*
|
||||
* @param progressiveStreamingBaseUrl the base URL of the progressive stream, which must not be
|
||||
* null
|
||||
* @param itagItem the {@link ItagItem} corresponding to the stream, which
|
||||
* must not be null
|
||||
* @param durationSecondsFallback the duration of the progressive stream which will be used
|
||||
* if the duration could not be extracted from the
|
||||
* {@link ItagItem}
|
||||
* @return the manifest generated into a string
|
||||
*/
|
||||
@Nonnull
|
||||
public static String fromProgressiveStreamingUrl(
|
||||
@Nonnull final String progressiveStreamingBaseUrl,
|
||||
@Nonnull final ItagItem itagItem,
|
||||
final long durationSecondsFallback) throws CreationException {
|
||||
if (PROGRESSIVE_STREAMS_CACHE.containsKey(progressiveStreamingBaseUrl)) {
|
||||
return Objects.requireNonNull(
|
||||
PROGRESSIVE_STREAMS_CACHE.get(progressiveStreamingBaseUrl)).getSecond();
|
||||
}
|
||||
|
||||
final long itagItemDuration = itagItem.getApproxDurationMs();
|
||||
final long streamDuration;
|
||||
if (itagItemDuration != -1) {
|
||||
streamDuration = itagItemDuration;
|
||||
} else {
|
||||
if (durationSecondsFallback > 0) {
|
||||
streamDuration = durationSecondsFallback * 1000;
|
||||
} else {
|
||||
throw CreationException.couldNotAddElement(MPD, "the duration of the stream "
|
||||
+ "could not be determined and durationSecondsFallback is <= 0");
|
||||
}
|
||||
}
|
||||
|
||||
final Document doc = generateDocumentAndDoCommonElementsGeneration(itagItem,
|
||||
streamDuration);
|
||||
|
||||
generateBaseUrlElement(doc, progressiveStreamingBaseUrl);
|
||||
generateSegmentBaseElement(doc, itagItem);
|
||||
generateInitializationElement(doc, itagItem);
|
||||
|
||||
return buildAndCacheResult(progressiveStreamingBaseUrl, doc,
|
||||
PROGRESSIVE_STREAMS_CACHE);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the cache of DASH manifests generated for progressive streams
|
||||
*/
|
||||
@Nonnull
|
||||
public static ManifestCreatorCache<String, String> getCache() {
|
||||
return PROGRESSIVE_STREAMS_CACHE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <BaseURL>} element, appended as a child of the
|
||||
* {@code <Representation>} element.
|
||||
*
|
||||
* <p>
|
||||
* The {@code <Representation>} element needs to be generated before this element with
|
||||
* {@link YoutubeDashManifestCreatorsUtils#generateRepresentationElement(Document, ItagItem)}).
|
||||
* </p>
|
||||
*
|
||||
* @param doc the {@link Document} on which the {@code <BaseURL>} element will be appended
|
||||
* @param baseUrl the base URL of the stream, which must not be null and will be set as the
|
||||
* content of the {@code <BaseURL>} element
|
||||
*/
|
||||
private static void generateBaseUrlElement(@Nonnull final Document doc,
|
||||
@Nonnull final String baseUrl)
|
||||
throws CreationException {
|
||||
try {
|
||||
final Element representationElement = (Element) doc.getElementsByTagName(
|
||||
REPRESENTATION).item(0);
|
||||
final Element baseURLElement = doc.createElement(BASE_URL);
|
||||
baseURLElement.setTextContent(baseUrl);
|
||||
representationElement.appendChild(baseURLElement);
|
||||
} catch (final DOMException e) {
|
||||
throw CreationException.couldNotAddElement(BASE_URL, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <SegmentBase>} element, appended as a child of the
|
||||
* {@code <Representation>} element.
|
||||
*
|
||||
* <p>
|
||||
* It generates the following element:
|
||||
* <br>
|
||||
* {@code <SegmentBase indexRange="indexStart-indexEnd" />}
|
||||
* <br>
|
||||
* (where {@code indexStart} and {@code indexEnd} are gotten from the {@link ItagItem} passed
|
||||
* as the second parameter)
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The {@code <Representation>} element needs to be generated before this element with
|
||||
* {@link YoutubeDashManifestCreatorsUtils#generateRepresentationElement(Document, ItagItem)}),
|
||||
* and the {@code BaseURL} element with {@link #generateBaseUrlElement(Document, String)}
|
||||
* should be generated too.
|
||||
* </p>
|
||||
*
|
||||
* @param doc the {@link Document} on which the {@code <SegmentBase>} element will be appended
|
||||
* @param itagItem the {@link ItagItem} to use, which must not be null
|
||||
*/
|
||||
private static void generateSegmentBaseElement(@Nonnull final Document doc,
|
||||
@Nonnull final ItagItem itagItem)
|
||||
throws CreationException {
|
||||
try {
|
||||
final Element representationElement = (Element) doc.getElementsByTagName(
|
||||
REPRESENTATION).item(0);
|
||||
final Element segmentBaseElement = doc.createElement(SEGMENT_BASE);
|
||||
|
||||
final String range = itagItem.getIndexStart() + "-" + itagItem.getIndexEnd();
|
||||
if (itagItem.getIndexStart() < 0 || itagItem.getIndexEnd() < 0) {
|
||||
throw CreationException.couldNotAddElement(SEGMENT_BASE,
|
||||
"ItagItem's indexStart or " + "indexEnd are < 0: " + range);
|
||||
}
|
||||
setAttribute(segmentBaseElement, doc, "indexRange", range);
|
||||
|
||||
representationElement.appendChild(segmentBaseElement);
|
||||
} catch (final DOMException e) {
|
||||
throw CreationException.couldNotAddElement(SEGMENT_BASE, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the {@code <Initialization>} element, appended as a child of the
|
||||
* {@code <SegmentBase>} element.
|
||||
*
|
||||
* <p>
|
||||
* It generates the following element:
|
||||
* <br>
|
||||
* {@code <Initialization range="initStart-initEnd"/>}
|
||||
* <br>
|
||||
* (where {@code indexStart} and {@code indexEnd} are gotten from the {@link ItagItem} passed
|
||||
* as the second parameter)
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The {@code <SegmentBase>} element needs to be generated before this element with
|
||||
* {@link #generateSegmentBaseElement(Document, ItagItem)}).
|
||||
* </p>
|
||||
*
|
||||
* @param doc the {@link Document} on which the {@code <Initialization>} element will be
|
||||
* appended
|
||||
* @param itagItem the {@link ItagItem} to use, which must not be null
|
||||
*/
|
||||
private static void generateInitializationElement(@Nonnull final Document doc,
|
||||
@Nonnull final ItagItem itagItem)
|
||||
throws CreationException {
|
||||
try {
|
||||
final Element segmentBaseElement = (Element) doc.getElementsByTagName(
|
||||
SEGMENT_BASE).item(0);
|
||||
final Element initializationElement = doc.createElement(INITIALIZATION);
|
||||
|
||||
final String range = itagItem.getInitStart() + "-" + itagItem.getInitEnd();
|
||||
if (itagItem.getInitStart() < 0 || itagItem.getInitEnd() < 0) {
|
||||
throw CreationException.couldNotAddElement(INITIALIZATION,
|
||||
"ItagItem's initStart and/or " + "initEnd are/is < 0: " + range);
|
||||
}
|
||||
setAttribute(initializationElement, doc, "range", range);
|
||||
|
||||
segmentBaseElement.appendChild(initializationElement);
|
||||
} catch (final DOMException e) {
|
||||
throw CreationException.couldNotAddElement(INITIALIZATION, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -9,11 +9,19 @@ public class ItagInfoRange {
|
|||
this.end = end;
|
||||
}
|
||||
|
||||
public int getStart() {
|
||||
public int start() {
|
||||
return start;
|
||||
}
|
||||
|
||||
public int getEnd() {
|
||||
public int end() {
|
||||
return end;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "ItagInfoRange{" +
|
||||
"start=" + start +
|
||||
", end=" + end +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
package org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
/**
|
||||
* Exception that is thrown when a DASH manifest creator encounters a problem
|
||||
* while creating a manifest.
|
||||
*/
|
||||
public class DashManifestCreationException extends RuntimeException {
|
||||
|
||||
public DashManifestCreationException(final String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public DashManifestCreationException(final String message, final Exception cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
|
||||
// Methods to create exceptions easily without having to use big exception messages and to
|
||||
// reduce duplication
|
||||
|
||||
/**
|
||||
* Create a new {@link DashManifestCreationException} with a cause and the following detail
|
||||
* message format:
|
||||
* <br>
|
||||
* {@code "Could not add " + element + " element", cause}, where {@code element} is an element
|
||||
* of a DASH manifest.
|
||||
*
|
||||
* @param element the element which was not added to the DASH document
|
||||
* @param cause the exception which prevented addition of the element to the DASH document
|
||||
* @return a new {@link DashManifestCreationException}
|
||||
*/
|
||||
@Nonnull
|
||||
public static DashManifestCreationException couldNotAddElement(final String element,
|
||||
final Exception cause) {
|
||||
return new DashManifestCreationException("Could not add " + element + " element", cause);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new {@link DashManifestCreationException} with a cause and the following detail
|
||||
* message format:
|
||||
* <br>
|
||||
* {@code "Could not add " + element + " element: " + reason}, where {@code element} is an
|
||||
* element of a DASH manifest and {@code reason} the reason why this element cannot be added to
|
||||
* the DASH document.
|
||||
*
|
||||
* @param element the element which was not added to the DASH document
|
||||
* @param reason the reason message of why the element has been not added to the DASH document
|
||||
* @return a new {@link DashManifestCreationException}
|
||||
*/
|
||||
@Nonnull
|
||||
public static DashManifestCreationException couldNotAddElement(final String element,
|
||||
final String reason) {
|
||||
return new DashManifestCreationException("Could not add " + element + " element: " + reason);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
package org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
public interface DashManifestCreator {
|
||||
|
||||
/**
|
||||
* Generates the DASH manifest.
|
||||
* @return The dash manifest as string.
|
||||
* @throws DashManifestCreationException May throw a CreationException
|
||||
*/
|
||||
@Nonnull
|
||||
String generateManifest();
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
package org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator;
|
||||
|
||||
public final class DashManifestCreatorConstants {
|
||||
private DashManifestCreatorConstants() {
|
||||
// No impl!
|
||||
}
|
||||
|
||||
// XML elements of DASH MPD manifests
|
||||
// see https://www.brendanlong.com/the-structure-of-an-mpeg-dash-mpd.html
|
||||
public static final String MPD = "MPD";
|
||||
public static final String PERIOD = "Period";
|
||||
public static final String ADAPTATION_SET = "AdaptationSet";
|
||||
public static final String ROLE = "Role";
|
||||
public static final String REPRESENTATION = "Representation";
|
||||
public static final String AUDIO_CHANNEL_CONFIGURATION = "AudioChannelConfiguration";
|
||||
public static final String SEGMENT_TEMPLATE = "SegmentTemplate";
|
||||
public static final String SEGMENT_TIMELINE = "SegmentTimeline";
|
||||
public static final String BASE_URL = "BaseURL";
|
||||
public static final String SEGMENT_BASE = "SegmentBase";
|
||||
public static final String INITIALIZATION = "Initialization";
|
||||
}
|
||||
|
|
@ -1,30 +1,5 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.schabi.newpipe.downloader.DownloaderTestImpl;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.CreationException;
|
||||
import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeOtfDashManifestCreator;
|
||||
import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeProgressiveDashManifestCreator;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.DeliveryMethod;
|
||||
import org.schabi.newpipe.extractor.stream.Stream;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.NodeList;
|
||||
import org.xml.sax.InputSource;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import java.io.StringReader;
|
||||
import java.util.List;
|
||||
import java.util.Random;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertAll;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
|
|
@ -36,19 +11,45 @@ import static org.schabi.newpipe.extractor.ExtractorAsserts.assertGreaterOrEqual
|
|||
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsValidUrl;
|
||||
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertNotBlank;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ADAPTATION_SET;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.AUDIO_CHANNEL_CONFIGURATION;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.BASE_URL;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.INITIALIZATION;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.MPD;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.PERIOD;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.REPRESENTATION;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ROLE;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_BASE;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TEMPLATE;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TIMELINE;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeDashManifestCreatorsUtils.ADAPTATION_SET;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeDashManifestCreatorsUtils.AUDIO_CHANNEL_CONFIGURATION;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeDashManifestCreatorsUtils.BASE_URL;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeDashManifestCreatorsUtils.INITIALIZATION;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeDashManifestCreatorsUtils.MPD;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeDashManifestCreatorsUtils.PERIOD;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeDashManifestCreatorsUtils.REPRESENTATION;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeDashManifestCreatorsUtils.ROLE;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeDashManifestCreatorsUtils.SEGMENT_BASE;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeDashManifestCreatorsUtils.SEGMENT_TEMPLATE;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeDashManifestCreatorsUtils.SEGMENT_TIMELINE;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isBlank;
|
||||
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.schabi.newpipe.downloader.DownloaderTestImpl;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeOtfDashManifestCreator;
|
||||
import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeProgressiveDashManifestCreator;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.DeliveryMethod;
|
||||
import org.schabi.newpipe.extractor.stream.Stream;
|
||||
import org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreationException;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.NodeList;
|
||||
import org.xml.sax.InputSource;
|
||||
|
||||
import java.io.StringReader;
|
||||
import java.util.List;
|
||||
import java.util.Random;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
|
||||
/**
|
||||
* Test for YouTube DASH manifest creators.
|
||||
*
|
||||
|
|
@ -112,7 +113,7 @@ class YoutubeDashManifestCreatorsTest {
|
|||
assertProgressiveStreams(extractor.getAudioStreams());
|
||||
|
||||
// we are not able to generate DASH manifests of video formats with audio
|
||||
assertThrows(CreationException.class,
|
||||
assertThrows(DashManifestCreationException.class,
|
||||
() -> assertProgressiveStreams(extractor.getVideoStreams()));
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue