diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/DeliveryType.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/DeliveryType.java
deleted file mode 100644
index 5ce0010f8..000000000
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/DeliveryType.java
+++ /dev/null
@@ -1,56 +0,0 @@
-package org.schabi.newpipe.extractor.services.youtube;
-
-/**
- * Streaming format types used by YouTube in their streams.
- *
- *
- * It is different from {@link org.schabi.newpipe.extractor.stream.DeliveryMethod delivery methods}!
- *
- */
-// TODO: Kill
-public enum DeliveryType {
-
- /**
- * YouTube's progressive delivery method, which works with HTTP range headers.
- * (Note that official clients use the corresponding parameter instead.)
- *
- *
- * Initialization and index ranges are available to get metadata (the corresponding values
- * are returned in the player response).
- *
- */
- PROGRESSIVE,
-
- /**
- * YouTube's OTF delivery method which uses a sequence parameter to get segments of
- * streams.
- *
- *
- * The first sequence (which can be fetched with the {@code &sq=0} parameter) contains all the
- * metadata needed to build the stream source (sidx boxes, segment length, segment count,
- * duration, ...).
- *
- *
- *
- * Only used for videos; mostly those with a small amount of views, or ended livestreams
- * which have just been re-encoded as normal videos.
- *
- */
- OTF,
-
- /**
- * YouTube's delivery method for livestreams which uses a sequence parameter to get
- * segments of streams.
- *
- *
- * Each sequence (which can be fetched with the {@code &sq=0} parameter) contains its own
- * metadata (sidx boxes, segment length, ...), which make no need of an initialization
- * segment.
- *
- *
- *
- * Only used for livestreams (ended or running).
- *
- */
- LIVE
-}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/AbstractYoutubeDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/AbstractYoutubeDashManifestCreator.java
new file mode 100644
index 000000000..95a4fc906
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/AbstractYoutubeDashManifestCreator.java
@@ -0,0 +1,650 @@
+package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator;
+
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isIosStreamingUrl;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5SimplyEmbeddedPlayerStreamingUrl;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebStreamingUrl;
+import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.ADAPTATION_SET;
+import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.AUDIO_CHANNEL_CONFIGURATION;
+import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.MPD;
+import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.PERIOD;
+import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.REPRESENTATION;
+import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.ROLE;
+import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.SEGMENT_TEMPLATE;
+import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.SEGMENT_TIMELINE;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.downloader.Downloader;
+import org.schabi.newpipe.extractor.downloader.Response;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.services.youtube.itag.format.AudioItagFormat;
+import org.schabi.newpipe.extractor.services.youtube.itag.format.VideoItagFormat;
+import org.schabi.newpipe.extractor.services.youtube.itag.info.ItagInfo;
+import org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreationException;
+import org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreator;
+import org.schabi.newpipe.extractor.streamdata.stream.quality.VideoQualityData;
+import org.w3c.dom.Attr;
+import org.w3c.dom.DOMException;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+
+import javax.annotation.Nonnull;
+import javax.xml.XMLConstants;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+// TODO: Doc
+public abstract class AbstractYoutubeDashManifestCreator implements DashManifestCreator {
+
+ /**
+ * The redirect count limit that this class uses, which is the same limit as OkHttp.
+ */
+ public static final int MAXIMUM_REDIRECT_COUNT = 20;
+
+ /**
+ * URL parameter of the first sequence for live, post-live-DVR and OTF streams.
+ */
+ public static final String SQ_0 = "&sq=0";
+
+ /**
+ * URL parameter of the first stream request made by official clients.
+ */
+ public static final String RN_0 = "&rn=0";
+
+ /**
+ * URL parameter specific to web clients. When this param is added, if a redirection occurs,
+ * the server will not redirect clients to the redirect URL. Instead, it will provide this URL
+ * as the response body.
+ */
+ public static final String ALR_YES = "&alr=yes";
+
+ protected final ItagInfo itagInfo;
+ protected final long durationSecondsFallback;
+
+ protected Document document;
+
+ protected AbstractYoutubeDashManifestCreator(
+ @Nonnull final ItagInfo itagInfo,
+ final long durationSecondsFallback) {
+ this.itagInfo = Objects.requireNonNull(itagInfo);
+ this.durationSecondsFallback = durationSecondsFallback;
+ }
+
+ protected boolean isLiveDelivery() {
+ return false;
+ }
+
+ // region generate manifest elements
+
+ /**
+ * Generate a {@link Document} with common manifest creator elements added to it.
+ *
+ *
+ * Those are:
+ *
+ * - {@code MPD} (using {@link #generateDocumentAndMpdElement(long)});
+ * - {@code Period} (using {@link #generatePeriodElement()});
+ * - {@code AdaptationSet} (using {@link #generateAdaptationSetElement()});
+ * - {@code Role} (using {@link #generateRoleElement()});
+ * - {@code Representation} (using {@link #generateRepresentationElement()});
+ * - and, for audio streams, {@code AudioChannelConfiguration} (using
+ * {@link #generateAudioChannelConfigurationElement()}).
+ *
+ *
+ *
+ * @param streamDurationMs the duration of the stream, in milliseconds
+ * @throws DashManifestCreationException May throw a CreationException
+ */
+ protected void generateDocumentAndCommonElements(final long streamDurationMs) {
+ generateDocumentAndMpdElement(streamDurationMs);
+
+ generatePeriodElement();
+ generateAdaptationSetElement();
+ generateRoleElement();
+ generateRepresentationElement();
+ if (itagInfo.getItagFormat() instanceof AudioItagFormat) {
+ generateAudioChannelConfigurationElement();
+ }
+ }
+
+ /**
+ * Create a {@link Document} instance and generate the {@code } element of the manifest.
+ *
+ *
+ * The generated {@code } element looks like the manifest returned into the player
+ * response of videos:
+ *
+ *
+ *
+ * {@code }
+ * (where {@code $duration$} represents the duration in seconds (a number with 3 digits after
+ * the decimal point)).
+ *
+ *
+ * @param durationMs the duration of the stream, in milliseconds
+ * @throws DashManifestCreationException May throw a CreationException
+ */
+ protected void generateDocumentAndMpdElement(final long durationMs) {
+ try {
+ newDocument();
+
+ final Element mpdElement = createElement(MPD);
+ document.appendChild(mpdElement);
+
+ appendNewAttrWithValue(
+ mpdElement, "xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
+
+ appendNewAttrWithValue(mpdElement, "xmlns", "urn:mpeg:DASH:schema:MPD:2011");
+
+ appendNewAttrWithValue(
+ mpdElement,
+ "xsi:schemaLocation",
+ "urn:mpeg:DASH:schema:MPD:2011 DASH-MPD.xsd");
+
+ appendNewAttrWithValue(mpdElement, "minBufferTime", "PT1.500S");
+
+ appendNewAttrWithValue(
+ mpdElement, "profiles", "urn:mpeg:dash:profile:full:2011");
+
+ appendNewAttrWithValue(mpdElement, "type", "static");
+
+ final String durationSeconds =
+ String.format(Locale.ENGLISH, "%.3f", durationMs / 1000.0);
+ appendNewAttrWithValue(
+ mpdElement, "mediaPresentationDuration", "PT" + durationSeconds + "S");
+ } catch (final Exception e) {
+ throw new DashManifestCreationException(
+ "Could not generate the DASH manifest or append the MPD document to it", e);
+ }
+ }
+
+ /**
+ * Generate the {@code } element, appended as a child of the {@code } element.
+ *
+ *
+ * The {@code } element needs to be generated before this element with
+ * {@link #generateDocumentAndMpdElement(long)}.
+ *
+ *
+ * @throws DashManifestCreationException May throw a CreationException
+ */
+ protected void generatePeriodElement() {
+ try {
+ getFirstElementByName(MPD).appendChild(createElement(PERIOD));
+ } catch (final DOMException e) {
+ throw DashManifestCreationException.couldNotAddElement(PERIOD, e);
+ }
+ }
+
+ /**
+ * Generate the {@code } element, appended as a child of the {@code }
+ * element.
+ *
+ *
+ * The {@code } element needs to be generated before this element with
+ * {@link #generatePeriodElement()}.
+ *
+ *
+ * @throws DashManifestCreationException May throw a CreationException
+ */
+ protected void generateAdaptationSetElement() {
+ try {
+ final Element adaptationSetElement = createElement(ADAPTATION_SET);
+
+ appendNewAttrWithValue(adaptationSetElement, "id", "0");
+
+ appendNewAttrWithValue(
+ adaptationSetElement,
+ "mimeType",
+ itagInfo.getItagFormat().mediaFormat().mimeType());
+
+ appendNewAttrWithValue(adaptationSetElement, "subsegmentAlignment", "true");
+
+ getFirstElementByName(PERIOD).appendChild(adaptationSetElement);
+ } catch (final DOMException e) {
+ throw DashManifestCreationException.couldNotAddElement(ADAPTATION_SET, e);
+ }
+ }
+
+ /**
+ * Generate the {@code } element, appended as a child of the {@code }
+ * element.
+ *
+ *
+ * This element, with its attributes and values, is:
+ *
+ *
+ *
+ * {@code }
+ *
+ *
+ *
+ * The {@code } element needs to be generated before this element with
+ * {@link #generateAdaptationSetElement(Document)}).
+ *
+ *
+ * @throws DashManifestCreationException May throw a CreationException
+ */
+ protected void generateRoleElement() {
+ try {
+ final Element roleElement = createElement(ROLE);
+
+ appendNewAttrWithValue(roleElement, "schemeIdUri", "urn:mpeg:DASH:role:2011");
+
+ appendNewAttrWithValue(roleElement, "value", "main");
+
+ getFirstElementByName(ADAPTATION_SET).appendChild(roleElement);
+ } catch (final DOMException e) {
+ throw DashManifestCreationException.couldNotAddElement(ROLE, e);
+ }
+ }
+
+ /**
+ * Generate the {@code } element, appended as a child of the
+ * {@code } element.
+ *
+ *
+ * The {@code } element needs to be generated before this element with
+ * {@link #generateAdaptationSetElement()}).
+ *
+ * @throws DashManifestCreationException May throw a CreationException
+ */
+ protected void generateRepresentationElement() {
+ try {
+ final Element representationElement = createElement(REPRESENTATION);
+
+ appendNewAttrWithValue(
+ representationElement, "id", itagInfo.getItagFormat().id());
+
+ final String codec = itagInfo.getCodec();
+ if (isNullOrEmpty(codec)) {
+ throw DashManifestCreationException.couldNotAddElement(ADAPTATION_SET,
+ "the codec value of the ItagItem is null or empty");
+ }
+
+ appendNewAttrWithValue(
+ representationElement, "codecs", codec);
+
+ appendNewAttrWithValue(
+ representationElement, "startWithSAP", "1");
+
+ appendNewAttrWithValue(
+ representationElement, "maxPlayoutRate", "1");
+
+ final Integer bitrate = itagInfo.getBitRate();
+ if (bitrate == null || bitrate <= 0) {
+ throw DashManifestCreationException.couldNotAddElement(
+ REPRESENTATION,
+ "invalid bitrate=" + bitrate);
+ }
+
+ appendNewAttrWithValue(
+ representationElement, "bandwidth", bitrate);
+
+ if (itagInfo.getItagFormat() instanceof VideoItagFormat) {
+
+ final VideoQualityData videoQualityData = itagInfo.getCombinedVideoQualityData();
+
+ if (videoQualityData.height() <= 0 && videoQualityData.width() <= 0) {
+ throw DashManifestCreationException.couldNotAddElement(
+ REPRESENTATION,
+ "both width and height are <= 0");
+ }
+
+ if (videoQualityData.width() > 0) {
+ appendNewAttrWithValue(
+ representationElement, "width", videoQualityData.width());
+ }
+
+ appendNewAttrWithValue(
+ representationElement, "height", videoQualityData.height());
+
+ if (videoQualityData.fps() > 0) {
+ appendNewAttrWithValue(
+ representationElement, "height", videoQualityData.fps());
+ }
+ }
+
+ if (itagInfo.getItagFormat() instanceof AudioItagFormat
+ && itagInfo.getAudioSampleRate() != null
+ && itagInfo.getAudioSampleRate() > 0) {
+
+ appendNewAttrWithValue(
+ representationElement,
+ "audioSamplingRate",
+ itagInfo.getAudioSampleRate());
+ }
+
+ getFirstElementByName(ADAPTATION_SET).appendChild(representationElement);
+ } catch (final DOMException e) {
+ throw DashManifestCreationException.couldNotAddElement(REPRESENTATION, e);
+ }
+ }
+
+ /**
+ * Generate the {@code } element, appended as a child of the
+ * {@code } element.
+ *
+ *
+ * This method is only used when generating DASH manifests of audio streams.
+ *
+ *
+ *
+ * It will produce the following element:
+ *
+ * {@code
+ * (where {@code audioChannelsValue} is get from the {@link ItagInfo} passed as the second
+ * parameter of this method)
+ *
+ *
+ *
+ * The {@code } element needs to be generated before this element with
+ * {@link #generateRepresentationElement()}).
+ *
+ *
+ * @throws DashManifestCreationException May throw a CreationException
+ */
+ protected void generateAudioChannelConfigurationElement() {
+ try {
+ final Element audioChannelConfigElement = createElement(AUDIO_CHANNEL_CONFIGURATION);
+
+ appendNewAttrWithValue(
+ audioChannelConfigElement,
+ "schemeIdUri",
+ "urn:mpeg:dash:23003:3:audio_channel_configuration:2011");
+
+ final Integer audioChannels = itagInfo.getAudioChannels();
+ if (audioChannels != null && audioChannels <= 0) {
+ throw new DashManifestCreationException(
+ "Invalid value for 'audioChannels'=" + audioChannels);
+ }
+
+ appendNewAttrWithValue(
+ audioChannelConfigElement, "value", itagInfo.getAudioChannels());
+
+ getFirstElementByName(REPRESENTATION).appendChild(audioChannelConfigElement);
+ } catch (final DOMException e) {
+ throw DashManifestCreationException.couldNotAddElement(AUDIO_CHANNEL_CONFIGURATION, e);
+ }
+ }
+
+ /**
+ * Generate the {@code } element, appended as a child of the
+ * {@code } element.
+ *
+ *
+ * This method is only used when generating DASH manifests from OTF and post-live-DVR streams.
+ *
+ *
+ *
+ * It will produce a {@code } element with the following attributes:
+ *
+ * - {@code startNumber}, which takes the value {@code 0} for post-live-DVR streams and
+ * {@code 1} for OTF streams;
+ * - {@code timescale}, which is always {@code 1000};
+ * - {@code media}, which is the base URL of the stream on which is appended
+ * {@code &sq=$Number$};
+ * - {@code initialization} (only for OTF streams), which is the base URL of the stream
+ * on which is appended {@link #SQ_0}.
+ *
+ *
+ *
+ *
+ * The {@code } element needs to be generated before this element with
+ * {@link #generateRepresentationElement()}).
+ *
+ *
+ * @param baseUrl the base URL of the OTF/post-live-DVR stream
+ * @throws DashManifestCreationException May throw a CreationException
+ */
+ protected void generateSegmentTemplateElement(@Nonnull final String baseUrl) {
+ try {
+ final Element segmentTemplateElement = createElement(SEGMENT_TEMPLATE);
+
+ // The first sequence of post DVR streams is the beginning of the video stream and not
+ // an initialization segment
+ appendNewAttrWithValue(
+ segmentTemplateElement, "startNumber", isLiveDelivery() ? "0" : "1");
+
+ appendNewAttrWithValue(
+ segmentTemplateElement, "timescale", "1000");
+
+ // Post-live-DVR/ended livestreams streams don't require an initialization sequence
+ if (!isLiveDelivery()) {
+ appendNewAttrWithValue(
+ segmentTemplateElement, "initialization", baseUrl + SQ_0);
+ }
+
+ appendNewAttrWithValue(
+ segmentTemplateElement, "media", baseUrl + "&sq=$Number$");
+
+ getFirstElementByName(REPRESENTATION).appendChild(segmentTemplateElement);
+ } catch (final DOMException e) {
+ throw DashManifestCreationException.couldNotAddElement(SEGMENT_TEMPLATE, e);
+ }
+ }
+
+ /**
+ * Generate the {@code } element, appended as a child of the
+ * {@code } element.
+ *
+ *
+ * The {@code } element needs to be generated before this element with
+ * {@link #generateSegmentTemplateElement(String)}.
+ *
+ *
+ * @throws DashManifestCreationException May throw a CreationException
+ */
+ protected void generateSegmentTimelineElement()
+ throws DashManifestCreationException {
+ try {
+ final Element segmentTemplateElement = getFirstElementByName(SEGMENT_TEMPLATE);
+ final Element segmentTimelineElement = createElement(SEGMENT_TIMELINE);
+
+ segmentTemplateElement.appendChild(segmentTimelineElement);
+ } catch (final DOMException e) {
+ throw DashManifestCreationException.couldNotAddElement(SEGMENT_TIMELINE, e);
+ }
+ }
+ // endregion
+
+ // region initResponse
+
+ @Nonnull
+ protected Response getInitializationResponse(@Nonnull String baseStreamingUrl) {
+ final boolean isHtml5StreamingUrl = isWebStreamingUrl(baseStreamingUrl)
+ || isTvHtml5SimplyEmbeddedPlayerStreamingUrl(baseStreamingUrl);
+ final boolean isAndroidStreamingUrl = isAndroidStreamingUrl(baseStreamingUrl);
+ final boolean isIosStreamingUrl = isIosStreamingUrl(baseStreamingUrl);
+ if (isHtml5StreamingUrl) {
+ baseStreamingUrl += ALR_YES;
+ }
+ baseStreamingUrl = appendBaseStreamingUrlParams(baseStreamingUrl);
+
+ final Downloader downloader = NewPipe.getDownloader();
+ if (isHtml5StreamingUrl) {
+ return getStreamingWebUrlWithoutRedirects(downloader, baseStreamingUrl);
+ } else if (isAndroidStreamingUrl || isIosStreamingUrl) {
+ try {
+ final Map> headers = new HashMap<>();
+ headers.put("User-Agent", Collections.singletonList(
+ isAndroidStreamingUrl
+ ? getAndroidUserAgent(null)
+ : getIosUserAgent(null)));
+ final byte[] emptyBody = "".getBytes(StandardCharsets.UTF_8);
+ return downloader.post(baseStreamingUrl, headers, emptyBody);
+ } catch (final IOException | ExtractionException e) {
+ throw new DashManifestCreationException("Could not get the "
+ + (isIosStreamingUrl ? "IOS" : "ANDROID") + " streaming URL response", e);
+ }
+ }
+
+ try {
+ return downloader.get(baseStreamingUrl);
+ } catch (final IOException | ExtractionException e) {
+ throw new DashManifestCreationException("Could not get the streaming URL response", e);
+ }
+ }
+
+ @Nonnull
+ protected Response getStreamingWebUrlWithoutRedirects(
+ @Nonnull final Downloader downloader,
+ @Nonnull String streamingUrl) {
+ try {
+ final Map> headers = new HashMap<>();
+ addClientInfoHeaders(headers);
+
+ for (int r = 0; r < MAXIMUM_REDIRECT_COUNT; r++) {
+ final Response response = downloader.get(streamingUrl, headers);
+
+ final int responseCode = response.responseCode();
+ if (responseCode != 200) {
+ throw new DashManifestCreationException(
+ "Could not get the initialization URL: HTTP response code "
+ + responseCode);
+ }
+
+ // A valid HTTP 1.0+ response should include a Content-Type header, so we can
+ // require that the response from video servers has this header.
+ final String responseMimeType =
+ Objects.requireNonNull(
+ response.getHeader("Content-Type"),
+ "Could not get the Content-Type header from the response headers");
+
+ // The response body is not the redirection URL
+ if (!responseMimeType.equals("text/plain")) {
+ return response;
+ }
+
+ streamingUrl = response.responseBody();
+ }
+
+ throw new DashManifestCreationException("Too many redirects");
+
+ } catch (final IOException | ExtractionException e) {
+ throw new DashManifestCreationException(
+ "Could not get the streaming URL response of a HTML5 client", e);
+ }
+ }
+
+ @Nonnull
+ protected String appendBaseStreamingUrlParams(@Nonnull final String baseStreamingUrl) {
+ return baseStreamingUrl + SQ_0 + RN_0;
+ }
+
+ // endregion
+
+ // region document util
+
+ /**
+ * Generate a new {@link DocumentBuilder} secured from XXE attacks, on platforms which
+ * support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and
+ * {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link DocumentBuilderFactory} instances.
+ */
+ protected void newDocument() throws ParserConfigurationException {
+ final DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
+ try {
+ documentBuilderFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+ documentBuilderFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
+ } catch (final Exception ignored) {
+ // Ignore exceptions as setting these attributes to secure XML generation is not
+ // supported by all platforms (like the Android implementation)
+ }
+
+ final DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
+ document = documentBuilder.newDocument();
+ }
+
+ protected Attr appendNewAttrWithValue(
+ final Element baseElement,
+ final String name,
+ final Object value
+ ) {
+ return appendNewAttrWithValue(baseElement, name, String.valueOf(value));
+ }
+
+ protected Attr appendNewAttrWithValue(
+ final Element baseElement,
+ final String name,
+ final String value
+ ) {
+ final Attr attr = document.createAttribute(name);
+ attr.setValue(value);
+ baseElement.setAttributeNode(attr);
+
+ return attr;
+ }
+
+ protected Element getFirstElementByName(final String name) {
+ return (Element) document.getElementsByTagName(name).item(0);
+ }
+
+ protected Element createElement(final String name) {
+ return document.createElement(name);
+ }
+
+ @SuppressWarnings("squid:S2755") // warning is still shown despite applied solution
+ protected String documentToXml() throws TransformerException {
+ /*
+ * Generate a new {@link TransformerFactory} secured from XXE attacks, on platforms which
+ * support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and
+ * {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link TransformerFactory} instances.
+ */
+ final TransformerFactory transformerFactory = TransformerFactory.newInstance();
+ try {
+ transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+ transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
+ } catch (final Exception ignored) {
+ // Ignore exceptions as setting these attributes to secure XML generation is not
+ // supported by all platforms (like the Android implementation)
+ }
+
+ final Transformer transformer = transformerFactory.newTransformer();
+ transformer.setOutputProperty(OutputKeys.VERSION, "1.0");
+ transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");
+ transformer.setOutputProperty(OutputKeys.STANDALONE, "no");
+
+ final StringWriter result = new StringWriter();
+ transformer.transform(new DOMSource(document), new StreamResult(result));
+
+ return result.toString();
+ }
+
+ protected String documentToXmlSafe() {
+ try {
+ return documentToXml();
+ } catch (final TransformerException e) {
+ throw new DashManifestCreationException("Failed to convert XML-document to string", e);
+ }
+ }
+
+ // endregion
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/YoutubeOtfDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/YoutubeOtfDashManifestCreator.java
new file mode 100644
index 000000000..ad4bc5593
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/YoutubeOtfDashManifestCreator.java
@@ -0,0 +1,162 @@
+package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator;
+
+import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.SEGMENT_TIMELINE;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
+import static org.schabi.newpipe.extractor.utils.Utils.isBlank;
+import static org.schabi.newpipe.extractor.utils.Utils.removeNonDigitCharacters;
+
+import org.schabi.newpipe.extractor.downloader.Response;
+import org.schabi.newpipe.extractor.services.youtube.itag.info.ItagInfo;
+import org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreationException;
+import org.w3c.dom.Element;
+
+import java.util.Arrays;
+import java.util.stream.Stream;
+
+import javax.annotation.Nonnull;
+
+public class YoutubeOtfDashManifestCreator extends AbstractYoutubeDashManifestCreator {
+
+ public YoutubeOtfDashManifestCreator(@Nonnull final ItagInfo itagInfo,
+ final long durationSecondsFallback) {
+ super(itagInfo, durationSecondsFallback);
+ }
+
+ @Nonnull
+ @Override
+ public String generateManifest() {
+ // Try to avoid redirects when streaming the content by saving the last URL we get
+ // from video servers.
+ final Response response = getInitializationResponse(itagInfo.getStreamUrl());
+ final String otfBaseStreamingUrl = response.latestUrl()
+ .replace(SQ_0, EMPTY_STRING)
+ .replace(RN_0, EMPTY_STRING)
+ .replace(ALR_YES, EMPTY_STRING);
+
+ final int responseCode = response.responseCode();
+ if (responseCode != 200) {
+ throw new DashManifestCreationException("Could not get the initialization URL: " +
+ "response code "
+ + responseCode);
+ }
+
+ final String[] segmentDurations;
+
+ try {
+ final String[] segmentsAndDurationsResponseSplit = response.responseBody()
+ // Get the lines with the durations and the following
+ .split("Segment-Durations-Ms: ")[1]
+ // Remove the other lines
+ .split("\n")[0]
+ // Get all durations and repetitions which are separated by a comma
+ .split(",");
+ final int lastIndex = segmentsAndDurationsResponseSplit.length - 1;
+ if (isBlank(segmentsAndDurationsResponseSplit[lastIndex])) {
+ segmentDurations = Arrays.copyOf(segmentsAndDurationsResponseSplit, lastIndex);
+ } else {
+ segmentDurations = segmentsAndDurationsResponseSplit;
+ }
+ } catch (final Exception e) {
+ throw new DashManifestCreationException("Could not get segment durations", e);
+ }
+
+ long streamDurationMs;
+ try {
+ streamDurationMs = getStreamDuration(segmentDurations);
+ } catch (final DashManifestCreationException e) {
+ streamDurationMs = durationSecondsFallback * 1000;
+ }
+
+ generateDocumentAndCommonElements(streamDurationMs);
+ generateSegmentTemplateElement(otfBaseStreamingUrl);
+ generateSegmentTimelineElement();
+ generateSegmentElementsForOtfStreams(segmentDurations);
+
+ return documentToXmlSafe();
+ }
+
+ /**
+ * Generate segment elements for OTF streams.
+ *
+ *
+ * By parsing by the first media sequence, we know how many durations and repetitions there are
+ * so we just have to loop into segment durations to generate the following elements for each
+ * duration repeated X times:
+ *
+ *
+ *
+ * {@code
+ *
+ *
+ * If there is no repetition of the duration between two segments, the {@code r} attribute is
+ * not added to the {@code S} element, as it is not needed.
+ *
+ *
+ *
+ * These elements will be appended as children of the {@code } element, which
+ * needs to be generated before these elements with
+ * {@link #generateSegmentTimelineElement()}.
+ *
+ *
+ * @param segmentDurations the sequences "length" or "length(r=repeat_count" extracted with the
+ * regular expressions
+ * @throws DashManifestCreationException May throw a CreationException
+ */
+ protected void generateSegmentElementsForOtfStreams(@Nonnull final String[] segmentDurations) {
+ try {
+ final Element segmentTimelineElement = getFirstElementByName(SEGMENT_TIMELINE);
+ streamAndSplitSegmentDurations(segmentDurations)
+ .map(segmentLengthRepeat -> {
+ final Element sElement = createElement("S");
+ // There are repetitions of a segment duration in other segments
+ if (segmentLengthRepeat.length > 1) {
+ appendNewAttrWithValue(sElement, "r", Integer.parseInt(
+ removeNonDigitCharacters(segmentLengthRepeat[1])));
+ }
+
+ appendNewAttrWithValue(
+ sElement, "d", Integer.parseInt(segmentLengthRepeat[0]));
+ return sElement;
+ })
+ .forEach(segmentTimelineElement::appendChild);
+ } catch (final Exception e) {
+ throw DashManifestCreationException.couldNotAddElement("segment (S)", e);
+ }
+ }
+
+ /**
+ * Get the duration of an OTF stream.
+ *
+ *
+ * The duration of OTF streams is not returned into the player response and needs to be
+ * calculated by adding the duration of each segment.
+ *
+ *
+ * @param segmentDurations the segment duration object extracted from the initialization
+ * sequence of the stream
+ * @return the duration of the OTF stream, in milliseconds
+ * @throws DashManifestCreationException May throw a CreationException
+ */
+ protected long getStreamDuration(@Nonnull final String[] segmentDurations) {
+ try {
+ return streamAndSplitSegmentDurations(segmentDurations)
+ .mapToLong(segmentLengthRepeat -> {
+ final long segmentLength = Integer.parseInt(segmentLengthRepeat[0]);
+ final long segmentRepeatCount = segmentLengthRepeat.length > 1
+ ? Long.parseLong(removeNonDigitCharacters(segmentLengthRepeat[1]))
+ : 0;
+ return segmentLength + segmentRepeatCount * segmentLength;
+ })
+ .sum();
+ } catch (final NumberFormatException e) {
+ throw new DashManifestCreationException("Could not get stream length from sequences " +
+ "list", e);
+ }
+ }
+
+ protected Stream streamAndSplitSegmentDurations(@Nonnull final String[] segmentDurations) {
+ return Stream.of(segmentDurations)
+ .map(segDuration -> segDuration.split("\\(r="));
+ }
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/YoutubePostLiveStreamDvrDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/YoutubePostLiveStreamDvrDashManifestCreator.java
new file mode 100644
index 000000000..3c6bd1da8
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/YoutubePostLiveStreamDvrDashManifestCreator.java
@@ -0,0 +1,121 @@
+package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator;
+
+import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.SEGMENT_TIMELINE;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
+import org.schabi.newpipe.extractor.downloader.Response;
+import org.schabi.newpipe.extractor.services.youtube.itag.info.ItagInfo;
+import org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreationException;
+import org.w3c.dom.Attr;
+import org.w3c.dom.DOMException;
+import org.w3c.dom.Element;
+
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nonnull;
+
+public class YoutubePostLiveStreamDvrDashManifestCreator extends AbstractYoutubeDashManifestCreator {
+
+ protected YoutubePostLiveStreamDvrDashManifestCreator(@Nonnull final ItagInfo itagInfo,
+ final long durationSecondsFallback) {
+ super(itagInfo, durationSecondsFallback);
+ }
+
+ @Override
+ protected boolean isLiveDelivery() {
+ return true;
+ }
+
+ @Nonnull
+ @Override
+ public String generateManifest() {
+ final Integer targetDurationSec = itagInfo.getTargetDurationSec();
+ if (targetDurationSec == null || targetDurationSec <= 0) {
+ throw new DashManifestCreationException(
+ "Invalid value for 'targetDurationSec'=" + targetDurationSec);
+ }
+
+ // Try to avoid redirects when streaming the content by saving the latest URL we get
+ // from video servers.
+ final Response response = getInitializationResponse(itagInfo.getStreamUrl());
+ final String realPostLiveStreamDvrStreamingUrl = response.latestUrl()
+ .replace(SQ_0, "")
+ .replace(RN_0, "")
+ .replace(ALR_YES, "");
+
+ final int responseCode = response.responseCode();
+ if (responseCode != 200) {
+ throw new DashManifestCreationException(
+ "Could not get the initialization sequence: response code " + responseCode);
+ }
+
+ final String streamDurationMsString;
+ final String segmentCount;
+ try {
+ final Map> responseHeaders = response.responseHeaders();
+ streamDurationMsString = responseHeaders.get("X-Head-Time-Millis").get(0);
+ segmentCount = responseHeaders.get("X-Head-Seqnum").get(0);
+ } catch (final IndexOutOfBoundsException e) {
+ throw new DashManifestCreationException(
+ "Could not get the value of the X-Head-Time-Millis or the X-Head-Seqnum header",
+ e);
+ }
+
+ if (isNullOrEmpty(segmentCount)) {
+ throw new DashManifestCreationException("Could not get the number of segments");
+ }
+
+ long streamDurationMs;
+ try {
+ streamDurationMs = Long.parseLong(streamDurationMsString);
+ } catch (final NumberFormatException e) {
+ streamDurationMs = durationSecondsFallback * 1000;
+ }
+
+ generateDocumentAndCommonElements(streamDurationMs);
+
+ generateSegmentTemplateElement(realPostLiveStreamDvrStreamingUrl);
+ generateSegmentTimelineElement();
+ generateSegmentElementForPostLiveDvrStreams(targetDurationSec, segmentCount);
+
+ return documentToXmlSafe();
+ }
+
+ /**
+ * Generate the segment ({@code }) element.
+ *
+ *
+ * We don't know the exact duration of segments for post-live-DVR streams but an
+ * average instead (which is the {@code targetDurationSec} value), so we can use the following
+ * structure to generate the segment timeline for DASH manifests of ended livestreams:
+ *
+ * {@code
+ *
+ * @param targetDurationSeconds the {@code targetDurationSec} value from YouTube player
+ * response's stream
+ * @param segmentCount the number of segments
+ * @throws DashManifestCreationException May throw a CreationException
+ */
+ private void generateSegmentElementForPostLiveDvrStreams(
+ final int targetDurationSeconds,
+ @Nonnull final String segmentCount
+ ) {
+ try {
+ final Element sElement = document.createElement("S");
+
+ appendNewAttrWithValue(sElement, "d", targetDurationSeconds * 1000);
+
+ final Attr rAttribute = document.createAttribute("r");
+ rAttribute.setValue(segmentCount);
+ sElement.setAttributeNode(rAttribute);
+
+ appendNewAttrWithValue(sElement, "r", segmentCount);
+
+ getFirstElementByName(SEGMENT_TIMELINE).appendChild(sElement);
+ } catch (final DOMException e) {
+ throw DashManifestCreationException.couldNotAddElement("segment (S)", e);
+ }
+ }
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/YoutubeProgressiveDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/YoutubeProgressiveDashManifestCreator.java
new file mode 100644
index 000000000..c368712a6
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreator/YoutubeProgressiveDashManifestCreator.java
@@ -0,0 +1,159 @@
+package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator;
+
+import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.BASE_URL;
+import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.INITIALIZATION;
+import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.MPD;
+import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.REPRESENTATION;
+import static org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreatorConstants.SEGMENT_BASE;
+
+import org.schabi.newpipe.extractor.services.youtube.itag.info.ItagInfo;
+import org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreationException;
+import org.w3c.dom.DOMException;
+import org.w3c.dom.Element;
+
+import javax.annotation.Nonnull;
+
+public class YoutubeProgressiveDashManifestCreator extends AbstractYoutubeDashManifestCreator {
+
+ public YoutubeProgressiveDashManifestCreator(@Nonnull final ItagInfo itagInfo,
+ final long durationSecondsFallback) {
+ super(itagInfo, durationSecondsFallback);
+ }
+
+ @Nonnull
+ @Override
+ public String generateManifest() {
+ final long streamDurationMs;
+ if (itagInfo.getApproxDurationMs() != null) {
+ streamDurationMs = itagInfo.getApproxDurationMs();
+ } else if (durationSecondsFallback > 0) {
+ streamDurationMs = durationSecondsFallback * 1000;
+ } else {
+ throw DashManifestCreationException.couldNotAddElement(MPD, "the duration of the " +
+ "stream could not be determined and durationSecondsFallback is <= 0");
+ }
+
+ generateDocumentAndCommonElements(streamDurationMs);
+ generateSegmentBaseElement();
+ generateInitializationElement();
+
+ return documentToXmlSafe();
+ }
+
+ /**
+ * Generate the {@code } element, appended as a child of the
+ * {@code } element.
+ *
+ *
+ * The {@code } element needs to be generated before this element with
+ * {@link #generateRepresentationElement()}).
+ *
+ *
+ * @param baseUrl the base URL of the stream, which must not be null and will be set as the
+ * content of the {@code } element
+ * @throws DashManifestCreationException May throw a CreationException
+ */
+ protected void generateBaseUrlElement(@Nonnull final String baseUrl)
+ throws DashManifestCreationException {
+ try {
+ final Element representationElement = getFirstElementByName(REPRESENTATION);
+
+ appendNewAttrWithValue(representationElement, BASE_URL, baseUrl);
+ } catch (final DOMException e) {
+ throw DashManifestCreationException.couldNotAddElement(BASE_URL, e);
+ }
+ }
+
+ /**
+ * Generate the {@code } element, appended as a child of the
+ * {@code } element.
+ *
+ *
+ * It generates the following element:
+ *
+ * {@code }
+ *
+ * (where {@code indexStart} and {@code indexEnd} are gotten from the {@link ItagInfo} passed
+ * as the second parameter)
+ *
+ *
+ *
+ * The {@code } element needs to be generated before this element with
+ * {@link #generateRepresentationElement()}),
+ * and the {@code BaseURL} element with {@link #generateBaseUrlElement(String)}
+ * should be generated too.
+ *
+ *
+ * @throws DashManifestCreationException May throw a CreationException
+ */
+ protected void generateSegmentBaseElement() {
+ try {
+ final Element segmentBaseElement = createElement(SEGMENT_BASE);
+
+ if (itagInfo.getIndexRange() == null
+ || itagInfo.getIndexRange().start() < 0
+ || itagInfo.getIndexRange().end() < 0) {
+ throw DashManifestCreationException.couldNotAddElement(SEGMENT_BASE,
+ "invalid index-range: " + itagInfo.getIndexRange());
+ }
+
+ appendNewAttrWithValue(
+ segmentBaseElement,
+ "indexRange",
+ itagInfo.getIndexRange().start() + "-" + itagInfo.getIndexRange().end());
+
+ getFirstElementByName(REPRESENTATION).appendChild(segmentBaseElement);
+ } catch (final DOMException e) {
+ throw DashManifestCreationException.couldNotAddElement(SEGMENT_BASE, e);
+ }
+ }
+
+ /**
+ * Generate the {@code } element, appended as a child of the
+ * {@code } element.
+ *
+ *
+ * It generates the following element:
+ *
+ * {@code }
+ *
+ * (where {@code indexStart} and {@code indexEnd} are gotten from the {@link ItagInfo} passed
+ * as the second parameter)
+ *
+ *
+ *
+ * The {@code } element needs to be generated before this element with
+ * {@link #generateSegmentBaseElement()}).
+ *
+ *
+ * @throws DashManifestCreationException May throw a CreationException
+ */
+ protected void generateInitializationElement() {
+ try {
+ final Element initializationElement = createElement(INITIALIZATION);
+
+ if (itagInfo.getInitRange() == null
+ || itagInfo.getInitRange().start() < 0
+ || itagInfo.getInitRange().end() < 0) {
+ throw DashManifestCreationException.couldNotAddElement(SEGMENT_BASE,
+ "invalid (init)-range: " + itagInfo.getInitRange());
+ }
+
+ appendNewAttrWithValue(
+ initializationElement,
+ "range",
+ itagInfo.getInitRange().start() + "-" + itagInfo.getInitRange().end());
+
+ getFirstElementByName(SEGMENT_BASE).appendChild(initializationElement);
+ } catch (final DOMException e) {
+ throw DashManifestCreationException.couldNotAddElement(INITIALIZATION, e);
+ }
+ }
+
+
+ @Nonnull
+ @Override
+ protected String appendBaseStreamingUrlParams(@Nonnull final String baseStreamingUrl) {
+ return baseStreamingUrl + RN_0;
+ }
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/CreationException.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/CreationException.java
deleted file mode 100644
index 46f32664b..000000000
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/CreationException.java
+++ /dev/null
@@ -1,63 +0,0 @@
-package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
-
-import javax.annotation.Nonnull;
-
-/**
- * Exception that is thrown when a YouTube DASH manifest creator encounters a problem
- * while creating a manifest.
- */
-public final class CreationException extends RuntimeException {
-
- /**
- * Create a new {@link CreationException} with a detail message.
- *
- * @param message the detail message to add in the exception
- */
- public CreationException(final String message) {
- super(message);
- }
-
- /**
- * Create a new {@link CreationException} with a detail message and a cause.
- * @param message the detail message to add in the exception
- * @param cause the exception cause of this {@link CreationException}
- */
- public CreationException(final String message, final Exception cause) {
- super(message, cause);
- }
-
- // Methods to create exceptions easily without having to use big exception messages and to
- // reduce duplication
-
- /**
- * Create a new {@link CreationException} with a cause and the following detail message format:
- *
- * {@code "Could not add " + element + " element", cause}, where {@code element} is an element
- * of a DASH manifest.
- *
- * @param element the element which was not added to the DASH document
- * @param cause the exception which prevented addition of the element to the DASH document
- * @return a new {@link CreationException}
- */
- @Nonnull
- public static CreationException couldNotAddElement(final String element,
- final Exception cause) {
- return new CreationException("Could not add " + element + " element", cause);
- }
-
- /**
- * Create a new {@link CreationException} with a cause and the following detail message format:
- *
- * {@code "Could not add " + element + " element: " + reason}, where {@code element} is an
- * element of a DASH manifest and {@code reason} the reason why this element cannot be added to
- * the DASH document.
- *
- * @param element the element which was not added to the DASH document
- * @param reason the reason message of why the element has been not added to the DASH document
- * @return a new {@link CreationException}
- */
- @Nonnull
- public static CreationException couldNotAddElement(final String element, final String reason) {
- return new CreationException("Could not add " + element + " element: " + reason);
- }
-}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java
deleted file mode 100644
index 045e5dda4..000000000
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java
+++ /dev/null
@@ -1,756 +0,0 @@
-package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
-
-import org.schabi.newpipe.extractor.MediaFormat;
-import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.downloader.Downloader;
-import org.schabi.newpipe.extractor.downloader.Response;
-import org.schabi.newpipe.extractor.exceptions.ExtractionException;
-import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
-import org.schabi.newpipe.extractor.services.youtube.ItagItem;
-import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
-import org.w3c.dom.Attr;
-import org.w3c.dom.DOMException;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-
-import javax.annotation.Nonnull;
-import javax.xml.XMLConstants;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.transform.OutputKeys;
-import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerException;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
-
-import java.io.IOException;
-import java.io.StringWriter;
-import java.nio.charset.StandardCharsets;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isIosStreamingUrl;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5SimplyEmbeddedPlayerStreamingUrl;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebStreamingUrl;
-import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
-
-/**
- * Utilities and constants for YouTube DASH manifest creators.
- *
- *
- * This class includes common methods of manifest creators and useful constants.
- *
- *
- *
- * Generation of DASH documents and their conversion as a string is done using external classes
- * from {@link org.w3c.dom} and {@link javax.xml} packages.
- *
- */
-public final class YoutubeDashManifestCreatorsUtils {
-
- private YoutubeDashManifestCreatorsUtils() {
- }
-
- /**
- * The redirect count limit that this class uses, which is the same limit as OkHttp.
- */
- public static final int MAXIMUM_REDIRECT_COUNT = 20;
-
- /**
- * URL parameter of the first sequence for live, post-live-DVR and OTF streams.
- */
- public static final String SQ_0 = "&sq=0";
-
- /**
- * URL parameter of the first stream request made by official clients.
- */
- public static final String RN_0 = "&rn=0";
-
- /**
- * URL parameter specific to web clients. When this param is added, if a redirection occurs,
- * the server will not redirect clients to the redirect URL. Instead, it will provide this URL
- * as the response body.
- */
- public static final String ALR_YES = "&alr=yes";
-
- // XML elements of DASH MPD manifests
- // see https://www.brendanlong.com/the-structure-of-an-mpeg-dash-mpd.html
- public static final String MPD = "MPD";
- public static final String PERIOD = "Period";
- public static final String ADAPTATION_SET = "AdaptationSet";
- public static final String ROLE = "Role";
- public static final String REPRESENTATION = "Representation";
- public static final String AUDIO_CHANNEL_CONFIGURATION = "AudioChannelConfiguration";
- public static final String SEGMENT_TEMPLATE = "SegmentTemplate";
- public static final String SEGMENT_TIMELINE = "SegmentTimeline";
- public static final String BASE_URL = "BaseURL";
- public static final String SEGMENT_BASE = "SegmentBase";
- public static final String INITIALIZATION = "Initialization";
-
- /**
- * Create an attribute with {@link Document#createAttribute(String)}, assign to it the provided
- * name and value, then add it to the provided element using {@link
- * Element#setAttributeNode(Attr)}.
- *
- * @param element element to which to add the created node
- * @param doc document to use to create the attribute
- * @param name name of the attribute
- * @param value value of the attribute, will be set using {@link Attr#setValue(String)}
- */
- public static void setAttribute(final Element element,
- final Document doc,
- final String name,
- final String value) {
- final Attr attr = doc.createAttribute(name);
- attr.setValue(value);
- element.setAttributeNode(attr);
- }
-
- /**
- * Generate a {@link Document} with common manifest creator elements added to it.
- *
- *
- * Those are:
- *
- * - {@code MPD} (using {@link #generateDocumentAndMpdElement(long)});
- * - {@code Period} (using {@link #generatePeriodElement(Document)});
- * - {@code AdaptationSet} (using {@link #generateAdaptationSetElement(Document,
- * ItagItem)});
- * - {@code Role} (using {@link #generateRoleElement(Document)});
- * - {@code Representation} (using {@link #generateRepresentationElement(Document,
- * ItagItem)});
- * - and, for audio streams, {@code AudioChannelConfiguration} (using
- * {@link #generateAudioChannelConfigurationElement(Document, ItagItem)}).
- *
- *
- *
- * @param itagItem the {@link ItagItem} associated to the stream, which must not be null
- * @param streamDuration the duration of the stream, in milliseconds
- * @return a {@link Document} with the common elements added in it
- */
- @Nonnull
- public static Document generateDocumentAndDoCommonElementsGeneration(
- @Nonnull final ItagItem itagItem,
- final long streamDuration) throws CreationException {
- final Document doc = generateDocumentAndMpdElement(streamDuration);
-
- generatePeriodElement(doc);
- generateAdaptationSetElement(doc, itagItem);
- generateRoleElement(doc);
- generateRepresentationElement(doc, itagItem);
- if (itagItem.itagType == ItagItem.ItagType.AUDIO) {
- generateAudioChannelConfigurationElement(doc, itagItem);
- }
-
- return doc;
- }
-
- /**
- * Create a {@link Document} instance and generate the {@code } element of the manifest.
- *
- *
- * The generated {@code } element looks like the manifest returned into the player
- * response of videos:
- *
- *
- *
- * {@code }
- * (where {@code $duration$} represents the duration in seconds (a number with 3 digits after
- * the decimal point)).
- *
- *
- * @param duration the duration of the stream, in milliseconds
- * @return a {@link Document} instance which contains a {@code } element
- */
- @Nonnull
- public static Document generateDocumentAndMpdElement(final long duration)
- throws CreationException {
- try {
- final Document doc = newDocument();
-
- final Element mpdElement = doc.createElement(MPD);
- doc.appendChild(mpdElement);
-
- setAttribute(mpdElement, doc, "xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
- setAttribute(mpdElement, doc, "xmlns", "urn:mpeg:DASH:schema:MPD:2011");
- setAttribute(mpdElement, doc, "xsi:schemaLocation",
- "urn:mpeg:DASH:schema:MPD:2011 DASH-MPD.xsd");
- setAttribute(mpdElement, doc, "minBufferTime", "PT1.500S");
- setAttribute(mpdElement, doc, "profiles", "urn:mpeg:dash:profile:full:2011");
- setAttribute(mpdElement, doc, "type", "static");
- setAttribute(mpdElement, doc, "mediaPresentationDuration",
- String.format(Locale.ENGLISH, "PT%.3fS", duration / 1000.0));
-
- return doc;
- } catch (final Exception e) {
- throw new CreationException(
- "Could not generate the DASH manifest or append the MPD doc to it", e);
- }
- }
-
- /**
- * Generate the {@code } element, appended as a child of the {@code } element.
- *
- *
- * The {@code } element needs to be generated before this element with
- * {@link #generateDocumentAndMpdElement(long)}.
- *
- *
- * @param doc the {@link Document} on which the the {@code } element will be appended
- */
- public static void generatePeriodElement(@Nonnull final Document doc)
- throws CreationException {
- try {
- final Element mpdElement = (Element) doc.getElementsByTagName(MPD).item(0);
- final Element periodElement = doc.createElement(PERIOD);
- mpdElement.appendChild(periodElement);
- } catch (final DOMException e) {
- throw CreationException.couldNotAddElement(PERIOD, e);
- }
- }
-
- /**
- * Generate the {@code } element, appended as a child of the {@code }
- * element.
- *
- *
- * The {@code } element needs to be generated before this element with
- * {@link #generatePeriodElement(Document)}.
- *
- *
- * @param doc the {@link Document} on which the {@code } element will be appended
- * @param itagItem the {@link ItagItem} corresponding to the stream, which must not be null
- */
- public static void generateAdaptationSetElement(@Nonnull final Document doc,
- @Nonnull final ItagItem itagItem)
- throws CreationException {
- try {
- final Element periodElement = (Element) doc.getElementsByTagName(PERIOD)
- .item(0);
- final Element adaptationSetElement = doc.createElement(ADAPTATION_SET);
-
- setAttribute(adaptationSetElement, doc, "id", "0");
-
- final MediaFormat mediaFormat = itagItem.getMediaFormat();
- if (mediaFormat == null || isNullOrEmpty(mediaFormat.getMimeType())) {
- throw CreationException.couldNotAddElement(ADAPTATION_SET,
- "the MediaFormat or its mime type is null or empty");
- }
-
- setAttribute(adaptationSetElement, doc, "mimeType", mediaFormat.getMimeType());
- setAttribute(adaptationSetElement, doc, "subsegmentAlignment", "true");
-
- periodElement.appendChild(adaptationSetElement);
- } catch (final DOMException e) {
- throw CreationException.couldNotAddElement(ADAPTATION_SET, e);
- }
- }
-
- /**
- * Generate the {@code } element, appended as a child of the {@code }
- * element.
- *
- *
- * This element, with its attributes and values, is:
- *
- *
- *
- * {@code }
- *
- *
- *
- * The {@code } element needs to be generated before this element with
- * {@link #generateAdaptationSetElement(Document, ItagItem)}).
- *
- *
- * @param doc the {@link Document} on which the the {@code } element will be appended
- */
- public static void generateRoleElement(@Nonnull final Document doc)
- throws CreationException {
- try {
- final Element adaptationSetElement = (Element) doc.getElementsByTagName(
- ADAPTATION_SET).item(0);
- final Element roleElement = doc.createElement(ROLE);
-
- setAttribute(roleElement, doc, "schemeIdUri", "urn:mpeg:DASH:role:2011");
- setAttribute(roleElement, doc, "value", "main");
-
- adaptationSetElement.appendChild(roleElement);
- } catch (final DOMException e) {
- throw CreationException.couldNotAddElement(ROLE, e);
- }
- }
-
- /**
- * Generate the {@code } element, appended as a child of the
- * {@code } element.
- *
- *
- * The {@code } element needs to be generated before this element with
- * {@link #generateAdaptationSetElement(Document, ItagItem)}).
- *
- *
- * @param doc the {@link Document} on which the the {@code } element will be
- * appended
- * @param itagItem the {@link ItagItem} to use, which must not be null
- */
- public static void generateRepresentationElement(@Nonnull final Document doc,
- @Nonnull final ItagItem itagItem)
- throws CreationException {
- try {
- final Element adaptationSetElement = (Element) doc.getElementsByTagName(
- ADAPTATION_SET).item(0);
- final Element representationElement = doc.createElement(REPRESENTATION);
-
- final int id = itagItem.id;
- if (id <= 0) {
- throw CreationException.couldNotAddElement(REPRESENTATION,
- "the id of the ItagItem is <= 0");
- }
- setAttribute(representationElement, doc, "id", String.valueOf(id));
-
- final String codec = itagItem.getCodec();
- if (isNullOrEmpty(codec)) {
- throw CreationException.couldNotAddElement(ADAPTATION_SET,
- "the codec value of the ItagItem is null or empty");
- }
- setAttribute(representationElement, doc, "codecs", codec);
- setAttribute(representationElement, doc, "startWithSAP", "1");
- setAttribute(representationElement, doc, "maxPlayoutRate", "1");
-
- final int bitrate = itagItem.getBitrate();
- if (bitrate <= 0) {
- throw CreationException.couldNotAddElement(REPRESENTATION,
- "the bitrate of the ItagItem is <= 0");
- }
- setAttribute(representationElement, doc, "bandwidth", String.valueOf(bitrate));
-
- if (itagItem.itagType == ItagItem.ItagType.VIDEO
- || itagItem.itagType == ItagItem.ItagType.VIDEO_ONLY) {
- final int height = itagItem.getHeight();
- final int width = itagItem.getWidth();
- if (height <= 0 && width <= 0) {
- throw CreationException.couldNotAddElement(REPRESENTATION,
- "both width and height of the ItagItem are <= 0");
- }
-
- if (width > 0) {
- setAttribute(representationElement, doc, "width", String.valueOf(width));
- }
- setAttribute(representationElement, doc, "height",
- String.valueOf(itagItem.getHeight()));
-
- final int fps = itagItem.getFps();
- if (fps > 0) {
- setAttribute(representationElement, doc, "frameRate", String.valueOf(fps));
- }
- }
-
- if (itagItem.itagType == ItagItem.ItagType.AUDIO && itagItem.getSampleRate() > 0) {
- final Attr audioSamplingRateAttribute = doc.createAttribute(
- "audioSamplingRate");
- audioSamplingRateAttribute.setValue(String.valueOf(itagItem.getSampleRate()));
- }
-
- adaptationSetElement.appendChild(representationElement);
- } catch (final DOMException e) {
- throw CreationException.couldNotAddElement(REPRESENTATION, e);
- }
- }
-
- /**
- * Generate the {@code } element, appended as a child of the
- * {@code } element.
- *
- *
- * This method is only used when generating DASH manifests of audio streams.
- *
- *
- *
- * It will produce the following element:
- *
- * {@code
- * (where {@code audioChannelsValue} is get from the {@link ItagItem} passed as the second
- * parameter of this method)
- *
- *
- *
- * The {@code } element needs to be generated before this element with
- * {@link #generateRepresentationElement(Document, ItagItem)}).
- *
- *
- * @param doc the {@link Document} on which the {@code } element will
- * be appended
- * @param itagItem the {@link ItagItem} to use, which must not be null
- */
- public static void generateAudioChannelConfigurationElement(
- @Nonnull final Document doc,
- @Nonnull final ItagItem itagItem) throws CreationException {
- try {
- final Element representationElement = (Element) doc.getElementsByTagName(
- REPRESENTATION).item(0);
- final Element audioChannelConfigurationElement = doc.createElement(
- AUDIO_CHANNEL_CONFIGURATION);
-
- setAttribute(audioChannelConfigurationElement, doc, "schemeIdUri",
- "urn:mpeg:dash:23003:3:audio_channel_configuration:2011");
-
- if (itagItem.getAudioChannels() <= 0) {
- throw new CreationException("the number of audioChannels in the ItagItem is <= 0: "
- + itagItem.getAudioChannels());
- }
- setAttribute(audioChannelConfigurationElement, doc, "value",
- String.valueOf(itagItem.getAudioChannels()));
-
- representationElement.appendChild(audioChannelConfigurationElement);
- } catch (final DOMException e) {
- throw CreationException.couldNotAddElement(AUDIO_CHANNEL_CONFIGURATION, e);
- }
- }
-
- /**
- * Convert a DASH manifest {@link Document doc} to a string and cache it.
- *
- * @param originalBaseStreamingUrl the original base URL of the stream
- * @param doc the doc to be converted
- * @param manifestCreatorCache the {@link ManifestCreatorCache} on which store the string
- * generated
- * @return the DASH manifest {@link Document doc} converted to a string
- */
- public static String buildAndCacheResult(
- @Nonnull final String originalBaseStreamingUrl,
- @Nonnull final Document doc,
- @Nonnull final ManifestCreatorCache manifestCreatorCache)
- throws CreationException {
-
- try {
- final String documentXml = documentToXml(doc);
- manifestCreatorCache.put(originalBaseStreamingUrl, documentXml);
- return documentXml;
- } catch (final Exception e) {
- throw new CreationException(
- "Could not convert the DASH manifest generated to a string", e);
- }
- }
-
- /**
- * Generate the {@code } element, appended as a child of the
- * {@code } element.
- *
- *
- * This method is only used when generating DASH manifests from OTF and post-live-DVR streams.
- *
- *
- *
- * It will produce a {@code } element with the following attributes:
- *
- * - {@code startNumber}, which takes the value {@code 0} for post-live-DVR streams and
- * {@code 1} for OTF streams;
- * - {@code timescale}, which is always {@code 1000};
- * - {@code media}, which is the base URL of the stream on which is appended
- * {@code &sq=$Number$};
- * - {@code initialization} (only for OTF streams), which is the base URL of the stream
- * on which is appended {@link #SQ_0}.
- *
- *
- *
- *
- * The {@code } element needs to be generated before this element with
- * {@link #generateRepresentationElement(Document, ItagItem)}).
- *
- *
- * @param doc the {@link Document} on which the {@code } element will
- * be appended
- * @param baseUrl the base URL of the OTF/post-live-DVR stream
- * @param deliveryType the stream {@link DeliveryType delivery type}, which must be either
- * {@link DeliveryType#OTF OTF} or {@link DeliveryType#LIVE LIVE}
- */
- public static void generateSegmentTemplateElement(@Nonnull final Document doc,
- @Nonnull final String baseUrl,
- final DeliveryType deliveryType)
- throws CreationException {
- if (deliveryType != DeliveryType.OTF && deliveryType != DeliveryType.LIVE) {
- throw CreationException.couldNotAddElement(SEGMENT_TEMPLATE, "invalid delivery type: "
- + deliveryType);
- }
-
- try {
- final Element representationElement = (Element) doc.getElementsByTagName(
- REPRESENTATION).item(0);
- final Element segmentTemplateElement = doc.createElement(SEGMENT_TEMPLATE);
-
- // The first sequence of post DVR streams is the beginning of the video stream and not
- // an initialization segment
- setAttribute(segmentTemplateElement, doc, "startNumber",
- deliveryType == DeliveryType.LIVE ? "0" : "1");
- setAttribute(segmentTemplateElement, doc, "timescale", "1000");
-
- // Post-live-DVR/ended livestreams streams don't require an initialization sequence
- if (deliveryType != DeliveryType.LIVE) {
- setAttribute(segmentTemplateElement, doc, "initialization", baseUrl + SQ_0);
- }
-
- setAttribute(segmentTemplateElement, doc, "media", baseUrl + "&sq=$Number$");
-
- representationElement.appendChild(segmentTemplateElement);
- } catch (final DOMException e) {
- throw CreationException.couldNotAddElement(SEGMENT_TEMPLATE, e);
- }
- }
-
- /**
- * Generate the {@code } element, appended as a child of the
- * {@code } element.
- *
- *
- * The {@code } element needs to be generated before this element with
- * {@link #generateSegmentTemplateElement(Document, String, DeliveryType)}.
- *
- *
- * @param doc the {@link Document} on which the the {@code } element will be
- * appended
- */
- public static void generateSegmentTimelineElement(@Nonnull final Document doc)
- throws CreationException {
- try {
- final Element segmentTemplateElement = (Element) doc.getElementsByTagName(
- SEGMENT_TEMPLATE).item(0);
- final Element segmentTimelineElement = doc.createElement(SEGMENT_TIMELINE);
-
- segmentTemplateElement.appendChild(segmentTimelineElement);
- } catch (final DOMException e) {
- throw CreationException.couldNotAddElement(SEGMENT_TIMELINE, e);
- }
- }
-
- /**
- * Get the "initialization" {@link Response response} of a stream.
- *
- * This method fetches, for OTF streams and for post-live-DVR streams:
- *
- * - the base URL of the stream, to which are appended {@link #SQ_0} and
- * {@link #RN_0} parameters, with a {@code GET} request for streaming URLs from HTML5
- * clients and a {@code POST} request for the ones from the {@code ANDROID} and the
- * {@code IOS} clients;
- * - for streaming URLs from HTML5 clients, the {@link #ALR_YES} param is also added.
- *
- *
- *
- *
- * @param baseStreamingUrl the base URL of the stream, which must not be null
- * @param itagItem the {@link ItagItem} of stream, which must not be null
- * @param deliveryType the {@link DeliveryType} of the stream
- * @return the "initialization" response, without redirections on the network on which the
- * request(s) is/are made
- */
- @SuppressWarnings("checkstyle:FinalParameters")
- @Nonnull
- public static Response getInitializationResponse(@Nonnull String baseStreamingUrl,
- @Nonnull final ItagItem itagItem,
- final DeliveryType deliveryType)
- throws CreationException {
- final boolean isHtml5StreamingUrl = isWebStreamingUrl(baseStreamingUrl)
- || isTvHtml5SimplyEmbeddedPlayerStreamingUrl(baseStreamingUrl);
- final boolean isAndroidStreamingUrl = isAndroidStreamingUrl(baseStreamingUrl);
- final boolean isIosStreamingUrl = isIosStreamingUrl(baseStreamingUrl);
- if (isHtml5StreamingUrl) {
- baseStreamingUrl += ALR_YES;
- }
- baseStreamingUrl = appendRnSqParamsIfNeeded(baseStreamingUrl, deliveryType);
-
- final Downloader downloader = NewPipe.getDownloader();
- if (isHtml5StreamingUrl) {
- final String mimeTypeExpected = itagItem.getMediaFormat().getMimeType();
- if (!isNullOrEmpty(mimeTypeExpected)) {
- return getStreamingWebUrlWithoutRedirects(downloader, baseStreamingUrl,
- mimeTypeExpected);
- }
- } else if (isAndroidStreamingUrl || isIosStreamingUrl) {
- try {
- final Map> headers = Collections.singletonMap("User-Agent",
- Collections.singletonList(isAndroidStreamingUrl
- ? getAndroidUserAgent(null) : getIosUserAgent(null)));
- final byte[] emptyBody = "".getBytes(StandardCharsets.UTF_8);
- return downloader.post(baseStreamingUrl, headers, emptyBody);
- } catch (final IOException | ExtractionException e) {
- throw new CreationException("Could not get the "
- + (isIosStreamingUrl ? "ANDROID" : "IOS") + " streaming URL response", e);
- }
- }
-
- try {
- return downloader.get(baseStreamingUrl);
- } catch (final IOException | ExtractionException e) {
- throw new CreationException("Could not get the streaming URL response", e);
- }
- }
-
- /**
- * Generate a new {@link DocumentBuilder} secured from XXE attacks, on platforms which
- * support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and
- * {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link DocumentBuilderFactory} instances.
- *
- * @return an instance of {@link Document} secured against XXE attacks on supported platforms,
- * that should then be convertible to an XML string without security problems
- */
- private static Document newDocument() throws ParserConfigurationException {
- final DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
- try {
- documentBuilderFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
- documentBuilderFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
- } catch (final Exception ignored) {
- // Ignore exceptions as setting these attributes to secure XML generation is not
- // supported by all platforms (like the Android implementation)
- }
-
- return documentBuilderFactory.newDocumentBuilder().newDocument();
- }
-
- /**
- * Generate a new {@link TransformerFactory} secured from XXE attacks, on platforms which
- * support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and
- * {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link TransformerFactory} instances.
- *
- * @param doc the doc to convert, which must have been created using {@link #newDocument()} to
- * properly prevent XXE attacks
- * @return the doc converted to an XML string, making sure there can't be XXE attacks
- */
- // Sonar warning is suppressed because it is still shown even if we apply its solution
- @SuppressWarnings("squid:S2755")
- private static String documentToXml(@Nonnull final Document doc)
- throws TransformerException {
-
- final TransformerFactory transformerFactory = TransformerFactory.newInstance();
- try {
- transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
- transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
- } catch (final Exception ignored) {
- // Ignore exceptions as setting these attributes to secure XML generation is not
- // supported by all platforms (like the Android implementation)
- }
-
- final Transformer transformer = transformerFactory.newTransformer();
- transformer.setOutputProperty(OutputKeys.VERSION, "1.0");
- transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");
- transformer.setOutputProperty(OutputKeys.STANDALONE, "no");
-
- final StringWriter result = new StringWriter();
- transformer.transform(new DOMSource(doc), new StreamResult(result));
-
- return result.toString();
- }
-
- /**
- * Append {@link #SQ_0} for post-live-DVR and OTF streams and {@link #RN_0} to all streams.
- *
- * @param baseStreamingUrl the base streaming URL to which the parameter(s) are being appended
- * @param deliveryType the {@link DeliveryType} of the stream
- * @return the base streaming URL to which the param(s) are appended, depending on the
- * {@link DeliveryType} of the stream
- */
- @Nonnull
- private static String appendRnSqParamsIfNeeded(@Nonnull final String baseStreamingUrl,
- @Nonnull final DeliveryType deliveryType) {
- return baseStreamingUrl + (deliveryType == DeliveryType.PROGRESSIVE ? "" : SQ_0) + RN_0;
- }
-
- /**
- * Get a URL on which no redirection between playback hosts should be present on the network
- * and/or IP used to fetch the streaming URL, for HTML5 clients.
- *
- * This method will follow redirects which works in the following way:
- *
- * - the {@link #ALR_YES} param is appended to all streaming URLs
- * - if no redirection occurs, the video server will return the streaming data;
- * - if a redirection occurs, the server will respond with HTTP status code 200 and a
- * {@code text/plain} mime type. The redirection URL is the response body;
- * - the redirection URL is requested and the steps above from step 2 are repeated,
- * until too many redirects are reached of course (the maximum number of redirects is
- * {@link #MAXIMUM_REDIRECT_COUNT the same as OkHttp}).
- *
- *
- *
- *
- * For non-HTML5 clients, redirections are managed in the standard way in
- * {@link #getInitializationResponse(String, ItagItem, DeliveryType)}.
- *
- *
- * @param downloader the {@link Downloader} instance to be used
- * @param streamingUrl the streaming URL which we are trying to get a streaming URL
- * without any redirection on the network and/or IP used
- * @param responseMimeTypeExpected the response mime type expected from Google video servers
- * @return the {@link Response} of the stream, which should have no redirections
- */
- @SuppressWarnings("checkstyle:FinalParameters")
- @Nonnull
- private static Response getStreamingWebUrlWithoutRedirects(
- @Nonnull final Downloader downloader,
- @Nonnull String streamingUrl,
- @Nonnull final String responseMimeTypeExpected)
- throws CreationException {
- try {
- final Map> headers = new HashMap<>();
- addClientInfoHeaders(headers);
-
- String responseMimeType = "";
-
- int redirectsCount = 0;
- while (!responseMimeType.equals(responseMimeTypeExpected)
- && redirectsCount < MAXIMUM_REDIRECT_COUNT) {
- final Response response = downloader.get(streamingUrl, headers);
-
- final int responseCode = response.responseCode();
- if (responseCode != 200) {
- throw new CreationException(
- "Could not get the initialization URL: HTTP response code "
- + responseCode);
- }
-
- // A valid HTTP 1.0+ response should include a Content-Type header, so we can
- // require that the response from video servers has this header.
- responseMimeType = Objects.requireNonNull(response.getHeader("Content-Type"),
- "Could not get the Content-Type header from the response headers");
-
- // The response body is the redirection URL
- if (responseMimeType.equals("text/plain")) {
- streamingUrl = response.responseBody();
- redirectsCount++;
- } else {
- return response;
- }
- }
-
- if (redirectsCount >= MAXIMUM_REDIRECT_COUNT) {
- throw new CreationException(
- "Too many redirects when trying to get the the streaming URL response of a "
- + "HTML5 client");
- }
-
- // This should never be reached, but is required because we don't want to return null
- // here
- throw new CreationException(
- "Could not get the streaming URL response of a HTML5 client: unreachable code "
- + "reached!");
- } catch (final IOException | ExtractionException e) {
- throw new CreationException(
- "Could not get the streaming URL response of a HTML5 client", e);
- }
- }
-}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeOtfDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeOtfDashManifestCreator.java
deleted file mode 100644
index 46e84df1d..000000000
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeOtfDashManifestCreator.java
+++ /dev/null
@@ -1,265 +0,0 @@
-package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
-
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ALR_YES;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.RN_0;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TIMELINE;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SQ_0;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.buildAndCacheResult;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateDocumentAndDoCommonElementsGeneration;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTemplateElement;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTimelineElement;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.getInitializationResponse;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.setAttribute;
-import static org.schabi.newpipe.extractor.utils.Utils.isBlank;
-
-import org.schabi.newpipe.extractor.downloader.Response;
-import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
-import org.schabi.newpipe.extractor.services.youtube.ItagItem;
-import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
-import org.schabi.newpipe.extractor.utils.Utils;
-import org.w3c.dom.DOMException;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-
-import java.util.Arrays;
-import java.util.Objects;
-
-import javax.annotation.Nonnull;
-
-/**
- * Class which generates DASH manifests of YouTube {@link DeliveryType#OTF OTF streams}.
- */
-public final class YoutubeOtfDashManifestCreator {
-
- /**
- * Cache of DASH manifests generated for OTF streams.
- */
- private static final ManifestCreatorCache OTF_STREAMS_CACHE
- = new ManifestCreatorCache<>();
-
- private YoutubeOtfDashManifestCreator() {
- }
-
- /**
- * Create DASH manifests from a YouTube OTF stream.
- *
- *
- * OTF streams are YouTube-DASH specific streams which work with sequences and without the need
- * to get a manifest (even if one is provided, it is not used by official clients).
- *
- *
- *
- * They can be found only on videos; mostly those with a small amount of views, or ended
- * livestreams which have just been re-encoded as normal videos.
- *
- *
- * This method needs:
- *
- * - the base URL of the stream (which, if you try to access to it, returns HTTP
- * status code 404 after redirects, and if the URL is valid);
- * - an {@link ItagItem}, which needs to contain the following information:
- *
- * - its type (see {@link ItagItem.ItagType}), to identify if the content is
- * an audio or a video stream;
- * - its bitrate;
- * - its mime type;
- * - its codec(s);
- * - for an audio stream: its audio channels;
- * - for a video stream: its width and height.
- *
- *
- * - the duration of the video, which will be used if the duration could not be
- * parsed from the first sequence of the stream.
- *
- *
- *
- * In order to generate the DASH manifest, this method will:
- *
- * - request the first sequence of the stream (the base URL on which the first
- * sequence parameter is appended (see {@link YoutubeDashManifestCreatorsUtils#SQ_0}))
- * with a {@code POST} or {@code GET} request (depending of the client on which the
- * streaming URL comes from is a mobile one ({@code POST}) or not ({@code GET}));
- * - follow its redirection(s), if any;
- * - save the last URL, remove the first sequence parameter;
- * - use the information provided in the {@link ItagItem} to generate all
- * elements of the DASH manifest.
- *
- *
- *
- *
- * If the duration cannot be extracted, the {@code durationSecondsFallback} value will be used
- * as the stream duration.
- *
- *
- * @param otfBaseStreamingUrl the base URL of the OTF stream, which must not be null
- * @param itagItem the {@link ItagItem} corresponding to the stream, which
- * must not be null
- * @param durationSecondsFallback the duration of the video, which will be used if the duration
- * could not be extracted from the first sequence
- * @return the manifest generated into a string
- */
- @Nonnull
- public static String fromOtfStreamingUrl(
- @Nonnull final String otfBaseStreamingUrl,
- @Nonnull final ItagItem itagItem,
- final long durationSecondsFallback) throws CreationException {
- if (OTF_STREAMS_CACHE.containsKey(otfBaseStreamingUrl)) {
- return Objects.requireNonNull(OTF_STREAMS_CACHE.get(otfBaseStreamingUrl)).getSecond();
- }
-
- String realOtfBaseStreamingUrl = otfBaseStreamingUrl;
- // Try to avoid redirects when streaming the content by saving the last URL we get
- // from video servers.
- final Response response = getInitializationResponse(realOtfBaseStreamingUrl,
- itagItem, DeliveryType.OTF);
- realOtfBaseStreamingUrl = response.latestUrl().replace(SQ_0, "")
- .replace(RN_0, "").replace(ALR_YES, "");
-
- final int responseCode = response.responseCode();
- if (responseCode != 200) {
- throw new CreationException("Could not get the initialization URL: response code "
- + responseCode);
- }
-
- final String[] segmentDuration;
-
- try {
- final String[] segmentsAndDurationsResponseSplit = response.responseBody()
- // Get the lines with the durations and the following
- .split("Segment-Durations-Ms: ")[1]
- // Remove the other lines
- .split("\n")[0]
- // Get all durations and repetitions which are separated by a comma
- .split(",");
- final int lastIndex = segmentsAndDurationsResponseSplit.length - 1;
- if (isBlank(segmentsAndDurationsResponseSplit[lastIndex])) {
- segmentDuration = Arrays.copyOf(segmentsAndDurationsResponseSplit, lastIndex);
- } else {
- segmentDuration = segmentsAndDurationsResponseSplit;
- }
- } catch (final Exception e) {
- throw new CreationException("Could not get segment durations", e);
- }
-
- long streamDuration;
- try {
- streamDuration = getStreamDuration(segmentDuration);
- } catch (final CreationException e) {
- streamDuration = durationSecondsFallback * 1000;
- }
-
- final Document doc = generateDocumentAndDoCommonElementsGeneration(itagItem,
- streamDuration);
-
- generateSegmentTemplateElement(doc, realOtfBaseStreamingUrl, DeliveryType.OTF);
- generateSegmentTimelineElement(doc);
- generateSegmentElementsForOtfStreams(segmentDuration, doc);
-
- return buildAndCacheResult(otfBaseStreamingUrl, doc, OTF_STREAMS_CACHE);
- }
-
- /**
- * @return the cache of DASH manifests generated for OTF streams
- */
- @Nonnull
- public static ManifestCreatorCache getCache() {
- return OTF_STREAMS_CACHE;
- }
-
- /**
- * Generate segment elements for OTF streams.
- *
- *
- * By parsing by the first media sequence, we know how many durations and repetitions there are
- * so we just have to loop into segment durations to generate the following elements for each
- * duration repeated X times:
- *
- *
- *
- * {@code
- *
- *
- * If there is no repetition of the duration between two segments, the {@code r} attribute is
- * not added to the {@code S} element, as it is not needed.
- *
- *
- *
- * These elements will be appended as children of the {@code } element, which
- * needs to be generated before these elements with
- * {@link YoutubeDashManifestCreatorsUtils#generateSegmentTimelineElement(Document)}.
- *
- *
- * @param segmentDurations the sequences "length" or "length(r=repeat_count" extracted with the
- * regular expressions
- * @param doc the {@link Document} on which the {@code } elements will be
- * appended
- */
- private static void generateSegmentElementsForOtfStreams(
- @Nonnull final String[] segmentDurations,
- @Nonnull final Document doc) throws CreationException {
- try {
- final Element segmentTimelineElement = (Element) doc.getElementsByTagName(
- SEGMENT_TIMELINE).item(0);
-
- for (final String segmentDuration : segmentDurations) {
- final Element sElement = doc.createElement("S");
-
- final String[] segmentLengthRepeat = segmentDuration.split("\\(r=");
- // make sure segmentLengthRepeat[0], which is the length, is convertible to int
- Integer.parseInt(segmentLengthRepeat[0]);
-
- // There are repetitions of a segment duration in other segments
- if (segmentLengthRepeat.length > 1) {
- final int segmentRepeatCount = Integer.parseInt(
- Utils.removeNonDigitCharacters(segmentLengthRepeat[1]));
- setAttribute(sElement, doc, "r", String.valueOf(segmentRepeatCount));
- }
- setAttribute(sElement, doc, "d", segmentLengthRepeat[0]);
-
- segmentTimelineElement.appendChild(sElement);
- }
-
- } catch (final DOMException | IllegalStateException | IndexOutOfBoundsException
- | NumberFormatException e) {
- throw CreationException.couldNotAddElement("segment (S)", e);
- }
- }
-
- /**
- * Get the duration of an OTF stream.
- *
- *
- * The duration of OTF streams is not returned into the player response and needs to be
- * calculated by adding the duration of each segment.
- *
- *
- * @param segmentDuration the segment duration object extracted from the initialization
- * sequence of the stream
- * @return the duration of the OTF stream, in milliseconds
- */
- private static long getStreamDuration(@Nonnull final String[] segmentDuration)
- throws CreationException {
- try {
- long streamLengthMs = 0;
-
- for (final String segDuration : segmentDuration) {
- final String[] segmentLengthRepeat = segDuration.split("\\(r=");
- long segmentRepeatCount = 0;
-
- // There are repetitions of a segment duration in other segments
- if (segmentLengthRepeat.length > 1) {
- segmentRepeatCount = Long.parseLong(Utils.removeNonDigitCharacters(
- segmentLengthRepeat[1]));
- }
-
- final long segmentLength = Integer.parseInt(segmentLengthRepeat[0]);
- streamLengthMs += segmentLength + segmentRepeatCount * segmentLength;
- }
-
- return streamLengthMs;
- } catch (final NumberFormatException e) {
- throw new CreationException("Could not get stream length from sequences list", e);
- }
- }
-}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubePostLiveStreamDvrDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubePostLiveStreamDvrDashManifestCreator.java
deleted file mode 100644
index 3a5a7dd23..000000000
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubePostLiveStreamDvrDashManifestCreator.java
+++ /dev/null
@@ -1,217 +0,0 @@
-package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
-
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ALR_YES;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.RN_0;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TIMELINE;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SQ_0;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.buildAndCacheResult;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateDocumentAndDoCommonElementsGeneration;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTemplateElement;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTimelineElement;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.getInitializationResponse;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.setAttribute;
-import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
-
-import org.schabi.newpipe.extractor.downloader.Response;
-import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
-import org.schabi.newpipe.extractor.services.youtube.ItagItem;
-import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
-import org.w3c.dom.DOMException;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-import javax.annotation.Nonnull;
-
-/**
- * Class which generates DASH manifests of YouTube post-live DVR streams (which use the
- * {@link DeliveryType#LIVE LIVE delivery type}).
- */
-public final class YoutubePostLiveStreamDvrDashManifestCreator {
-
- /**
- * Cache of DASH manifests generated for post-live-DVR streams.
- */
- private static final ManifestCreatorCache POST_LIVE_DVR_STREAMS_CACHE
- = new ManifestCreatorCache<>();
-
- private YoutubePostLiveStreamDvrDashManifestCreator() {
- }
-
- /**
- * Create DASH manifests from a YouTube post-live-DVR stream/ended livestream.
- *
- *
- * Post-live-DVR streams/ended livestreams are one of the YouTube DASH specific streams which
- * works with sequences and without the need to get a manifest (even if one is provided but not
- * used by main clients (and is not complete for big ended livestreams because it doesn't
- * return the full stream)).
- *
- *
- *
- * They can be found only on livestreams which have ended very recently (a few hours, most of
- * the time)
- *
- *
- * This method needs:
- *
- * - the base URL of the stream (which, if you try to access to it, returns HTTP
- * status code 404 after redirects, and if the URL is valid);
- * - an {@link ItagItem}, which needs to contain the following information:
- *
- * - its type (see {@link ItagItem.ItagType}), to identify if the content is
- * an audio or a video stream;
- * - its bitrate;
- * - its mime type;
- * - its codec(s);
- * - for an audio stream: its audio channels;
- * - for a video stream: its width and height.
- *
- *
- * - the duration of the video, which will be used if the duration could not be
- * parsed from the first sequence of the stream.
- *
- *
- *
- * In order to generate the DASH manifest, this method will:
- *
- * - request the first sequence of the stream (the base URL on which the first
- * sequence parameter is appended (see {@link YoutubeDashManifestCreatorsUtils#SQ_0}))
- * with a {@code POST} or {@code GET} request (depending of the client on which the
- * streaming URL comes from is a mobile one ({@code POST}) or not ({@code GET}));
- * - follow its redirection(s), if any;
- * - save the last URL, remove the first sequence parameters;
- * - use the information provided in the {@link ItagItem} to generate all elements
- * of the DASH manifest.
- *
- *
- *
- *
- * If the duration cannot be extracted, the {@code durationSecondsFallback} value will be used
- * as the stream duration.
- *
- *
- * @param postLiveStreamDvrStreamingUrl the base URL of the post-live-DVR stream/ended
- * livestream, which must not be null
- * @param itagItem the {@link ItagItem} corresponding to the stream, which
- * must not be null
- * @param targetDurationSec the target duration of each sequence, in seconds (this
- * value is returned with the {@code targetDurationSec}
- * field for each stream in YouTube's player response)
- * @param durationSecondsFallback the duration of the ended livestream, which will be
- * used if the duration could not be extracted from the
- * first sequence
- * @return the manifest generated into a string
- */
- @Nonnull
- public static String fromPostLiveStreamDvrStreamingUrl(
- @Nonnull final String postLiveStreamDvrStreamingUrl,
- @Nonnull final ItagItem itagItem,
- final int targetDurationSec,
- final long durationSecondsFallback) throws CreationException {
- if (POST_LIVE_DVR_STREAMS_CACHE.containsKey(postLiveStreamDvrStreamingUrl)) {
- return Objects.requireNonNull(
- POST_LIVE_DVR_STREAMS_CACHE.get(postLiveStreamDvrStreamingUrl)).getSecond();
- }
-
- String realPostLiveStreamDvrStreamingUrl = postLiveStreamDvrStreamingUrl;
- final String streamDurationString;
- final String segmentCount;
-
- if (targetDurationSec <= 0) {
- throw new CreationException("targetDurationSec value is <= 0: " + targetDurationSec);
- }
-
- try {
- // Try to avoid redirects when streaming the content by saving the latest URL we get
- // from video servers.
- final Response response = getInitializationResponse(realPostLiveStreamDvrStreamingUrl,
- itagItem, DeliveryType.LIVE);
- realPostLiveStreamDvrStreamingUrl = response.latestUrl().replace(SQ_0, "")
- .replace(RN_0, "").replace(ALR_YES, "");
-
- final int responseCode = response.responseCode();
- if (responseCode != 200) {
- throw new CreationException(
- "Could not get the initialization sequence: response code " + responseCode);
- }
-
- final Map> responseHeaders = response.responseHeaders();
- streamDurationString = responseHeaders.get("X-Head-Time-Millis").get(0);
- segmentCount = responseHeaders.get("X-Head-Seqnum").get(0);
- } catch (final IndexOutOfBoundsException e) {
- throw new CreationException(
- "Could not get the value of the X-Head-Time-Millis or the X-Head-Seqnum header",
- e);
- }
-
- if (isNullOrEmpty(segmentCount)) {
- throw new CreationException("Could not get the number of segments");
- }
-
- long streamDuration;
- try {
- streamDuration = Long.parseLong(streamDurationString);
- } catch (final NumberFormatException e) {
- streamDuration = durationSecondsFallback;
- }
-
- final Document doc = generateDocumentAndDoCommonElementsGeneration(itagItem,
- streamDuration);
-
- generateSegmentTemplateElement(doc, realPostLiveStreamDvrStreamingUrl,
- DeliveryType.LIVE);
- generateSegmentTimelineElement(doc);
- generateSegmentElementForPostLiveDvrStreams(doc, targetDurationSec, segmentCount);
-
- return buildAndCacheResult(postLiveStreamDvrStreamingUrl, doc,
- POST_LIVE_DVR_STREAMS_CACHE);
- }
-
- /**
- * @return the cache of DASH manifests generated for post-live-DVR streams
- */
- @Nonnull
- public static ManifestCreatorCache getCache() {
- return POST_LIVE_DVR_STREAMS_CACHE;
- }
-
- /**
- * Generate the segment ({@code }) element.
- *
- *
- * We don't know the exact duration of segments for post-live-DVR streams but an
- * average instead (which is the {@code targetDurationSec} value), so we can use the following
- * structure to generate the segment timeline for DASH manifests of ended livestreams:
- *
- * {@code
- *
- * @param doc the {@link Document} on which the {@code } element will
- * be appended
- * @param targetDurationSeconds the {@code targetDurationSec} value from YouTube player
- * response's stream
- * @param segmentCount the number of segments, extracted by {@link
- * #fromPostLiveStreamDvrStreamingUrl(String, ItagItem, int, long)}
- */
- private static void generateSegmentElementForPostLiveDvrStreams(
- @Nonnull final Document doc,
- final int targetDurationSeconds,
- @Nonnull final String segmentCount) throws CreationException {
- try {
- final Element segmentTimelineElement = (Element) doc.getElementsByTagName(
- SEGMENT_TIMELINE).item(0);
- final Element sElement = doc.createElement("S");
-
- setAttribute(sElement, doc, "d", String.valueOf(targetDurationSeconds * 1000));
- setAttribute(sElement, doc, "r", segmentCount);
-
- segmentTimelineElement.appendChild(sElement);
- } catch (final DOMException e) {
- throw CreationException.couldNotAddElement("segment (S)", e);
- }
- }
-}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeProgressiveDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeProgressiveDashManifestCreator.java
deleted file mode 100644
index 0f69895bb..000000000
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeProgressiveDashManifestCreator.java
+++ /dev/null
@@ -1,235 +0,0 @@
-package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
-
-import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
-import org.schabi.newpipe.extractor.services.youtube.ItagItem;
-import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
-import org.w3c.dom.DOMException;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-
-import javax.annotation.Nonnull;
-import java.util.Objects;
-
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.BASE_URL;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.INITIALIZATION;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.MPD;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.REPRESENTATION;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_BASE;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.buildAndCacheResult;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateDocumentAndDoCommonElementsGeneration;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.setAttribute;
-
-/**
- * Class which generates DASH manifests of {@link DeliveryType#PROGRESSIVE YouTube progressive}
- * streams.
- */
-public final class YoutubeProgressiveDashManifestCreator {
-
- /**
- * Cache of DASH manifests generated for progressive streams.
- */
- private static final ManifestCreatorCache PROGRESSIVE_STREAMS_CACHE
- = new ManifestCreatorCache<>();
-
- private YoutubeProgressiveDashManifestCreator() {
- }
-
- /**
- * Create DASH manifests from a YouTube progressive stream.
- *
- *
- * Progressive streams are YouTube DASH streams which work with range requests and without the
- * need to get a manifest.
- *
- *
- *
- * They can be found on all videos, and for all streams for most of videos which come from a
- * YouTube partner, and on videos with a large number of views.
- *
- *
- * This method needs:
- *
- * - the base URL of the stream (which, if you try to access to it, returns the whole
- * stream, after redirects, and if the URL is valid);
- * - an {@link ItagItem}, which needs to contain the following information:
- *
- * - its type (see {@link ItagItem.ItagType}), to identify if the content is
- * an audio or a video stream;
- * - its bitrate;
- * - its mime type;
- * - its codec(s);
- * - for an audio stream: its audio channels;
- * - for a video stream: its width and height.
- *
- *
- * - the duration of the video (parameter {@code durationSecondsFallback}), which
- * will be used as the stream duration if the duration could not be parsed from the
- * {@link ItagItem}.
- *
- *
- *
- * @param progressiveStreamingBaseUrl the base URL of the progressive stream, which must not be
- * null
- * @param itagItem the {@link ItagItem} corresponding to the stream, which
- * must not be null
- * @param durationSecondsFallback the duration of the progressive stream which will be used
- * if the duration could not be extracted from the
- * {@link ItagItem}
- * @return the manifest generated into a string
- */
- @Nonnull
- public static String fromProgressiveStreamingUrl(
- @Nonnull final String progressiveStreamingBaseUrl,
- @Nonnull final ItagItem itagItem,
- final long durationSecondsFallback) throws CreationException {
- if (PROGRESSIVE_STREAMS_CACHE.containsKey(progressiveStreamingBaseUrl)) {
- return Objects.requireNonNull(
- PROGRESSIVE_STREAMS_CACHE.get(progressiveStreamingBaseUrl)).getSecond();
- }
-
- final long itagItemDuration = itagItem.getApproxDurationMs();
- final long streamDuration;
- if (itagItemDuration != -1) {
- streamDuration = itagItemDuration;
- } else {
- if (durationSecondsFallback > 0) {
- streamDuration = durationSecondsFallback * 1000;
- } else {
- throw CreationException.couldNotAddElement(MPD, "the duration of the stream "
- + "could not be determined and durationSecondsFallback is <= 0");
- }
- }
-
- final Document doc = generateDocumentAndDoCommonElementsGeneration(itagItem,
- streamDuration);
-
- generateBaseUrlElement(doc, progressiveStreamingBaseUrl);
- generateSegmentBaseElement(doc, itagItem);
- generateInitializationElement(doc, itagItem);
-
- return buildAndCacheResult(progressiveStreamingBaseUrl, doc,
- PROGRESSIVE_STREAMS_CACHE);
- }
-
- /**
- * @return the cache of DASH manifests generated for progressive streams
- */
- @Nonnull
- public static ManifestCreatorCache getCache() {
- return PROGRESSIVE_STREAMS_CACHE;
- }
-
- /**
- * Generate the {@code } element, appended as a child of the
- * {@code } element.
- *
- *
- * The {@code } element needs to be generated before this element with
- * {@link YoutubeDashManifestCreatorsUtils#generateRepresentationElement(Document, ItagItem)}).
- *
- *
- * @param doc the {@link Document} on which the {@code } element will be appended
- * @param baseUrl the base URL of the stream, which must not be null and will be set as the
- * content of the {@code } element
- */
- private static void generateBaseUrlElement(@Nonnull final Document doc,
- @Nonnull final String baseUrl)
- throws CreationException {
- try {
- final Element representationElement = (Element) doc.getElementsByTagName(
- REPRESENTATION).item(0);
- final Element baseURLElement = doc.createElement(BASE_URL);
- baseURLElement.setTextContent(baseUrl);
- representationElement.appendChild(baseURLElement);
- } catch (final DOMException e) {
- throw CreationException.couldNotAddElement(BASE_URL, e);
- }
- }
-
- /**
- * Generate the {@code } element, appended as a child of the
- * {@code } element.
- *
- *
- * It generates the following element:
- *
- * {@code }
- *
- * (where {@code indexStart} and {@code indexEnd} are gotten from the {@link ItagItem} passed
- * as the second parameter)
- *
- *
- *
- * The {@code } element needs to be generated before this element with
- * {@link YoutubeDashManifestCreatorsUtils#generateRepresentationElement(Document, ItagItem)}),
- * and the {@code BaseURL} element with {@link #generateBaseUrlElement(Document, String)}
- * should be generated too.
- *
- *
- * @param doc the {@link Document} on which the {@code } element will be appended
- * @param itagItem the {@link ItagItem} to use, which must not be null
- */
- private static void generateSegmentBaseElement(@Nonnull final Document doc,
- @Nonnull final ItagItem itagItem)
- throws CreationException {
- try {
- final Element representationElement = (Element) doc.getElementsByTagName(
- REPRESENTATION).item(0);
- final Element segmentBaseElement = doc.createElement(SEGMENT_BASE);
-
- final String range = itagItem.getIndexStart() + "-" + itagItem.getIndexEnd();
- if (itagItem.getIndexStart() < 0 || itagItem.getIndexEnd() < 0) {
- throw CreationException.couldNotAddElement(SEGMENT_BASE,
- "ItagItem's indexStart or " + "indexEnd are < 0: " + range);
- }
- setAttribute(segmentBaseElement, doc, "indexRange", range);
-
- representationElement.appendChild(segmentBaseElement);
- } catch (final DOMException e) {
- throw CreationException.couldNotAddElement(SEGMENT_BASE, e);
- }
- }
-
- /**
- * Generate the {@code } element, appended as a child of the
- * {@code } element.
- *
- *
- * It generates the following element:
- *
- * {@code }
- *
- * (where {@code indexStart} and {@code indexEnd} are gotten from the {@link ItagItem} passed
- * as the second parameter)
- *
- *
- *
- * The {@code } element needs to be generated before this element with
- * {@link #generateSegmentBaseElement(Document, ItagItem)}).
- *
- *
- * @param doc the {@link Document} on which the {@code } element will be
- * appended
- * @param itagItem the {@link ItagItem} to use, which must not be null
- */
- private static void generateInitializationElement(@Nonnull final Document doc,
- @Nonnull final ItagItem itagItem)
- throws CreationException {
- try {
- final Element segmentBaseElement = (Element) doc.getElementsByTagName(
- SEGMENT_BASE).item(0);
- final Element initializationElement = doc.createElement(INITIALIZATION);
-
- final String range = itagItem.getInitStart() + "-" + itagItem.getInitEnd();
- if (itagItem.getInitStart() < 0 || itagItem.getInitEnd() < 0) {
- throw CreationException.couldNotAddElement(INITIALIZATION,
- "ItagItem's initStart and/or " + "initEnd are/is < 0: " + range);
- }
- setAttribute(initializationElement, doc, "range", range);
-
- segmentBaseElement.appendChild(initializationElement);
- } catch (final DOMException e) {
- throw CreationException.couldNotAddElement(INITIALIZATION, e);
- }
- }
-}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/info/ItagInfoRange.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/info/ItagInfoRange.java
index 2e38f0b11..cf3bd2f14 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/info/ItagInfoRange.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/info/ItagInfoRange.java
@@ -9,11 +9,19 @@ public class ItagInfoRange {
this.end = end;
}
- public int getStart() {
+ public int start() {
return start;
}
- public int getEnd() {
+ public int end() {
return end;
}
+
+ @Override
+ public String toString() {
+ return "ItagInfoRange{" +
+ "start=" + start +
+ ", end=" + end +
+ '}';
+ }
}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/dashmanifestcreator/DashManifestCreationException.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/dashmanifestcreator/DashManifestCreationException.java
new file mode 100644
index 000000000..b2aa4927c
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/dashmanifestcreator/DashManifestCreationException.java
@@ -0,0 +1,56 @@
+package org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator;
+
+import javax.annotation.Nonnull;
+
+/**
+ * Exception that is thrown when a DASH manifest creator encounters a problem
+ * while creating a manifest.
+ */
+public class DashManifestCreationException extends RuntimeException {
+
+ public DashManifestCreationException(final String message) {
+ super(message);
+ }
+
+ public DashManifestCreationException(final String message, final Exception cause) {
+ super(message, cause);
+ }
+
+ // Methods to create exceptions easily without having to use big exception messages and to
+ // reduce duplication
+
+ /**
+ * Create a new {@link DashManifestCreationException} with a cause and the following detail
+ * message format:
+ *
+ * {@code "Could not add " + element + " element", cause}, where {@code element} is an element
+ * of a DASH manifest.
+ *
+ * @param element the element which was not added to the DASH document
+ * @param cause the exception which prevented addition of the element to the DASH document
+ * @return a new {@link DashManifestCreationException}
+ */
+ @Nonnull
+ public static DashManifestCreationException couldNotAddElement(final String element,
+ final Exception cause) {
+ return new DashManifestCreationException("Could not add " + element + " element", cause);
+ }
+
+ /**
+ * Create a new {@link DashManifestCreationException} with a cause and the following detail
+ * message format:
+ *
+ * {@code "Could not add " + element + " element: " + reason}, where {@code element} is an
+ * element of a DASH manifest and {@code reason} the reason why this element cannot be added to
+ * the DASH document.
+ *
+ * @param element the element which was not added to the DASH document
+ * @param reason the reason message of why the element has been not added to the DASH document
+ * @return a new {@link DashManifestCreationException}
+ */
+ @Nonnull
+ public static DashManifestCreationException couldNotAddElement(final String element,
+ final String reason) {
+ return new DashManifestCreationException("Could not add " + element + " element: " + reason);
+ }
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/dashmanifestcreator/DashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/dashmanifestcreator/DashManifestCreator.java
new file mode 100644
index 000000000..66cd2d860
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/dashmanifestcreator/DashManifestCreator.java
@@ -0,0 +1,14 @@
+package org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator;
+
+import javax.annotation.Nonnull;
+
+public interface DashManifestCreator {
+
+ /**
+ * Generates the DASH manifest.
+ * @return The dash manifest as string.
+ * @throws DashManifestCreationException May throw a CreationException
+ */
+ @Nonnull
+ String generateManifest();
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/dashmanifestcreator/DashManifestCreatorConstants.java b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/dashmanifestcreator/DashManifestCreatorConstants.java
new file mode 100644
index 000000000..10d74239c
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/streamdata/delivery/dashmanifestcreator/DashManifestCreatorConstants.java
@@ -0,0 +1,21 @@
+package org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator;
+
+public final class DashManifestCreatorConstants {
+ private DashManifestCreatorConstants() {
+ // No impl!
+ }
+
+ // XML elements of DASH MPD manifests
+ // see https://www.brendanlong.com/the-structure-of-an-mpeg-dash-mpd.html
+ public static final String MPD = "MPD";
+ public static final String PERIOD = "Period";
+ public static final String ADAPTATION_SET = "AdaptationSet";
+ public static final String ROLE = "Role";
+ public static final String REPRESENTATION = "Representation";
+ public static final String AUDIO_CHANNEL_CONFIGURATION = "AudioChannelConfiguration";
+ public static final String SEGMENT_TEMPLATE = "SegmentTemplate";
+ public static final String SEGMENT_TIMELINE = "SegmentTimeline";
+ public static final String BASE_URL = "BaseURL";
+ public static final String SEGMENT_BASE = "SegmentBase";
+ public static final String INITIALIZATION = "Initialization";
+}
diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorsTest.java
index 0d276f901..9a396a7f9 100644
--- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorsTest.java
+++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorsTest.java
@@ -1,30 +1,5 @@
package org.schabi.newpipe.extractor.services.youtube;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Test;
-import org.schabi.newpipe.downloader.DownloaderTestImpl;
-import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.CreationException;
-import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeOtfDashManifestCreator;
-import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeProgressiveDashManifestCreator;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
-import org.schabi.newpipe.extractor.stream.DeliveryMethod;
-import org.schabi.newpipe.extractor.stream.Stream;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.NodeList;
-import org.xml.sax.InputSource;
-
-import javax.annotation.Nonnull;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import java.io.StringReader;
-import java.util.List;
-import java.util.Random;
-import java.util.function.Consumer;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-
import static org.junit.jupiter.api.Assertions.assertAll;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -36,19 +11,45 @@ import static org.schabi.newpipe.extractor.ExtractorAsserts.assertGreaterOrEqual
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsValidUrl;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertNotBlank;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ADAPTATION_SET;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.AUDIO_CHANNEL_CONFIGURATION;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.BASE_URL;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.INITIALIZATION;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.MPD;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.PERIOD;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.REPRESENTATION;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ROLE;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_BASE;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TEMPLATE;
-import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TIMELINE;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeDashManifestCreatorsUtils.ADAPTATION_SET;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeDashManifestCreatorsUtils.AUDIO_CHANNEL_CONFIGURATION;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeDashManifestCreatorsUtils.BASE_URL;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeDashManifestCreatorsUtils.INITIALIZATION;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeDashManifestCreatorsUtils.MPD;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeDashManifestCreatorsUtils.PERIOD;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeDashManifestCreatorsUtils.REPRESENTATION;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeDashManifestCreatorsUtils.ROLE;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeDashManifestCreatorsUtils.SEGMENT_BASE;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeDashManifestCreatorsUtils.SEGMENT_TEMPLATE;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeDashManifestCreatorsUtils.SEGMENT_TIMELINE;
import static org.schabi.newpipe.extractor.utils.Utils.isBlank;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeOtfDashManifestCreator;
+import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreator.YoutubeProgressiveDashManifestCreator;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
+import org.schabi.newpipe.extractor.stream.DeliveryMethod;
+import org.schabi.newpipe.extractor.stream.Stream;
+import org.schabi.newpipe.extractor.streamdata.delivery.dashmanifestcreator.DashManifestCreationException;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+
+import java.io.StringReader;
+import java.util.List;
+import java.util.Random;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import javax.annotation.Nonnull;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
/**
* Test for YouTube DASH manifest creators.
*
@@ -112,7 +113,7 @@ class YoutubeDashManifestCreatorsTest {
assertProgressiveStreams(extractor.getAudioStreams());
// we are not able to generate DASH manifests of video formats with audio
- assertThrows(CreationException.class,
+ assertThrows(DashManifestCreationException.class,
() -> assertProgressiveStreams(extractor.getVideoStreams()));
}