PROGRESSIVE_CACHE
- = new ManifestCreatorCache<>();
-
-
- /**
- * URL parameter of the first sequence for live, post-live-DVR and OTF streams.
- */
- private static final String SQ_0 = "&sq=0";
-
- /**
- * URL parameter of the first stream request made by official clients.
- */
- private static final String RN_0 = "&rn=0";
-
- /**
- * URL parameter specific to web clients. When this param is added, if a redirection occurs,
- * the server will not redirect clients to the redirect URL. Instead, it will provide this URL
- * as the response body.
- */
- private static final String ALR_YES = "&alr=yes";
-
- public static final String SEGMENT_TIMELINE = "SegmentTimeline";
- public static final String ADAPTATION_SET = "AdaptationSet";
- public static final String REPRESENTATION = "Representation";
- public static final String SEGMENT_TEMPLATE = "SegmentTemplate";
- public static final String INITIALIZATION = "Initialization";
- public static final String PERIOD = "Period";
- public static final String SEGMENT_BASE = "SegmentBase";
-
- /**
- * Enum of streaming format types used by YouTube in their streams.
- */
- private enum DeliveryType {
-
- /**
- * YouTube's progressive delivery method, which works with HTTP range headers.
- * (Note that official clients use the corresponding parameter instead.)
- *
- *
- * Initialization and index ranges are available to get metadata (the corresponding values
- * are returned in the player response).
- *
- */
- PROGRESSIVE,
-
- /**
- * YouTube's OTF delivery method which uses a sequence parameter to get segments of
- * streams.
- *
- *
- * The first sequence (which can be fetched with the {@link #SQ_0} param) contains all the
- * metadata needed to build the stream source (sidx boxes, segment length, segment count,
- * duration, ...)
- *
- *
- *
- * Only used for videos; mostly those with a small amount of views, or ended livestreams
- * which have just been re-encoded as normal videos.
- *
- */
- OTF,
-
- /**
- * YouTube's delivery method for livestreams which uses a sequence parameter to get
- * segments of streams.
- *
- *
- * Each sequence (which can be fetched with the {@link #SQ_0} param) contains its own
- * metadata (sidx boxes, segment length, ...), which make no need of an initialization
- * segment.
- *
- *
- *
- * Only used for livestreams (ended or running).
- *
- */
- LIVE
- }
-
- private YoutubeDashManifestCreator() {
- }
-
- /**
- * Exception that is thrown when the {@link YoutubeDashManifestCreator} encounters a problem
- * while creating a manifest.
- */
- public static final class CreationException extends Exception {
-
- CreationException(final String message) {
- super(message);
- }
-
- CreationException(final String message, final Exception e) {
- super(message, e);
- }
-
- public static CreationException couldNotAdd(final String element, final Exception e) {
- return new CreationException("Could not add " + element + " element", e);
- }
-
- public static CreationException couldNotAdd(final String element, final String reason) {
- return new CreationException("Could not add " + element + " element: " + reason);
- }
- }
-
- /**
- * Create DASH manifests from a YouTube OTF stream.
- *
- *
- * OTF streams are YouTube-DASH specific streams which work with sequences and without the need
- * to get a manifest (even if one is provided, it is not used by official clients).
- *
- *
- * They can be found only on videos; mostly those with a small amount of views, or ended
- * livestreams which have just been re-encoded as normal videos.
- *
- *
- * This method needs:
- *
- * - the base URL of the stream (which, if you try to access to it, returns HTTP
- * status code 404 after redirects, and if the URL is valid);
- * - an {@link ItagItem}, which needs to contain the following information:
- *
- * - its type (see {@link ItagItem.ItagType}), to identify if the content is
- * an audio or a video stream;
- * - its bitrate;
- * - its mime type;
- * - its codec(s);
- * - for an audio stream: its audio channels;
- * - for a video stream: its width and height.
- *
- *
- * - the duration of the video, which will be used if the duration could not be
- * parsed from the first sequence of the stream.
- *
- *
- *
- * In order to generate the DASH manifest, this method will:
- *
- * - request the first sequence of the stream (the base URL on which the first
- * sequence parameter is appended (see {@link #SQ_0})) with a POST or GET request
- * (depending of the client on which the streaming URL comes from);
- *
- * - follow its redirection(s), if any;
- * - save the last URL, remove the first sequence parameter;
- * - use the information provided in the {@link ItagItem} to generate all
- * elements of the DASH manifest.
- *
- *
- *
- *
- * If the duration cannot be extracted, the {@code durationSecondsFallback} value will be used
- * as the stream duration.
- *
- *
- * @param otfBaseStreamingUrl the base URL of the OTF stream, which cannot be null
- * @param itagItem the {@link ItagItem} corresponding to the stream, which
- * cannot be null
- * @param durationSecondsFallback the duration of the video, which will be used if the duration
- * could not be extracted from the first sequence
- * @return the manifest generated into a string
- */
- @Nonnull
- public static String fromOtfStreamingUrl(
- @Nonnull final String otfBaseStreamingUrl,
- @Nonnull final ItagItem itagItem,
- final long durationSecondsFallback) throws CreationException {
- if (OTF_CACHE.containsKey(otfBaseStreamingUrl)) {
- return Objects.requireNonNull(OTF_CACHE.get(otfBaseStreamingUrl)).getSecond();
- }
-
- String realOtfBaseStreamingUrl = otfBaseStreamingUrl;
- // Try to avoid redirects when streaming the content by saving the last URL we get
- // from video servers.
- final Response response = getInitializationResponse(realOtfBaseStreamingUrl,
- itagItem, DeliveryType.OTF);
- realOtfBaseStreamingUrl = response.latestUrl().replace(SQ_0, EMPTY_STRING)
- .replace(RN_0, EMPTY_STRING).replace(ALR_YES, EMPTY_STRING);
-
- final int responseCode = response.responseCode();
- if (responseCode != 200) {
- throw new CreationException("Could not get the initialization URL of "
- + "the OTF stream: response code " + responseCode);
- }
-
- final String[] segmentDuration;
-
- try {
- final String[] segmentsAndDurationsResponseSplit = response.responseBody()
- // Get the lines with the durations and the following
- .split("Segment-Durations-Ms: ")[1]
- // Remove the other lines
- .split("\n")[0]
- // Get all durations and repetitions which are separated by a comma
- .split(",");
- final int lastIndex = segmentsAndDurationsResponseSplit.length - 1;
- if (isBlank(segmentsAndDurationsResponseSplit[lastIndex])) {
- segmentDuration = Arrays.copyOf(segmentsAndDurationsResponseSplit, lastIndex);
- } else {
- segmentDuration = segmentsAndDurationsResponseSplit;
- }
- } catch (final Exception e) {
- throw new CreationException("Could not get segment durations", e);
- }
-
- final Document document = generateDocumentAndMpdElement(segmentDuration, DeliveryType.OTF,
- itagItem, durationSecondsFallback);
- generatePeriodElement(document);
- generateAdaptationSetElement(document, itagItem);
- generateRoleElement(document);
- generateRepresentationElement(document, itagItem);
- if (itagItem.itagType == ItagItem.ItagType.AUDIO) {
- generateAudioChannelConfigurationElement(document, itagItem);
- }
- generateSegmentTemplateElement(document, realOtfBaseStreamingUrl, DeliveryType.OTF);
- generateSegmentTimelineElement(document);
- generateSegmentElementsForOtfStreams(segmentDuration, document);
-
- return buildAndCacheResult(otfBaseStreamingUrl, document, OTF_CACHE);
- }
-
- /**
- * Create DASH manifests from a YouTube post-live-DVR stream/ended livestream.
- *
- *
- * Post-live-DVR streams/ended livestreams are one of the YouTube DASH specific streams which
- * works with sequences and without the need to get a manifest (even if one is provided but not
- * used by main clients (and is complete for big ended livestreams because it doesn't return
- * the full stream)).
- *
- *
- *
- * They can be found only on livestreams which have ended very recently (a few hours, most of
- * the time)
- *
- *
- * This method needs:
- *
- * - the base URL of the stream (which, if you try to access to it, returns HTTP
- * status code 404 after redirects, and if the URL is valid);
- * - an {@link ItagItem}, which needs to contain the following information:
- *
- * - its type (see {@link ItagItem.ItagType}), to identify if the content is
- * an audio or a video stream;
- * - its bitrate;
- * - its mime type;
- * - its codec(s);
- * - for an audio stream: its audio channels;
- * - for a video stream: its width and height.
- *
- *
- * - the duration of the video, which will be used if the duration could not be
- * parsed from the first sequence of the stream.
- *
- *
- *
- * In order to generate the DASH manifest, this method will:
- *
- * - request the first sequence of the stream (the base URL on which the first
- * sequence parameter is appended (see {@link #SQ_0})) with a POST or GET request
- * (depending of the client on which the streaming URL comes from);
- *
- * - follow its redirection(s), if any;
- * - save the last URL, remove the first sequence parameters;
- * - use the information provided in the {@link ItagItem} to generate all
- * elements of the DASH manifest.
- *
- *
- *
- *
- * If the duration cannot be extracted, the {@code durationSecondsFallback} value will be used
- * as the stream duration.
- *
- *
- * @param postLiveStreamDvrStreamingUrl the base URL of the post-live-DVR stream/ended
- * livestream, which cannot be null
- * @param itagItem the {@link ItagItem} corresponding to the stream, which
- * cannot be null
- * @param targetDurationSec the target duration of each sequence, in seconds (this
- * value is returned with the targetDurationSec field for
- * each stream in YouTube player response)
- * @param durationSecondsFallback the duration of the ended livestream which will be used
- * if the duration could not be extracted from the first
- * sequence
- * @return the manifest generated into a string
- */
- @Nonnull
- public static String fromPostLiveStreamDvrStreamingUrl(
- @Nonnull final String postLiveStreamDvrStreamingUrl,
- @Nonnull final ItagItem itagItem,
- final int targetDurationSec,
- final long durationSecondsFallback) throws CreationException {
- if (POST_LIVE_DVR_CACHE.containsKey(postLiveStreamDvrStreamingUrl)) {
- return Objects.requireNonNull(POST_LIVE_DVR_CACHE.get(postLiveStreamDvrStreamingUrl))
- .getSecond();
- }
- String realPostLiveStreamDvrStreamingUrl = postLiveStreamDvrStreamingUrl;
- final String streamDuration;
- final String segmentCount;
-
- if (targetDurationSec <= 0) {
- throw new CreationException("targetDurationSec value is <= 0: " + targetDurationSec);
- }
-
- try {
- // Try to avoid redirects when streaming the content by saving the latest URL we get
- // from video servers.
- final Response response = getInitializationResponse(realPostLiveStreamDvrStreamingUrl,
- itagItem, DeliveryType.LIVE);
- realPostLiveStreamDvrStreamingUrl = response.latestUrl().replace(SQ_0, EMPTY_STRING)
- .replace(RN_0, EMPTY_STRING).replace(ALR_YES, EMPTY_STRING);
-
- final int responseCode = response.responseCode();
- if (responseCode != 200) {
- throw new CreationException("Could not get the initialization "
- + "segment of the post-live-DVR stream: response code " + responseCode);
- }
-
- final Map> responseHeaders = response.responseHeaders();
- streamDuration = responseHeaders.get("X-Head-Time-Millis").get(0);
- segmentCount = responseHeaders.get("X-Head-Seqnum").get(0);
- } catch (final IndexOutOfBoundsException e) {
- throw new CreationException("Could not get the value of the X-Head-Time-Millis or the "
- + "X-Head-Seqnum header of the post-live-DVR streaming URL", e);
- }
-
- if (isNullOrEmpty(segmentCount)) {
- throw new CreationException(
- "Could not get the number of segments of the post-live-DVR stream");
- }
-
- final Document document = generateDocumentAndMpdElement(new String[] {streamDuration},
- DeliveryType.LIVE, itagItem, durationSecondsFallback);
- generatePeriodElement(document);
- generateAdaptationSetElement(document, itagItem);
- generateRoleElement(document);
- generateRepresentationElement(document, itagItem);
- if (itagItem.itagType == ItagItem.ItagType.AUDIO) {
- generateAudioChannelConfigurationElement(document, itagItem);
- }
- generateSegmentTemplateElement(document, realPostLiveStreamDvrStreamingUrl,
- DeliveryType.LIVE);
- generateSegmentTimelineElement(document);
- generateSegmentElementForPostLiveDvrStreams(document, targetDurationSec, segmentCount);
-
- return buildAndCacheResult(postLiveStreamDvrStreamingUrl, document,
- POST_LIVE_DVR_CACHE);
- }
-
- /**
- * Create DASH manifests from a YouTube progressive stream.
- *
- *
- * Progressive streams are YouTube DASH streams which work with range requests and without the
- * need to get a manifest.
- *
- *
- *
- * They can be found on all videos, and for all streams for most of videos which come from a
- * YouTube partner, and on videos with a large number of views.
- *
- *
- * This method needs:
- *
- * - the base URL of the stream (which, if you try to access to it, returns the whole
- * stream, after redirects, and if the URL is valid);
- * - an {@link ItagItem}, which needs to contain the following information:
- *
- * - its type (see {@link ItagItem.ItagType}), to identify if the content is
- * an audio or a video stream;
- * - its bitrate;
- * - its mime type;
- * - its codec(s);
- * - for an audio stream: its audio channels;
- * - for a video stream: its width and height.
- *
- *
- * - the duration of the video, which will be used if the duration could not be
- * parsed from the {@link ItagItem}.
- *
- *
- *
- * In order to generate the DASH manifest, this method will:
- *
- * - request the base URL of the stream with a HEAD request;
- * - follow its redirection(s), if any;
- * - save the last URL;
- * - use the information provided in the {@link ItagItem} to generate all
- * elements of the DASH manifest.
- *
- *
- *
- *
- * If the duration cannot be extracted, the {@code durationSecondsFallback} value will be used
- * as the stream duration.
- *
- *
- * @param progressiveStreamingBaseUrl the base URL of the progressive stream, which cannot be
- * null
- * @param itagItem the {@link ItagItem} corresponding to the stream, which
- * cannot be null
- * @param durationSecondsFallback the duration of the progressive stream which will be used
- * if the duration could not be extracted from the first
- * sequence
- * @return the manifest generated into a string
- */
- @Nonnull
- public static String fromProgressiveStreamingUrl(
- @Nonnull final String progressiveStreamingBaseUrl,
- @Nonnull final ItagItem itagItem,
- final long durationSecondsFallback) throws CreationException {
- if (PROGRESSIVE_CACHE.containsKey(progressiveStreamingBaseUrl)) {
- return Objects.requireNonNull(PROGRESSIVE_CACHE.get(progressiveStreamingBaseUrl))
- .getSecond();
- }
-
- final Document document = generateDocumentAndMpdElement(new String[]{},
- DeliveryType.PROGRESSIVE, itagItem, durationSecondsFallback);
- generatePeriodElement(document);
- generateAdaptationSetElement(document, itagItem);
- generateRoleElement(document);
- generateRepresentationElement(document, itagItem);
- if (itagItem.itagType == ItagItem.ItagType.AUDIO) {
- generateAudioChannelConfigurationElement(document, itagItem);
- }
- generateBaseUrlElement(document, progressiveStreamingBaseUrl);
- generateSegmentBaseElement(document, itagItem);
- generateInitializationElement(document, itagItem);
-
- return buildAndCacheResult(progressiveStreamingBaseUrl, document,
- PROGRESSIVE_CACHE);
- }
-
- /**
- * Get the "initialization" {@link Response response} of a stream.
- *
- *
- * This method fetches, for OTF streams and for post-live-DVR streams:
- *
- * - the base URL of the stream, to which are appended {@link #SQ_0} and {@link #RN_0}
- * parameters, with a GET request for streaming URLs from the WEB client and a POST request
- * for the ones from the Android client;
- * - for streaming URLs from the WEB client, the {@link #ALR_YES} param is also added.
- *
- *
- *
- *
- * @param baseStreamingUrl the base URL of the stream, which cannot be null
- * @param itagItem the {@link ItagItem} of stream, which cannot be null
- * @param deliveryType the {@link DeliveryType} of the stream
- * @return the "initialization" response, without redirections on the network on which the
- * request(s) is/are made
- */
- @SuppressWarnings("checkstyle:FinalParameters")
- @Nonnull
- private static Response getInitializationResponse(@Nonnull String baseStreamingUrl,
- @Nonnull final ItagItem itagItem,
- final DeliveryType deliveryType)
- throws CreationException {
- final boolean isAHtml5StreamingUrl = isWebStreamingUrl(baseStreamingUrl)
- || isTvHtml5SimplyEmbeddedPlayerStreamingUrl(baseStreamingUrl);
- final boolean isAnAndroidStreamingUrl = isAndroidStreamingUrl(baseStreamingUrl);
- final boolean isAnIosStreamingUrl = isIosStreamingUrl(baseStreamingUrl);
- if (isAHtml5StreamingUrl) {
- baseStreamingUrl += ALR_YES;
- }
- baseStreamingUrl = appendRnParamAndSqParamIfNeeded(baseStreamingUrl, deliveryType);
-
- final Downloader downloader = NewPipe.getDownloader();
- if (isAHtml5StreamingUrl) {
- final String mimeTypeExpected = itagItem.getMediaFormat().getMimeType();
- if (!isNullOrEmpty(mimeTypeExpected)) {
- return getStreamingWebUrlWithoutRedirects(downloader, baseStreamingUrl,
- mimeTypeExpected, deliveryType);
- }
- } else if (isAnAndroidStreamingUrl || isAnIosStreamingUrl) {
- try {
- final Map> headers = new HashMap<>();
- headers.put("User-Agent", Collections.singletonList(
- isAnAndroidStreamingUrl ? getAndroidUserAgent(null)
- : getIosUserAgent(null)));
- final byte[] emptyBody = "".getBytes(StandardCharsets.UTF_8);
- return downloader.post(baseStreamingUrl, headers, emptyBody);
- } catch (final IOException | ExtractionException e) {
- throw new CreationException("Could not get the "
- + (isAnIosStreamingUrl ? "ANDROID" : "IOS") + " streaming URL response", e);
- }
- }
-
- try {
- return downloader.get(baseStreamingUrl);
- } catch (final IOException | ExtractionException e) {
- throw new CreationException("Could not get the streaming URL response", e);
- }
- }
-
- /**
- * Append {@link #SQ_0} for post-live-DVR and OTF streams and {@link #RN_0} to all streams.
- *
- * @param baseStreamingUrl the base streaming URL to which the parameter(s) are being appended
- * @param deliveryType the {@link DeliveryType} of the stream
- * @return the base streaming URL to which the param(s) are appended, depending on the
- * {@link DeliveryType} of the stream
- */
- @SuppressWarnings({"checkstyle:FinalParameters", "checkstyle:FinalLocalVariable"})
- @Nonnull
- private static String appendRnParamAndSqParamIfNeeded(
- @Nonnull String baseStreamingUrl,
- @Nonnull final DeliveryType deliveryType) {
- if (deliveryType != DeliveryType.PROGRESSIVE) {
- baseStreamingUrl += SQ_0;
- }
- return baseStreamingUrl + RN_0;
- }
-
- /**
- * Get a URL on which no redirection between playback hosts should be present on the network
- * and/or IP used to fetch the streaming URL.
- *
- *
- * This method will follow redirects for web clients, which works in the following way:
- *
- * - the {@link #ALR_YES} param is appended to all streaming URLs
- * - if no redirection occurs, the video server will return the streaming data;
- * - if a redirection occurs, the server will respond with HTTP status code 200 and a
- * text/plain mime type. The redirection URL is the response body;
- * - the redirection URL is requested and the steps above from step 2 are repeated (until
- * too many redirects are reached, of course).
- *
- *
- *
- * @param downloader the {@link Downloader} instance to be used
- * @param streamingUrl the streaming URL which we are trying to get a streaming URL
- * without any redirection on the network and/or IP used
- * @param responseMimeTypeExpected the response mime type expected from Google video servers
- * @param deliveryType the {@link DeliveryType} of the stream
- * @return the response of the stream which should have no redirections
- */
- @SuppressWarnings("checkstyle:FinalParameters")
- @Nonnull
- private static Response getStreamingWebUrlWithoutRedirects(
- @Nonnull final Downloader downloader,
- @Nonnull String streamingUrl,
- @Nonnull final String responseMimeTypeExpected,
- @Nonnull final DeliveryType deliveryType) throws CreationException {
- try {
- final Map> headers = new HashMap<>();
- addClientInfoHeaders(headers);
-
- String responseMimeType = "";
-
- int redirectsCount = 0;
- while (!responseMimeType.equals(responseMimeTypeExpected)
- && redirectsCount < MAXIMUM_REDIRECT_COUNT) {
- final Response response;
- // We can use head requests to reduce the request size, but only for progressive
- // streams
- if (deliveryType == DeliveryType.PROGRESSIVE) {
- response = downloader.head(streamingUrl, headers);
- } else {
- response = downloader.get(streamingUrl, headers);
- }
-
- final int responseCode = response.responseCode();
- if (responseCode != 200) {
- throw new CreationException("Could not get the initialization URL of the "
- + deliveryType + " stream: response code " + responseCode);
- }
-
- // A valid response must include a Content-Type header, so we can require that
- // the response from video servers has this header.
- responseMimeType = Objects.requireNonNull(response.getHeader("Content-Type"),
- "Could not get the Content-Type header from the streaming URL");
-
- // The response body is the redirection URL
- if (responseMimeType.equals("text/plain")) {
- streamingUrl = response.responseBody();
- redirectsCount++;
- } else {
- return response;
- }
- }
-
- if (redirectsCount >= MAXIMUM_REDIRECT_COUNT) {
- throw new CreationException(
- "Too many redirects when trying to get the WEB streaming URL response");
- }
-
- // This should never be reached, but is required because we don't want to return null
- // here
- throw new CreationException(
- "Could not get the WEB streaming URL response: unreachable code reached!");
- } catch (final IOException | ExtractionException e) {
- throw new CreationException("Could not get the WEB streaming URL response", e);
- }
- }
-
- /**
- * Get the duration of an OTF stream.
- *
- *
- * The duration of OTF streams is not returned into the player response and needs to be
- * calculated by adding the duration of each segment.
- *
- *
- * @param segmentDuration the segment duration object extracted from the initialization
- * sequence of the stream
- * @return the duration of the OTF stream
- */
- private static int getStreamDuration(@Nonnull final String[] segmentDuration)
- throws CreationException {
- try {
- int streamLengthMs = 0;
- for (final String segDuration : segmentDuration) {
- final String[] segmentLengthRepeat = segDuration.split("\\(r=");
- int segmentRepeatCount = 0;
- // There are repetitions of a segment duration in other segments
- if (segmentLengthRepeat.length > 1) {
- segmentRepeatCount = Integer.parseInt(Utils.removeNonDigitCharacters(
- segmentLengthRepeat[1]));
- }
- final int segmentLength = Integer.parseInt(segmentLengthRepeat[0]);
- streamLengthMs += segmentLength + segmentRepeatCount * segmentLength;
- }
- return streamLengthMs;
- } catch (final NumberFormatException e) {
- throw new CreationException("Could not get stream length", e);
- }
- }
-
- /**
- * Create a {@link Document} object and generate the {@code } element of the manifest.
- *
- *
- * The generated {@code } element looks like the manifest returned into the player
- * response of videos with OTF streams:
- *
- *
- *
- * {@code }
- * (where {@code $duration$} represents the duration in seconds (a number with 3 digits after
- * the decimal point)
- *
- *
- *
- * If the duration is an integer or a double with less than 3 digits after the decimal point,
- * it will be converted into a double with 3 digits after the decimal point.
- *
- *
- * @param segmentDuration the segment duration object extracted from the initialization
- * sequence of the stream
- * @param deliveryType the {@link DeliveryType} of the stream, see the enum for
- * possible values
- * @param itagItem the {@link ItagItem} which will be used to get the duration
- * of progressive streams
- * @param durationSecondsFallback the duration in seconds, extracted from player response, used
- * as a fallback if the duration could not be determined
- * @return a {@link Document} object which contains a {@code } element
- */
- private static Document generateDocumentAndMpdElement(@Nonnull final String[] segmentDuration,
- final DeliveryType deliveryType,
- @Nonnull final ItagItem itagItem,
- final long durationSecondsFallback)
- throws CreationException {
- try {
- final Document document = newDocument();
-
- final Element mpdElement = document.createElement("MPD");
- document.appendChild(mpdElement);
-
- final Attr xmlnsXsiAttribute = document.createAttribute("xmlns:xsi");
- xmlnsXsiAttribute.setValue("http://www.w3.org/2001/XMLSchema-instance");
- mpdElement.setAttributeNode(xmlnsXsiAttribute);
-
- final Attr xmlns = document.createAttribute("xmlns");
- xmlns.setValue("urn:mpeg:DASH:schema:MPD:2011");
- mpdElement.setAttributeNode(xmlns);
-
- final Attr xsiSchemaLocationAttribute = document.createAttribute("xsi:schemaLocation");
- xsiSchemaLocationAttribute.setValue("urn:mpeg:DASH:schema:MPD:2011 DASH-MPD.xsd");
- mpdElement.setAttributeNode(xsiSchemaLocationAttribute);
-
- final Attr minBufferTimeAttribute = document.createAttribute("minBufferTime");
- minBufferTimeAttribute.setValue("PT1.500S");
- mpdElement.setAttributeNode(minBufferTimeAttribute);
-
- final Attr profilesAttribute = document.createAttribute("profiles");
- profilesAttribute.setValue("urn:mpeg:dash:profile:full:2011");
- mpdElement.setAttributeNode(profilesAttribute);
-
- final Attr typeAttribute = document.createAttribute("type");
- typeAttribute.setValue("static");
- mpdElement.setAttributeNode(typeAttribute);
-
- final Attr mediaPresentationDurationAttribute = document.createAttribute(
- "mediaPresentationDuration");
- final long streamDuration;
- if (deliveryType == DeliveryType.LIVE) {
- streamDuration = Integer.parseInt(segmentDuration[0]);
- } else if (deliveryType == DeliveryType.OTF) {
- streamDuration = getStreamDuration(segmentDuration);
- } else {
- final long itagItemDuration = itagItem.getApproxDurationMs();
- if (itagItemDuration != -1) {
- streamDuration = itagItemDuration;
- } else {
- if (durationSecondsFallback > 0) {
- streamDuration = durationSecondsFallback * 1000;
- } else {
- throw CreationException.couldNotAdd("MPD",
- "the duration of the stream could not be determined and the "
- + "durationSecondsFallback is <= 0");
- }
- }
- }
- final double duration = streamDuration / 1000.0;
- final String durationSeconds = String.format(Locale.ENGLISH, "%.3f", duration);
- mediaPresentationDurationAttribute.setValue("PT" + durationSeconds + "S");
- mpdElement.setAttributeNode(mediaPresentationDurationAttribute);
-
- return document;
- } catch (final Exception e) {
- throw CreationException.couldNotAdd("MPD", e);
- }
- }
-
- /**
- * Generate the {@code } element, appended as a child of the {@code } element.
- *
- *
- * The {@code } element needs to be generated before this element with
- * {@link #generateDocumentAndMpdElement(String[], DeliveryType, ItagItem, long)}.
- *
- *
- * @param document the {@link Document} on which the the {@code } element will be
- * appended
- */
- private static void generatePeriodElement(@Nonnull final Document document)
- throws CreationException {
- try {
- final Element mpdElement = (Element) document.getElementsByTagName("MPD").item(0);
- final Element periodElement = document.createElement(PERIOD);
- mpdElement.appendChild(periodElement);
- } catch (final DOMException e) {
- throw CreationException.couldNotAdd(PERIOD, e);
- }
- }
-
- /**
- * Generate the {@code } element, appended as a child of the {@code } element.
- *
- *
- * The {@code } element needs to be generated before this element with
- * {@link #generateDocumentAndMpdElement(String[], DeliveryType, ItagItem, long)}.
- *
- *
- * @param document the {@link Document} on which the the {@code } element will be
- * appended
- * @param itagItem the {@link ItagItem} corresponding to the stream, which cannot be null
- */
- private static void generateAdaptationSetElement(@Nonnull final Document document,
- @Nonnull final ItagItem itagItem)
- throws CreationException {
- try {
- final Element periodElement = (Element) document.getElementsByTagName(PERIOD)
- .item(0);
- final Element adaptationSetElement = document.createElement(ADAPTATION_SET);
-
- final Attr idAttribute = document.createAttribute("id");
- idAttribute.setValue("0");
- adaptationSetElement.setAttributeNode(idAttribute);
-
- final MediaFormat mediaFormat = itagItem.getMediaFormat();
- if (mediaFormat == null || isNullOrEmpty(mediaFormat.mimeType)) {
- throw CreationException.couldNotAdd(ADAPTATION_SET,
- "the MediaFormat or its mime type are null or empty");
- }
-
- final Attr mimeTypeAttribute = document.createAttribute("mimeType");
- mimeTypeAttribute.setValue(mediaFormat.mimeType);
- adaptationSetElement.setAttributeNode(mimeTypeAttribute);
-
- final Attr subsegmentAlignmentAttribute = document.createAttribute(
- "subsegmentAlignment");
- subsegmentAlignmentAttribute.setValue("true");
- adaptationSetElement.setAttributeNode(subsegmentAlignmentAttribute);
-
- periodElement.appendChild(adaptationSetElement);
- } catch (final DOMException e) {
- throw CreationException.couldNotAdd(ADAPTATION_SET, e);
- }
- }
-
- /**
- * Generate the {@code } element, appended as a child of the {@code }
- * element.
- *
- *
- * This element, with its attributes and values, is:
- *
- *
- *
- * {@code }
- *
- *
- *
- * The {@code } element needs to be generated before this element with
- * {@link #generateAdaptationSetElement(Document, ItagItem)}).
- *
- *
- * @param document the {@link Document} on which the the {@code } element will be
- * appended
- */
- private static void generateRoleElement(@Nonnull final Document document)
- throws CreationException {
- try {
- final Element adaptationSetElement = (Element) document.getElementsByTagName(
- ADAPTATION_SET).item(0);
- final Element roleElement = document.createElement("Role");
-
- final Attr schemeIdUriAttribute = document.createAttribute("schemeIdUri");
- schemeIdUriAttribute.setValue("urn:mpeg:DASH:role:2011");
- roleElement.setAttributeNode(schemeIdUriAttribute);
-
- final Attr valueAttribute = document.createAttribute("value");
- valueAttribute.setValue("main");
- roleElement.setAttributeNode(valueAttribute);
-
- adaptationSetElement.appendChild(roleElement);
- } catch (final DOMException e) {
- throw CreationException.couldNotAdd("Role", e);
- }
- }
-
- /**
- * Generate the {@code } element, appended as a child of the
- * {@code } element.
- *
- *
- * The {@code } element needs to be generated before this element with
- * {@link #generateAdaptationSetElement(Document, ItagItem)}).
- *
- *
- * @param document the {@link Document} on which the the {@code } element will
- * be appended
- * @param itagItem the {@link ItagItem} to use, which cannot be null
- */
- private static void generateRepresentationElement(@Nonnull final Document document,
- @Nonnull final ItagItem itagItem)
- throws CreationException {
- try {
- final Element adaptationSetElement = (Element) document.getElementsByTagName(
- ADAPTATION_SET).item(0);
- final Element representationElement = document.createElement(REPRESENTATION);
-
- final int id = itagItem.id;
- if (id <= 0) {
- throw CreationException.couldNotAdd(REPRESENTATION,
- "the id of the ItagItem is <= 0");
- }
- final Attr idAttribute = document.createAttribute("id");
- idAttribute.setValue(String.valueOf(id));
- representationElement.setAttributeNode(idAttribute);
-
- final String codec = itagItem.getCodec();
- if (isNullOrEmpty(codec)) {
- throw CreationException.couldNotAdd(ADAPTATION_SET,
- "the codec value is null or empty");
- }
- final Attr codecsAttribute = document.createAttribute("codecs");
- codecsAttribute.setValue(codec);
- representationElement.setAttributeNode(codecsAttribute);
-
- final Attr startWithSAPAttribute = document.createAttribute("startWithSAP");
- startWithSAPAttribute.setValue("1");
- representationElement.setAttributeNode(startWithSAPAttribute);
-
- final Attr maxPlayoutRateAttribute = document.createAttribute("maxPlayoutRate");
- maxPlayoutRateAttribute.setValue("1");
- representationElement.setAttributeNode(maxPlayoutRateAttribute);
-
- final int bitrate = itagItem.getBitrate();
- if (bitrate <= 0) {
- throw CreationException.couldNotAdd(REPRESENTATION,
- "the bitrate of the ItagItem is <= 0");
- }
- final Attr bandwidthAttribute = document.createAttribute("bandwidth");
- bandwidthAttribute.setValue(String.valueOf(bitrate));
- representationElement.setAttributeNode(bandwidthAttribute);
-
- final ItagItem.ItagType itagType = itagItem.itagType;
-
- if (itagType == ItagItem.ItagType.VIDEO || itagType == ItagItem.ItagType.VIDEO_ONLY) {
- final int height = itagItem.getHeight();
- final int width = itagItem.getWidth();
- if (height <= 0 && width <= 0) {
- throw CreationException.couldNotAdd(REPRESENTATION,
- "both width and height of the ItagItem are <= 0");
- }
-
- if (width > 0) {
- final Attr widthAttribute = document.createAttribute("width");
- widthAttribute.setValue(String.valueOf(width));
- representationElement.setAttributeNode(widthAttribute);
- }
-
- final Attr heightAttribute = document.createAttribute("height");
- heightAttribute.setValue(String.valueOf(itagItem.getHeight()));
- representationElement.setAttributeNode(heightAttribute);
-
- final int fps = itagItem.getFps();
- if (fps > 0) {
- final Attr frameRateAttribute = document.createAttribute("frameRate");
- frameRateAttribute.setValue(String.valueOf(fps));
- representationElement.setAttributeNode(frameRateAttribute);
- }
- }
-
- if (itagType == ItagItem.ItagType.AUDIO && itagItem.getSampleRate() > 0) {
- final Attr audioSamplingRateAttribute = document.createAttribute(
- "audioSamplingRate");
- audioSamplingRateAttribute.setValue(String.valueOf(itagItem.getSampleRate()));
- }
-
- adaptationSetElement.appendChild(representationElement);
- } catch (final DOMException e) {
- throw CreationException.couldNotAdd(REPRESENTATION, e);
- }
- }
-
- /**
- * Generate the {@code } element, appended as a child of the
- * {@code } element.
- *
- *
- * This method is only used when generating DASH manifests of audio streams.
- *
- *
- *
- * It will produce the following element:
- *
- * {@code
- * (where {@code audioChannelsValue} is get from the {@link ItagItem} passed as the second
- * parameter of this method)
- *
- *
- *
- * The {@code } element needs to be generated before this element with
- * {@link #generateRepresentationElement(Document, ItagItem)}).
- *
- *
- * @param document the {@link Document} on which the {@code }
- * element will be appended
- * @param itagItem the {@link ItagItem} to use, which cannot be null
- */
- private static void generateAudioChannelConfigurationElement(
- @Nonnull final Document document,
- @Nonnull final ItagItem itagItem) throws CreationException {
- try {
- final Element representationElement = (Element) document.getElementsByTagName(
- REPRESENTATION).item(0);
- final Element audioChannelConfigurationElement = document.createElement(
- "AudioChannelConfiguration");
-
- final Attr schemeIdUriAttribute = document.createAttribute("schemeIdUri");
- schemeIdUriAttribute.setValue(
- "urn:mpeg:dash:23003:3:audio_channel_configuration:2011");
- audioChannelConfigurationElement.setAttributeNode(schemeIdUriAttribute);
-
- final Attr valueAttribute = document.createAttribute("value");
- final int audioChannels = itagItem.getAudioChannels();
- if (audioChannels <= 0) {
- throw new CreationException("audioChannels is <= 0: " + audioChannels);
- }
- valueAttribute.setValue(String.valueOf(itagItem.getAudioChannels()));
- audioChannelConfigurationElement.setAttributeNode(valueAttribute);
-
- representationElement.appendChild(audioChannelConfigurationElement);
- } catch (final DOMException e) {
- throw CreationException.couldNotAdd("AudioChannelConfiguration", e);
- }
- }
-
- /**
- * Generate the {@code } element, appended as a child of the
- * {@code } element.
- *
- *
- * This method is only used when generating DASH manifests from progressive streams.
- *
- *
- *
- * The {@code } element needs to be generated before this element with
- * {@link #generateRepresentationElement(Document, ItagItem)}).
- *
- *
- * @param document the {@link Document} on which the {@code } element will
- * be appended
- * @param baseUrl the base URL of the stream, which cannot be null and will be set as the
- * content of the {@code } element
- */
- private static void generateBaseUrlElement(@Nonnull final Document document,
- @Nonnull final String baseUrl)
- throws CreationException {
- try {
- final Element representationElement = (Element) document.getElementsByTagName(
- REPRESENTATION).item(0);
- final Element baseURLElement = document.createElement("BaseURL");
- baseURLElement.setTextContent(baseUrl);
- representationElement.appendChild(baseURLElement);
- } catch (final DOMException e) {
- throw CreationException.couldNotAdd("BaseURL", e);
- }
- }
-
- // CHECKSTYLE:OFF
- /**
- * Generate the {@code } element, appended as a child of the
- * {@code } element.
- *
- *
- * This method is only used when generating DASH manifests from progressive streams.
- *
- *
- *
- * It generates the following element:
- *
- * {@code }
- *
- * (where {@code indexStart} and {@code indexEnd} are gotten from the {@link ItagItem} passed
- * as the second parameter)
- *
- *
- *
- * The {@code } element needs to be generated before this element with
- * {@link #generateRepresentationElement(Document, ItagItem)}).
- *
- *
- * @param document the {@link Document} on which the {@code } element will
- * be appended
- * @param itagItem the {@link ItagItem} to use, which cannot be null
- */
- // CHECKSTYLE:ON
- private static void generateSegmentBaseElement(@Nonnull final Document document,
- @Nonnull final ItagItem itagItem)
- throws CreationException {
- try {
- final Element representationElement = (Element) document.getElementsByTagName(
- REPRESENTATION).item(0);
-
- final Element segmentBaseElement = document.createElement(SEGMENT_BASE);
- final Attr indexRangeAttribute = document.createAttribute("indexRange");
-
- if (itagItem.getIndexStart() < 0 || itagItem.getIndexEnd() < 0) {
- throw CreationException.couldNotAdd(SEGMENT_BASE, "ItagItem's indexStart or "
- + "indexEnd are < 0: " + itagItem.getIndexStart() + "-"
- + itagItem.getIndexEnd());
- }
-
- indexRangeAttribute.setValue(itagItem.getIndexStart() + "-" + itagItem.getIndexEnd());
- segmentBaseElement.setAttributeNode(indexRangeAttribute);
-
- representationElement.appendChild(segmentBaseElement);
- } catch (final DOMException e) {
- throw CreationException.couldNotAdd(SEGMENT_BASE, e);
- }
- }
-
- /**
- * Generate the {@code } element, appended as a child of the
- * {@code } element.
- *
- *
- * This method is only used when generating DASH manifests from progressive streams.
- *
- *
- *
- * It generates the following element:
- *
- * {@code }
- *
- * (where {@code indexStart} and {@code indexEnd} are gotten from the {@link ItagItem} passed
- * as the second parameter)
- *
- *
- *
- * The {@code } element needs to be generated before this element with
- * {@link #generateSegmentBaseElement(Document, ItagItem)}).
- *
- *
- * @param document the {@link Document} on which the {@code } element will
- * be appended
- * @param itagItem the {@link ItagItem} to use, which cannot be null
- */
- private static void generateInitializationElement(@Nonnull final Document document,
- @Nonnull final ItagItem itagItem)
- throws CreationException {
- try {
- final Element segmentBaseElement = (Element) document.getElementsByTagName(
- SEGMENT_BASE).item(0);
-
- final Element initializationElement = document.createElement(INITIALIZATION);
- final Attr rangeAttribute = document.createAttribute("range");
-
- if (itagItem.getInitStart() < 0 || itagItem.getInitEnd() < 0) {
- throw CreationException.couldNotAdd(INITIALIZATION, "ItagItem's initStart or "
- + "initEnd are < 0: " + itagItem.getInitStart() + "-"
- + itagItem.getInitEnd());
- }
-
- rangeAttribute.setValue(itagItem.getInitStart() + "-" + itagItem.getInitEnd());
- initializationElement.setAttributeNode(rangeAttribute);
-
- segmentBaseElement.appendChild(initializationElement);
- } catch (final DOMException e) {
- throw CreationException.couldNotAdd(INITIALIZATION, e);
- }
- }
-
- /**
- * Generate the {@code } element, appended as a child of the
- * {@code } element.
- *
- *
- * This method is only used when generating DASH manifests from OTF and post-live-DVR streams.
- *
- *
- *
- * It will produce a {@code } element with the following attributes:
- *
- * - {@code startNumber}, which takes the value {@code 0} for post-live-DVR streams and
- * {@code 1} for OTF streams;
- * - {@code timescale}, which is always {@code 1000};
- * - {@code media}, which is the base URL of the stream on which is appended
- * {@code &sq=$Number$};
- * - {@code initialization} (only for OTF streams), which is the base URL of the stream
- * on which is appended {@link #SQ_0}.
- *
- *
- *
- *
- * The {@code } element needs to be generated before this element with
- * {@link #generateRepresentationElement(Document, ItagItem)}).
- *
- *
- * @param document the {@link Document} on which the {@code } element will
- * be appended
- * @param baseUrl the base URL of the OTF/post-live-DVR stream
- * @param deliveryType the stream {@link DeliveryType delivery type}
- */
- private static void generateSegmentTemplateElement(@Nonnull final Document document,
- @Nonnull final String baseUrl,
- final DeliveryType deliveryType)
- throws CreationException {
- try {
- final Element representationElement = (Element) document.getElementsByTagName(
- REPRESENTATION).item(0);
- final Element segmentTemplateElement = document.createElement(SEGMENT_TEMPLATE);
-
- final Attr startNumberAttribute = document.createAttribute("startNumber");
- final boolean isDeliveryTypeLive = deliveryType == DeliveryType.LIVE;
- // The first sequence of post DVR streams is the beginning of the video stream and not
- // an initialization segment
- final String startNumberValue = isDeliveryTypeLive ? "0" : "1";
- startNumberAttribute.setValue(startNumberValue);
- segmentTemplateElement.setAttributeNode(startNumberAttribute);
-
- final Attr timescaleAttribute = document.createAttribute("timescale");
- timescaleAttribute.setValue("1000");
- segmentTemplateElement.setAttributeNode(timescaleAttribute);
-
- // Post-live-DVR/ended livestreams streams don't require an initialization sequence
- if (!isDeliveryTypeLive) {
- final Attr initializationAttribute = document.createAttribute("initialization");
- initializationAttribute.setValue(baseUrl + SQ_0);
- segmentTemplateElement.setAttributeNode(initializationAttribute);
- }
-
- final Attr mediaAttribute = document.createAttribute("media");
- mediaAttribute.setValue(baseUrl + "&sq=$Number$");
- segmentTemplateElement.setAttributeNode(mediaAttribute);
-
- representationElement.appendChild(segmentTemplateElement);
- } catch (final DOMException e) {
- throw CreationException.couldNotAdd(SEGMENT_TEMPLATE, e);
- }
- }
-
- /**
- * Generate the {@code } element, appended as a child of the
- * {@code } element.
- *
- *
- * The {@code } element needs to be generated before this element with
- * {@link #generateSegmentTemplateElement(Document, String, DeliveryType)}.
- *
- *
- * @param document the {@link Document} on which the the {@code } element will
- * be appended
- */
- private static void generateSegmentTimelineElement(@Nonnull final Document document)
- throws CreationException {
- try {
- final Element segmentTemplateElement = (Element) document.getElementsByTagName(
- SEGMENT_TEMPLATE).item(0);
- final Element segmentTimelineElement = document.createElement(SEGMENT_TIMELINE);
-
- segmentTemplateElement.appendChild(segmentTimelineElement);
- } catch (final DOMException e) {
- throw CreationException.couldNotAdd(SEGMENT_TIMELINE, e);
- }
- }
-
- /**
- * Generate segment elements for OTF streams.
- *
- *
- * By parsing by the first media sequence, we know how many durations and repetitions there are
- * so we just have to loop into segment durations to generate the following elements for each:
- *
- *
- *
- * {@code
- *
- *
- * If there is no repetition of the duration between two segments, the {@code r} attribute is
- * not added to the {@code S} element.
- *
- *
- *
- * These elements will be appended as children of the {@code } element.
- *
- *
- *
- * The {@code } element needs to be generated before this element with
- * {@link #generateSegmentTimelineElement(Document)}.
- *
- *
- * @param segmentDurations the sequences "length" or "length(r=repeat_count" extracted with the
- * regexes
- * @param document the {@link Document} on which the the {@code } elements will be appended
- */
- private static void generateSegmentElementsForOtfStreams(final String[] segmentDurations,
- final Document document)
- throws CreationException {
-
- try {
- final Element segmentTimelineElement = (Element) document.getElementsByTagName(
- SEGMENT_TIMELINE).item(0);
-
- for (final String segmentDuration : segmentDurations) {
- final Element sElement = document.createElement("S");
-
- final String[] segmentLengthRepeat = segmentDuration.split("\\(r=");
- // make sure segmentLengthRepeat[0], which is the length, is convertible to int
- Integer.parseInt(segmentLengthRepeat[0]);
-
- // There are repetitions of a segment duration in other segments
- if (segmentLengthRepeat.length > 1) {
- final int segmentRepeatCount = Integer.parseInt(
- Utils.removeNonDigitCharacters(segmentLengthRepeat[1]));
- final Attr rAttribute = document.createAttribute("r");
- rAttribute.setValue(String.valueOf(segmentRepeatCount));
- sElement.setAttributeNode(rAttribute);
- }
-
- final Attr dAttribute = document.createAttribute("d");
- dAttribute.setValue(segmentLengthRepeat[0]);
- sElement.setAttributeNode(dAttribute);
-
- segmentTimelineElement.appendChild(sElement);
- }
-
- } catch (final DOMException | IllegalStateException | IndexOutOfBoundsException
- | NumberFormatException e) {
- throw CreationException.couldNotAdd("segment (S)", e);
- }
- }
-
- /**
- * Generate the segment element for post-live-DVR streams.
- *
- *
- * We don't know the exact duration of segments for post-live-DVR streams but an
- * average instead (which is the {@code targetDurationSec} value), so we can use the following
- * structure to generate the segment timeline for DASH manifests of ended livestreams:
- *
- * {@code
- *
- * @param document the {@link Document} on which the the {@code } element will
- * be appended
- * @param targetDurationSeconds the {@code targetDurationSec} value from player response's
- * stream
- * @param segmentCount the number of segments, extracted by the main method which
- * generates manifests of post DVR livestreams
- */
- private static void generateSegmentElementForPostLiveDvrStreams(
- @Nonnull final Document document,
- final int targetDurationSeconds,
- @Nonnull final String segmentCount) throws CreationException {
- try {
- final Element segmentTimelineElement = (Element) document.getElementsByTagName(
- SEGMENT_TIMELINE).item(0);
- final Element sElement = document.createElement("S");
-
- final Attr dAttribute = document.createAttribute("d");
- dAttribute.setValue(String.valueOf(targetDurationSeconds * 1000));
- sElement.setAttributeNode(dAttribute);
-
- final Attr rAttribute = document.createAttribute("r");
- rAttribute.setValue(segmentCount);
- sElement.setAttributeNode(rAttribute);
-
- segmentTimelineElement.appendChild(sElement);
- } catch (final DOMException e) {
- throw CreationException.couldNotAdd("segment (S)", e);
- }
- }
-
- /**
- * Convert a DASH manifest {@link Document document} to a string and cache it.
- *
- * @param originalBaseStreamingUrl the original base URL of the stream
- * @param document the document to be converted
- * @param manifestCreatorCache the {@link ManifestCreatorCache} on which store the
- * string generated (use either {@link #OTF_CACHE},
- * {@link #POST_LIVE_DVR_CACHE} or {@link #PROGRESSIVE_CACHE})
- * @return the DASH manifest {@link Document document} converted to a string
- */
- private static String buildAndCacheResult(
- @Nonnull final String originalBaseStreamingUrl,
- @Nonnull final Document document,
- @Nonnull final ManifestCreatorCache manifestCreatorCache)
- throws CreationException {
-
- try {
- final String documentXml = documentToXml(document);
- manifestCreatorCache.put(originalBaseStreamingUrl, documentXml);
- return documentXml;
- } catch (final Exception e) {
- throw new CreationException(
- "Could not convert the DASH manifest generated to a string", e);
- }
- }
-
- /**
- * Securing against XEE is done by passing {@code false} to {@link
- * DocumentBuilderFactory#setExpandEntityReferences(boolean)}, also see
- * ChuckerTeam/chucker#201.
- *
- * @return an instance of document secured against XEE attacks, that should then be convertible
- * to an XML string without security problems
- * @see #documentToXml(Document) Use documentToXml to convert the created document to XML, which
- * is also secured against XEE!
- */
- private static Document newDocument() throws ParserConfigurationException {
- final DocumentBuilderFactory documentBuilderFactory
- = DocumentBuilderFactory.newInstance();
- documentBuilderFactory.setExpandEntityReferences(false);
-
- final DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
- return documentBuilder.newDocument();
- }
-
- /**
- * Securing against XEE is done by setting {@link XMLConstants#FEATURE_SECURE_PROCESSING} to
- * {@code true} in the {@link TransformerFactory}, also see
- * ChuckerTeam/chucker#201.
- * The best way to do this would be setting the attributes {@link
- * XMLConstants#ACCESS_EXTERNAL_DTD} and {@link XMLConstants#ACCESS_EXTERNAL_STYLESHEET}, but
- * unfortunately the engine on Android does not support them.
- *
- * @param document the document to convert; must have been created using {@link #newDocument()}
- * to properly prevent XEE attacks!
- * @return the document converted to an XML string, making sure there can't be XEE attacks
- */
- private static String documentToXml(@Nonnull final Document document)
- throws TransformerException {
-
- @SuppressWarnings("java:S2755") // see javadoc: this is actually taken care of
- final TransformerFactory transformerFactory = TransformerFactory.newInstance();
- transformerFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
-
- final Transformer transformer = transformerFactory.newTransformer();
- transformer.setOutputProperty(OutputKeys.VERSION, "1.0");
- transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");
- transformer.setOutputProperty(OutputKeys.STANDALONE, "no");
-
- final StringWriter result = new StringWriter();
- transformer.transform(new DOMSource(document), new StreamResult(result));
-
- return result.toString();
- }
-
- /**
- * @return the cache of DASH manifests generated for OTF streams
- */
- public static ManifestCreatorCache getOtfManifestsCache() {
- return OTF_CACHE;
- }
-
- /**
- * @return the cache of DASH manifests generated for post-live-DVR streams
- */
- public static ManifestCreatorCache getPostLiveDvrManifestsCache() {
- return POST_LIVE_DVR_CACHE;
- }
-
- /**
- * @return the cache of DASH manifests generated for progressive streams
- */
- public static ManifestCreatorCache getProgressiveManifestsCache() {
- return PROGRESSIVE_CACHE;
- }
-}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/CreationException.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/CreationException.java
new file mode 100644
index 000000000..46f32664b
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/CreationException.java
@@ -0,0 +1,63 @@
+package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
+
+import javax.annotation.Nonnull;
+
+/**
+ * Exception that is thrown when a YouTube DASH manifest creator encounters a problem
+ * while creating a manifest.
+ */
+public final class CreationException extends RuntimeException {
+
+ /**
+ * Create a new {@link CreationException} with a detail message.
+ *
+ * @param message the detail message to add in the exception
+ */
+ public CreationException(final String message) {
+ super(message);
+ }
+
+ /**
+ * Create a new {@link CreationException} with a detail message and a cause.
+ * @param message the detail message to add in the exception
+ * @param cause the exception cause of this {@link CreationException}
+ */
+ public CreationException(final String message, final Exception cause) {
+ super(message, cause);
+ }
+
+ // Methods to create exceptions easily without having to use big exception messages and to
+ // reduce duplication
+
+ /**
+ * Create a new {@link CreationException} with a cause and the following detail message format:
+ *
+ * {@code "Could not add " + element + " element", cause}, where {@code element} is an element
+ * of a DASH manifest.
+ *
+ * @param element the element which was not added to the DASH document
+ * @param cause the exception which prevented addition of the element to the DASH document
+ * @return a new {@link CreationException}
+ */
+ @Nonnull
+ public static CreationException couldNotAddElement(final String element,
+ final Exception cause) {
+ return new CreationException("Could not add " + element + " element", cause);
+ }
+
+ /**
+ * Create a new {@link CreationException} with a cause and the following detail message format:
+ *
+ * {@code "Could not add " + element + " element: " + reason}, where {@code element} is an
+ * element of a DASH manifest and {@code reason} the reason why this element cannot be added to
+ * the DASH document.
+ *
+ * @param element the element which was not added to the DASH document
+ * @param reason the reason message of why the element has been not added to the DASH document
+ * @return a new {@link CreationException}
+ */
+ @Nonnull
+ public static CreationException couldNotAddElement(final String element, final String reason) {
+ return new CreationException("Could not add " + element + " element: " + reason);
+ }
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java
new file mode 100644
index 000000000..48b0bf41a
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java
@@ -0,0 +1,856 @@
+package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
+
+import org.schabi.newpipe.extractor.MediaFormat;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.downloader.Downloader;
+import org.schabi.newpipe.extractor.downloader.Response;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
+import org.schabi.newpipe.extractor.services.youtube.ItagItem;
+import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
+import org.w3c.dom.Attr;
+import org.w3c.dom.DOMException;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import javax.annotation.Nonnull;
+import javax.xml.XMLConstants;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isIosStreamingUrl;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5SimplyEmbeddedPlayerStreamingUrl;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebStreamingUrl;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
+/**
+ * Utilities and constants for YouTube DASH manifest creators.
+ *
+ *
+ * This class includes common methods of manifest creators and useful constants.
+ *
+ *
+ *
+ * Generation of DASH documents and their conversion as a string is done using external classes
+ * from {@link org.w3c.dom} and {@link javax.xml} packages.
+ *
+ */
+public final class YoutubeDashManifestCreatorsUtils {
+
+ private YoutubeDashManifestCreatorsUtils() {
+ }
+
+ /**
+ * The redirect count limit that this class uses, which is the same limit as OkHttp.
+ */
+ public static final int MAXIMUM_REDIRECT_COUNT = 20;
+
+ /**
+ * URL parameter of the first sequence for live, post-live-DVR and OTF streams.
+ */
+ public static final String SQ_0 = "&sq=0";
+
+ /**
+ * URL parameter of the first stream request made by official clients.
+ */
+ public static final String RN_0 = "&rn=0";
+
+ /**
+ * URL parameter specific to web clients. When this param is added, if a redirection occurs,
+ * the server will not redirect clients to the redirect URL. Instead, it will provide this URL
+ * as the response body.
+ */
+ public static final String ALR_YES = "&alr=yes";
+
+ /**
+ * Constant which represents the {@code MPD} element of DASH manifests.
+ */
+ public static final String MPD = "MPD";
+
+ /**
+ * Constant which represents the {@code Period} element of DASH manifests.
+ */
+ public static final String PERIOD = "Period";
+
+ /**
+ * Constant which represents the {@code AdaptationSet} element of DASH manifests.
+ */
+ public static final String ADAPTATION_SET = "AdaptationSet";
+
+ /**
+ * Constant which represents the {@code Role} element of DASH manifests.
+ */
+ public static final String ROLE = "Role";
+
+ /**
+ * Constant which represents the {@code Representation} element of DASH manifests.
+ */
+ public static final String REPRESENTATION = "Representation";
+
+ /**
+ * Constant which represents the {@code AudioChannelConfiguration} element of DASH manifests.
+ */
+ public static final String AUDIO_CHANNEL_CONFIGURATION = "AudioChannelConfiguration";
+
+ /**
+ * Constant which represents the {@code SegmentTemplate} element of DASH manifests.
+ */
+ public static final String SEGMENT_TEMPLATE = "SegmentTemplate";
+
+ /**
+ * Constant which represents the {@code SegmentTimeline} element of DASH manifests.
+ */
+ public static final String SEGMENT_TIMELINE = "SegmentTimeline";
+
+ /**
+ * Constant which represents the {@code SegmentBase} element of DASH manifests.
+ */
+ public static final String BASE_URL = "BaseURL";
+
+ /**
+ * Constant which represents the {@code SegmentBase} element of DASH manifests.
+ */
+ public static final String SEGMENT_BASE = "SegmentBase";
+
+ /**
+ * Constant which represents the {@code Initialization} element of DASH manifests.
+ */
+ public static final String INITIALIZATION = "Initialization";
+
+ /**
+ * Generate a {@link Document} with common manifest creator elements added to it.
+ *
+ *
+ * Those are:
+ *
+ * - {@code MPD} (using {@link #generateDocumentAndMpdElement(long)});
+ * - {@code Period} (using {@link #generatePeriodElement(Document)});
+ * - {@code AdaptationSet} (using {@link #generateAdaptationSetElement(Document,
+ * ItagItem)});
+ * - {@code Role} (using {@link #generateRoleElement(Document)});
+ * - {@code Representation} (using {@link #generateRepresentationElement(Document,
+ * ItagItem)});
+ * - and, for audio streams, {@code AudioChannelConfiguration} (using
+ * {@link #generateAudioChannelConfigurationElement(Document, ItagItem)}).
+ *
+ *
+ *
+ * @param itagItem the {@link ItagItem} associated to the stream, which must not be null
+ * @param streamDuration the duration of the stream, in milliseconds
+ * @return a {@link Document} with the common elements added in it
+ */
+ @Nonnull
+ public static Document generateDocumentAndDoCommonElementsGeneration(
+ @Nonnull final ItagItem itagItem,
+ final long streamDuration) throws CreationException {
+ final Document document = generateDocumentAndMpdElement(streamDuration);
+
+ generatePeriodElement(document);
+ generateAdaptationSetElement(document, itagItem);
+ generateRoleElement(document);
+ generateRepresentationElement(document, itagItem);
+ if (itagItem.itagType == ItagItem.ItagType.AUDIO) {
+ generateAudioChannelConfigurationElement(document, itagItem);
+ }
+
+ return document;
+ }
+
+ /**
+ * Create a {@link Document} instance and generate the {@code } element of the manifest.
+ *
+ *
+ * The generated {@code } element looks like the manifest returned into the player
+ * response of videos:
+ *
+ *
+ *
+ * {@code }
+ * (where {@code $duration$} represents the duration in seconds (a number with 3 digits after
+ * the decimal point)).
+ *
+ *
+ * @param duration the duration of the stream, in milliseconds
+ * @return a {@link Document} instance which contains a {@code } element
+ */
+ @Nonnull
+ public static Document generateDocumentAndMpdElement(final long duration)
+ throws CreationException {
+ try {
+ final Document document = newDocument();
+
+ final Element mpdElement = document.createElement(MPD);
+ document.appendChild(mpdElement);
+
+ final Attr xmlnsXsiAttribute = document.createAttribute("xmlns:xsi");
+ xmlnsXsiAttribute.setValue("http://www.w3.org/2001/XMLSchema-instance");
+ mpdElement.setAttributeNode(xmlnsXsiAttribute);
+
+ final Attr xmlns = document.createAttribute("xmlns");
+ xmlns.setValue("urn:mpeg:DASH:schema:MPD:2011");
+ mpdElement.setAttributeNode(xmlns);
+
+ final Attr xsiSchemaLocationAttribute = document.createAttribute("xsi:schemaLocation");
+ xsiSchemaLocationAttribute.setValue("urn:mpeg:DASH:schema:MPD:2011 DASH-MPD.xsd");
+ mpdElement.setAttributeNode(xsiSchemaLocationAttribute);
+
+ final Attr minBufferTimeAttribute = document.createAttribute("minBufferTime");
+ minBufferTimeAttribute.setValue("PT1.500S");
+ mpdElement.setAttributeNode(minBufferTimeAttribute);
+
+ final Attr profilesAttribute = document.createAttribute("profiles");
+ profilesAttribute.setValue("urn:mpeg:dash:profile:full:2011");
+ mpdElement.setAttributeNode(profilesAttribute);
+
+ final Attr typeAttribute = document.createAttribute("type");
+ typeAttribute.setValue("static");
+ mpdElement.setAttributeNode(typeAttribute);
+
+ final Attr mediaPresentationDurationAttribute = document.createAttribute(
+ "mediaPresentationDuration");
+ final String durationSeconds = String.format(Locale.ENGLISH, "%.3f",
+ duration / 1000.0);
+ mediaPresentationDurationAttribute.setValue("PT" + durationSeconds + "S");
+ mpdElement.setAttributeNode(mediaPresentationDurationAttribute);
+
+ return document;
+ } catch (final Exception e) {
+ throw new CreationException(
+ "Could not generate the DASH manifest or append the MPD document to it", e);
+ }
+ }
+
+ /**
+ * Generate the {@code } element, appended as a child of the {@code } element.
+ *
+ *
+ * The {@code } element needs to be generated before this element with
+ * {@link #generateDocumentAndMpdElement(long)}.
+ *
+ *
+ * @param document the {@link Document} on which the the {@code } element will be
+ * appended
+ */
+ public static void generatePeriodElement(@Nonnull final Document document)
+ throws CreationException {
+ try {
+ final Element mpdElement = (Element) document.getElementsByTagName(MPD).item(0);
+ final Element periodElement = document.createElement(PERIOD);
+ mpdElement.appendChild(periodElement);
+ } catch (final DOMException e) {
+ throw CreationException.couldNotAddElement(PERIOD, e);
+ }
+ }
+
+ /**
+ * Generate the {@code } element, appended as a child of the {@code }
+ * element.
+ *
+ *
+ * The {@code } element needs to be generated before this element with
+ * {@link #generatePeriodElement(Document)}.
+ *
+ *
+ * @param document the {@link Document} on which the {@code } element will be
+ * appended
+ * @param itagItem the {@link ItagItem} corresponding to the stream, which must not be null
+ */
+ public static void generateAdaptationSetElement(@Nonnull final Document document,
+ @Nonnull final ItagItem itagItem)
+ throws CreationException {
+ try {
+ final Element periodElement = (Element) document.getElementsByTagName(PERIOD)
+ .item(0);
+ final Element adaptationSetElement = document.createElement(ADAPTATION_SET);
+
+ final Attr idAttribute = document.createAttribute("id");
+ idAttribute.setValue("0");
+ adaptationSetElement.setAttributeNode(idAttribute);
+
+ final MediaFormat mediaFormat = itagItem.getMediaFormat();
+ if (mediaFormat == null || isNullOrEmpty(mediaFormat.getMimeType())) {
+ throw CreationException.couldNotAddElement(ADAPTATION_SET,
+ "the MediaFormat or its mime type is null or empty");
+ }
+
+ final Attr mimeTypeAttribute = document.createAttribute("mimeType");
+ mimeTypeAttribute.setValue(mediaFormat.getMimeType());
+ adaptationSetElement.setAttributeNode(mimeTypeAttribute);
+
+ final Attr subsegmentAlignmentAttribute = document.createAttribute(
+ "subsegmentAlignment");
+ subsegmentAlignmentAttribute.setValue("true");
+ adaptationSetElement.setAttributeNode(subsegmentAlignmentAttribute);
+
+ periodElement.appendChild(adaptationSetElement);
+ } catch (final DOMException e) {
+ throw CreationException.couldNotAddElement(ADAPTATION_SET, e);
+ }
+ }
+
+ /**
+ * Generate the {@code } element, appended as a child of the {@code }
+ * element.
+ *
+ *
+ * This element, with its attributes and values, is:
+ *
+ *
+ *
+ * {@code }
+ *
+ *
+ *
+ * The {@code } element needs to be generated before this element with
+ * {@link #generateAdaptationSetElement(Document, ItagItem)}).
+ *
+ *
+ * @param document the {@link Document} on which the the {@code } element will be
+ * appended
+ */
+ public static void generateRoleElement(@Nonnull final Document document)
+ throws CreationException {
+ try {
+ final Element adaptationSetElement = (Element) document.getElementsByTagName(
+ ADAPTATION_SET).item(0);
+ final Element roleElement = document.createElement(ROLE);
+
+ final Attr schemeIdUriAttribute = document.createAttribute("schemeIdUri");
+ schemeIdUriAttribute.setValue("urn:mpeg:DASH:role:2011");
+ roleElement.setAttributeNode(schemeIdUriAttribute);
+
+ final Attr valueAttribute = document.createAttribute("value");
+ valueAttribute.setValue("main");
+ roleElement.setAttributeNode(valueAttribute);
+
+ adaptationSetElement.appendChild(roleElement);
+ } catch (final DOMException e) {
+ throw CreationException.couldNotAddElement(ROLE, e);
+ }
+ }
+
+ /**
+ * Generate the {@code } element, appended as a child of the
+ * {@code } element.
+ *
+ *
+ * The {@code } element needs to be generated before this element with
+ * {@link #generateAdaptationSetElement(Document, ItagItem)}).
+ *
+ *
+ * @param document the {@link Document} on which the the {@code } element will
+ * be appended
+ * @param itagItem the {@link ItagItem} to use, which must not be null
+ */
+ public static void generateRepresentationElement(@Nonnull final Document document,
+ @Nonnull final ItagItem itagItem)
+ throws CreationException {
+ try {
+ final Element adaptationSetElement = (Element) document.getElementsByTagName(
+ ADAPTATION_SET).item(0);
+ final Element representationElement = document.createElement(REPRESENTATION);
+
+ final int id = itagItem.id;
+ if (id <= 0) {
+ throw CreationException.couldNotAddElement(REPRESENTATION,
+ "the id of the ItagItem is <= 0");
+ }
+ final Attr idAttribute = document.createAttribute("id");
+ idAttribute.setValue(String.valueOf(id));
+ representationElement.setAttributeNode(idAttribute);
+
+ final String codec = itagItem.getCodec();
+ if (isNullOrEmpty(codec)) {
+ throw CreationException.couldNotAddElement(ADAPTATION_SET,
+ "the codec value of the ItagItem is null or empty");
+ }
+ final Attr codecsAttribute = document.createAttribute("codecs");
+ codecsAttribute.setValue(codec);
+ representationElement.setAttributeNode(codecsAttribute);
+
+ final Attr startWithSAPAttribute = document.createAttribute("startWithSAP");
+ startWithSAPAttribute.setValue("1");
+ representationElement.setAttributeNode(startWithSAPAttribute);
+
+ final Attr maxPlayoutRateAttribute = document.createAttribute("maxPlayoutRate");
+ maxPlayoutRateAttribute.setValue("1");
+ representationElement.setAttributeNode(maxPlayoutRateAttribute);
+
+ final int bitrate = itagItem.getBitrate();
+ if (bitrate <= 0) {
+ throw CreationException.couldNotAddElement(REPRESENTATION,
+ "the bitrate of the ItagItem is <= 0");
+ }
+ final Attr bandwidthAttribute = document.createAttribute("bandwidth");
+ bandwidthAttribute.setValue(String.valueOf(bitrate));
+ representationElement.setAttributeNode(bandwidthAttribute);
+
+ final ItagItem.ItagType itagType = itagItem.itagType;
+
+ if (itagType == ItagItem.ItagType.VIDEO || itagType == ItagItem.ItagType.VIDEO_ONLY) {
+ final int height = itagItem.getHeight();
+ final int width = itagItem.getWidth();
+ if (height <= 0 && width <= 0) {
+ throw CreationException.couldNotAddElement(REPRESENTATION,
+ "both width and height of the ItagItem are <= 0");
+ }
+
+ if (width > 0) {
+ final Attr widthAttribute = document.createAttribute("width");
+ widthAttribute.setValue(String.valueOf(width));
+ representationElement.setAttributeNode(widthAttribute);
+ }
+
+ final Attr heightAttribute = document.createAttribute("height");
+ heightAttribute.setValue(String.valueOf(itagItem.getHeight()));
+ representationElement.setAttributeNode(heightAttribute);
+
+ final int fps = itagItem.getFps();
+ if (fps > 0) {
+ final Attr frameRateAttribute = document.createAttribute("frameRate");
+ frameRateAttribute.setValue(String.valueOf(fps));
+ representationElement.setAttributeNode(frameRateAttribute);
+ }
+ }
+
+ if (itagType == ItagItem.ItagType.AUDIO && itagItem.getSampleRate() > 0) {
+ final Attr audioSamplingRateAttribute = document.createAttribute(
+ "audioSamplingRate");
+ audioSamplingRateAttribute.setValue(String.valueOf(itagItem.getSampleRate()));
+ }
+
+ adaptationSetElement.appendChild(representationElement);
+ } catch (final DOMException e) {
+ throw CreationException.couldNotAddElement(REPRESENTATION, e);
+ }
+ }
+
+ /**
+ * Generate the {@code } element, appended as a child of the
+ * {@code } element.
+ *
+ *
+ * This method is only used when generating DASH manifests of audio streams.
+ *
+ *
+ *
+ * It will produce the following element:
+ *
+ * {@code
+ * (where {@code audioChannelsValue} is get from the {@link ItagItem} passed as the second
+ * parameter of this method)
+ *
+ *
+ *
+ * The {@code } element needs to be generated before this element with
+ * {@link #generateRepresentationElement(Document, ItagItem)}).
+ *
+ *
+ * @param document the {@link Document} on which the {@code }
+ * element will be appended
+ * @param itagItem the {@link ItagItem} to use, which must not be null
+ */
+ public static void generateAudioChannelConfigurationElement(
+ @Nonnull final Document document,
+ @Nonnull final ItagItem itagItem) throws CreationException {
+ try {
+ final Element representationElement = (Element) document.getElementsByTagName(
+ REPRESENTATION).item(0);
+ final Element audioChannelConfigurationElement = document.createElement(
+ AUDIO_CHANNEL_CONFIGURATION);
+
+ final Attr schemeIdUriAttribute = document.createAttribute("schemeIdUri");
+ schemeIdUriAttribute.setValue(
+ "urn:mpeg:dash:23003:3:audio_channel_configuration:2011");
+ audioChannelConfigurationElement.setAttributeNode(schemeIdUriAttribute);
+
+ final Attr valueAttribute = document.createAttribute("value");
+ final int audioChannels = itagItem.getAudioChannels();
+ if (audioChannels <= 0) {
+ throw new CreationException("the number of audioChannels in the ItagItem is <= 0: "
+ + audioChannels);
+ }
+ valueAttribute.setValue(String.valueOf(itagItem.getAudioChannels()));
+ audioChannelConfigurationElement.setAttributeNode(valueAttribute);
+
+ representationElement.appendChild(audioChannelConfigurationElement);
+ } catch (final DOMException e) {
+ throw CreationException.couldNotAddElement(AUDIO_CHANNEL_CONFIGURATION, e);
+ }
+ }
+
+ /**
+ * Convert a DASH manifest {@link Document document} to a string and cache it.
+ *
+ * @param originalBaseStreamingUrl the original base URL of the stream
+ * @param document the document to be converted
+ * @param manifestCreatorCache the {@link ManifestCreatorCache} on which store the string
+ * generated
+ * @return the DASH manifest {@link Document document} converted to a string
+ */
+ public static String buildAndCacheResult(
+ @Nonnull final String originalBaseStreamingUrl,
+ @Nonnull final Document document,
+ @Nonnull final ManifestCreatorCache manifestCreatorCache)
+ throws CreationException {
+
+ try {
+ final String documentXml = documentToXml(document);
+ manifestCreatorCache.put(originalBaseStreamingUrl, documentXml);
+ return documentXml;
+ } catch (final Exception e) {
+ throw new CreationException(
+ "Could not convert the DASH manifest generated to a string", e);
+ }
+ }
+
+ /**
+ * Generate the {@code } element, appended as a child of the
+ * {@code } element.
+ *
+ *
+ * This method is only used when generating DASH manifests from OTF and post-live-DVR streams.
+ *
+ *
+ *
+ * It will produce a {@code } element with the following attributes:
+ *
+ * - {@code startNumber}, which takes the value {@code 0} for post-live-DVR streams and
+ * {@code 1} for OTF streams;
+ * - {@code timescale}, which is always {@code 1000};
+ * - {@code media}, which is the base URL of the stream on which is appended
+ * {@code &sq=$Number$};
+ * - {@code initialization} (only for OTF streams), which is the base URL of the stream
+ * on which is appended {@link #SQ_0}.
+ *
+ *
+ *
+ *
+ * The {@code } element needs to be generated before this element with
+ * {@link #generateRepresentationElement(Document, ItagItem)}).
+ *
+ *
+ * @param document the {@link Document} on which the {@code } element will
+ * be appended
+ * @param baseUrl the base URL of the OTF/post-live-DVR stream
+ * @param deliveryType the stream {@link DeliveryType delivery type}, which must be either
+ * {@link DeliveryType#OTF OTF} or {@link DeliveryType#LIVE LIVE}
+ */
+ public static void generateSegmentTemplateElement(@Nonnull final Document document,
+ @Nonnull final String baseUrl,
+ final DeliveryType deliveryType)
+ throws CreationException {
+ if (deliveryType != DeliveryType.OTF && deliveryType != DeliveryType.LIVE) {
+ throw CreationException.couldNotAddElement(SEGMENT_TEMPLATE, "invalid delivery type: "
+ + deliveryType);
+ }
+
+ try {
+ final Element representationElement = (Element) document.getElementsByTagName(
+ REPRESENTATION).item(0);
+ final Element segmentTemplateElement = document.createElement(SEGMENT_TEMPLATE);
+
+ final Attr startNumberAttribute = document.createAttribute("startNumber");
+ final boolean isDeliveryTypeLive = deliveryType == DeliveryType.LIVE;
+ // The first sequence of post DVR streams is the beginning of the video stream and not
+ // an initialization segment
+ final String startNumberValue = isDeliveryTypeLive ? "0" : "1";
+ startNumberAttribute.setValue(startNumberValue);
+ segmentTemplateElement.setAttributeNode(startNumberAttribute);
+
+ final Attr timescaleAttribute = document.createAttribute("timescale");
+ timescaleAttribute.setValue("1000");
+ segmentTemplateElement.setAttributeNode(timescaleAttribute);
+
+ // Post-live-DVR/ended livestreams streams don't require an initialization sequence
+ if (!isDeliveryTypeLive) {
+ final Attr initializationAttribute = document.createAttribute("initialization");
+ initializationAttribute.setValue(baseUrl + SQ_0);
+ segmentTemplateElement.setAttributeNode(initializationAttribute);
+ }
+
+ final Attr mediaAttribute = document.createAttribute("media");
+ mediaAttribute.setValue(baseUrl + "&sq=$Number$");
+ segmentTemplateElement.setAttributeNode(mediaAttribute);
+
+ representationElement.appendChild(segmentTemplateElement);
+ } catch (final DOMException e) {
+ throw CreationException.couldNotAddElement(SEGMENT_TEMPLATE, e);
+ }
+ }
+
+ /**
+ * Generate the {@code } element, appended as a child of the
+ * {@code } element.
+ *
+ *
+ * The {@code } element needs to be generated before this element with
+ * {@link #generateSegmentTemplateElement(Document, String, DeliveryType)}.
+ *
+ *
+ * @param document the {@link Document} on which the the {@code } element will
+ * be appended
+ */
+ public static void generateSegmentTimelineElement(@Nonnull final Document document)
+ throws CreationException {
+ try {
+ final Element segmentTemplateElement = (Element) document.getElementsByTagName(
+ SEGMENT_TEMPLATE).item(0);
+ final Element segmentTimelineElement = document.createElement(SEGMENT_TIMELINE);
+
+ segmentTemplateElement.appendChild(segmentTimelineElement);
+ } catch (final DOMException e) {
+ throw CreationException.couldNotAddElement(SEGMENT_TIMELINE, e);
+ }
+ }
+
+ /**
+ * Get the "initialization" {@link Response response} of a stream.
+ *
+ * This method fetches, for OTF streams and for post-live-DVR streams:
+ *
+ * - the base URL of the stream, to which are appended {@link #SQ_0} and
+ * {@link #RN_0} parameters, with a {@code GET} request for streaming URLs from HTML5
+ * clients and a {@code POST} request for the ones from the {@code ANDROID} and the
+ * {@code IOS} clients;
+ * - for streaming URLs from HTML5 clients, the {@link #ALR_YES} param is also added.
+ *
+ *
+ *
+ *
+ * @param baseStreamingUrl the base URL of the stream, which must not be null
+ * @param itagItem the {@link ItagItem} of stream, which must not be null
+ * @param deliveryType the {@link DeliveryType} of the stream
+ * @return the "initialization" response, without redirections on the network on which the
+ * request(s) is/are made
+ */
+ @SuppressWarnings("checkstyle:FinalParameters")
+ @Nonnull
+ public static Response getInitializationResponse(@Nonnull String baseStreamingUrl,
+ @Nonnull final ItagItem itagItem,
+ final DeliveryType deliveryType)
+ throws CreationException {
+ final boolean isHtml5StreamingUrl = isWebStreamingUrl(baseStreamingUrl)
+ || isTvHtml5SimplyEmbeddedPlayerStreamingUrl(baseStreamingUrl);
+ final boolean isAndroidStreamingUrl = isAndroidStreamingUrl(baseStreamingUrl);
+ final boolean isIosStreamingUrl = isIosStreamingUrl(baseStreamingUrl);
+ if (isHtml5StreamingUrl) {
+ baseStreamingUrl += ALR_YES;
+ }
+ baseStreamingUrl = appendRnParamAndSqParamIfNeeded(baseStreamingUrl, deliveryType);
+
+ final Downloader downloader = NewPipe.getDownloader();
+ if (isHtml5StreamingUrl) {
+ final String mimeTypeExpected = itagItem.getMediaFormat().getMimeType();
+ if (!isNullOrEmpty(mimeTypeExpected)) {
+ return getStreamingWebUrlWithoutRedirects(downloader, baseStreamingUrl,
+ mimeTypeExpected);
+ }
+ } else if (isAndroidStreamingUrl || isIosStreamingUrl) {
+ try {
+ final Map> headers = new HashMap<>();
+ headers.put("User-Agent", Collections.singletonList(
+ isAndroidStreamingUrl ? getAndroidUserAgent(null)
+ : getIosUserAgent(null)));
+ final byte[] emptyBody = "".getBytes(StandardCharsets.UTF_8);
+ return downloader.post(baseStreamingUrl, headers, emptyBody);
+ } catch (final IOException | ExtractionException e) {
+ throw new CreationException("Could not get the "
+ + (isIosStreamingUrl ? "ANDROID" : "IOS") + " streaming URL response", e);
+ }
+ }
+
+ try {
+ return downloader.get(baseStreamingUrl);
+ } catch (final IOException | ExtractionException e) {
+ throw new CreationException("Could not get the streaming URL response", e);
+ }
+ }
+
+ /**
+ * Generate a new {@link DocumentBuilder} secured from XEE attacks, on platforms which
+ * support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and
+ * {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link DocumentBuilderFactory} instances.
+ *
+ * @return an instance of {@link Document} secured against XEE attacks on supported platforms,
+ * that should then be convertible to an XML string without security problems
+ */
+ private static Document newDocument() throws ParserConfigurationException {
+ final DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
+ try {
+ documentBuilderFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+ documentBuilderFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
+ } catch (final Exception ignored) {
+ // Ignore exceptions as setting these attributes to secure XML generation is supported
+ // by all platforms (like the Android implementation)
+ }
+
+ final DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
+ return documentBuilder.newDocument();
+ }
+
+ /**
+ * Generate a new {@link TransformerFactory} secured from XEE attacks, on platforms which
+ * support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and
+ * {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link TransformerFactory} instances.
+ *
+ * @param document the document to convert, which must have been created using
+ * {@link #newDocument()} to properly prevent XEE attacks
+ * @return the document converted to an XML string, making sure there can't be XEE attacks
+ */
+ // Sonar warning is suppressed because it is still shown even if we apply its solution
+ @SuppressWarnings("squid:S2755")
+ private static String documentToXml(@Nonnull final Document document)
+ throws TransformerException {
+
+ final TransformerFactory transformerFactory = TransformerFactory.newInstance();
+ try {
+ transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+ transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
+ } catch (final Exception ignored) {
+ // Ignore exceptions as setting these attributes to secure XML generation is supported
+ // by all platforms (like the Android implementation)
+ }
+
+ final Transformer transformer = transformerFactory.newTransformer();
+ transformer.setOutputProperty(OutputKeys.VERSION, "1.0");
+ transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");
+ transformer.setOutputProperty(OutputKeys.STANDALONE, "no");
+
+ final StringWriter result = new StringWriter();
+ transformer.transform(new DOMSource(document), new StreamResult(result));
+
+ return result.toString();
+ }
+
+ /**
+ * Append {@link #SQ_0} for post-live-DVR and OTF streams and {@link #RN_0} to all streams.
+ *
+ * @param baseStreamingUrl the base streaming URL to which the parameter(s) are being appended
+ * @param deliveryType the {@link DeliveryType} of the stream
+ * @return the base streaming URL to which the param(s) are appended, depending on the
+ * {@link DeliveryType} of the stream
+ */
+ @SuppressWarnings({"checkstyle:FinalParameters", "checkstyle:FinalLocalVariable"})
+ @Nonnull
+ private static String appendRnParamAndSqParamIfNeeded(
+ @Nonnull String baseStreamingUrl,
+ @Nonnull final DeliveryType deliveryType) {
+ if (deliveryType != DeliveryType.PROGRESSIVE) {
+ baseStreamingUrl += SQ_0;
+ }
+
+ return baseStreamingUrl + RN_0;
+ }
+
+ /**
+ * Get a URL on which no redirection between playback hosts should be present on the network
+ * and/or IP used to fetch the streaming URL, for HTML5 clients.
+ *
+ * This method will follow redirects which works in the following way:
+ *
+ * - the {@link #ALR_YES} param is appended to all streaming URLs
+ * - if no redirection occurs, the video server will return the streaming data;
+ * - if a redirection occurs, the server will respond with HTTP status code 200 and a
+ * {@code text/plain} mime type. The redirection URL is the response body;
+ * - the redirection URL is requested and the steps above from step 2 are repeated,
+ * until too many redirects are reached of course (the maximum number of redirects is
+ * {@link #MAXIMUM_REDIRECT_COUNT the same as OkHttp}).
+ *
+ *
+ *
+ *
+ * For non-HTML5 clients, redirections are managed in the standard way in
+ * {@link #getInitializationResponse(String, ItagItem, DeliveryType)}.
+ *
+ *
+ * @param downloader the {@link Downloader} instance to be used
+ * @param streamingUrl the streaming URL which we are trying to get a streaming URL
+ * without any redirection on the network and/or IP used
+ * @param responseMimeTypeExpected the response mime type expected from Google video servers
+ * @return the {@link Response} of the stream, which should have no redirections
+ */
+ @SuppressWarnings("checkstyle:FinalParameters")
+ @Nonnull
+ private static Response getStreamingWebUrlWithoutRedirects(
+ @Nonnull final Downloader downloader,
+ @Nonnull String streamingUrl,
+ @Nonnull final String responseMimeTypeExpected)
+ throws CreationException {
+ try {
+ final Map> headers = new HashMap<>();
+ addClientInfoHeaders(headers);
+
+ String responseMimeType = "";
+
+ int redirectsCount = 0;
+ while (!responseMimeType.equals(responseMimeTypeExpected)
+ && redirectsCount < MAXIMUM_REDIRECT_COUNT) {
+ final Response response = downloader.get(streamingUrl, headers);
+
+ final int responseCode = response.responseCode();
+ if (responseCode != 200) {
+ throw new CreationException(
+ "Could not get the initialization URL: HTTP response code "
+ + responseCode);
+ }
+
+ // A valid HTTP 1.0+ response should include a Content-Type header, so we can
+ // require that the response from video servers has this header.
+ responseMimeType = Objects.requireNonNull(response.getHeader("Content-Type"),
+ "Could not get the Content-Type header from the response headers");
+
+ // The response body is the redirection URL
+ if (responseMimeType.equals("text/plain")) {
+ streamingUrl = response.responseBody();
+ redirectsCount++;
+ } else {
+ return response;
+ }
+ }
+
+ if (redirectsCount >= MAXIMUM_REDIRECT_COUNT) {
+ throw new CreationException(
+ "Too many redirects when trying to get the the streaming URL response of a "
+ + "HTML5 client");
+ }
+
+ // This should never be reached, but is required because we don't want to return null
+ // here
+ throw new CreationException(
+ "Could not get the streaming URL response of a HTML5 client: unreachable code "
+ + "reached!");
+ } catch (final IOException | ExtractionException e) {
+ throw new CreationException(
+ "Could not get the streaming URL response of a HTML5 client", e);
+ }
+ }
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeOtfDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeOtfDashManifestCreator.java
new file mode 100644
index 000000000..375fd7421
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeOtfDashManifestCreator.java
@@ -0,0 +1,268 @@
+package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
+
+import org.schabi.newpipe.extractor.downloader.Response;
+import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
+import org.schabi.newpipe.extractor.services.youtube.ItagItem;
+import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
+import org.schabi.newpipe.extractor.utils.Utils;
+import org.w3c.dom.Attr;
+import org.w3c.dom.DOMException;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import javax.annotation.Nonnull;
+import java.util.Arrays;
+import java.util.Objects;
+
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ALR_YES;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.RN_0;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TIMELINE;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SQ_0;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.buildAndCacheResult;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateDocumentAndDoCommonElementsGeneration;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTemplateElement;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTimelineElement;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.getInitializationResponse;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
+import static org.schabi.newpipe.extractor.utils.Utils.isBlank;
+
+/**
+ * Class which generates DASH manifests of YouTube {@link DeliveryType#OTF OTF streams}.
+ */
+public final class YoutubeOtfDashManifestCreator {
+
+ /**
+ * Cache of DASH manifests generated for OTF streams.
+ */
+ private static final ManifestCreatorCache OTF_STREAMS_CACHE
+ = new ManifestCreatorCache<>();
+
+ private YoutubeOtfDashManifestCreator() {
+ }
+
+ /**
+ * Create DASH manifests from a YouTube OTF stream.
+ *
+ *
+ * OTF streams are YouTube-DASH specific streams which work with sequences and without the need
+ * to get a manifest (even if one is provided, it is not used by official clients).
+ *
+ *
+ *
+ * They can be found only on videos; mostly those with a small amount of views, or ended
+ * livestreams which have just been re-encoded as normal videos.
+ *
+ *
+ * This method needs:
+ *
+ * - the base URL of the stream (which, if you try to access to it, returns HTTP
+ * status code 404 after redirects, and if the URL is valid);
+ * - an {@link ItagItem}, which needs to contain the following information:
+ *
+ * - its type (see {@link ItagItem.ItagType}), to identify if the content is
+ * an audio or a video stream;
+ * - its bitrate;
+ * - its mime type;
+ * - its codec(s);
+ * - for an audio stream: its audio channels;
+ * - for a video stream: its width and height.
+ *
+ *
+ * - the duration of the video, which will be used if the duration could not be
+ * parsed from the first sequence of the stream.
+ *
+ *
+ *
+ * In order to generate the DASH manifest, this method will:
+ *
+ * - request the first sequence of the stream (the base URL on which the first
+ * sequence parameter is appended (see {@link YoutubeDashManifestCreatorsUtils#SQ_0}))
+ * with a {@code POST} or {@code GET} request (depending of the client on which the
+ * streaming URL comes from is a mobile one ({@code POST}) or not ({@code GET}));
+ * - follow its redirection(s), if any;
+ * - save the last URL, remove the first sequence parameter;
+ * - use the information provided in the {@link ItagItem} to generate all
+ * elements of the DASH manifest.
+ *
+ *
+ *
+ *
+ * If the duration cannot be extracted, the {@code durationSecondsFallback} value will be used
+ * as the stream duration.
+ *
+ *
+ * @param otfBaseStreamingUrl the base URL of the OTF stream, which must not be null
+ * @param itagItem the {@link ItagItem} corresponding to the stream, which
+ * must not be null
+ * @param durationSecondsFallback the duration of the video, which will be used if the duration
+ * could not be extracted from the first sequence
+ * @return the manifest generated into a string
+ */
+ @Nonnull
+ public static String fromOtfStreamingUrl(
+ @Nonnull final String otfBaseStreamingUrl,
+ @Nonnull final ItagItem itagItem,
+ final long durationSecondsFallback) throws CreationException {
+ if (OTF_STREAMS_CACHE.containsKey(otfBaseStreamingUrl)) {
+ return Objects.requireNonNull(OTF_STREAMS_CACHE.get(otfBaseStreamingUrl)).getSecond();
+ }
+
+ String realOtfBaseStreamingUrl = otfBaseStreamingUrl;
+ // Try to avoid redirects when streaming the content by saving the last URL we get
+ // from video servers.
+ final Response response = getInitializationResponse(realOtfBaseStreamingUrl,
+ itagItem, DeliveryType.OTF);
+ realOtfBaseStreamingUrl = response.latestUrl().replace(SQ_0, EMPTY_STRING)
+ .replace(RN_0, EMPTY_STRING).replace(ALR_YES, EMPTY_STRING);
+
+ final int responseCode = response.responseCode();
+ if (responseCode != 200) {
+ throw new CreationException("Could not get the initialization URL: response code "
+ + responseCode);
+ }
+
+ final String[] segmentDuration;
+
+ try {
+ final String[] segmentsAndDurationsResponseSplit = response.responseBody()
+ // Get the lines with the durations and the following
+ .split("Segment-Durations-Ms: ")[1]
+ // Remove the other lines
+ .split("\n")[0]
+ // Get all durations and repetitions which are separated by a comma
+ .split(",");
+ final int lastIndex = segmentsAndDurationsResponseSplit.length - 1;
+ if (isBlank(segmentsAndDurationsResponseSplit[lastIndex])) {
+ segmentDuration = Arrays.copyOf(segmentsAndDurationsResponseSplit, lastIndex);
+ } else {
+ segmentDuration = segmentsAndDurationsResponseSplit;
+ }
+ } catch (final Exception e) {
+ throw new CreationException("Could not get segment durations", e);
+ }
+
+ long streamDuration;
+ try {
+ streamDuration = getStreamDuration(segmentDuration);
+ } catch (final CreationException e) {
+ streamDuration = durationSecondsFallback * 1000;
+ }
+
+ final Document document = generateDocumentAndDoCommonElementsGeneration(itagItem,
+ streamDuration);
+
+ generateSegmentTemplateElement(document, realOtfBaseStreamingUrl, DeliveryType.OTF);
+ generateSegmentTimelineElement(document);
+ generateSegmentElementsForOtfStreams(segmentDuration, document);
+
+ return buildAndCacheResult(otfBaseStreamingUrl, document, OTF_STREAMS_CACHE);
+ }
+
+ /**
+ * @return the cache of DASH manifests generated for OTF streams
+ */
+ public static ManifestCreatorCache getCache() {
+ return OTF_STREAMS_CACHE;
+ }
+
+ /**
+ * Generate segment elements for OTF streams.
+ *
+ *
+ * By parsing by the first media sequence, we know how many durations and repetitions there are
+ * so we just have to loop into segment durations to generate the following elements for each
+ * duration repeated X times:
+ *
+ *
+ *
+ * {@code
+ *
+ *
+ * If there is no repetition of the duration between two segments, the {@code r} attribute is
+ * not added to the {@code S} element, as it is not needed.
+ *
+ *
+ *
+ * These elements will be appended as children of the {@code } element, which
+ * needs to be generated before these elements with
+ * {@link YoutubeDashManifestCreatorsUtils#generateSegmentTimelineElement(Document)}.
+ *
+ *
+ * @param segmentDurations the sequences "length" or "length(r=repeat_count" extracted with the
+ * regular expressions
+ * @param document the {@link Document} on which the {@code } elements will be appended
+ */
+ private static void generateSegmentElementsForOtfStreams(
+ @Nonnull final String[] segmentDurations,
+ @Nonnull final Document document) throws CreationException {
+ try {
+ final Element segmentTimelineElement = (Element) document.getElementsByTagName(
+ SEGMENT_TIMELINE).item(0);
+
+ for (final String segmentDuration : segmentDurations) {
+ final Element sElement = document.createElement("S");
+
+ final String[] segmentLengthRepeat = segmentDuration.split("\\(r=");
+ // make sure segmentLengthRepeat[0], which is the length, is convertible to int
+ Integer.parseInt(segmentLengthRepeat[0]);
+
+ // There are repetitions of a segment duration in other segments
+ if (segmentLengthRepeat.length > 1) {
+ final int segmentRepeatCount = Integer.parseInt(
+ Utils.removeNonDigitCharacters(segmentLengthRepeat[1]));
+ final Attr rAttribute = document.createAttribute("r");
+ rAttribute.setValue(String.valueOf(segmentRepeatCount));
+ sElement.setAttributeNode(rAttribute);
+ }
+
+ final Attr dAttribute = document.createAttribute("d");
+ dAttribute.setValue(segmentLengthRepeat[0]);
+ sElement.setAttributeNode(dAttribute);
+
+ segmentTimelineElement.appendChild(sElement);
+ }
+
+ } catch (final DOMException | IllegalStateException | IndexOutOfBoundsException
+ | NumberFormatException e) {
+ throw CreationException.couldNotAddElement("segment (S)", e);
+ }
+ }
+
+ /**
+ * Get the duration of an OTF stream.
+ *
+ *
+ * The duration of OTF streams is not returned into the player response and needs to be
+ * calculated by adding the duration of each segment.
+ *
+ *
+ * @param segmentDuration the segment duration object extracted from the initialization
+ * sequence of the stream
+ * @return the duration of the OTF stream, in milliseconds
+ */
+ private static long getStreamDuration(@Nonnull final String[] segmentDuration)
+ throws CreationException {
+ try {
+ long streamLengthMs = 0;
+
+ for (final String segDuration : segmentDuration) {
+ final String[] segmentLengthRepeat = segDuration.split("\\(r=");
+ long segmentRepeatCount = 0;
+
+ // There are repetitions of a segment duration in other segments
+ if (segmentLengthRepeat.length > 1) {
+ segmentRepeatCount = Long.parseLong(Utils.removeNonDigitCharacters(
+ segmentLengthRepeat[1]));
+ }
+
+ final long segmentLength = Integer.parseInt(segmentLengthRepeat[0]);
+ streamLengthMs += segmentLength + segmentRepeatCount * segmentLength;
+ }
+
+ return streamLengthMs;
+ } catch (final NumberFormatException e) {
+ throw new CreationException("Could not get stream length from sequences list", e);
+ }
+ }
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubePostLiveStreamDvrDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubePostLiveStreamDvrDashManifestCreator.java
new file mode 100644
index 000000000..21a21d2c6
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubePostLiveStreamDvrDashManifestCreator.java
@@ -0,0 +1,221 @@
+package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
+
+import org.schabi.newpipe.extractor.downloader.Response;
+import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
+import org.schabi.newpipe.extractor.services.youtube.ItagItem;
+import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
+import org.w3c.dom.Attr;
+import org.w3c.dom.DOMException;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import javax.annotation.Nonnull;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ALR_YES;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.RN_0;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TIMELINE;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SQ_0;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.buildAndCacheResult;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateDocumentAndDoCommonElementsGeneration;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTemplateElement;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTimelineElement;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.getInitializationResponse;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
+/**
+ * Class which generates DASH manifests of YouTube post-live DVR streams (which use the
+ * {@link DeliveryType#LIVE LIVE delivery type}).
+ */
+public final class YoutubePostLiveStreamDvrDashManifestCreator {
+
+ /**
+ * Cache of DASH manifests generated for post-live-DVR streams.
+ */
+ private static final ManifestCreatorCache POST_LIVE_DVR_STREAMS_CACHE
+ = new ManifestCreatorCache<>();
+
+ private YoutubePostLiveStreamDvrDashManifestCreator() {
+ }
+
+ /**
+ * Create DASH manifests from a YouTube post-live-DVR stream/ended livestream.
+ *
+ *
+ * Post-live-DVR streams/ended livestreams are one of the YouTube DASH specific streams which
+ * works with sequences and without the need to get a manifest (even if one is provided but not
+ * used by main clients (and is not complete for big ended livestreams because it doesn't
+ * return the full stream)).
+ *
+ *
+ *
+ * They can be found only on livestreams which have ended very recently (a few hours, most of
+ * the time)
+ *
+ *
+ * This method needs:
+ *
+ * - the base URL of the stream (which, if you try to access to it, returns HTTP
+ * status code 404 after redirects, and if the URL is valid);
+ * - an {@link ItagItem}, which needs to contain the following information:
+ *
+ * - its type (see {@link ItagItem.ItagType}), to identify if the content is
+ * an audio or a video stream;
+ * - its bitrate;
+ * - its mime type;
+ * - its codec(s);
+ * - for an audio stream: its audio channels;
+ * - for a video stream: its width and height.
+ *
+ *
+ * - the duration of the video, which will be used if the duration could not be
+ * parsed from the first sequence of the stream.
+ *
+ *
+ *
+ * In order to generate the DASH manifest, this method will:
+ *
+ * - request the first sequence of the stream (the base URL on which the first
+ * sequence parameter is appended (see {@link YoutubeDashManifestCreatorsUtils#SQ_0}))
+ * with a {@code POST} or {@code GET} request (depending of the client on which the
+ * streaming URL comes from is a mobile one ({@code POST}) or not ({@code GET}));
+ * - follow its redirection(s), if any;
+ * - save the last URL, remove the first sequence parameters;
+ * - use the information provided in the {@link ItagItem} to generate all elements
+ * of the DASH manifest.
+ *
+ *
+ *
+ *
+ * If the duration cannot be extracted, the {@code durationSecondsFallback} value will be used
+ * as the stream duration.
+ *
+ *
+ * @param postLiveStreamDvrStreamingUrl the base URL of the post-live-DVR stream/ended
+ * livestream, which must not be null
+ * @param itagItem the {@link ItagItem} corresponding to the stream, which
+ * must not be null
+ * @param targetDurationSec the target duration of each sequence, in seconds (this
+ * value is returned with the {@code targetDurationSec}
+ * field for each stream in YouTube's player response)
+ * @param durationSecondsFallback the duration of the ended livestream, which will be
+ * used if the duration could not be extracted from the
+ * first sequence
+ * @return the manifest generated into a string
+ */
+ @Nonnull
+ public static String fromPostLiveStreamDvrStreamingUrl(
+ @Nonnull final String postLiveStreamDvrStreamingUrl,
+ @Nonnull final ItagItem itagItem,
+ final int targetDurationSec,
+ final long durationSecondsFallback) throws CreationException {
+ if (POST_LIVE_DVR_STREAMS_CACHE.containsKey(postLiveStreamDvrStreamingUrl)) {
+ return Objects.requireNonNull(
+ POST_LIVE_DVR_STREAMS_CACHE.get(postLiveStreamDvrStreamingUrl)).getSecond();
+ }
+
+ String realPostLiveStreamDvrStreamingUrl = postLiveStreamDvrStreamingUrl;
+ final String streamDurationString;
+ final String segmentCount;
+
+ if (targetDurationSec <= 0) {
+ throw new CreationException("targetDurationSec value is <= 0: " + targetDurationSec);
+ }
+
+ try {
+ // Try to avoid redirects when streaming the content by saving the latest URL we get
+ // from video servers.
+ final Response response = getInitializationResponse(realPostLiveStreamDvrStreamingUrl,
+ itagItem, DeliveryType.LIVE);
+ realPostLiveStreamDvrStreamingUrl = response.latestUrl().replace(SQ_0, EMPTY_STRING)
+ .replace(RN_0, EMPTY_STRING).replace(ALR_YES, EMPTY_STRING);
+
+ final int responseCode = response.responseCode();
+ if (responseCode != 200) {
+ throw new CreationException(
+ "Could not get the initialization sequence: response code " + responseCode);
+ }
+
+ final Map> responseHeaders = response.responseHeaders();
+ streamDurationString = responseHeaders.get("X-Head-Time-Millis").get(0);
+ segmentCount = responseHeaders.get("X-Head-Seqnum").get(0);
+ } catch (final IndexOutOfBoundsException e) {
+ throw new CreationException(
+ "Could not get the value of the X-Head-Time-Millis or the X-Head-Seqnum header",
+ e);
+ }
+
+ if (isNullOrEmpty(segmentCount)) {
+ throw new CreationException("Could not get the number of segments");
+ }
+
+ long streamDuration;
+ try {
+ streamDuration = Long.parseLong(streamDurationString);
+ } catch (final NumberFormatException e) {
+ streamDuration = durationSecondsFallback;
+ }
+
+ final Document document = generateDocumentAndDoCommonElementsGeneration(itagItem,
+ streamDuration);
+
+ generateSegmentTemplateElement(document, realPostLiveStreamDvrStreamingUrl,
+ DeliveryType.LIVE);
+ generateSegmentTimelineElement(document);
+ generateSegmentElementForPostLiveDvrStreams(document, targetDurationSec, segmentCount);
+
+ return buildAndCacheResult(postLiveStreamDvrStreamingUrl, document,
+ POST_LIVE_DVR_STREAMS_CACHE);
+ }
+
+ /**
+ * @return the cache of DASH manifests generated for post-live-DVR streams
+ */
+ public static ManifestCreatorCache getCache() {
+ return POST_LIVE_DVR_STREAMS_CACHE;
+ }
+
+ /**
+ * Generate the segment ({@code }) element.
+ *
+ *
+ * We don't know the exact duration of segments for post-live-DVR streams but an
+ * average instead (which is the {@code targetDurationSec} value), so we can use the following
+ * structure to generate the segment timeline for DASH manifests of ended livestreams:
+ *
+ * {@code
+ *
+ * @param document the {@link Document} on which the {@code } element will
+ * be appended
+ * @param targetDurationSeconds the {@code targetDurationSec} value from YouTube player
+ * response's stream
+ * @param segmentCount the number of segments, extracted by {@link
+ * #fromPostLiveStreamDvrStreamingUrl(String, ItagItem, int, long)}
+ */
+ private static void generateSegmentElementForPostLiveDvrStreams(
+ @Nonnull final Document document,
+ final int targetDurationSeconds,
+ @Nonnull final String segmentCount) throws CreationException {
+ try {
+ final Element segmentTimelineElement = (Element) document.getElementsByTagName(
+ SEGMENT_TIMELINE).item(0);
+ final Element sElement = document.createElement("S");
+
+ final Attr dAttribute = document.createAttribute("d");
+ dAttribute.setValue(String.valueOf(targetDurationSeconds * 1000));
+ sElement.setAttributeNode(dAttribute);
+
+ final Attr rAttribute = document.createAttribute("r");
+ rAttribute.setValue(segmentCount);
+ sElement.setAttributeNode(rAttribute);
+
+ segmentTimelineElement.appendChild(sElement);
+ } catch (final DOMException e) {
+ throw CreationException.couldNotAddElement("segment (S)", e);
+ }
+ }
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeProgressiveDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeProgressiveDashManifestCreator.java
new file mode 100644
index 000000000..0a8ed4533
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeProgressiveDashManifestCreator.java
@@ -0,0 +1,244 @@
+package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
+
+import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
+import org.schabi.newpipe.extractor.services.youtube.ItagItem;
+import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
+import org.w3c.dom.Attr;
+import org.w3c.dom.DOMException;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import javax.annotation.Nonnull;
+import java.util.Objects;
+
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.BASE_URL;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.INITIALIZATION;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.MPD;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.REPRESENTATION;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_BASE;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.buildAndCacheResult;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateDocumentAndDoCommonElementsGeneration;
+
+/**
+ * Class which generates DASH manifests of {@link DeliveryType#PROGRESSIVE YouTube progressive}
+ * streams.
+ */
+public final class YoutubeProgressiveDashManifestCreator {
+
+ /**
+ * Cache of DASH manifests generated for progressive streams.
+ */
+ private static final ManifestCreatorCache PROGRESSIVE_STREAMS_CACHE
+ = new ManifestCreatorCache<>();
+
+ private YoutubeProgressiveDashManifestCreator() {
+ }
+
+ /**
+ * Create DASH manifests from a YouTube progressive stream.
+ *
+ *
+ * Progressive streams are YouTube DASH streams which work with range requests and without the
+ * need to get a manifest.
+ *
+ *
+ *
+ * They can be found on all videos, and for all streams for most of videos which come from a
+ * YouTube partner, and on videos with a large number of views.
+ *
+ *
+ * This method needs:
+ *
+ * - the base URL of the stream (which, if you try to access to it, returns the whole
+ * stream, after redirects, and if the URL is valid);
+ * - an {@link ItagItem}, which needs to contain the following information:
+ *
+ * - its type (see {@link ItagItem.ItagType}), to identify if the content is
+ * an audio or a video stream;
+ * - its bitrate;
+ * - its mime type;
+ * - its codec(s);
+ * - for an audio stream: its audio channels;
+ * - for a video stream: its width and height.
+ *
+ *
+ * - the duration of the video (parameter {@code durationSecondsFallback}), which
+ * will be used as the stream duration if the duration could not be parsed from the
+ * {@link ItagItem}.
+ *
+ *
+ *
+ * @param progressiveStreamingBaseUrl the base URL of the progressive stream, which must not be
+ * null
+ * @param itagItem the {@link ItagItem} corresponding to the stream, which
+ * must not be null
+ * @param durationSecondsFallback the duration of the progressive stream which will be used
+ * if the duration could not be extracted from the
+ * {@link ItagItem}
+ * @return the manifest generated into a string
+ */
+ @Nonnull
+ public static String fromProgressiveStreamingUrl(
+ @Nonnull final String progressiveStreamingBaseUrl,
+ @Nonnull final ItagItem itagItem,
+ final long durationSecondsFallback) throws CreationException {
+ if (PROGRESSIVE_STREAMS_CACHE.containsKey(progressiveStreamingBaseUrl)) {
+ return Objects.requireNonNull(
+ PROGRESSIVE_STREAMS_CACHE.get(progressiveStreamingBaseUrl)).getSecond();
+ }
+
+ final long itagItemDuration = itagItem.getApproxDurationMs();
+ final long streamDuration;
+ if (itagItemDuration != -1) {
+ streamDuration = itagItemDuration;
+ } else {
+ if (durationSecondsFallback > 0) {
+ streamDuration = durationSecondsFallback * 1000;
+ } else {
+ throw CreationException.couldNotAddElement(MPD, "the duration of the stream "
+ + "could not be determined and durationSecondsFallback is <= 0");
+ }
+ }
+
+ final Document document = generateDocumentAndDoCommonElementsGeneration(itagItem,
+ streamDuration);
+
+ generateBaseUrlElement(document, progressiveStreamingBaseUrl);
+ generateSegmentBaseElement(document, itagItem);
+ generateInitializationElement(document, itagItem);
+
+ return buildAndCacheResult(progressiveStreamingBaseUrl, document,
+ PROGRESSIVE_STREAMS_CACHE);
+ }
+
+ /**
+ * @return the cache of DASH manifests generated for progressive streams
+ */
+ public static ManifestCreatorCache getCache() {
+ return PROGRESSIVE_STREAMS_CACHE;
+ }
+
+ /**
+ * Generate the {@code } element, appended as a child of the
+ * {@code } element.
+ *
+ *
+ * The {@code } element needs to be generated before this element with
+ * {@link YoutubeDashManifestCreatorsUtils#generateRepresentationElement(Document, ItagItem)}).
+ *
+ *
+ * @param document the {@link Document} on which the {@code } element will
+ * be appended
+ * @param baseUrl the base URL of the stream, which must not be null and will be set as the
+ * content of the {@code } element
+ */
+ private static void generateBaseUrlElement(@Nonnull final Document document,
+ @Nonnull final String baseUrl)
+ throws CreationException {
+ try {
+ final Element representationElement = (Element) document.getElementsByTagName(
+ REPRESENTATION).item(0);
+ final Element baseURLElement = document.createElement(BASE_URL);
+ baseURLElement.setTextContent(baseUrl);
+ representationElement.appendChild(baseURLElement);
+ } catch (final DOMException e) {
+ throw CreationException.couldNotAddElement(BASE_URL, e);
+ }
+ }
+
+ /**
+ * Generate the {@code } element, appended as a child of the
+ * {@code } element.
+ *
+ *
+ * It generates the following element:
+ *
+ * {@code }
+ *
+ * (where {@code indexStart} and {@code indexEnd} are gotten from the {@link ItagItem} passed
+ * as the second parameter)
+ *
+ *
+ *
+ * The {@code } element needs to be generated before this element with
+ * {@link YoutubeDashManifestCreatorsUtils#generateRepresentationElement(Document, ItagItem)}),
+ * and the {@code BaseURL} element with {@link #generateBaseUrlElement(Document, String)}
+ * should be generated too.
+ *
+ *
+ * @param document the {@link Document} on which the {@code } element will be
+ * appended
+ * @param itagItem the {@link ItagItem} to use, which must not be null
+ */
+ private static void generateSegmentBaseElement(@Nonnull final Document document,
+ @Nonnull final ItagItem itagItem)
+ throws CreationException {
+ try {
+ final Element representationElement = (Element) document.getElementsByTagName(
+ REPRESENTATION).item(0);
+
+ final Element segmentBaseElement = document.createElement(SEGMENT_BASE);
+ final Attr indexRangeAttribute = document.createAttribute("indexRange");
+
+ if (itagItem.getIndexStart() < 0 || itagItem.getIndexEnd() < 0) {
+ throw CreationException.couldNotAddElement(SEGMENT_BASE,
+ "ItagItem's indexStart or " + "indexEnd are < 0: "
+ + itagItem.getIndexStart() + "-" + itagItem.getIndexEnd());
+ }
+
+ indexRangeAttribute.setValue(itagItem.getIndexStart() + "-" + itagItem.getIndexEnd());
+ segmentBaseElement.setAttributeNode(indexRangeAttribute);
+
+ representationElement.appendChild(segmentBaseElement);
+ } catch (final DOMException e) {
+ throw CreationException.couldNotAddElement(SEGMENT_BASE, e);
+ }
+ }
+
+ /**
+ * Generate the {@code } element, appended as a child of the
+ * {@code } element.
+ *
+ *
+ * It generates the following element:
+ *
+ * {@code }
+ *
+ * (where {@code indexStart} and {@code indexEnd} are gotten from the {@link ItagItem} passed
+ * as the second parameter)
+ *
+ *
+ *
+ * The {@code } element needs to be generated before this element with
+ * {@link #generateSegmentBaseElement(Document, ItagItem)}).
+ *
+ *
+ * @param document the {@link Document} on which the {@code } element will
+ * be appended
+ * @param itagItem the {@link ItagItem} to use, which must not be null
+ */
+ private static void generateInitializationElement(@Nonnull final Document document,
+ @Nonnull final ItagItem itagItem)
+ throws CreationException {
+ try {
+ final Element segmentBaseElement = (Element) document.getElementsByTagName(
+ SEGMENT_BASE).item(0);
+
+ final Element initializationElement = document.createElement(INITIALIZATION);
+ final Attr rangeAttribute = document.createAttribute("range");
+
+ if (itagItem.getInitStart() < 0 || itagItem.getInitEnd() < 0) {
+ throw CreationException.couldNotAddElement(INITIALIZATION,
+ "ItagItem's initStart and/or " + "initEnd are/is < 0: "
+ + itagItem.getInitStart() + "-" + itagItem.getInitEnd());
+ }
+
+ rangeAttribute.setValue(itagItem.getInitStart() + "-" + itagItem.getInitEnd());
+ initializationElement.setAttributeNode(rangeAttribute);
+
+ segmentBaseElement.appendChild(initializationElement);
+ } catch (final DOMException e) {
+ throw CreationException.couldNotAddElement(INITIALIZATION, e);
+ }
+ }
+}
diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorsTest.java
similarity index 80%
rename from extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorTest.java
rename to extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorsTest.java
index ca8185b9f..0d276f901 100644
--- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorTest.java
+++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorsTest.java
@@ -1,5 +1,30 @@
package org.schabi.newpipe.extractor.services.youtube;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.CreationException;
+import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeOtfDashManifestCreator;
+import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeProgressiveDashManifestCreator;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
+import org.schabi.newpipe.extractor.stream.DeliveryMethod;
+import org.schabi.newpipe.extractor.stream.Stream;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+
+import javax.annotation.Nonnull;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import java.io.StringReader;
+import java.util.List;
+import java.util.Random;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
import static org.junit.jupiter.api.Assertions.assertAll;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -11,41 +36,25 @@ import static org.schabi.newpipe.extractor.ExtractorAsserts.assertGreaterOrEqual
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsValidUrl;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertNotBlank;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeDashManifestCreator.ADAPTATION_SET;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeDashManifestCreator.INITIALIZATION;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeDashManifestCreator.PERIOD;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeDashManifestCreator.REPRESENTATION;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeDashManifestCreator.SEGMENT_BASE;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeDashManifestCreator.SEGMENT_TEMPLATE;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeDashManifestCreator.SEGMENT_TIMELINE;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ADAPTATION_SET;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.AUDIO_CHANNEL_CONFIGURATION;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.BASE_URL;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.INITIALIZATION;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.MPD;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.PERIOD;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.REPRESENTATION;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ROLE;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_BASE;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TEMPLATE;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TIMELINE;
import static org.schabi.newpipe.extractor.utils.Utils.isBlank;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Test;
-import org.schabi.newpipe.downloader.DownloaderTestImpl;
-import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
-import org.schabi.newpipe.extractor.stream.DeliveryMethod;
-import org.schabi.newpipe.extractor.stream.Stream;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.NodeList;
-import org.xml.sax.InputSource;
-
-import java.io.StringReader;
-import java.util.List;
-import java.util.Random;
-import java.util.function.Consumer;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-
-import javax.annotation.Nonnull;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-
/**
- * Test for {@link YoutubeDashManifestCreator}. Tests the generation of OTF and Progressive
- * manifests.
+ * Test for YouTube DASH manifest creators.
+ *
+ *
+ * Tests the generation of OTF and progressive manifests.
+ *
*
*
* We cannot test the generation of DASH manifests for ended livestreams because these videos will
@@ -54,8 +63,9 @@ import javax.xml.parsers.DocumentBuilderFactory;
*
*
* The generation of DASH manifests for OTF streams, which can be tested, uses a video licenced
- * under the Creative Commons Attribution licence (reuse allowed):
- * {@code https://www.youtube.com/watch?v=DJ8GQUNUXGM}
+ * under the Creative Commons Attribution licence (reuse allowed): {@code A New Era of Open?
+ * COVID-19 and the Pursuit for Equitable Solutions} (https://www.youtube.com/watch?v=DJ8GQUNUXGM)
*
*
*
@@ -68,8 +78,8 @@ import javax.xml.parsers.DocumentBuilderFactory;
* So the real downloader will be used everytime on this test class.
*
*/
-class YoutubeDashManifestCreatorTest {
- // Setting a higher number may let Google video servers return a lot of 403s
+class YoutubeDashManifestCreatorsTest {
+ // Setting a higher number may let Google video servers return 403s
private static final int MAX_STREAMS_TO_TEST_PER_METHOD = 3;
private static final String url = "https://www.youtube.com/watch?v=DJ8GQUNUXGM";
private static YoutubeStreamExtractor extractor;
@@ -102,7 +112,7 @@ class YoutubeDashManifestCreatorTest {
assertProgressiveStreams(extractor.getAudioStreams());
// we are not able to generate DASH manifests of video formats with audio
- assertThrows(YoutubeDashManifestCreator.CreationException.class,
+ assertThrows(CreationException.class,
() -> assertProgressiveStreams(extractor.getVideoStreams()));
}
@@ -110,7 +120,7 @@ class YoutubeDashManifestCreatorTest {
for (final Stream stream : assertFilterStreams(streams, DeliveryMethod.DASH)) {
//noinspection ConstantConditions
- final String manifest = YoutubeDashManifestCreator.fromOtfStreamingUrl(
+ final String manifest = YoutubeOtfDashManifestCreator.fromOtfStreamingUrl(
stream.getContent(), stream.getItagItem(), videoLength);
assertNotBlank(manifest);
@@ -129,8 +139,9 @@ class YoutubeDashManifestCreatorTest {
for (final Stream stream : assertFilterStreams(streams, DeliveryMethod.PROGRESSIVE_HTTP)) {
//noinspection ConstantConditions
- final String manifest = YoutubeDashManifestCreator.fromProgressiveStreamingUrl(
- stream.getContent(), stream.getItagItem(), videoLength);
+ final String manifest =
+ YoutubeProgressiveDashManifestCreator.fromProgressiveStreamingUrl(
+ stream.getContent(), stream.getItagItem(), videoLength);
assertNotBlank(manifest);
assertManifestGenerated(
@@ -145,8 +156,10 @@ class YoutubeDashManifestCreatorTest {
}
}
- private List assertFilterStreams(final List streams,
- final DeliveryMethod deliveryMethod) {
+ @Nonnull
+ private List assertFilterStreams(
+ @Nonnull final List streams,
+ final DeliveryMethod deliveryMethod) {
final List filteredStreams = streams.stream()
.filter(stream -> stream.getDeliveryMethod() == deliveryMethod)
@@ -190,7 +203,7 @@ class YoutubeDashManifestCreatorTest {
}
private void assertMpdElement(@Nonnull final Document document) {
- final Element element = (Element) document.getElementsByTagName("MPD").item(0);
+ final Element element = (Element) document.getElementsByTagName(MPD).item(0);
assertNotNull(element);
assertNull(element.getParentNode().getNodeValue());
@@ -200,7 +213,7 @@ class YoutubeDashManifestCreatorTest {
}
private void assertPeriodElement(@Nonnull final Document document) {
- assertGetElement(document, PERIOD, "MPD");
+ assertGetElement(document, PERIOD, MPD);
}
private void assertAdaptationSetElement(@Nonnull final Document document,
@@ -210,7 +223,7 @@ class YoutubeDashManifestCreatorTest {
}
private void assertRoleElement(@Nonnull final Document document) {
- assertGetElement(document, "Role", ADAPTATION_SET);
+ assertGetElement(document, ROLE, ADAPTATION_SET);
}
private void assertRepresentationElement(@Nonnull final Document document,
@@ -232,8 +245,8 @@ class YoutubeDashManifestCreatorTest {
private void assertAudioChannelConfigurationElement(@Nonnull final Document document,
@Nonnull final ItagItem itagItem) {
- final Element element = assertGetElement(document,
- "AudioChannelConfiguration", REPRESENTATION);
+ final Element element = assertGetElement(document, AUDIO_CHANNEL_CONFIGURATION,
+ REPRESENTATION);
assertAttrEquals(itagItem.getAudioChannels(), element, "value");
}
@@ -276,7 +289,7 @@ class YoutubeDashManifestCreatorTest {
}
private void assertBaseUrlElement(@Nonnull final Document document) {
- final Element element = assertGetElement(document, "BaseURL", REPRESENTATION);
+ final Element element = assertGetElement(document, BASE_URL, REPRESENTATION);
assertIsValidUrl(element.getTextContent());
}
@@ -294,7 +307,7 @@ class YoutubeDashManifestCreatorTest {
private void assertAttrEquals(final int expected,
- final Element element,
+ @Nonnull final Element element,
final String attribute) {
final int actual = Integer.parseInt(element.getAttribute(attribute));
@@ -305,7 +318,7 @@ class YoutubeDashManifestCreatorTest {
}
private void assertAttrEquals(final String expected,
- final Element element,
+ @Nonnull final Element element,
final String attribute) {
final String actual = element.getAttribute(attribute);
assertAll(
@@ -316,7 +329,7 @@ class YoutubeDashManifestCreatorTest {
private void assertRangeEquals(final int expectedStart,
final int expectedEnd,
- final Element element,
+ @Nonnull final Element element,
final String attribute) {
final String range = element.getAttribute(attribute);
assertNotBlank(range);
@@ -334,7 +347,8 @@ class YoutubeDashManifestCreatorTest {
);
}
- private Element assertGetElement(final Document document,
+ @Nonnull
+ private Element assertGetElement(@Nonnull final Document document,
final String tagName,
final String expectedParentTagName) {