} element.
*
*
* This method is only used when generating DASH manifests from OTF and post-live-DVR streams.
*
*
*
* It will produce a {@code } element with the following attributes:
*
* - {@code startNumber}, which takes the value {@code 0} for post-live-DVR streams and
* {@code 1} for OTF streams;
* - {@code timescale}, which is always {@code 1000};
* - {@code media}, which is the base URL of the stream on which is appended
* {@code &sq=$Number$};
* - {@code initialization} (only for OTF streams), which is the base URL of the stream
* on which is appended {@link #SQ_0}.
*
*
*
*
* The {@code } element needs to be generated before this element with
* {@link #generateRepresentationElement(Document, ItagItem)}).
*
*
* @param doc the {@link Document} on which the {@code } element will
* be appended
* @param baseUrl the base URL of the OTF/post-live-DVR stream
* @param deliveryType the stream {@link DeliveryType delivery type}, which must be either
* {@link DeliveryType#OTF OTF} or {@link DeliveryType#LIVE LIVE}
*/
public static void generateSegmentTemplateElement(@Nonnull final Document doc,
@Nonnull final String baseUrl,
final DeliveryType deliveryType)
throws CreationException {
if (deliveryType != DeliveryType.OTF && deliveryType != DeliveryType.LIVE) {
throw CreationException.couldNotAddElement(SEGMENT_TEMPLATE, "invalid delivery type: "
+ deliveryType);
}
try {
final Element representationElement = (Element) doc.getElementsByTagName(
REPRESENTATION).item(0);
final Element segmentTemplateElement = doc.createElement(SEGMENT_TEMPLATE);
// The first sequence of post DVR streams is the beginning of the video stream and not
// an initialization segment
setAttribute(segmentTemplateElement, doc, "startNumber",
deliveryType == DeliveryType.LIVE ? "0" : "1");
setAttribute(segmentTemplateElement, doc, "timescale", "1000");
// Post-live-DVR/ended livestreams streams don't require an initialization sequence
if (deliveryType != DeliveryType.LIVE) {
setAttribute(segmentTemplateElement, doc, "initialization", baseUrl + SQ_0);
}
setAttribute(segmentTemplateElement, doc, "media", baseUrl + "&sq=$Number$");
representationElement.appendChild(segmentTemplateElement);
} catch (final DOMException e) {
throw CreationException.couldNotAddElement(SEGMENT_TEMPLATE, e);
}
}
/**
* Generate the {@code } element, appended as a child of the
* {@code } element.
*
*
* The {@code } element needs to be generated before this element with
* {@link #generateSegmentTemplateElement(Document, String, DeliveryType)}.
*
*
* @param doc the {@link Document} on which the {@code } element will be
* appended
*/
public static void generateSegmentTimelineElement(@Nonnull final Document doc)
throws CreationException {
try {
final Element segmentTemplateElement = (Element) doc.getElementsByTagName(
SEGMENT_TEMPLATE).item(0);
final Element segmentTimelineElement = doc.createElement(SEGMENT_TIMELINE);
segmentTemplateElement.appendChild(segmentTimelineElement);
} catch (final DOMException e) {
throw CreationException.couldNotAddElement(SEGMENT_TIMELINE, e);
}
}
/**
* Get the "initialization" {@link Response response} of a stream.
*
* This method fetches, for OTF streams and for post-live-DVR streams:
*
* - the base URL of the stream, to which are appended {@link #SQ_0} and
* {@link #RN_0} parameters, with a {@code GET} request for streaming URLs from HTML5
* clients and a {@code POST} request for the ones from the {@code ANDROID} and the
* {@code IOS} clients;
* - for streaming URLs from HTML5 clients, the {@link #ALR_YES} param is also added.
*
*
*
*
* @param baseStreamingUrl the base URL of the stream, which must not be null
* @param itagItem the {@link ItagItem} of stream, which must not be null
* @param deliveryType the {@link DeliveryType} of the stream
* @return the "initialization" response, without redirections on the network on which the
* request(s) is/are made
*/
@SuppressWarnings("checkstyle:FinalParameters")
@Nonnull
public static Response getInitializationResponse(@Nonnull String baseStreamingUrl,
@Nonnull final ItagItem itagItem,
final DeliveryType deliveryType)
throws CreationException {
final boolean isHtml5StreamingUrl = isWebStreamingUrl(baseStreamingUrl)
|| isTvHtml5SimplyEmbeddedPlayerStreamingUrl(baseStreamingUrl);
final boolean isAndroidStreamingUrl = isAndroidStreamingUrl(baseStreamingUrl);
final boolean isIosStreamingUrl = isIosStreamingUrl(baseStreamingUrl);
if (isHtml5StreamingUrl) {
baseStreamingUrl += ALR_YES;
}
baseStreamingUrl = appendRnSqParamsIfNeeded(baseStreamingUrl, deliveryType);
final Downloader downloader = NewPipe.getDownloader();
if (isHtml5StreamingUrl) {
final String mimeTypeExpected = itagItem.getMediaFormat().getMimeType();
if (!isNullOrEmpty(mimeTypeExpected)) {
return getStreamingWebUrlWithoutRedirects(downloader, baseStreamingUrl,
mimeTypeExpected);
}
} else if (isAndroidStreamingUrl || isIosStreamingUrl) {
try {
final var headers = Map.of("User-Agent",
List.of(isAndroidStreamingUrl ? getAndroidUserAgent(null)
: getIosUserAgent(null)));
final byte[] emptyBody = "".getBytes(StandardCharsets.UTF_8);
return downloader.post(baseStreamingUrl, headers, emptyBody);
} catch (final IOException | ExtractionException e) {
throw new CreationException("Could not get the "
+ (isIosStreamingUrl ? "ANDROID" : "IOS") + " streaming URL response", e);
}
}
try {
return downloader.get(baseStreamingUrl);
} catch (final IOException | ExtractionException e) {
throw new CreationException("Could not get the streaming URL response", e);
}
}
/**
* Generate a new {@link DocumentBuilder} secured from XXE attacks, on platforms which
* support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and
* {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link DocumentBuilderFactory} instances.
*
* @return an instance of {@link Document} secured against XXE attacks on supported platforms,
* that should then be convertible to an XML string without security problems
*/
private static Document newDocument() throws ParserConfigurationException {
final DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
try {
documentBuilderFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
documentBuilderFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
} catch (final Exception ignored) {
// Ignore exceptions as setting these attributes to secure XML generation is not
// supported by all platforms (like the Android implementation)
}
return documentBuilderFactory.newDocumentBuilder().newDocument();
}
/**
* Generate a new {@link TransformerFactory} secured from XXE attacks, on platforms which
* support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and
* {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link TransformerFactory} instances.
*
* @param doc the doc to convert, which must have been created using {@link #newDocument()} to
* properly prevent XXE attacks
* @return the doc converted to an XML string, making sure there can't be XXE attacks
*/
// Sonar warning is suppressed because it is still shown even if we apply its solution
@SuppressWarnings("squid:S2755")
private static String documentToXml(@Nonnull final Document doc)
throws TransformerException {
final TransformerFactory transformerFactory = TransformerFactory.newInstance();
try {
transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
} catch (final Exception ignored) {
// Ignore exceptions as setting these attributes to secure XML generation is not
// supported by all platforms (like the Android implementation)
}
final Transformer transformer = transformerFactory.newTransformer();
transformer.setOutputProperty(OutputKeys.VERSION, "1.0");
transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");
transformer.setOutputProperty(OutputKeys.STANDALONE, "no");
final StringWriter result = new StringWriter();
transformer.transform(new DOMSource(doc), new StreamResult(result));
return result.toString();
}
/**
* Append {@link #SQ_0} for post-live-DVR and OTF streams and {@link #RN_0} to all streams.
*
* @param baseStreamingUrl the base streaming URL to which the parameter(s) are being appended
* @param deliveryType the {@link DeliveryType} of the stream
* @return the base streaming URL to which the param(s) are appended, depending on the
* {@link DeliveryType} of the stream
*/
@Nonnull
private static String appendRnSqParamsIfNeeded(@Nonnull final String baseStreamingUrl,
@Nonnull final DeliveryType deliveryType) {
return baseStreamingUrl + (deliveryType == DeliveryType.PROGRESSIVE ? "" : SQ_0) + RN_0;
}
/**
* Get a URL on which no redirection between playback hosts should be present on the network
* and/or IP used to fetch the streaming URL, for HTML5 clients.
*
* This method will follow redirects which works in the following way:
*
* - the {@link #ALR_YES} param is appended to all streaming URLs
* - if no redirection occurs, the video server will return the streaming data;
* - if a redirection occurs, the server will respond with HTTP status code 200 and a
* {@code text/plain} mime type. The redirection URL is the response body;
* - the redirection URL is requested and the steps above from step 2 are repeated,
* until too many redirects are reached of course (the maximum number of redirects is
* {@link #MAXIMUM_REDIRECT_COUNT the same as OkHttp}).
*
*
*
*
* For non-HTML5 clients, redirections are managed in the standard way in
* {@link #getInitializationResponse(String, ItagItem, DeliveryType)}.
*
*
* @param downloader the {@link Downloader} instance to be used
* @param streamingUrl the streaming URL which we are trying to get a streaming URL
* without any redirection on the network and/or IP used
* @param responseMimeTypeExpected the response mime type expected from Google video servers
* @return the {@link Response} of the stream, which should have no redirections
*/
@SuppressWarnings("checkstyle:FinalParameters")
@Nonnull
private static Response getStreamingWebUrlWithoutRedirects(
@Nonnull final Downloader downloader,
@Nonnull String streamingUrl,
@Nonnull final String responseMimeTypeExpected)
throws CreationException {
try {
final var headers = getClientInfoHeaders();
String responseMimeType = "";
int redirectsCount = 0;
while (!responseMimeType.equals(responseMimeTypeExpected)
&& redirectsCount < MAXIMUM_REDIRECT_COUNT) {
final Response response = downloader.get(streamingUrl, headers);
final int responseCode = response.responseCode();
if (responseCode != 200) {
throw new CreationException(
"Could not get the initialization URL: HTTP response code "
+ responseCode);
}
// A valid HTTP 1.0+ response should include a Content-Type header, so we can
// require that the response from video servers has this header.
responseMimeType = Objects.requireNonNull(response.getHeader("Content-Type"),
"Could not get the Content-Type header from the response headers");
// The response body is the redirection URL
if (responseMimeType.equals("text/plain")) {
streamingUrl = response.responseBody();
redirectsCount++;
} else {
return response;
}
}
if (redirectsCount >= MAXIMUM_REDIRECT_COUNT) {
throw new CreationException(
"Too many redirects when trying to get the the streaming URL response of a "
+ "HTML5 client");
}
// This should never be reached, but is required because we don't want to return null
// here
throw new CreationException(
"Could not get the streaming URL response of a HTML5 client: unreachable code "
+ "reached!");
} catch (final IOException | ExtractionException e) {
throw new CreationException(
"Could not get the streaming URL response of a HTML5 client", e);
}
}
}