lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Rebase + some code improvements + fix extraction of age-restricted videos + update clients version 

Here is now the requests which will be made by the `onFetchPage` method of `YoutubeStreamExtractor`:

- the desktop API is fetched.

If there is no streaming data, the desktop player API with the embed client screen will be fetched (and also the player code), then the Android mobile API.
- if there is no streaming data, a `ContentNotAvailableException` will be thrown by using the message provided in playability status

If the video is age restricted, a request to the next endpoint of the desktop player with the embed client screen will be sent.
Otherwise, the next endpoint will be fetched normally, if the content is available.

If the video is not age-restricted, a request to the player endpoint of the Android mobile API will be made.

We can get more streams by using the Android mobile API but some streams may be not available on this API, so the streaming data of the Android mobile API will be first used to get itags and then the streaming data of the desktop internal API will be used.
If the parsing of the Android mobile API went wrong, only the streams of the desktop API will be used.

Other code changes:

- `prepareJsonBuilder` in `YoutubeParsingHelper` was renamed to `prepareDesktopJsonBuilder`
- `prepareMobileJsonBuilder` in `YoutubeParsingHelper` was renamed to `prepareAndroidMobileJsonBuilder`
- two new methods in `YoutubeParsingHelper` were added: `prepareDesktopEmbedVideoJsonBuilder` and `prepareAndroidMobileEmbedVideoJsonBuilder`
- `createPlayerBodyWithSts` is now public and was moved to `YoutubeParsingHelper`
- a new method in `YoutubeJavaScriptExtractor` was added: `resetJavaScriptCode`, which was needed for the method `resetDebofuscationCode` of `YoutubeStreamExtractor`
- `areHardcodedClientVersionAndKeyValid` in `YoutubeParsingHelper` returns now a `boolean` instead of an `Optional<Boolean>`
- the `fetchVideoInfoPage` method of `YoutubeStreamExtractor` was removed because YouTube returns now 404 for every client with the `get_video_info` page
- some unused objects and some warnings in `YoutubeStreamExtractor` were removed and fixed

Co-authored-by: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com>
This commit is contained in:
FireMasterK 2021-07-29 03:25:09 +05:30 committed by TiA4f8R
parent 7753556e66
commit 2eeb0a3403
No known key found for this signature in database
GPG Key ID: E6D3E7F5949450DD
10 changed files with 366 additions and 275 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractor.java
View File

@ -60,6 +60,14 @@ public class YoutubeJavaScriptExtractor {
return extractJavaScriptCode("d4IGg5dqeO8");
}
/**
* Reset the JavaScript code. It will be fetched again the next time
* {@link #extractJavaScriptCode()} or {@link #extractJavaScriptCode(String)} is called.
*/
public static void resetJavaScriptCode() {
cachedJavaScriptCode = null;
}
private static String extractJavaScriptUrl(final String videoId) throws ParsingException {
try {
final String embedUrl = "https://www.youtube.com/embed/" + videoId;

120
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -66,15 +66,15 @@ public class YoutubeParsingHelper {
public static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/";
private static final String HARDCODED_CLIENT_VERSION = "2.20210701.00.00";
private static final String HARDCODED_CLIENT_VERSION = "2.20210728.00.00";
private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
private static final String MOBILE_YOUTUBE_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w";
private static final String MOBILE_YOUTUBE_CLIENT_VERSION = "16.25.37";
private static final String MOBILE_YOUTUBE_CLIENT_VERSION = "16.29.38";
private static String clientVersion;
private static String key;
private static final String[] HARDCODED_YOUTUBE_MUSIC_KEY =
{"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "1.20210628.00.00"};
{"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "1.20210726.00.01"};
private static String[] youtubeMusicKey;
private static boolean keyAndVersionExtracted = false;
@ -309,10 +309,10 @@ public class YoutubeParsingHelper {
}
}
public static Optional<Boolean> areHardcodedClientVersionAndKeyValid()
public static boolean areHardcodedClientVersionAndKeyValid()
throws IOException, ExtractionException {
if (hardcodedClientVersionAndKeyValid.isPresent()) {
return hardcodedClientVersionAndKeyValid;
return hardcodedClientVersionAndKeyValid.get();
}
// @formatter:off
final byte[] body = JsonWriter.string()
@ -344,8 +344,9 @@ public class YoutubeParsingHelper {
final String responseBody = response.responseBody();
final int responseCode = response.responseCode();
return hardcodedClientVersionAndKeyValid = Optional.of(responseBody.length() > 5000
hardcodedClientVersionAndKeyValid = Optional.of(responseBody.length() > 5000
&& responseCode == 200); // Ensure to have a valid response
return hardcodedClientVersionAndKeyValid.get();
}
private static void extractClientVersionAndKey() throws IOException, ExtractionException {
@ -425,7 +426,7 @@ public class YoutubeParsingHelper {
*/
public static String getClientVersion() throws IOException, ExtractionException {
if (!isNullOrEmpty(clientVersion)) return clientVersion;
if (areHardcodedClientVersionAndKeyValid().orElse(false)) {
if (areHardcodedClientVersionAndKeyValid()) {
return clientVersion = HARDCODED_CLIENT_VERSION;
}
@ -438,7 +439,7 @@ public class YoutubeParsingHelper {
*/
public static String getKey() throws IOException, ExtractionException {
if (!isNullOrEmpty(key)) return key;
if (areHardcodedClientVersionAndKeyValid().orElse(false)) {
if (areHardcodedClientVersionAndKeyValid()) {
return key = HARDCODED_KEY;
}
@ -799,10 +800,9 @@ public class YoutubeParsingHelper {
}
@Nonnull
public static JsonBuilder<JsonObject> prepareJsonBuilder(@Nonnull final Localization
localization,
@Nonnull final ContentCountry
contentCountry)
public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry)
throws IOException, ExtractionException {
// @formatter:off
return JsonObject.builder()
@ -823,10 +823,9 @@ public class YoutubeParsingHelper {
}
@Nonnull
public static JsonBuilder<JsonObject> prepareMobileJsonBuilder(@Nonnull final Localization
localization,
@Nonnull final ContentCountry
contentCountry) {
public static JsonBuilder<JsonObject> prepareAndroidMobileJsonBuilder(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry) {
// @formatter:off
return JsonObject.builder()
.object("context")
@ -845,6 +844,95 @@ public class YoutubeParsingHelper {
// @formatter:on
}
@Nonnull
public static JsonBuilder<JsonObject> prepareDesktopEmbedVideoJsonBuilder(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nonnull final String videoId) throws IOException, ExtractionException {
// @formatter:off
return JsonObject.builder()
.object("context")
.object("client")
.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.value("clientName", "WEB")
.value("clientVersion", getClientVersion())
.value("clientScreen", "EMBED")
.end()
.object("thirdParty")
.value("embedUrl", "https://www.youtube.com/watch?v=" + videoId)
.end()
.object("user")
// TO DO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.end()
.value("videoId", videoId);
// @formatter:on
}
@Nonnull
public static JsonBuilder<JsonObject> prepareAndroidMobileEmbedVideoJsonBuilder(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nonnull final String videoId) {
// @formatter:off
return JsonObject.builder()
.object("context")
.object("client")
.value("clientName", "ANDROID")
.value("clientVersion", MOBILE_YOUTUBE_CLIENT_VERSION)
.value("clientScreen", "EMBED")
.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.end()
.object("thirdParty")
.value("embedUrl", "https://www.youtube.com/watch?v=" + videoId)
.end()
.object("user")
// TO DO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.end()
.value("videoId", videoId);
// @formatter:on
}
@Nonnull
public static byte[] createPlayerBodyWithSts(final Localization localization,
final ContentCountry contentCountry,
final String videoId,
final boolean withThirdParty,
@Nullable final String sts)
throws IOException, ExtractionException {
if (withThirdParty) {
// @formatter:off
return JsonWriter.string(prepareDesktopEmbedVideoJsonBuilder(localization, contentCountry, videoId)
.object("playbackContext")
.object("contentPlaybackContext")
.value("signatureTimestamp", sts)
.end()
.end()
.done())
.getBytes(UTF_8);
// @formatter:on
} else {
// @formatter:off
return JsonWriter.string(prepareDesktopJsonBuilder(localization, contentCountry)
.value("videoId", videoId)
.object("playbackContext")
.object("contentPlaybackContext")
.value("signatureTimestamp", sts)
.end()
.end()
.done())
.getBytes(UTF_8);
// @formatter:on
}
}
/**
* Add required headers and cookies to an existing headers Map.
* @see #addClientInfoHeaders(Map)

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -88,8 +88,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
// navigation/resolve_url endpoint of the InnerTube API to get the channel id. Otherwise,
// we couldn't get information about the channel associated with this URL, if there is one.
if (!channelId[0].equals("channel")) {
final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry())
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
getExtractorLocalization(), getExtractorContentCountry())
.value("url", "https://www.youtube.com/" + channelPath)
.done())
.getBytes(UTF_8);
@ -135,8 +135,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
int level = 0;
while (level < 3) {
final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry())
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
getExtractorLocalization(), getExtractorContentCountry())
.value("browseId", id)
.value("params", "EgZ2aWRlb3M%3D") // Equal to videos
.done())
@ -384,7 +384,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final String continuation = continuationEndpoint.getObject("continuationCommand")
.getString("token");
final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(),
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry())
.value("continuation", continuation)
.done())

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
View File

@ -62,7 +62,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
final String videoId = getQueryValue(url, "v");
final String playlistIndexString = getQueryValue(url, "index");
final JsonBuilder<JsonObject> jsonBody = prepareJsonBuilder(localization,
final JsonBuilder<JsonObject> jsonBody = prepareDesktopJsonBuilder(localization,
getExtractorContentCountry()).value("playlistId", mixPlaylistId);
if (videoId != null) {
jsonBody.value("videoId", videoId);
@ -174,7 +174,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
final String videoId = watchEndpoint.getString("videoId");
final int index = watchEndpoint.getInt("index");
final String params = watchEndpoint.getString("params");
final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(),
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry())
.value("videoId", videoId)
.value("playlistId", playlistId)

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -44,7 +44,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
final Localization localization = getExtractorLocalization();
final byte[] body = JsonWriter.string(prepareJsonBuilder(localization,
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization,
getExtractorContentCountry())
.value("browseId", "VL" + getId())
.value("params", "wgYCCAA%3D") // Show unavailable videos
@ -251,8 +251,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
.getObject("continuationCommand")
.getString("token");
final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry())
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
getExtractorLocalization(), getExtractorContentCountry())
.value("continuation", continuation)
.done())
.getBytes(UTF_8);

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -70,7 +70,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
params = "";
}
final JsonBuilder<JsonObject> jsonBody = prepareJsonBuilder(localization,
final JsonBuilder<JsonObject> jsonBody = prepareDesktopJsonBuilder(localization,
getExtractorContentCountry())
.value("query", query);
if (!isNullOrEmpty(params)) {
@ -166,7 +166,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
// @formatter:off
final byte[] json = JsonWriter.string(prepareJsonBuilder(localization,
final byte[] json = JsonWriter.string(prepareDesktopJsonBuilder(localization,
getExtractorContentCountry())
.value("continuation", page.getId())
.done())

475
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -2,20 +2,14 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException;
@ -25,7 +19,6 @@ import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException;
import org.schabi.newpipe.extractor.exceptions.PaidContentException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.PrivateContentException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.ContentCountry;
@ -93,25 +86,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nullable
private static String cachedDeobfuscationCode = null;
@Nullable
private static String playerJsUrl = null;
@Nullable
private static String sts = null;
@Nullable
private static String playerCode = null;
@Nonnull
private final Map<String, String> videoInfoPage = new HashMap<>();
private JsonArray initialAjaxJson;
private JsonObject initialData;
private JsonObject playerResponse;
private JsonObject nextResponse;
@Nullable
private JsonObject streamingData;
private JsonObject desktopStreamingData;
@Nullable
private JsonObject mobileStreamingData;
private JsonObject videoPrimaryInfoRenderer;
private JsonObject videoSecondaryInfoRenderer;
private int ageLimit = -1;
private boolean isGetVideoInfoPlayerResponse = false;
@Nullable
private List<SubtitlesStream> subtitles = null;
@ -290,12 +278,17 @@ public class YoutubeStreamExtractor extends StreamExtractor {
.getString("lengthSeconds");
return Long.parseLong(duration);
} catch (final Exception e) {
try {
final JsonArray adaptiveFormats = streamingData.getArray("adaptiveFormats");
if (desktopStreamingData != null) {
final JsonArray adaptiveFormats = desktopStreamingData.getArray("adaptiveFormats");
final String durationMs = adaptiveFormats.getObject(0)
.getString("approxDurationMs");
return Math.round(Long.parseLong(durationMs) / 1000f);
} catch (final Exception ignored) {
} else if (mobileStreamingData != null) {
final JsonArray adaptiveFormats = mobileStreamingData.getArray("adaptiveFormats");
final String durationMs = adaptiveFormats.getObject(0)
.getString("approxDurationMs");
return Math.round(Long.parseLong(durationMs) / 1000f);
} else {
throw new ParsingException("Could not get duration", e);
}
}
@ -484,29 +477,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public String getDashMpdUrl() throws ParsingException {
assertPageFetched();
try {
String dashManifestUrl;
if (streamingData.isString("dashManifestUrl")) {
return streamingData.getString("dashManifestUrl");
} else if (videoInfoPage.containsKey("dashmpd")) {
dashManifestUrl = videoInfoPage.get("dashmpd");
} else {
return "";
}
if (!dashManifestUrl.contains("/signature/")) {
String obfuscatedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)",
dashManifestUrl);
final String deobfuscatedSig;
deobfuscatedSig = deobfuscateSignature(obfuscatedSig);
dashManifestUrl = dashManifestUrl.replace("/s/" + obfuscatedSig,
"/signature/" + deobfuscatedSig);
}
return dashManifestUrl;
} catch (final Exception e) {
throw new ParsingException("Could not get DASH manifest url", e);
if (desktopStreamingData != null) {
return desktopStreamingData.getString("dashManifestUrl");
} else if (mobileStreamingData != null) {
return mobileStreamingData.getString("dashManifestUrl");
} else {
return EMPTY_STRING;
}
}
@ -515,10 +491,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public String getHlsUrl() throws ParsingException {
assertPageFetched();
try {
return streamingData.getString("hlsManifestUrl");
} catch (final Exception e) {
throw new ParsingException("Could not get HLS manifest url", e);
if (desktopStreamingData != null) {
return desktopStreamingData.getString("hlsManifestUrl");
} else if (mobileStreamingData != null) {
return mobileStreamingData.getString("hlsManifestUrl");
} else {
return EMPTY_STRING;
}
}
@ -704,7 +682,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private static final String FORMATS = "formats";
private static final String ADAPTIVE_FORMATS = "adaptiveFormats";
private static final String HTTPS = "https:";
private static final String DEOBFUSCATION_FUNC_NAME = "deobfuscate";
private static final String[] REGEXES = {
@ -721,7 +698,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final String videoId = getId();
final Localization localization = getExtractorLocalization();
final ContentCountry contentCountry = getExtractorContentCountry();
final byte[] body = JsonWriter.string(prepareJsonBuilder(localization, contentCountry)
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
localization, contentCountry)
.value("videoId", videoId)
.done())
.getBytes(UTF_8);
@ -731,7 +709,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// API.
if (sts != null) {
playerResponse = getJsonPostResponse("player", createPlayerBodyWithSts(localization,
contentCountry, videoId), localization);
contentCountry, videoId, false, sts), localization);
} else {
playerResponse = getJsonPostResponse("player", body, localization);
}
@ -740,34 +718,55 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// there can be restrictions on the embedded player.
// E.g. if a video is age-restricted, the embedded player's playabilityStatus says that
// the video cannot be played outside of YouTube, but does not show the original message.
JsonObject youtubePlayerResponse = playerResponse;
final JsonObject youtubePlayerResponse = playerResponse;
if (playerResponse == null || !playerResponse.has("streamingData")) {
// Try to get the player response by fetching video info page
fetchVideoInfoPage();
}
if (playerResponse == null && youtubePlayerResponse == null) {
if (playerResponse == null) {
throw new ExtractionException("Could not get playerResponse");
} else if (youtubePlayerResponse == null) {
youtubePlayerResponse = playerResponse;
}
final JsonObject playabilityStatus = (playerResponse == null ? youtubePlayerResponse
: playerResponse).getObject("playabilityStatus");
final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus");
checkPlayabilityStatus(youtubePlayerResponse, playabilityStatus);
boolean ageRestricted = playabilityStatus.getString("reason", EMPTY_STRING)
.contains("age");
nextResponse = getJsonPostResponse("next", body, localization);
if (!playerResponse.has("streamingData")) {
try {
fetchDesktopEmbedJsonPlayer(contentCountry, localization, videoId);
} catch (final Exception ignored) {
}
try {
fetchAndroidEmbedJsonPlayer(contentCountry, localization, videoId);
} catch (final Exception ignored) {
}
}
// Workaround for rate limits on web streaming URLs.
// TODO: add ability to deobfuscate the n param of these URLs
if (desktopStreamingData == null && playerResponse.has("streamingData")) {
desktopStreamingData = playerResponse.getObject("streamingData");
}
// It's not needed to request the mobile API for age-restricted videos
if (!isGetVideoInfoPlayerResponse) {
fetchAndroidMobileJsonPlayer(contentCountry, localization, videoId);
if (desktopStreamingData == null) {
checkPlayabilityStatus(youtubePlayerResponse, playabilityStatus);
}
if (ageRestricted) {
final byte[] ageRestrictedBody = JsonWriter.string(prepareDesktopEmbedVideoJsonBuilder(
localization, contentCountry, videoId)
.done())
.getBytes(UTF_8);
nextResponse = getJsonPostResponse("next", ageRestrictedBody, localization);
} else {
streamingData = playerResponse.getObject("streamingData");
nextResponse = getJsonPostResponse("next", body, localization);
}
if (!ageRestricted) {
try {
fetchAndroidMobileJsonPlayer(contentCountry, localization, videoId);
} catch (final Exception ignored) {
}
}
if (isCipherProtectedContent()) {
fetchDesktopJsonPlayerWithSts(contentCountry, localization, videoId);
}
}
@ -826,133 +825,114 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
/**
* Fetch the Android Mobile API or fallback to the desktop streams.
* If something went wrong when parsing this API, fallback to the desktop JSON player, fetched
* again if the {@code signatureTimestamp} of the JS player is unknown (because signatures
* without a {@code signatureTimestamp} included in the player request are invalid).
* Fetch the Android Mobile API and assign the streaming data to the mobileStreamingData JSON
* object.
*/
private void fetchAndroidMobileJsonPlayer(final ContentCountry contentCountry,
final Localization localization,
final String videoId) throws ExtractionException,
IOException {
JsonObject mobilePlayerResponse = null;
final byte[] mobileBody = JsonWriter.string(prepareMobileJsonBuilder(localization,
contentCountry)
.value("videoId", videoId)
.done())
final String videoId)
throws IOException, ExtractionException {
final byte[] mobileBody = JsonWriter.string(prepareAndroidMobileJsonBuilder(
localization, contentCountry)
.value("videoId", videoId)
.done())
.getBytes(UTF_8);
try {
mobilePlayerResponse = getJsonMobilePostResponse("player", mobileBody,
contentCountry, localization);
} catch (final Exception ignored) {
}
if (mobilePlayerResponse != null && mobilePlayerResponse.has("streamingData")) {
final JsonObject mobileStreamingData = mobilePlayerResponse.getObject(
"streamingData");
if (!isNullOrEmpty(mobileStreamingData)) streamingData = mobileStreamingData;
} else {
// Fallback to the desktop JSON player endpoint
final JsonObject mobilePlayerResponse = getJsonMobilePostResponse("player",
mobileBody, contentCountry, localization);
// The cipher signatures from the player endpoint without a timestamp are invalid so
// download it again only if we didn't have a signatureTimestamp before fetching the
// data of this video (the sts string).
if (sts == null && isCipherProtectedContent()) {
getStsFromPlayerJs();
final JsonObject playerResponseWithSignatureTimestamp = getJsonPostResponse(
"player", createPlayerBodyWithSts(localization, contentCountry, videoId),
localization);
if (playerResponseWithSignatureTimestamp.has("streamingData")) {
streamingData = playerResponseWithSignatureTimestamp.getObject(
"streamingData");
}
} else {
streamingData = playerResponse.getObject("streamingData");
final JsonObject streamingData = mobilePlayerResponse.getObject("streamingData");
if (!isNullOrEmpty(streamingData)) {
mobileStreamingData = streamingData;
if (desktopStreamingData == null) {
playerResponse = mobilePlayerResponse;
}
}
}
private void fetchVideoInfoPage() throws ParsingException, ReCaptchaException, IOException {
getStsFromPlayerJs();
final String videoInfoUrl = getVideoInfoUrl(getId(), sts);
final String infoPageResponse = NewPipe.getDownloader()
.get(videoInfoUrl, getExtractorLocalization()).responseBody();
videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
try {
playerResponse = JsonParser.object().from(videoInfoPage.get("player_response"));
} catch (final JsonParserException e) {
throw new ParsingException(
"Could not parse YouTube player response from video info page", e);
/**
* Fetch the desktop API with the {@code signatureTimestamp} and assign the streaming data to
* the {@code desktopStreamingData} JSON object.
* The cipher signatures from the player endpoint without a signatureTimestamp are invalid so
* if the content is protected by signatureCiphers and if signatureTimestamp is not known, we
* need to fetch again the desktop InnerTube API.
*/
private void fetchDesktopJsonPlayerWithSts(final ContentCountry contentCountry,
final Localization localization,
final String videoId)
throws IOException, ExtractionException {
if (sts == null) {
getStsFromPlayerJs();
}
final JsonObject playerResponseWithSignatureTimestamp = getJsonPostResponse(
"player", createPlayerBodyWithSts(
localization, contentCountry, videoId, false, sts),
localization);
if (playerResponseWithSignatureTimestamp.has("streamingData")) {
desktopStreamingData = playerResponseWithSignatureTimestamp.getObject("streamingData");
}
isGetVideoInfoPlayerResponse = true;
}
@Nonnull
private byte[] createPlayerBodyWithSts(final Localization localization,
final ContentCountry contentCountry,
final String videoId) throws ExtractionException,
IOException {
// @formatter:off
return JsonWriter.string(prepareJsonBuilder(localization,
contentCountry)
.value("videoId", videoId)
.object("playbackContext")
.object("contentPlaybackContext")
.value("signatureTimestamp", sts)
.end()
.end()
.done())
/**
* Download again the desktop JSON player as an embed client to bypass some age-restrictions.
* <p>
* We need also to get the {@code signatureTimestamp}, if it isn't known because we don't know
* if the video will have signature ciphers or not.
* </p>
*/
private void fetchDesktopEmbedJsonPlayer(final ContentCountry contentCountry,
final Localization localization,
final String videoId)
throws IOException, ExtractionException {
if (sts == null) {
getStsFromPlayerJs();
}
final JsonObject desktopWebEmbedPlayerResponse = getJsonPostResponse(
"player", createPlayerBodyWithSts(
localization, contentCountry, videoId, true, sts),
localization);
final JsonObject streamingData = desktopWebEmbedPlayerResponse.getObject(
"streamingData");
if (!isNullOrEmpty(streamingData)) {
playerResponse = desktopWebEmbedPlayerResponse;
desktopStreamingData = streamingData;
}
}
/**
* Download the Android mobile JSON player as an embed client to bypass some age-restrictions.
*/
private void fetchAndroidEmbedJsonPlayer(final ContentCountry contentCountry,
final Localization localization,
final String videoId)
throws IOException, ExtractionException {
final byte[] androidMobileEmbedBody = JsonWriter.string(
prepareAndroidMobileEmbedVideoJsonBuilder(localization, contentCountry, videoId)
.done())
.getBytes(UTF_8);
// @formatter:on
final JsonObject androidMobileEmbedPlayerResponse = getJsonMobilePostResponse("player",
androidMobileEmbedBody, contentCountry, localization);
final JsonObject streamingData = androidMobileEmbedPlayerResponse.getObject(
"streamingData");
if (!isNullOrEmpty(streamingData)) {
if (desktopStreamingData == null) {
playerResponse = androidMobileEmbedPlayerResponse;
}
mobileStreamingData = androidMobileEmbedPlayerResponse.getObject("streamingData");
}
}
private void storePlayerJs() throws ParsingException {
try {
// The JavaScript player was not found in any page fetched so far and there is
// nothing cached, so try fetching embedded info.
// Don't provide a video id to get a smaller response (around 9Kb instead of 21 Kb
// with a video)
final String embedUrl = "https://www.youtube.com/embed/";
final String embedPageContent = NewPipe.getDownloader()
.get(embedUrl, getExtractorLocalization()).responseBody();
try {
final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")";
playerJsUrl = Parser.matchGroup1(assetsPattern, embedPageContent)
.replace("\\", "").replace("\"", "");
} catch (final Parser.RegexException ex) {
// playerJsUrl is still available in the file, just somewhere else TODO
// It is ok not to find it, see how that's handled in getDeobfuscationCode()
final Document doc = Jsoup.parse(embedPageContent);
final Elements elems = doc.select("script").attr("name", "player_ias/base");
for (final Element elem : elems) {
if (elem.attr("src").contains("base.js")) {
playerJsUrl = elem.attr("src");
break;
}
}
}
if (playerJsUrl != null) {
if (playerJsUrl.startsWith("//")) {
playerJsUrl = HTTPS + playerJsUrl;
} else if (playerJsUrl.startsWith("/")) {
// Sometimes https://www.youtube.com part has to be added manually
playerJsUrl = HTTPS + "//www.youtube.com" + playerJsUrl;
}
playerCode = NewPipe.getDownloader().get(playerJsUrl, getExtractorLocalization())
.responseBody();
} else {
throw new ExtractionException("Could not extract JS player URL");
}
playerCode = YoutubeJavaScriptExtractor.extractJavaScriptCode();
} catch (final Exception e) {
throw new ParsingException("Could not store JavaScript player", e);
}
}
private boolean isCipherProtectedContent() {
if (streamingData != null) {
if (streamingData.has("adaptiveFormats")) {
final JsonArray adaptiveFormats = streamingData.getArray("adaptiveFormats");
if (desktopStreamingData != null) {
if (desktopStreamingData.has("adaptiveFormats")) {
final JsonArray adaptiveFormats = desktopStreamingData.getArray("adaptiveFormats");
if (!isNullOrEmpty(adaptiveFormats)) {
for (final Object adaptiveFormat : adaptiveFormats) {
final JsonObject adaptiveFormatJsonObject = ((JsonObject) adaptiveFormat);
@ -963,8 +943,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
}
if (streamingData.has("formats")) {
final JsonArray formats = streamingData.getArray("formats");
if (desktopStreamingData.has("formats")) {
final JsonArray formats = desktopStreamingData.getArray("formats");
if (!isNullOrEmpty(formats)) {
for (final Object format : formats) {
final JsonObject formatJsonObject = ((JsonObject) format);
@ -1027,7 +1007,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull
private String getDeobfuscationCode() throws ParsingException {
if (cachedDeobfuscationCode == null) {
if (isNullOrEmpty(playerCode)) throw new ParsingException("playerCode is null");
if (isNullOrEmpty(playerCode)) {
throw new ParsingException("playerCode is null");
}
cachedDeobfuscationCode = loadDeobfuscationCode();
}
@ -1038,7 +1020,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (!isNullOrEmpty(sts)) return;
if (playerCode == null) {
storePlayerJs();
if (playerCode == null) throw new ParsingException("playerCode is null");
if (playerCode == null) {
throw new ParsingException("playerCode is null");
}
}
sts = Parser.matchGroup1(STS_REGEX, playerCode);
}
@ -1114,81 +1098,92 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return videoSecondaryInfoRenderer;
}
@Nonnull
private static String getVideoInfoUrl(final String id, final String sts) {
// TODO: Try parsing embedded_player_response first
return "https://www.youtube.com/get_video_info?" + "video_id=" + id
+ "&eurl=https://youtube.googleapis.com/v/" + id + "&sts=" + sts
+ "&html5=1&c=TVHTML5&cver=6.20180913&hl=en&gl=US";
}
@Nonnull
private Map<String, ItagItem> getItags(final String streamingDataKey,
final ItagItem.ItagType itagTypeWanted)
throws ParsingException {
final ItagItem.ItagType itagTypeWanted) {
final Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();
if (streamingData == null || !streamingData.has(streamingDataKey)) {
if (desktopStreamingData == null && mobileStreamingData == null) {
return urlAndItags;
}
final JsonArray formats = streamingData.getArray(streamingDataKey);
for (int i = 0; i != formats.size(); ++i) {
JsonObject formatData = formats.getObject(i);
int itag = formatData.getInt("itag");
// Use the mobileStreamingData object first because there is no n param and no
// signatureCiphers in streaming URLs of the Android client
urlAndItags.putAll(getStreamsFromStreamingDataKey(
mobileStreamingData, streamingDataKey, itagTypeWanted));
urlAndItags.putAll(getStreamsFromStreamingDataKey(
desktopStreamingData, streamingDataKey, itagTypeWanted));
if (ItagItem.isSupported(itag)) {
try {
final ItagItem itagItem = ItagItem.getItag(itag);
if (itagItem.itagType == itagTypeWanted) {
// Ignore streams that are delivered using YouTube's OTF format,
// as those only work with DASH and not with progressive HTTP.
if (formatData.getString("type", EMPTY_STRING)
.equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")) {
continue;
return urlAndItags;
}
@Nonnull
private Map<String, ItagItem> getStreamsFromStreamingDataKey(
final JsonObject streamingData,
final String streamingDataKey,
final ItagItem.ItagType itagTypeWanted) {
final Map<String, ItagItem> urlAndItagsFromStreamingDataObject = new LinkedHashMap<>();
if (streamingData != null && streamingData.has(streamingDataKey)) {
final JsonArray formats = streamingData.getArray(streamingDataKey);
for (int i = 0; i != formats.size(); ++i) {
JsonObject formatData = formats.getObject(i);
int itag = formatData.getInt("itag");
if (ItagItem.isSupported(itag)) {
try {
final ItagItem itagItem = ItagItem.getItag(itag);
if (itagItem.itagType == itagTypeWanted) {
// Ignore streams that are delivered using YouTube's OTF format,
// as those only work with DASH and not with progressive HTTP.
if (formatData.getString("type", EMPTY_STRING)
.equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")) {
continue;
}
final String streamUrl;
if (formatData.has("url")) {
streamUrl = formatData.getString("url");
} else {
// This url has an obfuscated signature
final String cipherString = formatData.has("cipher")
? formatData.getString("cipher")
: formatData.getString("signatureCipher");
final Map<String, String> cipher = Parser.compatParseMap(
cipherString);
streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "="
+ deobfuscateSignature(cipher.get("s"));
}
final JsonObject initRange = formatData.getObject("initRange");
final JsonObject indexRange = formatData.getObject("indexRange");
final String mimeType = formatData.getString("mimeType", EMPTY_STRING);
final String codec = mimeType.contains("codecs")
? mimeType.split("\"")[1] : EMPTY_STRING;
itagItem.setBitrate(formatData.getInt("bitrate"));
itagItem.setWidth(formatData.getInt("width"));
itagItem.setHeight(formatData.getInt("height"));
itagItem.setInitStart(Integer.parseInt(initRange.getString("start",
"-1")));
itagItem.setInitEnd(Integer.parseInt(initRange.getString("end",
"-1")));
itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start",
"-1")));
itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end",
"-1")));
itagItem.fps = formatData.getInt("fps");
itagItem.setQuality(formatData.getString("quality"));
itagItem.setCodec(codec);
urlAndItagsFromStreamingDataObject.put(streamUrl, itagItem);
}
final String streamUrl;
if (formatData.has("url")) {
streamUrl = formatData.getString("url");
} else {
// This url has an obfuscated signature
final String cipherString = formatData.has("cipher")
? formatData.getString("cipher")
: formatData.getString("signatureCipher");
final Map<String, String> cipher = Parser.compatParseMap(cipherString);
streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "="
+ deobfuscateSignature(cipher.get("s"));
}
final JsonObject initRange = formatData.getObject("initRange");
final JsonObject indexRange = formatData.getObject("indexRange");
final String mimeType = formatData.getString("mimeType", EMPTY_STRING);
final String codec = mimeType.contains("codecs")
? mimeType.split("\"")[1] : EMPTY_STRING;
itagItem.setBitrate(formatData.getInt("bitrate"));
itagItem.setWidth(formatData.getInt("width"));
itagItem.setHeight(formatData.getInt("height"));
itagItem.setInitStart(Integer.parseInt(initRange.getString("start",
"-1")));
itagItem.setInitEnd(Integer.parseInt(initRange.getString("end",
"-1")));
itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start",
"-1")));
itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end",
"-1")));
itagItem.fps = formatData.getInt("fps");
itagItem.setQuality(formatData.getString("quality"));
itagItem.setCodec(codec);
urlAndItags.put(streamUrl, itagItem);
} catch (final UnsupportedEncodingException | ParsingException ignored) {
}
} catch (final UnsupportedEncodingException ignored) {
}
}
}
return urlAndItags;
return urlAndItagsFromStreamingDataObject;
}
@Nonnull
@ -1381,7 +1376,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
/**
* Reset YouTube deobfuscation code.
* Reset YouTube's deobfuscation code.
* <p>
* This is needed for mocks in YouTube stream tests, because when they are ran, the
* {@code signatureTimestamp} is known (the {@code sts} string) so a different body than the
@ -1393,7 +1388,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public static void resetDeobfuscationCode() {
cachedDeobfuscationCode = null;
playerCode = null;
playerJsUrl = null;
sts = null;
YoutubeJavaScriptExtractor.resetJavaScriptCode();
}
}

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View File

@ -41,7 +41,7 @@ import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextAtKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -57,7 +57,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
@Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
// @formatter:off
final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(),
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry())
.value("browseId", "FEtrending")
.done())

8
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java
View File

@ -92,7 +92,7 @@ public class YoutubeMixPlaylistExtractorTest {
@Test
public void getPage() throws Exception {
final byte[] body = JsonWriter.string(prepareJsonBuilder(
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
.value("videoId", VIDEO_ID)
.value("playlistId", "RD" + VIDEO_ID)
@ -176,7 +176,7 @@ public class YoutubeMixPlaylistExtractorTest {
@Test
public void getPage() throws Exception {
final byte[] body = JsonWriter.string(prepareJsonBuilder(
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
.value("videoId", VIDEO_ID)
.value("playlistId", "RD" + VIDEO_ID)
@ -259,7 +259,7 @@ public class YoutubeMixPlaylistExtractorTest {
@Test
public void getPage() throws Exception {
final byte[] body = JsonWriter.string(prepareJsonBuilder(
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
.value("videoId", VIDEO_ID)
.value("playlistId", "RDMM" + VIDEO_ID)
@ -373,7 +373,7 @@ public class YoutubeMixPlaylistExtractorTest {
@Test
public void getPage() throws Exception {
final byte[] body = JsonWriter.string(prepareJsonBuilder(
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
.value("videoId", VIDEO_ID_OF_CHANNEL)
.value("playlistId", "RDCM" + CHANNEL_ID)

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java
View File

@ -27,7 +27,7 @@ public class YoutubeParsingHelperTest {
@Test
public void testAreHardcodedClientVersionAndKeyValid() throws IOException, ExtractionException {
assertTrue("Hardcoded client version and key are not valid anymore",
YoutubeParsingHelper.areHardcodedClientVersionAndKeyValid().orElse(false));
YoutubeParsingHelper.areHardcodedClientVersionAndKeyValid());
}
@Test