diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractor.java index 7b8d83dd6..22631b6fe 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractor.java @@ -60,6 +60,14 @@ public class YoutubeJavaScriptExtractor { return extractJavaScriptCode("d4IGg5dqeO8"); } + /** + * Reset the JavaScript code. It will be fetched again the next time + * {@link #extractJavaScriptCode()} or {@link #extractJavaScriptCode(String)} is called. + */ + public static void resetJavaScriptCode() { + cachedJavaScriptCode = null; + } + private static String extractJavaScriptUrl(final String videoId) throws ParsingException { try { final String embedUrl = "https://www.youtube.com/embed/" + videoId; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index ed34a2507..190bbbe16 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -66,15 +66,15 @@ public class YoutubeParsingHelper { public static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/"; - private static final String HARDCODED_CLIENT_VERSION = "2.20210701.00.00"; + private static final String HARDCODED_CLIENT_VERSION = "2.20210728.00.00"; private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; private static final String MOBILE_YOUTUBE_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w"; - private static final String MOBILE_YOUTUBE_CLIENT_VERSION = "16.25.37"; + private static final String MOBILE_YOUTUBE_CLIENT_VERSION = "16.29.38"; private static String clientVersion; private static String key; private static final String[] HARDCODED_YOUTUBE_MUSIC_KEY = - {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "1.20210628.00.00"}; + {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "1.20210726.00.01"}; private static String[] youtubeMusicKey; private static boolean keyAndVersionExtracted = false; @@ -309,10 +309,10 @@ public class YoutubeParsingHelper { } } - public static Optional areHardcodedClientVersionAndKeyValid() + public static boolean areHardcodedClientVersionAndKeyValid() throws IOException, ExtractionException { if (hardcodedClientVersionAndKeyValid.isPresent()) { - return hardcodedClientVersionAndKeyValid; + return hardcodedClientVersionAndKeyValid.get(); } // @formatter:off final byte[] body = JsonWriter.string() @@ -344,8 +344,9 @@ public class YoutubeParsingHelper { final String responseBody = response.responseBody(); final int responseCode = response.responseCode(); - return hardcodedClientVersionAndKeyValid = Optional.of(responseBody.length() > 5000 + hardcodedClientVersionAndKeyValid = Optional.of(responseBody.length() > 5000 && responseCode == 200); // Ensure to have a valid response + return hardcodedClientVersionAndKeyValid.get(); } private static void extractClientVersionAndKey() throws IOException, ExtractionException { @@ -425,7 +426,7 @@ public class YoutubeParsingHelper { */ public static String getClientVersion() throws IOException, ExtractionException { if (!isNullOrEmpty(clientVersion)) return clientVersion; - if (areHardcodedClientVersionAndKeyValid().orElse(false)) { + if (areHardcodedClientVersionAndKeyValid()) { return clientVersion = HARDCODED_CLIENT_VERSION; } @@ -438,7 +439,7 @@ public class YoutubeParsingHelper { */ public static String getKey() throws IOException, ExtractionException { if (!isNullOrEmpty(key)) return key; - if (areHardcodedClientVersionAndKeyValid().orElse(false)) { + if (areHardcodedClientVersionAndKeyValid()) { return key = HARDCODED_KEY; } @@ -799,10 +800,9 @@ public class YoutubeParsingHelper { } @Nonnull - public static JsonBuilder prepareJsonBuilder(@Nonnull final Localization - localization, - @Nonnull final ContentCountry - contentCountry) + public static JsonBuilder prepareDesktopJsonBuilder( + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry) throws IOException, ExtractionException { // @formatter:off return JsonObject.builder() @@ -823,10 +823,9 @@ public class YoutubeParsingHelper { } @Nonnull - public static JsonBuilder prepareMobileJsonBuilder(@Nonnull final Localization - localization, - @Nonnull final ContentCountry - contentCountry) { + public static JsonBuilder prepareAndroidMobileJsonBuilder( + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry) { // @formatter:off return JsonObject.builder() .object("context") @@ -845,6 +844,95 @@ public class YoutubeParsingHelper { // @formatter:on } + @Nonnull + public static JsonBuilder prepareDesktopEmbedVideoJsonBuilder( + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId) throws IOException, ExtractionException { + // @formatter:off + return JsonObject.builder() + .object("context") + .object("client") + .value("hl", localization.getLocalizationCode()) + .value("gl", contentCountry.getCountryCode()) + .value("clientName", "WEB") + .value("clientVersion", getClientVersion()) + .value("clientScreen", "EMBED") + .end() + .object("thirdParty") + .value("embedUrl", "https://www.youtube.com/watch?v=" + videoId) + .end() + .object("user") + // TO DO: provide a way to enable restricted mode with: + // .value("enableSafetyMode", boolean) + .value("lockedSafetyMode", false) + .end() + .end() + .value("videoId", videoId); + // @formatter:on + } + + @Nonnull + public static JsonBuilder prepareAndroidMobileEmbedVideoJsonBuilder( + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId) { + // @formatter:off + return JsonObject.builder() + .object("context") + .object("client") + .value("clientName", "ANDROID") + .value("clientVersion", MOBILE_YOUTUBE_CLIENT_VERSION) + .value("clientScreen", "EMBED") + .value("hl", localization.getLocalizationCode()) + .value("gl", contentCountry.getCountryCode()) + .end() + .object("thirdParty") + .value("embedUrl", "https://www.youtube.com/watch?v=" + videoId) + .end() + .object("user") + // TO DO: provide a way to enable restricted mode with: + // .value("enableSafetyMode", boolean) + .value("lockedSafetyMode", false) + .end() + .end() + .value("videoId", videoId); + // @formatter:on + } + + @Nonnull + public static byte[] createPlayerBodyWithSts(final Localization localization, + final ContentCountry contentCountry, + final String videoId, + final boolean withThirdParty, + @Nullable final String sts) + throws IOException, ExtractionException { + if (withThirdParty) { + // @formatter:off + return JsonWriter.string(prepareDesktopEmbedVideoJsonBuilder(localization, contentCountry, videoId) + .object("playbackContext") + .object("contentPlaybackContext") + .value("signatureTimestamp", sts) + .end() + .end() + .done()) + .getBytes(UTF_8); + // @formatter:on + } else { + // @formatter:off + return JsonWriter.string(prepareDesktopJsonBuilder(localization, contentCountry) + .value("videoId", videoId) + .object("playbackContext") + .object("contentPlaybackContext") + .value("signatureTimestamp", sts) + .end() + .end() + .done()) + .getBytes(UTF_8); + // @formatter:on + } + } + /** * Add required headers and cookies to an existing headers Map. * @see #addClientInfoHeaders(Map) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 6480bc8ae..d7ba0712c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -88,8 +88,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { // navigation/resolve_url endpoint of the InnerTube API to get the channel id. Otherwise, // we couldn't get information about the channel associated with this URL, if there is one. if (!channelId[0].equals("channel")) { - final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(), - getExtractorContentCountry()) + final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder( + getExtractorLocalization(), getExtractorContentCountry()) .value("url", "https://www.youtube.com/" + channelPath) .done()) .getBytes(UTF_8); @@ -135,8 +135,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { int level = 0; while (level < 3) { - final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(), - getExtractorContentCountry()) + final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder( + getExtractorLocalization(), getExtractorContentCountry()) .value("browseId", id) .value("params", "EgZ2aWRlb3M%3D") // Equal to videos .done()) @@ -384,7 +384,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final String continuation = continuationEndpoint.getObject("continuationCommand") .getString("token"); - final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(), + final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(), getExtractorContentCountry()) .value("continuation", continuation) .done()) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 524fd201b..d37320dfc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -62,7 +62,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final String videoId = getQueryValue(url, "v"); final String playlistIndexString = getQueryValue(url, "index"); - final JsonBuilder jsonBody = prepareJsonBuilder(localization, + final JsonBuilder jsonBody = prepareDesktopJsonBuilder(localization, getExtractorContentCountry()).value("playlistId", mixPlaylistId); if (videoId != null) { jsonBody.value("videoId", videoId); @@ -174,7 +174,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final String videoId = watchEndpoint.getString("videoId"); final int index = watchEndpoint.getInt("index"); final String params = watchEndpoint.getString("params"); - final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(), + final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(), getExtractorContentCountry()) .value("videoId", videoId) .value("playlistId", playlistId) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index be58b7348..d6daa954b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -44,7 +44,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { final Localization localization = getExtractorLocalization(); - final byte[] body = JsonWriter.string(prepareJsonBuilder(localization, + final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization, getExtractorContentCountry()) .value("browseId", "VL" + getId()) .value("params", "wgYCCAA%3D") // Show unavailable videos @@ -251,8 +251,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { .getObject("continuationCommand") .getString("token"); - final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(), - getExtractorContentCountry()) + final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder( + getExtractorLocalization(), getExtractorContentCountry()) .value("continuation", continuation) .done()) .getBytes(UTF_8); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index be8a9f0bc..746e4964f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -70,7 +70,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { params = ""; } - final JsonBuilder jsonBody = prepareJsonBuilder(localization, + final JsonBuilder jsonBody = prepareDesktopJsonBuilder(localization, getExtractorContentCountry()) .value("query", query); if (!isNullOrEmpty(params)) { @@ -166,7 +166,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); // @formatter:off - final byte[] json = JsonWriter.string(prepareJsonBuilder(localization, + final byte[] json = JsonWriter.string(prepareDesktopJsonBuilder(localization, getExtractorContentCountry()) .value("continuation", page.getId()) .done()) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 5b87f61e8..387ba4f75 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -2,20 +2,14 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; + import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.MetaInfo; -import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException; @@ -25,7 +19,6 @@ import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException; import org.schabi.newpipe.extractor.exceptions.PaidContentException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.PrivateContentException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.ContentCountry; @@ -93,25 +86,20 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nullable private static String cachedDeobfuscationCode = null; @Nullable - private static String playerJsUrl = null; - @Nullable private static String sts = null; @Nullable private static String playerCode = null; - @Nonnull - private final Map videoInfoPage = new HashMap<>(); - private JsonArray initialAjaxJson; - private JsonObject initialData; private JsonObject playerResponse; private JsonObject nextResponse; @Nullable - private JsonObject streamingData; + private JsonObject desktopStreamingData; + @Nullable + private JsonObject mobileStreamingData; private JsonObject videoPrimaryInfoRenderer; private JsonObject videoSecondaryInfoRenderer; private int ageLimit = -1; - private boolean isGetVideoInfoPlayerResponse = false; @Nullable private List subtitles = null; @@ -290,12 +278,17 @@ public class YoutubeStreamExtractor extends StreamExtractor { .getString("lengthSeconds"); return Long.parseLong(duration); } catch (final Exception e) { - try { - final JsonArray adaptiveFormats = streamingData.getArray("adaptiveFormats"); + if (desktopStreamingData != null) { + final JsonArray adaptiveFormats = desktopStreamingData.getArray("adaptiveFormats"); final String durationMs = adaptiveFormats.getObject(0) .getString("approxDurationMs"); return Math.round(Long.parseLong(durationMs) / 1000f); - } catch (final Exception ignored) { + } else if (mobileStreamingData != null) { + final JsonArray adaptiveFormats = mobileStreamingData.getArray("adaptiveFormats"); + final String durationMs = adaptiveFormats.getObject(0) + .getString("approxDurationMs"); + return Math.round(Long.parseLong(durationMs) / 1000f); + } else { throw new ParsingException("Could not get duration", e); } } @@ -484,29 +477,12 @@ public class YoutubeStreamExtractor extends StreamExtractor { public String getDashMpdUrl() throws ParsingException { assertPageFetched(); - try { - String dashManifestUrl; - if (streamingData.isString("dashManifestUrl")) { - return streamingData.getString("dashManifestUrl"); - } else if (videoInfoPage.containsKey("dashmpd")) { - dashManifestUrl = videoInfoPage.get("dashmpd"); - } else { - return ""; - } - - if (!dashManifestUrl.contains("/signature/")) { - String obfuscatedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", - dashManifestUrl); - final String deobfuscatedSig; - - deobfuscatedSig = deobfuscateSignature(obfuscatedSig); - dashManifestUrl = dashManifestUrl.replace("/s/" + obfuscatedSig, - "/signature/" + deobfuscatedSig); - } - - return dashManifestUrl; - } catch (final Exception e) { - throw new ParsingException("Could not get DASH manifest url", e); + if (desktopStreamingData != null) { + return desktopStreamingData.getString("dashManifestUrl"); + } else if (mobileStreamingData != null) { + return mobileStreamingData.getString("dashManifestUrl"); + } else { + return EMPTY_STRING; } } @@ -515,10 +491,12 @@ public class YoutubeStreamExtractor extends StreamExtractor { public String getHlsUrl() throws ParsingException { assertPageFetched(); - try { - return streamingData.getString("hlsManifestUrl"); - } catch (final Exception e) { - throw new ParsingException("Could not get HLS manifest url", e); + if (desktopStreamingData != null) { + return desktopStreamingData.getString("hlsManifestUrl"); + } else if (mobileStreamingData != null) { + return mobileStreamingData.getString("hlsManifestUrl"); + } else { + return EMPTY_STRING; } } @@ -704,7 +682,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { private static final String FORMATS = "formats"; private static final String ADAPTIVE_FORMATS = "adaptiveFormats"; - private static final String HTTPS = "https:"; private static final String DEOBFUSCATION_FUNC_NAME = "deobfuscate"; private static final String[] REGEXES = { @@ -721,7 +698,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { final String videoId = getId(); final Localization localization = getExtractorLocalization(); final ContentCountry contentCountry = getExtractorContentCountry(); - final byte[] body = JsonWriter.string(prepareJsonBuilder(localization, contentCountry) + final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder( + localization, contentCountry) .value("videoId", videoId) .done()) .getBytes(UTF_8); @@ -731,7 +709,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { // API. if (sts != null) { playerResponse = getJsonPostResponse("player", createPlayerBodyWithSts(localization, - contentCountry, videoId), localization); + contentCountry, videoId, false, sts), localization); } else { playerResponse = getJsonPostResponse("player", body, localization); } @@ -740,34 +718,55 @@ public class YoutubeStreamExtractor extends StreamExtractor { // there can be restrictions on the embedded player. // E.g. if a video is age-restricted, the embedded player's playabilityStatus says that // the video cannot be played outside of YouTube, but does not show the original message. - JsonObject youtubePlayerResponse = playerResponse; + final JsonObject youtubePlayerResponse = playerResponse; - if (playerResponse == null || !playerResponse.has("streamingData")) { - // Try to get the player response by fetching video info page - fetchVideoInfoPage(); - } - - if (playerResponse == null && youtubePlayerResponse == null) { + if (playerResponse == null) { throw new ExtractionException("Could not get playerResponse"); - } else if (youtubePlayerResponse == null) { - youtubePlayerResponse = playerResponse; } - final JsonObject playabilityStatus = (playerResponse == null ? youtubePlayerResponse - : playerResponse).getObject("playabilityStatus"); + final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus"); - checkPlayabilityStatus(youtubePlayerResponse, playabilityStatus); + boolean ageRestricted = playabilityStatus.getString("reason", EMPTY_STRING) + .contains("age"); - nextResponse = getJsonPostResponse("next", body, localization); + if (!playerResponse.has("streamingData")) { + try { + fetchDesktopEmbedJsonPlayer(contentCountry, localization, videoId); + } catch (final Exception ignored) { + } + try { + fetchAndroidEmbedJsonPlayer(contentCountry, localization, videoId); + } catch (final Exception ignored) { + } + } - // Workaround for rate limits on web streaming URLs. - // TODO: add ability to deobfuscate the n param of these URLs + if (desktopStreamingData == null && playerResponse.has("streamingData")) { + desktopStreamingData = playerResponse.getObject("streamingData"); + } - // It's not needed to request the mobile API for age-restricted videos - if (!isGetVideoInfoPlayerResponse) { - fetchAndroidMobileJsonPlayer(contentCountry, localization, videoId); + if (desktopStreamingData == null) { + checkPlayabilityStatus(youtubePlayerResponse, playabilityStatus); + } + + if (ageRestricted) { + final byte[] ageRestrictedBody = JsonWriter.string(prepareDesktopEmbedVideoJsonBuilder( + localization, contentCountry, videoId) + .done()) + .getBytes(UTF_8); + nextResponse = getJsonPostResponse("next", ageRestrictedBody, localization); } else { - streamingData = playerResponse.getObject("streamingData"); + nextResponse = getJsonPostResponse("next", body, localization); + } + + if (!ageRestricted) { + try { + fetchAndroidMobileJsonPlayer(contentCountry, localization, videoId); + } catch (final Exception ignored) { + } + } + + if (isCipherProtectedContent()) { + fetchDesktopJsonPlayerWithSts(contentCountry, localization, videoId); } } @@ -826,133 +825,114 @@ public class YoutubeStreamExtractor extends StreamExtractor { } /** - * Fetch the Android Mobile API or fallback to the desktop streams. - * If something went wrong when parsing this API, fallback to the desktop JSON player, fetched - * again if the {@code signatureTimestamp} of the JS player is unknown (because signatures - * without a {@code signatureTimestamp} included in the player request are invalid). + * Fetch the Android Mobile API and assign the streaming data to the mobileStreamingData JSON + * object. */ private void fetchAndroidMobileJsonPlayer(final ContentCountry contentCountry, final Localization localization, - final String videoId) throws ExtractionException, - IOException { - JsonObject mobilePlayerResponse = null; - final byte[] mobileBody = JsonWriter.string(prepareMobileJsonBuilder(localization, - contentCountry) - .value("videoId", videoId) - .done()) + final String videoId) + throws IOException, ExtractionException { + final byte[] mobileBody = JsonWriter.string(prepareAndroidMobileJsonBuilder( + localization, contentCountry) + .value("videoId", videoId) + .done()) .getBytes(UTF_8); - try { - mobilePlayerResponse = getJsonMobilePostResponse("player", mobileBody, - contentCountry, localization); - } catch (final Exception ignored) { - } - if (mobilePlayerResponse != null && mobilePlayerResponse.has("streamingData")) { - final JsonObject mobileStreamingData = mobilePlayerResponse.getObject( - "streamingData"); - if (!isNullOrEmpty(mobileStreamingData)) streamingData = mobileStreamingData; - } else { - // Fallback to the desktop JSON player endpoint + final JsonObject mobilePlayerResponse = getJsonMobilePostResponse("player", + mobileBody, contentCountry, localization); - // The cipher signatures from the player endpoint without a timestamp are invalid so - // download it again only if we didn't have a signatureTimestamp before fetching the - // data of this video (the sts string). - if (sts == null && isCipherProtectedContent()) { - getStsFromPlayerJs(); - final JsonObject playerResponseWithSignatureTimestamp = getJsonPostResponse( - "player", createPlayerBodyWithSts(localization, contentCountry, videoId), - localization); - if (playerResponseWithSignatureTimestamp.has("streamingData")) { - streamingData = playerResponseWithSignatureTimestamp.getObject( - "streamingData"); - } - } else { - streamingData = playerResponse.getObject("streamingData"); + final JsonObject streamingData = mobilePlayerResponse.getObject("streamingData"); + if (!isNullOrEmpty(streamingData)) { + mobileStreamingData = streamingData; + if (desktopStreamingData == null) { + playerResponse = mobilePlayerResponse; } } } - private void fetchVideoInfoPage() throws ParsingException, ReCaptchaException, IOException { - getStsFromPlayerJs(); - final String videoInfoUrl = getVideoInfoUrl(getId(), sts); - final String infoPageResponse = NewPipe.getDownloader() - .get(videoInfoUrl, getExtractorLocalization()).responseBody(); - videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse)); - - try { - playerResponse = JsonParser.object().from(videoInfoPage.get("player_response")); - } catch (final JsonParserException e) { - throw new ParsingException( - "Could not parse YouTube player response from video info page", e); + /** + * Fetch the desktop API with the {@code signatureTimestamp} and assign the streaming data to + * the {@code desktopStreamingData} JSON object. + * The cipher signatures from the player endpoint without a signatureTimestamp are invalid so + * if the content is protected by signatureCiphers and if signatureTimestamp is not known, we + * need to fetch again the desktop InnerTube API. + */ + private void fetchDesktopJsonPlayerWithSts(final ContentCountry contentCountry, + final Localization localization, + final String videoId) + throws IOException, ExtractionException { + if (sts == null) { + getStsFromPlayerJs(); + } + final JsonObject playerResponseWithSignatureTimestamp = getJsonPostResponse( + "player", createPlayerBodyWithSts( + localization, contentCountry, videoId, false, sts), + localization); + if (playerResponseWithSignatureTimestamp.has("streamingData")) { + desktopStreamingData = playerResponseWithSignatureTimestamp.getObject("streamingData"); } - isGetVideoInfoPlayerResponse = true; } - @Nonnull - private byte[] createPlayerBodyWithSts(final Localization localization, - final ContentCountry contentCountry, - final String videoId) throws ExtractionException, - IOException { - // @formatter:off - return JsonWriter.string(prepareJsonBuilder(localization, - contentCountry) - .value("videoId", videoId) - .object("playbackContext") - .object("contentPlaybackContext") - .value("signatureTimestamp", sts) - .end() - .end() - .done()) + /** + * Download again the desktop JSON player as an embed client to bypass some age-restrictions. + *

+ * We need also to get the {@code signatureTimestamp}, if it isn't known because we don't know + * if the video will have signature ciphers or not. + *

+ */ + private void fetchDesktopEmbedJsonPlayer(final ContentCountry contentCountry, + final Localization localization, + final String videoId) + throws IOException, ExtractionException { + if (sts == null) { + getStsFromPlayerJs(); + } + final JsonObject desktopWebEmbedPlayerResponse = getJsonPostResponse( + "player", createPlayerBodyWithSts( + localization, contentCountry, videoId, true, sts), + localization); + final JsonObject streamingData = desktopWebEmbedPlayerResponse.getObject( + "streamingData"); + if (!isNullOrEmpty(streamingData)) { + playerResponse = desktopWebEmbedPlayerResponse; + desktopStreamingData = streamingData; + } + } + + /** + * Download the Android mobile JSON player as an embed client to bypass some age-restrictions. + */ + private void fetchAndroidEmbedJsonPlayer(final ContentCountry contentCountry, + final Localization localization, + final String videoId) + throws IOException, ExtractionException { + final byte[] androidMobileEmbedBody = JsonWriter.string( + prepareAndroidMobileEmbedVideoJsonBuilder(localization, contentCountry, videoId) + .done()) .getBytes(UTF_8); - // @formatter:on + final JsonObject androidMobileEmbedPlayerResponse = getJsonMobilePostResponse("player", + androidMobileEmbedBody, contentCountry, localization); + final JsonObject streamingData = androidMobileEmbedPlayerResponse.getObject( + "streamingData"); + if (!isNullOrEmpty(streamingData)) { + if (desktopStreamingData == null) { + playerResponse = androidMobileEmbedPlayerResponse; + } + mobileStreamingData = androidMobileEmbedPlayerResponse.getObject("streamingData"); + } } private void storePlayerJs() throws ParsingException { try { - // The JavaScript player was not found in any page fetched so far and there is - // nothing cached, so try fetching embedded info. - // Don't provide a video id to get a smaller response (around 9Kb instead of 21 Kb - // with a video) - final String embedUrl = "https://www.youtube.com/embed/"; - final String embedPageContent = NewPipe.getDownloader() - .get(embedUrl, getExtractorLocalization()).responseBody(); - try { - final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")"; - playerJsUrl = Parser.matchGroup1(assetsPattern, embedPageContent) - .replace("\\", "").replace("\"", ""); - } catch (final Parser.RegexException ex) { - // playerJsUrl is still available in the file, just somewhere else TODO - // It is ok not to find it, see how that's handled in getDeobfuscationCode() - final Document doc = Jsoup.parse(embedPageContent); - final Elements elems = doc.select("script").attr("name", "player_ias/base"); - for (final Element elem : elems) { - if (elem.attr("src").contains("base.js")) { - playerJsUrl = elem.attr("src"); - break; - } - } - } - - if (playerJsUrl != null) { - if (playerJsUrl.startsWith("//")) { - playerJsUrl = HTTPS + playerJsUrl; - } else if (playerJsUrl.startsWith("/")) { - // Sometimes https://www.youtube.com part has to be added manually - playerJsUrl = HTTPS + "//www.youtube.com" + playerJsUrl; - } - playerCode = NewPipe.getDownloader().get(playerJsUrl, getExtractorLocalization()) - .responseBody(); - } else { - throw new ExtractionException("Could not extract JS player URL"); - } + playerCode = YoutubeJavaScriptExtractor.extractJavaScriptCode(); } catch (final Exception e) { throw new ParsingException("Could not store JavaScript player", e); } } private boolean isCipherProtectedContent() { - if (streamingData != null) { - if (streamingData.has("adaptiveFormats")) { - final JsonArray adaptiveFormats = streamingData.getArray("adaptiveFormats"); + if (desktopStreamingData != null) { + if (desktopStreamingData.has("adaptiveFormats")) { + final JsonArray adaptiveFormats = desktopStreamingData.getArray("adaptiveFormats"); if (!isNullOrEmpty(adaptiveFormats)) { for (final Object adaptiveFormat : adaptiveFormats) { final JsonObject adaptiveFormatJsonObject = ((JsonObject) adaptiveFormat); @@ -963,8 +943,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } } - if (streamingData.has("formats")) { - final JsonArray formats = streamingData.getArray("formats"); + if (desktopStreamingData.has("formats")) { + final JsonArray formats = desktopStreamingData.getArray("formats"); if (!isNullOrEmpty(formats)) { for (final Object format : formats) { final JsonObject formatJsonObject = ((JsonObject) format); @@ -1027,7 +1007,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull private String getDeobfuscationCode() throws ParsingException { if (cachedDeobfuscationCode == null) { - if (isNullOrEmpty(playerCode)) throw new ParsingException("playerCode is null"); + if (isNullOrEmpty(playerCode)) { + throw new ParsingException("playerCode is null"); + } cachedDeobfuscationCode = loadDeobfuscationCode(); } @@ -1038,7 +1020,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { if (!isNullOrEmpty(sts)) return; if (playerCode == null) { storePlayerJs(); - if (playerCode == null) throw new ParsingException("playerCode is null"); + if (playerCode == null) { + throw new ParsingException("playerCode is null"); + } } sts = Parser.matchGroup1(STS_REGEX, playerCode); } @@ -1114,81 +1098,92 @@ public class YoutubeStreamExtractor extends StreamExtractor { return videoSecondaryInfoRenderer; } - @Nonnull - private static String getVideoInfoUrl(final String id, final String sts) { - // TODO: Try parsing embedded_player_response first - return "https://www.youtube.com/get_video_info?" + "video_id=" + id - + "&eurl=https://youtube.googleapis.com/v/" + id + "&sts=" + sts - + "&html5=1&c=TVHTML5&cver=6.20180913&hl=en&gl=US"; - } - @Nonnull private Map getItags(final String streamingDataKey, - final ItagItem.ItagType itagTypeWanted) - throws ParsingException { + final ItagItem.ItagType itagTypeWanted) { final Map urlAndItags = new LinkedHashMap<>(); - if (streamingData == null || !streamingData.has(streamingDataKey)) { + if (desktopStreamingData == null && mobileStreamingData == null) { return urlAndItags; } - final JsonArray formats = streamingData.getArray(streamingDataKey); - for (int i = 0; i != formats.size(); ++i) { - JsonObject formatData = formats.getObject(i); - int itag = formatData.getInt("itag"); + // Use the mobileStreamingData object first because there is no n param and no + // signatureCiphers in streaming URLs of the Android client + urlAndItags.putAll(getStreamsFromStreamingDataKey( + mobileStreamingData, streamingDataKey, itagTypeWanted)); + urlAndItags.putAll(getStreamsFromStreamingDataKey( + desktopStreamingData, streamingDataKey, itagTypeWanted)); - if (ItagItem.isSupported(itag)) { - try { - final ItagItem itagItem = ItagItem.getItag(itag); - if (itagItem.itagType == itagTypeWanted) { - // Ignore streams that are delivered using YouTube's OTF format, - // as those only work with DASH and not with progressive HTTP. - if (formatData.getString("type", EMPTY_STRING) - .equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")) { - continue; + return urlAndItags; + } + + @Nonnull + private Map getStreamsFromStreamingDataKey( + final JsonObject streamingData, + final String streamingDataKey, + final ItagItem.ItagType itagTypeWanted) { + + final Map urlAndItagsFromStreamingDataObject = new LinkedHashMap<>(); + if (streamingData != null && streamingData.has(streamingDataKey)) { + final JsonArray formats = streamingData.getArray(streamingDataKey); + for (int i = 0; i != formats.size(); ++i) { + JsonObject formatData = formats.getObject(i); + int itag = formatData.getInt("itag"); + + if (ItagItem.isSupported(itag)) { + try { + final ItagItem itagItem = ItagItem.getItag(itag); + if (itagItem.itagType == itagTypeWanted) { + // Ignore streams that are delivered using YouTube's OTF format, + // as those only work with DASH and not with progressive HTTP. + if (formatData.getString("type", EMPTY_STRING) + .equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")) { + continue; + } + + final String streamUrl; + if (formatData.has("url")) { + streamUrl = formatData.getString("url"); + } else { + // This url has an obfuscated signature + final String cipherString = formatData.has("cipher") + ? formatData.getString("cipher") + : formatData.getString("signatureCipher"); + final Map cipher = Parser.compatParseMap( + cipherString); + streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + + deobfuscateSignature(cipher.get("s")); + } + + final JsonObject initRange = formatData.getObject("initRange"); + final JsonObject indexRange = formatData.getObject("indexRange"); + final String mimeType = formatData.getString("mimeType", EMPTY_STRING); + final String codec = mimeType.contains("codecs") + ? mimeType.split("\"")[1] : EMPTY_STRING; + + itagItem.setBitrate(formatData.getInt("bitrate")); + itagItem.setWidth(formatData.getInt("width")); + itagItem.setHeight(formatData.getInt("height")); + itagItem.setInitStart(Integer.parseInt(initRange.getString("start", + "-1"))); + itagItem.setInitEnd(Integer.parseInt(initRange.getString("end", + "-1"))); + itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start", + "-1"))); + itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end", + "-1"))); + itagItem.fps = formatData.getInt("fps"); + itagItem.setQuality(formatData.getString("quality")); + itagItem.setCodec(codec); + + urlAndItagsFromStreamingDataObject.put(streamUrl, itagItem); } - - final String streamUrl; - if (formatData.has("url")) { - streamUrl = formatData.getString("url"); - } else { - // This url has an obfuscated signature - final String cipherString = formatData.has("cipher") - ? formatData.getString("cipher") - : formatData.getString("signatureCipher"); - final Map cipher = Parser.compatParseMap(cipherString); - streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" - + deobfuscateSignature(cipher.get("s")); - } - - final JsonObject initRange = formatData.getObject("initRange"); - final JsonObject indexRange = formatData.getObject("indexRange"); - final String mimeType = formatData.getString("mimeType", EMPTY_STRING); - final String codec = mimeType.contains("codecs") - ? mimeType.split("\"")[1] : EMPTY_STRING; - - itagItem.setBitrate(formatData.getInt("bitrate")); - itagItem.setWidth(formatData.getInt("width")); - itagItem.setHeight(formatData.getInt("height")); - itagItem.setInitStart(Integer.parseInt(initRange.getString("start", - "-1"))); - itagItem.setInitEnd(Integer.parseInt(initRange.getString("end", - "-1"))); - itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start", - "-1"))); - itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end", - "-1"))); - itagItem.fps = formatData.getInt("fps"); - itagItem.setQuality(formatData.getString("quality")); - itagItem.setCodec(codec); - - urlAndItags.put(streamUrl, itagItem); + } catch (final UnsupportedEncodingException | ParsingException ignored) { } - } catch (final UnsupportedEncodingException ignored) { } } } - return urlAndItags; + return urlAndItagsFromStreamingDataObject; } @Nonnull @@ -1381,7 +1376,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } /** - * Reset YouTube deobfuscation code. + * Reset YouTube's deobfuscation code. *

* This is needed for mocks in YouTube stream tests, because when they are ran, the * {@code signatureTimestamp} is known (the {@code sts} string) so a different body than the @@ -1393,7 +1388,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { public static void resetDeobfuscationCode() { cachedDeobfuscationCode = null; playerCode = null; - playerJsUrl = null; sts = null; + YoutubeJavaScriptExtractor.resetJavaScriptCode(); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index b9e826f8d..b038cd923 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -41,7 +41,7 @@ import javax.annotation.Nonnull; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextAtKey; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -57,7 +57,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor { @Override public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { // @formatter:off - final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(), + final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(), getExtractorContentCountry()) .value("browseId", "FEtrending") .done()) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java index 43bce9bfa..869341360 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -92,7 +92,7 @@ public class YoutubeMixPlaylistExtractorTest { @Test public void getPage() throws Exception { - final byte[] body = JsonWriter.string(prepareJsonBuilder( + final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder( NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry()) .value("videoId", VIDEO_ID) .value("playlistId", "RD" + VIDEO_ID) @@ -176,7 +176,7 @@ public class YoutubeMixPlaylistExtractorTest { @Test public void getPage() throws Exception { - final byte[] body = JsonWriter.string(prepareJsonBuilder( + final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder( NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry()) .value("videoId", VIDEO_ID) .value("playlistId", "RD" + VIDEO_ID) @@ -259,7 +259,7 @@ public class YoutubeMixPlaylistExtractorTest { @Test public void getPage() throws Exception { - final byte[] body = JsonWriter.string(prepareJsonBuilder( + final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder( NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry()) .value("videoId", VIDEO_ID) .value("playlistId", "RDMM" + VIDEO_ID) @@ -373,7 +373,7 @@ public class YoutubeMixPlaylistExtractorTest { @Test public void getPage() throws Exception { - final byte[] body = JsonWriter.string(prepareJsonBuilder( + final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder( NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry()) .value("videoId", VIDEO_ID_OF_CHANNEL) .value("playlistId", "RDCM" + CHANNEL_ID) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java index 3c6c6e354..98d25e0b5 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java @@ -27,7 +27,7 @@ public class YoutubeParsingHelperTest { @Test public void testAreHardcodedClientVersionAndKeyValid() throws IOException, ExtractionException { assertTrue("Hardcoded client version and key are not valid anymore", - YoutubeParsingHelper.areHardcodedClientVersionAndKeyValid().orElse(false)); + YoutubeParsingHelper.areHardcodedClientVersionAndKeyValid()); } @Test