Rebase + some code improvements + fix extraction of age-restricted videos + update clients version

Here is now the requests which will be made by the `onFetchPage` method of `YoutubeStreamExtractor`:

- the desktop API is fetched.

If there is no streaming data, the desktop player API with the embed client screen will be fetched (and also the player code), then the Android mobile API.
- if there is no streaming data, a `ContentNotAvailableException` will be thrown by using the message provided in playability status

If the video is age restricted, a request to the next endpoint of the desktop player with the embed client screen will be sent.
Otherwise, the next endpoint will be fetched normally, if the content is available.

If the video is not age-restricted, a request to the player endpoint of the Android mobile API will be made.

We can get more streams by using the Android mobile API but some streams may be not available on this API, so the streaming data of the Android mobile API will be first used to get itags and then the streaming data of the desktop internal API will be used.
If the parsing of the Android mobile API went wrong, only the streams of the desktop API will be used.

Other code changes:

- `prepareJsonBuilder` in `YoutubeParsingHelper` was renamed to `prepareDesktopJsonBuilder`
- `prepareMobileJsonBuilder` in `YoutubeParsingHelper` was renamed to `prepareAndroidMobileJsonBuilder`
- two new methods in `YoutubeParsingHelper` were added: `prepareDesktopEmbedVideoJsonBuilder` and `prepareAndroidMobileEmbedVideoJsonBuilder`
- `createPlayerBodyWithSts` is now public and was moved to `YoutubeParsingHelper`
- a new method in `YoutubeJavaScriptExtractor` was added: `resetJavaScriptCode`, which was needed for the method `resetDebofuscationCode` of `YoutubeStreamExtractor`
- `areHardcodedClientVersionAndKeyValid` in `YoutubeParsingHelper` returns now a `boolean` instead of an `Optional<Boolean>`
- the `fetchVideoInfoPage` method of `YoutubeStreamExtractor` was removed because YouTube returns now 404 for every client with the `get_video_info` page
- some unused objects and some warnings in `YoutubeStreamExtractor` were removed and fixed

Co-authored-by: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com>

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptExtractor.java

@@ -60,6 +60,14 @@ public class YoutubeJavaScriptExtractor {
         return extractJavaScriptCode("d4IGg5dqeO8");
     }
 
+    /**
+     * Reset the JavaScript code. It will be fetched again the next time
+     * {@link #extractJavaScriptCode()} or {@link #extractJavaScriptCode(String)} is called.
+     */
+    public static void resetJavaScriptCode() {
+        cachedJavaScriptCode = null;
+    }
+
     private static String extractJavaScriptUrl(final String videoId) throws ParsingException {
         try {
             final String embedUrl = "https://www.youtube.com/embed/" + videoId;

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -66,15 +66,15 @@ public class YoutubeParsingHelper {
 
     public static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/";
 
-    private static final String HARDCODED_CLIENT_VERSION = "2.20210701.00.00";
+    private static final String HARDCODED_CLIENT_VERSION = "2.20210728.00.00";
     private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
     private static final String MOBILE_YOUTUBE_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w";
-    private static final String MOBILE_YOUTUBE_CLIENT_VERSION = "16.25.37";
+    private static final String MOBILE_YOUTUBE_CLIENT_VERSION = "16.29.38";
     private static String clientVersion;
     private static String key;
 
     private static final String[] HARDCODED_YOUTUBE_MUSIC_KEY =
-            {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "1.20210628.00.00"};
+            {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "1.20210726.00.01"};
     private static String[] youtubeMusicKey;
 
     private static boolean keyAndVersionExtracted = false;
@@ -309,10 +309,10 @@ public class YoutubeParsingHelper {
         }
     }
 
-    public static Optional<Boolean> areHardcodedClientVersionAndKeyValid()
+    public static boolean areHardcodedClientVersionAndKeyValid()
             throws IOException, ExtractionException {
         if (hardcodedClientVersionAndKeyValid.isPresent()) {
-            return hardcodedClientVersionAndKeyValid;
+            return hardcodedClientVersionAndKeyValid.get();
         }
         // @formatter:off
         final byte[] body = JsonWriter.string()
@@ -344,8 +344,9 @@ public class YoutubeParsingHelper {
         final String responseBody = response.responseBody();
         final int responseCode = response.responseCode();
 
-        return hardcodedClientVersionAndKeyValid = Optional.of(responseBody.length() > 5000
+        hardcodedClientVersionAndKeyValid = Optional.of(responseBody.length() > 5000
                 && responseCode == 200); // Ensure to have a valid response
+        return hardcodedClientVersionAndKeyValid.get();
     }
 
     private static void extractClientVersionAndKey() throws IOException, ExtractionException {
@@ -425,7 +426,7 @@ public class YoutubeParsingHelper {
      */
     public static String getClientVersion() throws IOException, ExtractionException {
         if (!isNullOrEmpty(clientVersion)) return clientVersion;
-        if (areHardcodedClientVersionAndKeyValid().orElse(false)) {
+        if (areHardcodedClientVersionAndKeyValid()) {
             return clientVersion = HARDCODED_CLIENT_VERSION;
         }
 
@@ -438,7 +439,7 @@ public class YoutubeParsingHelper {
      */
     public static String getKey() throws IOException, ExtractionException {
         if (!isNullOrEmpty(key)) return key;
-        if (areHardcodedClientVersionAndKeyValid().orElse(false)) {
+        if (areHardcodedClientVersionAndKeyValid()) {
             return key = HARDCODED_KEY;
         }
 
@@ -799,10 +800,9 @@ public class YoutubeParsingHelper {
     }
 
     @Nonnull
-    public static JsonBuilder<JsonObject> prepareJsonBuilder(@Nonnull final Localization
+    public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
-                                                                         localization,
+            @Nonnull final Localization localization,
-                                                             @Nonnull final ContentCountry
+            @Nonnull final ContentCountry contentCountry)
-                                                                     contentCountry)
             throws IOException, ExtractionException {
         // @formatter:off
         return JsonObject.builder()
@@ -823,10 +823,9 @@ public class YoutubeParsingHelper {
     }
 
     @Nonnull
-    public static JsonBuilder<JsonObject> prepareMobileJsonBuilder(@Nonnull final Localization
+    public static JsonBuilder<JsonObject> prepareAndroidMobileJsonBuilder(
-                                                                               localization,
+            @Nonnull final Localization localization,
-                                                                   @Nonnull final ContentCountry
+            @Nonnull final ContentCountry contentCountry) {
-                                                                           contentCountry) {
         // @formatter:off
         return JsonObject.builder()
                 .object("context")
@@ -845,6 +844,95 @@ public class YoutubeParsingHelper {
         // @formatter:on
     }
 
+    @Nonnull
+    public static JsonBuilder<JsonObject> prepareDesktopEmbedVideoJsonBuilder(
+            @Nonnull final Localization localization,
+            @Nonnull final ContentCountry contentCountry,
+            @Nonnull final String videoId) throws IOException, ExtractionException {
+        // @formatter:off
+        return JsonObject.builder()
+                .object("context")
+                    .object("client")
+                        .value("hl", localization.getLocalizationCode())
+                        .value("gl", contentCountry.getCountryCode())
+                        .value("clientName", "WEB")
+                        .value("clientVersion", getClientVersion())
+                        .value("clientScreen", "EMBED")
+                    .end()
+                    .object("thirdParty")
+                        .value("embedUrl", "https://www.youtube.com/watch?v=" + videoId)
+                    .end()
+                    .object("user")
+                        // TO DO: provide a way to enable restricted mode with:
+                        // .value("enableSafetyMode", boolean)
+                        .value("lockedSafetyMode", false)
+                    .end()
+                .end()
+                .value("videoId", videoId);
+        // @formatter:on
+    }
+
+    @Nonnull
+    public static JsonBuilder<JsonObject> prepareAndroidMobileEmbedVideoJsonBuilder(
+            @Nonnull final Localization localization,
+            @Nonnull final ContentCountry contentCountry,
+            @Nonnull final String videoId) {
+        // @formatter:off
+        return JsonObject.builder()
+                .object("context")
+                    .object("client")
+                        .value("clientName", "ANDROID")
+                        .value("clientVersion", MOBILE_YOUTUBE_CLIENT_VERSION)
+                        .value("clientScreen", "EMBED")
+                        .value("hl", localization.getLocalizationCode())
+                        .value("gl", contentCountry.getCountryCode())
+                    .end()
+                    .object("thirdParty")
+                        .value("embedUrl", "https://www.youtube.com/watch?v=" + videoId)
+                    .end()
+                    .object("user")
+                        // TO DO: provide a way to enable restricted mode with:
+                        // .value("enableSafetyMode", boolean)
+                        .value("lockedSafetyMode", false)
+                    .end()
+                .end()
+                .value("videoId", videoId);
+        // @formatter:on
+    }
+
+    @Nonnull
+    public static byte[] createPlayerBodyWithSts(final Localization localization,
+                                                 final ContentCountry contentCountry,
+                                                 final String videoId,
+                                                 final boolean withThirdParty,
+                                                 @Nullable final String sts)
+            throws IOException, ExtractionException {
+        if (withThirdParty) {
+            // @formatter:off
+            return JsonWriter.string(prepareDesktopEmbedVideoJsonBuilder(localization, contentCountry, videoId)
+                    .object("playbackContext")
+                        .object("contentPlaybackContext")
+                            .value("signatureTimestamp", sts)
+                        .end()
+                    .end()
+                    .done())
+                    .getBytes(UTF_8);
+            // @formatter:on
+        } else {
+            // @formatter:off
+            return JsonWriter.string(prepareDesktopJsonBuilder(localization, contentCountry)
+                    .value("videoId", videoId)
+                    .object("playbackContext")
+                        .object("contentPlaybackContext")
+                            .value("signatureTimestamp", sts)
+                        .end()
+                    .end()
+                    .done())
+                    .getBytes(UTF_8);
+            // @formatter:on
+        }
+    }
+
     /**
      * Add required headers and cookies to an existing headers Map.
      * @see #addClientInfoHeaders(Map)

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -88,8 +88,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
         // navigation/resolve_url endpoint of the InnerTube API to get the channel id. Otherwise,
         // we couldn't get information about the channel associated with this URL, if there is one.
         if (!channelId[0].equals("channel")) {
-            final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(),
+            final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
-                    getExtractorContentCountry())
+                    getExtractorLocalization(), getExtractorContentCountry())
                     .value("url", "https://www.youtube.com/" + channelPath)
                     .done())
                     .getBytes(UTF_8);
@@ -135,8 +135,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
 
         int level = 0;
         while (level < 3) {
-            final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(),
+            final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
-                    getExtractorContentCountry())
+                    getExtractorLocalization(), getExtractorContentCountry())
                     .value("browseId", id)
                     .value("params", "EgZ2aWRlb3M%3D") // Equal to videos
                     .done())
@@ -384,7 +384,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
         final String continuation = continuationEndpoint.getObject("continuationCommand")
                 .getString("token");
 
-        final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(),
+        final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
                 getExtractorContentCountry())
                 .value("continuation", continuation)
                 .done())

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java

@@ -62,7 +62,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
         final String videoId = getQueryValue(url, "v");
         final String playlistIndexString = getQueryValue(url, "index");
 
-        final JsonBuilder<JsonObject> jsonBody = prepareJsonBuilder(localization,
+        final JsonBuilder<JsonObject> jsonBody = prepareDesktopJsonBuilder(localization,
                 getExtractorContentCountry()).value("playlistId", mixPlaylistId);
         if (videoId != null) {
             jsonBody.value("videoId", videoId);
@@ -174,7 +174,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
         final String videoId = watchEndpoint.getString("videoId");
         final int index = watchEndpoint.getInt("index");
         final String params = watchEndpoint.getString("params");
-        final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(),
+        final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
                 getExtractorContentCountry())
                 .value("videoId", videoId)
                 .value("playlistId", playlistId)

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -44,7 +44,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
             ExtractionException {
         final Localization localization = getExtractorLocalization();
-        final byte[] body = JsonWriter.string(prepareJsonBuilder(localization,
+        final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization,
                 getExtractorContentCountry())
                 .value("browseId", "VL" + getId())
                 .value("params", "wgYCCAA%3D") // Show unavailable videos
@@ -251,8 +251,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
                     .getObject("continuationCommand")
                     .getString("token");
 
-            final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(),
+            final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
-                    getExtractorContentCountry())
+                    getExtractorLocalization(), getExtractorContentCountry())
                     .value("continuation", continuation)
                     .done())
                     .getBytes(UTF_8);

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java

@@ -70,7 +70,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
             params = "";
         }
 
-        final JsonBuilder<JsonObject> jsonBody = prepareJsonBuilder(localization,
+        final JsonBuilder<JsonObject> jsonBody = prepareDesktopJsonBuilder(localization,
                 getExtractorContentCountry())
                 .value("query", query);
         if (!isNullOrEmpty(params)) {
@@ -166,7 +166,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
         final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
 
         // @formatter:off
-        final byte[] json = JsonWriter.string(prepareJsonBuilder(localization,
+        final byte[] json = JsonWriter.string(prepareDesktopJsonBuilder(localization,
                 getExtractorContentCountry())
                 .value("continuation", page.getId())
                 .done())

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -2,20 +2,14 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
-import com.grack.nanojson.JsonParser;
-import com.grack.nanojson.JsonParserException;
 import com.grack.nanojson.JsonWriter;
 
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
-import org.jsoup.select.Elements;
 import org.mozilla.javascript.Context;
 import org.mozilla.javascript.Function;
 import org.mozilla.javascript.ScriptableObject;
+
 import org.schabi.newpipe.extractor.MediaFormat;
 import org.schabi.newpipe.extractor.MetaInfo;
-import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException;
@@ -25,7 +19,6 @@ import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException;
 import org.schabi.newpipe.extractor.exceptions.PaidContentException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.PrivateContentException;
-import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.localization.ContentCountry;
@@ -93,25 +86,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Nullable
     private static String cachedDeobfuscationCode = null;
     @Nullable
-    private static String playerJsUrl = null;
-    @Nullable
     private static String sts = null;
     @Nullable
     private static String playerCode = null;
 
-    @Nonnull
-    private final Map<String, String> videoInfoPage = new HashMap<>();
-    private JsonArray initialAjaxJson;
-    private JsonObject initialData;
     private JsonObject playerResponse;
     private JsonObject nextResponse;
 
     @Nullable
-    private JsonObject streamingData;
+    private JsonObject desktopStreamingData;
+    @Nullable
+    private JsonObject mobileStreamingData;
     private JsonObject videoPrimaryInfoRenderer;
     private JsonObject videoSecondaryInfoRenderer;
     private int ageLimit = -1;
-    private boolean isGetVideoInfoPlayerResponse = false;
     @Nullable
     private List<SubtitlesStream> subtitles = null;
 
@@ -290,12 +278,17 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                     .getString("lengthSeconds");
             return Long.parseLong(duration);
         } catch (final Exception e) {
-            try {
+            if (desktopStreamingData != null) {
-                final JsonArray adaptiveFormats = streamingData.getArray("adaptiveFormats");
+                final JsonArray adaptiveFormats = desktopStreamingData.getArray("adaptiveFormats");
                 final String durationMs = adaptiveFormats.getObject(0)
                         .getString("approxDurationMs");
                 return Math.round(Long.parseLong(durationMs) / 1000f);
-            } catch (final Exception ignored) {
+            } else if (mobileStreamingData != null) {
+                final JsonArray adaptiveFormats = mobileStreamingData.getArray("adaptiveFormats");
+                final String durationMs = adaptiveFormats.getObject(0)
+                        .getString("approxDurationMs");
+                return Math.round(Long.parseLong(durationMs) / 1000f);
+            } else {
                 throw new ParsingException("Could not get duration", e);
             }
         }
@@ -484,29 +477,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     public String getDashMpdUrl() throws ParsingException {
         assertPageFetched();
 
-        try {
+        if (desktopStreamingData != null) {
-            String dashManifestUrl;
+            return desktopStreamingData.getString("dashManifestUrl");
-            if (streamingData.isString("dashManifestUrl")) {
+        } else if (mobileStreamingData != null) {
-                return streamingData.getString("dashManifestUrl");
+            return mobileStreamingData.getString("dashManifestUrl");
-            } else if (videoInfoPage.containsKey("dashmpd")) {
+        } else {
-                dashManifestUrl = videoInfoPage.get("dashmpd");
+            return EMPTY_STRING;
-            } else {
-                return "";
-            }
-
-            if (!dashManifestUrl.contains("/signature/")) {
-                String obfuscatedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)",
-                        dashManifestUrl);
-                final String deobfuscatedSig;
-
-                deobfuscatedSig = deobfuscateSignature(obfuscatedSig);
-                dashManifestUrl = dashManifestUrl.replace("/s/" + obfuscatedSig,
-                        "/signature/" + deobfuscatedSig);
-            }
-
-            return dashManifestUrl;
-        } catch (final Exception e) {
-            throw new ParsingException("Could not get DASH manifest url", e);
         }
     }
 
@@ -515,10 +491,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     public String getHlsUrl() throws ParsingException {
         assertPageFetched();
 
-        try {
+        if (desktopStreamingData != null) {
-            return streamingData.getString("hlsManifestUrl");
+            return desktopStreamingData.getString("hlsManifestUrl");
-        } catch (final Exception e) {
+        } else if (mobileStreamingData != null) {
-            throw new ParsingException("Could not get HLS manifest url", e);
+            return mobileStreamingData.getString("hlsManifestUrl");
+        } else {
+            return EMPTY_STRING;
         }
     }
 
@@ -704,7 +682,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
     private static final String FORMATS = "formats";
     private static final String ADAPTIVE_FORMATS = "adaptiveFormats";
-    private static final String HTTPS = "https:";
     private static final String DEOBFUSCATION_FUNC_NAME = "deobfuscate";
 
     private static final String[] REGEXES = {
@@ -721,7 +698,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         final String videoId = getId();
         final Localization localization = getExtractorLocalization();
         final ContentCountry contentCountry = getExtractorContentCountry();
-        final byte[] body = JsonWriter.string(prepareJsonBuilder(localization, contentCountry)
+        final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
+                localization, contentCountry)
                 .value("videoId", videoId)
                 .done())
                 .getBytes(UTF_8);
@@ -731,7 +709,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         // API.
         if (sts != null) {
             playerResponse = getJsonPostResponse("player", createPlayerBodyWithSts(localization,
-                    contentCountry, videoId), localization);
+                    contentCountry, videoId, false, sts), localization);
         } else {
             playerResponse = getJsonPostResponse("player", body, localization);
         }
@@ -740,34 +718,55 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         // there can be restrictions on the embedded player.
         // E.g. if a video is age-restricted, the embedded player's playabilityStatus says that
         // the video cannot be played outside of YouTube, but does not show the original message.
-        JsonObject youtubePlayerResponse = playerResponse;
+        final JsonObject youtubePlayerResponse = playerResponse;
 
-        if (playerResponse == null || !playerResponse.has("streamingData")) {
+        if (playerResponse == null) {
-            // Try to get the player response by fetching video info page
-            fetchVideoInfoPage();
-        }
-
-        if (playerResponse == null && youtubePlayerResponse == null) {
             throw new ExtractionException("Could not get playerResponse");
-        } else if (youtubePlayerResponse == null) {
-            youtubePlayerResponse = playerResponse;
         }
 
-        final JsonObject playabilityStatus = (playerResponse == null ? youtubePlayerResponse
+        final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus");
-                : playerResponse).getObject("playabilityStatus");
 
-        checkPlayabilityStatus(youtubePlayerResponse, playabilityStatus);
+        boolean ageRestricted = playabilityStatus.getString("reason", EMPTY_STRING)
+                .contains("age");
 
-        nextResponse = getJsonPostResponse("next", body, localization);
+        if (!playerResponse.has("streamingData")) {
+            try {
+                fetchDesktopEmbedJsonPlayer(contentCountry, localization, videoId);
+            } catch (final Exception ignored) {
+            }
+            try {
+                fetchAndroidEmbedJsonPlayer(contentCountry, localization, videoId);
+            } catch (final Exception ignored) {
+            }
+        }
 
-        // Workaround for rate limits on web streaming URLs.
+        if (desktopStreamingData == null && playerResponse.has("streamingData")) {
-        // TODO: add ability to deobfuscate the n param of these URLs
+            desktopStreamingData = playerResponse.getObject("streamingData");
+        }
 
-        // It's not needed to request the mobile API for age-restricted videos
+        if (desktopStreamingData == null) {
-        if (!isGetVideoInfoPlayerResponse) {
+            checkPlayabilityStatus(youtubePlayerResponse, playabilityStatus);
-            fetchAndroidMobileJsonPlayer(contentCountry, localization, videoId);
+        }
+
+        if (ageRestricted) {
+            final byte[] ageRestrictedBody = JsonWriter.string(prepareDesktopEmbedVideoJsonBuilder(
+                    localization, contentCountry, videoId)
+                    .done())
+                    .getBytes(UTF_8);
+            nextResponse = getJsonPostResponse("next", ageRestrictedBody, localization);
         } else {
-            streamingData = playerResponse.getObject("streamingData");
+            nextResponse = getJsonPostResponse("next", body, localization);
+        }
+
+        if (!ageRestricted) {
+            try {
+                fetchAndroidMobileJsonPlayer(contentCountry, localization, videoId);
+            } catch (final Exception ignored) {
+            }
+        }
+
+        if (isCipherProtectedContent()) {
+            fetchDesktopJsonPlayerWithSts(contentCountry, localization, videoId);
         }
     }
 
@@ -826,133 +825,114 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     }
 
     /**
-     * Fetch the Android Mobile API or fallback to the desktop streams.
+     * Fetch the Android Mobile API and assign the streaming data to the mobileStreamingData JSON
-     * If something went wrong when parsing this API, fallback to the desktop JSON player, fetched
+     * object.
-     * again if the {@code signatureTimestamp} of the JS player is unknown (because signatures
-     * without a {@code signatureTimestamp} included in the player request are invalid).
      */
     private void fetchAndroidMobileJsonPlayer(final ContentCountry contentCountry,
                                               final Localization localization,
-                                              final String videoId) throws ExtractionException,
+                                              final String videoId)
-            IOException {
+            throws IOException, ExtractionException {
-        JsonObject mobilePlayerResponse = null;
+        final byte[] mobileBody = JsonWriter.string(prepareAndroidMobileJsonBuilder(
-        final byte[] mobileBody = JsonWriter.string(prepareMobileJsonBuilder(localization,
+                localization, contentCountry)
-                contentCountry)
+                        .value("videoId", videoId)
-                .value("videoId", videoId)
+                        .done())
-                .done())
                 .getBytes(UTF_8);
-        try {
+        final JsonObject mobilePlayerResponse = getJsonMobilePostResponse("player",
-            mobilePlayerResponse = getJsonMobilePostResponse("player", mobileBody,
+                mobileBody, contentCountry, localization);
-                    contentCountry, localization);
-        } catch (final Exception ignored) {
-        }
-        if (mobilePlayerResponse != null && mobilePlayerResponse.has("streamingData")) {
-            final JsonObject mobileStreamingData = mobilePlayerResponse.getObject(
-                    "streamingData");
-            if (!isNullOrEmpty(mobileStreamingData)) streamingData = mobileStreamingData;
-        } else {
-            // Fallback to the desktop JSON player endpoint
 
-            // The cipher signatures from the player endpoint without a timestamp are invalid so
+        final JsonObject streamingData = mobilePlayerResponse.getObject("streamingData");
-            // download it again only if we didn't have a signatureTimestamp before fetching the
+        if (!isNullOrEmpty(streamingData)) {
-            // data of this video (the sts string).
+            mobileStreamingData = streamingData;
-            if (sts == null && isCipherProtectedContent()) {
+            if (desktopStreamingData == null) {
-                getStsFromPlayerJs();
+                playerResponse = mobilePlayerResponse;
-                final JsonObject playerResponseWithSignatureTimestamp = getJsonPostResponse(
-                        "player", createPlayerBodyWithSts(localization, contentCountry, videoId),
-                        localization);
-                if (playerResponseWithSignatureTimestamp.has("streamingData")) {
-                    streamingData = playerResponseWithSignatureTimestamp.getObject(
-                            "streamingData");
-                }
-            } else {
-                streamingData = playerResponse.getObject("streamingData");
             }
         }
     }
 
-    private void fetchVideoInfoPage() throws ParsingException, ReCaptchaException, IOException {
+    /**
-        getStsFromPlayerJs();
+     * Fetch the desktop API with the {@code signatureTimestamp} and assign the streaming data to
-        final String videoInfoUrl = getVideoInfoUrl(getId(), sts);
+     * the {@code desktopStreamingData} JSON object.
-        final String infoPageResponse = NewPipe.getDownloader()
+     * The cipher signatures from the player endpoint without a signatureTimestamp are invalid so
-                .get(videoInfoUrl, getExtractorLocalization()).responseBody();
+     * if the content is protected by signatureCiphers and if signatureTimestamp is not known, we
-        videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
+     * need to fetch again the desktop InnerTube API.
-
+     */
-        try {
+    private void fetchDesktopJsonPlayerWithSts(final ContentCountry contentCountry,
-            playerResponse = JsonParser.object().from(videoInfoPage.get("player_response"));
+                                               final Localization localization,
-        } catch (final JsonParserException e) {
+                                               final String videoId)
-            throw new ParsingException(
+            throws IOException, ExtractionException {
-                    "Could not parse YouTube player response from video info page", e);
+        if (sts == null) {
+            getStsFromPlayerJs();
+        }
+        final JsonObject playerResponseWithSignatureTimestamp = getJsonPostResponse(
+                "player", createPlayerBodyWithSts(
+                        localization, contentCountry, videoId, false, sts),
+                localization);
+        if (playerResponseWithSignatureTimestamp.has("streamingData")) {
+            desktopStreamingData = playerResponseWithSignatureTimestamp.getObject("streamingData");
         }
-        isGetVideoInfoPlayerResponse = true;
     }
 
-    @Nonnull
+    /**
-    private byte[] createPlayerBodyWithSts(final Localization localization,
+     * Download again the desktop JSON player as an embed client to bypass some age-restrictions.
-                                           final ContentCountry contentCountry,
+     * <p>
-                                           final String videoId) throws ExtractionException,
+     * We need also to get the {@code signatureTimestamp}, if it isn't known because we don't know
-            IOException {
+     * if the video will have signature ciphers or not.
-        // @formatter:off
+     * </p>
-        return JsonWriter.string(prepareJsonBuilder(localization,
+     */
-                contentCountry)
+    private void fetchDesktopEmbedJsonPlayer(final ContentCountry contentCountry,
-                .value("videoId", videoId)
+                                             final Localization localization,
-                .object("playbackContext")
+                                             final String videoId)
-                    .object("contentPlaybackContext")
+            throws IOException, ExtractionException {
-                        .value("signatureTimestamp", sts)
+        if (sts == null) {
-                    .end()
+            getStsFromPlayerJs();
-                .end()
+        }
-                .done())
+        final JsonObject desktopWebEmbedPlayerResponse = getJsonPostResponse(
+                "player", createPlayerBodyWithSts(
+                        localization, contentCountry, videoId, true, sts),
+                localization);
+        final JsonObject streamingData = desktopWebEmbedPlayerResponse.getObject(
+                "streamingData");
+        if (!isNullOrEmpty(streamingData)) {
+            playerResponse = desktopWebEmbedPlayerResponse;
+            desktopStreamingData = streamingData;
+        }
+    }
+
+    /**
+     * Download the Android mobile JSON player as an embed client to bypass some age-restrictions.
+     */
+    private void fetchAndroidEmbedJsonPlayer(final ContentCountry contentCountry,
+                                             final Localization localization,
+                                             final String videoId)
+            throws IOException, ExtractionException {
+        final byte[] androidMobileEmbedBody = JsonWriter.string(
+                prepareAndroidMobileEmbedVideoJsonBuilder(localization, contentCountry, videoId)
+                        .done())
                 .getBytes(UTF_8);
-        // @formatter:on
+        final JsonObject androidMobileEmbedPlayerResponse = getJsonMobilePostResponse("player",
+                androidMobileEmbedBody, contentCountry, localization);
+        final JsonObject streamingData = androidMobileEmbedPlayerResponse.getObject(
+                "streamingData");
+        if (!isNullOrEmpty(streamingData)) {
+            if (desktopStreamingData == null) {
+                playerResponse = androidMobileEmbedPlayerResponse;
+            }
+            mobileStreamingData = androidMobileEmbedPlayerResponse.getObject("streamingData");
+        }
     }
 
     private void storePlayerJs() throws ParsingException {
         try {
-            // The JavaScript player was not found in any page fetched so far and there is
+            playerCode = YoutubeJavaScriptExtractor.extractJavaScriptCode();
-            // nothing cached, so try fetching embedded info.
-            // Don't provide a video id to get a smaller response (around 9Kb instead of 21 Kb
-            // with a video)
-            final String embedUrl = "https://www.youtube.com/embed/";
-            final String embedPageContent = NewPipe.getDownloader()
-                    .get(embedUrl, getExtractorLocalization()).responseBody();
-            try {
-                final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")";
-                playerJsUrl = Parser.matchGroup1(assetsPattern, embedPageContent)
-                        .replace("\\", "").replace("\"", "");
-            } catch (final Parser.RegexException ex) {
-                // playerJsUrl is still available in the file, just somewhere else TODO
-                // It is ok not to find it, see how that's handled in getDeobfuscationCode()
-                final Document doc = Jsoup.parse(embedPageContent);
-                final Elements elems = doc.select("script").attr("name", "player_ias/base");
-                for (final Element elem : elems) {
-                    if (elem.attr("src").contains("base.js")) {
-                        playerJsUrl = elem.attr("src");
-                        break;
-                    }
-                }
-            }
-
-            if (playerJsUrl != null) {
-                if (playerJsUrl.startsWith("//")) {
-                    playerJsUrl = HTTPS + playerJsUrl;
-                } else if (playerJsUrl.startsWith("/")) {
-                    // Sometimes https://www.youtube.com part has to be added manually
-                    playerJsUrl = HTTPS + "//www.youtube.com" + playerJsUrl;
-                }
-                playerCode = NewPipe.getDownloader().get(playerJsUrl, getExtractorLocalization())
-                        .responseBody();
-            } else {
-                throw new ExtractionException("Could not extract JS player URL");
-            }
         } catch (final Exception e) {
             throw new ParsingException("Could not store JavaScript player", e);
         }
     }
 
     private boolean isCipherProtectedContent() {
-        if (streamingData != null) {
+        if (desktopStreamingData != null) {
-            if (streamingData.has("adaptiveFormats")) {
+            if (desktopStreamingData.has("adaptiveFormats")) {
-                final JsonArray adaptiveFormats = streamingData.getArray("adaptiveFormats");
+                final JsonArray adaptiveFormats = desktopStreamingData.getArray("adaptiveFormats");
                 if (!isNullOrEmpty(adaptiveFormats)) {
                     for (final Object adaptiveFormat : adaptiveFormats) {
                         final JsonObject adaptiveFormatJsonObject = ((JsonObject) adaptiveFormat);
@@ -963,8 +943,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                     }
                 }
             }
-            if (streamingData.has("formats")) {
+            if (desktopStreamingData.has("formats")) {
-                final JsonArray formats = streamingData.getArray("formats");
+                final JsonArray formats = desktopStreamingData.getArray("formats");
                 if (!isNullOrEmpty(formats)) {
                     for (final Object format : formats) {
                         final JsonObject formatJsonObject = ((JsonObject) format);
@@ -1027,7 +1007,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Nonnull
     private String getDeobfuscationCode() throws ParsingException {
         if (cachedDeobfuscationCode == null) {
-            if (isNullOrEmpty(playerCode)) throw new ParsingException("playerCode is null");
+            if (isNullOrEmpty(playerCode)) {
+                throw new ParsingException("playerCode is null");
+            }
 
             cachedDeobfuscationCode = loadDeobfuscationCode();
         }
@@ -1038,7 +1020,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         if (!isNullOrEmpty(sts)) return;
         if (playerCode == null) {
             storePlayerJs();
-            if (playerCode == null) throw new ParsingException("playerCode is null");
+            if (playerCode == null) {
+                throw new ParsingException("playerCode is null");
+            }
         }
         sts = Parser.matchGroup1(STS_REGEX, playerCode);
     }
@@ -1114,81 +1098,92 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         return videoSecondaryInfoRenderer;
     }
 
-    @Nonnull
-    private static String getVideoInfoUrl(final String id, final String sts) {
-        // TODO: Try parsing embedded_player_response first
-        return "https://www.youtube.com/get_video_info?" + "video_id=" + id
-                + "&eurl=https://youtube.googleapis.com/v/" + id + "&sts=" + sts
-                + "&html5=1&c=TVHTML5&cver=6.20180913&hl=en&gl=US";
-    }
-
     @Nonnull
     private Map<String, ItagItem> getItags(final String streamingDataKey,
-                                           final ItagItem.ItagType itagTypeWanted)
+                                           final ItagItem.ItagType itagTypeWanted) {
-            throws ParsingException {
         final Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();
-        if (streamingData == null || !streamingData.has(streamingDataKey)) {
+        if (desktopStreamingData == null && mobileStreamingData == null) {
             return urlAndItags;
         }
 
-        final JsonArray formats = streamingData.getArray(streamingDataKey);
+        // Use the mobileStreamingData object first because there is no n param and no
-        for (int i = 0; i != formats.size(); ++i) {
+        // signatureCiphers in streaming URLs of the Android client
-            JsonObject formatData = formats.getObject(i);
+        urlAndItags.putAll(getStreamsFromStreamingDataKey(
-            int itag = formatData.getInt("itag");
+                mobileStreamingData, streamingDataKey, itagTypeWanted));
+        urlAndItags.putAll(getStreamsFromStreamingDataKey(
+                desktopStreamingData, streamingDataKey, itagTypeWanted));
 
-            if (ItagItem.isSupported(itag)) {
+        return urlAndItags;
-                try {
+    }
-                    final ItagItem itagItem = ItagItem.getItag(itag);
+
-                    if (itagItem.itagType == itagTypeWanted) {
+    @Nonnull
-                        // Ignore streams that are delivered using YouTube's OTF format,
+    private Map<String, ItagItem> getStreamsFromStreamingDataKey(
-                        // as those only work with DASH and not with progressive HTTP.
+            final JsonObject streamingData,
-                        if (formatData.getString("type", EMPTY_STRING)
+            final String streamingDataKey,
-                                .equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")) {
+            final ItagItem.ItagType itagTypeWanted) {
-                            continue;
+
+        final Map<String, ItagItem> urlAndItagsFromStreamingDataObject = new LinkedHashMap<>();
+        if (streamingData != null && streamingData.has(streamingDataKey)) {
+            final JsonArray formats = streamingData.getArray(streamingDataKey);
+            for (int i = 0; i != formats.size(); ++i) {
+                JsonObject formatData = formats.getObject(i);
+                int itag = formatData.getInt("itag");
+
+                if (ItagItem.isSupported(itag)) {
+                    try {
+                        final ItagItem itagItem = ItagItem.getItag(itag);
+                        if (itagItem.itagType == itagTypeWanted) {
+                            // Ignore streams that are delivered using YouTube's OTF format,
+                            // as those only work with DASH and not with progressive HTTP.
+                            if (formatData.getString("type", EMPTY_STRING)
+                                    .equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")) {
+                                continue;
+                            }
+
+                            final String streamUrl;
+                            if (formatData.has("url")) {
+                                streamUrl = formatData.getString("url");
+                            } else {
+                                // This url has an obfuscated signature
+                                final String cipherString = formatData.has("cipher")
+                                        ? formatData.getString("cipher")
+                                        : formatData.getString("signatureCipher");
+                                final Map<String, String> cipher = Parser.compatParseMap(
+                                        cipherString);
+                                streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "="
+                                        + deobfuscateSignature(cipher.get("s"));
+                            }
+
+                            final JsonObject initRange = formatData.getObject("initRange");
+                            final JsonObject indexRange = formatData.getObject("indexRange");
+                            final String mimeType = formatData.getString("mimeType", EMPTY_STRING);
+                            final String codec = mimeType.contains("codecs")
+                                    ? mimeType.split("\"")[1] : EMPTY_STRING;
+
+                            itagItem.setBitrate(formatData.getInt("bitrate"));
+                            itagItem.setWidth(formatData.getInt("width"));
+                            itagItem.setHeight(formatData.getInt("height"));
+                            itagItem.setInitStart(Integer.parseInt(initRange.getString("start",
+                                    "-1")));
+                            itagItem.setInitEnd(Integer.parseInt(initRange.getString("end",
+                                    "-1")));
+                            itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start",
+                                    "-1")));
+                            itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end",
+                                    "-1")));
+                            itagItem.fps = formatData.getInt("fps");
+                            itagItem.setQuality(formatData.getString("quality"));
+                            itagItem.setCodec(codec);
+
+                            urlAndItagsFromStreamingDataObject.put(streamUrl, itagItem);
                         }
-
+                    } catch (final UnsupportedEncodingException | ParsingException ignored) {
-                        final String streamUrl;
-                        if (formatData.has("url")) {
-                            streamUrl = formatData.getString("url");
-                        } else {
-                            // This url has an obfuscated signature
-                            final String cipherString = formatData.has("cipher")
-                                    ? formatData.getString("cipher")
-                                    : formatData.getString("signatureCipher");
-                            final Map<String, String> cipher = Parser.compatParseMap(cipherString);
-                            streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "="
-                                    + deobfuscateSignature(cipher.get("s"));
-                        }
-
-                        final JsonObject initRange = formatData.getObject("initRange");
-                        final JsonObject indexRange = formatData.getObject("indexRange");
-                        final String mimeType = formatData.getString("mimeType", EMPTY_STRING);
-                        final String codec = mimeType.contains("codecs")
-                                ? mimeType.split("\"")[1] : EMPTY_STRING;
-
-                        itagItem.setBitrate(formatData.getInt("bitrate"));
-                        itagItem.setWidth(formatData.getInt("width"));
-                        itagItem.setHeight(formatData.getInt("height"));
-                        itagItem.setInitStart(Integer.parseInt(initRange.getString("start",
-                                "-1")));
-                        itagItem.setInitEnd(Integer.parseInt(initRange.getString("end",
-                                "-1")));
-                        itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start",
-                                "-1")));
-                        itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end",
-                                "-1")));
-                        itagItem.fps = formatData.getInt("fps");
-                        itagItem.setQuality(formatData.getString("quality"));
-                        itagItem.setCodec(codec);
-
-                        urlAndItags.put(streamUrl, itagItem);
                     }
-                } catch (final UnsupportedEncodingException ignored) {
                 }
             }
         }
 
-        return urlAndItags;
+        return urlAndItagsFromStreamingDataObject;
     }
 
     @Nonnull
@@ -1381,7 +1376,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     }
 
     /**
-     * Reset YouTube deobfuscation code.
+     * Reset YouTube's deobfuscation code.
      * <p>
      * This is needed for mocks in YouTube stream tests, because when they are ran, the
      * {@code signatureTimestamp} is known (the {@code sts} string) so a different body than the
@@ -1393,7 +1388,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     public static void resetDeobfuscationCode() {
         cachedDeobfuscationCode = null;
         playerCode = null;
-        playerJsUrl = null;
         sts = null;
+        YoutubeJavaScriptExtractor.resetJavaScriptCode();
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java

@@ -41,7 +41,7 @@ import javax.annotation.Nonnull;
 
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextAtKey;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
 import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
@@ -57,7 +57,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
     @Override
     public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
         // @formatter:off
-        final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(),
+        final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
                 getExtractorContentCountry())
                 .value("browseId", "FEtrending")
                 .done())

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java

@@ -92,7 +92,7 @@ public class YoutubeMixPlaylistExtractorTest {
 
         @Test
         public void getPage() throws Exception {
-            final byte[] body = JsonWriter.string(prepareJsonBuilder(
+            final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
                     NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
                     .value("videoId", VIDEO_ID)
                     .value("playlistId", "RD" + VIDEO_ID)
@@ -176,7 +176,7 @@ public class YoutubeMixPlaylistExtractorTest {
 
         @Test
         public void getPage() throws Exception {
-            final byte[] body = JsonWriter.string(prepareJsonBuilder(
+            final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
                     NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
                     .value("videoId", VIDEO_ID)
                     .value("playlistId", "RD" + VIDEO_ID)
@@ -259,7 +259,7 @@ public class YoutubeMixPlaylistExtractorTest {
 
         @Test
         public void getPage() throws Exception {
-            final byte[] body = JsonWriter.string(prepareJsonBuilder(
+            final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
                     NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
                     .value("videoId", VIDEO_ID)
                     .value("playlistId", "RDMM" + VIDEO_ID)
@@ -373,7 +373,7 @@ public class YoutubeMixPlaylistExtractorTest {
 
         @Test
         public void getPage() throws Exception {
-            final byte[] body = JsonWriter.string(prepareJsonBuilder(
+            final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
                     NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
                     .value("videoId", VIDEO_ID_OF_CHANNEL)
                     .value("playlistId", "RDCM" + CHANNEL_ID)

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java

@@ -27,7 +27,7 @@ public class YoutubeParsingHelperTest {
     @Test
     public void testAreHardcodedClientVersionAndKeyValid() throws IOException, ExtractionException {
         assertTrue("Hardcoded client version and key are not valid anymore",
-                YoutubeParsingHelper.areHardcodedClientVersionAndKeyValid().orElse(false));
+                YoutubeParsingHelper.areHardcodedClientVersionAndKeyValid());
     }
 
     @Test