diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 4442e278a..88da1cacf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -15,7 +15,6 @@ import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; import org.schabi.newpipe.extractor.MetaInfo; -import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.AccountTerminatedException; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; @@ -35,6 +34,8 @@ import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.security.SecureRandom; import java.time.LocalDate; import java.time.OffsetDateTime; import java.time.ZoneOffset; @@ -78,11 +79,15 @@ public final class YoutubeParsingHelper { } public static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/"; + public static final String CPN = "cpn"; + public static final String VIDEO_ID = "videoId"; private static final String HARDCODED_CLIENT_VERSION = "2.20220107.00.00"; private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; - private static final String MOBILE_YOUTUBE_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w"; + + private static final String ANDROID_YOUTUBE_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w"; private static final String MOBILE_YOUTUBE_CLIENT_VERSION = "16.49.37"; + private static String clientVersion; private static String key; @@ -94,6 +99,9 @@ public final class YoutubeParsingHelper { @SuppressWarnings("OptionalUsedAsFieldOrParameterType") private static Optional hardcodedClientVersionAndKeyValid = Optional.empty(); + private static final String CONTENT_PLAYBACK_NONCE_ALPHABET = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + private static Random numberGenerator = new Random(); /** @@ -593,7 +601,7 @@ public final class YoutubeParsingHelper { // The ANDROID API key is also valid with the WEB client so return it if we couldn't // extract the WEB API key. - return MOBILE_YOUTUBE_KEY; + return ANDROID_YOUTUBE_KEY; } /** @@ -769,7 +777,7 @@ public final class YoutubeParsingHelper { } else if (navigationEndpoint.has("watchEndpoint")) { final StringBuilder url = new StringBuilder(); url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint - .getObject("watchEndpoint").getString("videoId")); + .getObject("watchEndpoint").getString(VIDEO_ID)); if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) { url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint") .getString("playlistId")); @@ -906,17 +914,6 @@ public final class YoutubeParsingHelper { return responseBody; } - public static Response getResponse(final String url, final Localization localization) - throws IOException, ExtractionException { - final Map> headers = new HashMap<>(); - addYouTubeHeaders(headers); - - final Response response = getDownloader().get(url, headers, localization); - getValidJsonResponseBody(response); - - return response; - } - public static JsonObject getJsonPostResponse(final String endpoint, final byte[] body, final Localization localization) @@ -931,48 +928,30 @@ public final class YoutubeParsingHelper { return JsonUtils.toJsonObject(getValidJsonResponseBody(response)); } - public static JsonObject getJsonMobilePostResponse(final String endpoint, - final byte[] body, - @Nonnull final ContentCountry - contentCountry, - final Localization localization) - throws IOException, ExtractionException { + public static JsonObject getJsonAndroidPostResponse( + final String endpoint, + final byte[] body, + @Nonnull final ContentCountry contentCountry, + final Localization localization, + @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException { final Map> headers = new HashMap<>(); headers.put("Content-Type", Collections.singletonList("application/json")); // Spoofing an Android 11 device with the hardcoded version of the Android app headers.put("User-Agent", Collections.singletonList("com.google.android.youtube/" - + MOBILE_YOUTUBE_CLIENT_VERSION + "Linux; U; Android 11; " + + MOBILE_YOUTUBE_CLIENT_VERSION + " (Linux; U; Android 11; " + contentCountry.getCountryCode() + ") gzip")); headers.put("x-goog-api-format-version", Collections.singletonList("2")); - final Response response = getDownloader().post( - "https://youtubei.googleapis.com/youtubei/v1/" + endpoint + "?key=" - + MOBILE_YOUTUBE_KEY, headers, body, localization); + final String baseEndpointUrl = "https://youtubei.googleapis.com/youtubei/v1/" + endpoint + + "?key=" + ANDROID_YOUTUBE_KEY; + + final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest) + ? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest, + headers, body, localization); return JsonUtils.toJsonObject(getValidJsonResponseBody(response)); } - public static JsonArray getJsonResponse(final String url, final Localization localization) - throws IOException, ExtractionException { - final Map> headers = new HashMap<>(); - addYouTubeHeaders(headers); - - final Response response = getDownloader().get(url, headers, localization); - - return JsonUtils.toJsonArray(getValidJsonResponseBody(response)); - } - - public static JsonArray getJsonResponse(@Nonnull final Page page, - final Localization localization) - throws IOException, ExtractionException { - final Map> headers = new HashMap<>(); - addYouTubeHeaders(headers); - - final Response response = getDownloader().get(page.getUrl(), headers, localization); - - return JsonUtils.toJsonArray(getValidJsonResponseBody(response)); - } - @Nonnull public static JsonBuilder prepareDesktopJsonBuilder( @Nonnull final Localization localization, @@ -986,6 +965,13 @@ public final class YoutubeParsingHelper { .value("gl", contentCountry.getCountryCode()) .value("clientName", "WEB") .value("clientVersion", getClientVersion()) + .value("originalUrl", "https://www.youtube.com") + .value("platform", "DESKTOP") + .end() + .object("request") + .array("internalExperimentFlags") + .end() + .value("useSsl", true) .end() .object("user") // TO DO: provide a way to enable restricted mode with: @@ -1032,17 +1018,23 @@ public final class YoutubeParsingHelper { .value("clientName", "WEB") .value("clientVersion", getClientVersion()) .value("clientScreen", "EMBED") + .value("originalUrl", "https://www.youtube.com") + .value("platform", "DESKTOP") .end() .object("thirdParty") .value("embedUrl", "https://www.youtube.com/watch?v=" + videoId) .end() + .object("request") + .array("internalExperimentFlags") + .end() + .value("useSsl", true) + .end() .object("user") // TO DO: provide a way to enable restricted mode with: // .value("enableSafetyMode", boolean) .value("lockedSafetyMode", false) .end() - .end() - .value("videoId", videoId); + .end(); // @formatter:on } @@ -1050,7 +1042,8 @@ public final class YoutubeParsingHelper { public static JsonBuilder prepareAndroidMobileEmbedVideoJsonBuilder( @Nonnull final Localization localization, @Nonnull final ContentCountry contentCountry, - @Nonnull final String videoId) { + @Nonnull final String videoId, + @Nonnull final String contentPlaybackNonce) { // @formatter:off return JsonObject.builder() .object("context") @@ -1064,48 +1057,53 @@ public final class YoutubeParsingHelper { .object("thirdParty") .value("embedUrl", "https://www.youtube.com/watch?v=" + videoId) .end() + .object("request") + .array("internalExperimentFlags") + .end() + .value("useSsl", true) + .end() .object("user") // TO DO: provide a way to enable restricted mode with: // .value("enableSafetyMode", boolean) .value("lockedSafetyMode", false) .end() .end() - .value("videoId", videoId); + .value(CPN, contentPlaybackNonce) + .value(VIDEO_ID, videoId); // @formatter:on } @Nonnull - public static byte[] createPlayerBodyWithSts(final Localization localization, - final ContentCountry contentCountry, - final String videoId, - final boolean withThirdParty, - @Nullable final String sts) - throws IOException, ExtractionException { - if (withThirdParty) { - // @formatter:off - return JsonWriter.string(prepareDesktopEmbedVideoJsonBuilder( - localization, contentCountry, videoId) - .object("playbackContext") - .object("contentPlaybackContext") - .value("signatureTimestamp", sts) - .end() + public static byte[] createDesktopPlayerBody( + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId, + @Nonnull final String sts, + final boolean isEmbedClientScreen, + @Nonnull final String contentPlaybackNonce) throws IOException, ExtractionException { + // @formatter:off + return JsonWriter.string((isEmbedClientScreen + ? prepareDesktopEmbedVideoJsonBuilder(localization, contentCountry, + videoId) + : prepareDesktopJsonBuilder(localization, contentCountry)) + .object("playbackContext") + .object("contentPlaybackContext") + .value("currentUrl", "/watch?v=" + videoId) + .value("vis", 0) + .value("splay", false) + .value("autoCaptionsDefaultOn", false) + .value("autonavState", "STATE_NONE") + .value("html5Preference", "HTML5_PREF_WANTS") + .value("signatureTimestamp", sts) + .value("referer", "https://www.youtube.com/watch?v=" + videoId) + .value("lactMilliseconds", "-1") .end() - .done()) - .getBytes(UTF_8); - // @formatter:on - } else { - // @formatter:off - return JsonWriter.string(prepareDesktopJsonBuilder(localization, contentCountry) - .value("videoId", videoId) - .object("playbackContext") - .object("contentPlaybackContext") - .value("signatureTimestamp", sts) - .end() - .end() - .done()) - .getBytes(UTF_8); - // @formatter:on - } + .end() + .value(CPN, contentPlaybackNonce) + .value(VIDEO_ID, videoId) + .done()) + .getBytes(StandardCharsets.UTF_8); + // @formatter:on } /** @@ -1381,4 +1379,47 @@ public final class YoutubeParsingHelper { .replaceAll("\\\\x5b", "[") .replaceAll("\\\\x5d", "]"); } + + /** + * Generate a content playback nonce (also called {@code cpn}), sent by YouTube clients in + * playback requests (and also for some clients, in the player request body). + * + * @return a content playback nonce string + */ + @Nonnull + public static String generateContentPlaybackNonce() { + final SecureRandom random = new SecureRandom(); + final StringBuilder stringBuilder = new StringBuilder(); + + for (int i = 0; i < 16; i++) { + stringBuilder.append(CONTENT_PLAYBACK_NONCE_ALPHABET.charAt( + (random.nextInt(128) + 1) & 63)); + } + + return stringBuilder.toString(); + } + + /** + * Try to generate a {@code t} parameter, sent by mobile clients as a query of the player + * request. + * + *

+ * Some researches needs to be done to know how this parameter, unique at each request, is + * generated. + *

+ * + * @return a 12 characters string to try to reproduce the {@code} parameter + */ + @Nonnull + public static String generateTParameter() { + final SecureRandom random = new SecureRandom(); + final StringBuilder stringBuilder = new StringBuilder(); + + for (int i = 0; i < 12; i++) { + stringBuilder.append(CONTENT_PLAYBACK_NONCE_ALPHABET.charAt( + (random.nextInt(128) + 1) & 63)); + } + + return stringBuilder.toString(); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index e95a2fcd7..c07d9e33d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -1,8 +1,12 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.createPlayerBodyWithSts; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.VIDEO_ID; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.createDesktopPlayerBody; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonMobilePostResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateContentPlaybackNonce; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonAndroidPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareAndroidMobileEmbedVideoJsonBuilder; @@ -69,7 +73,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; - import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -119,13 +122,16 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nullable private JsonObject desktopStreamingData; @Nullable - private JsonObject mobileStreamingData; + private JsonObject androidStreamingData; private JsonObject videoPrimaryInfoRenderer; private JsonObject videoSecondaryInfoRenderer; private int ageLimit = -1; @Nullable private List subtitles = null; + private String desktopCpn; + private String androidCpn; + public YoutubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler) { super(service, linkHandler); } @@ -310,8 +316,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { final String durationMs = adaptiveFormats.getObject(0) .getString("approxDurationMs"); return Math.round(Long.parseLong(durationMs) / 1000f); - } else if (mobileStreamingData != null) { - final JsonArray adaptiveFormats = mobileStreamingData.getArray(ADAPTIVE_FORMATS); + } else if (androidStreamingData != null) { + final JsonArray adaptiveFormats = androidStreamingData.getArray(ADAPTIVE_FORMATS); final String durationMs = adaptiveFormats.getObject(0) .getString("approxDurationMs"); return Math.round(Long.parseLong(durationMs) / 1000f); @@ -493,8 +499,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { if (desktopStreamingData != null) { return desktopStreamingData.getString("dashManifestUrl"); - } else if (mobileStreamingData != null) { - return mobileStreamingData.getString("dashManifestUrl"); + } else if (androidStreamingData != null) { + return androidStreamingData.getString("dashManifestUrl"); } else { return EMPTY_STRING; } @@ -507,8 +513,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { if (desktopStreamingData != null) { return desktopStreamingData.getString("hlsManifestUrl"); - } else if (mobileStreamingData != null) { - return mobileStreamingData.getString("hlsManifestUrl"); + } else if (androidStreamingData != null) { + return androidStreamingData.getString("hlsManifestUrl"); } else { return EMPTY_STRING; } @@ -710,6 +716,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { private static final String FORMATS = "formats"; private static final String ADAPTIVE_FORMATS = "adaptiveFormats"; private static final String DEOBFUSCATION_FUNC_NAME = "deobfuscate"; + private static final String STREAMING_DATA = "streamingData"; + private static final String PLAYER = "player"; + private static final String NEXT = "next"; private static final String[] REGEXES = { "(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2,})\\s*=\\s*function\\(\\s*a\\s*\\)" @@ -725,24 +734,19 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { + if (sts == null) { + getStsFromPlayerJs(); + } + final String videoId = getId(); final Localization localization = getExtractorLocalization(); final ContentCountry contentCountry = getExtractorContentCountry(); - final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder( - localization, contentCountry) - .value("videoId", videoId) - .done()) - .getBytes(UTF_8); + desktopCpn = generateContentPlaybackNonce(); - // Put the sts string if we already know it so we don't have to fetch again the player - // endpoint of the desktop internal API if something went wrong when parsing the Android - // API. - if (sts != null) { - playerResponse = getJsonPostResponse("player", createPlayerBodyWithSts(localization, - contentCountry, videoId, false, sts), localization); - } else { - playerResponse = getJsonPostResponse("player", body, localization); - } + playerResponse = getJsonPostResponse(PLAYER, + createDesktopPlayerBody(localization, contentCountry, videoId, sts, false, + desktopCpn), + localization); // Save the playerResponse from the player endpoint of the desktop internal API because // there can be restrictions on the embedded player. @@ -759,7 +763,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { final boolean ageRestricted = playabilityStatus.getString("reason", EMPTY_STRING) .contains("age"); - if (!playerResponse.has("streamingData")) { + if (!playerResponse.has(STREAMING_DATA)) { try { fetchDesktopEmbedJsonPlayer(contentCountry, localization, videoId); } catch (final Exception ignored) { @@ -770,8 +774,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - if (desktopStreamingData == null && playerResponse.has("streamingData")) { - desktopStreamingData = playerResponse.getObject("streamingData"); + if (desktopStreamingData == null && playerResponse.has(STREAMING_DATA)) { + desktopStreamingData = playerResponse.getObject(STREAMING_DATA); } if (desktopStreamingData == null) { @@ -781,11 +785,17 @@ public class YoutubeStreamExtractor extends StreamExtractor { if (ageRestricted) { final byte[] ageRestrictedBody = JsonWriter.string(prepareDesktopEmbedVideoJsonBuilder( localization, contentCountry, videoId) + .value(VIDEO_ID, videoId) .done()) .getBytes(UTF_8); - nextResponse = getJsonPostResponse("next", ageRestrictedBody, localization); + nextResponse = getJsonPostResponse(NEXT, ageRestrictedBody, localization); } else { - nextResponse = getJsonPostResponse("next", body, localization); + final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization, + contentCountry) + .value(VIDEO_ID, videoId) + .done()) + .getBytes(UTF_8); + nextResponse = getJsonPostResponse(NEXT, body, localization); } if (!ageRestricted) { @@ -794,10 +804,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { } catch (final Exception ignored) { } } - - if (isCipherProtectedContent()) { - fetchDesktopJsonPlayerWithSts(contentCountry, localization, videoId); - } } private void checkPlayabilityStatus(final JsonObject youtubePlayerResponse, @@ -825,28 +831,26 @@ public class YoutubeStreamExtractor extends StreamExtractor { "This age-restricted video cannot be watched."); } } - if (status.equalsIgnoreCase("unplayable")) { - if (reason != null) { - if (reason.contains("Music Premium")) { - throw new YoutubeMusicPremiumContentException(); - } - if (reason.contains("payment")) { - throw new PaidContentException("This video is a paid video"); - } - if (reason.contains("members-only")) { - throw new PaidContentException("This video is only available" - + " for members of the channel of this video"); - } - if (reason.contains("unavailable")) { - final String detailedErrorMessage = getTextFromObject(newPlayabilityStatus - .getObject("errorScreen").getObject("playerErrorMessageRenderer") - .getObject("subreason")); - if (detailedErrorMessage != null) { - if (detailedErrorMessage.contains("country")) { - throw new GeographicRestrictionException( - "This video is not available in user's country."); - } - } + + if (status.equalsIgnoreCase("unplayable") && reason != null) { + if (reason.contains("Music Premium")) { + throw new YoutubeMusicPremiumContentException(); + } + if (reason.contains("payment")) { + throw new PaidContentException("This video is a paid video"); + } + if (reason.contains("members-only")) { + throw new PaidContentException("This video is only available" + + " for members of the channel of this video"); + } + + if (reason.contains("unavailable")) { + final String detailedErrorMessage = getTextFromObject(newPlayabilityStatus + .getObject("errorScreen").getObject("playerErrorMessageRenderer") + .getObject("subreason")); + if (detailedErrorMessage != null && detailedErrorMessage.contains("country")) { + throw new GeographicRestrictionException( + "This video is not available in client's country."); } } } @@ -859,70 +863,50 @@ public class YoutubeStreamExtractor extends StreamExtractor { * Fetch the Android Mobile API and assign the streaming data to the mobileStreamingData JSON * object. */ - private void fetchAndroidMobileJsonPlayer(final ContentCountry contentCountry, - final Localization localization, - final String videoId) + private void fetchAndroidMobileJsonPlayer(@Nonnull final ContentCountry contentCountry, + @Nonnull final Localization localization, + @Nonnull final String videoId) throws IOException, ExtractionException { final byte[] mobileBody = JsonWriter.string(prepareAndroidMobileJsonBuilder( localization, contentCountry) - .value("videoId", videoId) + .value(VIDEO_ID, videoId) + .value(CPN, androidCpn) .done()) .getBytes(UTF_8); - final JsonObject mobilePlayerResponse = getJsonMobilePostResponse("player", - mobileBody, contentCountry, localization); - final JsonObject streamingData = mobilePlayerResponse.getObject("streamingData"); + final JsonObject androidPlayerResponse = getJsonAndroidPostResponse(PLAYER, + mobileBody, contentCountry, localization, "&t=" + generateTParameter() + + "&id=" + videoId); + + final JsonObject streamingData = androidPlayerResponse.getObject(STREAMING_DATA); if (!isNullOrEmpty(streamingData)) { - mobileStreamingData = streamingData; + androidStreamingData = streamingData; if (desktopStreamingData == null) { - playerResponse = mobilePlayerResponse; + playerResponse = androidPlayerResponse; } } } - /** - * Fetch the desktop API with the {@code signatureTimestamp} and assign the streaming data to - * the {@code desktopStreamingData} JSON object. - * The cipher signatures from the player endpoint without a signatureTimestamp are invalid so - * if the content is protected by signatureCiphers and if signatureTimestamp is not known, we - * need to fetch again the desktop InnerTube API. - */ - private void fetchDesktopJsonPlayerWithSts(final ContentCountry contentCountry, - final Localization localization, - final String videoId) - throws IOException, ExtractionException { - if (sts == null) { - getStsFromPlayerJs(); - } - final JsonObject playerResponseWithSignatureTimestamp = getJsonPostResponse( - "player", createPlayerBodyWithSts( - localization, contentCountry, videoId, false, sts), - localization); - if (playerResponseWithSignatureTimestamp.has("streamingData")) { - desktopStreamingData = playerResponseWithSignatureTimestamp.getObject("streamingData"); - } - } - /** * Download again the desktop JSON player as an embed client to bypass some age-restrictions. - *

- * We need also to get the {@code signatureTimestamp}, if it isn't known because we don't know - * if the video will have signature ciphers or not. - *

*/ - private void fetchDesktopEmbedJsonPlayer(final ContentCountry contentCountry, - final Localization localization, - final String videoId) + private void fetchDesktopEmbedJsonPlayer(@Nonnull final ContentCountry contentCountry, + @Nonnull final Localization localization, + @Nonnull final String videoId) throws IOException, ExtractionException { if (sts == null) { getStsFromPlayerJs(); } - final JsonObject desktopWebEmbedPlayerResponse = getJsonPostResponse( - "player", createPlayerBodyWithSts( - localization, contentCountry, videoId, true, sts), + + // Because a cpn is unique to each request, we need to generate it again + desktopCpn = generateContentPlaybackNonce(); + + final JsonObject desktopWebEmbedPlayerResponse = getJsonPostResponse(PLAYER, + createDesktopPlayerBody(localization, contentCountry, videoId, sts, true, + desktopCpn), localization); final JsonObject streamingData = desktopWebEmbedPlayerResponse.getObject( - "streamingData"); + STREAMING_DATA); if (!isNullOrEmpty(streamingData)) { playerResponse = desktopWebEmbedPlayerResponse; desktopStreamingData = streamingData; @@ -932,27 +916,32 @@ public class YoutubeStreamExtractor extends StreamExtractor { /** * Download the Android mobile JSON player as an embed client to bypass some age-restrictions. */ - private void fetchAndroidEmbedJsonPlayer(final ContentCountry contentCountry, - final Localization localization, - final String videoId) + private void fetchAndroidEmbedJsonPlayer(@Nonnull final ContentCountry contentCountry, + @Nonnull final Localization localization, + @Nonnull final String videoId) throws IOException, ExtractionException { + // Because a cpn is unique to each request, we need to generate it again + androidCpn = generateContentPlaybackNonce(); + final byte[] androidMobileEmbedBody = JsonWriter.string( - prepareAndroidMobileEmbedVideoJsonBuilder(localization, contentCountry, videoId) - .done()) + prepareAndroidMobileEmbedVideoJsonBuilder(localization, contentCountry, videoId, + androidCpn) + .done()) .getBytes(UTF_8); - final JsonObject androidMobileEmbedPlayerResponse = getJsonMobilePostResponse("player", - androidMobileEmbedBody, contentCountry, localization); + final JsonObject androidMobileEmbedPlayerResponse = getJsonAndroidPostResponse(PLAYER, + androidMobileEmbedBody, contentCountry, localization, "&t=" + generateTParameter() + + "&id=" + videoId); final JsonObject streamingData = androidMobileEmbedPlayerResponse.getObject( - "streamingData"); + STREAMING_DATA); if (!isNullOrEmpty(streamingData)) { if (desktopStreamingData == null) { playerResponse = androidMobileEmbedPlayerResponse; } - mobileStreamingData = androidMobileEmbedPlayerResponse.getObject("streamingData"); + androidStreamingData = androidMobileEmbedPlayerResponse.getObject(STREAMING_DATA); } } - private void storePlayerJs() throws ParsingException { + private static void storePlayerJs() throws ParsingException { try { playerCode = YoutubeJavaScriptExtractor.extractJavaScriptCode(); } catch (final Exception e) { @@ -960,38 +949,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - private boolean isCipherProtectedContent() { - if (desktopStreamingData != null) { - if (desktopStreamingData.has(ADAPTIVE_FORMATS)) { - final JsonArray adaptiveFormats = desktopStreamingData.getArray(ADAPTIVE_FORMATS); - if (!isNullOrEmpty(adaptiveFormats)) { - for (final Object adaptiveFormat : adaptiveFormats) { - final JsonObject adaptiveFormatJsonObject = ((JsonObject) adaptiveFormat); - if (adaptiveFormatJsonObject.has("signatureCipher") - || adaptiveFormatJsonObject.has("cipher")) { - return true; - } - } - } - } - if (desktopStreamingData.has(FORMATS)) { - final JsonArray formats = desktopStreamingData.getArray(FORMATS); - if (!isNullOrEmpty(formats)) { - for (final Object format : formats) { - final JsonObject formatJsonObject = ((JsonObject) format); - if (formatJsonObject.has("signatureCipher") - || formatJsonObject.has("cipher")) { - return true; - } - } - } - } - } - return false; - } - - private String getDeobfuscationFuncName(final String thePlayerCode) - throws DeobfuscateException { + private static String getDeobfuscationFuncName(final String thePlayerCode) throws DeobfuscateException { Parser.RegexException exception = null; for (final String regex : REGEXES) { try { @@ -1007,7 +965,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Nonnull - private String loadDeobfuscationCode() throws DeobfuscateException { + private static String loadDeobfuscationCode() throws DeobfuscateException { try { final String deobfuscationFunctionName = getDeobfuscationFuncName(playerCode); @@ -1024,7 +982,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { "(var " + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)"; final String helperObject = - Parser.matchGroup1(helperPattern, playerCode.replace("\n", "")); + Parser.matchGroup1(helperPattern, Objects.requireNonNull(playerCode).replace( + "\n", "")); final String callerFunction = "function " + DEOBFUSCATION_FUNC_NAME + "(a){return " @@ -1037,7 +996,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Nonnull - private String getDeobfuscationCode() throws ParsingException { + private static String getDeobfuscationCode() throws ParsingException { if (cachedDeobfuscationCode == null) { if (isNullOrEmpty(playerCode)) { throw new ParsingException("playerCode is null"); @@ -1048,7 +1007,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { return cachedDeobfuscationCode; } - private void getStsFromPlayerJs() throws ParsingException { + private static void getStsFromPlayerJs() throws ParsingException { if (!isNullOrEmpty(sts)) { return; } @@ -1085,8 +1044,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { //////////////////////////////////////////////////////////////////////////*/ private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException { - if (this.videoPrimaryInfoRenderer != null) { - return this.videoPrimaryInfoRenderer; + if (videoPrimaryInfoRenderer != null) { + return videoPrimaryInfoRenderer; } final JsonArray contents = nextResponse.getObject("contents") @@ -1106,18 +1065,19 @@ public class YoutubeStreamExtractor extends StreamExtractor { throw new ParsingException("Could not find videoPrimaryInfoRenderer"); } - this.videoPrimaryInfoRenderer = theVideoPrimaryInfoRenderer; + videoPrimaryInfoRenderer = theVideoPrimaryInfoRenderer; return theVideoPrimaryInfoRenderer; } private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException { - if (this.videoSecondaryInfoRenderer != null) { - return this.videoSecondaryInfoRenderer; + if (videoSecondaryInfoRenderer != null) { + return videoSecondaryInfoRenderer; } final JsonArray contents = nextResponse.getObject("contents") .getObject("twoColumnWatchNextResults").getObject("results").getObject("results") .getArray("contents"); + JsonObject theVideoSecondaryInfoRenderer = null; for (final Object content : contents) { @@ -1132,24 +1092,24 @@ public class YoutubeStreamExtractor extends StreamExtractor { throw new ParsingException("Could not find videoSecondaryInfoRenderer"); } - this.videoSecondaryInfoRenderer = theVideoSecondaryInfoRenderer; + videoSecondaryInfoRenderer = theVideoSecondaryInfoRenderer; return theVideoSecondaryInfoRenderer; } @Nonnull - private Map getItags(final String streamingDataKey, - final ItagItem.ItagType itagTypeWanted) { + private Map getItags(@Nonnull final String streamingDataKey, + @Nonnull final ItagItem.ItagType itagTypeWanted) { final Map urlAndItags = new LinkedHashMap<>(); - if (desktopStreamingData == null && mobileStreamingData == null) { + if (desktopStreamingData == null && androidStreamingData == null) { return urlAndItags; } // Use the mobileStreamingData object first because there is no n param and no // signatureCiphers in streaming URLs of the Android client urlAndItags.putAll(getStreamsFromStreamingDataKey( - mobileStreamingData, streamingDataKey, itagTypeWanted)); + androidStreamingData, streamingDataKey, itagTypeWanted, androidCpn)); urlAndItags.putAll(getStreamsFromStreamingDataKey( - desktopStreamingData, streamingDataKey, itagTypeWanted)); + desktopStreamingData, streamingDataKey, itagTypeWanted, desktopCpn)); return urlAndItags; } @@ -1157,8 +1117,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull private Map getStreamsFromStreamingDataKey( final JsonObject streamingData, - final String streamingDataKey, - final ItagItem.ItagType itagTypeWanted) { + @Nonnull final String streamingDataKey, + @Nonnull final ItagItem.ItagType itagTypeWanted, + @Nonnull final String contentPlaybackNonce) { final Map urlAndItagsFromStreamingDataObject = new LinkedHashMap<>(); if (streamingData != null && streamingData.has(streamingDataKey)) { @@ -1180,7 +1141,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { final String streamUrl; if (formatData.has("url")) { - streamUrl = formatData.getString("url"); + streamUrl = formatData.getString("url") + "&cpn=" + + contentPlaybackNonce; } else { // This url has an obfuscated signature final String cipherString = formatData.has("cipher")