lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

[YouTube] Add the cpn param to playback requests and try to spoof better the Android client 

The cpn param, aka the content playback nonce param, is a parameter sent by YouTube web client in videoplayback requests, and for some of them, in the player request body. This PR adds it everywhere.

For the desktop/WEB client, some params were missing from the playbackContext object, which seemed (or not) to make YouTube throttle streams extracted from the WEB client. This PR adds them.

Fingerprinting on the WEB client basing on the client version used is not possible anymore, because the latest client version is extracted at the first time of a YouTube request on a session which require the extractor to fetch again the website (and this may come back the reCaptcha issues again unfortunately, but it seems there is no other way to get it).

For the Android client, the video id is now also sent as a query parameter, like a 12 characters string, in the t query parameter, in order to spoof better this client. Researches need to be done on this parameter, unique to each request, and how it is generated by clients.

This commit also fixes a small bug with the Android User-Agent string.

Some code improvements have been also made.
This commit is contained in:
TiA4f8R 2021-12-22 17:55:41 +01:00
parent 83f374bff1
commit 05b7fee23b
No known key found for this signature in database
GPG Key ID: E6D3E7F5949450DD
2 changed files with 241 additions and 238 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

201
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -15,7 +15,6 @@ import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.AccountTerminatedException;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
@ -35,6 +34,8 @@ import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.security.SecureRandom;
import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
@ -78,11 +79,15 @@ public final class YoutubeParsingHelper {
}
public static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/";
public static final String CPN = "cpn";
public static final String VIDEO_ID = "videoId";
private static final String HARDCODED_CLIENT_VERSION = "2.20220107.00.00";
private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
private static final String MOBILE_YOUTUBE_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w";
private static final String ANDROID_YOUTUBE_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w";
private static final String MOBILE_YOUTUBE_CLIENT_VERSION = "16.49.37";
private static String clientVersion;
private static String key;
@ -94,6 +99,9 @@ public final class YoutubeParsingHelper {
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
private static Optional<Boolean> hardcodedClientVersionAndKeyValid = Optional.empty();
private static final String CONTENT_PLAYBACK_NONCE_ALPHABET =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
private static Random numberGenerator = new Random();
/**
@ -593,7 +601,7 @@ public final class YoutubeParsingHelper {
// The ANDROID API key is also valid with the WEB client so return it if we couldn't
// extract the WEB API key.
return MOBILE_YOUTUBE_KEY;
return ANDROID_YOUTUBE_KEY;
}
/**
@ -769,7 +777,7 @@ public final class YoutubeParsingHelper {
} else if (navigationEndpoint.has("watchEndpoint")) {
final StringBuilder url = new StringBuilder();
url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint
.getObject("watchEndpoint").getString("videoId"));
.getObject("watchEndpoint").getString(VIDEO_ID));
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) {
url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint")
.getString("playlistId"));
@ -906,17 +914,6 @@ public final class YoutubeParsingHelper {
return responseBody;
}
public static Response getResponse(final String url, final Localization localization)
throws IOException, ExtractionException {
final Map<String, List<String>> headers = new HashMap<>();
addYouTubeHeaders(headers);
final Response response = getDownloader().get(url, headers, localization);
getValidJsonResponseBody(response);
return response;
}
public static JsonObject getJsonPostResponse(final String endpoint,
final byte[] body,
final Localization localization)
@ -931,48 +928,30 @@ public final class YoutubeParsingHelper {
return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
}
public static JsonObject getJsonMobilePostResponse(final String endpoint,
final byte[] body,
@Nonnull final ContentCountry
contentCountry,
final Localization localization)
throws IOException, ExtractionException {
public static JsonObject getJsonAndroidPostResponse(
final String endpoint,
final byte[] body,
@Nonnull final ContentCountry contentCountry,
final Localization localization,
@Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
final Map<String, List<String>> headers = new HashMap<>();
headers.put("Content-Type", Collections.singletonList("application/json"));
// Spoofing an Android 11 device with the hardcoded version of the Android app
headers.put("User-Agent", Collections.singletonList("com.google.android.youtube/"
+ MOBILE_YOUTUBE_CLIENT_VERSION + "Linux; U; Android 11; "
+ MOBILE_YOUTUBE_CLIENT_VERSION + " (Linux; U; Android 11; "
+ contentCountry.getCountryCode() + ") gzip"));
headers.put("x-goog-api-format-version", Collections.singletonList("2"));
final Response response = getDownloader().post(
"https://youtubei.googleapis.com/youtubei/v1/" + endpoint + "?key="
+ MOBILE_YOUTUBE_KEY, headers, body, localization);
final String baseEndpointUrl = "https://youtubei.googleapis.com/youtubei/v1/" + endpoint
+ "?key=" + ANDROID_YOUTUBE_KEY;
final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest,
headers, body, localization);
return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
}
public static JsonArray getJsonResponse(final String url, final Localization localization)
throws IOException, ExtractionException {
final Map<String, List<String>> headers = new HashMap<>();
addYouTubeHeaders(headers);
final Response response = getDownloader().get(url, headers, localization);
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
}
public static JsonArray getJsonResponse(@Nonnull final Page page,
final Localization localization)
throws IOException, ExtractionException {
final Map<String, List<String>> headers = new HashMap<>();
addYouTubeHeaders(headers);
final Response response = getDownloader().get(page.getUrl(), headers, localization);
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
}
@Nonnull
public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
@Nonnull final Localization localization,
@ -986,6 +965,13 @@ public final class YoutubeParsingHelper {
.value("gl", contentCountry.getCountryCode())
.value("clientName", "WEB")
.value("clientVersion", getClientVersion())
.value("originalUrl", "https://www.youtube.com")
.value("platform", "DESKTOP")
.end()
.object("request")
.array("internalExperimentFlags")
.end()
.value("useSsl", true)
.end()
.object("user")
// TO DO: provide a way to enable restricted mode with:
@ -1032,17 +1018,23 @@ public final class YoutubeParsingHelper {
.value("clientName", "WEB")
.value("clientVersion", getClientVersion())
.value("clientScreen", "EMBED")
.value("originalUrl", "https://www.youtube.com")
.value("platform", "DESKTOP")
.end()
.object("thirdParty")
.value("embedUrl", "https://www.youtube.com/watch?v=" + videoId)
.end()
.object("request")
.array("internalExperimentFlags")
.end()
.value("useSsl", true)
.end()
.object("user")
// TO DO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.end()
.value("videoId", videoId);
.end();
// @formatter:on
}
@ -1050,7 +1042,8 @@ public final class YoutubeParsingHelper {
public static JsonBuilder<JsonObject> prepareAndroidMobileEmbedVideoJsonBuilder(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nonnull final String videoId) {
@Nonnull final String videoId,
@Nonnull final String contentPlaybackNonce) {
// @formatter:off
return JsonObject.builder()
.object("context")
@ -1064,48 +1057,53 @@ public final class YoutubeParsingHelper {
.object("thirdParty")
.value("embedUrl", "https://www.youtube.com/watch?v=" + videoId)
.end()
.object("request")
.array("internalExperimentFlags")
.end()
.value("useSsl", true)
.end()
.object("user")
// TO DO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.end()
.value("videoId", videoId);
.value(CPN, contentPlaybackNonce)
.value(VIDEO_ID, videoId);
// @formatter:on
}
@Nonnull
public static byte[] createPlayerBodyWithSts(final Localization localization,
final ContentCountry contentCountry,
final String videoId,
final boolean withThirdParty,
@Nullable final String sts)
throws IOException, ExtractionException {
if (withThirdParty) {
// @formatter:off
return JsonWriter.string(prepareDesktopEmbedVideoJsonBuilder(
localization, contentCountry, videoId)
.object("playbackContext")
.object("contentPlaybackContext")
.value("signatureTimestamp", sts)
.end()
public static byte[] createDesktopPlayerBody(
@Nonnull final Localization localization,
@Nonnull final ContentCountry contentCountry,
@Nonnull final String videoId,
@Nonnull final String sts,
final boolean isEmbedClientScreen,
@Nonnull final String contentPlaybackNonce) throws IOException, ExtractionException {
// @formatter:off
return JsonWriter.string((isEmbedClientScreen
? prepareDesktopEmbedVideoJsonBuilder(localization, contentCountry,
videoId)
: prepareDesktopJsonBuilder(localization, contentCountry))
.object("playbackContext")
.object("contentPlaybackContext")
.value("currentUrl", "/watch?v=" + videoId)
.value("vis", 0)
.value("splay", false)
.value("autoCaptionsDefaultOn", false)
.value("autonavState", "STATE_NONE")
.value("html5Preference", "HTML5_PREF_WANTS")
.value("signatureTimestamp", sts)
.value("referer", "https://www.youtube.com/watch?v=" + videoId)
.value("lactMilliseconds", "-1")
.end()
.done())
.getBytes(UTF_8);
// @formatter:on
} else {
// @formatter:off
return JsonWriter.string(prepareDesktopJsonBuilder(localization, contentCountry)
.value("videoId", videoId)
.object("playbackContext")
.object("contentPlaybackContext")
.value("signatureTimestamp", sts)
.end()
.end()
.done())
.getBytes(UTF_8);
// @formatter:on
}
.end()
.value(CPN, contentPlaybackNonce)
.value(VIDEO_ID, videoId)
.done())
.getBytes(StandardCharsets.UTF_8);
// @formatter:on
}
/**
@ -1381,4 +1379,47 @@ public final class YoutubeParsingHelper {
.replaceAll("\\\\x5b", "[")
.replaceAll("\\\\x5d", "]");
}
/**
* Generate a content playback nonce (also called {@code cpn}), sent by YouTube clients in
* playback requests (and also for some clients, in the player request body).
*
* @return a content playback nonce string
*/
@Nonnull
public static String generateContentPlaybackNonce() {
final SecureRandom random = new SecureRandom();
final StringBuilder stringBuilder = new StringBuilder();
for (int i = 0; i < 16; i++) {
stringBuilder.append(CONTENT_PLAYBACK_NONCE_ALPHABET.charAt(
(random.nextInt(128) + 1) & 63));
}
return stringBuilder.toString();
}
/**
* Try to generate a {@code t} parameter, sent by mobile clients as a query of the player
* request.
*
* <p>
* Some researches needs to be done to know how this parameter, unique at each request, is
* generated.
* </p>
*
* @return a 12 characters string to try to reproduce the {@code} parameter
*/
@Nonnull
public static String generateTParameter() {
final SecureRandom random = new SecureRandom();
final StringBuilder stringBuilder = new StringBuilder();
for (int i = 0; i < 12; i++) {
stringBuilder.append(CONTENT_PLAYBACK_NONCE_ALPHABET.charAt(
(random.nextInt(128) + 1) & 63));
}
return stringBuilder.toString();
}
}

278
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -1,8 +1,12 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.createPlayerBodyWithSts;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.VIDEO_ID;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.createDesktopPlayerBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonMobilePostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateContentPlaybackNonce;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonAndroidPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareAndroidMobileEmbedVideoJsonBuilder;
@ -69,7 +73,6 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@ -119,13 +122,16 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nullable
private JsonObject desktopStreamingData;
@Nullable
private JsonObject mobileStreamingData;
private JsonObject androidStreamingData;
private JsonObject videoPrimaryInfoRenderer;
private JsonObject videoSecondaryInfoRenderer;
private int ageLimit = -1;
@Nullable
private List<SubtitlesStream> subtitles = null;
private String desktopCpn;
private String androidCpn;
public YoutubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler) {
super(service, linkHandler);
}
@ -310,8 +316,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final String durationMs = adaptiveFormats.getObject(0)
.getString("approxDurationMs");
return Math.round(Long.parseLong(durationMs) / 1000f);
} else if (mobileStreamingData != null) {
final JsonArray adaptiveFormats = mobileStreamingData.getArray(ADAPTIVE_FORMATS);
} else if (androidStreamingData != null) {
final JsonArray adaptiveFormats = androidStreamingData.getArray(ADAPTIVE_FORMATS);
final String durationMs = adaptiveFormats.getObject(0)
.getString("approxDurationMs");
return Math.round(Long.parseLong(durationMs) / 1000f);
@ -493,8 +499,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (desktopStreamingData != null) {
return desktopStreamingData.getString("dashManifestUrl");
} else if (mobileStreamingData != null) {
return mobileStreamingData.getString("dashManifestUrl");
} else if (androidStreamingData != null) {
return androidStreamingData.getString("dashManifestUrl");
} else {
return EMPTY_STRING;
}
@ -507,8 +513,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (desktopStreamingData != null) {
return desktopStreamingData.getString("hlsManifestUrl");
} else if (mobileStreamingData != null) {
return mobileStreamingData.getString("hlsManifestUrl");
} else if (androidStreamingData != null) {
return androidStreamingData.getString("hlsManifestUrl");
} else {
return EMPTY_STRING;
}
@ -710,6 +716,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private static final String FORMATS = "formats";
private static final String ADAPTIVE_FORMATS = "adaptiveFormats";
private static final String DEOBFUSCATION_FUNC_NAME = "deobfuscate";
private static final String STREAMING_DATA = "streamingData";
private static final String PLAYER = "player";
private static final String NEXT = "next";
private static final String[] REGEXES = {
"(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2,})\\s*=\\s*function\\(\\s*a\\s*\\)"
@ -725,24 +734,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
if (sts == null) {
getStsFromPlayerJs();
}
final String videoId = getId();
final Localization localization = getExtractorLocalization();
final ContentCountry contentCountry = getExtractorContentCountry();
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
localization, contentCountry)
.value("videoId", videoId)
.done())
.getBytes(UTF_8);
desktopCpn = generateContentPlaybackNonce();
// Put the sts string if we already know it so we don't have to fetch again the player
// endpoint of the desktop internal API if something went wrong when parsing the Android
// API.
if (sts != null) {
playerResponse = getJsonPostResponse("player", createPlayerBodyWithSts(localization,
contentCountry, videoId, false, sts), localization);
} else {
playerResponse = getJsonPostResponse("player", body, localization);
}
playerResponse = getJsonPostResponse(PLAYER,
createDesktopPlayerBody(localization, contentCountry, videoId, sts, false,
desktopCpn),
localization);
// Save the playerResponse from the player endpoint of the desktop internal API because
// there can be restrictions on the embedded player.
@ -759,7 +763,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final boolean ageRestricted = playabilityStatus.getString("reason", EMPTY_STRING)
.contains("age");
if (!playerResponse.has("streamingData")) {
if (!playerResponse.has(STREAMING_DATA)) {
try {
fetchDesktopEmbedJsonPlayer(contentCountry, localization, videoId);
} catch (final Exception ignored) {
@ -770,8 +774,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
if (desktopStreamingData == null && playerResponse.has("streamingData")) {
desktopStreamingData = playerResponse.getObject("streamingData");
if (desktopStreamingData == null && playerResponse.has(STREAMING_DATA)) {
desktopStreamingData = playerResponse.getObject(STREAMING_DATA);
}
if (desktopStreamingData == null) {
@ -781,11 +785,17 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (ageRestricted) {
final byte[] ageRestrictedBody = JsonWriter.string(prepareDesktopEmbedVideoJsonBuilder(
localization, contentCountry, videoId)
.value(VIDEO_ID, videoId)
.done())
.getBytes(UTF_8);
nextResponse = getJsonPostResponse("next", ageRestrictedBody, localization);
nextResponse = getJsonPostResponse(NEXT, ageRestrictedBody, localization);
} else {
nextResponse = getJsonPostResponse("next", body, localization);
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization,
contentCountry)
.value(VIDEO_ID, videoId)
.done())
.getBytes(UTF_8);
nextResponse = getJsonPostResponse(NEXT, body, localization);
}
if (!ageRestricted) {
@ -794,10 +804,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} catch (final Exception ignored) {
}
}
if (isCipherProtectedContent()) {
fetchDesktopJsonPlayerWithSts(contentCountry, localization, videoId);
}
}
private void checkPlayabilityStatus(final JsonObject youtubePlayerResponse,
@ -825,28 +831,26 @@ public class YoutubeStreamExtractor extends StreamExtractor {
"This age-restricted video cannot be watched.");
}
}
if (status.equalsIgnoreCase("unplayable")) {
if (reason != null) {
if (reason.contains("Music Premium")) {
throw new YoutubeMusicPremiumContentException();
}
if (reason.contains("payment")) {
throw new PaidContentException("This video is a paid video");
}
if (reason.contains("members-only")) {
throw new PaidContentException("This video is only available"
+ " for members of the channel of this video");
}
if (reason.contains("unavailable")) {
final String detailedErrorMessage = getTextFromObject(newPlayabilityStatus
.getObject("errorScreen").getObject("playerErrorMessageRenderer")
.getObject("subreason"));
if (detailedErrorMessage != null) {
if (detailedErrorMessage.contains("country")) {
throw new GeographicRestrictionException(
"This video is not available in user's country.");
}
}
if (status.equalsIgnoreCase("unplayable") && reason != null) {
if (reason.contains("Music Premium")) {
throw new YoutubeMusicPremiumContentException();
}
if (reason.contains("payment")) {
throw new PaidContentException("This video is a paid video");
}
if (reason.contains("members-only")) {
throw new PaidContentException("This video is only available"
+ " for members of the channel of this video");
}
if (reason.contains("unavailable")) {
final String detailedErrorMessage = getTextFromObject(newPlayabilityStatus
.getObject("errorScreen").getObject("playerErrorMessageRenderer")
.getObject("subreason"));
if (detailedErrorMessage != null && detailedErrorMessage.contains("country")) {
throw new GeographicRestrictionException(
"This video is not available in client's country.");
}
}
}
@ -859,70 +863,50 @@ public class YoutubeStreamExtractor extends StreamExtractor {
* Fetch the Android Mobile API and assign the streaming data to the mobileStreamingData JSON
* object.
*/
private void fetchAndroidMobileJsonPlayer(final ContentCountry contentCountry,
final Localization localization,
final String videoId)
private void fetchAndroidMobileJsonPlayer(@Nonnull final ContentCountry contentCountry,
@Nonnull final Localization localization,
@Nonnull final String videoId)
throws IOException, ExtractionException {
final byte[] mobileBody = JsonWriter.string(prepareAndroidMobileJsonBuilder(
localization, contentCountry)
.value("videoId", videoId)
.value(VIDEO_ID, videoId)
.value(CPN, androidCpn)
.done())
.getBytes(UTF_8);
final JsonObject mobilePlayerResponse = getJsonMobilePostResponse("player",
mobileBody, contentCountry, localization);
final JsonObject streamingData = mobilePlayerResponse.getObject("streamingData");
final JsonObject androidPlayerResponse = getJsonAndroidPostResponse(PLAYER,
mobileBody, contentCountry, localization, "&t=" + generateTParameter()
+ "&id=" + videoId);
final JsonObject streamingData = androidPlayerResponse.getObject(STREAMING_DATA);
if (!isNullOrEmpty(streamingData)) {
mobileStreamingData = streamingData;
androidStreamingData = streamingData;
if (desktopStreamingData == null) {
playerResponse = mobilePlayerResponse;
playerResponse = androidPlayerResponse;
}
}
}
/**
* Fetch the desktop API with the {@code signatureTimestamp} and assign the streaming data to
* the {@code desktopStreamingData} JSON object.
* The cipher signatures from the player endpoint without a signatureTimestamp are invalid so
* if the content is protected by signatureCiphers and if signatureTimestamp is not known, we
* need to fetch again the desktop InnerTube API.
*/
private void fetchDesktopJsonPlayerWithSts(final ContentCountry contentCountry,
final Localization localization,
final String videoId)
throws IOException, ExtractionException {
if (sts == null) {
getStsFromPlayerJs();
}
final JsonObject playerResponseWithSignatureTimestamp = getJsonPostResponse(
"player", createPlayerBodyWithSts(
localization, contentCountry, videoId, false, sts),
localization);
if (playerResponseWithSignatureTimestamp.has("streamingData")) {
desktopStreamingData = playerResponseWithSignatureTimestamp.getObject("streamingData");
}
}
/**
* Download again the desktop JSON player as an embed client to bypass some age-restrictions.
* <p>
* We need also to get the {@code signatureTimestamp}, if it isn't known because we don't know
* if the video will have signature ciphers or not.
* </p>
*/
private void fetchDesktopEmbedJsonPlayer(final ContentCountry contentCountry,
final Localization localization,
final String videoId)
private void fetchDesktopEmbedJsonPlayer(@Nonnull final ContentCountry contentCountry,
@Nonnull final Localization localization,
@Nonnull final String videoId)
throws IOException, ExtractionException {
if (sts == null) {
getStsFromPlayerJs();
}
final JsonObject desktopWebEmbedPlayerResponse = getJsonPostResponse(
"player", createPlayerBodyWithSts(
localization, contentCountry, videoId, true, sts),
// Because a cpn is unique to each request, we need to generate it again
desktopCpn = generateContentPlaybackNonce();
final JsonObject desktopWebEmbedPlayerResponse = getJsonPostResponse(PLAYER,
createDesktopPlayerBody(localization, contentCountry, videoId, sts, true,
desktopCpn),
localization);
final JsonObject streamingData = desktopWebEmbedPlayerResponse.getObject(
"streamingData");
STREAMING_DATA);
if (!isNullOrEmpty(streamingData)) {
playerResponse = desktopWebEmbedPlayerResponse;
desktopStreamingData = streamingData;
@ -932,27 +916,32 @@ public class YoutubeStreamExtractor extends StreamExtractor {
/**
* Download the Android mobile JSON player as an embed client to bypass some age-restrictions.
*/
private void fetchAndroidEmbedJsonPlayer(final ContentCountry contentCountry,
final Localization localization,
final String videoId)
private void fetchAndroidEmbedJsonPlayer(@Nonnull final ContentCountry contentCountry,
@Nonnull final Localization localization,
@Nonnull final String videoId)
throws IOException, ExtractionException {
// Because a cpn is unique to each request, we need to generate it again
androidCpn = generateContentPlaybackNonce();
final byte[] androidMobileEmbedBody = JsonWriter.string(
prepareAndroidMobileEmbedVideoJsonBuilder(localization, contentCountry, videoId)
.done())
prepareAndroidMobileEmbedVideoJsonBuilder(localization, contentCountry, videoId,
androidCpn)
.done())
.getBytes(UTF_8);
final JsonObject androidMobileEmbedPlayerResponse = getJsonMobilePostResponse("player",
androidMobileEmbedBody, contentCountry, localization);
final JsonObject androidMobileEmbedPlayerResponse = getJsonAndroidPostResponse(PLAYER,
androidMobileEmbedBody, contentCountry, localization, "&t=" + generateTParameter()
+ "&id=" + videoId);
final JsonObject streamingData = androidMobileEmbedPlayerResponse.getObject(
"streamingData");
STREAMING_DATA);
if (!isNullOrEmpty(streamingData)) {
if (desktopStreamingData == null) {
playerResponse = androidMobileEmbedPlayerResponse;
}
mobileStreamingData = androidMobileEmbedPlayerResponse.getObject("streamingData");
androidStreamingData = androidMobileEmbedPlayerResponse.getObject(STREAMING_DATA);
}
}
private void storePlayerJs() throws ParsingException {
private static void storePlayerJs() throws ParsingException {
try {
playerCode = YoutubeJavaScriptExtractor.extractJavaScriptCode();
} catch (final Exception e) {
@ -960,38 +949,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
private boolean isCipherProtectedContent() {
if (desktopStreamingData != null) {
if (desktopStreamingData.has(ADAPTIVE_FORMATS)) {
final JsonArray adaptiveFormats = desktopStreamingData.getArray(ADAPTIVE_FORMATS);
if (!isNullOrEmpty(adaptiveFormats)) {
for (final Object adaptiveFormat : adaptiveFormats) {
final JsonObject adaptiveFormatJsonObject = ((JsonObject) adaptiveFormat);
if (adaptiveFormatJsonObject.has("signatureCipher")
|| adaptiveFormatJsonObject.has("cipher")) {
return true;
}
}
}
}
if (desktopStreamingData.has(FORMATS)) {
final JsonArray formats = desktopStreamingData.getArray(FORMATS);
if (!isNullOrEmpty(formats)) {
for (final Object format : formats) {
final JsonObject formatJsonObject = ((JsonObject) format);
if (formatJsonObject.has("signatureCipher")
|| formatJsonObject.has("cipher")) {
return true;
}
}
}
}
}
return false;
}
private String getDeobfuscationFuncName(final String thePlayerCode)
throws DeobfuscateException {
private static String getDeobfuscationFuncName(final String thePlayerCode) throws DeobfuscateException {
Parser.RegexException exception = null;
for (final String regex : REGEXES) {
try {
@ -1007,7 +965,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
@Nonnull
private String loadDeobfuscationCode() throws DeobfuscateException {
private static String loadDeobfuscationCode() throws DeobfuscateException {
try {
final String deobfuscationFunctionName = getDeobfuscationFuncName(playerCode);
@ -1024,7 +982,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
"(var " + helperObjectName.replace("$", "\\$")
+ "=\\{.+?\\}\\};)";
final String helperObject =
Parser.matchGroup1(helperPattern, playerCode.replace("\n", ""));
Parser.matchGroup1(helperPattern, Objects.requireNonNull(playerCode).replace(
"\n", ""));
final String callerFunction =
"function " + DEOBFUSCATION_FUNC_NAME + "(a){return "
@ -1037,7 +996,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
@Nonnull
private String getDeobfuscationCode() throws ParsingException {
private static String getDeobfuscationCode() throws ParsingException {
if (cachedDeobfuscationCode == null) {
if (isNullOrEmpty(playerCode)) {
throw new ParsingException("playerCode is null");
@ -1048,7 +1007,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return cachedDeobfuscationCode;
}
private void getStsFromPlayerJs() throws ParsingException {
private static void getStsFromPlayerJs() throws ParsingException {
if (!isNullOrEmpty(sts)) {
return;
}
@ -1085,8 +1044,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
//////////////////////////////////////////////////////////////////////////*/
private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException {
if (this.videoPrimaryInfoRenderer != null) {
return this.videoPrimaryInfoRenderer;
if (videoPrimaryInfoRenderer != null) {
return videoPrimaryInfoRenderer;
}
final JsonArray contents = nextResponse.getObject("contents")
@ -1106,18 +1065,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
throw new ParsingException("Could not find videoPrimaryInfoRenderer");
}
this.videoPrimaryInfoRenderer = theVideoPrimaryInfoRenderer;
videoPrimaryInfoRenderer = theVideoPrimaryInfoRenderer;
return theVideoPrimaryInfoRenderer;
}
private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException {
if (this.videoSecondaryInfoRenderer != null) {
return this.videoSecondaryInfoRenderer;
if (videoSecondaryInfoRenderer != null) {
return videoSecondaryInfoRenderer;
}
final JsonArray contents = nextResponse.getObject("contents")
.getObject("twoColumnWatchNextResults").getObject("results").getObject("results")
.getArray("contents");
JsonObject theVideoSecondaryInfoRenderer = null;
for (final Object content : contents) {
@ -1132,24 +1092,24 @@ public class YoutubeStreamExtractor extends StreamExtractor {
throw new ParsingException("Could not find videoSecondaryInfoRenderer");
}
this.videoSecondaryInfoRenderer = theVideoSecondaryInfoRenderer;
videoSecondaryInfoRenderer = theVideoSecondaryInfoRenderer;
return theVideoSecondaryInfoRenderer;
}
@Nonnull
private Map<String, ItagItem> getItags(final String streamingDataKey,
final ItagItem.ItagType itagTypeWanted) {
private Map<String, ItagItem> getItags(@Nonnull final String streamingDataKey,
@Nonnull final ItagItem.ItagType itagTypeWanted) {
final Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();
if (desktopStreamingData == null && mobileStreamingData == null) {
if (desktopStreamingData == null && androidStreamingData == null) {
return urlAndItags;
}
// Use the mobileStreamingData object first because there is no n param and no
// signatureCiphers in streaming URLs of the Android client
urlAndItags.putAll(getStreamsFromStreamingDataKey(
mobileStreamingData, streamingDataKey, itagTypeWanted));
androidStreamingData, streamingDataKey, itagTypeWanted, androidCpn));
urlAndItags.putAll(getStreamsFromStreamingDataKey(
desktopStreamingData, streamingDataKey, itagTypeWanted));
desktopStreamingData, streamingDataKey, itagTypeWanted, desktopCpn));
return urlAndItags;
}
@ -1157,8 +1117,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull
private Map<String, ItagItem> getStreamsFromStreamingDataKey(
final JsonObject streamingData,
final String streamingDataKey,
final ItagItem.ItagType itagTypeWanted) {
@Nonnull final String streamingDataKey,
@Nonnull final ItagItem.ItagType itagTypeWanted,
@Nonnull final String contentPlaybackNonce) {
final Map<String, ItagItem> urlAndItagsFromStreamingDataObject = new LinkedHashMap<>();
if (streamingData != null && streamingData.has(streamingDataKey)) {
@ -1180,7 +1141,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final String streamUrl;
if (formatData.has("url")) {
streamUrl = formatData.getString("url");
streamUrl = formatData.getString("url") + "&cpn="
+ contentPlaybackNonce;
} else {
// This url has an obfuscated signature
final String cipherString = formatData.has("cipher")