lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix: YoutubeParsingHelper formatting

This commit is contained in:
ThetaDev 2022-11-22 01:59:51 +01:00
parent 2245de1197
commit 8d3bc2bc4b
1 changed files with 131 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

266
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -483,7 +483,7 @@ public final class YoutubeParsingHelper {
/**
* @param playlistId the playlist id to parse
* @return the {@link PlaylistInfo.PlaylistType} extracted from the playlistId (mix playlist
* types included)
* types included)
* @throws ParsingException if the playlistId is null or empty, if the playlistId is not a mix,
* if it is a mix but it's not based on a specific stream (this is the
* case for channel or genre mixes)
@ -516,7 +516,7 @@ public final class YoutubeParsingHelper {
// 11 characters then it can't be a video id, hence we are dealing with a different
// type of mix (e.g. genre mixes handled above, of the form RDGMEM{garbage})
throw new ParsingException("Video id could not be determined from mix id: "
+ playlistId);
+ playlistId);
}
return playlistId.substring(2);
@ -529,7 +529,7 @@ public final class YoutubeParsingHelper {
/**
* @param playlistId the playlist id to parse
* @return the {@link PlaylistInfo.PlaylistType} extracted from the playlistId (mix playlist
* types included)
* types included)
* @throws ParsingException if the playlistId is null or empty
*/
@Nonnull
@ -557,7 +557,7 @@ public final class YoutubeParsingHelper {
/**
* @param playlistUrl the playlist url to parse
* @return the {@link PlaylistInfo.PlaylistType} extracted from the playlistUrl's list param
* (mix playlist types included)
* (mix playlist types included)
* @throws ParsingException if the playlistUrl is malformed, if has no list param or if the list
* param is empty
*/
@ -587,20 +587,20 @@ public final class YoutubeParsingHelper {
}
// @formatter:off
final byte[] body = JsonWriter.string()
.object()
.object()
.object("context")
.object("client")
.value("hl", "en-GB")
.value("gl", "GB")
.value("clientName", "WEB")
.value("clientVersion", HARDCODED_CLIENT_VERSION)
.end()
.object("client")
.value("hl", "en-GB")
.value("gl", "GB")
.value("clientName", "WEB")
.value("clientVersion", HARDCODED_CLIENT_VERSION)
.end()
.object("user")
.value("lockedSafetyMode", false)
.value("lockedSafetyMode", false)
.end()
.value("fetchLiveState", true)
.end()
.end().done().getBytes(UTF_8);
.end().done().getBytes(UTF_8);
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();
@ -611,7 +611,7 @@ public final class YoutubeParsingHelper {
// This endpoint is fetched by the YouTube website to get the items of its main menu and is
// pretty lightweight (around 30kB)
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "guide?key="
+ HARDCODED_KEY + DISABLE_PRETTY_PRINT_PARAMETER, headers, body);
+ HARDCODED_KEY + DISABLE_PRETTY_PRINT_PARAMETER, headers, body);
final String responseBody = response.responseBody();
final int responseCode = response.responseCode();
@ -688,14 +688,14 @@ public final class YoutubeParsingHelper {
throw new ParsingException(
// CHECKSTYLE:OFF
"Could not extract YouTube WEB InnerTube API key from HTML search results page");
// CHECKSTYLE:ON
// CHECKSTYLE:ON
}
if (clientVersion == null) {
throw new ParsingException(
// CHECKSTYLE:OFF
"Could not extract YouTube WEB InnerTube client version from HTML search results page");
// CHECKSTYLE:ON
// CHECKSTYLE:ON
}
keyAndVersionExtracted = true;
@ -822,30 +822,30 @@ public final class YoutubeParsingHelper {
// @formatter:off
final byte[] json = JsonWriter.string()
.object()
.object()
.object("context")
.object("client")
.value("clientName", "WEB_REMIX")
.value("clientVersion", HARDCODED_YOUTUBE_MUSIC_KEY[2])
.value("hl", "en-GB")
.value("gl", "GB")
.array("experimentIds").end()
.value("experimentsToken", "")
.object("locationInfo").end()
.object("musicAppInfo").end()
.end()
.object("capabilities").end()
.object("request")
.array("internalExperimentFlags").end()
.object("sessionIndex").end()
.end()
.object("activePlayers").end()
.object("user")
.value("enableSafetyMode", false)
.end()
.object("client")
.value("clientName", "WEB_REMIX")
.value("clientVersion", HARDCODED_YOUTUBE_MUSIC_KEY[2])
.value("hl", "en-GB")
.value("gl", "GB")
.array("experimentIds").end()
.value("experimentsToken", "")
.object("locationInfo").end()
.object("musicAppInfo").end()
.end()
.object("capabilities").end()
.object("request")
.array("internalExperimentFlags").end()
.object("sessionIndex").end()
.end()
.object("activePlayers").end()
.object("user")
.value("enableSafetyMode", false)
.end()
.end()
.value("input", "")
.end().done().getBytes(UTF_8);
.end().done().getBytes(UTF_8);
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();
@ -882,10 +882,10 @@ public final class YoutubeParsingHelper {
headers.put("Origin", singletonList("https://music.youtube.com"));
headers.put("Referer", singletonList("https://music.youtube.com"));
final String response = getDownloader().get(url, headers).responseBody();
musicClientVersion = getStringResultFromRegexArray(response,
INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1);
musicKey = getStringResultFromRegexArray(response, INNERTUBE_API_KEY_REGEXES, 1);
musicClientName = Parser.matchGroup1(INNERTUBE_CLIENT_NAME_REGEX, response);
musicClientVersion = getStringResultFromRegexArray(response,
INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1);
musicKey = getStringResultFromRegexArray(response, INNERTUBE_API_KEY_REGEXES, 1);
musicClientName = Parser.matchGroup1(INNERTUBE_CLIENT_NAME_REGEX, response);
} catch (final Exception e) {
final String url = "https://music.youtube.com/?ucbcb=1";
final String html = getDownloader().get(url, getCookieHeader()).responseBody();
@ -896,7 +896,7 @@ public final class YoutubeParsingHelper {
musicClientName = Parser.matchGroup1(INNERTUBE_CLIENT_NAME_REGEX, html);
}
youtubeMusicKey = new String[]{musicKey, musicClientName, musicClientVersion};
youtubeMusicKey = new String[] {musicKey, musicClientName, musicClientVersion};
return youtubeMusicKey;
}
@ -1253,29 +1253,29 @@ public final class YoutubeParsingHelper {
// @formatter:off
final JsonBuilder<JsonObject> builder = JsonObject.builder()
.object("context")
.object("client")
.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.value("clientName", "WEB")
.value("clientVersion", getClientVersion())
.value("originalUrl", "https://www.youtube.com")
.value("platform", "DESKTOP");
.object("client")
.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.value("clientName", "WEB")
.value("clientVersion", getClientVersion())
.value("originalUrl", "https://www.youtube.com")
.value("platform", "DESKTOP");
if (visitorData != null) {
builder.value("visitorData", visitorData);
}
builder.end()
.object("request")
.array("internalExperimentFlags")
.end()
.value("useSsl", true)
.end()
.object("user")
// TO DO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.object("request")
.array("internalExperimentFlags")
.end()
.value("useSsl", true)
.end()
.object("user")
// TO DO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.end();
// @formatter:on
@ -1289,32 +1289,32 @@ public final class YoutubeParsingHelper {
// @formatter:off
return JsonObject.builder()
.object("context")
.object("client")
.value("clientName", "ANDROID")
.value("clientVersion", ANDROID_YOUTUBE_CLIENT_VERSION)
.value("platform", "MOBILE")
.value("osName", "Android")
.value("osVersion", "12")
/*
A valid Android SDK version is required to be sure to get a valid player
response
If this parameter is not provided, the player response may be replaced by
the one of a 5-minute video saying the message "The following content is
not available on this app. Watch this content on the latest version on
YouTube"
See https://github.com/TeamNewPipe/NewPipe/issues/8713
The Android SDK version corresponding to the Android version used in
requests is sent
*/
.value("androidSdkVersion", 31)
.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.end()
.object("user")
// TO DO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.object("client")
.value("clientName", "ANDROID")
.value("clientVersion", ANDROID_YOUTUBE_CLIENT_VERSION)
.value("platform", "MOBILE")
.value("osName", "Android")
.value("osVersion", "12")
/*
A valid Android SDK version is required to be sure to get a valid player
response
If this parameter is not provided, the player response may be replaced by
the one of a 5-minute video saying the message "The following content is
not available on this app. Watch this content on the latest version on
YouTube"
See https://github.com/TeamNewPipe/NewPipe/issues/8713
The Android SDK version corresponding to the Android version used in
requests is sent
*/
.value("androidSdkVersion", 31)
.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.end()
.object("user")
// TO DO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.end();
// @formatter:on
}
@ -1326,27 +1326,27 @@ public final class YoutubeParsingHelper {
// @formatter:off
return JsonObject.builder()
.object("context")
.object("client")
.value("clientName", "IOS")
.value("clientVersion", IOS_YOUTUBE_CLIENT_VERSION)
.value("deviceMake", "Apple")
// Device model is required to get 60fps streams
.value("deviceModel", IOS_DEVICE_MODEL)
.value("platform", "MOBILE")
.value("osName", "iOS")
// The value of this field seems to use the following structure:
// "iOS version.0.build version"
// The build version corresponding to the iOS version used can be found on
// https://www.theiphonewiki.com/wiki/Firmware/iPhone/15.x#iPhone_13
.value("osVersion", "15.6.0.19G71")
.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.end()
.object("user")
// TO DO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.object("client")
.value("clientName", "IOS")
.value("clientVersion", IOS_YOUTUBE_CLIENT_VERSION)
.value("deviceMake", "Apple")
// Device model is required to get 60fps streams
.value("deviceModel", IOS_DEVICE_MODEL)
.value("platform", "MOBILE")
.value("osName", "iOS")
// The value of this field seems to use the following structure:
// "iOS version.0.build version"
// The build version corresponding to the iOS version used can be found on
// https://www.theiphonewiki.com/wiki/Firmware/iPhone/15.x#iPhone_13
.value("osVersion", "15.6.0.19G71")
.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.end()
.object("user")
// TO DO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.end();
// @formatter:on
}
@ -1359,22 +1359,22 @@ public final class YoutubeParsingHelper {
// @formatter:off
return JsonObject.builder()
.object("context")
.object("client")
.value("clientName", "TVHTML5_SIMPLY_EMBEDDED_PLAYER")
.value("clientVersion", TVHTML5_SIMPLY_EMBED_CLIENT_VERSION)
.value("clientScreen", "EMBED")
.value("platform", "TV")
.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.end()
.object("thirdParty")
.value("embedUrl", "https://www.youtube.com/watch?v=" + videoId)
.end()
.object("user")
// TO DO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.object("client")
.value("clientName", "TVHTML5_SIMPLY_EMBEDDED_PLAYER")
.value("clientVersion", TVHTML5_SIMPLY_EMBED_CLIENT_VERSION)
.value("clientScreen", "EMBED")
.value("platform", "TV")
.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.end()
.object("thirdParty")
.value("embedUrl", "https://www.youtube.com/watch?v=" + videoId)
.end()
.object("user")
// TO DO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.end();
// @formatter:on
}
@ -1391,19 +1391,19 @@ public final class YoutubeParsingHelper {
return JsonWriter.string((isTvHtml5DesktopJsonBuilder
? prepareTvHtml5EmbedJsonBuilder(localization, contentCountry, videoId)
: prepareDesktopJsonBuilder(localization, contentCountry))
.object("playbackContext")
.object("contentPlaybackContext")
.object("playbackContext")
.object("contentPlaybackContext")
// Signature timestamp from the JavaScript base player is needed to get
// working obfuscated URLs
.value("signatureTimestamp", sts)
.value("referer", "https://www.youtube.com/watch?v=" + videoId)
.end()
.end()
.value(CPN, contentPlaybackNonce)
.value(VIDEO_ID, videoId)
.value(CONTENT_CHECK_OK, true)
.value(RACY_CHECK_OK, true)
.done())
.end()
.end()
.value(CPN, contentPlaybackNonce)
.value(VIDEO_ID, videoId)
.value(CONTENT_CHECK_OK, true)
.value(RACY_CHECK_OK, true)
.done())
.getBytes(StandardCharsets.UTF_8);
// @formatter:on
}
@ -1454,7 +1454,6 @@ public final class YoutubeParsingHelper {
/**
* Add required headers and cookies to an existing headers Map.
*
* @see #addClientInfoHeaders(Map)
* @see #addCookieHeader(Map)
*/
@ -1467,7 +1466,6 @@ public final class YoutubeParsingHelper {
/**
* Add the <code>X-YouTube-Client-Name</code>, <code>X-YouTube-Client-Version</code>,
* <code>Origin</code>, and <code>Referer</code> headers.
*
* @param headers The headers which should be completed
*/
public static void addClientInfoHeaders(@Nonnull final Map<String, List<String>> headers)
@ -1482,7 +1480,6 @@ public final class YoutubeParsingHelper {
/**
* Create a map with the required cookie header.
*
* @return A singleton map containing the header.
*/
public static Map<String, List<String>> getCookieHeader() {
@ -1491,7 +1488,6 @@ public final class YoutubeParsingHelper {
/**
* Add the <code>CONSENT</code> cookie to prevent redirect to <code>consent.youtube.com</code>
*
* @param headers the headers which should be completed
*/
public static void addCookieHeader(@Nonnull final Map<String, List<String>> headers) {