diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index b6ea901d3..7acfeb46e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -100,17 +100,17 @@ public final class YoutubeParsingHelper { * sizes. * *

- * Sent in query parameters of the requests, after the API key. + * Sent in query parameters of the requests. *

**/ - public static final String DISABLE_PRETTY_PRINT_PARAMETER = "&prettyPrint=false"; + public static final String DISABLE_PRETTY_PRINT_PARAMETER = "prettyPrint=false"; /** * A parameter sent by official clients named {@code contentPlaybackNonce}. * *

- * It is sent by official clients on videoplayback requests, and by all clients (except the - * {@code WEB} one to the player requests. + * It is sent by official clients on videoplayback requests and InnerTube player requests in + * most cases. *

* *

@@ -148,13 +148,7 @@ public final class YoutubeParsingHelper { * The client version for InnerTube requests with the {@code WEB} client, used as the last * fallback if the extraction of the real one failed. */ - private static final String HARDCODED_CLIENT_VERSION = "2.20231208.01.00"; - - /** - * The InnerTube API key which should be used by YouTube's desktop website, used as a fallback - * if the extraction of the real one failed. - */ - private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; + private static final String HARDCODED_CLIENT_VERSION = "2.20240410.01.00"; /** * The hardcoded client version of the Android app used for InnerTube requests with this @@ -165,17 +159,10 @@ public final class YoutubeParsingHelper { * such as APKMirror. *

*/ - private static final String ANDROID_YOUTUBE_CLIENT_VERSION = "18.48.37"; + private static final String ANDROID_YOUTUBE_CLIENT_VERSION = "19.13.36"; /** - * The InnerTube API key used by the {@code ANDROID} client. Found with the help of - * reverse-engineering app network requests. - */ - private static final String ANDROID_YOUTUBE_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w"; - - /** - * The hardcoded client version of the iOS app used for InnerTube requests with this - * client. + * The hardcoded client version of the iOS app used for InnerTube requests with this client. * *

* It can be extracted by getting the latest release version of the app on @@ -183,42 +170,34 @@ public final class YoutubeParsingHelper { * Store page of the YouTube app, in the {@code What’s New} section. *

*/ - private static final String IOS_YOUTUBE_CLIENT_VERSION = "18.48.3"; - - /** - * The InnerTube API key used by the {@code iOS} client. Found with the help of - * reverse-engineering app network requests. - */ - private static final String IOS_YOUTUBE_KEY = "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc"; + private static final String IOS_YOUTUBE_CLIENT_VERSION = "19.14.3"; /** * The hardcoded client version used for InnerTube requests with the TV HTML5 embed client. */ private static final String TVHTML5_SIMPLY_EMBED_CLIENT_VERSION = "2.0"; + /** + * The hardcoded client version used for InnerTube requests with the YouTube Music desktop + * client. + */ + private static final String HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION = "1.20240403.01.00"; + private static String clientVersion; - private static String key; - private static final String[] HARDCODED_YOUTUBE_MUSIC_KEY = - {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "1.20231204.01.00"}; - private static String[] youtubeMusicKey; + private static String youtubeMusicClientVersion; - private static boolean keyAndVersionExtracted = false; + private static boolean clientVersionExtracted = false; @SuppressWarnings("OptionalUsedAsFieldOrParameterType") - private static Optional hardcodedClientVersionAndKeyValid = Optional.empty(); + private static Optional hardcodedClientVersionValid = Optional.empty(); private static final String[] INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES = {"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", "innertube_context_client_version\":\"([0-9\\.]+?)\"", "client.version=([0-9\\.]+)"}; - private static final String[] INNERTUBE_API_KEY_REGEXES = - {"INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", - "innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\""}; private static final String[] INITIAL_DATA_REGEXES = {"window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", "var\\s*ytInitialData\\s*=\\s*(\\{.*?\\});"}; - private static final String INNERTUBE_CLIENT_NAME_REGEX = - "INNERTUBE_CONTEXT_CLIENT_NAME\":([0-9]+?),"; private static final String CONTENT_PLAYBACK_NONCE_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; @@ -518,10 +497,10 @@ public final class YoutubeParsingHelper { } } - public static boolean areHardcodedClientVersionAndKeyValid() + public static boolean isHardcodedClientVersionValid() throws IOException, ExtractionException { - if (hardcodedClientVersionAndKeyValid.isPresent()) { - return hardcodedClientVersionAndKeyValid.get(); + if (hardcodedClientVersionValid.isPresent()) { + return hardcodedClientVersionValid.get(); } // @formatter:off final byte[] body = JsonWriter.string() @@ -555,20 +534,20 @@ public final class YoutubeParsingHelper { // This endpoint is fetched by the YouTube website to get the items of its main menu and is // pretty lightweight (around 30kB) final Response response = getDownloader().postWithContentTypeJson( - YOUTUBEI_V1_URL + "guide?key=" + HARDCODED_KEY + DISABLE_PRETTY_PRINT_PARAMETER, + YOUTUBEI_V1_URL + "guide?" + DISABLE_PRETTY_PRINT_PARAMETER, headers, body); final String responseBody = response.responseBody(); final int responseCode = response.responseCode(); - hardcodedClientVersionAndKeyValid = Optional.of(responseBody.length() > 5000 + hardcodedClientVersionValid = Optional.of(responseBody.length() > 5000 && responseCode == 200); // Ensure to have a valid response - return hardcodedClientVersionAndKeyValid.get(); + return hardcodedClientVersionValid.get(); } - private static void extractClientVersionAndKeyFromSwJs() + private static void extractClientVersionFromSwJs() throws IOException, ExtractionException { - if (keyAndVersionExtracted) { + if (clientVersionExtracted) { return; } final String url = "https://www.youtube.com/sw.js"; @@ -577,18 +556,17 @@ public final class YoutubeParsingHelper { try { clientVersion = getStringResultFromRegexArray(response, INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1); - key = getStringResultFromRegexArray(response, INNERTUBE_API_KEY_REGEXES, 1); } catch (final Parser.RegexException e) { throw new ParsingException("Could not extract YouTube WEB InnerTube client version " - + "and API key from sw.js", e); + + "from sw.js", e); } - keyAndVersionExtracted = true; + clientVersionExtracted = true; } - private static void extractClientVersionAndKeyFromHtmlSearchResultsPage() + private static void extractClientVersionFromHtmlSearchResultsPage() throws IOException, ExtractionException { - // Don't extract the client version and the InnerTube key if it has been already extracted - if (keyAndVersionExtracted) { + // Don't extract the InnerTube client version if it has been already extracted + if (clientVersionExtracted) { return; } @@ -622,18 +600,6 @@ public final class YoutubeParsingHelper { serviceTrackingParamsStream, "ECATCHER", "client.version"); } - try { - key = getStringResultFromRegexArray(html, INNERTUBE_API_KEY_REGEXES, 1); - } catch (final Parser.RegexException ignored) { - } - - if (isNullOrEmpty(key)) { - throw new ParsingException( - // CHECKSTYLE:OFF - "Could not extract YouTube WEB InnerTube API key from HTML search results page"); - // CHECKSTYLE:ON - } - if (clientVersion == null) { throw new ParsingException( // CHECKSTYLE:OFF @@ -641,7 +607,7 @@ public final class YoutubeParsingHelper { // CHECKSTYLE:ON } - keyAndVersionExtracted = true; + clientVersionExtracted = true; } @Nullable @@ -676,17 +642,17 @@ public final class YoutubeParsingHelper { // JavaScript service worker, then from HTML search results page as a fallback, to prevent // fingerprinting based on the client version used try { - extractClientVersionAndKeyFromSwJs(); + extractClientVersionFromSwJs(); } catch (final Exception e) { - extractClientVersionAndKeyFromHtmlSearchResultsPage(); + extractClientVersionFromHtmlSearchResultsPage(); } - if (keyAndVersionExtracted) { + if (clientVersionExtracted) { return clientVersion; } // Fallback to the hardcoded one if it is valid - if (areHardcodedClientVersionAndKeyValid()) { + if (isHardcodedClientVersionValid()) { clientVersion = HARDCODED_CLIENT_VERSION; return clientVersion; } @@ -694,39 +660,6 @@ public final class YoutubeParsingHelper { throw new ExtractionException("Could not get YouTube WEB client version"); } - /** - * Get the internal API key used by YouTube website on InnerTube requests. - */ - public static String getKey() throws IOException, ExtractionException { - if (!isNullOrEmpty(key)) { - return key; - } - - // Always extract the key used by the website, by trying first to extract it from the - // JavaScript service worker, then from HTML search results page as a fallback, to prevent - // fingerprinting based on the key and/or invalid key issues - try { - extractClientVersionAndKeyFromSwJs(); - } catch (final Exception e) { - extractClientVersionAndKeyFromHtmlSearchResultsPage(); - } - - if (keyAndVersionExtracted) { - return key; - } - - // Fallback to the hardcoded one if it's valid - if (areHardcodedClientVersionAndKeyValid()) { - key = HARDCODED_KEY; - return key; - } - - // The ANDROID API key is also valid with the WEB client so return it if we couldn't - // extract the WEB API key. This can be used as a way to fingerprint the extractor in this - // case - return ANDROID_YOUTUBE_KEY; - } - /** *

* Only used in tests. @@ -742,10 +675,9 @@ public final class YoutubeParsingHelper { * tests with mocks will fail, because the mock is missing. *

*/ - public static void resetClientVersionAndKey() { + public static void resetClientVersion() { clientVersion = null; - key = null; - keyAndVersionExtracted = false; + clientVersionExtracted = false; } /** @@ -757,11 +689,11 @@ public final class YoutubeParsingHelper { numberGenerator = random; } - public static boolean isHardcodedYoutubeMusicKeyValid() throws IOException, + public static boolean isHardcodedYoutubeMusicClientVersionValid() throws IOException, ReCaptchaException { final String url = - "https://music.youtube.com/youtubei/v1/music/get_search_suggestions?key=" - + HARDCODED_YOUTUBE_MUSIC_KEY[0] + DISABLE_PRETTY_PRINT_PARAMETER; + "https://music.youtube.com/youtubei/v1/music/get_search_suggestions?" + + DISABLE_PRETTY_PRINT_PARAMETER; // @formatter:off final byte[] json = JsonWriter.string() @@ -769,7 +701,7 @@ public final class YoutubeParsingHelper { .object("context") .object("client") .value("clientName", "WEB_REMIX") - .value("clientVersion", HARDCODED_YOUTUBE_MUSIC_KEY[2]) + .value("clientVersion", HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION) .value("hl", "en-GB") .value("gl", "GB") .value("platform", "DESKTOP") @@ -791,48 +723,39 @@ public final class YoutubeParsingHelper { // @formatter:on final var headers = new HashMap<>(getOriginReferrerHeaders(YOUTUBE_MUSIC_URL)); - headers.putAll(getClientHeaders(HARDCODED_YOUTUBE_MUSIC_KEY[1], - HARDCODED_YOUTUBE_MUSIC_KEY[2])); + headers.putAll(getClientHeaders("67", HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION)); final Response response = getDownloader().postWithContentTypeJson(url, headers, json); // Ensure to have a valid response return response.responseBody().length() > 500 && response.responseCode() == 200; } - public static String[] getYoutubeMusicKey() + public static String getYoutubeMusicClientVersion() throws IOException, ReCaptchaException, Parser.RegexException { - if (youtubeMusicKey != null && youtubeMusicKey.length == 3) { - return youtubeMusicKey; + if (!isNullOrEmpty(youtubeMusicClientVersion)) { + return youtubeMusicClientVersion; } - if (isHardcodedYoutubeMusicKeyValid()) { - youtubeMusicKey = HARDCODED_YOUTUBE_MUSIC_KEY; - return youtubeMusicKey; + if (isHardcodedYoutubeMusicClientVersionValid()) { + youtubeMusicClientVersion = HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION; + return youtubeMusicClientVersion; } - String musicClientVersion; - String musicKey; - String musicClientName; - try { final String url = "https://music.youtube.com/sw.js"; final var headers = getOriginReferrerHeaders(YOUTUBE_MUSIC_URL); final String response = getDownloader().get(url, headers).responseBody(); - musicClientVersion = getStringResultFromRegexArray(response, + + youtubeMusicClientVersion = getStringResultFromRegexArray(response, INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1); - musicKey = getStringResultFromRegexArray(response, INNERTUBE_API_KEY_REGEXES, 1); - musicClientName = Parser.matchGroup1(INNERTUBE_CLIENT_NAME_REGEX, response); } catch (final Exception e) { final String url = "https://music.youtube.com/?ucbcb=1"; final String html = getDownloader().get(url, getCookieHeader()).responseBody(); - musicKey = getStringResultFromRegexArray(html, INNERTUBE_API_KEY_REGEXES, 1); - musicClientVersion = getStringResultFromRegexArray(html, + youtubeMusicClientVersion = getStringResultFromRegexArray(html, INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1); - musicClientName = Parser.matchGroup1(INNERTUBE_CLIENT_NAME_REGEX, html); } - youtubeMusicKey = new String[] {musicKey, musicClientName, musicClientVersion}; - return youtubeMusicKey; + return youtubeMusicClientVersion; } @Nullable @@ -1150,8 +1073,8 @@ public final class YoutubeParsingHelper { final var headers = getYouTubeHeaders(); return JsonUtils.toJsonObject(getValidJsonResponseBody( - getDownloader().postWithContentTypeJson(YOUTUBEI_V1_URL + endpoint + "?key=" - + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization))); + getDownloader().postWithContentTypeJson(YOUTUBEI_V1_URL + endpoint + "?" + + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization))); } public static JsonObject getJsonAndroidPostResponse( @@ -1160,7 +1083,7 @@ public final class YoutubeParsingHelper { @Nonnull final Localization localization, @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException { return getMobilePostResponse(endpoint, body, localization, - getAndroidUserAgent(localization), ANDROID_YOUTUBE_KEY, endPartOfUrlRequest); + getAndroidUserAgent(localization), endPartOfUrlRequest); } public static JsonObject getJsonIosPostResponse( @@ -1169,7 +1092,7 @@ public final class YoutubeParsingHelper { @Nonnull final Localization localization, @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException { return getMobilePostResponse(endpoint, body, localization, getIosUserAgent(localization), - IOS_YOUTUBE_KEY, endPartOfUrlRequest); + endPartOfUrlRequest); } private static JsonObject getMobilePostResponse( @@ -1177,12 +1100,11 @@ public final class YoutubeParsingHelper { final byte[] body, @Nonnull final Localization localization, @Nonnull final String userAgent, - @Nonnull final String innerTubeApiKey, @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException { final var headers = Map.of("User-Agent", List.of(userAgent), "X-Goog-Api-Format-Version", List.of("2")); - final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?key=" + innerTubeApiKey + final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?" + DISABLE_PRETTY_PRINT_PARAMETER; return JsonUtils.toJsonObject(getValidJsonResponseBody( @@ -1302,7 +1224,7 @@ public final class YoutubeParsingHelper { The build version corresponding to the iOS version used can be found on https://theapplewiki.com/wiki/Firmware/iPhone/17.x#iPhone_15 */ - .value("osVersion", "17.1.2.21B101") + .value("osVersion", "17.4.1.21E237") .value("hl", localization.getLocalizationCode()) .value("gl", contentCountry.getCountryCode()) .value("utcOffsetMinutes", 0) @@ -1421,9 +1343,9 @@ public final class YoutubeParsingHelper { */ @Nonnull public static String getIosUserAgent(@Nullable final Localization localization) { - // Spoofing an iPhone 15 running iOS 17.1.2 with the hardcoded version of the iOS app + // Spoofing an iPhone 15 running iOS 17.4.1 with the hardcoded version of the iOS app return "com.google.ios.youtube/" + IOS_YOUTUBE_CLIENT_VERSION - + "(" + IOS_DEVICE_MODEL + "; U; CPU iOS 17_1_2 like Mac OS X; " + + "(" + IOS_DEVICE_MODEL + "; U; CPU iOS 17_4_1 like Mac OS X; " + (localization != null ? localization : Localization.DEFAULT).getCountryCode() + ")"; } @@ -1434,7 +1356,7 @@ public final class YoutubeParsingHelper { @Nonnull public static Map> getYoutubeMusicHeaders() { final var headers = new HashMap<>(getOriginReferrerHeaders(YOUTUBE_MUSIC_URL)); - headers.putAll(getClientHeaders(youtubeMusicKey[1], youtubeMusicKey[2])); + headers.putAll(getClientHeaders("67", youtubeMusicClientVersion)); return headers; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java index 49049457a..d2ec32ca6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java @@ -29,7 +29,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -422,8 +421,8 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor { .done()) .getBytes(StandardCharsets.UTF_8); - return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey() - + DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body); + return new Page(YOUTUBEI_V1_URL + "browse?" + DISABLE_PRETTY_PRINT_PARAMETER, null, + channelIds, null, body); } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 29695366d..1df90f967 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -4,7 +4,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYouTubeHeaders; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; @@ -103,8 +102,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final var headers = getYouTubeHeaders(); final Response response = getDownloader().postWithContentTypeJson( - YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, - headers, body, localization); + YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, + localization); initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response)); playlistData = initialData @@ -225,8 +224,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { .done()) .getBytes(StandardCharsets.UTF_8); - return new Page(YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, - null, null, cookies, body); + return new Page(YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, null, null, + cookies, body); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index dd2309448..4ed5449fc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYoutubeMusicClientVersion; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYoutubeMusicHeaders; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS; @@ -25,10 +26,8 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.search.SearchExtractor; -import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.JsonUtils; import java.io.IOException; @@ -52,10 +51,8 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { @Override public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { - final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey(); - - final String url = "https://music.youtube.com/youtubei/v1/search?key=" - + youtubeMusicKeys[0] + DISABLE_PRETTY_PRINT_PARAMETER; + final String url = "https://music.youtube.com/youtubei/v1/search?" + + DISABLE_PRETTY_PRINT_PARAMETER; final String params; @@ -86,7 +83,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { .object("context") .object("client") .value("clientName", "WEB_REMIX") - .value("clientVersion", youtubeMusicKeys[2]) + .value("clientVersion", getYoutubeMusicClientVersion()) .value("hl", "en-GB") .value("gl", getExtractorContentCountry().getCountryCode()) .value("platform", "DESKTOP") @@ -206,15 +203,13 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); - final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey(); - // @formatter:off final byte[] json = JsonWriter.string() .object() .object("context") .object("client") .value("clientName", "WEB_REMIX") - .value("clientVersion", youtubeMusicKeys[2]) + .value("clientVersion", getYoutubeMusicClientVersion()) .value("hl", "en-GB") .value("gl", getExtractorContentCountry().getCountryCode()) .value("platform", "DESKTOP") @@ -295,8 +290,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { } @Nullable - private Page getNextPageFrom(final JsonArray continuations) - throws IOException, ParsingException, ReCaptchaException { + private Page getNextPageFrom(final JsonArray continuations) { if (isNullOrEmpty(continuations)) { return null; } @@ -306,7 +300,6 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { final String continuation = nextContinuationData.getString("continuation"); return new Page("https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation - + "&continuation=" + continuation + "&key=" - + YoutubeParsingHelper.getYoutubeMusicKey()[0] + DISABLE_PRETTY_PRINT_PARAMETER); + + "&continuation=" + continuation + "&" + DISABLE_PRETTY_PRINT_PARAMETER); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index a8999fdd3..79afd5542 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -4,7 +4,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; @@ -387,8 +386,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { .done()) .getBytes(StandardCharsets.UTF_8); - return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey() - + DISABLE_PRETTY_PRINT_PARAMETER, body); + return new Page(YOUTUBEI_V1_URL + "browse?" + DISABLE_PRETTY_PRINT_PARAMETER, body); } else { return null; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 90d5cab01..491d53164 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.ALL; @@ -247,8 +246,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { } @Nullable - private Page getNextPageFrom(final JsonObject continuationItemRenderer) throws IOException, - ExtractionException { + private Page getNextPageFrom(final JsonObject continuationItemRenderer) { if (isNullOrEmpty(continuationItemRenderer)) { return null; } @@ -257,8 +255,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { .getObject("continuationCommand") .getString("token"); - final String url = YOUTUBEI_V1_URL + "search?key=" + getKey() - + DISABLE_PRETTY_PRINT_PARAMETER; + final String url = YOUTUBEI_V1_URL + "search?" + DISABLE_PRETTY_PRINT_PARAMETER; return new Page(url, token); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorsTest.java index 3974c3117..d172d8a8a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorsTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorsTest.java @@ -8,7 +8,6 @@ import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.Creati import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeOtfDashManifestCreator; import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeProgressiveDashManifestCreator; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; -import org.schabi.newpipe.extractor.stream.AudioTrackType; import org.schabi.newpipe.extractor.stream.DeliveryMethod; import org.schabi.newpipe.extractor.stream.Stream; import org.w3c.dom.Document; @@ -89,7 +88,7 @@ class YoutubeDashManifestCreatorsTest { @BeforeAll public static void setUp() throws Exception { - YoutubeParsingHelper.resetClientVersionAndKey(); + YoutubeParsingHelper.resetClientVersion(); YoutubeParsingHelper.setNumberGenerator(new Random(1)); NewPipe.init(DownloaderTestImpl.getInstance()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java index 00e4a75ca..1a6e596d8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -7,7 +7,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import com.grack.nanojson.JsonWriter; @@ -91,7 +90,7 @@ public class YoutubeMixPlaylistExtractorTest { .getBytes(StandardCharsets.UTF_8); final InfoItemsPage streams = extractor.getPage(new Page( - YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, + YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, null, null, dummyCookie, body)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); @@ -180,7 +179,7 @@ public class YoutubeMixPlaylistExtractorTest { .getBytes(StandardCharsets.UTF_8); final InfoItemsPage streams = extractor.getPage(new Page( - YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, + YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, null, null, dummyCookie, body)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); @@ -268,7 +267,7 @@ public class YoutubeMixPlaylistExtractorTest { .getBytes(StandardCharsets.UTF_8); final InfoItemsPage streams = extractor.getPage(new Page( - YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, + YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, null, null, dummyCookie, body)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); @@ -386,7 +385,7 @@ public class YoutubeMixPlaylistExtractorTest { .getBytes(StandardCharsets.UTF_8); final InfoItemsPage streams = extractor.getPage(new Page( - YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, + YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, null, null, dummyCookie, body)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); @@ -453,7 +452,7 @@ public class YoutubeMixPlaylistExtractorTest { .getBytes(StandardCharsets.UTF_8); final InfoItemsPage streams = extractor.getPage(new Page( - YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, + YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, null, null, dummyCookie, body)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); @@ -542,7 +541,7 @@ public class YoutubeMixPlaylistExtractorTest { .getBytes(StandardCharsets.UTF_8); final InfoItemsPage streams = extractor.getPage(new Page( - YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, + YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, null, null, dummyCookie, body)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java index 905b3a01e..dc7733344 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java @@ -25,15 +25,15 @@ public class YoutubeParsingHelperTest { } @Test - void testAreHardcodedClientVersionAndKeyValid() throws IOException, ExtractionException { - assertTrue(YoutubeParsingHelper.areHardcodedClientVersionAndKeyValid(), - "Hardcoded client version and key are not valid anymore"); + void testIsHardcodedClientVersionValid() throws IOException, ExtractionException { + assertTrue(YoutubeParsingHelper.isHardcodedClientVersionValid(), + "Hardcoded client version is not valid anymore"); } @Test - void testAreHardcodedYoutubeMusicKeysValid() throws IOException, ExtractionException { - assertTrue(YoutubeParsingHelper.isHardcodedYoutubeMusicKeyValid(), - "Hardcoded YouTube Music keys are not valid anymore"); + void testIsHardcodedYoutubeMusicClientVersionValid() throws IOException, ExtractionException { + assertTrue(YoutubeParsingHelper.isHardcodedYoutubeMusicClientVersionValid(), + "Hardcoded YouTube Music client version is not valid anymore"); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java index ca26eaf35..68d496415 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java @@ -26,7 +26,7 @@ public final class YoutubeTestsUtils { */ public static void ensureStateless() { YoutubeParsingHelper.setConsentAccepted(false); - YoutubeParsingHelper.resetClientVersionAndKey(); + YoutubeParsingHelper.resetClientVersion(); YoutubeParsingHelper.setNumberGenerator(new Random(1)); YoutubeJavaScriptPlayerManager.clearAllCaches(); }