lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Extract YouTube's key

This commit is contained in:
wb9688 2020-07-26 12:00:56 +02:00
parent 4c987a5302
commit f345f667e2
2 changed files with 55 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -64,6 +64,8 @@ public class YoutubeParsingHelper {
private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
private static String clientVersion;
private static String key;
private static final String[] HARDCODED_YOUTUBE_MUSIC_KEYS = {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"};
private static String[] youtubeMusicKeys;
@ -214,39 +216,31 @@ public class YoutubeParsingHelper {
return response.length() > 50; // ensure to have a valid response
}
/**
* Get the client version from a page
* @return
* @throws ParsingException
*/
public static String getClientVersion() throws IOException, ExtractionException {
if (!isNullOrEmpty(clientVersion)) return clientVersion;
if (isHardcodedClientVersionValid()) return clientVersion = HARDCODED_CLIENT_VERSION;
private static void getClientVersionAndKey() throws IOException, ExtractionException {
final String url = "https://www.youtube.com/results?search_query=test";
final String html = getDownloader().get(url).responseBody();
JsonObject initialData = getInitialData(html);
JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
final JsonObject initialData = getInitialData(html);
final JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
String shortClientVersion = null;
// try to get version from initial data first
for (Object service : serviceTrackingParams) {
JsonObject s = (JsonObject) service;
for (final Object service : serviceTrackingParams) {
final JsonObject s = (JsonObject) service;
if (s.getString("service").equals("CSI")) {
JsonArray params = s.getArray("params");
for (Object param : params) {
JsonObject p = (JsonObject) param;
String key = p.getString("key");
final JsonArray params = s.getArray("params");
for (final Object param : params) {
final JsonObject p = (JsonObject) param;
final String key = p.getString("key");
if (key != null && key.equals("cver")) {
return clientVersion = p.getString("value");
clientVersion = p.getString("value");
}
}
} else if (s.getString("service").equals("ECATCHER")) {
// fallback to get a shortened client version which does not contain the last two digits
JsonArray params = s.getArray("params");
for (Object param : params) {
JsonObject p = (JsonObject) param;
String key = p.getString("key");
final JsonArray params = s.getArray("params");
for (final Object param : params) {
final JsonObject p = (JsonObject) param;
final String key = p.getString("key");
if (key != null && key.equals("client.version")) {
shortClientVersion = p.getString("value");
}
@ -255,26 +249,54 @@ public class YoutubeParsingHelper {
}
String contextClientVersion;
String[] patterns = {
final String[] patterns = {
"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"",
"innertube_context_client_version\":\"([0-9\\.]+?)\"",
"client.version=([0-9\\.]+)"
};
for (String pattern : patterns) {
for (final String pattern : patterns) {
try {
contextClientVersion = Parser.matchGroup1(pattern, html);
if (!isNullOrEmpty(contextClientVersion)) {
return clientVersion = contextClientVersion;
clientVersion = contextClientVersion;
}
} catch (Exception ignored) {
}
} catch (Parser.RegexException ignored) { }
}
if (shortClientVersion != null) {
return clientVersion = shortClientVersion;
clientVersion = shortClientVersion;
}
throw new ParsingException("Could not get client version");
try {
key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html);
} catch (Parser.RegexException e) {
try {
key = Parser.matchGroup1("innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\"", html);
} catch (Parser.RegexException ignored) { }
}
}
/**
* Get the client version
*/
public static String getClientVersion() throws IOException, ExtractionException {
if (!isNullOrEmpty(clientVersion)) return clientVersion;
if (isHardcodedClientVersionValid()) return clientVersion = HARDCODED_CLIENT_VERSION;
getClientVersionAndKey();
if (isNullOrEmpty(key)) throw new ParsingException("Could not extract client version");
return clientVersion;
}
/**
* Get the key
*/
public static String getKey() throws IOException, ExtractionException {
if (!isNullOrEmpty(key)) return key;
getClientVersionAndKey();
if (isNullOrEmpty(key)) throw new ParsingException("Could not extract key");
return key;
}
public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, ReCaptchaException {

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -28,6 +28,7 @@ import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -107,7 +108,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws ExtractionException {
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
final JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
@ -221,7 +222,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
+ "&itct=" + clickTrackingParams);
}
private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) {
private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) throws IOException, ExtractionException {
if (isNullOrEmpty(continuationItemRenderer)) {
return null;
}
@ -229,8 +230,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
final String token = continuationItemRenderer.getObject("continuationEndpoint")
.getObject("continuationCommand").getString("token");
// FIXME: Key needs to be extracted
final String url = "https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
final String url = "https://www.youtube.com/youtubei/v1/search?key=" + getKey();
return new Page(url, token);
}