mirror of
https://github.com/TeamNewPipe/NewPipeExtractor
synced 2024-12-14 13:00:26 +01:00
Rewrite youtube throttling solution and add tests
This commit is contained in:
XiangRongLin
parent
6956b72af7
commit
a02ee2e952
@ -0,0 +1,101 @@
|
|||||||
|
package org.schabi.newpipe.extractor.services.youtube;
|
||||||
|
|
||||||
|
import org.jsoup.Jsoup;
|
||||||
|
import org.jsoup.nodes.Document;
|
||||||
|
import org.jsoup.nodes.Element;
|
||||||
|
import org.jsoup.select.Elements;
|
||||||
|
import org.schabi.newpipe.extractor.NewPipe;
|
||||||
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||||
|
import org.schabi.newpipe.extractor.localization.Localization;
|
||||||
|
import org.schabi.newpipe.extractor.utils.Parser;
|
||||||
|
|
||||||
|
import javax.annotation.Nonnull;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Youtube restricts streaming their media in multiple ways by requiring clients to apply a cipher function
|
||||||
|
* on parameters of requests.
|
||||||
|
* The cipher function is sent alongside as a JavaScript function.
|
||||||
|
* <p>
|
||||||
|
* This class handling fetching the JavaScript file in order to allow other classes to extract the needed functions.
|
||||||
|
*/
|
||||||
|
public class YoutubeJavascriptExtractor {
|
||||||
|
|
||||||
|
private static final String HTTPS = "https:";
|
||||||
|
private static String cachedJavascriptCode;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extracts the JavaScript file. The result is cached, so subsequent calls use the result of previous calls.
|
||||||
|
*
|
||||||
|
* @param videoId Does not influence the result, but a valid video id can prevent tracking
|
||||||
|
* @return The whole javascript file as a string.
|
||||||
|
* @throws ParsingException If the extraction failed.
|
||||||
|
*/
|
||||||
|
@Nonnull
|
||||||
|
public static String extractJavascriptCode(String videoId) throws ParsingException {
|
||||||
|
if (cachedJavascriptCode == null) {
|
||||||
|
final YoutubeJavascriptExtractor extractor = new YoutubeJavascriptExtractor();
|
||||||
|
String playerJsUrl = extractor.cleanJavascriptUrl(extractor.extractJavascriptUrl(videoId));
|
||||||
|
cachedJavascriptCode = extractor.downloadJavascriptCode(playerJsUrl);
|
||||||
|
}
|
||||||
|
|
||||||
|
return cachedJavascriptCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Same as {@link YoutubeJavascriptExtractor#extractJavascriptCode(String)} but with a constant value for videoId.
|
||||||
|
* Possible because the videoId has no influence on the result.
|
||||||
|
*
|
||||||
|
* For tracking avoidance purposes it may make sense to pass in valid video ids.
|
||||||
|
*/
|
||||||
|
@Nonnull
|
||||||
|
public static String extractJavascriptCode() throws ParsingException {
|
||||||
|
return extractJavascriptCode("d4IGg5dqeO8");
|
||||||
|
}
|
||||||
|
|
||||||
|
private String extractJavascriptUrl(String videoId) throws ParsingException {
|
||||||
|
try {
|
||||||
|
final String embedUrl = "https://www.youtube.com/embed/" + videoId;
|
||||||
|
final String embedPageContent = NewPipe.getDownloader()
|
||||||
|
.get(embedUrl, Localization.DEFAULT).responseBody();
|
||||||
|
|
||||||
|
try {
|
||||||
|
final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")";
|
||||||
|
return Parser.matchGroup1(assetsPattern, embedPageContent)
|
||||||
|
.replace("\\", "").replace("\"", "");
|
||||||
|
} catch (final Parser.RegexException ex) {
|
||||||
|
// playerJsUrl is still available in the file, just somewhere else TODO
|
||||||
|
// it is ok not to find it, see how that's handled in getDeobfuscationCode()
|
||||||
|
final Document doc = Jsoup.parse(embedPageContent);
|
||||||
|
final Elements elems = doc.select("script").attr("name", "player_ias/base");
|
||||||
|
for (final Element elem : elems) {
|
||||||
|
if (elem.attr("src").contains("base.js")) {
|
||||||
|
return elem.attr("src");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (final Exception i) {
|
||||||
|
throw new ParsingException("Embedded info did not provide YouTube player js url");
|
||||||
|
}
|
||||||
|
throw new ParsingException("Embedded info did not provide YouTube player js url");
|
||||||
|
}
|
||||||
|
|
||||||
|
private String cleanJavascriptUrl(String playerJsUrl) {
|
||||||
|
if (playerJsUrl.startsWith("//")) {
|
||||||
|
return HTTPS + playerJsUrl;
|
||||||
|
} else if (playerJsUrl.startsWith("/")) {
|
||||||
|
// sometimes https://www.youtube.com part has to be added manually
|
||||||
|
return HTTPS + "//www.youtube.com" + playerJsUrl;
|
||||||
|
} else {
|
||||||
|
return playerJsUrl;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String downloadJavascriptCode(String playerJsUrl) throws ParsingException {
|
||||||
|
try {
|
||||||
|
return NewPipe.getDownloader().get(playerJsUrl, Localization.DEFAULT).responseBody();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new ParsingException("Could not get player js code from url: " + playerJsUrl);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,104 +0,0 @@
|
|||||||
package org.schabi.newpipe.extractor.services.youtube;
|
|
||||||
|
|
||||||
import org.jsoup.Jsoup;
|
|
||||||
import org.jsoup.nodes.Document;
|
|
||||||
import org.jsoup.nodes.Element;
|
|
||||||
import org.jsoup.select.Elements;
|
|
||||||
import org.schabi.newpipe.extractor.NewPipe;
|
|
||||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
|
||||||
import org.schabi.newpipe.extractor.localization.Localization;
|
|
||||||
import org.schabi.newpipe.extractor.utils.Javascript;
|
|
||||||
import org.schabi.newpipe.extractor.utils.Parser;
|
|
||||||
|
|
||||||
import java.util.regex.Pattern;
|
|
||||||
|
|
||||||
public class YoutubeThrottlingDecoder {
|
|
||||||
|
|
||||||
private static final String HTTPS = "https:";
|
|
||||||
private static final String N_PARAM_REGEX = "[&?]n=([^&]+)";
|
|
||||||
private static String playerJsCode;
|
|
||||||
|
|
||||||
private final String functionName;
|
|
||||||
private final String function;
|
|
||||||
|
|
||||||
public YoutubeThrottlingDecoder(String videoId, Localization localization) throws ParsingException {
|
|
||||||
if (playerJsCode == null) {
|
|
||||||
String playerJsUrl = cleanPlayerJsUrl(extractPlayerJsUrl(videoId, localization));
|
|
||||||
playerJsCode = downloadPlayerJsCode(localization, playerJsUrl);
|
|
||||||
}
|
|
||||||
|
|
||||||
functionName = parseDecodeFunctionName(playerJsCode);
|
|
||||||
function = parseDecodeFunction(playerJsCode, functionName);
|
|
||||||
}
|
|
||||||
|
|
||||||
private String extractPlayerJsUrl(String videoId, Localization localization) throws ParsingException {
|
|
||||||
try {
|
|
||||||
final String embedUrl = "https://www.youtube.com/embed/" + videoId;
|
|
||||||
final String embedPageContent = NewPipe.getDownloader()
|
|
||||||
.get(embedUrl, localization).responseBody();
|
|
||||||
|
|
||||||
try {
|
|
||||||
final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")";
|
|
||||||
return Parser.matchGroup1(assetsPattern, embedPageContent)
|
|
||||||
.replace("\\", "").replace("\"", "");
|
|
||||||
} catch (final Parser.RegexException ex) {
|
|
||||||
// playerJsUrl is still available in the file, just somewhere else TODO
|
|
||||||
// it is ok not to find it, see how that's handled in getDeobfuscationCode()
|
|
||||||
final Document doc = Jsoup.parse(embedPageContent);
|
|
||||||
final Elements elems = doc.select("script").attr("name", "player_ias/base");
|
|
||||||
for (final Element elem : elems) {
|
|
||||||
if (elem.attr("src").contains("base.js")) {
|
|
||||||
return elem.attr("src");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
} catch (final Exception i) {
|
|
||||||
throw new ParsingException("Embedded info did not provide YouTube player js url");
|
|
||||||
}
|
|
||||||
throw new ParsingException("Embedded info did not provide YouTube player js url");
|
|
||||||
}
|
|
||||||
|
|
||||||
private String cleanPlayerJsUrl(String playerJsUrl) {
|
|
||||||
if (playerJsUrl.startsWith("//")) {
|
|
||||||
return HTTPS + playerJsUrl;
|
|
||||||
} else if (playerJsUrl.startsWith("/")) {
|
|
||||||
// sometimes https://www.youtube.com part has to be added manually
|
|
||||||
return HTTPS + "//www.youtube.com" + playerJsUrl;
|
|
||||||
} else {
|
|
||||||
return playerJsUrl;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private String downloadPlayerJsCode(Localization localization, String playerJsUrl) throws ParsingException {
|
|
||||||
try {
|
|
||||||
return NewPipe.getDownloader().get(playerJsUrl, localization).responseBody();
|
|
||||||
} catch (Exception e) {
|
|
||||||
throw new ParsingException("Could not get player js code from url: " + playerJsUrl);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private String parseDecodeFunctionName(String playerJsCode) throws Parser.RegexException {
|
|
||||||
Pattern pattern = Pattern.compile("b=a\\.get\\(\"n\"\\)\\)&&\\(b=(\\w+)\\(b\\),a\\.set\\(\"n\",b\\)");
|
|
||||||
return Parser.matchGroup1(pattern, playerJsCode);
|
|
||||||
}
|
|
||||||
|
|
||||||
private String parseDecodeFunction(String playerJsCode, String functionName) throws Parser.RegexException {
|
|
||||||
Pattern functionPattern = Pattern.compile(functionName + "=function(.*?;)\n", Pattern.DOTALL);
|
|
||||||
return "function " + functionName + Parser.matchGroup1(functionPattern, playerJsCode);
|
|
||||||
}
|
|
||||||
|
|
||||||
public String parseNParam(String url) throws Parser.RegexException {
|
|
||||||
Pattern nValuePattern = Pattern.compile(N_PARAM_REGEX);
|
|
||||||
return Parser.matchGroup1(nValuePattern, url);
|
|
||||||
}
|
|
||||||
|
|
||||||
public String decodeNParam(String nParam) {
|
|
||||||
Javascript javascript = new Javascript();
|
|
||||||
return javascript.run(function, functionName, nParam);
|
|
||||||
}
|
|
||||||
|
|
||||||
public String replaceNParam(String url, String oldValue, String newValue) {
|
|
||||||
return url.replace(oldValue, newValue);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -0,0 +1,88 @@
|
|||||||
|
package org.schabi.newpipe.extractor.services.youtube;
|
||||||
|
|
||||||
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||||
|
import org.schabi.newpipe.extractor.utils.Javascript;
|
||||||
|
import org.schabi.newpipe.extractor.utils.Parser;
|
||||||
|
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <p>
|
||||||
|
* YouTube's media is protected with a cipher, which modifies the "n" query parameter of it's video playback urls.
|
||||||
|
* This class handles extracting that "n" query parameter, applying the cipher on it and returning the resulting url
|
||||||
|
* which is not throttled.
|
||||||
|
* </p>
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* https://r5---sn-4g5ednsz.googlevideo.com/videoplayback?n=VVF2xyZLVRZZxHXZ&other=other
|
||||||
|
* </p>
|
||||||
|
* becomes
|
||||||
|
* <p>
|
||||||
|
* https://r5---sn-4g5ednsz.googlevideo.com/videoplayback?n=iHywZkMipkszqA&other=other
|
||||||
|
* </p>
|
||||||
|
*/
|
||||||
|
public class YoutubeThrottlingDecrypter {
|
||||||
|
|
||||||
|
private static final String N_PARAM_REGEX = "[&?]n=([^&]+)";
|
||||||
|
|
||||||
|
private final String functionName;
|
||||||
|
private final String function;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <p>
|
||||||
|
* Use this if you care about the off chance that YouTube tracks with which videoId the cipher is requested.
|
||||||
|
* </p>
|
||||||
|
* Otherwise use the no-arg constructor which uses a constant value.
|
||||||
|
*/
|
||||||
|
public YoutubeThrottlingDecrypter(String videoId) throws ParsingException {
|
||||||
|
final String playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode(videoId);
|
||||||
|
|
||||||
|
functionName = parseDecodeFunctionName(playerJsCode);
|
||||||
|
function = parseDecodeFunction(playerJsCode, functionName);
|
||||||
|
}
|
||||||
|
|
||||||
|
public YoutubeThrottlingDecrypter() throws ParsingException {
|
||||||
|
final String playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode();
|
||||||
|
|
||||||
|
functionName = parseDecodeFunctionName(playerJsCode);
|
||||||
|
function = parseDecodeFunction(playerJsCode, functionName);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String parseDecodeFunctionName(String playerJsCode) throws Parser.RegexException {
|
||||||
|
Pattern pattern = Pattern.compile("b=a\\.get\\(\"n\"\\)\\)&&\\(b=(\\w+)\\(b\\),a\\.set\\(\"n\",b\\)");
|
||||||
|
return Parser.matchGroup1(pattern, playerJsCode);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String parseDecodeFunction(String playerJsCode, String functionName) throws Parser.RegexException {
|
||||||
|
Pattern functionPattern = Pattern.compile(functionName + "=function(.*?;)\n", Pattern.DOTALL);
|
||||||
|
return "function " + functionName + Parser.matchGroup1(functionPattern, playerJsCode);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String apply(String url) throws Parser.RegexException {
|
||||||
|
if (containsNParam(url)) {
|
||||||
|
String oldNParam = parseNParam(url);
|
||||||
|
String newNParam = decryptNParam(oldNParam);
|
||||||
|
return replaceNParam(url, oldNParam, newNParam);
|
||||||
|
} else {
|
||||||
|
return url;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean containsNParam(String url) {
|
||||||
|
return Parser.isMatch(N_PARAM_REGEX, url);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String parseNParam(String url) throws Parser.RegexException {
|
||||||
|
Pattern nValuePattern = Pattern.compile(N_PARAM_REGEX);
|
||||||
|
return Parser.matchGroup1(nValuePattern, url);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String decryptNParam(String nParam) {
|
||||||
|
Javascript javascript = new Javascript();
|
||||||
|
return javascript.run(function, functionName, nParam);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String replaceNParam(String url, String oldValue, String newValue) {
|
||||||
|
return url.replace(oldValue, newValue);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -4,7 +4,6 @@ import com.grack.nanojson.JsonArray;
|
|||||||
import com.grack.nanojson.JsonObject;
|
import com.grack.nanojson.JsonObject;
|
||||||
import com.grack.nanojson.JsonParser;
|
import com.grack.nanojson.JsonParser;
|
||||||
import com.grack.nanojson.JsonParserException;
|
import com.grack.nanojson.JsonParserException;
|
||||||
|
|
||||||
import org.jsoup.Jsoup;
|
import org.jsoup.Jsoup;
|
||||||
import org.jsoup.nodes.Document;
|
import org.jsoup.nodes.Document;
|
||||||
import org.jsoup.nodes.Element;
|
import org.jsoup.nodes.Element;
|
||||||
@ -18,15 +17,7 @@ import org.schabi.newpipe.extractor.NewPipe;
|
|||||||
import org.schabi.newpipe.extractor.StreamingService;
|
import org.schabi.newpipe.extractor.StreamingService;
|
||||||
import org.schabi.newpipe.extractor.downloader.Downloader;
|
import org.schabi.newpipe.extractor.downloader.Downloader;
|
||||||
import org.schabi.newpipe.extractor.downloader.Response;
|
import org.schabi.newpipe.extractor.downloader.Response;
|
||||||
import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException;
|
import org.schabi.newpipe.extractor.exceptions.*;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
|
|
||||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
|
||||||
import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException;
|
|
||||||
import org.schabi.newpipe.extractor.exceptions.PaidContentException;
|
|
||||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
|
||||||
import org.schabi.newpipe.extractor.exceptions.PrivateContentException;
|
|
||||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
|
||||||
import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException;
|
|
||||||
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
|
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
|
||||||
import org.schabi.newpipe.extractor.localization.DateWrapper;
|
import org.schabi.newpipe.extractor.localization.DateWrapper;
|
||||||
import org.schabi.newpipe.extractor.localization.Localization;
|
import org.schabi.newpipe.extractor.localization.Localization;
|
||||||
@ -34,44 +25,23 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser;
|
|||||||
import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager;
|
import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager;
|
||||||
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
|
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
|
||||||
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
|
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
|
||||||
import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecoder;
|
import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecrypter;
|
||||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
|
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
|
||||||
import org.schabi.newpipe.extractor.stream.AudioStream;
|
import org.schabi.newpipe.extractor.stream.*;
|
||||||
import org.schabi.newpipe.extractor.stream.Description;
|
|
||||||
import org.schabi.newpipe.extractor.stream.Frameset;
|
|
||||||
import org.schabi.newpipe.extractor.stream.Stream;
|
|
||||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
|
||||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
|
|
||||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
|
||||||
import org.schabi.newpipe.extractor.stream.StreamSegment;
|
|
||||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
|
||||||
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
|
|
||||||
import org.schabi.newpipe.extractor.stream.VideoStream;
|
|
||||||
import org.schabi.newpipe.extractor.utils.JsonUtils;
|
import org.schabi.newpipe.extractor.utils.JsonUtils;
|
||||||
import org.schabi.newpipe.extractor.utils.Parser;
|
import org.schabi.newpipe.extractor.utils.Parser;
|
||||||
import org.schabi.newpipe.extractor.utils.Utils;
|
import org.schabi.newpipe.extractor.utils.Utils;
|
||||||
|
|
||||||
|
import javax.annotation.Nonnull;
|
||||||
|
import javax.annotation.Nullable;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.UnsupportedEncodingException;
|
import java.io.UnsupportedEncodingException;
|
||||||
import java.time.LocalDate;
|
import java.time.LocalDate;
|
||||||
import java.time.OffsetDateTime;
|
import java.time.OffsetDateTime;
|
||||||
import java.time.format.DateTimeFormatter;
|
import java.time.format.DateTimeFormatter;
|
||||||
import java.util.ArrayList;
|
import java.util.*;
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.LinkedHashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Locale;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Objects;
|
|
||||||
|
|
||||||
import javax.annotation.Nonnull;
|
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
|
||||||
import javax.annotation.Nullable;
|
|
||||||
|
|
||||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
|
|
||||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
|
|
||||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
|
|
||||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
|
|
||||||
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
|
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
|
||||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||||
|
|
||||||
@ -553,18 +523,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||||||
public List<VideoStream> getVideoStreams() throws ExtractionException {
|
public List<VideoStream> getVideoStreams() throws ExtractionException {
|
||||||
assertPageFetched();
|
assertPageFetched();
|
||||||
final List<VideoStream> videoStreams = new ArrayList<>();
|
final List<VideoStream> videoStreams = new ArrayList<>();
|
||||||
YoutubeThrottlingDecoder throttlingDecoder = new YoutubeThrottlingDecoder(getId(), getExtractorLocalization());
|
YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId());
|
||||||
|
|
||||||
try {
|
try {
|
||||||
for (final Map.Entry<String, ItagItem> entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) {
|
for (final Map.Entry<String, ItagItem> entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) {
|
||||||
final ItagItem itag = entry.getValue();
|
final ItagItem itag = entry.getValue();
|
||||||
final String url = entry.getKey();
|
String url = entry.getKey();
|
||||||
|
url = throttlingDecrypter.apply(url);
|
||||||
|
|
||||||
String oldNParam = throttlingDecoder.parseNParam(url);
|
final VideoStream videoStream = new VideoStream(url, false, itag);
|
||||||
String newNParam = throttlingDecoder.decodeNParam(oldNParam);
|
|
||||||
String newUrl = throttlingDecoder.replaceNParam(url, oldNParam, newNParam);
|
|
||||||
|
|
||||||
final VideoStream videoStream = new VideoStream(newUrl, false, itag);
|
|
||||||
if (!Stream.containSimilarStream(videoStream, videoStreams)) {
|
if (!Stream.containSimilarStream(videoStream, videoStreams)) {
|
||||||
videoStreams.add(videoStream);
|
videoStreams.add(videoStream);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,47 @@
|
|||||||
|
package org.schabi.newpipe.extractor.services.youtube;
|
||||||
|
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.schabi.newpipe.downloader.DownloaderTestImpl;
|
||||||
|
import org.schabi.newpipe.extractor.NewPipe;
|
||||||
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import static org.hamcrest.CoreMatchers.allOf;
|
||||||
|
import static org.hamcrest.CoreMatchers.containsString;
|
||||||
|
import static org.hamcrest.MatcherAssert.assertThat;
|
||||||
|
|
||||||
|
public class YoutubeJavascriptExtractorTest {
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setup() throws IOException {
|
||||||
|
NewPipe.init(DownloaderTestImpl.getInstance());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExtractJavascript__success() throws ParsingException {
|
||||||
|
String playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode("d4IGg5dqeO8");
|
||||||
|
assertPlayerJsCode(playerJsCode);
|
||||||
|
|
||||||
|
playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode();
|
||||||
|
assertPlayerJsCode(playerJsCode);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExtractJavascript__invalidVideoId__success() throws ParsingException {
|
||||||
|
String playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode("not_a_video_id");
|
||||||
|
assertPlayerJsCode(playerJsCode);
|
||||||
|
|
||||||
|
playerJsCode = YoutubeJavascriptExtractor.extractJavascriptCode("11-chars123");
|
||||||
|
assertPlayerJsCode(playerJsCode);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private void assertPlayerJsCode(String playerJsCode) {
|
||||||
|
assertThat(playerJsCode, allOf(
|
||||||
|
containsString(" Copyright The Closure Library Authors.\n"
|
||||||
|
+ " SPDX-License-Identifier: Apache-2.0"),
|
||||||
|
containsString("var _yt_player")));
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,40 @@
|
|||||||
|
package org.schabi.newpipe.extractor.services.youtube;
|
||||||
|
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.schabi.newpipe.downloader.DownloaderTestImpl;
|
||||||
|
import org.schabi.newpipe.extractor.NewPipe;
|
||||||
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import static org.hamcrest.CoreMatchers.equalTo;
|
||||||
|
import static org.hamcrest.MatcherAssert.assertThat;
|
||||||
|
|
||||||
|
public class YoutubeThrottlingDecrypterTest {
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setup() throws IOException {
|
||||||
|
NewPipe.init(DownloaderTestImpl.getInstance());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDecode__success() throws ParsingException {
|
||||||
|
// urls extracted from browser with the dev tools
|
||||||
|
final String encryptedUrl = "https://r6---sn-4g5ednek.googlevideo.com/videoplayback?expire=1626562120&ei=6AnzYO_YBpql1gLGkb_IBQ&ip=127.0.0.1&id=o-ANhBEf36Z5h-8U9DDddtPDqtS0ZNwf0XJAAigudKI2uI&itag=278&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&source=youtube&requiressl=yes&vprv=1&mime=video%2Fwebm&ns=TvecOReN0vPuXb3j_zq157IG&gir=yes&clen=2915100&dur=270.203&lmt=1608157174907785&keepalive=yes&fexp=24001373,24007246&c=WEB&txp=5535432&n=N9BWSTFT7vvBJrvQ&sparams=expire%2Cei%2Cip%2Cid%2Caitags%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&alr=yes&sig=AOq0QJ8wRQIgW6XnUDKPDSxiT0_KE_tDDMpcaCJl2Un5p0Fu9qZNQGkCIQDWxsDHi_s2BEmRqIbd1C5g_gzfihB7RZLsScKWNMwzzA%3D%3D&cpn=9r2yt3BqcYmeb2Yu&cver=2.20210716.00.00&redirect_counter=1&cm2rm=sn-4g5ezy7s&cms_redirect=yes&mh=Y5&mm=34&mn=sn-4g5ednek&ms=ltu&mt=1626540524&mv=m&mvi=6&pl=43&lsparams=mh,mm,mn,ms,mv,mvi,pl&lsig=AG3C_xAwRQIhAIUzxTn9Vw1-vm-_7OQ5-0h1M6AZsY9Bx1FlCCTeMICzAiADtGggbn4Znsrh2EnvyOsGnYdRGcbxn4mW9JMOQiInDQ%3D%3D&range=259165-480735&rn=11&rbuf=20190";
|
||||||
|
final String decryptedUrl = "https://r6---sn-4g5ednek.googlevideo.com/videoplayback?expire=1626562120&ei=6AnzYO_YBpql1gLGkb_IBQ&ip=127.0.0.1&id=o-ANhBEf36Z5h-8U9DDddtPDqtS0ZNwf0XJAAigudKI2uI&itag=278&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&source=youtube&requiressl=yes&vprv=1&mime=video%2Fwebm&ns=TvecOReN0vPuXb3j_zq157IG&gir=yes&clen=2915100&dur=270.203&lmt=1608157174907785&keepalive=yes&fexp=24001373,24007246&c=WEB&txp=5535432&n=HI_QWjzacNoiJw&sparams=expire%2Cei%2Cip%2Cid%2Caitags%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&alr=yes&sig=AOq0QJ8wRQIgW6XnUDKPDSxiT0_KE_tDDMpcaCJl2Un5p0Fu9qZNQGkCIQDWxsDHi_s2BEmRqIbd1C5g_gzfihB7RZLsScKWNMwzzA%3D%3D&cpn=9r2yt3BqcYmeb2Yu&cver=2.20210716.00.00&redirect_counter=1&cm2rm=sn-4g5ezy7s&cms_redirect=yes&mh=Y5&mm=34&mn=sn-4g5ednek&ms=ltu&mt=1626540524&mv=m&mvi=6&pl=43&lsparams=mh,mm,mn,ms,mv,mvi,pl&lsig=AG3C_xAwRQIhAIUzxTn9Vw1-vm-_7OQ5-0h1M6AZsY9Bx1FlCCTeMICzAiADtGggbn4Znsrh2EnvyOsGnYdRGcbxn4mW9JMOQiInDQ%3D%3D&range=259165-480735&rn=11&rbuf=20190";
|
||||||
|
|
||||||
|
final String actualDecryptedUrl = new YoutubeThrottlingDecrypter().apply(encryptedUrl);
|
||||||
|
|
||||||
|
assertThat(actualDecryptedUrl, equalTo(decryptedUrl));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDecode__noNParam__success() throws ParsingException {
|
||||||
|
final String noNParamUrl = "https://r5---sn-4g5ednsz.googlevideo.com/videoplayback?expire=1626553257&ei=SefyYPmIFoKT1wLtqbjgCQ&ip=127.0.0.1&id=o-AIT5xGifsaEAdEOAb5vd06J9VNtm-KHHolnaZRGPjHZi&itag=140&source=youtube&requiressl=yes&mh=xO&mm=31%2C29&mn=sn-4g5ednsz%2Csn-4g5e6nsr&ms=au%2Crdu&mv=m&mvi=5&pl=24&initcwndbps=1322500&vprv=1&mime=audio%2Fmp4&ns=cA2SS5atEe0mH8tMwGTO4RIG&gir=yes&clen=3009275&dur=185.898&lmt=1626356984653961&mt=1626531173&fvip=5&keepalive=yes&fexp=24001373%2C24007246&beids=23886212&c=WEB&txp=6411222&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&lsig=AG3C_xAwRgIhAPueRlTutSlzPafxrqBmgZz5m7-Zfbw3QweDp3j4XO9SAiEA5tF7_ZCJFKmS-D6I1jlUURjpjoiTbsYyKuarV4u6E8Y%3D&sig=AOq0QJ8wRQIgRD_4WwkPeTEKGVSQqPsznMJGqq4nVJ8o1ChGBCgi4Y0CIQCZT3tI40YLKBWJCh2Q7AlvuUIpN0ficzdSElLeQpJdrw==";
|
||||||
|
String decrypted = new YoutubeThrottlingDecrypter().apply(noNParamUrl);
|
||||||
|
|
||||||
|
assertThat(decrypted, equalTo(noNParamUrl));
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user