lasers
/
NewPipe
mirror of https://github.com/TeamNewPipe/NewPipe
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'rrooij-age_restriction'

This commit is contained in:
Christian Schabesberger 2016-02-22 19:18:22 +01:00
parent 65b6aaec8e 7041e63268
commit ff89dd00b6
8 changed files with 258 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
app/src/androidTest/java/org/schabi/newpipe/extractor/youtube/YoutubeStreamExtractorRestrictedTest.java Normal file
View File

@ -0,0 +1,84 @@
package org.schabi.newpipe.extractor.youtube;
import android.test.AndroidTestCase;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.ExtractionException;
import org.schabi.newpipe.extractor.ParsingException;
import org.schabi.newpipe.extractor.VideoInfo;
import org.schabi.newpipe.extractor.services.youtube.YoutubeStreamExtractor;
import java.io.IOException;
public class YoutubeStreamExtractorRestrictedTest extends AndroidTestCase {
private YoutubeStreamExtractor extractor;
public void setUp() throws IOException, ExtractionException {
extractor = new YoutubeStreamExtractor("https://www.youtube.com/watch?v=i6JTvzrpBy0",
new Downloader());
}
public void testGetInvalidTimeStamp() throws ParsingException {
assertTrue(Integer.toString(extractor.getTimeStamp()),
extractor.getTimeStamp() <= 0);
}
public void testGetValidTimeStamp() throws ExtractionException, IOException {
YoutubeStreamExtractor extractor =
new YoutubeStreamExtractor("https://youtu.be/FmG385_uUys?t=174", new Downloader());
assertTrue(Integer.toString(extractor.getTimeStamp()),
extractor.getTimeStamp() == 174);
}
public void testGetAgeLimit() throws ParsingException {
assertTrue(extractor.getAgeLimit() == 18);
}
public void testGetTitle() throws ParsingException {
assertTrue(!extractor.getTitle().isEmpty());
}
public void testGetDescription() throws ParsingException {
assertTrue(extractor.getDescription() != null);
}
public void testGetUploader() throws ParsingException {
assertTrue(!extractor.getUploader().isEmpty());
}
public void testGetLength() throws ParsingException {
assertTrue(extractor.getLength() > 0);
}
public void testGetViews() throws ParsingException {
assertTrue(extractor.getLength() > 0);
}
public void testGetUploadDate() throws ParsingException {
assertTrue(extractor.getUploadDate().length() > 0);
}
public void testGetThumbnailUrl() throws ParsingException {
assertTrue(extractor.getThumbnailUrl(),
extractor.getThumbnailUrl().contains("https://"));
}
public void testGetUploaderThumbnailUrl() throws ParsingException {
assertTrue(extractor.getUploaderThumbnailUrl(),
extractor.getUploaderThumbnailUrl().contains("https://"));
}
public void testGetAudioStreams() throws ParsingException {
assertTrue(!extractor.getAudioStreams().isEmpty());
}
public void testGetVideoStreams() throws ParsingException {
for(VideoInfo.VideoStream s : extractor.getVideoStreams()) {
assertTrue(s.url,
s.url.contains("https://"));
assertTrue(s.resolution.length() > 0);
assertTrue(Integer.toString(s.format),
0 <= s.format && s.format <= 4);
}
}
}

12
app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java
View File

@ -181,9 +181,13 @@ public class VideoItemDetailFragment extends Fragment {
}
@Override
public void run() {
//todo: fix expired thread error:
// If the thread calling this runnable is expired, the following function will crash.
updateInfo(videoInfo);
boolean show_age_restricted_content = PreferenceManager.getDefaultSharedPreferences(getActivity())
.getBoolean(activity.getString(R.string.show_age_restricted_content), false);
if(videoInfo.age_limit == 0 || show_age_restricted_content) {
updateInfo(videoInfo);
} else {
onNotSpecifiedContentErrorWithMessage(R.string.video_is_age_restricted);
}
}
}
@ -418,7 +422,7 @@ public class VideoItemDetailFragment extends Fragment {
imageLoader.displayImage(info.uploader_thumbnail_url,
uploaderThumb, displayImageOptions, new ThumbnailLoadingListener());
}
if(info.thumbnail_url != null && !info.thumbnail_url.isEmpty()) {
if(info.thumbnail_url != null && !info.thumbnail_url.isEmpty() && info.next_video != null) {
imageLoader.displayImage(info.next_video.thumbnail_url,
nextVideoThumb, displayImageOptions, new ThumbnailLoadingListener());
}

8
app/src/main/java/org/schabi/newpipe/extractor/Parser.java
View File

@ -1,5 +1,7 @@
package org.schabi.newpipe.extractor;
import android.util.Log;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.util.HashMap;
@ -53,7 +55,11 @@ public class Parser {
Map<String, String> map = new HashMap<>();
for(String arg : input.split("&")) {
String[] split_arg = arg.split("=");
map.put(split_arg[0], URLDecoder.decode(split_arg[1], "UTF-8"));
if(split_arg.length > 1) {
map.put(split_arg[0], URLDecoder.decode(split_arg[1], "UTF-8"));
} else {
map.put(split_arg[0], "");
}
}
return map;
}

11
app/src/main/java/org/schabi/newpipe/extractor/VideoInfo.java
View File

@ -52,10 +52,12 @@ public class VideoInfo extends AbstractVideoInfo {
videoInfo.webpage_url = extractor.getPageUrl();
videoInfo.id = uiconv.getVideoId(extractor.getPageUrl());
videoInfo.title = extractor.getTitle();
videoInfo.age_limit = extractor.getAgeLimit();
if((videoInfo.webpage_url == null || videoInfo.webpage_url.isEmpty())
|| (videoInfo.id == null || videoInfo.id.isEmpty())
|| (videoInfo.title == null /* videoInfo.title can be empty of course */));
|| (videoInfo.title == null /* videoInfo.title can be empty of course */)
|| (videoInfo.age_limit == -1));
return videoInfo;
}
@ -192,6 +194,11 @@ public class VideoInfo extends AbstractVideoInfo {
} catch(Exception e) {
videoInfo.addException(e);
}
try {
} catch (Exception e) {
videoInfo.addException(e);
}
return videoInfo;
}
@ -209,7 +216,7 @@ public class VideoInfo extends AbstractVideoInfo {
public String dashMpdUrl = "";
public int duration = -1;
public int age_limit = 0;
public int age_limit = -1;
public int like_count = -1;
public int dislike_count = -1;
public String average_rating = "";

183
app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java
View File

@ -24,6 +24,8 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Vector;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Created by Christian Schabesberger on 06.08.15.
@ -168,7 +170,8 @@ public class YoutubeStreamExtractor implements StreamExtractor {
private static final String TAG = YoutubeStreamExtractor.class.toString();
private final Document doc;
private JSONObject playerArgs;
//private Map<String, String> videoInfoPage;
private boolean isAgeRestricted;
private Map<String, String> videoInfoPage;
// static values
private static final String DECRYPTION_FUNC_NAME="decrypt";
@ -187,79 +190,123 @@ public class YoutubeStreamExtractor implements StreamExtractor {
this.pageUrl = pageUrl;
String pageContent = downloader.download(urlidhandler.cleanUrl(pageUrl));
doc = Jsoup.parse(pageContent, pageUrl);
String ytPlayerConfigRaw;
JSONObject ytPlayerConfig;
String playerUrl;
//attempt to load the youtube js player JSON arguments
boolean isLiveStream = false; //used to determine if this is a livestream or not
// Check if the video is age restricted
if (pageContent.contains("<meta property=\"og:restrictions:age")) {
String videoInfoUrl = GET_VIDEO_INFO_URL.replace("%%video_id%%",
urlidhandler.getVideoId(pageUrl)).replace("$$el_type$$", "&" + EL_INFO);
String videoInfoPageString = downloader.download(videoInfoUrl);
videoInfoPage = Parser.compatParseMap(videoInfoPageString);
playerUrl = getPlayerUrlFromRestrictedVideo(pageUrl);
isAgeRestricted = true;
} else {
ytPlayerConfig = getPlayerConfig(pageContent);
playerArgs = getPlayerArgs(ytPlayerConfig);
playerUrl = getPlayerUrl(ytPlayerConfig);
isAgeRestricted = false;
}
if(decryptionCode.isEmpty()) {
decryptionCode = loadDecryptionCode(playerUrl);
}
}
private JSONObject getPlayerConfig(String pageContent) throws ParsingException {
try {
ytPlayerConfigRaw =
String ytPlayerConfigRaw =
Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
ytPlayerConfig = new JSONObject(ytPlayerConfigRaw);
playerArgs = ytPlayerConfig.getJSONObject("args");
// check if we have a live stream. We need to filter it, since its not yet supported.
if((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))
|| (playerArgs.get("url_encoded_fmt_stream_map").toString().isEmpty())) {
isLiveStream = true;
}
return new JSONObject(ytPlayerConfigRaw);
} catch (Parser.RegexException e) {
String errorReason = findErrorReason(doc);
switch(errorReason) {
case "GEMA":
throw new GemaException(errorReason);
case "":
throw new ParsingException("player config empty", e);
throw new ContentNotAvailableException("Content not available: player config empty", e);
default:
throw new ContentNotAvailableException("Content not available", e);
}
} catch (JSONException e) {
throw new ParsingException("Could not parse yt player config", e);
}
}
private JSONObject getPlayerArgs(JSONObject playerConfig) throws ParsingException {
JSONObject playerArgs;
//attempt to load the youtube js player JSON arguments
boolean isLiveStream = false; //used to determine if this is a livestream or not
try {
playerArgs = playerConfig.getJSONObject("args");
// check if we have a live stream. We need to filter it, since its not yet supported.
if((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))
|| (playerArgs.get("url_encoded_fmt_stream_map").toString().isEmpty())) {
isLiveStream = true;
}
} catch (JSONException e) {
throw new ParsingException("Could not parse yt player config", e);
}
if (isLiveStream) {
throw new LiveStreamException();
}
return playerArgs;
}
/* not yet nececeary
// get videoInfo page
private String getPlayerUrl(JSONObject playerConfig) throws ParsingException {
try {
//Parser.unescapeEntities(url_data_str, true).split("&")
String getVideoInfoUrl = GET_VIDEO_INFO_URL.replace("%%video_id%%",
urlidhandler.getVideoId(pageUrl)).replace("$$el_type$$", "&" + EL_INFO);
videoInfoPage = Parser.compatParseMap(downloader.download(getVideoInfoUrl));
} catch(Exception e) {
throw new ParsingException("Could not load video info page.", e);
}
*/
// The Youtube service needs to be initialized by downloading the
// js-Youtube-player. This is done in order to get the algorithm
// for decrypting cryptic signatures inside certain stream urls.
String playerUrl = "";
//----------------------------------
// load and parse description code, if it isn't already initialised
//----------------------------------
if (decryptionCode.isEmpty()) {
try {
// The Youtube service needs to be initialized by downloading the
// js-Youtube-player. This is done in order to get the algorithm
// for decrypting cryptic signatures inside certain stream urls.
JSONObject ytAssets = ytPlayerConfig.getJSONObject("assets");
String playerUrl = ytAssets.getString("js");
JSONObject ytAssets = playerConfig.getJSONObject("assets");
playerUrl = ytAssets.getString("js");
if (playerUrl.startsWith("//")) {
playerUrl = "https:" + playerUrl;
}
decryptionCode = loadDecryptionCode(playerUrl);
} catch (JSONException e) {
throw new ParsingException(
"Could not load decryption code for the Youtube service.", e);
if (playerUrl.startsWith("//")) {
playerUrl = "https:" + playerUrl;
}
return playerUrl;
} catch (JSONException e) {
throw new ParsingException(
"Could not load decryption code for the Youtube service.", e);
}
}
private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException {
try {
String playerUrl = "";
String videoId = urlidhandler.getVideoId(pageUrl);
String embedUrl = "https://www.youtube.com/embed/" + videoId;
String embedPageContent = downloader.download(embedUrl);
//todo: find out if this can be reapaced by Parser.matchGroup1()
Pattern assetsPattern = Pattern.compile("\"assets\":.+?\"js\":\\s*(\"[^\"]+\")");
Matcher patternMatcher = assetsPattern.matcher(embedPageContent);
while (patternMatcher.find()) {
playerUrl = patternMatcher.group(1);
}
playerUrl = playerUrl.replace("\\", "").replace("\"", "");
if (playerUrl.startsWith("//")) {
playerUrl = "https:" + playerUrl;
}
return playerUrl;
} catch (IOException e) {
throw new ParsingException(
"Could load decryption code form restricted video for the Youtube service.", e);
}
}
@Override
public String getTitle() throws ParsingException {
try {//json player args method
try {
if (playerArgs == null) {
return videoInfoPage.get("title");
}
//json player args method
return playerArgs.getString("title");
} catch(JSONException je) {//html <meta> method
je.printStackTrace();
@ -283,7 +330,11 @@ public class YoutubeStreamExtractor implements StreamExtractor {
@Override
public String getUploader() throws ParsingException {
try {//json player args method
try {
if (playerArgs == null) {
return videoInfoPage.get("author");
}
//json player args method
return playerArgs.getString("author");
} catch(JSONException je) {
je.printStackTrace();
@ -299,6 +350,9 @@ public class YoutubeStreamExtractor implements StreamExtractor {
@Override
public int getLength() throws ParsingException {
try {
if (playerArgs == null) {
return Integer.valueOf(videoInfoPage.get("length_seconds"));
}
return playerArgs.getInt("length_seconds");
} catch (JSONException e) {//todo: find fallback method
throw new ParsingException("failed to load video duration from JSON args", e);
@ -339,6 +393,9 @@ public class YoutubeStreamExtractor implements StreamExtractor {
} catch (JSONException je) {
throw new ParsingException(
"failed to extract thumbnail URL from JSON args; trying to extract it from HTML", je);
} catch (NullPointerException ne) {
// Get from the video info page instead
return videoInfoPage.get("thumbnail_url");
}
}
@ -379,7 +436,13 @@ public class YoutubeStreamExtractor implements StreamExtractor {
public List<VideoInfo.AudioStream> getAudioStreams() throws ParsingException {
Vector<VideoInfo.AudioStream> audioStreams = new Vector<>();
try{
String encoded_url_map = playerArgs.getString("adaptive_fmts");
String encoded_url_map;
// playerArgs could be null if the video is age restricted
if (playerArgs == null) {
encoded_url_map = videoInfoPage.get("adaptive_fmts");
} else {
encoded_url_map = playerArgs.getString("adaptive_fmts");
}
for(String url_data_str : encoded_url_map.split(",")) {
// This loop iterates through multiple streams, therefor tags
// is related to one and the same stream at a time.
@ -416,7 +479,13 @@ public class YoutubeStreamExtractor implements StreamExtractor {
Vector<VideoInfo.VideoStream> videoStreams = new Vector<>();
try{
String encoded_url_map = playerArgs.getString("url_encoded_fmt_stream_map");
String encoded_url_map;
// playerArgs could be null if the video is age restricted
if (playerArgs == null) {
encoded_url_map = videoInfoPage.get("url_encoded_fmt_stream_map");
} else {
encoded_url_map = playerArgs.getString("url_encoded_fmt_stream_map");
}
for(String url_data_str : encoded_url_map.split(",")) {
try {
// This loop iterates through multiple streams, therefor tags
@ -499,7 +568,8 @@ public class YoutubeStreamExtractor implements StreamExtractor {
int minutes = (minutesString.isEmpty() ? 0 : Integer.parseInt(minutesString));
int hours = (hoursString.isEmpty() ? 0 : Integer.parseInt(hoursString));
int ret = seconds + (60 * minutes) + (3600 * hours);//don't trust BODMAS!
//don't trust BODMAS!
int ret = seconds + (60 * minutes) + (3600 * hours);
//Log.d(TAG, "derived timestamp value:"+ret);
return ret;
//the ordering varies internationally
@ -513,15 +583,24 @@ public class YoutubeStreamExtractor implements StreamExtractor {
@Override
public int getAgeLimit() throws ParsingException {
// Not yet implemented.
// Also you need to be logged in to see age restricted videos on youtube,
// therefore NP is not able to receive such videos.
return 0;
if (!isAgeRestricted) {
return 0;
}
try {
return Integer.valueOf(doc.head()
.getElementsByAttributeValue("property", "og:restrictions:age")
.attr("content").replace("+", ""));
} catch (Exception e) {
throw new ParsingException("Could not get age restriction");
}
}
@Override
public String getAverageRating() throws ParsingException {
try {
if (playerArgs == null) {
videoInfoPage.get("avg_rating");
}
return playerArgs.getString("avg_rating");
} catch (JSONException e) {
throw new ParsingException("Could not get Average rating", e);

2
app/src/main/res/values/settings_keys.xml
View File

@ -3,6 +3,7 @@
<!-- Categories -->
<string name="settings_category_video_audio">settings_category_video_audio</string>
<string name="settings_category_appearance">settings_category_appearance</string>
<string name="settings_content_options">settings_content_options</string>
<string name="settings_category_other">settings_category_other</string>
<!-- Key values -->
<string name="download_path_key">download_path</string>
@ -207,5 +208,6 @@
<item>日本語</item>
<item>한국어</item>
</string-array>
<string name="show_age_restricted_content">show_age_restricted_content</string>
<string name="use_tor_key">use_tor</string>
</resources>

3
app/src/main/res/values/strings.xml
View File

@ -74,6 +74,9 @@
<string name="background_player_playing_toast">Playing in background</string>
<string name="c3s_url" translatable="false">https://www.c3s.cc/</string>
<string name="play_btn_text">Play</string>
<string name="content">Content</string>
<string name="show_age_restricted_content_title">Show age restricted content</string>
<string name="video_is_age_restricted">Video is Age restricted. Enable age restricted videos in the settings first.</string>
<!-- error strings -->
<string name="general_error">Error</string>

16
app/src/main/res/xml/settings.xml
View File

@ -63,8 +63,8 @@
</PreferenceCategory>
<PreferenceCategory
android:key="@string/settings_category_other"
android:title="@string/settings_category_other_title"
android:key="@string/settings_content_options"
android:title="@string/content"
android:textAllCaps="true">
<ListPreference
@ -75,6 +75,18 @@
android:entryValues="@array/language_codes"
android:defaultValue="@string/default_language_value" />
<CheckBoxPreference
android:key="@string/show_age_restricted_content"
android:title="@string/show_age_restricted_content_title"
android:defaultValue="false"/>
</PreferenceCategory>
<PreferenceCategory
android:key="@string/settings_category_other"
android:title="@string/settings_category_other_title"
android:textAllCaps="true">
<EditTextPreference
android:key="@string/download_path_key"
android:title="@string/download_path_title"