lasers/NewPipe
1
0
Fork 0
mirror of https://github.com/TeamNewPipe/NewPipe synced 2024-12-21 08:09:38 +01:00
Code Issues Projects Releases Wiki Activity

Merge branch 'reCaptcha' of git://github.com/be-neth/NewPipe into be-neth-reCaptcha

Browse Source
This commit is contained in:
Christian Schabesberger 2016-12-24 14:13:12 +01:00
commit cd65f1dffc
14 changed files with 265 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
app/src/main/AndroidManifest.xml
View File

@ -153,6 +153,9 @@
android:label="@string/title_activity_channel"
android:theme="@style/AppTheme.NoActionBar" />
<activity
android:name=".ReCaptchaActivity"
android:label="@string/reCaptchaActivity" />
</application>
</manifest>

2
app/src/main/java/org/schabi/newpipe/App.java
View File

@ -62,7 +62,7 @@ public class App extends Application {
}
//init NewPipe
NewPipe.init(new Downloader());
NewPipe.init(Downloader.getInstance());
// Initialize image loader
ImageLoaderConfiguration config = new ImageLoaderConfiguration.Builder(this).build();

46
app/src/main/java/org/schabi/newpipe/Downloader.java
View File

@ -1,5 +1,7 @@
package org.schabi.newpipe;
import org.schabi.newpipe.extractor.exceptions.reCaptchaException;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
@ -35,13 +37,37 @@ import javax.net.ssl.HttpsURLConnection;
public class Downloader implements org.schabi.newpipe.extractor.Downloader {
private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0";
private static String mCookies = "";
private static Downloader instance = null;
private Downloader() {}
public static Downloader getInstance() {
if(instance == null) {
synchronized (Downloader.class) {
if (instance == null) {
instance = new Downloader();
}
}
}
return instance;
}
public static synchronized void setCookies(String cookies) {
Downloader.mCookies = cookies;
}
public static synchronized String getCookies() {
return Downloader.mCookies;
}
/**Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string.
* @param siteUrl the URL of the text file to return the contents of
* @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file*/
public String download(String siteUrl, String language) throws IOException {
public String download(String siteUrl, String language) throws IOException, reCaptchaException {
Map<String, String> requestProperties = new HashMap<>();
requestProperties.put("Accept-Language", language);
return download(siteUrl, requestProperties);
@ -54,7 +80,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
* @param customProperties set request header properties
* @return the contents of the specified text file
* @throws IOException*/
public String download(String siteUrl, Map<String, String> customProperties) throws IOException {
public String download(String siteUrl, Map<String, String> customProperties) throws IOException, reCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
Iterator it = customProperties.entrySet().iterator();
@ -66,7 +92,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
}
/**Common functionality between download(String url) and download(String url, String language)*/
private static String dl(HttpsURLConnection con) throws IOException {
private static String dl(HttpsURLConnection con) throws IOException, reCaptchaException {
StringBuilder response = new StringBuilder();
BufferedReader in = null;
@ -74,6 +100,10 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
con.setRequestMethod("GET");
con.setRequestProperty("User-Agent", USER_AGENT);
if (getCookies().length() > 0) {
con.setRequestProperty("Cookie", getCookies());
}
in = new BufferedReader(
new InputStreamReader(con.getInputStream()));
String inputLine;
@ -85,6 +115,14 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
throw new IOException("unknown host or no network", uhe);
//Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show();
} catch(Exception e) {
/*
* HTTP 429 == Too Many Request
* Receive from Youtube.com = ReCaptcha challenge request
* See : https://github.com/rg3/youtube-dl/issues/5138
*/
if (con.getResponseCode() == 429) {
throw new reCaptchaException("reCaptcha Challenge requested");
}
throw new IOException(e);
} finally {
if(in != null) {
@ -99,7 +137,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
* Primarily intended for downloading web pages.
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file*/
public String download(String siteUrl) throws IOException {
public String download(String siteUrl) throws IOException, reCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
//HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);

1
app/src/main/java/org/schabi/newpipe/MainActivity.java
View File

@ -46,6 +46,7 @@ public class MainActivity extends AppCompatActivity {
.findFragmentById(R.id.search_fragment);
}
@Override
public boolean onCreateOptionsMenu(Menu menu) {
super.onCreateOptionsMenu(menu);
MenuInflater inflater = getMenuInflater();

138
app/src/main/java/org/schabi/newpipe/ReCaptchaActivity.java Normal file
View File

@ -0,0 +1,138 @@
package org.schabi.newpipe;
import android.app.Activity;
import android.content.Intent;
import android.os.Build;
import android.os.Bundle;
import android.support.v4.app.NavUtils;
import android.support.v7.app.ActionBar;
import android.support.v7.app.AppCompatActivity;
import android.view.MenuItem;
import android.webkit.CookieManager;
import android.webkit.ValueCallback;
import android.webkit.WebSettings;
import android.webkit.WebView;
import android.webkit.WebViewClient;
/**
* Created by beneth <bmauduit@beneth.fr> on 06.12.16.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* ReCaptchaActivity.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class ReCaptchaActivity extends AppCompatActivity {
public static final String TAG = ReCaptchaActivity.class.toString();
public static final String YT_URL = "https://www.youtube.com";
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_recaptcha);
ActionBar actionBar = getSupportActionBar();
actionBar.setDisplayHomeAsUpEnabled(true);
actionBar.setTitle(R.string.reCaptcha_title);
actionBar.setDisplayShowTitleEnabled(true);
WebView myWebView = (WebView) findViewById(R.id.reCaptchaWebView);
// Enable Javascript
WebSettings webSettings = myWebView.getSettings();
webSettings.setJavaScriptEnabled(true);
ReCaptchaWebViewClient webClient = new ReCaptchaWebViewClient(this);
myWebView.setWebViewClient(webClient);
// Cleaning cache, history and cookies from webView
myWebView.clearCache(true);
myWebView.clearHistory();
android.webkit.CookieManager cookieManager = CookieManager.getInstance();
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
cookieManager.removeAllCookies(new ValueCallback<Boolean>() {
@Override
public void onReceiveValue(Boolean aBoolean) {}
});
} else {
cookieManager.removeAllCookie();
}
myWebView.loadUrl(YT_URL);
}
private class ReCaptchaWebViewClient extends WebViewClient {
private Activity context;
private String mCookies;
ReCaptchaWebViewClient(Activity ctx) {
context = ctx;
}
@Override
public void onPageFinished(WebView view, String url) {
String cookies = CookieManager.getInstance().getCookie(url);
// find cookies : s_gl & goojf and Add cookies to Downloader
if (find_access_cookies(cookies)) {
// Give cookies to Downloader class
Downloader.setCookies(mCookies);
// Closing activity and return to parent.
Intent intent = new Intent(context, org.schabi.newpipe.MainActivity.class);
intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP);
NavUtils.navigateUpTo(context, intent);
}
}
private boolean find_access_cookies(String cookies) {
boolean ret = false;
String c_s_gl = "";
String c_goojf = "";
String[] parts = cookies.split("; ");
for (String part : parts) {
if (part.trim().startsWith("s_gl")) {
c_s_gl = part.trim();
}
if (part.trim().startsWith("goojf")) {
c_goojf = part.trim();
}
}
if (c_s_gl.length() > 0 && c_goojf.length() > 0) {
ret = true;
//mCookies = c_s_gl + "; " + c_goojf;
// Youtube seems to also need the other cookies:
mCookies = cookies;
}
return ret;
}
}
@Override
public boolean onOptionsItemSelected(MenuItem item) {
int id = item.getItemId();
switch (id) {
case android.R.id.home: {
Intent intent = new Intent(this, org.schabi.newpipe.MainActivity.class);
intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP);
NavUtils.navigateUpTo(this, intent);
return true;
}
default:
return false;
}
}
}

5
app/src/main/java/org/schabi/newpipe/extractor/DashMpdParser.java
View File

@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor;
import android.util.Xml;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.reCaptchaException;
import org.schabi.newpipe.extractor.stream_info.AudioStream;
import org.xmlpull.v1.XmlPullParser;
@ -43,13 +44,15 @@ public class DashMpdParser {
}
public static List<AudioStream> getAudioStreams(String dashManifestUrl)
throws DashMpdParsingException {
throws DashMpdParsingException, reCaptchaException {
String dashDoc;
Downloader downloader = NewPipe.getDownloader();
try {
dashDoc = downloader.download(dashManifestUrl);
} catch(IOException ioe) {
throw new DashMpdParsingException("Could not get dash mpd: " + dashManifestUrl, ioe);
} catch (reCaptchaException e) {
throw new reCaptchaException("reCaptcha Challenge needed");
}
Vector<AudioStream> audioStreams = new Vector<>();
try {

8
app/src/main/java/org/schabi/newpipe/extractor/Downloader.java
View File

@ -1,5 +1,7 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.reCaptchaException;
import java.io.IOException;
import java.util.Map;
@ -31,7 +33,7 @@ public interface Downloader {
* @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file
* @throws IOException*/
String download(String siteUrl, String language) throws IOException;
String download(String siteUrl, String language) throws IOException, reCaptchaException;
/**Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string.
@ -39,12 +41,12 @@ public interface Downloader {
* @param customProperties set request header properties
* @return the contents of the specified text file
* @throws IOException*/
String download(String siteUrl, Map<String, String> customProperties) throws IOException;
String download(String siteUrl, Map<String, String> customProperties) throws IOException, reCaptchaException;
/**Download (via HTTP) the text file located at the supplied URL, and return its contents.
* Primarily intended for downloading web pages.
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file
* @throws IOException*/
String download(String siteUrl) throws IOException;
String download(String siteUrl) throws IOException, reCaptchaException;
}

27
app/src/main/java/org/schabi/newpipe/extractor/exceptions/reCaptchaException.java Normal file
View File

@ -0,0 +1,27 @@
package org.schabi.newpipe.extractor.exceptions;
/**
* Created by beneth <bmauduit@beneth.fr> on 07.12.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* reCaptchaException.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class reCaptchaException extends ExtractionException {
public reCaptchaException(String message) {
super(message);
}
}

5
app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java
View File

@ -11,6 +11,7 @@ import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.extractor.AbstractStreamInfo;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.reCaptchaException;
import org.schabi.newpipe.extractor.stream_info.AudioStream;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
@ -280,7 +281,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException {
private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException, reCaptchaException {
try {
Downloader downloader = NewPipe.getDownloader();
String playerUrl = "";
@ -302,6 +303,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} catch (IOException e) {
throw new ParsingException(
"Could load decryption code form restricted video for the Youtube service.", e);
} catch (reCaptchaException e) {
throw new reCaptchaException("reCaptcha Challenge requested");
}
}

2
app/src/main/java/org/schabi/newpipe/report/ErrorActivity.java
View File

@ -473,7 +473,7 @@ public class ErrorActivity extends AppCompatActivity {
public void run() {
String ipRange = "none";
try {
Downloader dl = new Downloader();
Downloader dl = Downloader.getInstance();
String ip = dl.download("https://ifcfg.me/ip");
ipRange = Parser.matchGroup1("([0-9]*\\.[0-9]*\\.)[0-9]*\\.[0-9]*", ip)

12
app/src/main/java/org/schabi/newpipe/search_fragment/SearchInfoItemFragment.java
View File

@ -18,6 +18,7 @@ import android.view.inputmethod.InputMethodManager;
import android.widget.ProgressBar;
import android.widget.Toast;
import org.schabi.newpipe.ReCaptchaActivity;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchResult;
import org.schabi.newpipe.info_list.InfoItemBuilder;
@ -149,7 +150,7 @@ public class SearchInfoItemFragment extends Fragment {
}
SearchWorker sw = SearchWorker.getInstance();
sw.setSearchWorkerResultListner(new SearchWorker.SearchWorkerResultListner() {
sw.setSearchWorkerResultListener(new SearchWorker.SearchWorkerResultListener() {
@Override
public void onResult(SearchResult result) {
infoListAdapter.addStreamItemList(result.resultList);
@ -174,6 +175,15 @@ public class SearchInfoItemFragment extends Fragment {
isLoading = false;
loadingIndicator.setVisibility(View.GONE);
}
@Override
public void onReCaptchaChallenge() {
Toast.makeText(getActivity(), "ReCaptcha Challenge requested",
Toast.LENGTH_LONG).show();
// Starting ReCaptcha Challenge Activity
Intent i = new Intent(getActivity(), ReCaptchaActivity.class);
getActivity().startActivity(i);
}
});
}

23
app/src/main/java/org/schabi/newpipe/search_fragment/SearchWorker.java
View File

@ -8,6 +8,7 @@ import android.util.Log;
import android.view.View;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.reCaptchaException;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchResult;
import org.schabi.newpipe.report.ErrorActivity;
@ -40,10 +41,11 @@ import java.io.IOException;
public class SearchWorker {
private static final String TAG = SearchWorker.class.toString();
public interface SearchWorkerResultListner {
public interface SearchWorkerResultListener {
void onResult(SearchResult result);
void onNothingFound(final int stringResource);
void onError(String message);
void onReCaptchaChallenge();
}
private class ResultRunnable implements Runnable {
@ -56,7 +58,7 @@ public class SearchWorker {
@Override
public void run() {
if(this.requestId == SearchWorker.this.requestId) {
searchWorkerResultListner.onResult(result);
searchWorkerResultListener.onResult(result);
}
}
}
@ -121,11 +123,18 @@ public class SearchWorker {
}
// hard errors:
} catch (reCaptchaException e) {
h.post(new Runnable() {
@Override
public void run() {
searchWorkerResultListener.onReCaptchaChallenge();
}
});
} catch(IOException e) {
h.post(new Runnable() {
@Override
public void run() {
searchWorkerResultListner.onNothingFound(R.string.network_error);
searchWorkerResultListener.onNothingFound(R.string.network_error);
}
});
e.printStackTrace();
@ -133,7 +142,7 @@ public class SearchWorker {
h.post(new Runnable() {
@Override
public void run() {
searchWorkerResultListner.onError(e.getMessage());
searchWorkerResultListener.onError(e.getMessage());
}
});
} catch(ExtractionException e) {
@ -155,7 +164,7 @@ public class SearchWorker {
}
private static SearchWorker searchWorker = null;
private SearchWorkerResultListner searchWorkerResultListner = null;
private SearchWorkerResultListener searchWorkerResultListener = null;
private SearchRunnable runnable = null;
private int requestId = 0; //prevents running requests that have already ben expired
@ -163,8 +172,8 @@ public class SearchWorker {
return searchWorker == null ? (searchWorker = new SearchWorker()) : searchWorker;
}
public void setSearchWorkerResultListner(SearchWorkerResultListner listener) {
searchWorkerResultListner = listener;
public void setSearchWorkerResultListener(SearchWorkerResultListener listener) {
searchWorkerResultListener = listener;
}
private SearchWorker() {

10
app/src/main/res/layout/activity_recaptcha.xml Normal file
View File

@ -0,0 +1,10 @@
<?xml version="1.0" encoding="utf-8"?>
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
android:orientation="vertical" android:layout_width="match_parent"
android:layout_height="match_parent">
<WebView
android:layout_width="match_parent"
android:layout_height="match_parent"
android:id="@+id/reCaptchaWebView" />
</LinearLayout>

2
app/src/main/res/values/strings.xml
View File

@ -269,6 +269,8 @@
"when a user tries to pick up one of cards.\n\n"
</string>
<string name="action_settings">Settings</string>
<string name="reCaptchaActivity">reCaptcha</string>
<string name="reCaptcha_title">reCaptcha Challenge</string>
<!-- End of GigaGet's Strings -->