lasers/NewPipe
1
0
Fork 0
mirror of https://github.com/TeamNewPipe/NewPipe synced 2024-12-12 11:49:42 +01:00
Code Issues Projects Releases Wiki Activity

detatch android related downloader from crawler

Browse Source
This commit is contained in:
Christian Schabesberger 2016-01-28 21:21:19 +01:00
parent f152d66cd8
commit 54d318bf04
12 changed files with 243 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/src/main/java/org/schabi/newpipe/DownloadDialog.java
View File

@ -107,7 +107,7 @@ public class DownloadDialog extends DialogFragment {
long id = 0;
if (App.isUsingTor()) {
// if using Tor, do not use DownloadManager because the proxy cannot be set
Downloader.downloadFile(getContext(), url, saveFilePath, title);
FileDownloader.downloadFile(getContext(), url, saveFilePath, title);
} else {
DownloadManager dm = (DownloadManager) context.getSystemService(Context.DOWNLOAD_SERVICE);
DownloadManager.Request request = new DownloadManager.Request(

145
app/src/main/java/org/schabi/newpipe/Downloader.java
View File

@ -1,24 +1,8 @@
package org.schabi.newpipe;
import android.app.NotificationManager;
import android.content.Context;
import android.content.SharedPreferences;
import android.graphics.drawable.BitmapDrawable;
import android.graphics.drawable.Drawable;
import android.os.AsyncTask;
import android.preference.PreferenceManager;
import android.support.v4.app.NotificationCompat;
import android.util.Log;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.UnknownHostException;
@ -27,7 +11,7 @@ import javax.net.ssl.HttpsURLConnection;
import info.guardianproject.netcipher.NetCipher;
/**
* Created by Christian Schabesberger on 14.08.15.
* Created by Christian Schabesberger on 28.01.16.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* Downloader.java is part of NewPipe.
@ -46,38 +30,16 @@ import info.guardianproject.netcipher.NetCipher;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class Downloader extends AsyncTask<Void, Integer, Void> {
public static final String TAG = "Downloader";
public class Downloader implements org.schabi.newpipe.services.Downloader {
private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0";
private NotificationManager nm;
private NotificationCompat.Builder builder;
private int notifyId = 0x1234;
private int fileSize = 0xffffffff;
private final Context context;
private final String fileURL;
private final File saveFilePath;
private final String title;
private final String debugContext;
public Downloader(Context context, String fileURL, File saveFilePath, String title) {
this.context = context;
this.fileURL = fileURL;
this.saveFilePath = saveFilePath;
this.title = title;
this.debugContext = "'" + fileURL +
"' => '" + saveFilePath + "'";
}
/**Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string.
* @param siteUrl the URL of the text file to return the contents of
* @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file*/
public static String download(String siteUrl, String language) {
public String download(String siteUrl, String language) {
String ret = "";
try {
URL url = new URL(siteUrl);
@ -118,11 +80,11 @@ public class Downloader extends AsyncTask<Void, Integer, Void> {
return response.toString();
}
/**Download (via HTTP) the text file located at the supplied URL, and return its contents.
/**Download (via HTTP) the text file located at the supplied URL, and return its contents.
* Primarily intended for downloading web pages.
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file*/
public static String download(String siteUrl) {
public String download(String siteUrl) {
String ret = "";
try {
@ -137,99 +99,4 @@ public class Downloader extends AsyncTask<Void, Integer, Void> {
return ret;
}
/**
* Downloads a file from a URL in the background using an {@link AsyncTask}.
*
* @param fileURL HTTP URL of the file to be downloaded
* @param saveFilePath path of the directory to save the file
* @param title
* @throws IOException
*/
public static void downloadFile(final Context context, final String fileURL, final File saveFilePath, String title) {
new Downloader(context, fileURL, saveFilePath, title).execute();
}
/** AsyncTask impl: executed in gui thread */
@Override
protected void onPreExecute() {
super.onPreExecute();
nm = (NotificationManager) context.getSystemService(Context.NOTIFICATION_SERVICE);
Drawable icon = context.getResources().getDrawable(R.mipmap.ic_launcher);
builder = new NotificationCompat.Builder(context)
.setSmallIcon(android.R.drawable.stat_sys_download)
.setLargeIcon(((BitmapDrawable) icon).getBitmap())
.setContentTitle(saveFilePath.getName())
.setContentText(saveFilePath.getAbsolutePath())
.setProgress(fileSize, 0, false);
nm.notify(notifyId, builder.build());
}
/** AsyncTask impl: executed in background thread does the download */
@Override
protected Void doInBackground(Void... voids) {
HttpsURLConnection con = null;
InputStream inputStream = null;
FileOutputStream outputStream = null;
try {
con = NetCipher.getHttpsURLConnection(fileURL);
int responseCode = con.getResponseCode();
// always check HTTP response code first
if (responseCode == HttpURLConnection.HTTP_OK) {
fileSize = con.getContentLength();
inputStream = new BufferedInputStream(con.getInputStream());
outputStream = new FileOutputStream(saveFilePath);
int bufferSize = 8192;
int downloaded = 0;
int bytesRead = -1;
byte[] buffer = new byte[bufferSize];
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
downloaded += bytesRead;
if (downloaded % 50000 < bufferSize) {
publishProgress(downloaded);
}
}
publishProgress(bufferSize);
} else {
Log.i(TAG, "No file to download. Server replied HTTP code: " + responseCode);
}
} catch (IOException e) {
Log.e(TAG, "No file to download. Server replied HTTP code: ", e);
e.printStackTrace();
} finally {
try {
if (outputStream != null) {
outputStream.close();
}
if (inputStream != null) {
inputStream.close();
}
} catch (IOException e) {
e.printStackTrace();
}
if (con != null) {
con.disconnect();
}
}
return null;
}
@Override
protected void onProgressUpdate(Integer... progress) {
builder.setProgress(fileSize, progress[0], false);
nm.notify(notifyId, builder.build());
}
@Override
protected void onPostExecute(Void aVoid) {
super.onPostExecute(aVoid);
nm.cancel(notifyId);
}
}

169
app/src/main/java/org/schabi/newpipe/FileDownloader.java Normal file
View File

@ -0,0 +1,169 @@
package org.schabi.newpipe;
import android.app.NotificationManager;
import android.content.Context;
import android.content.SharedPreferences;
import android.graphics.drawable.BitmapDrawable;
import android.graphics.drawable.Drawable;
import android.os.AsyncTask;
import android.preference.PreferenceManager;
import android.support.v4.app.NotificationCompat;
import android.util.Log;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.UnknownHostException;
import javax.net.ssl.HttpsURLConnection;
import info.guardianproject.netcipher.NetCipher;
/**
* Created by Christian Schabesberger on 14.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* FileDownloader.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class FileDownloader extends AsyncTask<Void, Integer, Void> {
public static final String TAG = "FileDownloader";
private NotificationManager nm;
private NotificationCompat.Builder builder;
private int notifyId = 0x1234;
private int fileSize = 0xffffffff;
private final Context context;
private final String fileURL;
private final File saveFilePath;
private final String title;
private final String debugContext;
public FileDownloader(Context context, String fileURL, File saveFilePath, String title) {
this.context = context;
this.fileURL = fileURL;
this.saveFilePath = saveFilePath;
this.title = title;
this.debugContext = "'" + fileURL +
"' => '" + saveFilePath + "'";
}
/**
* Downloads a file from a URL in the background using an {@link AsyncTask}.
*
* @param fileURL HTTP URL of the file to be downloaded
* @param saveFilePath path of the directory to save the file
* @param title
* @throws IOException
*/
public static void downloadFile(final Context context, final String fileURL, final File saveFilePath, String title) {
new FileDownloader(context, fileURL, saveFilePath, title).execute();
}
/** AsyncTask impl: executed in gui thread */
@Override
protected void onPreExecute() {
super.onPreExecute();
nm = (NotificationManager) context.getSystemService(Context.NOTIFICATION_SERVICE);
Drawable icon = context.getResources().getDrawable(R.mipmap.ic_launcher);
builder = new NotificationCompat.Builder(context)
.setSmallIcon(android.R.drawable.stat_sys_download)
.setLargeIcon(((BitmapDrawable) icon).getBitmap())
.setContentTitle(saveFilePath.getName())
.setContentText(saveFilePath.getAbsolutePath())
.setProgress(fileSize, 0, false);
nm.notify(notifyId, builder.build());
}
/** AsyncTask impl: executed in background thread does the download */
@Override
protected Void doInBackground(Void... voids) {
HttpsURLConnection con = null;
InputStream inputStream = null;
FileOutputStream outputStream = null;
try {
con = NetCipher.getHttpsURLConnection(fileURL);
int responseCode = con.getResponseCode();
// always check HTTP response code first
if (responseCode == HttpURLConnection.HTTP_OK) {
fileSize = con.getContentLength();
inputStream = new BufferedInputStream(con.getInputStream());
outputStream = new FileOutputStream(saveFilePath);
int bufferSize = 8192;
int downloaded = 0;
int bytesRead = -1;
byte[] buffer = new byte[bufferSize];
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
downloaded += bytesRead;
if (downloaded % 50000 < bufferSize) {
publishProgress(downloaded);
}
}
publishProgress(bufferSize);
} else {
Log.i(TAG, "No file to download. Server replied HTTP code: " + responseCode);
}
} catch (IOException e) {
Log.e(TAG, "No file to download. Server replied HTTP code: ", e);
e.printStackTrace();
} finally {
try {
if (outputStream != null) {
outputStream.close();
}
if (inputStream != null) {
inputStream.close();
}
} catch (IOException e) {
e.printStackTrace();
}
if (con != null) {
con.disconnect();
}
}
return null;
}
@Override
protected void onProgressUpdate(Integer... progress) {
builder.setProgress(fileSize, progress[0], false);
nm.notify(notifyId, builder.build());
}
@Override
protected void onPostExecute(Void aVoid) {
super.onPostExecute(aVoid);
nm.cancel(notifyId);
}
}

2
app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java
View File

@ -108,7 +108,7 @@ public class VideoItemDetailFragment extends Fragment {
@Override
public void run() {
try {
this.videoExtractor = service.getExtractorInstance(videoUrl);
this.videoExtractor = service.getExtractorInstance(videoUrl, new Downloader());
VideoInfo videoInfo = videoExtractor.getVideoInfo();
h.post(new VideoResultReturnedRunnable(videoInfo));
if (videoInfo.errorCode == VideoInfo.NO_ERROR) {

3
app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java
View File

@ -108,7 +108,8 @@ public class VideoItemListFragment extends ListFragment {
String searchLanguageKey = getContext().getString(R.string.search_language_key);
String searchLanguage = sp.getString(searchLanguageKey,
getString(R.string.default_language_value));
SearchEngine.Result result = engine.search(query, page, searchLanguage);
SearchEngine.Result result = engine.search(query, page, searchLanguage,
new Downloader());
Log.i(TAG, "language code passed:\""+searchLanguage+"\"");
if(runs) {

37
app/src/main/java/org/schabi/newpipe/services/Downloader.java Normal file
View File

@ -0,0 +1,37 @@
package org.schabi.newpipe.services;
/**
* Created by Christian Schabesberger on 28.01.16.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* Downloader.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public interface Downloader {
/**Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string.
* @param siteUrl the URL of the text file to return the contents of
* @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file*/
String download(String siteUrl, String language);
/**Download (via HTTP) the text file located at the supplied URL, and return its contents.
* Primarily intended for downloading web pages.
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file*/
String download(String siteUrl);
}

6
app/src/main/java/org/schabi/newpipe/services/SearchEngine.java
View File

@ -27,16 +27,14 @@ import java.util.Vector;
@SuppressWarnings("ALL")
public interface SearchEngine {
class Result {
public String errorMessage = "";
public String suggestion = "";
public final Vector<VideoPreviewInfo> resultList = new Vector<>();
}
ArrayList<String> suggestionList(String query);
ArrayList<String> suggestionList(String query, Downloader dl);
//Result search(String query, int page);
Result search(String query, int page, String contentCountry);
Result search(String query, int page, String contentCountry, Downloader dl);
}

2
app/src/main/java/org/schabi/newpipe/services/StreamingService.java
View File

@ -25,7 +25,7 @@ public interface StreamingService {
public String name = "";
}
ServiceInfo getServiceInfo();
VideoExtractor getExtractorInstance(String url);
VideoExtractor getExtractorInstance(String url, Downloader downloader);
SearchEngine getSearchEngineInstance();
/**When a VIEW_ACTION is caught this function will test if the url delivered within the calling

2
app/src/main/java/org/schabi/newpipe/services/VideoExtractor.java
View File

@ -28,7 +28,7 @@ public abstract class VideoExtractor {
protected VideoInfo videoInfo;
@SuppressWarnings("WeakerAccess")
public VideoExtractor(String url) {
public VideoExtractor(String url, Downloader dl) {
this.pageUrl = url;
}

12
app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngine.java
View File

@ -6,7 +6,7 @@ import android.util.Log;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.services.Downloader;
import org.schabi.newpipe.services.SearchEngine;
import org.schabi.newpipe.VideoPreviewInfo;
import org.w3c.dom.Node;
@ -49,7 +49,7 @@ public class YoutubeSearchEngine implements SearchEngine {
private static final String TAG = YoutubeSearchEngine.class.toString();
@Override
public Result search(String query, int page, String languageCode) {
public Result search(String query, int page, String languageCode, Downloader downloader) {
//String contentCountry = PreferenceManager.getDefaultSharedPreferences(this).getString(getString(R.string., "");
Uri.Builder builder = new Uri.Builder();
builder.scheme("https")
@ -64,10 +64,10 @@ public class YoutubeSearchEngine implements SearchEngine {
//if we've been passed a valid language code, append it to the URL
if(!languageCode.isEmpty()) {
//assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode);
site = Downloader.download(url, languageCode);
site = downloader.download(url, languageCode);
}
else {
site = Downloader.download(url);
site = downloader.download(url);
}
@ -140,7 +140,7 @@ public class YoutubeSearchEngine implements SearchEngine {
}
@Override
public ArrayList<String> suggestionList(String query) {
public ArrayList<String> suggestionList(String query, Downloader dl) {
ArrayList<String> suggestions = new ArrayList<>();
@ -155,7 +155,7 @@ public class YoutubeSearchEngine implements SearchEngine {
.appendQueryParameter("q", query);
String url = builder.build().toString();
String response = Downloader.download(url);
String response = dl.download(url);
//TODO: Parse xml data using Jsoup not done
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();

5
app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeService.java
View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.services.youtube;
import org.schabi.newpipe.services.Downloader;
import org.schabi.newpipe.services.StreamingService;
import org.schabi.newpipe.services.VideoExtractor;
import org.schabi.newpipe.services.SearchEngine;
@ -33,9 +34,9 @@ public class YoutubeService implements StreamingService {
return serviceInfo;
}
@Override
public VideoExtractor getExtractorInstance(String url) {
public VideoExtractor getExtractorInstance(String url, Downloader downloader) {
if(acceptUrl(url)) {
return new YoutubeVideoExtractor(url);
return new YoutubeVideoExtractor(url, downloader);
}
else {
throw new IllegalArgumentException("supplied String is not a valid Youtube URL");

17
app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractor.java
View File

@ -12,7 +12,8 @@ import org.jsoup.parser.Parser;
import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.FileDownloader;
import org.schabi.newpipe.services.Downloader;
import org.schabi.newpipe.services.VideoExtractor;
import org.schabi.newpipe.services.MediaFormat;
import org.schabi.newpipe.services.VideoInfo;
@ -62,9 +63,13 @@ public class YoutubeVideoExtractor extends VideoExtractor {
// cached values
private static volatile String decryptionCode = "";
public YoutubeVideoExtractor(String pageUrl) {
super(pageUrl);//most common videoInfo fields are now set in our superclass, for all services
String pageContent = Downloader.download(cleanUrl(pageUrl));
private Downloader downloader;
public YoutubeVideoExtractor(String pageUrl, Downloader dl) {
//most common videoInfo fields are now set in our superclass, for all services
super(pageUrl, dl);
downloader = dl;
String pageContent = downloader.download(cleanUrl(pageUrl));
doc = Jsoup.parse(pageContent, pageUrl);
//attempt to load the youtube js player JSON arguments
@ -472,7 +477,7 @@ public class YoutubeVideoExtractor extends VideoExtractor {
decryptedSig = decryptSignature(encryptedSig, decryptoinCode);
dashManifest = dashManifest.replace("/s/" + encryptedSig, "/signature/" + decryptedSig);
}
String dashDoc = Downloader.download(dashManifest);
String dashDoc = downloader.download(dashManifest);
Vector<VideoInfo.AudioStream> audioStreams = new Vector<>();
try {
XmlPullParser parser = Xml.newPullParser();
@ -574,7 +579,7 @@ public class YoutubeVideoExtractor extends VideoExtractor {
}
private String loadDecryptionCode(String playerUrl) {
String playerCode = Downloader.download(playerUrl);
String playerCode = downloader.download(playerUrl);
String decryptionFuncName = "";
String decryptionFunc = "";
String helperObjectName;