detatch android related downloader from crawler

app/src/main/java/org/schabi/newpipe/DownloadDialog.java

@@ -107,7 +107,7 @@ public class DownloadDialog extends DialogFragment {
                         long id = 0;
                         if (App.isUsingTor()) {
                             // if using Tor, do not use DownloadManager because the proxy cannot be set
-                            Downloader.downloadFile(getContext(), url, saveFilePath, title);
+                            FileDownloader.downloadFile(getContext(), url, saveFilePath, title);
                         } else {
                             DownloadManager dm = (DownloadManager) context.getSystemService(Context.DOWNLOAD_SERVICE);
                             DownloadManager.Request request = new DownloadManager.Request(

app/src/main/java/org/schabi/newpipe/Downloader.java

@@ -1,24 +1,8 @@
 package org.schabi.newpipe;
 
-
-import android.app.NotificationManager;
-import android.content.Context;
-import android.content.SharedPreferences;
-import android.graphics.drawable.BitmapDrawable;
-import android.graphics.drawable.Drawable;
-import android.os.AsyncTask;
-import android.preference.PreferenceManager;
-import android.support.v4.app.NotificationCompat;
-import android.util.Log;
-
-import java.io.BufferedInputStream;
 import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.net.HttpURLConnection;
 import java.net.URL;
 import java.net.UnknownHostException;
 
@@ -27,7 +11,7 @@ import javax.net.ssl.HttpsURLConnection;
 import info.guardianproject.netcipher.NetCipher;
 
 /**
- * Created by Christian Schabesberger on 14.08.15.
+ * Created by Christian Schabesberger on 28.01.16.
  *
  * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
  * Downloader.java is part of NewPipe.
@@ -46,38 +30,16 @@ import info.guardianproject.netcipher.NetCipher;
  * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-public class Downloader extends AsyncTask<Void, Integer, Void> {
+public class Downloader implements org.schabi.newpipe.services.Downloader {
-    public static final String TAG = "Downloader";
+    
     private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0";
 
-    private NotificationManager nm;
-    private NotificationCompat.Builder builder;
-    private int notifyId = 0x1234;
-    private int fileSize = 0xffffffff;
-
-    private final Context context;
-    private final String fileURL;
-    private final File saveFilePath;
-    private final String title;
-
-    private final String debugContext;
-
-    public Downloader(Context context, String fileURL, File saveFilePath, String title) {
-        this.context = context;
-        this.fileURL = fileURL;
-        this.saveFilePath = saveFilePath;
-        this.title = title;
-
-        this.debugContext = "'" + fileURL +
-                "' => '" + saveFilePath + "'";
-    }
-
     /**Download the text file at the supplied URL as in download(String),
      * but set the HTTP header field "Accept-Language" to the supplied string.
      * @param siteUrl the URL of the text file to return the contents of
      * @param language the language (usually a 2-character code) to set as the preferred language
      * @return the contents of the specified text file*/
-    public static String download(String siteUrl, String language) {
+    public String download(String siteUrl, String language) {
         String ret = "";
         try {
             URL url = new URL(siteUrl);
@@ -122,7 +84,7 @@ public class Downloader extends AsyncTask<Void, Integer, Void> {
      * Primarily intended for downloading web pages.
      * @param siteUrl the URL of the text file to download
      * @return the contents of the specified text file*/
-    public static String download(String siteUrl) {
+    public String download(String siteUrl) {
         String ret = "";
 
         try {
@@ -137,99 +99,4 @@ public class Downloader extends AsyncTask<Void, Integer, Void> {
 
         return ret;
     }
-
-    /**
-     * Downloads a file from a URL in the background using an {@link AsyncTask}.
-     *
-     * @param fileURL      HTTP URL of the file to be downloaded
-     * @param saveFilePath path of the directory to save the file
-     * @param title
-     * @throws IOException
-     */
-    public static void downloadFile(final Context context, final String fileURL, final File saveFilePath, String title) {
-        new Downloader(context, fileURL, saveFilePath, title).execute();
-    }
-
-    /** AsyncTask impl: executed in gui thread */
-    @Override
-    protected void onPreExecute() {
-        super.onPreExecute();
-        nm = (NotificationManager) context.getSystemService(Context.NOTIFICATION_SERVICE);
-        Drawable icon = context.getResources().getDrawable(R.mipmap.ic_launcher);
-        builder = new NotificationCompat.Builder(context)
-                .setSmallIcon(android.R.drawable.stat_sys_download)
-                .setLargeIcon(((BitmapDrawable) icon).getBitmap())
-                .setContentTitle(saveFilePath.getName())
-                .setContentText(saveFilePath.getAbsolutePath())
-                .setProgress(fileSize, 0, false);
-        nm.notify(notifyId, builder.build());
-    }
-
-    /** AsyncTask impl: executed in background thread does the download */
-    @Override
-    protected Void doInBackground(Void... voids) {
-        HttpsURLConnection con = null;
-        InputStream inputStream = null;
-        FileOutputStream outputStream = null;
-        try {
-            con = NetCipher.getHttpsURLConnection(fileURL);
-            int responseCode = con.getResponseCode();
-
-            // always check HTTP response code first
-            if (responseCode == HttpURLConnection.HTTP_OK) {
-                fileSize = con.getContentLength();
-                inputStream = new BufferedInputStream(con.getInputStream());
-                outputStream = new FileOutputStream(saveFilePath);
-
-                int bufferSize = 8192;
-                int downloaded = 0;
-
-                int bytesRead = -1;
-                byte[] buffer = new byte[bufferSize];
-                while ((bytesRead = inputStream.read(buffer)) != -1) {
-                    outputStream.write(buffer, 0, bytesRead);
-                    downloaded += bytesRead;
-                    if (downloaded % 50000 < bufferSize) {
-                        publishProgress(downloaded);
-                    }
-                }
-
-                publishProgress(bufferSize);
-
-            } else {
-                Log.i(TAG, "No file to download. Server replied HTTP code: " + responseCode);
-            }
-        } catch (IOException e) {
-            Log.e(TAG, "No file to download. Server replied HTTP code: ", e);
-            e.printStackTrace();
-        } finally {
-            try {
-                if (outputStream != null) {
-                    outputStream.close();
-                }
-                if (inputStream != null) {
-                    inputStream.close();
-                }
-            } catch (IOException e) {
-                e.printStackTrace();
-            }
-            if (con != null) {
-                con.disconnect();
-            }
-        }
-        return null;
-    }
-
-    @Override
-    protected void onProgressUpdate(Integer... progress) {
-        builder.setProgress(fileSize, progress[0], false);
-        nm.notify(notifyId, builder.build());
-    }
-
-    @Override
-    protected void onPostExecute(Void aVoid) {
-        super.onPostExecute(aVoid);
-        nm.cancel(notifyId);
-    }
-
 }

app/src/main/java/org/schabi/newpipe/FileDownloader.java

@@ -0,0 +1,169 @@
+package org.schabi.newpipe;
+
+
+import android.app.NotificationManager;
+import android.content.Context;
+import android.content.SharedPreferences;
+import android.graphics.drawable.BitmapDrawable;
+import android.graphics.drawable.Drawable;
+import android.os.AsyncTask;
+import android.preference.PreferenceManager;
+import android.support.v4.app.NotificationCompat;
+import android.util.Log;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.UnknownHostException;
+
+import javax.net.ssl.HttpsURLConnection;
+
+import info.guardianproject.netcipher.NetCipher;
+
+/**
+ * Created by Christian Schabesberger on 14.08.15.
+ *
+ * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
+ * FileDownloader.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class FileDownloader extends AsyncTask<Void, Integer, Void> {
+    public static final String TAG = "FileDownloader";
+
+
+    private NotificationManager nm;
+    private NotificationCompat.Builder builder;
+    private int notifyId = 0x1234;
+    private int fileSize = 0xffffffff;
+
+    private final Context context;
+    private final String fileURL;
+    private final File saveFilePath;
+    private final String title;
+
+    private final String debugContext;
+
+    public FileDownloader(Context context, String fileURL, File saveFilePath, String title) {
+        this.context = context;
+        this.fileURL = fileURL;
+        this.saveFilePath = saveFilePath;
+        this.title = title;
+
+        this.debugContext = "'" + fileURL +
+                "' => '" + saveFilePath + "'";
+    }
+
+    /**
+     * Downloads a file from a URL in the background using an {@link AsyncTask}.
+     *
+     * @param fileURL      HTTP URL of the file to be downloaded
+     * @param saveFilePath path of the directory to save the file
+     * @param title
+     * @throws IOException
+     */
+    public static void downloadFile(final Context context, final String fileURL, final File saveFilePath, String title) {
+        new FileDownloader(context, fileURL, saveFilePath, title).execute();
+    }
+
+    /** AsyncTask impl: executed in gui thread */
+    @Override
+    protected void onPreExecute() {
+        super.onPreExecute();
+        nm = (NotificationManager) context.getSystemService(Context.NOTIFICATION_SERVICE);
+        Drawable icon = context.getResources().getDrawable(R.mipmap.ic_launcher);
+        builder = new NotificationCompat.Builder(context)
+                .setSmallIcon(android.R.drawable.stat_sys_download)
+                .setLargeIcon(((BitmapDrawable) icon).getBitmap())
+                .setContentTitle(saveFilePath.getName())
+                .setContentText(saveFilePath.getAbsolutePath())
+                .setProgress(fileSize, 0, false);
+        nm.notify(notifyId, builder.build());
+    }
+
+    /** AsyncTask impl: executed in background thread does the download */
+    @Override
+    protected Void doInBackground(Void... voids) {
+        HttpsURLConnection con = null;
+        InputStream inputStream = null;
+        FileOutputStream outputStream = null;
+        try {
+            con = NetCipher.getHttpsURLConnection(fileURL);
+            int responseCode = con.getResponseCode();
+
+            // always check HTTP response code first
+            if (responseCode == HttpURLConnection.HTTP_OK) {
+                fileSize = con.getContentLength();
+                inputStream = new BufferedInputStream(con.getInputStream());
+                outputStream = new FileOutputStream(saveFilePath);
+
+                int bufferSize = 8192;
+                int downloaded = 0;
+
+                int bytesRead = -1;
+                byte[] buffer = new byte[bufferSize];
+                while ((bytesRead = inputStream.read(buffer)) != -1) {
+                    outputStream.write(buffer, 0, bytesRead);
+                    downloaded += bytesRead;
+                    if (downloaded % 50000 < bufferSize) {
+                        publishProgress(downloaded);
+                    }
+                }
+
+                publishProgress(bufferSize);
+
+            } else {
+                Log.i(TAG, "No file to download. Server replied HTTP code: " + responseCode);
+            }
+        } catch (IOException e) {
+            Log.e(TAG, "No file to download. Server replied HTTP code: ", e);
+            e.printStackTrace();
+        } finally {
+            try {
+                if (outputStream != null) {
+                    outputStream.close();
+                }
+                if (inputStream != null) {
+                    inputStream.close();
+                }
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+            if (con != null) {
+                con.disconnect();
+            }
+        }
+        return null;
+    }
+
+    @Override
+    protected void onProgressUpdate(Integer... progress) {
+        builder.setProgress(fileSize, progress[0], false);
+        nm.notify(notifyId, builder.build());
+    }
+
+    @Override
+    protected void onPostExecute(Void aVoid) {
+        super.onPostExecute(aVoid);
+        nm.cancel(notifyId);
+    }
+
+}

app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java

@@ -108,7 +108,7 @@ public class VideoItemDetailFragment extends Fragment {
         @Override
         public void run() {
             try {
-                this.videoExtractor = service.getExtractorInstance(videoUrl);
+                this.videoExtractor = service.getExtractorInstance(videoUrl, new Downloader());
                 VideoInfo videoInfo = videoExtractor.getVideoInfo();
                 h.post(new VideoResultReturnedRunnable(videoInfo));
                 if (videoInfo.errorCode == VideoInfo.NO_ERROR) {

app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java

@@ -108,7 +108,8 @@ public class VideoItemListFragment extends ListFragment {
                 String searchLanguageKey = getContext().getString(R.string.search_language_key);
                 String searchLanguage = sp.getString(searchLanguageKey,
                         getString(R.string.default_language_value));
-                SearchEngine.Result result = engine.search(query, page, searchLanguage);
+                SearchEngine.Result result = engine.search(query, page, searchLanguage,
+                        new Downloader());
 
                 Log.i(TAG, "language code passed:\""+searchLanguage+"\"");
                 if(runs) {

app/src/main/java/org/schabi/newpipe/services/Downloader.java

@@ -0,0 +1,37 @@
+package org.schabi.newpipe.services;
+
+/**
+ * Created by Christian Schabesberger on 28.01.16.
+ *
+ * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
+ * Downloader.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public interface Downloader {
+
+    /**Download the text file at the supplied URL as in download(String),
+     * but set the HTTP header field "Accept-Language" to the supplied string.
+     * @param siteUrl the URL of the text file to return the contents of
+     * @param language the language (usually a 2-character code) to set as the preferred language
+     * @return the contents of the specified text file*/
+    String download(String siteUrl, String language);
+
+    /**Download (via HTTP) the text file located at the supplied URL, and return its contents.
+     * Primarily intended for downloading web pages.
+     * @param siteUrl the URL of the text file to download
+     * @return the contents of the specified text file*/
+    String download(String siteUrl);
+}

app/src/main/java/org/schabi/newpipe/services/SearchEngine.java

@@ -27,16 +27,14 @@ import java.util.Vector;
 
 @SuppressWarnings("ALL")
 public interface SearchEngine {
-
-
     class Result {
         public String errorMessage = "";
         public String suggestion = "";
         public final Vector<VideoPreviewInfo> resultList = new Vector<>();
     }
 
-    ArrayList<String> suggestionList(String query);
+    ArrayList<String> suggestionList(String query, Downloader dl);
 
     //Result search(String query, int page);
-    Result search(String query, int page, String contentCountry);
+    Result search(String query, int page, String contentCountry, Downloader dl);
 }

app/src/main/java/org/schabi/newpipe/services/StreamingService.java

@@ -25,7 +25,7 @@ public interface StreamingService {
         public String name = "";
     }
     ServiceInfo getServiceInfo();
-    VideoExtractor getExtractorInstance(String url);
+    VideoExtractor getExtractorInstance(String url, Downloader downloader);
     SearchEngine getSearchEngineInstance();
 
     /**When a VIEW_ACTION is caught this function will test if the url delivered within the calling

app/src/main/java/org/schabi/newpipe/services/VideoExtractor.java

@@ -28,7 +28,7 @@ public abstract class VideoExtractor {
     protected VideoInfo videoInfo;
 
     @SuppressWarnings("WeakerAccess")
-    public VideoExtractor(String url) {
+    public VideoExtractor(String url, Downloader dl) {
         this.pageUrl = url;
     }
 

app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngine.java

@@ -6,7 +6,7 @@ import android.util.Log;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
-import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.services.Downloader;
 import org.schabi.newpipe.services.SearchEngine;
 import org.schabi.newpipe.VideoPreviewInfo;
 import org.w3c.dom.Node;
@@ -49,7 +49,7 @@ public class YoutubeSearchEngine implements SearchEngine {
     private static final String TAG = YoutubeSearchEngine.class.toString();
 
     @Override
-    public Result search(String query, int page, String languageCode) {
+    public Result search(String query, int page, String languageCode, Downloader downloader) {
         //String contentCountry = PreferenceManager.getDefaultSharedPreferences(this).getString(getString(R.string., "");
         Uri.Builder builder = new Uri.Builder();
         builder.scheme("https")
@@ -64,10 +64,10 @@ public class YoutubeSearchEngine implements SearchEngine {
         //if we've been passed a valid language code, append it to the URL
         if(!languageCode.isEmpty()) {
           //assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode);
-                site  = Downloader.download(url, languageCode);
+                site  = downloader.download(url, languageCode);
         }
         else {
-            site = Downloader.download(url);
+            site = downloader.download(url);
         }
 
 
@@ -140,7 +140,7 @@ public class YoutubeSearchEngine implements SearchEngine {
     }
 
     @Override
-    public ArrayList<String> suggestionList(String query) {
+    public ArrayList<String> suggestionList(String query, Downloader dl) {
 
         ArrayList<String> suggestions = new ArrayList<>();
 
@@ -155,7 +155,7 @@ public class YoutubeSearchEngine implements SearchEngine {
                 .appendQueryParameter("q", query);
         String url = builder.build().toString();
 
-        String response = Downloader.download(url);
+        String response = dl.download(url);
 
         //TODO: Parse xml data using Jsoup not done
         DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();

app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeService.java

@@ -1,5 +1,6 @@
 package org.schabi.newpipe.services.youtube;
 
+import org.schabi.newpipe.services.Downloader;
 import org.schabi.newpipe.services.StreamingService;
 import org.schabi.newpipe.services.VideoExtractor;
 import org.schabi.newpipe.services.SearchEngine;
@@ -33,9 +34,9 @@ public class YoutubeService implements StreamingService {
         return serviceInfo;
     }
     @Override
-    public VideoExtractor getExtractorInstance(String url) {
+    public VideoExtractor getExtractorInstance(String url, Downloader downloader) {
         if(acceptUrl(url)) {
-            return new YoutubeVideoExtractor(url);
+            return new YoutubeVideoExtractor(url, downloader);
         }
         else {
             throw new IllegalArgumentException("supplied String is not a valid Youtube URL");

app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractor.java

@@ -12,7 +12,8 @@ import org.jsoup.parser.Parser;
 import org.mozilla.javascript.Context;
 import org.mozilla.javascript.Function;
 import org.mozilla.javascript.ScriptableObject;
-import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.FileDownloader;
+import org.schabi.newpipe.services.Downloader;
 import org.schabi.newpipe.services.VideoExtractor;
 import org.schabi.newpipe.services.MediaFormat;
 import org.schabi.newpipe.services.VideoInfo;
@@ -62,9 +63,13 @@ public class YoutubeVideoExtractor extends VideoExtractor {
     // cached values
     private static volatile String decryptionCode = "";
 
-    public YoutubeVideoExtractor(String pageUrl) {
+    private Downloader downloader;
-        super(pageUrl);//most common videoInfo fields are now set in our superclass, for all services
+
-        String pageContent = Downloader.download(cleanUrl(pageUrl));
+    public YoutubeVideoExtractor(String pageUrl, Downloader dl) {
+        //most common videoInfo fields are now set in our superclass, for all services
+        super(pageUrl, dl);
+        downloader = dl;
+        String pageContent = downloader.download(cleanUrl(pageUrl));
         doc = Jsoup.parse(pageContent, pageUrl);
 
         //attempt to load the youtube js player JSON arguments
@@ -472,7 +477,7 @@ public class YoutubeVideoExtractor extends VideoExtractor {
             decryptedSig = decryptSignature(encryptedSig, decryptoinCode);
             dashManifest = dashManifest.replace("/s/" + encryptedSig, "/signature/" + decryptedSig);
         }
-        String dashDoc = Downloader.download(dashManifest);
+        String dashDoc = downloader.download(dashManifest);
         Vector<VideoInfo.AudioStream> audioStreams = new Vector<>();
         try {
             XmlPullParser parser = Xml.newPullParser();
@@ -574,7 +579,7 @@ public class YoutubeVideoExtractor extends VideoExtractor {
     }
 
     private String loadDecryptionCode(String playerUrl) {
-        String playerCode = Downloader.download(playerUrl);
+        String playerCode = downloader.download(playerUrl);
         String decryptionFuncName = "";
         String decryptionFunc = "";
         String helperObjectName;