diff --git a/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngineTest.java b/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngineTest.java index c01474416..d81c4b431 100644 --- a/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngineTest.java +++ b/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngineTest.java @@ -2,8 +2,10 @@ package org.schabi.newpipe.services.youtube; import android.test.AndroidTestCase; -import org.schabi.newpipe.VideoPreviewInfo; -import org.schabi.newpipe.services.SearchEngine; +import org.schabi.newpipe.crawler.VideoPreviewInfo; +import org.schabi.newpipe.crawler.SearchEngine; +import org.schabi.newpipe.crawler.services.youtube.YoutubeSearchEngine; +import org.schabi.newpipe.Downloader; import java.util.ArrayList; @@ -35,8 +37,9 @@ public class YoutubeSearchEngineTest extends AndroidTestCase { public void setUp() throws Exception{ super.setUp(); SearchEngine engine = new YoutubeSearchEngine(); - result = engine.search("https://www.youtube.com/results?search_query=bla", 0, "de"); - suggestionReply = engine.suggestionList("hello"); + result = engine.search("https://www.youtube.com/results?search_query=bla", + 0, "de", new Downloader()); + suggestionReply = engine.suggestionList("hello", new Downloader()); } public void testIfNoErrorOccur() { diff --git a/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorDefaultTest.java b/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorDefaultTest.java index 6f6ecaaad..b63e390c2 100644 --- a/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorDefaultTest.java +++ b/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorDefaultTest.java @@ -1,9 +1,14 @@ package org.schabi.newpipe.services.youtube; import android.test.AndroidTestCase; -import android.util.Log; -import org.schabi.newpipe.services.VideoInfo; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.crawler.CrawlingException; +import org.schabi.newpipe.crawler.ParsingException; +import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor; +import org.schabi.newpipe.crawler.VideoInfo; + +import java.io.IOException; /** * Created by the-scrabi on 30.12.15. @@ -28,67 +33,62 @@ import org.schabi.newpipe.services.VideoInfo; public class YoutubeVideoExtractorDefaultTest extends AndroidTestCase { private YoutubeVideoExtractor extractor; - public void setUp() { - extractor = new YoutubeVideoExtractor("https://www.youtube.com/watch?v=FmG385_uUys"); + public void setUp() throws IOException, CrawlingException { + extractor = new YoutubeVideoExtractor("https://www.youtube.com/watch?v=FmG385_uUys", + new Downloader()); } - public void testGetErrorCode() { - assertEquals(extractor.getErrorCode(), VideoInfo.NO_ERROR); - } - - public void testGetErrorMessage() { - assertEquals(extractor.getErrorMessage(), ""); - } - - public void testGetTimeStamp() { + public void testGetInvalidTimeStamp() throws ParsingException { assertTrue(Integer.toString(extractor.getTimeStamp()), - extractor.getTimeStamp() >= 0); + extractor.getTimeStamp() <= 0); } - public void testGetTitle() { + public void testGetValidTimeStamp() throws CrawlingException, IOException { + YoutubeVideoExtractor extractor = + new YoutubeVideoExtractor("https://youtu.be/FmG385_uUys?t=174", new Downloader()); + assertTrue(Integer.toString(extractor.getTimeStamp()), + extractor.getTimeStamp() == 174); + } + + public void testGetTitle() throws ParsingException { assertTrue(!extractor.getTitle().isEmpty()); } - public void testGetDescription() { + public void testGetDescription() throws ParsingException { assertTrue(extractor.getDescription() != null); } - public void testGetUploader() { + public void testGetUploader() throws ParsingException { assertTrue(!extractor.getUploader().isEmpty()); } - public void testGetLength() { + public void testGetLength() throws ParsingException { assertTrue(extractor.getLength() > 0); } - public void testGetViews() { + public void testGetViews() throws ParsingException { assertTrue(extractor.getLength() > 0); } - public void testGetUploadDate() { + public void testGetUploadDate() throws ParsingException { assertTrue(extractor.getUploadDate().length() > 0); } - public void testGetThumbnailUrl() { + public void testGetThumbnailUrl() throws ParsingException { assertTrue(extractor.getThumbnailUrl(), extractor.getThumbnailUrl().contains("https://")); } - public void testGetUploaderThumbnailUrl() { + public void testGetUploaderThumbnailUrl() throws ParsingException { assertTrue(extractor.getUploaderThumbnailUrl(), extractor.getUploaderThumbnailUrl().contains("https://")); } - public void testGetAudioStreams() { - for(VideoInfo.AudioStream s : extractor.getAudioStreams()) { - assertTrue(s.url, - s.url.contains("https://")); - assertTrue(s.bandwidth > 0); - assertTrue(s.samplingRate > 0); - } + public void testGetAudioStreams() throws ParsingException { + assertTrue(extractor.getAudioStreams() == null); } - public void testGetVideoStreams() { + public void testGetVideoStreams() throws ParsingException { for(VideoInfo.VideoStream s : extractor.getVideoStreams()) { assertTrue(s.url, s.url.contains("https://")); diff --git a/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorGemaTest.java b/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorGemaTest.java index 5e5cb40f6..8125ef5e3 100644 --- a/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorGemaTest.java +++ b/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorGemaTest.java @@ -2,7 +2,13 @@ package org.schabi.newpipe.services.youtube; import android.test.AndroidTestCase; -import org.schabi.newpipe.services.VideoInfo; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.crawler.CrawlingException; +import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor; +import org.schabi.newpipe.crawler.VideoInfo; +import org.schabi.newpipe.Downloader; + +import java.io.IOException; /** * Created by the-scrabi on 30.12.15. @@ -31,29 +37,15 @@ public class YoutubeVideoExtractorGemaTest extends AndroidTestCase { // Deaktivate this Test Case bevore uploading it githup, otherwise CI will fail. private static final boolean testActive = false; - - private YoutubeVideoExtractor extractor; - - public void setUp() { + public void testGemaError() throws IOException, CrawlingException { if(testActive) { - extractor = new YoutubeVideoExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8"); - } - } - - public void testGetErrorCode() { - if(testActive) { - assertEquals(extractor.getErrorCode(), VideoInfo.ERROR_BLOCKED_BY_GEMA); - } else { - assertTrue(true); - } - } - - public void testGetErrorMessage() { - if(testActive) { - assertTrue(extractor.getErrorMessage(), - extractor.getErrorMessage().contains("GEMA")); - } else { - assertTrue(true); + try { + new YoutubeVideoExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8", + new Downloader()); + assertTrue("Gema exception not thrown", false); + } catch(YoutubeVideoExtractor.GemaException ge) { + assertTrue(true); + } } } } diff --git a/app/src/main/java/org/schabi/newpipe/ActionBarHandler.java b/app/src/main/java/org/schabi/newpipe/ActionBarHandler.java index 490b8d6f4..a5e7c8cff 100644 --- a/app/src/main/java/org/schabi/newpipe/ActionBarHandler.java +++ b/app/src/main/java/org/schabi/newpipe/ActionBarHandler.java @@ -16,13 +16,15 @@ import android.view.MenuInflater; import android.view.MenuItem; import android.widget.ArrayAdapter; -import org.schabi.newpipe.services.MediaFormat; -import org.schabi.newpipe.services.VideoInfo; +import org.schabi.newpipe.crawler.MediaFormat; +import org.schabi.newpipe.crawler.VideoInfo; + +import java.util.List; /** * Created by Christian Schabesberger on 18.08.15. * - * Copyright (C) Christian Schabesberger 2015 + * Copyright (C) Christian Schabesberger 2016 * DetailsMenuHandler.java is part of NewPipe. * * NewPipe is free software: you can redistribute it and/or modify @@ -49,7 +51,7 @@ class ActionBarHandler { private Bitmap videoThumbnail = null; private String channelName = ""; private AppCompatActivity activity; - private VideoInfo.VideoStream[] videoStreams = null; + private List videoStreams = null; private VideoInfo.AudioStream audioStream = null; private int selectedStream = -1; private String videoTitle = ""; @@ -93,19 +95,21 @@ class ActionBarHandler { } @SuppressWarnings("deprecation") - public void setStreams(VideoInfo.VideoStream[] videoStreams, VideoInfo.AudioStream[] audioStreams) { + public void setStreams(List videoStreams, + List audioStreams) { this.videoStreams = videoStreams; selectedStream = 0; defaultPreferences = PreferenceManager.getDefaultSharedPreferences(activity); - String[] itemArray = new String[videoStreams.length]; + String[] itemArray = new String[videoStreams.size()]; String defaultResolution = defaultPreferences .getString(activity.getString(R.string.default_resolution_key), activity.getString(R.string.default_resolution_value)); int defaultResolutionPos = 0; - for(int i = 0; i < videoStreams.length; i++) { - itemArray[i] = MediaFormat.getNameById(videoStreams[i].format) + " " + videoStreams[i].resolution; - if(defaultResolution.equals(videoStreams[i].resolution)) { + for(int i = 0; i < videoStreams.size(); i++) { + VideoInfo.VideoStream item = videoStreams.get(i); + itemArray[i] = MediaFormat.getNameById(item.format) + " " + item.resolution; + if(defaultResolution.equals(item.resolution)) { defaultResolutionPos = i; } } @@ -209,6 +213,8 @@ class ActionBarHandler { public void playVideo() { // ----------- THE MAGIC MOMENT --------------- if(!videoTitle.isEmpty()) { + VideoInfo.VideoStream selectedStreamItem = videoStreams.get(selectedStream); + if (PreferenceManager.getDefaultSharedPreferences(activity) .getBoolean(activity.getString(R.string.use_external_video_player_key), false)) { @@ -217,8 +223,8 @@ class ActionBarHandler { try { intent.setAction(Intent.ACTION_VIEW); - intent.setDataAndType(Uri.parse(videoStreams[selectedStream].url), - MediaFormat.getMimeById(videoStreams[selectedStream].format)); + intent.setDataAndType(Uri.parse(selectedStreamItem.url), + MediaFormat.getMimeById(selectedStreamItem.format)); intent.putExtra(Intent.EXTRA_TITLE, videoTitle); intent.putExtra("title", videoTitle); @@ -248,7 +254,7 @@ class ActionBarHandler { // Internal Player Intent intent = new Intent(activity, PlayVideoActivity.class); intent.putExtra(PlayVideoActivity.VIDEO_TITLE, videoTitle); - intent.putExtra(PlayVideoActivity.STREAM_URL, videoStreams[selectedStream].url); + intent.putExtra(PlayVideoActivity.STREAM_URL, selectedStreamItem.url); intent.putExtra(PlayVideoActivity.VIDEO_URL, websiteUrl); intent.putExtra(PlayVideoActivity.START_POSITION, startPosition); activity.startActivity(intent); //also HERE !!! @@ -264,13 +270,14 @@ class ActionBarHandler { private void downloadVideo() { if(!videoTitle.isEmpty()) { - String videoSuffix = "." + MediaFormat.getSuffixById(videoStreams[selectedStream].format); + VideoInfo.VideoStream selectedStreamItem = videoStreams.get(selectedStream); + String videoSuffix = "." + MediaFormat.getSuffixById(selectedStreamItem.format); String audioSuffix = "." + MediaFormat.getSuffixById(audioStream.format); Bundle args = new Bundle(); args.putString(DownloadDialog.FILE_SUFFIX_VIDEO, videoSuffix); args.putString(DownloadDialog.FILE_SUFFIX_AUDIO, audioSuffix); args.putString(DownloadDialog.TITLE, videoTitle); - args.putString(DownloadDialog.VIDEO_URL, videoStreams[selectedStream].url); + args.putString(DownloadDialog.VIDEO_URL, selectedStreamItem.url); args.putString(DownloadDialog.AUDIO_URL, audioStream.url); DownloadDialog downloadDialog = new DownloadDialog(); downloadDialog.setArguments(args); diff --git a/app/src/main/java/org/schabi/newpipe/DownloadDialog.java b/app/src/main/java/org/schabi/newpipe/DownloadDialog.java index d7160ccdb..79d24823c 100644 --- a/app/src/main/java/org/schabi/newpipe/DownloadDialog.java +++ b/app/src/main/java/org/schabi/newpipe/DownloadDialog.java @@ -107,7 +107,7 @@ public class DownloadDialog extends DialogFragment { long id = 0; if (App.isUsingTor()) { // if using Tor, do not use DownloadManager because the proxy cannot be set - Downloader.downloadFile(getContext(), url, saveFilePath, title); + FileDownloader.downloadFile(getContext(), url, saveFilePath, title); } else { DownloadManager dm = (DownloadManager) context.getSystemService(Context.DOWNLOAD_SERVICE); DownloadManager.Request request = new DownloadManager.Request( diff --git a/app/src/main/java/org/schabi/newpipe/Downloader.java b/app/src/main/java/org/schabi/newpipe/Downloader.java index 6bd982575..80f1d0dd3 100644 --- a/app/src/main/java/org/schabi/newpipe/Downloader.java +++ b/app/src/main/java/org/schabi/newpipe/Downloader.java @@ -1,24 +1,8 @@ package org.schabi.newpipe; - -import android.app.NotificationManager; -import android.content.Context; -import android.content.SharedPreferences; -import android.graphics.drawable.BitmapDrawable; -import android.graphics.drawable.Drawable; -import android.os.AsyncTask; -import android.preference.PreferenceManager; -import android.support.v4.app.NotificationCompat; -import android.util.Log; - -import java.io.BufferedInputStream; import java.io.BufferedReader; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; import java.io.InputStreamReader; -import java.net.HttpURLConnection; import java.net.URL; import java.net.UnknownHostException; @@ -27,9 +11,9 @@ import javax.net.ssl.HttpsURLConnection; import info.guardianproject.netcipher.NetCipher; /** - * Created by Christian Schabesberger on 14.08.15. + * Created by Christian Schabesberger on 28.01.16. * - * Copyright (C) Christian Schabesberger 2015 + * Copyright (C) Christian Schabesberger 2016 * Downloader.java is part of NewPipe. * * NewPipe is free software: you can redistribute it and/or modify @@ -46,190 +30,61 @@ import info.guardianproject.netcipher.NetCipher; * along with NewPipe. If not, see . */ -public class Downloader extends AsyncTask { - public static final String TAG = "Downloader"; +public class Downloader implements org.schabi.newpipe.crawler.Downloader { + private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0"; - private NotificationManager nm; - private NotificationCompat.Builder builder; - private int notifyId = 0x1234; - private int fileSize = 0xffffffff; - - private final Context context; - private final String fileURL; - private final File saveFilePath; - private final String title; - - private final String debugContext; - - public Downloader(Context context, String fileURL, File saveFilePath, String title) { - this.context = context; - this.fileURL = fileURL; - this.saveFilePath = saveFilePath; - this.title = title; - - this.debugContext = "'" + fileURL + - "' => '" + saveFilePath + "'"; - } - /**Download the text file at the supplied URL as in download(String), * but set the HTTP header field "Accept-Language" to the supplied string. * @param siteUrl the URL of the text file to return the contents of * @param language the language (usually a 2-character code) to set as the preferred language * @return the contents of the specified text file*/ - public static String download(String siteUrl, String language) { - String ret = ""; - try { - URL url = new URL(siteUrl); - //HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); - HttpsURLConnection con = NetCipher.getHttpsURLConnection(url); - con.setRequestProperty("Accept-Language", language); - ret = dl(con); - } - catch(Exception e) { - e.printStackTrace(); - } - return ret; + public String download(String siteUrl, String language) throws IOException { + URL url = new URL(siteUrl); + //HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); + HttpsURLConnection con = NetCipher.getHttpsURLConnection(url); + con.setRequestProperty("Accept-Language", language); + return dl(con); } /**Common functionality between download(String url) and download(String url, String language)*/ private static String dl(HttpsURLConnection con) throws IOException { StringBuilder response = new StringBuilder(); + BufferedReader in = null; try { con.setRequestMethod("GET"); con.setRequestProperty("User-Agent", USER_AGENT); - BufferedReader in = new BufferedReader( + in = new BufferedReader( new InputStreamReader(con.getInputStream())); String inputLine; while((inputLine = in.readLine()) != null) { response.append(inputLine); } - in.close(); - - } - catch(UnknownHostException uhe) {//thrown when there's no internet connection - uhe.printStackTrace(); + } catch(UnknownHostException uhe) {//thrown when there's no internet connection + throw new IOException("unknown host or no network", uhe); //Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show(); + } catch(Exception e) { + throw new IOException(e); + } finally { + if(in != null) { + in.close(); + } } return response.toString(); } -/**Download (via HTTP) the text file located at the supplied URL, and return its contents. - * Primarily intended for downloading web pages. - * @param siteUrl the URL of the text file to download - * @return the contents of the specified text file*/ - public static String download(String siteUrl) { - String ret = ""; - - try { - URL url = new URL(siteUrl); - HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); - //HttpsURLConnection con = NetCipher.getHttpsURLConnection(url); - ret = dl(con); - } - catch(Exception e) { - e.printStackTrace(); - } - - return ret; + /**Download (via HTTP) the text file located at the supplied URL, and return its contents. + * Primarily intended for downloading web pages. + * @param siteUrl the URL of the text file to download + * @return the contents of the specified text file*/ + public String download(String siteUrl) throws IOException { + URL url = new URL(siteUrl); + HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); + //HttpsURLConnection con = NetCipher.getHttpsURLConnection(url); + return dl(con); } - - /** - * Downloads a file from a URL in the background using an {@link AsyncTask}. - * - * @param fileURL HTTP URL of the file to be downloaded - * @param saveFilePath path of the directory to save the file - * @param title - * @throws IOException - */ - public static void downloadFile(final Context context, final String fileURL, final File saveFilePath, String title) { - new Downloader(context, fileURL, saveFilePath, title).execute(); - } - - /** AsyncTask impl: executed in gui thread */ - @Override - protected void onPreExecute() { - super.onPreExecute(); - nm = (NotificationManager) context.getSystemService(Context.NOTIFICATION_SERVICE); - Drawable icon = context.getResources().getDrawable(R.mipmap.ic_launcher); - builder = new NotificationCompat.Builder(context) - .setSmallIcon(android.R.drawable.stat_sys_download) - .setLargeIcon(((BitmapDrawable) icon).getBitmap()) - .setContentTitle(saveFilePath.getName()) - .setContentText(saveFilePath.getAbsolutePath()) - .setProgress(fileSize, 0, false); - nm.notify(notifyId, builder.build()); - } - - /** AsyncTask impl: executed in background thread does the download */ - @Override - protected Void doInBackground(Void... voids) { - HttpsURLConnection con = null; - InputStream inputStream = null; - FileOutputStream outputStream = null; - try { - con = NetCipher.getHttpsURLConnection(fileURL); - int responseCode = con.getResponseCode(); - - // always check HTTP response code first - if (responseCode == HttpURLConnection.HTTP_OK) { - fileSize = con.getContentLength(); - inputStream = new BufferedInputStream(con.getInputStream()); - outputStream = new FileOutputStream(saveFilePath); - - int bufferSize = 8192; - int downloaded = 0; - - int bytesRead = -1; - byte[] buffer = new byte[bufferSize]; - while ((bytesRead = inputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, bytesRead); - downloaded += bytesRead; - if (downloaded % 50000 < bufferSize) { - publishProgress(downloaded); - } - } - - publishProgress(bufferSize); - - } else { - Log.i(TAG, "No file to download. Server replied HTTP code: " + responseCode); - } - } catch (IOException e) { - Log.e(TAG, "No file to download. Server replied HTTP code: ", e); - e.printStackTrace(); - } finally { - try { - if (outputStream != null) { - outputStream.close(); - } - if (inputStream != null) { - inputStream.close(); - } - } catch (IOException e) { - e.printStackTrace(); - } - if (con != null) { - con.disconnect(); - } - } - return null; - } - - @Override - protected void onProgressUpdate(Integer... progress) { - builder.setProgress(fileSize, progress[0], false); - nm.notify(notifyId, builder.build()); - } - - @Override - protected void onPostExecute(Void aVoid) { - super.onPostExecute(aVoid); - nm.cancel(notifyId); - } - } diff --git a/app/src/main/java/org/schabi/newpipe/FileDownloader.java b/app/src/main/java/org/schabi/newpipe/FileDownloader.java new file mode 100644 index 000000000..31ce9ecc8 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/FileDownloader.java @@ -0,0 +1,169 @@ +package org.schabi.newpipe; + + +import android.app.NotificationManager; +import android.content.Context; +import android.content.SharedPreferences; +import android.graphics.drawable.BitmapDrawable; +import android.graphics.drawable.Drawable; +import android.os.AsyncTask; +import android.preference.PreferenceManager; +import android.support.v4.app.NotificationCompat; +import android.util.Log; + +import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.UnknownHostException; + +import javax.net.ssl.HttpsURLConnection; + +import info.guardianproject.netcipher.NetCipher; + +/** + * Created by Christian Schabesberger on 14.08.15. + * + * Copyright (C) Christian Schabesberger 2015 + * FileDownloader.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +public class FileDownloader extends AsyncTask { + public static final String TAG = "FileDownloader"; + + + private NotificationManager nm; + private NotificationCompat.Builder builder; + private int notifyId = 0x1234; + private int fileSize = 0xffffffff; + + private final Context context; + private final String fileURL; + private final File saveFilePath; + private final String title; + + private final String debugContext; + + public FileDownloader(Context context, String fileURL, File saveFilePath, String title) { + this.context = context; + this.fileURL = fileURL; + this.saveFilePath = saveFilePath; + this.title = title; + + this.debugContext = "'" + fileURL + + "' => '" + saveFilePath + "'"; + } + + /** + * Downloads a file from a URL in the background using an {@link AsyncTask}. + * + * @param fileURL HTTP URL of the file to be downloaded + * @param saveFilePath path of the directory to save the file + * @param title + * @throws IOException + */ + public static void downloadFile(final Context context, final String fileURL, final File saveFilePath, String title) { + new FileDownloader(context, fileURL, saveFilePath, title).execute(); + } + + /** AsyncTask impl: executed in gui thread */ + @Override + protected void onPreExecute() { + super.onPreExecute(); + nm = (NotificationManager) context.getSystemService(Context.NOTIFICATION_SERVICE); + Drawable icon = context.getResources().getDrawable(R.mipmap.ic_launcher); + builder = new NotificationCompat.Builder(context) + .setSmallIcon(android.R.drawable.stat_sys_download) + .setLargeIcon(((BitmapDrawable) icon).getBitmap()) + .setContentTitle(saveFilePath.getName()) + .setContentText(saveFilePath.getAbsolutePath()) + .setProgress(fileSize, 0, false); + nm.notify(notifyId, builder.build()); + } + + /** AsyncTask impl: executed in background thread does the download */ + @Override + protected Void doInBackground(Void... voids) { + HttpsURLConnection con = null; + InputStream inputStream = null; + FileOutputStream outputStream = null; + try { + con = NetCipher.getHttpsURLConnection(fileURL); + int responseCode = con.getResponseCode(); + + // always check HTTP response code first + if (responseCode == HttpURLConnection.HTTP_OK) { + fileSize = con.getContentLength(); + inputStream = new BufferedInputStream(con.getInputStream()); + outputStream = new FileOutputStream(saveFilePath); + + int bufferSize = 8192; + int downloaded = 0; + + int bytesRead = -1; + byte[] buffer = new byte[bufferSize]; + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + downloaded += bytesRead; + if (downloaded % 50000 < bufferSize) { + publishProgress(downloaded); + } + } + + publishProgress(bufferSize); + + } else { + Log.i(TAG, "No file to download. Server replied HTTP code: " + responseCode); + } + } catch (IOException e) { + Log.e(TAG, "No file to download. Server replied HTTP code: ", e); + e.printStackTrace(); + } finally { + try { + if (outputStream != null) { + outputStream.close(); + } + if (inputStream != null) { + inputStream.close(); + } + } catch (IOException e) { + e.printStackTrace(); + } + if (con != null) { + con.disconnect(); + } + } + return null; + } + + @Override + protected void onProgressUpdate(Integer... progress) { + builder.setProgress(fileSize, progress[0], false); + nm.notify(notifyId, builder.build()); + } + + @Override + protected void onPostExecute(Void aVoid) { + super.onPostExecute(aVoid); + nm.cancel(notifyId); + } + +} diff --git a/app/src/main/java/org/schabi/newpipe/VideoInfoItemViewCreator.java b/app/src/main/java/org/schabi/newpipe/VideoInfoItemViewCreator.java index c2bbb069e..e254af02d 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoInfoItemViewCreator.java +++ b/app/src/main/java/org/schabi/newpipe/VideoInfoItemViewCreator.java @@ -7,6 +7,8 @@ import android.view.ViewGroup; import android.widget.ImageView; import android.widget.TextView; +import org.schabi.newpipe.crawler.VideoPreviewInfo; + /** * Created by Christian Schabesberger on 24.10.15. * diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemDetailActivity.java b/app/src/main/java/org/schabi/newpipe/VideoItemDetailActivity.java index 6ea3fba66..3be69fbd6 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemDetailActivity.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemDetailActivity.java @@ -11,8 +11,8 @@ import android.view.Menu; import android.view.MenuItem; import android.widget.Toast; -import org.schabi.newpipe.services.ServiceList; -import org.schabi.newpipe.services.StreamingService; +import org.schabi.newpipe.crawler.ServiceList; +import org.schabi.newpipe.crawler.StreamingService; /** @@ -73,7 +73,7 @@ public class VideoItemDetailActivity extends AppCompatActivity { StreamingService[] serviceList = ServiceList.getServices(); //VideoExtractor videoExtractor = null; for (int i = 0; i < serviceList.length; i++) { - if (serviceList[i].acceptUrl(videoUrl)) { + if (serviceList[i].getUrlIdHandler().acceptUrl(videoUrl)) { arguments.putInt(VideoItemDetailFragment.STREAMING_SERVICE, i); currentStreamingService = i; //videoExtractor = ServiceList.getService(i).getExtractorInstance(); diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java index e3a924c64..daa7719c0 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java @@ -16,7 +16,6 @@ import android.support.v4.app.Fragment; import android.support.v7.app.AppCompatActivity; import android.text.Html; import android.text.method.LinkMovementMethod; -import android.util.DisplayMetrics; import android.util.Log; import android.view.LayoutInflater; import android.view.Menu; @@ -32,14 +31,20 @@ import android.widget.TextView; import android.view.MenuItem; import android.widget.Toast; +import java.io.IOException; import java.net.URL; +import java.nio.charset.MalformedInputException; import java.util.ArrayList; import java.util.Vector; -import org.schabi.newpipe.services.VideoExtractor; -import org.schabi.newpipe.services.ServiceList; -import org.schabi.newpipe.services.StreamingService; -import org.schabi.newpipe.services.VideoInfo; +import org.schabi.newpipe.crawler.CrawlingException; +import org.schabi.newpipe.crawler.ParsingException; +import org.schabi.newpipe.crawler.VideoPreviewInfo; +import org.schabi.newpipe.crawler.VideoExtractor; +import org.schabi.newpipe.crawler.ServiceList; +import org.schabi.newpipe.crawler.StreamingService; +import org.schabi.newpipe.crawler.VideoInfo; +import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor; /** @@ -68,7 +73,6 @@ public class VideoItemDetailFragment extends Fragment { * The fragment argument representing the item ID that this fragment * represents. */ - //public static final String ARG_ITEM_ID = "item_id"; public static final String VIDEO_URL = "video_url"; public static final String STREAMING_SERVICE = "streaming_service"; public static final String AUTO_PLAY = "auto_play"; @@ -87,7 +91,6 @@ public class VideoItemDetailFragment extends Fragment { private FloatingActionButton playVideoButton; private final Point initialThumbnailPos = new Point(0, 0); - public interface OnInvokeCreateOptionsMenuListener { void createOptionsMenu(); } @@ -108,45 +111,65 @@ public class VideoItemDetailFragment extends Fragment { @Override public void run() { try { - this.videoExtractor = service.getExtractorInstance(videoUrl); - VideoInfo videoInfo = videoExtractor.getVideoInfo(); + videoExtractor = service.getExtractorInstance(videoUrl, new Downloader()); + VideoInfo videoInfo = VideoInfo.getVideoInfo(videoExtractor, new Downloader()); h.post(new VideoResultReturnedRunnable(videoInfo)); - if (videoInfo.errorCode == VideoInfo.NO_ERROR) { + h.post(new SetThumbnailRunnable( + //todo: make bitmaps not bypass tor + BitmapFactory.decodeStream( + new URL(videoInfo.thumbnail_url) + .openConnection() + .getInputStream()), + SetThumbnailRunnable.VIDEO_THUMBNAIL)); + h.post(new SetThumbnailRunnable( + BitmapFactory.decodeStream( + new URL(videoInfo.uploader_thumbnail_url) + .openConnection() + .getInputStream()), + SetThumbnailRunnable.CHANNEL_THUMBNAIL)); + if (showNextVideoItem) { h.post(new SetThumbnailRunnable( BitmapFactory.decodeStream( - new URL(videoInfo.thumbnail_url) + new URL(videoInfo.nextVideo.thumbnail_url) .openConnection() .getInputStream()), - SetThumbnailRunnable.VIDEO_THUMBNAIL)); - h.post(new SetThumbnailRunnable( - BitmapFactory.decodeStream( - new URL(videoInfo.uploader_thumbnail_url) - .openConnection() - .getInputStream()), - SetThumbnailRunnable.CHANNEL_THUMBNAIL)); - if(showNextVideoItem) { - h.post(new SetThumbnailRunnable( - BitmapFactory.decodeStream( - new URL(videoInfo.nextVideo.thumbnail_url) - .openConnection() - .getInputStream()), - SetThumbnailRunnable.NEXT_VIDEO_THUMBNAIL)); - } + SetThumbnailRunnable.NEXT_VIDEO_THUMBNAIL)); } - } catch (Exception e) { + } catch (MalformedInputException e) { + postNewErrorToast(h, R.string.could_not_load_thumbnails); + e.printStackTrace(); + } catch (IOException e) { + postNewErrorToast(h, R.string.network_error); + e.printStackTrace(); + } + // custom service related exceptions + catch (YoutubeVideoExtractor.DecryptException de) { + postNewErrorToast(h, R.string.youtube_signature_decryption_error); + de.printStackTrace(); + } catch (YoutubeVideoExtractor.GemaException ge) { h.post(new Runnable() { @Override public void run() { - progressBar.setVisibility(View.GONE); - // This is poor style, but unless we have better error handling in the - // crawler, this may not be better. - Toast.makeText(VideoItemDetailFragment.this.getActivity(), - R.string.network_error, Toast.LENGTH_LONG).show(); + onErrorBlockedByGema(); + } + }); + } + // ---------------------------------------- + catch(VideoExtractor.ContentNotAvailableException e) { + h.post(new Runnable() { + @Override + public void run() { + onNotSpecifiedContentError(); } }); e.printStackTrace(); + } catch (ParsingException e) { + postNewErrorToast(h, e.getMessage()); + e.printStackTrace(); + } catch(Exception e) { + postNewErrorToast(h, R.string.general_error); + e.printStackTrace(); } - } } @@ -213,7 +236,7 @@ public class VideoItemDetailFragment extends Fragment { private void updateInfo(VideoInfo info) { currentVideoInfo = info; - Resources res = activity.getResources(); + try { VideoInfoItemViewCreator videoItemViewCreator = new VideoInfoItemViewCreator(LayoutInflater.from(getActivity())); @@ -226,107 +249,77 @@ public class VideoItemDetailFragment extends Fragment { TextView thumbsDownView = (TextView) activity.findViewById(R.id.detailThumbsDownCountView); TextView uploadDateView = (TextView) activity.findViewById(R.id.detailUploadDateView); TextView descriptionView = (TextView) activity.findViewById(R.id.detailDescriptionView); - ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView); FrameLayout nextVideoFrame = (FrameLayout) activity.findViewById(R.id.detailNextVideoFrame); RelativeLayout nextVideoRootFrame = (RelativeLayout) activity.findViewById(R.id.detailNextVideoRootLayout); - Button backgroundButton = (Button) - activity.findViewById(R.id.detailVideoThumbnailWindowBackgroundButton); progressBar.setVisibility(View.GONE); - switch (info.errorCode) { - case VideoInfo.NO_ERROR: { - View nextVideoView = videoItemViewCreator - .getViewFromVideoInfoItem(null, nextVideoFrame, info.nextVideo, getContext()); - nextVideoFrame.addView(nextVideoView); + + View nextVideoView = videoItemViewCreator + .getViewFromVideoInfoItem(null, nextVideoFrame, info.nextVideo, getContext()); + nextVideoFrame.addView(nextVideoView); - Button nextVideoButton = (Button) activity.findViewById(R.id.detailNextVideoButton); - Button similarVideosButton = (Button) activity.findViewById(R.id.detailShowSimilarButton); + Button nextVideoButton = (Button) activity.findViewById(R.id.detailNextVideoButton); + Button similarVideosButton = (Button) activity.findViewById(R.id.detailShowSimilarButton); - textContentLayout.setVisibility(View.VISIBLE); - playVideoButton.setVisibility(View.VISIBLE); - if (!showNextVideoItem) { - nextVideoRootFrame.setVisibility(View.GONE); - similarVideosButton.setVisibility(View.GONE); - } + textContentLayout.setVisibility(View.VISIBLE); + playVideoButton.setVisibility(View.VISIBLE); + if (!showNextVideoItem) { + nextVideoRootFrame.setVisibility(View.GONE); + similarVideosButton.setVisibility(View.GONE); + } - videoTitleView.setText(info.title); - uploaderView.setText(info.uploader); - actionBarHandler.setChannelName(info.uploader); + videoTitleView.setText(info.title); + uploaderView.setText(info.uploader); + actionBarHandler.setChannelName(info.uploader); - String localizedViewCount = Localization.localizeViewCount(info.view_count, getContext()); - viewCountView.setText(localizedViewCount); + String localizedViewCount = Localization.localizeViewCount(info.view_count, getContext()); + viewCountView.setText(localizedViewCount); - String localizedLikeCount = Localization.localizeNumber(info.like_count, getContext()); - thumbsUpView.setText(localizedLikeCount); + String localizedLikeCount = Localization.localizeNumber(info.like_count, getContext()); + thumbsUpView.setText(localizedLikeCount); - String localizedDislikeCount = Localization.localizeNumber(info.dislike_count, getContext()); - thumbsDownView.setText(localizedDislikeCount); + String localizedDislikeCount = Localization.localizeNumber(info.dislike_count, getContext()); + thumbsDownView.setText(localizedDislikeCount); - String localizedDate = Localization.localizeDate(info.upload_date, getContext()); - uploadDateView.setText(localizedDate); + String localizedDate = Localization.localizeDate(info.upload_date, getContext()); + uploadDateView.setText(localizedDate); - descriptionView.setText(Html.fromHtml(info.description)); + descriptionView.setText(Html.fromHtml(info.description)); - descriptionView.setMovementMethod(LinkMovementMethod.getInstance()); + descriptionView.setMovementMethod(LinkMovementMethod.getInstance()); - actionBarHandler.setServiceId(streamingServiceId); - actionBarHandler.setVideoInfo(info.webpage_url, info.title); - actionBarHandler.setStartPosition(info.startPosition); + actionBarHandler.setServiceId(streamingServiceId); + actionBarHandler.setVideoInfo(info.webpage_url, info.title); + actionBarHandler.setStartPosition(info.startPosition); - // parse streams - Vector streamsToUse = new Vector<>(); - for (VideoInfo.VideoStream i : info.videoStreams) { - if (useStream(i, streamsToUse)) { - streamsToUse.add(i); - } - } - VideoInfo.VideoStream[] streamList = new VideoInfo.VideoStream[streamsToUse.size()]; - for (int i = 0; i < streamList.length; i++) { - streamList[i] = streamsToUse.get(i); - } - actionBarHandler.setStreams(streamList, info.audioStreams); + // parse streams + Vector streamsToUse = new Vector<>(); + for (VideoInfo.VideoStream i : info.videoStreams) { + if (useStream(i, streamsToUse)) { + streamsToUse.add(i); + } + } - nextVideoButton.setOnClickListener(new View.OnClickListener() { - @Override - public void onClick(View v) { - Intent detailIntent = - new Intent(getActivity(), VideoItemDetailActivity.class); + actionBarHandler.setStreams(streamsToUse, info.audioStreams); + + nextVideoButton.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View v) { + Intent detailIntent = + new Intent(getActivity(), VideoItemDetailActivity.class); /*detailIntent.putExtra( VideoItemDetailFragment.ARG_ITEM_ID, currentVideoInfo.nextVideo.id); */ - detailIntent.putExtra( - VideoItemDetailFragment.VIDEO_URL, currentVideoInfo.nextVideo.webpage_url); + detailIntent.putExtra( + VideoItemDetailFragment.VIDEO_URL, currentVideoInfo.nextVideo.webpage_url); - detailIntent.putExtra(VideoItemDetailFragment.STREAMING_SERVICE, streamingServiceId); - startActivity(detailIntent); - } - }); + detailIntent.putExtra(VideoItemDetailFragment.STREAMING_SERVICE, streamingServiceId); + startActivity(detailIntent); } - break; - case VideoInfo.ERROR_BLOCKED_BY_GEMA: - thumbnailView.setImageBitmap(BitmapFactory.decodeResource( - getResources(), R.drawable.gruese_die_gema)); - backgroundButton.setOnClickListener(new View.OnClickListener() { - @Override - public void onClick(View v) { - Intent intent = new Intent(); - intent.setAction(Intent.ACTION_VIEW); - intent.setData(Uri.parse(activity.getString(R.string.c3s_url))); - activity.startActivity(intent); - } - }); - break; - case VideoInfo.ERROR_NO_SPECIFIED_ERROR: - thumbnailView.setImageBitmap(BitmapFactory.decodeResource( - getResources(), R.drawable.not_available_monkey)); - Toast.makeText(activity, info.errorMessage, Toast.LENGTH_LONG) - .show(); - break; - default: - Log.e(TAG, "Video Available Status not known."); - } + }); + if(autoPlayEnabled) { actionBarHandler.playVideo(); @@ -337,6 +330,37 @@ public class VideoItemDetailFragment extends Fragment { } } + private void onErrorBlockedByGema() { + Button backgroundButton = (Button) + activity.findViewById(R.id.detailVideoThumbnailWindowBackgroundButton); + ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView); + + progressBar.setVisibility(View.GONE); + thumbnailView.setImageBitmap(BitmapFactory.decodeResource( + getResources(), R.drawable.gruese_die_gema)); + backgroundButton.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View v) { + Intent intent = new Intent(); + intent.setAction(Intent.ACTION_VIEW); + intent.setData(Uri.parse(activity.getString(R.string.c3s_url))); + activity.startActivity(intent); + } + }); + + Toast.makeText(VideoItemDetailFragment.this.getActivity(), + R.string.blocked_by_gema, Toast.LENGTH_LONG).show(); + } + + private void onNotSpecifiedContentError() { + ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView); + progressBar.setVisibility(View.GONE); + thumbnailView.setImageBitmap(BitmapFactory.decodeResource( + getResources(), R.drawable.not_available_monkey)); + Toast.makeText(activity, R.string.content_not_available, Toast.LENGTH_LONG) + .show(); + } + private boolean useStream(VideoInfo.VideoStream stream, Vector streams) { for(VideoInfo.VideoStream i : streams) { if(i.resolution.equals(stream.resolution)) { @@ -465,4 +489,24 @@ public class VideoItemDetailFragment extends Fragment { public void setOnInvokeCreateOptionsMenuListener(OnInvokeCreateOptionsMenuListener listener) { this.onInvokeCreateOptionsMenuListener = listener; } + + private void postNewErrorToast(Handler h, final int stringResource) { + h.post(new Runnable() { + @Override + public void run() { + Toast.makeText(VideoItemDetailFragment.this.getActivity(), + stringResource, Toast.LENGTH_LONG).show(); + } + }); + } + + private void postNewErrorToast(Handler h, final String message) { + h.post(new Runnable() { + @Override + public void run() { + Toast.makeText(VideoItemDetailFragment.this.getActivity(), + message, Toast.LENGTH_LONG).show(); + } + }); + } } \ No newline at end of file diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemListActivity.java b/app/src/main/java/org/schabi/newpipe/VideoItemListActivity.java index 204cab503..b40afa549 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemListActivity.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemListActivity.java @@ -17,7 +17,8 @@ import android.view.inputmethod.InputMethodManager; import java.util.ArrayList; -import org.schabi.newpipe.services.ServiceList; +import org.schabi.newpipe.crawler.VideoPreviewInfo; +import org.schabi.newpipe.crawler.ServiceList; /** * Copyright (C) Christian Schabesberger 2015 diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java index 16953df1d..479a4820d 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java @@ -15,12 +15,15 @@ import android.widget.AbsListView; import android.widget.ListView; import android.widget.Toast; +import java.io.IOException; import java.net.URL; import java.util.List; import java.util.Vector; -import org.schabi.newpipe.services.SearchEngine; -import org.schabi.newpipe.services.StreamingService; +import org.schabi.newpipe.crawler.CrawlingException; +import org.schabi.newpipe.crawler.VideoPreviewInfo; +import org.schabi.newpipe.crawler.SearchEngine; +import org.schabi.newpipe.crawler.StreamingService; /** @@ -108,23 +111,22 @@ public class VideoItemListFragment extends ListFragment { String searchLanguageKey = getContext().getString(R.string.search_language_key); String searchLanguage = sp.getString(searchLanguageKey, getString(R.string.default_language_value)); - SearchEngine.Result result = engine.search(query, page, searchLanguage); + SearchEngine.Result result = engine.search(query, page, searchLanguage, + new Downloader()); Log.i(TAG, "language code passed:\""+searchLanguage+"\""); if(runs) { h.post(new ResultRunnable(result, requestId)); } - } catch(Exception e) { + } catch(IOException e) { + postNewErrorToast(h, R.string.network_error); + e.printStackTrace(); + } catch(CrawlingException ce) { + postNewErrorToast(h, R.string.parsing_error); + ce.printStackTrace(); + } catch(Exception e) { + postNewErrorToast(h, R.string.general_error); e.printStackTrace(); - - h.post(new Runnable() { - @Override - public void run() { - setListShown(true); - Toast.makeText(getActivity(), getString(R.string.network_error), - Toast.LENGTH_SHORT).show(); - } - }); } } } @@ -155,6 +157,7 @@ public class VideoItemListFragment extends ListFragment { if(!downloadedList.get(i)) { Bitmap thumbnail; try { + //todo: make bitmaps not bypass tor thumbnail = BitmapFactory.decodeStream( new URL(thumbnailUrlList.get(i)).openConnection().getInputStream()); h.post(new SetThumbnailRunnable(i, thumbnail, requestId)); @@ -384,4 +387,14 @@ public class VideoItemListFragment extends ListFragment { mActivatedPosition = position; } + private void postNewErrorToast(Handler h, final int stringResource) { + h.post(new Runnable() { + @Override + public void run() { + setListShown(true); + Toast.makeText(getActivity(), getString(R.string.network_error), + Toast.LENGTH_SHORT).show(); + } + }); + } } diff --git a/app/src/main/java/org/schabi/newpipe/VideoListAdapter.java b/app/src/main/java/org/schabi/newpipe/VideoListAdapter.java index de08b2f03..54ff763f0 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoListAdapter.java +++ b/app/src/main/java/org/schabi/newpipe/VideoListAdapter.java @@ -9,6 +9,8 @@ import android.view.ViewGroup; import android.widget.BaseAdapter; import android.widget.ListView; +import org.schabi.newpipe.crawler.VideoPreviewInfo; + import java.util.List; import java.util.Vector; diff --git a/app/src/main/java/org/schabi/newpipe/crawler/AbstractVideoInfo.java b/app/src/main/java/org/schabi/newpipe/crawler/AbstractVideoInfo.java new file mode 100644 index 000000000..7a15a8af2 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/crawler/AbstractVideoInfo.java @@ -0,0 +1,33 @@ +package org.schabi.newpipe.crawler; + +import android.graphics.Bitmap; + +/** + * Copyright (C) Christian Schabesberger 2015 + * AbstractVideoInfo.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +/**Common properties between VideoInfo and VideoPreviewInfo.*/ +public abstract class AbstractVideoInfo { + public String id = ""; + public String title = ""; + public String uploader = ""; + public String thumbnail_url = ""; + public Bitmap thumbnail = null; + public String webpage_url = ""; + public String upload_date = ""; + public long view_count = -1; +} diff --git a/app/src/main/java/org/schabi/newpipe/crawler/CrawlingException.java b/app/src/main/java/org/schabi/newpipe/crawler/CrawlingException.java new file mode 100644 index 000000000..291670953 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/crawler/CrawlingException.java @@ -0,0 +1,37 @@ +package org.schabi.newpipe.crawler; + +/** + * Created by Christian Schabesberger on 30.01.16. + * + * Copyright (C) Christian Schabesberger 2016 + * CrawlingException.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +public class CrawlingException extends Exception { + public CrawlingException() {} + + public CrawlingException(String message) { + super(message); + } + + public CrawlingException(Throwable cause) { + super(cause); + } + + public CrawlingException(String message, Throwable cause) { + super(message, cause); + } +} \ No newline at end of file diff --git a/app/src/main/java/org/schabi/newpipe/crawler/DashMpdParser.java b/app/src/main/java/org/schabi/newpipe/crawler/DashMpdParser.java new file mode 100644 index 000000000..027cc66a5 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/crawler/DashMpdParser.java @@ -0,0 +1,102 @@ +package org.schabi.newpipe.crawler; + +import android.util.Xml; + +import org.xmlpull.v1.XmlPullParser; + +import java.io.IOException; +import java.io.StringReader; +import java.util.List; +import java.util.Vector; + +/** + * Created by Christian Schabesberger on 02.02.16. + * + * Copyright (C) Christian Schabesberger 2016 + * DashMpdParser.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +public class DashMpdParser { + + static class DashMpdParsingException extends ParsingException { + DashMpdParsingException(String message, Exception e) { + super(message, e); + } + } + + public static List getAudioStreams(String dashManifestUrl, + Downloader downloader) + throws DashMpdParsingException { + String dashDoc; + try { + dashDoc = downloader.download(dashManifestUrl); + } catch(IOException ioe) { + throw new DashMpdParsingException("Could not get dash mpd: " + dashManifestUrl, ioe); + } + Vector audioStreams = new Vector<>(); + try { + XmlPullParser parser = Xml.newPullParser(); + parser.setInput(new StringReader(dashDoc)); + String tagName = ""; + String currentMimeType = ""; + int currentBandwidth = -1; + int currentSamplingRate = -1; + boolean currentTagIsBaseUrl = false; + for(int eventType = parser.getEventType(); + eventType != XmlPullParser.END_DOCUMENT; + eventType = parser.next() ) { + switch(eventType) { + case XmlPullParser.START_TAG: + tagName = parser.getName(); + if(tagName.equals("AdaptationSet")) { + currentMimeType = parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "mimeType"); + } else if(tagName.equals("Representation") && currentMimeType.contains("audio")) { + currentBandwidth = Integer.parseInt( + parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "bandwidth")); + currentSamplingRate = Integer.parseInt( + parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "audioSamplingRate")); + } else if(tagName.equals("BaseURL")) { + currentTagIsBaseUrl = true; + } + break; + + case XmlPullParser.TEXT: + if(currentTagIsBaseUrl && + (currentMimeType.contains("audio"))) { + int format = -1; + if(currentMimeType.equals(MediaFormat.WEBMA.mimeType)) { + format = MediaFormat.WEBMA.id; + } else if(currentMimeType.equals(MediaFormat.M4A.mimeType)) { + format = MediaFormat.M4A.id; + } + audioStreams.add(new VideoInfo.AudioStream(parser.getText(), + format, currentBandwidth, currentSamplingRate)); + } + break; + case XmlPullParser.END_TAG: + if(tagName.equals("AdaptationSet")) { + currentMimeType = ""; + } else if(tagName.equals("BaseURL")) { + currentTagIsBaseUrl = false; + }//no break needed here + } + } + } catch(Exception e) { + throw new DashMpdParsingException("Could not parse Dash mpd", e); + } + return audioStreams; + } +} diff --git a/app/src/main/java/org/schabi/newpipe/crawler/Downloader.java b/app/src/main/java/org/schabi/newpipe/crawler/Downloader.java new file mode 100644 index 000000000..8732c0372 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/crawler/Downloader.java @@ -0,0 +1,41 @@ +package org.schabi.newpipe.crawler; + +import java.io.IOException; + +/** + * Created by Christian Schabesberger on 28.01.16. + * + * Copyright (C) Christian Schabesberger 2016 + * Downloader.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +public interface Downloader { + + /**Download the text file at the supplied URL as in download(String), + * but set the HTTP header field "Accept-Language" to the supplied string. + * @param siteUrl the URL of the text file to return the contents of + * @param language the language (usually a 2-character code) to set as the preferred language + * @return the contents of the specified text file + * @throws IOException*/ + String download(String siteUrl, String language) throws IOException; + + /**Download (via HTTP) the text file located at the supplied URL, and return its contents. + * Primarily intended for downloading web pages. + * @param siteUrl the URL of the text file to download + * @return the contents of the specified text file + * @throws IOException*/ + String download(String siteUrl) throws IOException; +} diff --git a/app/src/main/java/org/schabi/newpipe/services/MediaFormat.java b/app/src/main/java/org/schabi/newpipe/crawler/MediaFormat.java similarity index 97% rename from app/src/main/java/org/schabi/newpipe/services/MediaFormat.java rename to app/src/main/java/org/schabi/newpipe/crawler/MediaFormat.java index 867dc3035..63b94fd47 100644 --- a/app/src/main/java/org/schabi/newpipe/services/MediaFormat.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/MediaFormat.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.services; +package org.schabi.newpipe.crawler; /** * Created by Adam Howard on 08/11/15. @@ -6,7 +6,7 @@ package org.schabi.newpipe.services; * Copyright (c) Christian Schabesberger * and Adam Howard 2015 * - * VideoListAdapter.java is part of NewPipe. + * MediaFormat.java is part of NewPipe. * * NewPipe is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/app/src/main/java/org/schabi/newpipe/crawler/ParsingException.java b/app/src/main/java/org/schabi/newpipe/crawler/ParsingException.java new file mode 100644 index 000000000..25d46b119 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/crawler/ParsingException.java @@ -0,0 +1,35 @@ +package org.schabi.newpipe.crawler; + +/** + * Created by Christian Schabesberger on 31.01.16. + * + * Copyright (C) Christian Schabesberger 2016 + * ParsingException.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + + +public class ParsingException extends CrawlingException { + public ParsingException() {} + public ParsingException(String message) { + super(message); + } + public ParsingException(Throwable cause) { + super(cause); + } + public ParsingException(String message, Throwable cause) { + super(message, cause); + } +} \ No newline at end of file diff --git a/app/src/main/java/org/schabi/newpipe/crawler/RegexHelper.java b/app/src/main/java/org/schabi/newpipe/crawler/RegexHelper.java new file mode 100644 index 000000000..a82386182 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/crawler/RegexHelper.java @@ -0,0 +1,47 @@ +package org.schabi.newpipe.crawler; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Created by Christian Schabesberger on 02.02.16. + * + * Copyright (C) Christian Schabesberger 2016 + * RegexHelper.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +/** avoid using regex !!! */ +public class RegexHelper { + + public static class RegexException extends ParsingException { + public RegexException(String message) { + super(message); + } + } + + public static String matchGroup1(String pattern, String input) throws RegexException { + Pattern pat = Pattern.compile(pattern); + Matcher mat = pat.matcher(input); + boolean foundMatch = mat.find(); + if (foundMatch) { + return mat.group(1); + } + else { + //Log.e(TAG, "failed to find pattern \""+pattern+"\" inside of \""+input+"\""); + throw new RegexException("failed to find pattern \""+pattern+" inside of "+input+"\""); + } + } +} diff --git a/app/src/main/java/org/schabi/newpipe/services/SearchEngine.java b/app/src/main/java/org/schabi/newpipe/crawler/SearchEngine.java similarity index 72% rename from app/src/main/java/org/schabi/newpipe/services/SearchEngine.java rename to app/src/main/java/org/schabi/newpipe/crawler/SearchEngine.java index 98aa42ae5..845c78926 100644 --- a/app/src/main/java/org/schabi/newpipe/services/SearchEngine.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/SearchEngine.java @@ -1,8 +1,8 @@ -package org.schabi.newpipe.services; - -import org.schabi.newpipe.VideoPreviewInfo; +package org.schabi.newpipe.crawler; +import java.io.IOException; import java.util.ArrayList; +import java.util.List; import java.util.Vector; /** @@ -27,16 +27,16 @@ import java.util.Vector; @SuppressWarnings("ALL") public interface SearchEngine { - - class Result { public String errorMessage = ""; public String suggestion = ""; - public final Vector resultList = new Vector<>(); + public final List resultList = new Vector<>(); } - ArrayList suggestionList(String query); + ArrayList suggestionList(String query, Downloader dl) + throws CrawlingException, IOException; //Result search(String query, int page); - Result search(String query, int page, String contentCountry); + Result search(String query, int page, String contentCountry, Downloader dl) + throws CrawlingException, IOException; } diff --git a/app/src/main/java/org/schabi/newpipe/services/ServiceList.java b/app/src/main/java/org/schabi/newpipe/crawler/ServiceList.java similarity index 94% rename from app/src/main/java/org/schabi/newpipe/services/ServiceList.java rename to app/src/main/java/org/schabi/newpipe/crawler/ServiceList.java index da1b293fb..b1d98a73f 100644 --- a/app/src/main/java/org/schabi/newpipe/services/ServiceList.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/ServiceList.java @@ -1,8 +1,8 @@ -package org.schabi.newpipe.services; +package org.schabi.newpipe.crawler; import android.util.Log; -import org.schabi.newpipe.services.youtube.YoutubeService; +import org.schabi.newpipe.crawler.services.youtube.YoutubeService; /** * Created by Christian Schabesberger on 23.08.15. diff --git a/app/src/main/java/org/schabi/newpipe/services/StreamingService.java b/app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java similarity index 73% rename from app/src/main/java/org/schabi/newpipe/services/StreamingService.java rename to app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java index acf887b57..9b6f4e285 100644 --- a/app/src/main/java/org/schabi/newpipe/services/StreamingService.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java @@ -1,4 +1,6 @@ -package org.schabi.newpipe.services; +package org.schabi.newpipe.crawler; + +import java.io.IOException; /** * Created by Christian Schabesberger on 23.08.15. @@ -25,11 +27,11 @@ public interface StreamingService { public String name = ""; } ServiceInfo getServiceInfo(); - VideoExtractor getExtractorInstance(String url); + VideoExtractor getExtractorInstance(String url, Downloader downloader) + throws IOException, CrawlingException; SearchEngine getSearchEngineInstance(); - /**When a VIEW_ACTION is caught this function will test if the url delivered within the calling - Intent was meant to be watched with this Service. - Return false if this service shall not allow to be called through ACTIONs.*/ - boolean acceptUrl(String videoUrl); + VideoUrlIdHandler getUrlIdHandler(); + + } diff --git a/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java b/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java new file mode 100644 index 000000000..44b4e743d --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java @@ -0,0 +1,77 @@ +package org.schabi.newpipe.crawler; + +/** + * Created by Christian Schabesberger on 10.08.15. + * + * Copyright (C) Christian Schabesberger 2015 + * VideoExtractor.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +import java.util.List; + +/**Scrapes information from a video streaming service (eg, YouTube).*/ + + +@SuppressWarnings("ALL") +public interface VideoExtractor { + + public class ExctractorInitException extends CrawlingException { + public ExctractorInitException() {} + public ExctractorInitException(String message) { + super(message); + } + public ExctractorInitException(Throwable cause) { + super(cause); + } + public ExctractorInitException(String message, Throwable cause) { + super(message, cause); + } + } + + public class ContentNotAvailableException extends ParsingException { + public ContentNotAvailableException() {} + public ContentNotAvailableException(String message) { + super(message); + } + public ContentNotAvailableException(Throwable cause) { + super(cause); + } + public ContentNotAvailableException(String message, Throwable cause) { + super(message, cause); + } + } + + public abstract int getTimeStamp() throws ParsingException; + public abstract String getTitle() throws ParsingException; + public abstract String getDescription() throws ParsingException; + public abstract String getUploader() throws ParsingException; + public abstract int getLength() throws ParsingException; + public abstract long getViews() throws ParsingException; + public abstract String getUploadDate() throws ParsingException; + public abstract String getThumbnailUrl() throws ParsingException; + public abstract String getUploaderThumbnailUrl() throws ParsingException; + public abstract List getAudioStreams() throws ParsingException; + public abstract List getVideoStreams() throws ParsingException; + public abstract String getDashMpdUrl() throws ParsingException; + public abstract int getAgeLimit() throws ParsingException; + public abstract String getAverageRating() throws ParsingException; + public abstract int getLikeCount() throws ParsingException; + public abstract int getDislikeCount() throws ParsingException; + public abstract VideoPreviewInfo getNextVideo() throws ParsingException; + public abstract List getRelatedVideos() throws ParsingException; + public abstract VideoUrlIdHandler getUrlIdConverter(); + public abstract String getPageUrl(); +} diff --git a/app/src/main/java/org/schabi/newpipe/services/VideoInfo.java b/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java similarity index 50% rename from app/src/main/java/org/schabi/newpipe/services/VideoInfo.java rename to app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java index dcf871d8c..2542c501d 100644 --- a/app/src/main/java/org/schabi/newpipe/services/VideoInfo.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java @@ -1,9 +1,8 @@ -package org.schabi.newpipe.services; - -import org.schabi.newpipe.VideoPreviewInfo; -import org.schabi.newpipe.services.AbstractVideoInfo; +package org.schabi.newpipe.crawler; +import java.io.IOException; import java.util.List; +import java.util.Vector; /** * Created by Christian Schabesberger on 26.08.15. @@ -29,20 +28,60 @@ import java.util.List; @SuppressWarnings("ALL") public class VideoInfo extends AbstractVideoInfo { - // If a video could not be parsed, this predefined error codes - // will be returned AND can be parsed by the frontend of the app. - // Error codes: - public final static int NO_ERROR = 0x0; - public final static int ERROR_NO_SPECIFIED_ERROR = 0x1; - // GEMA a german music colecting society. - public final static int ERROR_BLOCKED_BY_GEMA = 0x2; + /**Fills out the video info fields which are common to all services. + * Probably needs to be overridden by subclasses*/ + public static VideoInfo getVideoInfo(VideoExtractor extractor, Downloader downloader) + throws CrawlingException, IOException { + VideoInfo videoInfo = new VideoInfo(); + + VideoUrlIdHandler uiconv = extractor.getUrlIdConverter(); + + videoInfo.webpage_url = extractor.getPageUrl(); + videoInfo.title = extractor.getTitle(); + videoInfo.duration = extractor.getLength(); + videoInfo.uploader = extractor.getUploader(); + videoInfo.description = extractor.getDescription(); + videoInfo.view_count = extractor.getViews(); + videoInfo.upload_date = extractor.getUploadDate(); + videoInfo.thumbnail_url = extractor.getThumbnailUrl(); + videoInfo.id = uiconv.getVideoId(extractor.getPageUrl()); + videoInfo.dashMpdUrl = extractor.getDashMpdUrl(); + /** Load and extract audio*/ + videoInfo.audioStreams = extractor.getAudioStreams(); + if(videoInfo.dashMpdUrl != null && !videoInfo.dashMpdUrl.isEmpty()) { + if(videoInfo.audioStreams == null) { + videoInfo.audioStreams = new Vector(); + } + videoInfo.audioStreams.addAll( + DashMpdParser.getAudioStreams(videoInfo.dashMpdUrl, downloader)); + } + /** Extract video stream url*/ + videoInfo.videoStreams = extractor.getVideoStreams(); + videoInfo.uploader_thumbnail_url = extractor.getUploaderThumbnailUrl(); + videoInfo.startPosition = extractor.getTimeStamp(); + videoInfo.average_rating = extractor.getAverageRating(); + videoInfo.like_count = extractor.getLikeCount(); + videoInfo.dislike_count = extractor.getDislikeCount(); + videoInfo.nextVideo = extractor.getNextVideo(); + videoInfo.relatedVideos = extractor.getRelatedVideos(); + + //Bitmap thumbnail = null; + //Bitmap uploader_thumbnail = null; + //int videoAvailableStatus = VIDEO_AVAILABLE; + return videoInfo; + } + public String uploader_thumbnail_url = ""; public String description = ""; - public VideoStream[] videoStreams = null; - public AudioStream[] audioStreams = null; - public int errorCode = NO_ERROR; - public String errorMessage = ""; + /*todo: make this lists over vectors*/ + public List videoStreams = null; + public List audioStreams = null; + // video streams provided by the dash mpd do not need to be provided as VideoStream. + // Later on this will also aplly to audio streams. Since dash mpd is standarized, + // crawling such a file is not service dependent. Therefore getting audio only streams by yust + // providing the dash mpd fille will be possible in the future. + public String dashMpdUrl = ""; public int duration = -1; /*YouTube-specific fields @@ -53,11 +92,11 @@ public class VideoInfo extends AbstractVideoInfo { public String average_rating = ""; public VideoPreviewInfo nextVideo = null; public List relatedVideos = null; - public int startPosition = -1;//in seconds. some metadata is not passed using a VideoInfo object! + //in seconds. some metadata is not passed using a VideoInfo object! + public int startPosition = -1; public VideoInfo() {} - /**Creates a new VideoInfo object from an existing AbstractVideoInfo. * All the shared properties are copied to the new VideoInfo.*/ @SuppressWarnings("WeakerAccess") @@ -73,7 +112,8 @@ public class VideoInfo extends AbstractVideoInfo { this.view_count = avi.view_count; //todo: better than this - if(avi instanceof VideoPreviewInfo) {//shitty String to convert code + if(avi instanceof VideoPreviewInfo) { + //shitty String to convert code String dur = ((VideoPreviewInfo)avi).duration; int minutes = Integer.parseInt(dur.substring(0, dur.indexOf(":"))); int seconds = Integer.parseInt(dur.substring(dur.indexOf(":")+1, dur.length())); @@ -82,7 +122,8 @@ public class VideoInfo extends AbstractVideoInfo { } public static class VideoStream { - public String url = ""; //url of the stream + //url of the stream + public String url = ""; public int format = -1; public String resolution = ""; diff --git a/app/src/main/java/org/schabi/newpipe/VideoPreviewInfo.java b/app/src/main/java/org/schabi/newpipe/crawler/VideoPreviewInfo.java similarity index 96% rename from app/src/main/java/org/schabi/newpipe/VideoPreviewInfo.java rename to app/src/main/java/org/schabi/newpipe/crawler/VideoPreviewInfo.java index 0832114e0..bca13a208 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoPreviewInfo.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/VideoPreviewInfo.java @@ -1,10 +1,10 @@ -package org.schabi.newpipe; +package org.schabi.newpipe.crawler; import android.graphics.Bitmap; import android.os.Parcel; import android.os.Parcelable; -import org.schabi.newpipe.services.AbstractVideoInfo; +import org.schabi.newpipe.crawler.AbstractVideoInfo; /** * Created by Christian Schabesberger on 26.08.15. diff --git a/app/src/main/java/org/schabi/newpipe/crawler/VideoUrlIdHandler.java b/app/src/main/java/org/schabi/newpipe/crawler/VideoUrlIdHandler.java new file mode 100644 index 000000000..66d8c3cd8 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/crawler/VideoUrlIdHandler.java @@ -0,0 +1,32 @@ +package org.schabi.newpipe.crawler; + +/** + * Created by Christian Schabesberger on 02.02.16. + * + * Copyright (C) Christian Schabesberger 2016 + * VideoUrlIdHandler.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +public interface VideoUrlIdHandler { + String getVideoUrl(String videoId); + String getVideoId(String siteUrl) throws ParsingException; + String cleanUrl(String siteUrl) throws ParsingException; + + /**When a VIEW_ACTION is caught this function will test if the url delivered within the calling + Intent was meant to be watched with this Service. + Return false if this service shall not allow to be called through ACTIONs.*/ + boolean acceptUrl(String videoUrl); +} diff --git a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeSearchEngine.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeSearchEngine.java new file mode 100644 index 000000000..a6d4857c1 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeSearchEngine.java @@ -0,0 +1,202 @@ +package org.schabi.newpipe.crawler.services.youtube; + +import android.net.Uri; +import android.util.Log; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.schabi.newpipe.crawler.CrawlingException; +import org.schabi.newpipe.crawler.Downloader; +import org.schabi.newpipe.crawler.ParsingException; +import org.schabi.newpipe.crawler.SearchEngine; +import org.schabi.newpipe.crawler.VideoExtractor; +import org.schabi.newpipe.crawler.VideoPreviewInfo; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +/** + * Created by Christian Schabesberger on 09.08.15. + * + * Copyright (C) Christian Schabesberger 2015 + * YoutubeSearchEngine.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +public class YoutubeSearchEngine implements SearchEngine { + + private static final String TAG = YoutubeSearchEngine.class.toString(); + + @Override + public Result search(String query, int page, String languageCode, Downloader downloader) + throws IOException, ParsingException { + Result result = new Result(); + Uri.Builder builder = new Uri.Builder(); + builder.scheme("https") + .authority("www.youtube.com") + .appendPath("results") + .appendQueryParameter("search_query", query) + .appendQueryParameter("page", Integer.toString(page)) + .appendQueryParameter("filters", "video"); + + String site; + String url = builder.build().toString(); + //if we've been passed a valid language code, append it to the URL + if(!languageCode.isEmpty()) { + //assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode); + site = downloader.download(url, languageCode); + } + else { + site = downloader.download(url); + } + + try { + + Document doc = Jsoup.parse(site, url); + Element list = doc.select("ol[class=\"item-section\"]").first(); + + for (Element item : list.children()) { + /* First we need to determine which kind of item we are working with. + Youtube depicts five different kinds of items on its search result page. These are + regular videos, playlists, channels, two types of video suggestions, and a "no video + found" item. Since we only want videos, we need to filter out all the others. + An example for this can be seen here: + https://www.youtube.com/results?search_query=asdf&page=1 + + We already applied a filter to the url, so we don't need to care about channels and + playlists now. + */ + + Element el; + + // both types of spell correction item + if (!((el = item.select("div[class*=\"spell-correction\"]").first()) == null)) { + result.suggestion = el.select("a").first().text(); + // search message item + } else if (!((el = item.select("div[class*=\"search-message\"]").first()) == null)) { + result.errorMessage = el.text(); + + // video item type + } else if (!((el = item.select("div[class*=\"yt-lockup-video\"").first()) == null)) { + VideoPreviewInfo resultItem = new VideoPreviewInfo(); + Element dl = el.select("h3").first().select("a").first(); + resultItem.webpage_url = dl.attr("abs:href"); + try { + Pattern p = Pattern.compile("v=([0-9a-zA-Z-]*)"); + Matcher m = p.matcher(resultItem.webpage_url); + resultItem.id = m.group(1); + } catch (Exception e) { + //e.printStackTrace(); + } + resultItem.title = dl.text(); + + resultItem.duration = item.select("span[class=\"video-time\"]").first().text(); + + resultItem.uploader = item.select("div[class=\"yt-lockup-byline\"]").first() + .select("a").first() + .text(); + resultItem.upload_date = item.select("div[class=\"yt-lockup-meta\"]").first() + .select("li").first() + .text(); + Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first() + .select("img").first(); + resultItem.thumbnail_url = te.attr("abs:src"); + // Sometimes youtube sends links to gif files which somehow seem to not exist + // anymore. Items with such gif also offer a secondary image source. So we are going + // to use that if we've caught such an item. + if (resultItem.thumbnail_url.contains(".gif")) { + resultItem.thumbnail_url = te.attr("abs:data-thumb"); + } + result.resultList.add(resultItem); + } else { + //noinspection ConstantConditions + Log.e(TAG, "unexpected element found:\"" + el + "\""); + } + } + } catch(Exception e) { + throw new ParsingException(e); + } + return result; + } + + @Override + public ArrayList suggestionList(String query, Downloader dl) + throws IOException, ParsingException { + + ArrayList suggestions = new ArrayList<>(); + + Uri.Builder builder = new Uri.Builder(); + builder.scheme("https") + .authority("suggestqueries.google.com") + .appendPath("complete") + .appendPath("search") + .appendQueryParameter("client", "") + .appendQueryParameter("output", "toolbar") + .appendQueryParameter("ds", "yt") + .appendQueryParameter("q", query); + String url = builder.build().toString(); + + + String response = dl.download(url); + + try { + + //TODO: Parse xml data using Jsoup not done + DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder dBuilder; + org.w3c.dom.Document doc = null; + + try { + dBuilder = dbFactory.newDocumentBuilder(); + doc = dBuilder.parse(new InputSource( + new ByteArrayInputStream(response.getBytes("utf-8")))); + doc.getDocumentElement().normalize(); + } catch (ParserConfigurationException | SAXException | IOException e) { + e.printStackTrace(); + } + + if (doc != null) { + NodeList nList = doc.getElementsByTagName("CompleteSuggestion"); + for (int temp = 0; temp < nList.getLength(); temp++) { + + NodeList nList1 = doc.getElementsByTagName("suggestion"); + Node nNode1 = nList1.item(temp); + if (nNode1.getNodeType() == Node.ELEMENT_NODE) { + org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1; + suggestions.add(eElement.getAttribute("data")); + } + } + } else { + Log.e(TAG, "GREAT FUCKING ERROR"); + } + return suggestions; + } catch(Exception e) { + throw new ParsingException(e); + } + } + +} diff --git a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeService.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java similarity index 62% rename from app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeService.java rename to app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java index 576d8c065..b49c55b87 100644 --- a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeService.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java @@ -1,8 +1,13 @@ -package org.schabi.newpipe.services.youtube; +package org.schabi.newpipe.crawler.services.youtube; -import org.schabi.newpipe.services.StreamingService; -import org.schabi.newpipe.services.VideoExtractor; -import org.schabi.newpipe.services.SearchEngine; +import org.schabi.newpipe.crawler.CrawlingException; +import org.schabi.newpipe.crawler.Downloader; +import org.schabi.newpipe.crawler.StreamingService; +import org.schabi.newpipe.crawler.VideoUrlIdHandler; +import org.schabi.newpipe.crawler.VideoExtractor; +import org.schabi.newpipe.crawler.SearchEngine; + +import java.io.IOException; /** @@ -33,9 +38,11 @@ public class YoutubeService implements StreamingService { return serviceInfo; } @Override - public VideoExtractor getExtractorInstance(String url) { - if(acceptUrl(url)) { - return new YoutubeVideoExtractor(url); + public VideoExtractor getExtractorInstance(String url, Downloader downloader) + throws CrawlingException, IOException { + VideoUrlIdHandler urlIdHandler = new YoutubeVideoUrlIdHandler(); + if(urlIdHandler.acceptUrl(url)) { + return new YoutubeVideoExtractor(url, downloader) ; } else { throw new IllegalArgumentException("supplied String is not a valid Youtube URL"); @@ -45,9 +52,9 @@ public class YoutubeService implements StreamingService { public SearchEngine getSearchEngineInstance() { return new YoutubeSearchEngine(); } + @Override - public boolean acceptUrl(String videoUrl) { - return videoUrl.contains("youtube") || - videoUrl.contains("youtu.be"); + public VideoUrlIdHandler getUrlIdHandler() { + return new YoutubeVideoUrlIdHandler(); } } diff --git a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java new file mode 100644 index 000000000..07d2b23a7 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java @@ -0,0 +1,600 @@ +package org.schabi.newpipe.crawler.services.youtube; + +import android.util.Log; + +import org.json.JSONException; +import org.json.JSONObject; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.parser.Parser; +import org.mozilla.javascript.Context; +import org.mozilla.javascript.Function; +import org.mozilla.javascript.ScriptableObject; +import org.schabi.newpipe.crawler.CrawlingException; +import org.schabi.newpipe.crawler.Downloader; +import org.schabi.newpipe.crawler.ParsingException; +import org.schabi.newpipe.crawler.RegexHelper; +import org.schabi.newpipe.crawler.VideoUrlIdHandler; +import org.schabi.newpipe.crawler.VideoExtractor; +import org.schabi.newpipe.crawler.MediaFormat; +import org.schabi.newpipe.crawler.VideoInfo; +import org.schabi.newpipe.crawler.VideoPreviewInfo; + +import java.io.IOException; +import java.net.URLDecoder; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Vector; + +/** + * Created by Christian Schabesberger on 06.08.15. + * + * Copyright (C) Christian Schabesberger 2015 + * YoutubeVideoExtractor.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +public class YoutubeVideoExtractor implements VideoExtractor { + + public class DecryptException extends ParsingException { + DecryptException(Throwable cause) { + super(cause); + } + DecryptException(String message, Throwable cause) { + super(message, cause); + } + } + + // special content not available exceptions + + public class GemaException extends ContentNotAvailableException { + GemaException(String message) { + super(message); + } + } + + // ---------------- + + private static final String TAG = YoutubeVideoExtractor.class.toString(); + private final Document doc; + private JSONObject playerArgs; + + // static values + private static final String DECRYPTION_FUNC_NAME="decrypt"; + + // cached values + private static volatile String decryptionCode = ""; + + VideoUrlIdHandler urlidhandler = new YoutubeVideoUrlIdHandler(); + String pageUrl = ""; + + private Downloader downloader; + + public YoutubeVideoExtractor(String pageUrl, Downloader dl) throws CrawlingException, IOException { + //most common videoInfo fields are now set in our superclass, for all services + downloader = dl; + this.pageUrl = pageUrl; + String pageContent = downloader.download(urlidhandler.cleanUrl(pageUrl)); + doc = Jsoup.parse(pageContent, pageUrl); + String ytPlayerConfigRaw; + JSONObject ytPlayerConfig; + + //attempt to load the youtube js player JSON arguments + try { + ytPlayerConfigRaw = + RegexHelper.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent); + ytPlayerConfig = new JSONObject(ytPlayerConfigRaw); + playerArgs = ytPlayerConfig.getJSONObject("args"); + } catch (RegexHelper.RegexException e) { + String errorReason = findErrorReason(doc); + switch(errorReason) { + case "GEMA": + throw new GemaException(errorReason); + case "": + throw new ParsingException("player config empty", e); + default: + throw new ContentNotAvailableException("Content not available", e); + } + } catch (JSONException e) { + throw new ParsingException("Could not parse yt player config"); + } + + //---------------------------------- + // load and parse description code, if it isn't already initialised + //---------------------------------- + if (decryptionCode.isEmpty()) { + try { + // The Youtube service needs to be initialized by downloading the + // js-Youtube-player. This is done in order to get the algorithm + // for decrypting cryptic signatures inside certain stream urls. + JSONObject ytAssets = ytPlayerConfig.getJSONObject("assets"); + String playerUrl = ytAssets.getString("js"); + + if (playerUrl.startsWith("//")) { + playerUrl = "https:" + playerUrl; + } + decryptionCode = loadDecryptionCode(playerUrl); + } catch (JSONException e) { + throw new ParsingException( + "Could not load decryption code for the Youtube service.", e); + } + } + } + + @Override + public String getTitle() throws ParsingException { + try {//json player args method + return playerArgs.getString("title"); + } catch(JSONException je) {//html method + je.printStackTrace(); + Log.w(TAG, "failed to load title from JSON args; trying to extract it from HTML"); + try { // fall through to fall-back + return doc.select("meta[name=title]").attr("content"); + } catch (Exception e) { + throw new ParsingException("failed permanently to load title.", e); + } + } + } + + @Override + public String getDescription() throws ParsingException { + try { + return doc.select("p[id=\"eow-description\"]").first().html(); + } catch (Exception e) {//todo: add fallback method <-- there is no ... as long as i know + throw new ParsingException("failed to load description.", e); + } + } + + @Override + public String getUploader() throws ParsingException { + try {//json player args method + return playerArgs.getString("author"); + } catch(JSONException je) { + je.printStackTrace(); + Log.w(TAG, + "failed to load uploader name from JSON args; trying to extract it from HTML"); + } try {//fall through to fallback HTML method + return doc.select("div.yt-user-info").first().text(); + } catch (Exception e) { + throw new ParsingException("failed permanently to load uploader name.", e); + } + } + + @Override + public int getLength() throws ParsingException { + try { + return playerArgs.getInt("length_seconds"); + } catch (JSONException e) {//todo: find fallback method + throw new ParsingException("failed to load video duration from JSON args", e); + } + } + + @Override + public long getViews() throws ParsingException { + try { + String viewCountString = doc.select("meta[itemprop=interactionCount]").attr("content"); + return Long.parseLong(viewCountString); + } catch (Exception e) {//todo: find fallback method + throw new ParsingException("failed to number of views", e); + } + } + + @Override + public String getUploadDate() throws ParsingException { + try { + return doc.select("meta[itemprop=datePublished]").attr("content"); + } catch (Exception e) {//todo: add fallback method + throw new ParsingException("failed to get upload date.", e); + } + } + + @Override + public String getThumbnailUrl() throws ParsingException { + //first attempt getting a small image version + //in the html extracting part we try to get a thumbnail with a higher resolution + // Try to get high resolution thumbnail if it fails use low res from the player instead + try { + return doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href"); + } catch(Exception e) { + Log.w(TAG, "Could not find high res Thumbnail. Using low res instead"); + } try { //fall through to fallback + return playerArgs.getString("thumbnail_url"); + } catch (JSONException je) { + throw new ParsingException( + "failed to extract thumbnail URL from JSON args; trying to extract it from HTML", je); + } + } + + @Override + public String getUploaderThumbnailUrl() throws ParsingException { + try { + return doc.select("a[class*=\"yt-user-photo\"]").first() + .select("img").first() + .attr("abs:data-thumb"); + } catch (Exception e) {//todo: add fallback method + throw new ParsingException("failed to get uploader thumbnail URL.", e); + } + } + + @Override + public String getDashMpdUrl() throws ParsingException { + try { + String dashManifest = playerArgs.getString("dashmpd"); + if(!dashManifest.contains("/signature/")) { + String encryptedSig = RegexHelper.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifest); + String decryptedSig; + + decryptedSig = decryptSignature(encryptedSig, decryptionCode); + dashManifest = dashManifest.replace("/s/" + encryptedSig, "/signature/" + decryptedSig); + } + + return dashManifest; + } catch(NullPointerException e) { + throw new ParsingException( + "Could not find \"dashmpd\" upon the player args (maybe no dash manifest available).", e); + } catch (Exception e) { + throw new ParsingException(e); + } + } + + @Override + public List getAudioStreams() throws ParsingException { + /* If we provide a valid dash manifest, we don't need to provide audio streams extra */ + return null; + } + + @Override + public List getVideoStreams() throws ParsingException { + Vector videoStreams = new Vector<>(); + try{ + String encoded_url_map = playerArgs.getString("url_encoded_fmt_stream_map"); + for(String url_data_str : encoded_url_map.split(",")) { + try { + Map tags = new HashMap<>(); + for (String raw_tag : Parser.unescapeEntities(url_data_str, true).split("&")) { + String[] split_tag = raw_tag.split("="); + tags.put(split_tag[0], split_tag[1]); + } + + int itag = Integer.parseInt(tags.get("itag")); + String streamUrl = URLDecoder.decode(tags.get("url"), "UTF-8"); + + // if video has a signature: decrypt it and add it to the url + if (tags.get("s") != null) { + streamUrl = streamUrl + "&signature=" + + decryptSignature(tags.get("s"), decryptionCode); + } + + if (resolveFormat(itag) != -1) { + videoStreams.add(new VideoInfo.VideoStream( + streamUrl, + resolveFormat(itag), + resolveResolutionString(itag))); + } + } catch (Exception e) { + Log.w(TAG, "Could not get Video stream."); + e.printStackTrace(); + } + } + + } catch (Exception e) { + throw new ParsingException("Failed to get video streams", e); + } + + if(videoStreams.isEmpty()) { + throw new ParsingException("Failed to get any video stream"); + } + + return videoStreams; + } + + /**Attempts to parse (and return) the offset to start playing the video from. + * @return the offset (in seconds), or 0 if no timestamp is found.*/ + @Override + public int getTimeStamp() throws ParsingException { + //todo: add unit test for timestamp + String timeStamp; + try { + timeStamp = RegexHelper.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl); + } catch (RegexHelper.RegexException e) { + // catch this instantly since an url does not necessarily have to have a time stamp + + // -2 because well the testing system will then know its the regex that failed :/ + // not good i know + return -2; + } + + //TODO: test this + if(!timeStamp.isEmpty()) { + try { + String secondsString = ""; + String minutesString = ""; + String hoursString = ""; + try { + secondsString = RegexHelper.matchGroup1("(\\d{1,3})s", timeStamp); + minutesString = RegexHelper.matchGroup1("(\\d{1,3})m", timeStamp); + hoursString = RegexHelper.matchGroup1("(\\d{1,3})h", timeStamp); + } catch (Exception e) { + //it could be that time is given in another method + if (secondsString.isEmpty() //if nothing was got, + && minutesString.isEmpty()//treat as unlabelled seconds + && hoursString.isEmpty()) { + secondsString = RegexHelper.matchGroup1("t=(\\d{1,3})", timeStamp); + } + } + + int seconds = (secondsString.isEmpty() ? 0 : Integer.parseInt(secondsString)); + int minutes = (minutesString.isEmpty() ? 0 : Integer.parseInt(minutesString)); + int hours = (hoursString.isEmpty() ? 0 : Integer.parseInt(hoursString)); + + int ret = seconds + (60 * minutes) + (3600 * hours);//don't trust BODMAS! + //Log.d(TAG, "derived timestamp value:"+ret); + return ret; + //the ordering varies internationally + } catch (ParsingException e) { + throw new ParsingException("Could not get timestamp.", e); + } + } else { + return -1; + } + } + + @Override + public int getAgeLimit() throws ParsingException { + // Not yet implemented. + // Also you need to be logged in to see age restricted videos on youtube, + // therefore NP is not able to receive such videos. + return 0; + } + + @Override + public String getAverageRating() throws ParsingException { + try { + return playerArgs.getString("avg_rating"); + } catch (JSONException e) { + throw new ParsingException("Could not get Average rating", e); + } + } + + @Override + public int getLikeCount() throws ParsingException { + String likesString = ""; + try { + likesString = doc.select("button.like-button-renderer-like-button").first() + .select("span.yt-uix-button-content").first().text(); + return Integer.parseInt(likesString.replaceAll("[^\\d]", "")); + } catch (NumberFormatException nfe) { + throw new ParsingException( + "failed to parse likesString \"" + likesString + "\" as integers", nfe); + } catch (Exception e) { + throw new ParsingException("Could not get like count", e); + } + } + + @Override + public int getDislikeCount() throws ParsingException { + String dislikesString = ""; + try { + dislikesString = doc.select("button.like-button-renderer-dislike-button").first() + .select("span.yt-uix-button-content").first().text(); + return Integer.parseInt(dislikesString.replaceAll("[^\\d]", "")); + } catch(NumberFormatException nfe) { + throw new ParsingException( + "failed to parse dislikesString \"" + dislikesString + "\" as integers", nfe); + } catch(Exception e) { + throw new ParsingException("Could not get dislike count", e); + } + } + + @Override + public VideoPreviewInfo getNextVideo() throws ParsingException { + try { + return extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first() + .select("li").first()); + } catch(Exception e) { + throw new ParsingException("Could not get next video", e); + } + } + + @Override + public Vector getRelatedVideos() throws ParsingException { + try { + Vector relatedVideos = new Vector<>(); + for (Element li : doc.select("ul[id=\"watch-related\"]").first().children()) { + // first check if we have a playlist. If so leave them out + if (li.select("a[class*=\"content-link\"]").first() != null) { + relatedVideos.add(extractVideoPreviewInfo(li)); + } + } + return relatedVideos; + } catch(Exception e) { + throw new ParsingException("Could not get related videos", e); + } + } + + @Override + public VideoUrlIdHandler getUrlIdConverter() { + return new YoutubeVideoUrlIdHandler(); + } + + @Override + public String getPageUrl() { + return pageUrl; + } + + /**Provides information about links to other videos on the video page, such as related videos. + * This is encapsulated in a VideoPreviewInfo object, + * which is a subset of the fields in a full VideoInfo.*/ + private VideoPreviewInfo extractVideoPreviewInfo(Element li) throws ParsingException { + VideoPreviewInfo info = new VideoPreviewInfo(); + + try { + info.webpage_url = li.select("a.content-link").first() + .attr("abs:href"); + + info.id = RegexHelper.matchGroup1("v=([0-9a-zA-Z-]*)", info.webpage_url); + + //todo: check NullPointerException causing + info.title = li.select("span.title").first().text(); + //this page causes the NullPointerException, after finding it by searching for "tjvg": + //https://www.youtube.com/watch?v=Uqg0aEhLFAg + + //this line is unused + //String views = li.select("span.view-count").first().text(); + + //Log.i(TAG, "title:"+info.title); + //Log.i(TAG, "view count:"+views); + + try { + info.view_count = Long.parseLong(li.select("span.view-count") + .first().text().replaceAll("[^\\d]", "")); + } catch (NullPointerException e) {//related videos sometimes have no view count + info.view_count = 0; + } + info.uploader = li.select("span.g-hovercard").first().text(); + + info.duration = li.select("span.video-time").first().text(); + + Element img = li.select("img").first(); + info.thumbnail_url = img.attr("abs:src"); + // Sometimes youtube sends links to gif files which somehow seem to not exist + // anymore. Items with such gif also offer a secondary image source. So we are going + // to use that if we caught such an item. + if (info.thumbnail_url.contains(".gif")) { + info.thumbnail_url = img.attr("data-thumb"); + } + if (info.thumbnail_url.startsWith("//")) { + info.thumbnail_url = "https:" + info.thumbnail_url; + } + } catch (Exception e) { + throw new ParsingException(e); + } + return info; + } + + private String loadDecryptionCode(String playerUrl) throws DecryptException { + String decryptionFuncName; + String decryptionFunc; + String helperObjectName; + String helperObject; + String callerFunc = "function " + DECRYPTION_FUNC_NAME + "(a){return %%(a);}"; + String decryptionCode; + + try { + String playerCode = downloader.download(playerUrl); + + decryptionFuncName = + RegexHelper.matchGroup1("\\.sig\\|\\|([a-zA-Z0-9$]+)\\(", playerCode); + + String functionPattern = "(" + + decryptionFuncName.replace("$", "\\$") + + "=function\\([a-zA-Z0-9_]*\\)\\{.+?\\})"; + decryptionFunc = "var " + RegexHelper.matchGroup1(functionPattern, playerCode) + ";"; + + helperObjectName = RegexHelper + .matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", decryptionFunc); + + String helperPattern = "(var " + + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)"; + helperObject = RegexHelper.matchGroup1(helperPattern, playerCode); + + + callerFunc = callerFunc.replace("%%", decryptionFuncName); + decryptionCode = helperObject + decryptionFunc + callerFunc; + } catch(IOException ioe) { + throw new DecryptException("Could not load decrypt function", ioe); + } catch(Exception e) { + throw new DecryptException("Could not parse decrypt function ", e); + } + + return decryptionCode; + } + + private String decryptSignature(String encryptedSig, String decryptionCode) + throws DecryptException{ + Context context = Context.enter(); + context.setOptimizationLevel(-1); + Object result = null; + try { + ScriptableObject scope = context.initStandardObjects(); + context.evaluateString(scope, decryptionCode, "decryptionCode", 1, null); + Function decryptionFunc = (Function) scope.get("decrypt", scope); + result = decryptionFunc.call(context, scope, scope, new Object[]{encryptedSig}); + } catch (Exception e) { + throw new DecryptException(e); + } finally { + Context.exit(); + } + return (result == null ? "" : result.toString()); + } + + private String findErrorReason(Document doc) { + String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text(); + if(errorMessage.contains("GEMA")) { + // Gema sometimes blocks youtube music content in germany: + // https://www.gema.de/en/ + // Detailed description: + // https://en.wikipedia.org/wiki/GEMA_%28German_organization%29 + return "GEMA"; + } + return ""; + } + + /**These lists only contain itag formats that are supported by the common Android Video player. + However if you are looking for a list showing all itag formats, look at + https://github.com/rg3/youtube-dl/issues/1687 */ + + @SuppressWarnings("WeakerAccess") + public static int resolveFormat(int itag) { + switch(itag) { + // !!! lists only supported formats !!! + // video + case 17: return MediaFormat.v3GPP.id; + case 18: return MediaFormat.MPEG_4.id; + case 22: return MediaFormat.MPEG_4.id; + case 36: return MediaFormat.v3GPP.id; + case 37: return MediaFormat.MPEG_4.id; + case 38: return MediaFormat.MPEG_4.id; + case 43: return MediaFormat.WEBM.id; + case 44: return MediaFormat.WEBM.id; + case 45: return MediaFormat.WEBM.id; + case 46: return MediaFormat.WEBM.id; + default: + //Log.i(TAG, "Itag " + Integer.toString(itag) + " not known or not supported."); + return -1; + } + } + + @SuppressWarnings("WeakerAccess") + public static String resolveResolutionString(int itag) { + switch(itag) { + case 17: return "144p"; + case 18: return "360p"; + case 22: return "720p"; + case 36: return "240p"; + case 37: return "1080p"; + case 38: return "1080p"; + case 43: return "360p"; + case 44: return "480p"; + case 45: return "720p"; + case 46: return "1080p"; + default: + //Log.i(TAG, "Itag " + Integer.toString(itag) + " not known or not supported."); + return null; + } + } +} diff --git a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoUrlIdHandler.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoUrlIdHandler.java new file mode 100644 index 000000000..b9d2b4fe8 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoUrlIdHandler.java @@ -0,0 +1,68 @@ +package org.schabi.newpipe.crawler.services.youtube; + +import org.schabi.newpipe.crawler.ParsingException; +import org.schabi.newpipe.crawler.RegexHelper; +import org.schabi.newpipe.crawler.VideoUrlIdHandler; + +/** + * Created by Christian Schabesberger on 02.02.16. + * + * Copyright (C) Christian Schabesberger 2016 + * YoutubeVideoUrlIdHandler.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +public class YoutubeVideoUrlIdHandler implements VideoUrlIdHandler { + @SuppressWarnings("WeakerAccess") + @Override + public String getVideoUrl(String videoId) { + return "https://www.youtube.com/watch?v=" + videoId; + } + + @SuppressWarnings("WeakerAccess") + @Override + public String getVideoId(String url) throws ParsingException { + String id; + String pat; + + if(url.contains("youtube")) { + pat = "youtube\\.com/watch\\?v=([\\-a-zA-Z0-9_]{11})"; + } + else if(url.contains("youtu.be")) { + pat = "youtu\\.be/([a-zA-Z0-9_-]{11})"; + } + else { + throw new ParsingException("Error no suitable url: " + url); + } + + id = RegexHelper.matchGroup1(pat, url); + if(!id.isEmpty()){ + //Log.i(TAG, "string \""+url+"\" matches!"); + return id; + } else { + throw new ParsingException("Error could not parse url: " + url); + } + } + + public String cleanUrl(String complexUrl) throws ParsingException { + return getVideoUrl(getVideoId(complexUrl)); + } + + @Override + public boolean acceptUrl(String videoUrl) { + return videoUrl.contains("youtube") || + videoUrl.contains("youtu.be"); + } +} diff --git a/app/src/main/java/org/schabi/newpipe/services/AbstractVideoInfo.java b/app/src/main/java/org/schabi/newpipe/services/AbstractVideoInfo.java deleted file mode 100644 index 72d43ebde..000000000 --- a/app/src/main/java/org/schabi/newpipe/services/AbstractVideoInfo.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.schabi.newpipe.services; - -import android.graphics.Bitmap; - -/**Common properties between VideoInfo and VideoPreviewInfo.*/ -public abstract class AbstractVideoInfo { - public String id = ""; - public String title = ""; - public String uploader = ""; - //public int duration = -1; - public String thumbnail_url = ""; - public Bitmap thumbnail = null; - public String webpage_url = ""; - public String upload_date = ""; - public long view_count = -1; -} diff --git a/app/src/main/java/org/schabi/newpipe/services/VideoExtractor.java b/app/src/main/java/org/schabi/newpipe/services/VideoExtractor.java deleted file mode 100644 index f57ef0894..000000000 --- a/app/src/main/java/org/schabi/newpipe/services/VideoExtractor.java +++ /dev/null @@ -1,128 +0,0 @@ -package org.schabi.newpipe.services; - -/** - * Created by Christian Schabesberger on 10.08.15. - * - * Copyright (C) Christian Schabesberger 2015 - * VideoExtractor.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -/**Scrapes information from a video streaming service (eg, YouTube).*/ - -@SuppressWarnings("ALL") -public abstract class VideoExtractor { - protected final String pageUrl; - protected VideoInfo videoInfo; - - @SuppressWarnings("WeakerAccess") - public VideoExtractor(String url) { - this.pageUrl = url; - } - - /**Fills out the video info fields which are common to all services. - * Probably needs to be overridden by subclasses*/ - public VideoInfo getVideoInfo() - { - if(videoInfo == null) { - videoInfo = new VideoInfo(); - } - - if(videoInfo.webpage_url.isEmpty()) { - videoInfo.webpage_url = pageUrl; - } - - if(getErrorCode() == VideoInfo.NO_ERROR) { - - if (videoInfo.title.isEmpty()) { - videoInfo.title = getTitle(); - } - - if (videoInfo.duration < 1) { - videoInfo.duration = getLength(); - } - - - if (videoInfo.uploader.isEmpty()) { - videoInfo.uploader = getUploader(); - } - - if (videoInfo.description.isEmpty()) { - videoInfo.description = getDescription(); - } - - if (videoInfo.view_count == -1) { - videoInfo.view_count = getViews(); - } - - if (videoInfo.upload_date.isEmpty()) { - videoInfo.upload_date = getUploadDate(); - } - - if (videoInfo.thumbnail_url.isEmpty()) { - videoInfo.thumbnail_url = getThumbnailUrl(); - } - - if (videoInfo.id.isEmpty()) { - videoInfo.id = getVideoId(pageUrl); - } - - /** Load and extract audio*/ - if (videoInfo.audioStreams == null) { - videoInfo.audioStreams = getAudioStreams(); - } - /** Extract video stream url*/ - if (videoInfo.videoStreams == null) { - videoInfo.videoStreams = getVideoStreams(); - } - - if (videoInfo.uploader_thumbnail_url.isEmpty()) { - videoInfo.uploader_thumbnail_url = getUploaderThumbnailUrl(); - } - - if (videoInfo.startPosition < 0) { - videoInfo.startPosition = getTimeStamp(); - } - } else { - videoInfo.errorCode = getErrorCode(); - videoInfo.errorMessage = getErrorMessage(); - } - - //Bitmap thumbnail = null; - //Bitmap uploader_thumbnail = null; - //int videoAvailableStatus = VIDEO_AVAILABLE; - return videoInfo; - } - - //todo: add licence field - public abstract int getErrorCode(); - public abstract String getErrorMessage(); - - //todo: remove these functions, or make them static, otherwise its useles, to have them here - public abstract String getVideoUrl(String videoId); - public abstract String getVideoId(String siteUrl); - /////////////////////////////////////////////////////////////////////////////////////////// - public abstract int getTimeStamp(); - public abstract String getTitle(); - public abstract String getDescription(); - public abstract String getUploader(); - public abstract int getLength(); - public abstract long getViews(); - public abstract String getUploadDate(); - public abstract String getThumbnailUrl(); - public abstract String getUploaderThumbnailUrl(); - public abstract VideoInfo.AudioStream[] getAudioStreams(); - public abstract VideoInfo.VideoStream[] getVideoStreams(); -} diff --git a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngine.java b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngine.java deleted file mode 100644 index 30bc92ce7..000000000 --- a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngine.java +++ /dev/null @@ -1,190 +0,0 @@ -package org.schabi.newpipe.services.youtube; - -import android.net.Uri; -import android.util.Log; - -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.schabi.newpipe.Downloader; -import org.schabi.newpipe.services.SearchEngine; -import org.schabi.newpipe.VideoPreviewInfo; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; - -/** - * Created by Christian Schabesberger on 09.08.15. - * - * Copyright (C) Christian Schabesberger 2015 - * YoutubeSearchEngine.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class YoutubeSearchEngine implements SearchEngine { - - private static final String TAG = YoutubeSearchEngine.class.toString(); - - @Override - public Result search(String query, int page, String languageCode) { - //String contentCountry = PreferenceManager.getDefaultSharedPreferences(this).getString(getString(R.string., ""); - Uri.Builder builder = new Uri.Builder(); - builder.scheme("https") - .authority("www.youtube.com") - .appendPath("results") - .appendQueryParameter("search_query", query) - .appendQueryParameter("page", Integer.toString(page)) - .appendQueryParameter("filters", "video"); - - String site; - String url = builder.build().toString(); - //if we've been passed a valid language code, append it to the URL - if(!languageCode.isEmpty()) { - //assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode); - site = Downloader.download(url, languageCode); - } - else { - site = Downloader.download(url); - } - - - Document doc = Jsoup.parse(site, url); - Result result = new Result(); - Element list = doc.select("ol[class=\"item-section\"]").first(); - - - int i = 0; - for(Element item : list.children()) { - i++; - /* First we need to determine which kind of item we are working with. - Youtube depicts five different kinds of items on its search result page. These are - regular videos, playlists, channels, two types of video suggestions, and a "no video - found" item. Since we only want videos, we need to filter out all the others. - An example for this can be seen here: - https://www.youtube.com/results?search_query=asdf&page=1 - - We already applied a filter to the url, so we don't need to care about channels and - playlists now. - */ - - Element el; - - // both types of spell correction item - if(!((el = item.select("div[class*=\"spell-correction\"]").first()) == null)) { - result.suggestion = el.select("a").first().text(); - // search message item - } else if(!((el = item.select("div[class*=\"search-message\"]").first()) == null)) { - result.errorMessage = el.text(); - - // video item type - } else if(!((el = item.select("div[class*=\"yt-lockup-video\"").first()) == null)) { - VideoPreviewInfo resultItem = new VideoPreviewInfo(); - Element dl = el.select("h3").first().select("a").first(); - resultItem.webpage_url = dl.attr("abs:href"); - try { - Pattern p = Pattern.compile("v=([0-9a-zA-Z-]*)"); - Matcher m = p.matcher(resultItem.webpage_url); - resultItem.id=m.group(1); - } catch (Exception e) { - //e.printStackTrace(); - } - resultItem.title = dl.text(); - - resultItem.duration = item.select("span[class=\"video-time\"]").first().text(); - - resultItem.uploader = item.select("div[class=\"yt-lockup-byline\"]").first() - .select("a").first() - .text(); - resultItem.upload_date = item.select("div[class=\"yt-lockup-meta\"]").first() - .select("li").first() - .text(); - Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first() - .select("img").first(); - resultItem.thumbnail_url = te.attr("abs:src"); - // Sometimes youtube sends links to gif files which somehow seem to not exist - // anymore. Items with such gif also offer a secondary image source. So we are going - // to use that if we've caught such an item. - if(resultItem.thumbnail_url.contains(".gif")) { - resultItem.thumbnail_url = te.attr("abs:data-thumb"); - } - result.resultList.add(resultItem); - } else { - //noinspection ConstantConditions - Log.e(TAG, "unexpected element found:\""+el+"\""); - } - } - return result; - } - - @Override - public ArrayList suggestionList(String query) { - - ArrayList suggestions = new ArrayList<>(); - - Uri.Builder builder = new Uri.Builder(); - builder.scheme("https") - .authority("suggestqueries.google.com") - .appendPath("complete") - .appendPath("search") - .appendQueryParameter("client", "") - .appendQueryParameter("output", "toolbar") - .appendQueryParameter("ds", "yt") - .appendQueryParameter("q", query); - String url = builder.build().toString(); - - String response = Downloader.download(url); - - //TODO: Parse xml data using Jsoup not done - DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder dBuilder; - org.w3c.dom.Document doc = null; - - try { - dBuilder = dbFactory.newDocumentBuilder(); - doc = dBuilder.parse(new InputSource(new ByteArrayInputStream(response.getBytes("utf-8")))); - doc.getDocumentElement().normalize(); - }catch (ParserConfigurationException | SAXException | IOException e) { - e.printStackTrace(); - } - - if(doc!=null){ - NodeList nList = doc.getElementsByTagName("CompleteSuggestion"); - for (int temp = 0; temp < nList.getLength(); temp++) { - - NodeList nList1 = doc.getElementsByTagName("suggestion"); - Node nNode1 = nList1.item(temp); - if (nNode1.getNodeType() == Node.ELEMENT_NODE) { - org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1; - suggestions.add(eElement.getAttribute("data")); - } - } - }else { - Log.e(TAG, "GREAT FUCKING ERROR"); - } - return suggestions; - } - -} diff --git a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractor.java b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractor.java deleted file mode 100644 index 6477305cf..000000000 --- a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractor.java +++ /dev/null @@ -1,647 +0,0 @@ -package org.schabi.newpipe.services.youtube; - -import android.util.Log; -import android.util.Xml; - -import org.json.JSONException; -import org.json.JSONObject; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.parser.Parser; -import org.mozilla.javascript.Context; -import org.mozilla.javascript.Function; -import org.mozilla.javascript.ScriptableObject; -import org.schabi.newpipe.Downloader; -import org.schabi.newpipe.services.VideoExtractor; -import org.schabi.newpipe.services.MediaFormat; -import org.schabi.newpipe.services.VideoInfo; -import org.schabi.newpipe.VideoPreviewInfo; -import org.xmlpull.v1.XmlPullParser; - -import java.io.StringReader; -import java.net.URLDecoder; -import java.util.HashMap; -import java.util.Map; -import java.util.Vector; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Created by Christian Schabesberger on 06.08.15. - * - * Copyright (C) Christian Schabesberger 2015 - * YoutubeVideoExtractor.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class YoutubeVideoExtractor extends VideoExtractor { - - private static final String TAG = YoutubeVideoExtractor.class.toString(); - private final Document doc; - private JSONObject jsonObj; - private JSONObject playerArgs; - private int errorCode = VideoInfo.NO_ERROR; - private String errorMessage = ""; - - // static values - private static final String DECRYPTION_FUNC_NAME="decrypt"; - - // cached values - private static volatile String decryptionCode = ""; - - public YoutubeVideoExtractor(String pageUrl) { - super(pageUrl);//most common videoInfo fields are now set in our superclass, for all services - String pageContent = Downloader.download(cleanUrl(pageUrl)); - doc = Jsoup.parse(pageContent, pageUrl); - - //attempt to load the youtube js player JSON arguments - try { - String jsonString = matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent); - //todo: implement this by try and catch. TESTING THE STRING AGAINST EMPTY IS CONSIDERED POOR STYLE !!! - if(jsonString.isEmpty()) { - errorCode = findErrorReason(doc); - return; - } - - jsonObj = new JSONObject(jsonString); - playerArgs = jsonObj.getJSONObject("args"); - - } catch (Exception e) {//if this fails, the video is most likely not available. - // Determining why is done later. - videoInfo.errorCode = VideoInfo.ERROR_NO_SPECIFIED_ERROR; - Log.e(TAG, "Could not load JSON data for Youtube video \""+pageUrl+"\". This most likely means the video is unavailable"); - } - - //---------------------------------- - // load and parse description code, if it isn't already initialised - //---------------------------------- - if (decryptionCode.isEmpty()) { - try { - // The Youtube service needs to be initialized by downloading the - // js-Youtube-player. This is done in order to get the algorithm - // for decrypting cryptic signatures inside certain stream urls. - JSONObject ytAssets = jsonObj.getJSONObject("assets"); - String playerUrl = ytAssets.getString("js"); - - if (playerUrl.startsWith("//")) { - playerUrl = "https:" + playerUrl; - } - decryptionCode = loadDecryptionCode(playerUrl); - } catch (Exception e){ - Log.e(TAG, "Could not load decryption code for the Youtube service."); - e.printStackTrace(); - } - } - } - - @Override - public String getTitle() { - try {//json player args method - return playerArgs.getString("title"); - } catch(JSONException je) {//html method - je.printStackTrace(); - Log.w(TAG, "failed to load title from JSON args; trying to extract it from HTML"); - } try { // fall through to fall-back - return doc.select("meta[name=title]").attr("content"); - } catch (Exception e) { - Log.e(TAG, "failed permanently to load title."); - e.printStackTrace(); - return ""; - } - } - - @Override - public String getDescription() { - try { - return doc.select("p[id=\"eow-description\"]").first().html(); - } catch (Exception e) {//todo: add fallback method - Log.e(TAG, "failed to load description."); - e.printStackTrace(); - return ""; - } - } - - @Override - public String getUploader() { - try {//json player args method - return playerArgs.getString("author"); - } catch(JSONException je) { - je.printStackTrace(); - Log.w(TAG, "failed to load uploader name from JSON args; trying to extract it from HTML"); - } try {//fall through to fallback HTML method - return doc.select("div.yt-user-info").first().text(); - } catch (Exception e) { - e.printStackTrace(); - Log.e(TAG, "failed permanently to load uploader name."); - return ""; - } - } - - @Override - public int getLength() { - try { - return playerArgs.getInt("length_seconds"); - } catch (JSONException je) {//todo: find fallback method - Log.e(TAG, "failed to load video duration from JSON args"); - je.printStackTrace(); - return -1; - } - } - - @Override - public long getViews() { - try { - String viewCountString = doc.select("meta[itemprop=interactionCount]").attr("content"); - return Long.parseLong(viewCountString); - } catch (Exception e) {//todo: find fallback method - Log.e(TAG, "failed to number of views"); - e.printStackTrace(); - return -1; - } - } - - @Override - public String getUploadDate() { - try { - return doc.select("meta[itemprop=datePublished]").attr("content"); - } catch (Exception e) {//todo: add fallback method - Log.e(TAG, "failed to get upload date."); - e.printStackTrace(); - return ""; - } - } - - @Override - public String getThumbnailUrl() { - //first attempt getting a small image version - //in the html extracting part we try to get a thumbnail with a higher resolution - // Try to get high resolution thumbnail if it fails use low res from the player instead - try { - return doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href"); - } catch(Exception e) { - Log.w(TAG, "Could not find high res Thumbnail. Using low res instead"); - //fall through to fallback - } try { - return playerArgs.getString("thumbnail_url"); - } catch (JSONException je) { - je.printStackTrace(); - Log.w(TAG, "failed to extract thumbnail URL from JSON args; trying to extract it from HTML"); - return ""; - } - } - - @Override - public String getUploaderThumbnailUrl() { - try { - return doc.select("a[class*=\"yt-user-photo\"]").first() - .select("img").first() - .attr("abs:data-thumb"); - } catch (Exception e) {//todo: add fallback method - Log.e(TAG, "failed to get uploader thumbnail URL."); - e.printStackTrace(); - return ""; - } - } - - @Override - public VideoInfo.AudioStream[] getAudioStreams() { - try { - String dashManifest = playerArgs.getString("dashmpd"); - return parseDashManifest(dashManifest, decryptionCode); - - } catch (NullPointerException e) { - Log.e(TAG, "Could not find \"dashmpd\" upon the player args (maybe no dash manifest available)."); - } catch (Exception e) { - e.printStackTrace(); - } - return new VideoInfo.AudioStream[0]; - } - - @Override - public VideoInfo.VideoStream[] getVideoStreams() { - try{ - //------------------------------------ - // extract video stream url - //------------------------------------ - String encoded_url_map = playerArgs.getString("url_encoded_fmt_stream_map"); - Vector videoStreams = new Vector<>(); - for(String url_data_str : encoded_url_map.split(",")) { - Map tags = new HashMap<>(); - for(String raw_tag : Parser.unescapeEntities(url_data_str, true).split("&")) { - String[] split_tag = raw_tag.split("="); - tags.put(split_tag[0], split_tag[1]); - } - - int itag = Integer.parseInt(tags.get("itag")); - String streamUrl = URLDecoder.decode(tags.get("url"), "UTF-8"); - - // if video has a signature: decrypt it and add it to the url - if(tags.get("s") != null) { - streamUrl = streamUrl + "&signature=" + decryptSignature(tags.get("s"), decryptionCode); - } - - if(resolveFormat(itag) != -1) { - videoStreams.add(new VideoInfo.VideoStream( - streamUrl, - resolveFormat(itag), - resolveResolutionString(itag))); - } - } - return videoStreams.toArray(new VideoInfo.VideoStream[videoStreams.size()]); - - } catch (Exception e) { - Log.e(TAG, "Failed to get video stream"); - e.printStackTrace(); - return new VideoInfo.VideoStream[0]; - } - } - - /**These lists only contain itag formats that are supported by the common Android Video player. - However if you are looking for a list showing all itag formats, look at - https://github.com/rg3/youtube-dl/issues/1687 */ - - @SuppressWarnings("WeakerAccess") - public static int resolveFormat(int itag) { - switch(itag) { - // video - case 17: return MediaFormat.v3GPP.id; - case 18: return MediaFormat.MPEG_4.id; - case 22: return MediaFormat.MPEG_4.id; - case 36: return MediaFormat.v3GPP.id; - case 37: return MediaFormat.MPEG_4.id; - case 38: return MediaFormat.MPEG_4.id; - case 43: return MediaFormat.WEBM.id; - case 44: return MediaFormat.WEBM.id; - case 45: return MediaFormat.WEBM.id; - case 46: return MediaFormat.WEBM.id; - default: - //Log.i(TAG, "Itag " + Integer.toString(itag) + " not known or not supported."); - return -1; - } - } - - @SuppressWarnings("WeakerAccess") - public static String resolveResolutionString(int itag) { - switch(itag) { - case 17: return "144p"; - case 18: return "360p"; - case 22: return "720p"; - case 36: return "240p"; - case 37: return "1080p"; - case 38: return "1080p"; - case 43: return "360p"; - case 44: return "480p"; - case 45: return "720p"; - case 46: return "1080p"; - default: - //Log.i(TAG, "Itag " + Integer.toString(itag) + " not known or not supported."); - return null; - } - } - - @SuppressWarnings("WeakerAccess") - @Override - public String getVideoId(String url) { - String id; - String pat; - - if(url.contains("youtube")) { - pat = "youtube\\.com/watch\\?v=([\\-a-zA-Z0-9_]{11})"; - } - else if(url.contains("youtu.be")) { - pat = "youtu\\.be/([a-zA-Z0-9_-]{11})"; - } - else { - Log.e(TAG, "Error could not parse url: " + url); - return ""; - } - id = matchGroup1(pat, url); - if(!id.isEmpty()){ - //Log.i(TAG, "string \""+url+"\" matches!"); - return id; - } - //Log.i(TAG, "string \""+url+"\" does not match."); - return ""; - } - - @SuppressWarnings("WeakerAccess") - @Override - public String getVideoUrl(String videoId) { - return "https://www.youtube.com/watch?v=" + videoId; - } - - /**Attempts to parse (and return) the offset to start playing the video from. - * @return the offset (in seconds), or 0 if no timestamp is found.*/ - @Override - public int getTimeStamp(){ - String timeStamp = matchGroup1("((#|&)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl); - - //TODO: test this - if(!timeStamp.isEmpty()) { - String secondsString = matchGroup1("(\\d{1,3})s", timeStamp); - String minutesString = matchGroup1("(\\d{1,3})m", timeStamp); - String hoursString = matchGroup1("(\\d{1,3})h", timeStamp); - - if(secondsString.isEmpty()//if nothing was got, - && minutesString.isEmpty()//treat as unlabelled seconds - && hoursString.isEmpty()) - secondsString = matchGroup1("t=(\\d{1,3})", timeStamp); - - int seconds = (secondsString.isEmpty() ? 0 : Integer.parseInt(secondsString)); - int minutes = (minutesString.isEmpty() ? 0 : Integer.parseInt(minutesString)); - int hours = (hoursString.isEmpty() ? 0 : Integer.parseInt(hoursString)); - - int ret = seconds + (60*minutes) + (3600*hours);//don't trust BODMAS! - //Log.d(TAG, "derived timestamp value:"+ret); - return ret; - //the ordering varies internationally - }//else, return default 0 - return 0; - } - - @Override - public VideoInfo getVideoInfo() { - //todo: @medovax i like your work, but what the fuck: - videoInfo = super.getVideoInfo(); - - if(errorCode == VideoInfo.NO_ERROR) { - //todo: replace this with a call to getVideoId, if possible - videoInfo.id = matchGroup1("v=([0-9a-zA-Z_-]{11})", pageUrl); - - if (videoInfo.audioStreams == null - || videoInfo.audioStreams.length == 0) { - Log.e(TAG, "uninitialised audio streams!"); - } - - if (videoInfo.videoStreams == null - || videoInfo.videoStreams.length == 0) { - Log.e(TAG, "uninitialised video streams!"); - } - - videoInfo.age_limit = 0; - - //average rating - try { - videoInfo.average_rating = playerArgs.getString("avg_rating"); - } catch (JSONException e) { - e.printStackTrace(); - } - - //--------------------------------------- - // extracting information from html page - //--------------------------------------- - - /* Code does not work here anymore. - // Determine what went wrong when the Video is not available - if(videoInfo.errorCode == VideoInfo.ERROR_NO_SPECIFIED_ERROR) { - if(doc.select("h1[id=\"unavailable-message\"]").first().text().contains("GEMA")) { - videoInfo.videoAvailableStatus = VideoInfo.VIDEO_UNAVAILABLE_GEMA; - } - } - */ - - String likesString = ""; - String dislikesString = ""; - try { - // likes - likesString = doc.select("button.like-button-renderer-like-button").first() - .select("span.yt-uix-button-content").first().text(); - videoInfo.like_count = Integer.parseInt(likesString.replaceAll("[^\\d]", "")); - // dislikes - dislikesString = doc.select("button.like-button-renderer-dislike-button").first() - .select("span.yt-uix-button-content").first().text(); - - videoInfo.dislike_count = Integer.parseInt(dislikesString.replaceAll("[^\\d]", "")); - } catch (NumberFormatException nfe) { - Log.e(TAG, "failed to parse likesString \"" + likesString + "\" and dislikesString \"" + - dislikesString + "\" as integers"); - } catch (Exception e) { - // if it fails we know that the video does not offer dislikes. - e.printStackTrace(); - videoInfo.like_count = 0; - videoInfo.dislike_count = 0; - } - - // next video - videoInfo.nextVideo = extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first() - .select("li").first()); - - // related videos - Vector relatedVideos = new Vector<>(); - for (Element li : doc.select("ul[id=\"watch-related\"]").first().children()) { - // first check if we have a playlist. If so leave them out - if (li.select("a[class*=\"content-link\"]").first() != null) { - relatedVideos.add(extractVideoPreviewInfo(li)); - } - } - //todo: replace conversion - videoInfo.relatedVideos = relatedVideos; - //videoInfo.relatedVideos = relatedVideos.toArray(new VideoPreviewInfo[relatedVideos.size()]); - } - return videoInfo; - } - - @Override - public int getErrorCode() { - return errorCode; - } - - @Override - public String getErrorMessage() { - return errorMessage; - } - - private VideoInfo.AudioStream[] parseDashManifest(String dashManifest, String decryptoinCode) { - if(!dashManifest.contains("/signature/")) { - String encryptedSig = matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifest); - String decryptedSig; - - decryptedSig = decryptSignature(encryptedSig, decryptoinCode); - dashManifest = dashManifest.replace("/s/" + encryptedSig, "/signature/" + decryptedSig); - } - String dashDoc = Downloader.download(dashManifest); - Vector audioStreams = new Vector<>(); - try { - XmlPullParser parser = Xml.newPullParser(); - parser.setInput(new StringReader(dashDoc)); - String tagName = ""; - String currentMimeType = ""; - int currentBandwidth = -1; - int currentSamplingRate = -1; - boolean currentTagIsBaseUrl = false; - for(int eventType = parser.getEventType(); - eventType != XmlPullParser.END_DOCUMENT; - eventType = parser.next() ) { - switch(eventType) { - case XmlPullParser.START_TAG: - tagName = parser.getName(); - if(tagName.equals("AdaptationSet")) { - currentMimeType = parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "mimeType"); - } else if(tagName.equals("Representation") && currentMimeType.contains("audio")) { - currentBandwidth = Integer.parseInt( - parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "bandwidth")); - currentSamplingRate = Integer.parseInt( - parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "audioSamplingRate")); - } else if(tagName.equals("BaseURL")) { - currentTagIsBaseUrl = true; - } - break; - - case XmlPullParser.TEXT: - if(currentTagIsBaseUrl && - (currentMimeType.contains("audio"))) { - int format = -1; - if(currentMimeType.equals(MediaFormat.WEBMA.mimeType)) { - format = MediaFormat.WEBMA.id; - } else if(currentMimeType.equals(MediaFormat.M4A.mimeType)) { - format = MediaFormat.M4A.id; - } - audioStreams.add(new VideoInfo.AudioStream(parser.getText(), - format, currentBandwidth, currentSamplingRate)); - } - //missing break here? - case XmlPullParser.END_TAG: - if(tagName.equals("AdaptationSet")) { - currentMimeType = ""; - } else if(tagName.equals("BaseURL")) { - currentTagIsBaseUrl = false; - }//no break needed here - } - } - } catch(Exception e) { - e.printStackTrace(); - } - return audioStreams.toArray(new VideoInfo.AudioStream[audioStreams.size()]); - } - /**Provides information about links to other videos on the video page, such as related videos. - * This is encapsulated in a VideoPreviewInfo object, - * which is a subset of the fields in a full VideoInfo.*/ - private VideoPreviewInfo extractVideoPreviewInfo(Element li) { - VideoPreviewInfo info = new VideoPreviewInfo(); - info.webpage_url = li.select("a.content-link").first() - .attr("abs:href"); - try { - info.id = matchGroup1("v=([0-9a-zA-Z-]*)", info.webpage_url); - } catch (Exception e) { - e.printStackTrace(); - } - - //todo: check NullPointerException causing - info.title = li.select("span.title").first().text(); - //this page causes the NullPointerException, after finding it by searching for "tjvg": - //https://www.youtube.com/watch?v=Uqg0aEhLFAg - - //this line is unused - //String views = li.select("span.view-count").first().text(); - - //Log.i(TAG, "title:"+info.title); - //Log.i(TAG, "view count:"+views); - try { - info.view_count = Long.parseLong(li.select("span.view-count") - .first().text().replaceAll("[^\\d]", "")); - } catch (NullPointerException e) {//related videos sometimes have no view count - info.view_count = 0; - } - info.uploader = li.select("span.g-hovercard").first().text(); - - info.duration = li.select("span.video-time").first().text(); - - Element img = li.select("img").first(); - info.thumbnail_url = img.attr("abs:src"); - // Sometimes youtube sends links to gif files which somehow seem to not exist - // anymore. Items with such gif also offer a secondary image source. So we are going - // to use that if we caught such an item. - if(info.thumbnail_url.contains(".gif")) { - info.thumbnail_url = img.attr("data-thumb"); - } - if(info.thumbnail_url.startsWith("//")) { - info.thumbnail_url = "https:" + info.thumbnail_url; - } - return info; - } - - private String loadDecryptionCode(String playerUrl) { - String playerCode = Downloader.download(playerUrl); - String decryptionFuncName = ""; - String decryptionFunc = ""; - String helperObjectName; - String helperObject = ""; - String callerFunc = "function " + DECRYPTION_FUNC_NAME + "(a){return %%(a);}"; - String decryptionCode; - - try { - decryptionFuncName = matchGroup1("\\.sig\\|\\|([a-zA-Z0-9$]+)\\(", playerCode); - - String functionPattern = "(" + decryptionFuncName.replace("$", "\\$") +"=function\\([a-zA-Z0-9_]*\\)\\{.+?\\})"; - decryptionFunc = "var " + matchGroup1(functionPattern, playerCode) + ";"; - - helperObjectName = matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", decryptionFunc); - - String helperPattern = "(var " + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)"; - helperObject = matchGroup1(helperPattern, playerCode); - - } catch (Exception e) { - e.printStackTrace(); - } - - callerFunc = callerFunc.replace("%%", decryptionFuncName); - decryptionCode = helperObject + decryptionFunc + callerFunc; - - return decryptionCode; - } - - private String decryptSignature(String encryptedSig, String decryptionCode) { - Context context = Context.enter(); - context.setOptimizationLevel(-1); - Object result = null; - try { - ScriptableObject scope = context.initStandardObjects(); - context.evaluateString(scope, decryptionCode, "decryptionCode", 1, null); - Function decryptionFunc = (Function) scope.get("decrypt", scope); - result = decryptionFunc.call(context, scope, scope, new Object[]{encryptedSig}); - } catch (Exception e) { - e.printStackTrace(); - } - Context.exit(); - return (result == null ? "" : result.toString()); - } - - private String cleanUrl(String complexUrl) { - return getVideoUrl(getVideoId(complexUrl)); - } - - private String matchGroup1(String pattern, String input) { - Pattern pat = Pattern.compile(pattern); - Matcher mat = pat.matcher(input); - boolean foundMatch = mat.find(); - if (foundMatch) { - return mat.group(1); - } - else { - Log.e(TAG, "failed to find pattern \""+pattern+"\" inside of \""+input+"\""); - new Exception("failed to find pattern \""+pattern+"\"").printStackTrace(); - return ""; - } - } - - private int findErrorReason(Document doc) { - errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text(); - if(errorMessage.contains("GEMA")) { - return VideoInfo.ERROR_BLOCKED_BY_GEMA; - } - return VideoInfo.ERROR_NO_SPECIFIED_ERROR; - } -} diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 01f742cad..91afd974c 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -65,7 +65,13 @@ Playing in background https://www.c3s.cc/ Play + Error Network error + Could not load Thumbnails + Could not decrypt video url signature. + Could not parse website. + Content not available. + Blocked by GEMA. Video preview thumbnail