From b0182ed60486083de9fe7d2befee588f723ae736 Mon Sep 17 00:00:00 2001 From: Adam Howard Date: Sat, 14 Nov 2015 11:47:21 +0000 Subject: [PATCH 1/7] halfway through implementing timestamps: * still need to use the start position value stored in the PlayVideoActivity's Intent bundle, to set the VideoView using seekTo() * need to test timestamp extraction regex, and possibly move it somewhere else *need to find a better way to get the startPosition value to ActionBarHandler, which I thought used VideoInfo objects, but apparently doesn't * currently there is a small setStartPosition() method --- .../org/schabi/newpipe/ActionBarHandler.java | 9 ++++++++- .../org/schabi/newpipe/PlayVideoActivity.java | 1 + .../java/org/schabi/newpipe/VideoInfo.java | 1 + .../newpipe/youtube/YoutubeExtractor.java | 19 ++++++++++++++++++- 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/ActionBarHandler.java b/app/src/main/java/org/schabi/newpipe/ActionBarHandler.java index b9bef5cfd..4f28b606d 100644 --- a/app/src/main/java/org/schabi/newpipe/ActionBarHandler.java +++ b/app/src/main/java/org/schabi/newpipe/ActionBarHandler.java @@ -48,6 +48,7 @@ public class ActionBarHandler { private String videoTitle = ""; SharedPreferences defaultPreferences = null; + private int startPosition; class FormatItemSelectListener implements ActionBar.OnNavigationListener { @Override @@ -216,12 +217,18 @@ public class ActionBarHandler { intent.putExtra(PlayVideoActivity.VIDEO_TITLE, videoTitle); intent.putExtra(PlayVideoActivity.STREAM_URL, videoStreams[selectedStream].url); intent.putExtra(PlayVideoActivity.VIDEO_URL, websiteUrl); - activity.startActivity(intent); + intent.putExtra(PlayVideoActivity.START_POSITION, startPosition); + activity.startActivity(intent); //also HERE !!! } } // -------------------------------------------- } + public void setStartPosition(int startPositionSeconds) + { + this.startPosition = startPositionSeconds; + } + public void downloadVideo() { if(!videoTitle.isEmpty()) { String videoSuffix = "." + MediaFormat.getSuffixById(videoStreams[selectedStream].format); diff --git a/app/src/main/java/org/schabi/newpipe/PlayVideoActivity.java b/app/src/main/java/org/schabi/newpipe/PlayVideoActivity.java index d05f721a2..7a691641a 100644 --- a/app/src/main/java/org/schabi/newpipe/PlayVideoActivity.java +++ b/app/src/main/java/org/schabi/newpipe/PlayVideoActivity.java @@ -52,6 +52,7 @@ public class PlayVideoActivity extends AppCompatActivity { public static final String STREAM_URL = "stream_url"; public static final String VIDEO_TITLE = "video_title"; private static final String POSITION = "position"; + public static final String START_POSITION = "start_position"; private static final long HIDING_DELAY = 3000; private static final long TAB_HIDING_DELAY = 100; diff --git a/app/src/main/java/org/schabi/newpipe/VideoInfo.java b/app/src/main/java/org/schabi/newpipe/VideoInfo.java index e0d985d8e..64b21fec6 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoInfo.java +++ b/app/src/main/java/org/schabi/newpipe/VideoInfo.java @@ -50,6 +50,7 @@ public class VideoInfo { public VideoInfoItem nextVideo = null; public VideoInfoItem[] relatedVideos = null; public int videoAvailableStatus = VIDEO_AVAILABLE; + public int startPosition = 0;//in seconds private static final String TAG = VideoInfo.class.toString(); diff --git a/app/src/main/java/org/schabi/newpipe/youtube/YoutubeExtractor.java b/app/src/main/java/org/schabi/newpipe/youtube/YoutubeExtractor.java index 6d23ae038..0b5a20a17 100644 --- a/app/src/main/java/org/schabi/newpipe/youtube/YoutubeExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/youtube/YoutubeExtractor.java @@ -169,6 +169,23 @@ public class YoutubeExtractor implements Extractor { Document doc = Jsoup.parse(site, siteUrl); videoInfo.id = matchGroup1("v=([0-9a-zA-Z_-]{11})", siteUrl); + String timeStamp = matchGroup1("((#|&)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s)", siteUrl); + Log.i(TAG, "time stamp:"+timeStamp); + //videoInfo.startPosition + + //TODO: test this! + if(timeStamp.length() > 0) { + String secondsString = matchGroup1("(\\d{1,3})s", timeStamp); + String minutesString = matchGroup1("(\\d{1,3})m", timeStamp); + String hoursString = matchGroup1("(\\d{1,3})h", timeStamp); + + int seconds = (secondsString.length() > 0 ? Integer.parseInt(secondsString) : 0); + int minutes = (minutesString.length() > 0 ? Integer.parseInt(minutesString) : 0); + int hours = (hoursString.length() > 0 ? Integer.parseInt(hoursString) : 0); + + videoInfo.startPosition = seconds + (60*minutes) + (3600*hours);//don't trust BODMAS! + //the ordering varies internationally + }//else, leave videoInfo.startPosition as default 0 videoInfo.age_limit = 0; videoInfo.webpage_url = siteUrl; @@ -410,7 +427,7 @@ public class YoutubeExtractor implements Extractor { Element img = li.select("img").first(); info.thumbnail_url = img.attr("abs:src"); - // Sometimes youtube sends links to gif files witch somehow seam to not exist + // Sometimes youtube sends links to gif files which somehow sesm to not exist // anymore. Items with such gif also offer a secondary image source. So we are going // to use that if we caught such an item. if(info.thumbnail_url.contains(".gif")) { From 320a4e2351674fa6b19aaa9b8322c0df4469b9c3 Mon Sep 17 00:00:00 2001 From: Adam Howard Date: Mon, 16 Nov 2015 23:32:00 +0000 Subject: [PATCH 2/7] Refactoring: * renamed VideoInfoItem to VideoPreviewInfo * moved streaming service-related classes into their own, new package services * added javadoc to some classes and methods (where functionality is known well enough to explain) * De-duplicated common fields between VideoInfo and VideoPreviewInfo by moving them into a common superclass: AbstractVideoInfo * Removed 2 methods which only call super(), and therefore are unnecessary: PlayVideoActivity.onResume() and PlayVideoActivity.onPostCreate(Bundle) --- .../org/schabi/newpipe/AbstractVideoInfo.java | 15 +++++ .../java/org/schabi/newpipe/Downloader.java | 12 +++- .../java/org/schabi/newpipe/MediaFormat.java | 14 +++++ .../org/schabi/newpipe/PlayVideoActivity.java | 12 +--- .../java/org/schabi/newpipe/VideoInfo.java | 21 ++----- .../newpipe/VideoInfoItemViewCreator.java | 4 +- .../newpipe/VideoItemDetailActivity.java | 7 ++- .../newpipe/VideoItemDetailFragment.java | 8 ++- .../schabi/newpipe/VideoItemListActivity.java | 9 +-- .../schabi/newpipe/VideoItemListFragment.java | 12 ++-- .../org/schabi/newpipe/VideoListAdapter.java | 6 +- ...deoInfoItem.java => VideoPreviewInfo.java} | 31 ++++------ .../newpipe/{ => services}/Extractor.java | 4 +- .../newpipe/{ => services}/SearchEngine.java | 6 +- .../newpipe/{ => services}/ServiceList.java | 6 +- .../{ => services}/StreamingService.java | 2 +- .../youtube/YoutubeExtractor.java | 59 ++++++++++--------- .../youtube/YoutubeSearchEngine.java | 8 +-- .../youtube/YoutubeService.java | 8 +-- 19 files changed, 136 insertions(+), 108 deletions(-) create mode 100644 app/src/main/java/org/schabi/newpipe/AbstractVideoInfo.java rename app/src/main/java/org/schabi/newpipe/{VideoInfoItem.java => VideoPreviewInfo.java} (69%) rename app/src/main/java/org/schabi/newpipe/{ => services}/Extractor.java (86%) rename app/src/main/java/org/schabi/newpipe/{ => services}/SearchEngine.java (88%) rename app/src/main/java/org/schabi/newpipe/{ => services}/ServiceList.java (87%) rename app/src/main/java/org/schabi/newpipe/{ => services}/StreamingService.java (97%) rename app/src/main/java/org/schabi/newpipe/{ => services}/youtube/YoutubeExtractor.java (95%) rename app/src/main/java/org/schabi/newpipe/{ => services}/youtube/YoutubeSearchEngine.java (97%) rename app/src/main/java/org/schabi/newpipe/{ => services}/youtube/YoutubeService.java (88%) diff --git a/app/src/main/java/org/schabi/newpipe/AbstractVideoInfo.java b/app/src/main/java/org/schabi/newpipe/AbstractVideoInfo.java new file mode 100644 index 000000000..1eaa332cd --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/AbstractVideoInfo.java @@ -0,0 +1,15 @@ +package org.schabi.newpipe; + +import android.graphics.Bitmap; + +/**Common properties between VideoInfo and VideoPreviewInfo.*/ +public abstract class AbstractVideoInfo { + public String id = ""; + public String title = ""; + public String uploader = ""; + public String thumbnail_url = ""; + public Bitmap thumbnail = null; + public String webpage_url = ""; + public String upload_date = ""; + public long view_count = 0; +} diff --git a/app/src/main/java/org/schabi/newpipe/Downloader.java b/app/src/main/java/org/schabi/newpipe/Downloader.java index b6d22c44e..9c2fa2948 100644 --- a/app/src/main/java/org/schabi/newpipe/Downloader.java +++ b/app/src/main/java/org/schabi/newpipe/Downloader.java @@ -31,6 +31,11 @@ public class Downloader { private static final String USER_AGENT = "Mozilla/5.0"; + /**Download the text file at the supplied URL as in download(String), + * but set the HTTP header field "Accept-Language" to the supplied string. + * @param siteUrl the URL of the text file to return the contents of + * @param language the language (usually a 2-character code) to set as the preferred language + * @return the contents of the specified text file*/ public static String download(String siteUrl, String language) { String ret = ""; try { @@ -44,7 +49,7 @@ public class Downloader { } return ret; } - + /**Common functionality between download(String url) and download(String url, String language)*/ private static String dl(HttpURLConnection con) { StringBuffer response = new StringBuffer(); @@ -72,7 +77,10 @@ public class Downloader { return response.toString(); } - +/**Download (via HTTP) the text file located at the supplied URL, and return its contents. + * Primarily intended for downloading web pages. + * @param siteUrl the URL of the text file to download + * @return the contents of the specified text file*/ public static String download(String siteUrl) { String ret = ""; diff --git a/app/src/main/java/org/schabi/newpipe/MediaFormat.java b/app/src/main/java/org/schabi/newpipe/MediaFormat.java index 6a37c3fd7..c9af7b8b7 100644 --- a/app/src/main/java/org/schabi/newpipe/MediaFormat.java +++ b/app/src/main/java/org/schabi/newpipe/MediaFormat.java @@ -21,6 +21,8 @@ package org.schabi.newpipe; * You should have received a copy of the GNU General Public License * along with NewPipe. If not, see . */ + +/**Static data about various media formats support by Newpipe, eg mime type, extension*/ public enum MediaFormat { // id name suffix mime type MPEG_4 (0x0, "MPEG-4", "mp4", "video/mp4"), @@ -41,6 +43,10 @@ public enum MediaFormat { this.mimeType = mimeType; } + /**Return the friendly name of the media format with the supplied id + * @param ident the id of the media format. Currently an arbitrary, NewPipe-specific number. + * @return the friendly name of the MediaFormat associated with this ids, + * or an empty String if none match it.*/ public static String getNameById(int ident) { for (MediaFormat vf : MediaFormat.values()) { if(vf.id == ident) return vf.name; @@ -48,6 +54,10 @@ public enum MediaFormat { return ""; } + /**Return the file extension of the media format with the supplied id + * @param ident the id of the media format. Currently an arbitrary, NewPipe-specific number. + * @return the file extension of the MediaFormat associated with this ids, + * or an empty String if none match it.*/ public static String getSuffixById(int ident) { for (MediaFormat vf : MediaFormat.values()) { if(vf.id == ident) return vf.suffix; @@ -55,6 +65,10 @@ public enum MediaFormat { return ""; } + /**Return the MIME type of the media format with the supplied id + * @param ident the id of the media format. Currently an arbitrary, NewPipe-specific number. + * @return the MIME type of the MediaFormat associated with this ids, + * or an empty String if none match it.*/ public static String getMimeById(int ident) { for (MediaFormat vf : MediaFormat.values()) { if(vf.id == ident) return vf.mimeType; diff --git a/app/src/main/java/org/schabi/newpipe/PlayVideoActivity.java b/app/src/main/java/org/schabi/newpipe/PlayVideoActivity.java index 7a691641a..6b90161e9 100644 --- a/app/src/main/java/org/schabi/newpipe/PlayVideoActivity.java +++ b/app/src/main/java/org/schabi/newpipe/PlayVideoActivity.java @@ -89,6 +89,8 @@ public class PlayVideoActivity extends AppCompatActivity { mediaController = new MediaController(this); } + position = intent.getIntExtra(START_POSITION, 0)*1000;//convert from seconds to milliseconds + videoView = (VideoView) findViewById(R.id.video_view); progressBar = (ProgressBar) findViewById(R.id.play_video_progress_bar); try { @@ -146,11 +148,6 @@ public class PlayVideoActivity extends AppCompatActivity { } } - @Override - protected void onPostCreate(Bundle savedInstanceState) { - super.onPostCreate(savedInstanceState); - } - @Override public boolean onCreatePanelMenu(int featured, Menu menu) { super.onCreatePanelMenu(featured, menu); @@ -160,11 +157,6 @@ public class PlayVideoActivity extends AppCompatActivity { return true; } - @Override - public void onResume() { - super.onResume(); - } - @Override public void onPause() { super.onPause(); diff --git a/app/src/main/java/org/schabi/newpipe/VideoInfo.java b/app/src/main/java/org/schabi/newpipe/VideoInfo.java index 64b21fec6..50ea90e4a 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoInfo.java +++ b/app/src/main/java/org/schabi/newpipe/VideoInfo.java @@ -1,10 +1,6 @@ package org.schabi.newpipe; import android.graphics.Bitmap; -import android.util.Log; - -import java.util.Date; -import java.util.Vector; /** * Created by Christian Schabesberger on 26.08.15. @@ -27,16 +23,7 @@ import java.util.Vector; */ /**Info object for opened videos, ie the video ready to play.*/ -public class VideoInfo { - public String id = ""; - public String title = ""; - public String uploader = ""; - public String thumbnail_url = ""; - public Bitmap thumbnail = null; - public String webpage_url = ""; - public String upload_date = ""; - public long view_count = 0; - +public class VideoInfo extends AbstractVideoInfo { public String uploader_thumbnail_url = ""; public Bitmap uploader_thumbnail = null; public String description = ""; @@ -47,10 +34,10 @@ public class VideoInfo { public String average_rating = ""; public VideoStream[] videoStreams = null; public AudioStream[] audioStreams = null; - public VideoInfoItem nextVideo = null; - public VideoInfoItem[] relatedVideos = null; + public VideoPreviewInfo nextVideo = null; + public VideoPreviewInfo[] relatedVideos = null; public int videoAvailableStatus = VIDEO_AVAILABLE; - public int startPosition = 0;//in seconds + //public int startPosition = 0;//in seconds. some metadata is not passed using a VideoInfo object! private static final String TAG = VideoInfo.class.toString(); diff --git a/app/src/main/java/org/schabi/newpipe/VideoInfoItemViewCreator.java b/app/src/main/java/org/schabi/newpipe/VideoInfoItemViewCreator.java index d15d6b86a..b33a66e94 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoInfoItemViewCreator.java +++ b/app/src/main/java/org/schabi/newpipe/VideoInfoItemViewCreator.java @@ -35,7 +35,7 @@ public class VideoInfoItemViewCreator { this.inflater = inflater; } - public View getViewByVideoInfoItem(View convertView, ViewGroup parent, VideoInfoItem info) { + public View getViewByVideoInfoItem(View convertView, ViewGroup parent, VideoPreviewInfo info) { ViewHolder holder; if(convertView == null) { convertView = inflater.inflate(R.layout.video_item, parent, false); @@ -62,7 +62,7 @@ public class VideoInfoItemViewCreator { holder.itemUploadDateView.setText(info.upload_date); } else { //tweak if necessary: This is a hack to prevent having white space in the layout :P - holder.itemUploadDateView.setText(info.view_count); + holder.itemUploadDateView.setText(""+info.view_count); } return convertView; diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemDetailActivity.java b/app/src/main/java/org/schabi/newpipe/VideoItemDetailActivity.java index 2ba09c189..b31fe970b 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemDetailActivity.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemDetailActivity.java @@ -7,10 +7,13 @@ import android.support.v4.app.NavUtils; import android.support.v7.app.AppCompatActivity; import android.util.Log; import android.view.Menu; -import android.view.MenuInflater; import android.view.MenuItem; import android.widget.Toast; +import org.schabi.newpipe.services.Extractor; +import org.schabi.newpipe.services.ServiceList; +import org.schabi.newpipe.services.StreamingService; + /** * Copyright (C) Christian Schabesberger 2015 @@ -81,7 +84,7 @@ public class VideoItemDetailActivity extends AppCompatActivity { .show(); } arguments.putString(VideoItemDetailFragment.VIDEO_URL, - extractor.getVideoUrl(extractor.getVideoId(videoUrl))); + extractor.getVideoUrl(extractor.getVideoId(videoUrl)));//cleans URL arguments.putBoolean(VideoItemDetailFragment.AUTO_PLAY, PreferenceManager.getDefaultSharedPreferences(this) .getBoolean(getString(R.string.autoPlayThroughIntent), false)); diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java index e27d02278..6383a836a 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java @@ -37,6 +37,10 @@ import java.util.Date; import java.util.Locale; import java.util.Vector; +import org.schabi.newpipe.services.Extractor; +import org.schabi.newpipe.services.ServiceList; +import org.schabi.newpipe.services.StreamingService; + /** * Copyright (C) Christian Schabesberger 2015 @@ -394,10 +398,12 @@ public class VideoItemDetailFragment extends Fragment { } } + /**Returns the java.util.Locale object which corresponds to the locale set in NewPipe's preferences. + * Currently not affected by the device's locale.*/ public Locale getPreferredLocale() { SharedPreferences sp = PreferenceManager.getDefaultSharedPreferences(getContext()); String languageKey = getContext().getString(R.string.searchLanguage); - String languageCode = "en";//i know the following lines defaults languageCode to "en", but java is picky about uninitialised values + String languageCode = "en";//i know the following line defaults languageCode to "en", but java is picky about uninitialised values languageCode = sp.getString(languageKey, "en"); if(languageCode.length() == 2) { diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemListActivity.java b/app/src/main/java/org/schabi/newpipe/VideoItemListActivity.java index 90892ca5e..7a89682f3 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemListActivity.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemListActivity.java @@ -3,21 +3,18 @@ package org.schabi.newpipe; import android.content.Context; import android.content.Intent; import android.os.Bundle; -import android.os.Parcel; -import android.os.Parcelable; import android.support.v4.app.NavUtils; import android.support.v7.app.AppCompatActivity; import android.support.v7.widget.SearchView; -import android.util.Log; import android.view.Menu; import android.view.MenuInflater; import android.view.MenuItem; import android.view.View; import android.view.inputmethod.InputMethodManager; -import android.widget.ImageView; import java.util.ArrayList; -import java.util.Arrays; + +import org.schabi.newpipe.services.ServiceList; /** * Copyright (C) Christian Schabesberger 2015 @@ -116,7 +113,7 @@ public class VideoItemListActivity extends AppCompatActivity if(arguments != null) { //Parcelable[] p = arguments.getParcelableArray(VIDEO_INFO_ITEMS); - ArrayList p = arguments.getParcelableArrayList(VIDEO_INFO_ITEMS); + ArrayList p = arguments.getParcelableArrayList(VIDEO_INFO_ITEMS); if(p != null) { mode = PRESENT_VIDEOS_MODE; getSupportActionBar().setDisplayHomeAsUpEnabled(true); diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java index 5bab9669b..2c8cb6de9 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java @@ -15,10 +15,12 @@ import android.widget.ListView; import android.widget.Toast; import java.net.URL; -import java.util.Arrays; import java.util.List; import java.util.Vector; +import org.schabi.newpipe.services.SearchEngine; +import org.schabi.newpipe.services.StreamingService; + /** * Copyright (C) Christian Schabesberger 2015 @@ -119,9 +121,9 @@ public class VideoItemListFragment extends ListFragment { Handler h = new Handler(); private volatile boolean run = true; private int requestId; - public LoadThumbsRunnable(Vector videoList, + public LoadThumbsRunnable(Vector videoList, Vector downloadedList, int requestId) { - for(VideoInfoItem item : videoList) { + for(VideoPreviewInfo item : videoList) { thumbnailUrlList.add(item.thumbnail_url); } this.downloadedList = downloadedList; @@ -168,7 +170,7 @@ public class VideoItemListFragment extends ListFragment { } } - public void present(List videoList) { + public void present(List videoList) { mode = PRESENT_VIDEOS_MODE; setListShown(true); getListView().smoothScrollToPosition(0); @@ -220,7 +222,7 @@ public class VideoItemListFragment extends ListFragment { } } - private void updateList(List list) { + private void updateList(List list) { try { videoListAdapter.addVideoList(list); terminateThreads(); diff --git a/app/src/main/java/org/schabi/newpipe/VideoListAdapter.java b/app/src/main/java/org/schabi/newpipe/VideoListAdapter.java index 87bd3b8aa..6f20a92db 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoListAdapter.java +++ b/app/src/main/java/org/schabi/newpipe/VideoListAdapter.java @@ -37,7 +37,7 @@ public class VideoListAdapter extends BaseAdapter { private Context context; private VideoInfoItemViewCreator viewCreator; - private Vector videoList = new Vector<>(); + private Vector videoList = new Vector<>(); private Vector downloadedThumbnailList = new Vector<>(); VideoItemListFragment videoListFragment; ListView listView; @@ -49,7 +49,7 @@ public class VideoListAdapter extends BaseAdapter { this.context = context; } - public void addVideoList(List videos) { + public void addVideoList(List videos) { videoList.addAll(videos); for(int i = 0; i < videos.size(); i++) { downloadedThumbnailList.add(false); @@ -63,7 +63,7 @@ public class VideoListAdapter extends BaseAdapter { notifyDataSetChanged(); } - public Vector getVideoList() { + public Vector getVideoList() { return videoList; } diff --git a/app/src/main/java/org/schabi/newpipe/VideoInfoItem.java b/app/src/main/java/org/schabi/newpipe/VideoPreviewInfo.java similarity index 69% rename from app/src/main/java/org/schabi/newpipe/VideoInfoItem.java rename to app/src/main/java/org/schabi/newpipe/VideoPreviewInfo.java index 7516723b6..6f644f3b1 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoInfoItem.java +++ b/app/src/main/java/org/schabi/newpipe/VideoPreviewInfo.java @@ -8,7 +8,7 @@ import android.os.Parcelable; * Created by Christian Schabesberger on 26.08.15. * * Copyright (C) Christian Schabesberger 2015 - * VideoInfoItem.java is part of NewPipe. + * VideoPreviewInfo.java is part of NewPipe. * * NewPipe is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -25,19 +25,12 @@ import android.os.Parcelable; */ /**Info object for previews of unopened videos, eg search results, related videos*/ -public class VideoInfoItem implements Parcelable { - public String id = ""; - public String title = ""; - public String uploader = ""; - public String thumbnail_url = ""; - public Bitmap thumbnail = null; - public String webpage_url = ""; - public String upload_date = ""; - public String view_count = ""; +public class VideoPreviewInfo extends AbstractVideoInfo implements Parcelable { + public String duration = ""; - protected VideoInfoItem(Parcel in) { + protected VideoPreviewInfo(Parcel in) { id = in.readString(); title = in.readString(); uploader = in.readString(); @@ -46,10 +39,10 @@ public class VideoInfoItem implements Parcelable { thumbnail = (Bitmap) in.readValue(Bitmap.class.getClassLoader()); webpage_url = in.readString(); upload_date = in.readString(); - view_count = in.readString(); + view_count = in.readLong(); } - public VideoInfoItem() { + public VideoPreviewInfo() { } @@ -68,19 +61,19 @@ public class VideoInfoItem implements Parcelable { dest.writeValue(thumbnail); dest.writeString(webpage_url); dest.writeString(upload_date); - dest.writeString(view_count); + dest.writeLong(view_count); } @SuppressWarnings("unused") - public static final Parcelable.Creator CREATOR = new Parcelable.Creator() { + public static final Parcelable.Creator CREATOR = new Parcelable.Creator() { @Override - public VideoInfoItem createFromParcel(Parcel in) { - return new VideoInfoItem(in); + public VideoPreviewInfo createFromParcel(Parcel in) { + return new VideoPreviewInfo(in); } @Override - public VideoInfoItem[] newArray(int size) { - return new VideoInfoItem[size]; + public VideoPreviewInfo[] newArray(int size) { + return new VideoPreviewInfo[size]; } }; } \ No newline at end of file diff --git a/app/src/main/java/org/schabi/newpipe/Extractor.java b/app/src/main/java/org/schabi/newpipe/services/Extractor.java similarity index 86% rename from app/src/main/java/org/schabi/newpipe/Extractor.java rename to app/src/main/java/org/schabi/newpipe/services/Extractor.java index 102a39494..7f2e18f50 100644 --- a/app/src/main/java/org/schabi/newpipe/Extractor.java +++ b/app/src/main/java/org/schabi/newpipe/services/Extractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe; +package org.schabi.newpipe.services; /** * Created by Christian Schabesberger on 10.08.15. @@ -20,7 +20,9 @@ package org.schabi.newpipe; * along with NewPipe. If not, see . */ +import org.schabi.newpipe.VideoInfo; +/**Scrapes information from a video streaming service (eg, YouTube). To implement*/ public interface Extractor { VideoInfo getVideoInfo(String siteUrl); String getVideoUrl(String videoId); diff --git a/app/src/main/java/org/schabi/newpipe/SearchEngine.java b/app/src/main/java/org/schabi/newpipe/services/SearchEngine.java similarity index 88% rename from app/src/main/java/org/schabi/newpipe/SearchEngine.java rename to app/src/main/java/org/schabi/newpipe/services/SearchEngine.java index 6d5509df7..4feda4edc 100644 --- a/app/src/main/java/org/schabi/newpipe/SearchEngine.java +++ b/app/src/main/java/org/schabi/newpipe/services/SearchEngine.java @@ -1,4 +1,6 @@ -package org.schabi.newpipe; +package org.schabi.newpipe.services; + +import org.schabi.newpipe.VideoPreviewInfo; import java.util.ArrayList; import java.util.Vector; @@ -29,7 +31,7 @@ public interface SearchEngine { class Result { public String errorMessage = ""; public String suggestion = ""; - public Vector resultList = new Vector<>(); + public Vector resultList = new Vector<>(); } ArrayList suggestionList(String query); diff --git a/app/src/main/java/org/schabi/newpipe/ServiceList.java b/app/src/main/java/org/schabi/newpipe/services/ServiceList.java similarity index 87% rename from app/src/main/java/org/schabi/newpipe/ServiceList.java rename to app/src/main/java/org/schabi/newpipe/services/ServiceList.java index cb79c7bcc..f0522a9cd 100644 --- a/app/src/main/java/org/schabi/newpipe/ServiceList.java +++ b/app/src/main/java/org/schabi/newpipe/services/ServiceList.java @@ -1,8 +1,8 @@ -package org.schabi.newpipe; +package org.schabi.newpipe.services; import android.util.Log; -import org.schabi.newpipe.youtube.YoutubeService; +import org.schabi.newpipe.services.youtube.YoutubeService; /** * Created by Christian Schabesberger on 23.08.15. @@ -24,6 +24,8 @@ import org.schabi.newpipe.youtube.YoutubeService; * along with NewPipe. If not, see . */ +/**Provides access to the video streaming services supported by NewPipe. + * Currently only Youtube until the API becomes more stable.*/ public class ServiceList { private static final String TAG = ServiceList.class.toString(); private static final StreamingService[] services = { diff --git a/app/src/main/java/org/schabi/newpipe/StreamingService.java b/app/src/main/java/org/schabi/newpipe/services/StreamingService.java similarity index 97% rename from app/src/main/java/org/schabi/newpipe/StreamingService.java rename to app/src/main/java/org/schabi/newpipe/services/StreamingService.java index 58185a5fe..0e9a022c2 100644 --- a/app/src/main/java/org/schabi/newpipe/StreamingService.java +++ b/app/src/main/java/org/schabi/newpipe/services/StreamingService.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe; +package org.schabi.newpipe.services; /** * Created by Christian Schabesberger on 23.08.15. diff --git a/app/src/main/java/org/schabi/newpipe/youtube/YoutubeExtractor.java b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeExtractor.java similarity index 95% rename from app/src/main/java/org/schabi/newpipe/youtube/YoutubeExtractor.java rename to app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeExtractor.java index 0b5a20a17..fd902c320 100644 --- a/app/src/main/java/org/schabi/newpipe/youtube/YoutubeExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.youtube; +package org.schabi.newpipe.services.youtube; import android.util.Log; import android.util.Xml; @@ -12,14 +12,13 @@ import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; import org.schabi.newpipe.Downloader; -import org.schabi.newpipe.Extractor; +import org.schabi.newpipe.services.Extractor; import org.schabi.newpipe.MediaFormat; import org.schabi.newpipe.VideoInfo; -import org.schabi.newpipe.VideoInfoItem; +import org.schabi.newpipe.VideoPreviewInfo; import org.xmlpull.v1.XmlPullParser; import java.io.StringReader; -import java.net.URI; import java.net.URLDecoder; import java.util.HashMap; import java.util.Map; @@ -161,6 +160,29 @@ public class YoutubeExtractor implements Extractor { return "https://www.youtube.com/watch?v=" + videoId; } + public int getStartPosition(String siteUrl){ + String timeStamp = matchGroup1("((#|&)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", siteUrl); + Log.i(TAG, "time stamp:"+timeStamp); + //videoInfo.startPosition + + //TODO: test this! + if(timeStamp.length() > 0) { + String secondsString = matchGroup1("(\\d{1,3})s", timeStamp); + if(secondsString.length() == 0)//try again with unspecified units as seconds + secondsString = matchGroup1("t=(\\d{1,3})", timeStamp); + String minutesString = matchGroup1("(\\d{1,3})m", timeStamp); + String hoursString = matchGroup1("(\\d{1,3})h", timeStamp); + + int seconds = (secondsString.length() > 0 ? Integer.parseInt(secondsString) : 0); + int minutes = (minutesString.length() > 0 ? Integer.parseInt(minutesString) : 0); + int hours = (hoursString.length() > 0 ? Integer.parseInt(hoursString) : 0); + + return seconds + (60*minutes) + (3600*hours);//don't trust BODMAS! + //the ordering varies internationally + }//else, return default 0 + return 0; + } + @Override public VideoInfo getVideoInfo(String siteUrl) { String site = Downloader.download(siteUrl); @@ -169,23 +191,6 @@ public class YoutubeExtractor implements Extractor { Document doc = Jsoup.parse(site, siteUrl); videoInfo.id = matchGroup1("v=([0-9a-zA-Z_-]{11})", siteUrl); - String timeStamp = matchGroup1("((#|&)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s)", siteUrl); - Log.i(TAG, "time stamp:"+timeStamp); - //videoInfo.startPosition - - //TODO: test this! - if(timeStamp.length() > 0) { - String secondsString = matchGroup1("(\\d{1,3})s", timeStamp); - String minutesString = matchGroup1("(\\d{1,3})m", timeStamp); - String hoursString = matchGroup1("(\\d{1,3})h", timeStamp); - - int seconds = (secondsString.length() > 0 ? Integer.parseInt(secondsString) : 0); - int minutes = (minutesString.length() > 0 ? Integer.parseInt(minutesString) : 0); - int hours = (hoursString.length() > 0 ? Integer.parseInt(hoursString) : 0); - - videoInfo.startPosition = seconds + (60*minutes) + (3600*hours);//don't trust BODMAS! - //the ordering varies internationally - }//else, leave videoInfo.startPosition as default 0 videoInfo.age_limit = 0; videoInfo.webpage_url = siteUrl; @@ -334,14 +339,14 @@ public class YoutubeExtractor implements Extractor { .select("li").first()); // related videos - Vector relatedVideos = new Vector<>(); + Vector relatedVideos = new Vector<>(); for(Element li : doc.select("ul[id=\"watch-related\"]").first().children()) { // first check if we have a playlist. If so leave them out if(li.select("a[class*=\"content-link\"]").first() != null) { relatedVideos.add(extractVideoInfoItem(li)); } } - videoInfo.relatedVideos = relatedVideos.toArray(new VideoInfoItem[relatedVideos.size()]); + videoInfo.relatedVideos = relatedVideos.toArray(new VideoPreviewInfo[relatedVideos.size()]); return videoInfo; } @@ -409,8 +414,8 @@ public class YoutubeExtractor implements Extractor { return audioStreams.toArray(new VideoInfo.AudioStream[audioStreams.size()]); } - private VideoInfoItem extractVideoInfoItem(Element li) { - VideoInfoItem info = new VideoInfoItem(); + private VideoPreviewInfo extractVideoInfoItem(Element li) { + VideoPreviewInfo info = new VideoPreviewInfo(); info.webpage_url = li.select("a[class*=\"content-link\"]").first() .attr("abs:href"); try { @@ -421,13 +426,13 @@ public class YoutubeExtractor implements Extractor { //todo: check NullPointerException causing info.title = li.select("span[class=\"title\"]").first().text(); - info.view_count = li.select("span[class*=\"view-count\"]").first().text(); + info.view_count = Long.parseLong(li.select("span[class*=\"view-count\"]").first().text()); info.uploader = li.select("span[class=\"g-hovercard\"]").first().text(); info.duration = li.select("span[class=\"video-time\"]").first().text(); Element img = li.select("img").first(); info.thumbnail_url = img.attr("abs:src"); - // Sometimes youtube sends links to gif files which somehow sesm to not exist + // Sometimes youtube sends links to gif files which somehow seem to not exist // anymore. Items with such gif also offer a secondary image source. So we are going // to use that if we caught such an item. if(info.thumbnail_url.contains(".gif")) { diff --git a/app/src/main/java/org/schabi/newpipe/youtube/YoutubeSearchEngine.java b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngine.java similarity index 97% rename from app/src/main/java/org/schabi/newpipe/youtube/YoutubeSearchEngine.java rename to app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngine.java index 18bbf4337..74af8cdf7 100644 --- a/app/src/main/java/org/schabi/newpipe/youtube/YoutubeSearchEngine.java +++ b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngine.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.youtube; +package org.schabi.newpipe.services.youtube; import android.net.Uri; import android.util.Log; @@ -7,8 +7,8 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.schabi.newpipe.Downloader; -import org.schabi.newpipe.SearchEngine; -import org.schabi.newpipe.VideoInfoItem; +import org.schabi.newpipe.services.SearchEngine; +import org.schabi.newpipe.VideoPreviewInfo; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.InputSource; @@ -101,7 +101,7 @@ public class YoutubeSearchEngine implements SearchEngine { // video item type } else if(!((el = item.select("div[class*=\"yt-lockup-video\"").first()) == null)) { - VideoInfoItem resultItem = new VideoInfoItem(); + VideoPreviewInfo resultItem = new VideoPreviewInfo(); Element dl = el.select("h3").first().select("a").first(); resultItem.webpage_url = dl.attr("abs:href"); try { diff --git a/app/src/main/java/org/schabi/newpipe/youtube/YoutubeService.java b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeService.java similarity index 88% rename from app/src/main/java/org/schabi/newpipe/youtube/YoutubeService.java rename to app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeService.java index ee33fdd85..284fcbb16 100644 --- a/app/src/main/java/org/schabi/newpipe/youtube/YoutubeService.java +++ b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeService.java @@ -1,8 +1,8 @@ -package org.schabi.newpipe.youtube; +package org.schabi.newpipe.services.youtube; -import org.schabi.newpipe.StreamingService; -import org.schabi.newpipe.Extractor; -import org.schabi.newpipe.SearchEngine; +import org.schabi.newpipe.services.StreamingService; +import org.schabi.newpipe.services.Extractor; +import org.schabi.newpipe.services.SearchEngine; /** From 91f98c125e59de5f36b8d3dfec0fe102501e4a68 Mon Sep 17 00:00:00 2001 From: Adam Howard Date: Tue, 17 Nov 2015 22:51:27 +0000 Subject: [PATCH 3/7] finished implementing timestamp, along with refactoring services * added VideoInfo(AbstractVideoInfo) constructor, to support later implementation for reusing info scraped into VideoPreviewInfo, into VideoInfo * Made the Extractor class behave as a per-video object; - most method return values are video-specific, so it makes sense (to me) to have Extractor be stateful. - The only stateless methods are getVideoUrl(), getVideoId() and loadDecryptionCode(String) * Implemented a constructor for YoutubeExtractor, which performs all initialisation work --- .../org/schabi/newpipe/AbstractVideoInfo.java | 3 +- .../java/org/schabi/newpipe/VideoInfo.java | 64 ++- .../newpipe/VideoInfoItemViewCreator.java | 2 +- .../newpipe/VideoItemDetailActivity.java | 18 +- .../newpipe/VideoItemDetailFragment.java | 13 +- .../org/schabi/newpipe/VideoPreviewInfo.java | 3 - .../schabi/newpipe/services/Extractor.java | 95 +++- .../newpipe/services/StreamingService.java | 2 +- .../services/youtube/YoutubeExtractor.java | 456 +++++++++++------- .../services/youtube/YoutubeSearchEngine.java | 10 +- .../services/youtube/YoutubeService.java | 9 +- 11 files changed, 441 insertions(+), 234 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/AbstractVideoInfo.java b/app/src/main/java/org/schabi/newpipe/AbstractVideoInfo.java index 1eaa332cd..43839b1f0 100644 --- a/app/src/main/java/org/schabi/newpipe/AbstractVideoInfo.java +++ b/app/src/main/java/org/schabi/newpipe/AbstractVideoInfo.java @@ -7,9 +7,10 @@ public abstract class AbstractVideoInfo { public String id = ""; public String title = ""; public String uploader = ""; + //public int duration = -1; public String thumbnail_url = ""; public Bitmap thumbnail = null; public String webpage_url = ""; public String upload_date = ""; - public long view_count = 0; + public long view_count = -1; } diff --git a/app/src/main/java/org/schabi/newpipe/VideoInfo.java b/app/src/main/java/org/schabi/newpipe/VideoInfo.java index 50ea90e4a..bfb90a292 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoInfo.java +++ b/app/src/main/java/org/schabi/newpipe/VideoInfo.java @@ -24,44 +24,76 @@ import android.graphics.Bitmap; /**Info object for opened videos, ie the video ready to play.*/ public class VideoInfo extends AbstractVideoInfo { + private static final String TAG = VideoInfo.class.toString(); + public String uploader_thumbnail_url = ""; public Bitmap uploader_thumbnail = null; public String description = ""; - public int duration = -1; - public int age_limit = 0; - public int like_count = 0; - public int dislike_count = 0; - public String average_rating = ""; public VideoStream[] videoStreams = null; public AudioStream[] audioStreams = null; + public int videoAvailableStatus = VIDEO_AVAILABLE; + public int duration = -1; + + /*YouTube-specific fields + todo: move these to a subclass*/ + public int age_limit = 0; + public int like_count = -1; + public int dislike_count = -1; + public String average_rating = ""; public VideoPreviewInfo nextVideo = null; public VideoPreviewInfo[] relatedVideos = null; - public int videoAvailableStatus = VIDEO_AVAILABLE; - //public int startPosition = 0;//in seconds. some metadata is not passed using a VideoInfo object! - - private static final String TAG = VideoInfo.class.toString(); + public int startPosition = -1;//in seconds. some metadata is not passed using a VideoInfo object! public static final int VIDEO_AVAILABLE = 0x00; public static final int VIDEO_UNAVAILABLE = 0x01; public static final int VIDEO_UNAVAILABLE_GEMA = 0x02;//German DRM organisation - public static class VideoStream { - public VideoStream(String url, int format, String res) { - this.url = url; this.format = format; resolution = res; + + public VideoInfo() {} + + + /**Creates a new VideoInfo object from an existing AbstractVideoInfo. + * All the shared properties are copied to the new VideoInfo.*/ + public VideoInfo(AbstractVideoInfo avi) { + this.id = avi.id; + this.title = avi.title; + this.uploader = avi.uploader; + this.thumbnail_url = avi.thumbnail_url; + this.thumbnail = avi.thumbnail; + this.webpage_url = avi.webpage_url; + this.upload_date = avi.upload_date; + this.upload_date = avi.upload_date; + this.view_count = avi.view_count; + + //todo: better than this + if(avi instanceof VideoPreviewInfo) {//shitty String to convert code + String dur = ((VideoPreviewInfo)avi).duration; + int minutes = Integer.parseInt(dur.substring(0, dur.indexOf(":"))); + int seconds = Integer.parseInt(dur.substring(dur.indexOf(":")+1, dur.length())); + this.duration = (minutes*60)+seconds; } + + } + + public static class VideoStream { public String url = ""; //url of the stream public int format = -1; public String resolution = ""; + + public VideoStream(String url, int format, String res) { + this.url = url; this.format = format; resolution = res; + } } public static class AudioStream { - public AudioStream(String url, int format, int bandwidth, int samplingRate) { - this.url = url; this.format = format; - this.bandwidth = bandwidth; this.samplingRate = samplingRate; - } public String url = ""; public int format = -1; public int bandwidth = -1; public int samplingRate = -1; + + public AudioStream(String url, int format, int bandwidth, int samplingRate) { + this.url = url; this.format = format; + this.bandwidth = bandwidth; this.samplingRate = samplingRate; + } } } \ No newline at end of file diff --git a/app/src/main/java/org/schabi/newpipe/VideoInfoItemViewCreator.java b/app/src/main/java/org/schabi/newpipe/VideoInfoItemViewCreator.java index b33a66e94..19064c211 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoInfoItemViewCreator.java +++ b/app/src/main/java/org/schabi/newpipe/VideoInfoItemViewCreator.java @@ -57,7 +57,7 @@ public class VideoInfoItemViewCreator { } holder.itemVideoTitleView.setText(info.title); holder.itemUploaderView.setText(info.uploader); - holder.itemDurationView.setText(info.duration); + holder.itemDurationView.setText(""+info.duration); if(!info.upload_date.isEmpty()) { holder.itemUploadDateView.setText(info.upload_date); } else { diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemDetailActivity.java b/app/src/main/java/org/schabi/newpipe/VideoItemDetailActivity.java index b31fe970b..0b87b7b7f 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemDetailActivity.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemDetailActivity.java @@ -64,27 +64,25 @@ public class VideoItemDetailActivity extends AppCompatActivity { // this means the video was called though another app if (getIntent().getData() != null) { videoUrl = getIntent().getData().toString(); - Log.i(TAG, "video URL passed:\"" + videoUrl + "\""); + //Log.i(TAG, "video URL passed:\"" + videoUrl + "\""); StreamingService[] serviceList = ServiceList.getServices(); Extractor extractor = null; for (int i = 0; i < serviceList.length; i++) { if (serviceList[i].acceptUrl(videoUrl)) { arguments.putInt(VideoItemDetailFragment.STREAMING_SERVICE, i); - try { - currentStreamingService = i; - extractor = ServiceList.getService(i).getExtractorInstance(); - } catch (Exception e) { - e.printStackTrace(); - } + currentStreamingService = i; + //extractor = ServiceList.getService(i).getExtractorInstance(); break; } } - if(extractor == null) { + if(currentStreamingService == -1) { Toast.makeText(this, R.string.urlNotSupportedText, Toast.LENGTH_LONG) .show(); } - arguments.putString(VideoItemDetailFragment.VIDEO_URL, - extractor.getVideoUrl(extractor.getVideoId(videoUrl)));//cleans URL + //arguments.putString(VideoItemDetailFragment.VIDEO_URL, + // extractor.getVideoUrl(extractor.getVideoId(videoUrl)));//cleans URL + arguments.putString(VideoItemDetailFragment.VIDEO_URL, videoUrl); + arguments.putBoolean(VideoItemDetailFragment.AUTO_PLAY, PreferenceManager.getDefaultSharedPreferences(this) .getBoolean(getString(R.string.autoPlayThroughIntent), false)); diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java index 6383a836a..351692f2a 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java @@ -90,16 +90,18 @@ public class VideoItemDetailFragment extends Fragment { private class ExtractorRunnable implements Runnable { private Handler h = new Handler(); private Extractor extractor; + private StreamingService service; private String videoUrl; - public ExtractorRunnable(String videoUrl, Extractor extractor, VideoItemDetailFragment f) { - this.extractor = extractor; + public ExtractorRunnable(String videoUrl, StreamingService service, VideoItemDetailFragment f) { + this.service = service; this.videoUrl = videoUrl; } @Override public void run() { try { - VideoInfo videoInfo = extractor.getVideoInfo(videoUrl); + this.extractor = service.getExtractorInstance(videoUrl); + VideoInfo videoInfo = extractor.getVideoInfo(); h.post(new VideoResultReturnedRunnable(videoInfo)); if (videoInfo.videoAvailableStatus == VideoInfo.VIDEO_AVAILABLE) { h.post(new SetThumbnailRunnable( @@ -239,7 +241,7 @@ public class VideoItemDetailFragment extends Fragment { //this is horribly convoluted //TODO: find a better way to convert YYYY-MM-DD to a locale-specific date - //suggestions welcome + //suggestions are welcome int year = Integer.parseInt(info.upload_date.substring(0, 4)); int month = Integer.parseInt(info.upload_date.substring(5, 7)); int date = Integer.parseInt(info.upload_date.substring(8, 10)); @@ -255,6 +257,7 @@ public class VideoItemDetailFragment extends Fragment { descriptionView.setMovementMethod(LinkMovementMethod.getInstance()); actionBarHandler.setVideoInfo(info.webpage_url, info.title); + actionBarHandler.setStartPosition(info.startPosition); // parse streams Vector streamsToUse = new Vector<>(); @@ -357,7 +360,7 @@ public class VideoItemDetailFragment extends Fragment { StreamingService streamingService = ServiceList.getService( getArguments().getInt(STREAMING_SERVICE)); extractorThread = new Thread(new ExtractorRunnable( - getArguments().getString(VIDEO_URL), streamingService.getExtractorInstance(), this)); + getArguments().getString(VIDEO_URL), streamingService, this)); autoPlayEnabled = getArguments().getBoolean(AUTO_PLAY); extractorThread.start(); diff --git a/app/src/main/java/org/schabi/newpipe/VideoPreviewInfo.java b/app/src/main/java/org/schabi/newpipe/VideoPreviewInfo.java index 6f644f3b1..f49bd0a0e 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoPreviewInfo.java +++ b/app/src/main/java/org/schabi/newpipe/VideoPreviewInfo.java @@ -26,10 +26,7 @@ import android.os.Parcelable; /**Info object for previews of unopened videos, eg search results, related videos*/ public class VideoPreviewInfo extends AbstractVideoInfo implements Parcelable { - - public String duration = ""; - protected VideoPreviewInfo(Parcel in) { id = in.readString(); title = in.readString(); diff --git a/app/src/main/java/org/schabi/newpipe/services/Extractor.java b/app/src/main/java/org/schabi/newpipe/services/Extractor.java index 7f2e18f50..a38b13f13 100644 --- a/app/src/main/java/org/schabi/newpipe/services/Extractor.java +++ b/app/src/main/java/org/schabi/newpipe/services/Extractor.java @@ -22,9 +22,94 @@ package org.schabi.newpipe.services; import org.schabi.newpipe.VideoInfo; -/**Scrapes information from a video streaming service (eg, YouTube). To implement*/ -public interface Extractor { - VideoInfo getVideoInfo(String siteUrl); - String getVideoUrl(String videoId); - String getVideoId(String videoUrl); +/**Scrapes information from a video streaming service (eg, YouTube).*/ +public abstract class Extractor { + public String pageUrl; + public VideoInfo videoInfo; + + public Extractor(String url) { + this.pageUrl = url; + } + + /**Fills out the video info fields which are common to all services. + * Probably needs to be overridden by subclasses*/ + public VideoInfo getVideoInfo() + { + if(videoInfo == null) { + videoInfo = new VideoInfo(); + } + + if(videoInfo.webpage_url.isEmpty()) { + videoInfo.webpage_url = pageUrl; + } + + if(videoInfo.title.isEmpty()) { + videoInfo.title = getTitle(); + } + + if(videoInfo.duration < 1) { + videoInfo.duration = getLength(); + } + + + if(videoInfo.uploader.isEmpty()) { + videoInfo.uploader = getUploader(); + } + + if(videoInfo.description.isEmpty()) { + videoInfo.description = getDescription(); + } + + if(videoInfo.view_count == -1) { + videoInfo.view_count = getViews(); + } + + if(videoInfo.upload_date.isEmpty()) { + videoInfo.upload_date = getUploadDate(); + } + + if(videoInfo.thumbnail_url.isEmpty()) { + videoInfo.thumbnail_url = getThumbnailUrl(); + } + + if(videoInfo.id.isEmpty()) { + videoInfo.id = getVideoId(pageUrl); + } + + /** Load and extract audio*/ + if(videoInfo.audioStreams == null) { + videoInfo.audioStreams = getAudioStreams(); + } + /** Extract video stream url*/ + if(videoInfo.videoStreams == null) { + videoInfo.videoStreams = getVideoStreams(); + } + + if(videoInfo.uploader_thumbnail_url.isEmpty()) { + videoInfo.uploader_thumbnail_url = getUploaderThumbnailUrl(); + } + + if(videoInfo.startPosition < 0) { + videoInfo.startPosition = getTimeStamp(); + } + + //Bitmap thumbnail = null; + //Bitmap uploader_thumbnail = null; + //int videoAvailableStatus = VIDEO_AVAILABLE; + return videoInfo; + } + + public abstract String getVideoUrl(String videoId); + public abstract String getVideoId(String siteUrl); + public abstract int getTimeStamp(); + public abstract String getTitle(); + public abstract String getDescription(); + public abstract String getUploader(); + public abstract int getLength(); + public abstract int getViews(); + public abstract String getUploadDate(); + public abstract String getThumbnailUrl(); + public abstract String getUploaderThumbnailUrl(); + public abstract VideoInfo.AudioStream[] getAudioStreams(); + public abstract VideoInfo.VideoStream[] getVideoStreams(); } diff --git a/app/src/main/java/org/schabi/newpipe/services/StreamingService.java b/app/src/main/java/org/schabi/newpipe/services/StreamingService.java index 0e9a022c2..4321340c6 100644 --- a/app/src/main/java/org/schabi/newpipe/services/StreamingService.java +++ b/app/src/main/java/org/schabi/newpipe/services/StreamingService.java @@ -25,7 +25,7 @@ public interface StreamingService { public String name = ""; } ServiceInfo getServiceInfo(); - Extractor getExtractorInstance(); + Extractor getExtractorInstance(String url); SearchEngine getSearchEngineInstance(); /**When a VIEW_ACTION is caught this function will test if the url delivered within the calling diff --git a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeExtractor.java b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeExtractor.java index fd902c320..9c47b76cc 100644 --- a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeExtractor.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.services.youtube; import android.util.Log; import android.util.Xml; +import org.json.JSONException; import org.json.JSONObject; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -46,14 +47,225 @@ import java.util.regex.Pattern; * along with NewPipe. If not, see . */ -public class YoutubeExtractor implements Extractor { +public class YoutubeExtractor extends Extractor { private static final String TAG = YoutubeExtractor.class.toString(); + private String pageContents; + private Document doc; + private JSONObject jsonObj; + private JSONObject playerArgs; - // These lists only contain itag formats that are supported by the common Android Video player. - // How ever if you are heading for a list showing all itag formats look at - // https://github.com/rg3/youtube-dl/issues/1687 + // static values + private static final String DECRYPTION_FUNC_NAME="decrypt"; + // cached values + private static volatile String decryptionCode = ""; + + + public YoutubeExtractor(String pageUrl) { + super(pageUrl);//most common videoInfo fields are now set in our superclass, for all services + pageContents = Downloader.download(cleanUrl(pageUrl)); + doc = Jsoup.parse(pageContents, pageUrl); + + //attempt to load the youtube js player JSON arguments + try { + String jsonString = matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContents); + jsonObj = new JSONObject(jsonString); + playerArgs = jsonObj.getJSONObject("args"); + + } catch (Exception e) {//if this fails, the video is most likely not available. + // Determining why is done later. + videoInfo.videoAvailableStatus = VideoInfo.VIDEO_UNAVAILABLE; + Log.d(TAG, "Could not load JSON data for Youtube video \""+pageUrl+"\". This most likely means the video is unavailable"); + } + + //---------------------------------- + // load and parse description code, if it isn't already initialised + //---------------------------------- + if (decryptionCode.isEmpty()) { + try { + // The Youtube service needs to be initialized by downloading the + // js-Youtube-player. This is done in order to get the algorithm + // for decrypting cryptic signatures inside certain stream urls. + JSONObject ytAssets = jsonObj.getJSONObject("assets"); + String playerUrl = ytAssets.getString("js"); + + if (playerUrl.startsWith("//")) { + playerUrl = "https:" + playerUrl; + } + decryptionCode = loadDecryptionCode(playerUrl); + } catch (Exception e){ + Log.d(TAG, "Could not load decryption code for the Youtube service."); + e.printStackTrace(); + } + } + } + + @Override + public String getTitle() { + try {//json player args method + return playerArgs.getString("title"); + } catch(JSONException je) {//html method + je.printStackTrace(); + Log.w(TAG, "failed to load title from JSON args; trying to extract it from HTML"); + } try { // fall through to fall-back + return doc.select("meta[name=title]").attr("content"); + } catch (Exception e) { + Log.e(TAG, "failed permanently to load title."); + e.printStackTrace(); + return ""; + } + } + + @Override + public String getDescription() { + try { + return doc.select("p[id=\"eow-description\"]").first().html(); + } catch (Exception e) {//todo: add fallback method + Log.e(TAG, "failed to load description."); + e.printStackTrace(); + return ""; + } + } + + @Override + public String getUploader() { + try {//json player args method + return playerArgs.getString("author"); + } catch(JSONException je) { + je.printStackTrace(); + Log.w(TAG, "failed to load uploader name from JSON args; trying to extract it from HTML"); + } try {//fall through to fallback HTML method + return doc.select("div.yt-user-info").first().text(); + } catch (Exception e) { + e.printStackTrace(); + Log.e(TAG, "failed permanently to load uploader name."); + return ""; + } + } + + @Override + public int getLength() { + try { + return playerArgs.getInt("length_seconds"); + } catch (JSONException je) {//todo: find fallback method + Log.e(TAG, "failed to load video duration from JSON args"); + je.printStackTrace(); + return -1; + } + } + + @Override + public int getViews() { + try { + String viewCountString = doc.select("meta[itemprop=interactionCount]").attr("content"); + return Integer.parseInt(viewCountString); + } catch (Exception e) {//todo: find fallback method + Log.e(TAG, "failed to number of views"); + e.printStackTrace(); + return -1; + } + } + + @Override + public String getUploadDate() { + try { + return doc.select("meta[itemprop=datePublished]").attr("content"); + } catch (Exception e) {//todo: add fallback method + Log.e(TAG, "failed to get upload date."); + e.printStackTrace(); + return ""; + } + } + + @Override + public String getThumbnailUrl() { + //first attempt getting a small image version + //in the html extracting part we try to get a thumbnail with a higher resolution + // Try to get high resolution thumbnail if it fails use low res from the player instead + try { + return doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href"); + } catch(Exception e) { + Log.w(TAG, "Could not find high res Thumbnail. Using low res instead"); + //fall through to fallback + } try { + return playerArgs.getString("thumbnail_url"); + } catch (JSONException je) { + je.printStackTrace(); + Log.w(TAG, "failed to extract thumbnail URL from JSON args; trying to extract it from HTML"); + return ""; + } + } + + @Override + public String getUploaderThumbnailUrl() { + try { + return doc.select("a[class*=\"yt-user-photo\"]").first() + .select("img").first() + .attr("abs:data-thumb"); + } catch (Exception e) {//todo: add fallback method + Log.e(TAG, "failed to get uploader thumbnail URL."); + e.printStackTrace(); + return ""; + } + } + + @Override + public VideoInfo.AudioStream[] getAudioStreams() { + try { + String dashManifest = playerArgs.getString("dashmpd"); + return parseDashManifest(dashManifest, decryptionCode); + + } catch (NullPointerException e) { + Log.e(TAG, "Could not find \"dashmpd\" upon the player args (maybe no dash manifest available)."); + } catch (Exception e) { + e.printStackTrace(); + } + return new VideoInfo.AudioStream[0]; + } + + @Override + public VideoInfo.VideoStream[] getVideoStreams() { + try{ + //------------------------------------ + // extract video stream url + //------------------------------------ + String encoded_url_map = playerArgs.getString("url_encoded_fmt_stream_map"); + Vector videoStreams = new Vector<>(); + for(String url_data_str : encoded_url_map.split(",")) { + Map tags = new HashMap<>(); + for(String raw_tag : Parser.unescapeEntities(url_data_str, true).split("&")) { + String[] split_tag = raw_tag.split("="); + tags.put(split_tag[0], split_tag[1]); + } + + int itag = Integer.parseInt(tags.get("itag")); + String streamUrl = URLDecoder.decode(tags.get("url"), "UTF-8"); + + // if video has a signature: decrypt it and add it to the url + if(tags.get("s") != null) { + streamUrl = streamUrl + "&signature=" + decryptSignature(tags.get("s"), decryptionCode); + } + + if(resolveFormat(itag) != -1) { + videoStreams.add(new VideoInfo.VideoStream( + streamUrl, + resolveFormat(itag), + resolveResolutionString(itag))); + } + } + return videoStreams.toArray(new VideoInfo.VideoStream[videoStreams.size()]); + + } catch (Exception e) { + Log.e(TAG, "Failed to get video stream"); + e.printStackTrace(); + return new VideoInfo.VideoStream[0]; + } + } + + /**These lists only contain itag formats that are supported by the common Android Video player. + However if you are looking for a list showing all itag formats, look at + https://github.com/rg3/youtube-dl/issues/1687 */ public static int resolveFormat(int itag) { switch(itag) { // video @@ -91,68 +303,28 @@ public class YoutubeExtractor implements Extractor { } } - - // static values - private static final String DECRYPTION_FUNC_NAME="decrypt"; - - // cached values - private static volatile String decryptionCode = ""; - - public void initService(String site) { - // The Youtube service needs to be initialized by downloading the - // js-Youtube-player. This is done in order to get the algorithm - // for decrypting cryptic signatures inside certain stream urls. - - // Star Wars Kid is used as a dummy video, in order to download the youtube player. - //String site = Downloader.download("https://www.youtube.com/watch?v=HPPj6viIBmU"); - //------------------------------------- - // extracting form player args - //------------------------------------- - try { - String jsonString = matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", site); - JSONObject jsonObj = new JSONObject(jsonString); - - //---------------------------------- - // load and parse description code - //---------------------------------- - if (decryptionCode.isEmpty()) { - JSONObject ytAssets = jsonObj.getJSONObject("assets"); - String playerUrl = ytAssets.getString("js"); - if (playerUrl.startsWith("//")) { - playerUrl = "https:" + playerUrl; - } - decryptionCode = loadDecryptionCode(playerUrl); - } - - } catch (Exception e){ - Log.d(TAG, "Could not initialize the extractor of the Youtube service."); - e.printStackTrace(); - } - } - @Override - public String getVideoId(String videoUrl) { - String id = ""; - Pattern pat; + public String getVideoId(String url) { + String id; + String pat; - if(videoUrl.contains("youtube")) { - pat = Pattern.compile("youtube\\.com/watch\\?v=([\\-a-zA-Z0-9_]{11})"); + if(url.contains("youtube")) { + pat = "youtube\\.com/watch\\?v=([\\-a-zA-Z0-9_]{11})"; } - else if(videoUrl.contains("youtu.be")) { - pat = Pattern.compile("youtu\\.be/([a-zA-Z0-9_-]{11})"); + else if(url.contains("youtu.be")) { + pat = "youtu\\.be/([a-zA-Z0-9_-]{11})"; } else { - Log.e(TAG, "Error could not parse url: " + videoUrl); + Log.e(TAG, "Error could not parse url: " + url); return ""; } - Matcher mat = pat.matcher(videoUrl); - boolean foundMatch = mat.find(); - if(foundMatch){ - id = mat.group(1); - Log.i(TAG, "string \""+videoUrl+"\" matches!"); + id = matchGroup1(pat, url); + if(!id.isEmpty()){ + Log.i(TAG, "string \""+url+"\" matches!"); + return id; } - Log.i(TAG, "string \""+videoUrl+"\" does not match."); - return id; + Log.i(TAG, "string \""+url+"\" does not match."); + return ""; } @Override @@ -160,118 +332,47 @@ public class YoutubeExtractor implements Extractor { return "https://www.youtube.com/watch?v=" + videoId; } - public int getStartPosition(String siteUrl){ - String timeStamp = matchGroup1("((#|&)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", siteUrl); - Log.i(TAG, "time stamp:"+timeStamp); - //videoInfo.startPosition + /**Attempts to parse (and return) the offset to start playing the video from. + * @return the offset (in seconds), or 0 if no timestamp is found.*/ + @Override + public int getTimeStamp(){ + String timeStamp = matchGroup1("((#|&)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl); - //TODO: test this! - if(timeStamp.length() > 0) { + //TODO: test this + if(!timeStamp.isEmpty()) { String secondsString = matchGroup1("(\\d{1,3})s", timeStamp); - if(secondsString.length() == 0)//try again with unspecified units as seconds - secondsString = matchGroup1("t=(\\d{1,3})", timeStamp); String minutesString = matchGroup1("(\\d{1,3})m", timeStamp); String hoursString = matchGroup1("(\\d{1,3})h", timeStamp); - int seconds = (secondsString.length() > 0 ? Integer.parseInt(secondsString) : 0); - int minutes = (minutesString.length() > 0 ? Integer.parseInt(minutesString) : 0); - int hours = (hoursString.length() > 0 ? Integer.parseInt(hoursString) : 0); + if(secondsString.isEmpty()//if nothing was got, + && minutesString.isEmpty()//treat as unlabelled seconds + && hoursString.isEmpty()) + secondsString = matchGroup1("t=(\\d{1,3})", timeStamp); - return seconds + (60*minutes) + (3600*hours);//don't trust BODMAS! + int seconds = (secondsString.isEmpty() ? 0 : Integer.parseInt(secondsString)); + int minutes = (minutesString.isEmpty() ? 0 : Integer.parseInt(minutesString)); + int hours = (hoursString.isEmpty() ? 0 : Integer.parseInt(hoursString)); + + int ret = seconds + (60*minutes) + (3600*hours);//don't trust BODMAS! + Log.d(TAG, "derived timestamp value:"+ret); + return ret; //the ordering varies internationally }//else, return default 0 return 0; } @Override - public VideoInfo getVideoInfo(String siteUrl) { - String site = Downloader.download(siteUrl); - VideoInfo videoInfo = new VideoInfo(); - - Document doc = Jsoup.parse(site, siteUrl); - - videoInfo.id = matchGroup1("v=([0-9a-zA-Z_-]{11})", siteUrl); + public VideoInfo getVideoInfo() { + videoInfo = super.getVideoInfo(); + //todo: replace this with a call to getVideoId, if possible + videoInfo.id = matchGroup1("v=([0-9a-zA-Z_-]{11})", pageUrl); videoInfo.age_limit = 0; - videoInfo.webpage_url = siteUrl; - initService(site); - - //------------------------------------- - // extracting form player args - //------------------------------------- - JSONObject playerArgs = null; - { - try { - String jsonString = matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", site); - JSONObject jsonObj = new JSONObject(jsonString); - playerArgs = jsonObj.getJSONObject("args"); - } - catch (Exception e) { - e.printStackTrace(); - // If we fail in this part the video is most likely not available. - // Determining why is done later. - videoInfo.videoAvailableStatus = VideoInfo.VIDEO_UNAVAILABLE; - } - } - - //----------------------- - // load and extract audio - //----------------------- + //average rating try { - String dashManifest = playerArgs.getString("dashmpd"); - videoInfo.audioStreams = parseDashManifest(dashManifest, decryptionCode); - - } catch (NullPointerException e) { - Log.e(TAG, "Could not find \"dashmpd\" upon the player args (maybe no dash manifest available)."); - } catch (Exception e) { - e.printStackTrace(); - } - - try { - //-------------------------------------------- - // extract general information about the video - //-------------------------------------------- - - videoInfo.uploader = playerArgs.getString("author"); - videoInfo.title = playerArgs.getString("title"); - //first attempt getting a small image version - //in the html extracting part we try to get a thumbnail with a higher resolution - videoInfo.thumbnail_url = playerArgs.getString("thumbnail_url"); - videoInfo.duration = playerArgs.getInt("length_seconds"); videoInfo.average_rating = playerArgs.getString("avg_rating"); - - //------------------------------------ - // extract video stream url - //------------------------------------ - String encoded_url_map = playerArgs.getString("url_encoded_fmt_stream_map"); - Vector videoStreams = new Vector<>(); - for(String url_data_str : encoded_url_map.split(",")) { - Map tags = new HashMap<>(); - for(String raw_tag : Parser.unescapeEntities(url_data_str, true).split("&")) { - String[] split_tag = raw_tag.split("="); - tags.put(split_tag[0], split_tag[1]); - } - - int itag = Integer.parseInt(tags.get("itag")); - String streamUrl = URLDecoder.decode(tags.get("url"), "UTF-8"); - - // if video has a signature: decrypt it and add it to the url - if(tags.get("s") != null) { - streamUrl = streamUrl + "&signature=" + decryptSignature(tags.get("s"), decryptionCode); - } - - if(resolveFormat(itag) != -1) { - videoStreams.add(new VideoInfo.VideoStream( - streamUrl, - resolveFormat(itag), - resolveResolutionString(itag))); - } - } - videoInfo.videoStreams = - videoStreams.toArray(new VideoInfo.VideoStream[videoStreams.size()]); - - } catch (Exception e) { + } catch (JSONException e) { e.printStackTrace(); } @@ -279,7 +380,6 @@ public class YoutubeExtractor implements Extractor { // extracting information from html page //--------------------------------------- - // Determine what went wrong when the Video is not available if(videoInfo.videoAvailableStatus == VideoInfo.VIDEO_UNAVAILABLE) { if(doc.select("h1[id=\"unavailable-message\"]").first().text().contains("GEMA")) { @@ -287,22 +387,6 @@ public class YoutubeExtractor implements Extractor { } } - // Try to get high resolution thumbnail if it fails use low res from the player instead - try { - videoInfo.thumbnail_url = doc.select("link[itemprop=\"thumbnailUrl\"]").first() - .attr("abs:href"); - } catch(Exception e) { - Log.i(TAG, "Could not find high res Thumbnail. Using low res instead"); - } - - // upload date - videoInfo.upload_date = doc.select("meta[itemprop=datePublished]").attr("content"); - - //TODO: Format date locale-specifically - - - // description - videoInfo.description = doc.select("p[id=\"eow-description\"]").first().html(); String likesString = ""; String dislikesString = ""; try { @@ -325,17 +409,8 @@ public class YoutubeExtractor implements Extractor { videoInfo.dislike_count = 0; } - // uploader thumbnail - videoInfo.uploader_thumbnail_url = doc.select("a[class*=\"yt-user-photo\"]").first() - .select("img").first() - .attr("abs:data-thumb"); - - // view count TODO: locale-specific formatting - String viewCountString = doc.select("meta[itemprop=interactionCount]").attr("content"); - videoInfo.view_count = Integer.parseInt(viewCountString); - // next video - videoInfo.nextVideo = extractVideoInfoItem(doc.select("div[class=\"watch-sidebar-section\"]").first() + videoInfo.nextVideo = extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first() .select("li").first()); // related videos @@ -343,13 +418,14 @@ public class YoutubeExtractor implements Extractor { for(Element li : doc.select("ul[id=\"watch-related\"]").first().children()) { // first check if we have a playlist. If so leave them out if(li.select("a[class*=\"content-link\"]").first() != null) { - relatedVideos.add(extractVideoInfoItem(li)); + relatedVideos.add(extractVideoPreviewInfo(li)); } } videoInfo.relatedVideos = relatedVideos.toArray(new VideoPreviewInfo[relatedVideos.size()]); return videoInfo; } + private VideoInfo.AudioStream[] parseDashManifest(String dashManifest, String decryptoinCode) { if(!dashManifest.contains("/signature/")) { String encryptedSig = matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifest); @@ -413,8 +489,10 @@ public class YoutubeExtractor implements Extractor { } return audioStreams.toArray(new VideoInfo.AudioStream[audioStreams.size()]); } - - private VideoPreviewInfo extractVideoInfoItem(Element li) { + /**Provides information about links to other videos on the video page, such as related videos. + * This is encapsulated in a VideoPreviewInfo object, + * which is a subset of the fields in a full VideoInfo.*/ + private VideoPreviewInfo extractVideoPreviewInfo(Element li) { VideoPreviewInfo info = new VideoPreviewInfo(); info.webpage_url = li.select("a[class*=\"content-link\"]").first() .attr("abs:href"); @@ -426,8 +504,10 @@ public class YoutubeExtractor implements Extractor { //todo: check NullPointerException causing info.title = li.select("span[class=\"title\"]").first().text(); - info.view_count = Long.parseLong(li.select("span[class*=\"view-count\"]").first().text()); + info.view_count = Long.parseLong(li.select("span[class*=\"view-count\"]") + .first().text().replaceAll("[^\\d]", "")); info.uploader = li.select("span[class=\"g-hovercard\"]").first().text(); + info.duration = li.select("span[class=\"video-time\"]").first().text(); Element img = li.select("img").first(); @@ -491,15 +571,19 @@ public class YoutubeExtractor implements Extractor { return result.toString(); } + private String cleanUrl(String complexUrl) { + return getVideoUrl(getVideoId(complexUrl)); + } + private String matchGroup1(String pattern, String input) { Pattern pat = Pattern.compile(pattern); Matcher mat = pat.matcher(input); boolean foundMatch = mat.find(); - if(foundMatch){ + if (foundMatch) { return mat.group(1); } else { - Log.e(TAG, "failed to find pattern \""+pattern+"\" inside of \""+input+"\""); + Log.w(TAG, "failed to find pattern \""+pattern+"\" inside of \""+input+"\""); new Exception("failed to find pattern \""+pattern+"\"").printStackTrace(); return ""; } diff --git a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngine.java b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngine.java index 74af8cdf7..d01718ed2 100644 --- a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngine.java +++ b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngine.java @@ -62,7 +62,7 @@ public class YoutubeSearchEngine implements SearchEngine { String site; String url = builder.build().toString(); //if we've been passed a valid language code, append it to the URL - if(languageCode.length() > 0) { + if(!languageCode.isEmpty()) { //assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode); site = Downloader.download(url, languageCode); } @@ -101,6 +101,7 @@ public class YoutubeSearchEngine implements SearchEngine { // video item type } else if(!((el = item.select("div[class*=\"yt-lockup-video\"").first()) == null)) { + //todo: de-duplicate this with YoutubeExtractor.getVideoPreviewInfo() VideoPreviewInfo resultItem = new VideoPreviewInfo(); Element dl = el.select("h3").first().select("a").first(); resultItem.webpage_url = dl.attr("abs:href"); @@ -113,8 +114,9 @@ public class YoutubeSearchEngine implements SearchEngine { e.printStackTrace(); } resultItem.title = dl.text(); - resultItem.duration = item.select("span[class=\"video-time\"]").first() - .text(); + + resultItem.duration = item.select("span[class=\"video-time\"]").first().text(); + resultItem.uploader = item.select("div[class=\"yt-lockup-byline\"]").first() .select("a").first() .text(); @@ -132,7 +134,7 @@ public class YoutubeSearchEngine implements SearchEngine { } result.resultList.add(resultItem); } else { - Log.e(TAG, "GREAT FUCKING ERROR"); + Log.e(TAG, "unexpected element found:\""+el+"\""); } } return result; diff --git a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeService.java b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeService.java index 284fcbb16..e606c805d 100644 --- a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeService.java +++ b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeService.java @@ -33,8 +33,13 @@ public class YoutubeService implements StreamingService { return serviceInfo; } @Override - public Extractor getExtractorInstance() { - return new YoutubeExtractor(); + public Extractor getExtractorInstance(String url) { + if(acceptUrl(url)) { + return new YoutubeExtractor(url); + } + else { + throw new IllegalArgumentException("supplied String is not a valid Youtube URL"); + } } @Override public SearchEngine getSearchEngineInstance() { From 23e0196fcc987a0188149ce0123f5b82699a2477 Mon Sep 17 00:00:00 2001 From: Adam Howard Date: Thu, 19 Nov 2015 00:08:51 +0000 Subject: [PATCH 4/7] * fixed counterintuitive back button behaviour in PlayVideoActivity; see https://github.com/theScrabi/NewPipe/issues/99 * fixed a rarely-caused NullPointerException caused by a related video's view-count field being missing --- .../org/schabi/newpipe/PlayVideoActivity.java | 26 ++++++++++++++++++- .../services/youtube/YoutubeExtractor.java | 21 ++++++++++----- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/PlayVideoActivity.java b/app/src/main/java/org/schabi/newpipe/PlayVideoActivity.java index 6b90161e9..cf1593903 100644 --- a/app/src/main/java/org/schabi/newpipe/PlayVideoActivity.java +++ b/app/src/main/java/org/schabi/newpipe/PlayVideoActivity.java @@ -15,6 +15,7 @@ import android.support.v7.app.AppCompatActivity; import android.util.DisplayMetrics; import android.util.Log; import android.view.Display; +import android.view.KeyEvent; import android.view.Menu; import android.view.MenuInflater; import android.view.MenuItem; @@ -86,7 +87,30 @@ public class PlayVideoActivity extends AppCompatActivity { actionBar.setDisplayHomeAsUpEnabled(true); Intent intent = getIntent(); if(mediaController == null) { - mediaController = new MediaController(this); + //prevents back button hiding media controller controls (after showing them) + //instead of exiting video + //see http://stackoverflow.com/questions/6051825 + //also solves https://github.com/theScrabi/NewPipe/issues/99 + mediaController = new MediaController(this) { + @Override + public boolean dispatchKeyEvent(KeyEvent event) { + int keyCode = event.getKeyCode(); + final boolean uniqueDown = event.getRepeatCount() == 0 + && event.getAction() == KeyEvent.ACTION_DOWN; + if (keyCode == KeyEvent.KEYCODE_BACK) { + if (uniqueDown) + { + if (isShowing()) { + finish(); + } else { + hide(); + } + } + return true; + } + return super.dispatchKeyEvent(event); + } + }; } position = intent.getIntExtra(START_POSITION, 0)*1000;//convert from seconds to milliseconds diff --git a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeExtractor.java b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeExtractor.java index 9c47b76cc..c3ebe3a40 100644 --- a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeExtractor.java @@ -494,7 +494,7 @@ public class YoutubeExtractor extends Extractor { * which is a subset of the fields in a full VideoInfo.*/ private VideoPreviewInfo extractVideoPreviewInfo(Element li) { VideoPreviewInfo info = new VideoPreviewInfo(); - info.webpage_url = li.select("a[class*=\"content-link\"]").first() + info.webpage_url = li.select("a.content-link").first() .attr("abs:href"); try { info.id = matchGroup1("v=([0-9a-zA-Z-]*)", info.webpage_url); @@ -503,12 +503,21 @@ public class YoutubeExtractor extends Extractor { } //todo: check NullPointerException causing - info.title = li.select("span[class=\"title\"]").first().text(); - info.view_count = Long.parseLong(li.select("span[class*=\"view-count\"]") - .first().text().replaceAll("[^\\d]", "")); - info.uploader = li.select("span[class=\"g-hovercard\"]").first().text(); + info.title = li.select("span.title").first().text(); + //this page causes the NullPointerException, after finding it by searching for "tjvg": + //https://www.youtube.com/watch?v=Uqg0aEhLFAg + String views = li.select("span.view-count").first().text(); + Log.i(TAG, "title:"+info.title); + Log.i(TAG, "view count:"+views); + try { + info.view_count = Long.parseLong(li.select("span.view-count") + .first().text().replaceAll("[^\\d]", "")); + } catch (NullPointerException e) {//related videos sometimes have no view count + info.view_count = 0; + } + info.uploader = li.select("span.g-hovercard").first().text(); - info.duration = li.select("span[class=\"video-time\"]").first().text(); + info.duration = li.select("span.video-time").first().text(); Element img = li.select("img").first(); info.thumbnail_url = img.attr("abs:src"); From 2c11bd188998715aefa6f56bcc5c4c190f19ffaa Mon Sep 17 00:00:00 2001 From: Adam Howard Date: Thu, 19 Nov 2015 14:40:35 +0000 Subject: [PATCH 5/7] fixed Bundle casting bug incurred during related videosdisplay --- app/src/main/java/org/schabi/newpipe/VideoInfo.java | 4 +++- .../java/org/schabi/newpipe/VideoItemDetailFragment.java | 8 +++++++- .../schabi/newpipe/services/youtube/YoutubeExtractor.java | 4 +++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/VideoInfo.java b/app/src/main/java/org/schabi/newpipe/VideoInfo.java index bfb90a292..5b8321a61 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoInfo.java +++ b/app/src/main/java/org/schabi/newpipe/VideoInfo.java @@ -2,6 +2,8 @@ package org.schabi.newpipe; import android.graphics.Bitmap; +import java.util.List; + /** * Created by Christian Schabesberger on 26.08.15. * @@ -41,7 +43,7 @@ public class VideoInfo extends AbstractVideoInfo { public int dislike_count = -1; public String average_rating = ""; public VideoPreviewInfo nextVideo = null; - public VideoPreviewInfo[] relatedVideos = null; + public List relatedVideos = null; public int startPosition = -1;//in seconds. some metadata is not passed using a VideoInfo object! public static final int VIDEO_AVAILABLE = 0x00; diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java index 351692f2a..81636d825 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java @@ -32,6 +32,7 @@ import android.view.MenuItem; import java.net.URL; import java.text.DateFormat; import java.text.NumberFormat; +import java.util.ArrayList; import java.util.Calendar; import java.util.Date; import java.util.Locale; @@ -394,7 +395,12 @@ public class VideoItemDetailFragment extends Fragment { @Override public void onClick(View v) { Intent intent = new Intent(activity, VideoItemListActivity.class); - intent.putExtra(VideoItemListActivity.VIDEO_INFO_ITEMS, currentVideoInfo.relatedVideos); + //todo: find more elegant way to do this - converting from List to ArrayList sucks + ArrayList toParcel = new ArrayList<>(currentVideoInfo.relatedVideos); + //why oh why does the parcelable array put method have to be so damn specific + // about the class of its argument? + //why not a List? + intent.putParcelableArrayListExtra(VideoItemListActivity.VIDEO_INFO_ITEMS, toParcel); activity.startActivity(intent); } }); diff --git a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeExtractor.java b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeExtractor.java index c3ebe3a40..cb996d2ef 100644 --- a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeExtractor.java @@ -421,7 +421,9 @@ public class YoutubeExtractor extends Extractor { relatedVideos.add(extractVideoPreviewInfo(li)); } } - videoInfo.relatedVideos = relatedVideos.toArray(new VideoPreviewInfo[relatedVideos.size()]); + //todo: replace conversion + videoInfo.relatedVideos = relatedVideos; + //videoInfo.relatedVideos = relatedVideos.toArray(new VideoPreviewInfo[relatedVideos.size()]); return videoInfo; } From 0392bf6a0278b679fdd4d0d493e6689702c71fb9 Mon Sep 17 00:00:00 2001 From: chschtsch Date: Fri, 20 Nov 2015 00:49:52 +0300 Subject: [PATCH 6/7] fix bug with upload date + better way to parse it --- .../newpipe/VideoItemDetailFragment.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java index 81636d825..4be7d74f5 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java @@ -32,6 +32,8 @@ import android.view.MenuItem; import java.net.URL; import java.text.DateFormat; import java.text.NumberFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; import java.util.Date; @@ -240,15 +242,14 @@ public class VideoItemDetailFragment extends Fragment { thumbsUpView.setText(nf.format(info.like_count)); thumbsDownView.setText(nf.format(info.dislike_count)); - //this is horribly convoluted - //TODO: find a better way to convert YYYY-MM-DD to a locale-specific date - //suggestions are welcome - int year = Integer.parseInt(info.upload_date.substring(0, 4)); - int month = Integer.parseInt(info.upload_date.substring(5, 7)); - int date = Integer.parseInt(info.upload_date.substring(8, 10)); - Calendar cal = Calendar.getInstance(); - cal.set(year, month, date); - Date datum = cal.getTime(); + SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); + Date datum = null; + try { + datum = formatter.parse(info.upload_date); + } catch (ParseException e) { + e.printStackTrace(); + } + DateFormat df = DateFormat.getDateInstance(DateFormat.MEDIUM, locale); String localisedDate = df.format(datum); From fa1d386fccec4491006bb1faf62ba9ce13b82810 Mon Sep 17 00:00:00 2001 From: chschtsch Date: Fri, 20 Nov 2015 00:52:43 +0300 Subject: [PATCH 7/7] cleanup --- .../main/java/org/schabi/newpipe/VideoItemDetailFragment.java | 1 - 1 file changed, 1 deletion(-) diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java index 4be7d74f5..c883e7f8d 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java @@ -35,7 +35,6 @@ import java.text.NumberFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Calendar; import java.util.Date; import java.util.Locale; import java.util.Vector;