From 53059bcb91aece69f14ee2b9690c9d064bfe2459 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sat, 10 Sep 2016 16:26:21 +0200 Subject: [PATCH] level 1 of making loading more content work --- app/src/main/AndroidManifest.xml | 2 +- .../org/schabi/newpipe/ChannelActivity.java | 29 +-- .../java/org/schabi/newpipe/Downloader.java | 24 ++- .../newpipe/extractor/ChannelExtractor.java | 3 +- .../schabi/newpipe/extractor/ChannelInfo.java | 2 + .../schabi/newpipe/extractor/Downloader.java | 9 + .../newpipe/extractor/StreamingService.java | 2 +- .../youtube/YoutubeChannelExtractor.java | 176 ++++++++++++------ .../services/youtube/YoutubeService.java | 4 +- .../newpipe/info_list/InfoListAdapter.java | 6 +- .../SearchInfoItemFragment.java | 2 +- 11 files changed, 181 insertions(+), 78 deletions(-) diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index f2bb0e715..c2716efb3 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -32,7 +32,7 @@ android:theme="@style/AppTheme"> + android:value=".MainActivity" /> diff --git a/app/src/main/java/org/schabi/newpipe/ChannelActivity.java b/app/src/main/java/org/schabi/newpipe/ChannelActivity.java index 80b75e61f..48afa06df 100644 --- a/app/src/main/java/org/schabi/newpipe/ChannelActivity.java +++ b/app/src/main/java/org/schabi/newpipe/ChannelActivity.java @@ -11,10 +11,8 @@ import android.support.v7.widget.LinearLayoutManager; import android.support.v7.widget.RecyclerView; import android.support.v7.widget.Toolbar; import android.util.Log; -import android.view.LayoutInflater; import android.view.View; import android.widget.ImageView; -import android.widget.LinearLayout; import android.widget.ProgressBar; import android.widget.Toast; @@ -27,12 +25,10 @@ import org.schabi.newpipe.extractor.ChannelInfo; import org.schabi.newpipe.extractor.ExtractionException; import org.schabi.newpipe.extractor.ParsingException; import org.schabi.newpipe.extractor.ServiceList; -import org.schabi.newpipe.extractor.StreamPreviewInfo; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.info_list.InfoListAdapter; import java.io.IOException; -import java.util.ArrayList; public class ChannelActivity extends AppCompatActivity { @@ -47,6 +43,7 @@ public class ChannelActivity extends AppCompatActivity { private int serviceId = -1; private String channelUrl = ""; private int pageNumber = 0; + private boolean hasNextPage = true; private boolean isLoading = false; private ImageLoader imageLoader = ImageLoader.getInstance(); @@ -91,22 +88,23 @@ public class ChannelActivity extends AppCompatActivity { totalItemCount = layoutManager.getItemCount(); pastVisiblesItems = layoutManager.findFirstVisibleItemPosition(); - if ( (visibleItemCount + pastVisiblesItems) >= totalItemCount && !isLoading) + if ( (visibleItemCount + pastVisiblesItems) >= totalItemCount + && !isLoading + && hasNextPage) { pageNumber++; - Log.d(TAG, "bottomn"); + requestData(true); } } } }); - requestData(pageNumber); + requestData(false); } private void updateUi(final ChannelInfo info) { - isLoading = false; CollapsingToolbarLayout ctl = (CollapsingToolbarLayout) findViewById(R.id.channel_toolbar_layout); ProgressBar progressBar = (ProgressBar) findViewById(R.id.progressBar); ImageView channelBanner = (ImageView) findViewById(R.id.channel_banner_image); @@ -144,11 +142,9 @@ public class ChannelActivity extends AppCompatActivity { } else { feedButton.setVisibility(View.GONE); } - - initVideos(info); } - private void initVideos(final ChannelInfo info) { + private void addVideos(final ChannelInfo info) { infoListAdapter.addStreamItemList(info.related_streams); } @@ -162,7 +158,7 @@ public class ChannelActivity extends AppCompatActivity { }); } - private void requestData(int page) { + private void requestData(final boolean onlyVideos) { // start processing isLoading = true; Thread channelExtractorThread = new Thread(new Runnable() { @@ -173,7 +169,7 @@ public class ChannelActivity extends AppCompatActivity { try { StreamingService service = ServiceList.getService(serviceId); ChannelExtractor extractor = service.getChannelExtractorInstance( - channelUrl, new Downloader()); + channelUrl, pageNumber, new Downloader()); final ChannelInfo info = ChannelInfo.getInfo(extractor, new Downloader()); @@ -181,7 +177,12 @@ public class ChannelActivity extends AppCompatActivity { h.post(new Runnable() { @Override public void run() { - updateUi(info); + isLoading = false; + if(!onlyVideos) { + updateUi(info); + } + hasNextPage = info.hasNextPage; + addVideos(info); } }); diff --git a/app/src/main/java/org/schabi/newpipe/Downloader.java b/app/src/main/java/org/schabi/newpipe/Downloader.java index 833891762..04098a4d7 100644 --- a/app/src/main/java/org/schabi/newpipe/Downloader.java +++ b/app/src/main/java/org/schabi/newpipe/Downloader.java @@ -5,10 +5,12 @@ import java.io.IOException; import java.io.InputStreamReader; import java.net.URL; import java.net.UnknownHostException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; import javax.net.ssl.HttpsURLConnection; -import info.guardianproject.netcipher.NetCipher; /** * Created by Christian Schabesberger on 28.01.16. @@ -40,10 +42,26 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader { * @param language the language (usually a 2-character code) to set as the preferred language * @return the contents of the specified text file*/ public String download(String siteUrl, String language) throws IOException { + Map requestProperties = new HashMap<>(); + requestProperties.put("Accept-Language", language); + return download(siteUrl, requestProperties); + } + + + /**Download the text file at the supplied URL as in download(String), + * but set the HTTP header field "Accept-Language" to the supplied string. + * @param siteUrl the URL of the text file to return the contents of + * @param customProperties set request header properties + * @return the contents of the specified text file + * @throws IOException*/ + public String download(String siteUrl, Map customProperties) throws IOException { URL url = new URL(siteUrl); HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); - //HttpsURLConnection con = NetCipher.getHttpsURLConnection(url); - con.setRequestProperty("Accept-Language", language); + Iterator it = customProperties.entrySet().iterator(); + while(it.hasNext()) { + Map.Entry pair = (Map.Entry)it.next(); + con.setRequestProperty((String)pair.getKey(), (String)pair.getValue()); + } return dl(con); } diff --git a/app/src/main/java/org/schabi/newpipe/extractor/ChannelExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/ChannelExtractor.java index bf6292bbf..d8c931d08 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/ChannelExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/ChannelExtractor.java @@ -29,7 +29,7 @@ public abstract class ChannelExtractor { private Downloader downloader; private StreamPreviewInfoCollector previewInfoCollector; - public ChannelExtractor(UrlIdHandler urlIdHandler, String url, Downloader dl, int serviceId) + public ChannelExtractor(UrlIdHandler urlIdHandler, String url, int page, Downloader dl, int serviceId) throws ExtractionException, IOException { this.serviceId = serviceId; this.urlIdHandler = urlIdHandler; @@ -48,6 +48,7 @@ public abstract class ChannelExtractor { public abstract String getBannerUrl() throws ParsingException; public abstract String getFeedUrl() throws ParsingException; public abstract StreamPreviewInfoCollector getStreams() throws ParsingException; + public abstract boolean hasNextPage() throws ParsingException; public int getServiceId() { return serviceId; } diff --git a/app/src/main/java/org/schabi/newpipe/extractor/ChannelInfo.java b/app/src/main/java/org/schabi/newpipe/extractor/ChannelInfo.java index 0151d895a..2f856dac0 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/ChannelInfo.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/ChannelInfo.java @@ -39,6 +39,7 @@ public class ChannelInfo { // importand data info.service_id = extractor.getServiceId(); info.channel_name = extractor.getChannelName(); + info.hasNextPage = extractor.hasNextPage(); try { info.avatar_url = extractor.getAvatarUrl(); @@ -72,6 +73,7 @@ public class ChannelInfo { public String banner_url = ""; public String feed_url = ""; public List related_streams = null; + public boolean hasNextPage = false; public List errors = new Vector<>(); } diff --git a/app/src/main/java/org/schabi/newpipe/extractor/Downloader.java b/app/src/main/java/org/schabi/newpipe/extractor/Downloader.java index a4cde400d..14475dba7 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/Downloader.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/Downloader.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor; import java.io.IOException; +import java.util.Map; /** * Created by Christian Schabesberger on 28.01.16. @@ -32,6 +33,14 @@ public interface Downloader { * @throws IOException*/ String download(String siteUrl, String language) throws IOException; + /**Download the text file at the supplied URL as in download(String), + * but set the HTTP header field "Accept-Language" to the supplied string. + * @param siteUrl the URL of the text file to return the contents of + * @param customProperties set request header properties + * @return the contents of the specified text file + * @throws IOException*/ + String download(String siteUrl, Map customProperties) throws IOException; + /**Download (via HTTP) the text file located at the supplied URL, and return its contents. * Primarily intended for downloading web pages. * @param siteUrl the URL of the text file to download diff --git a/app/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/app/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index d9acdefff..1805b48db 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -40,7 +40,7 @@ public abstract class StreamingService { public abstract SearchEngine getSearchEngineInstance(Downloader downloader); public abstract UrlIdHandler getUrlIdHandlerInstance(); public abstract UrlIdHandler getChannelUrlIdHandlerInstance(); - public abstract ChannelExtractor getChannelExtractorInstance(String url, Downloader downloader) + public abstract ChannelExtractor getChannelExtractorInstance(String url, int page, Downloader downloader) throws ExtractionException, IOException; public final int getServiceId() { diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java index e697f0d3b..b4f856a77 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java @@ -1,20 +1,9 @@ package org.schabi.newpipe.extractor.services.youtube; -import android.util.Log; -/* -import com.steadystate.css.dom.CSSStyleDeclarationImpl; -import com.steadystate.css.dom.CSSStyleSheetImpl; -import com.steadystate.css.parser.CSSOMParser; -import com.steadystate.css.parser.SACParserCSS3; -import org.w3c.css.sac.CSSParseException; -import org.w3c.css.sac.InputSource; -import org.w3c.dom.css.CSSRule; -import org.w3c.dom.css.CSSRuleList; -import org.w3c.dom.css.CSSStyleSheet; -import java.io.StringReader; -*/ +import org.json.JSONException; +import org.json.JSONObject; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -30,6 +19,8 @@ import org.schabi.newpipe.extractor.UrlIdHandler; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; /** * Created by Christian Schabesberger on 25.07.16. @@ -58,42 +49,79 @@ public class YoutubeChannelExtractor extends ChannelExtractor { // private CSSOMParser cssParser = new CSSOMParser(new SACParserCSS3()); private Downloader downloader; - private final Document doc; - private final String channelUrl; - private String vUrl =""; + private Document doc = null; + private boolean isAjaxPage = false; + private static String userUrl = ""; + private static String channelName = ""; + private static String avatarUrl = ""; + private static String bannerUrl = ""; + private static String feedUrl = ""; + // the fist page is html all other pages are ajax. Every new page can be requested by sending + // this request url. + private static String nextPageUrl = ""; - public YoutubeChannelExtractor(UrlIdHandler urlIdHandler, String url, Downloader dl, int serviceId) + public YoutubeChannelExtractor(UrlIdHandler urlIdHandler, String url, int page, Downloader dl, int serviceId) throws ExtractionException, IOException { - super(urlIdHandler, url, dl, serviceId); + super(urlIdHandler, url, page, dl, serviceId); - channelUrl = urlIdHandler.cleanUrl(url) ; //+ "/video?veiw=0&flow=list&sort=dd"; + url = urlIdHandler.cleanUrl(url) ; //+ "/video?veiw=0&flow=list&sort=dd"; downloader = dl; - // we first need to get the user url. Otherwise we can't find videos - String channelPageContent = downloader.download(channelUrl); - Document channelDoc = Jsoup.parse(channelPageContent, channelUrl); - String userUrl = getUserUrl(channelDoc); - vUrl = userUrl + "/videos?veiw=0&flow=list&sort=dd"; - String pageContent = downloader.download(vUrl); - doc = Jsoup.parse(pageContent, vUrl); + if(page == 0) { + if (isUserUrl(url)) { + userUrl = url; + } else { + // we first need to get the user url. Otherwise we can't find videos + String channelPageContent = downloader.download(url); + Document channelDoc = Jsoup.parse(channelPageContent, url); + userUrl = getUserUrl(channelDoc); + } + + userUrl = userUrl + "/videos?veiw=0&flow=list&sort=dd&live_view=500"; + String pageContent = downloader.download(userUrl); + doc = Jsoup.parse(pageContent, userUrl); + nextPageUrl = getNextPageUrl(doc); + isAjaxPage = false; + } else { + Map userProperties = new HashMap<>(); + userProperties.put("Referer", userUrl); + String ajaxDataRaw = downloader.download(nextPageUrl, userProperties); + JSONObject ajaxData; + String htmlDataRaw; + try { + ajaxData = new JSONObject(ajaxDataRaw); + htmlDataRaw = ajaxData.getString("content_html"); + } catch (JSONException e) { + throw new ParsingException("Could not parse json data for next page", e); + } + doc = Jsoup.parse(htmlDataRaw, nextPageUrl); + nextPageUrl = getNextPageUrl(ajaxData); + isAjaxPage = true; + } } @Override public String getChannelName() throws ParsingException { - try { - return doc.select("span[class=\"qualified-channel-title-text\"]").first() - .select("a").first().text(); - } catch(Exception e) { - throw new ParsingException("Could not get channel name"); - } + try { + if(!isAjaxPage) { + channelName = doc.select("span[class=\"qualified-channel-title-text\"]").first() + .select("a").first().text(); + } + return channelName; + } catch(Exception e) { + throw new ParsingException("Could not get channel name"); + } } @Override public String getAvatarUrl() throws ParsingException { try { - return doc.select("img[class=\"channel-header-profile-image\"]") - .first().attr("abs:src"); + if(!isAjaxPage) { + avatarUrl = doc.select("img[class=\"channel-header-profile-image\"]") + .first().attr("abs:src"); + } + return avatarUrl; } catch(Exception e) { throw new ParsingException("Could not get avatar", e); } @@ -101,16 +129,18 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getBannerUrl() throws ParsingException { - try { - Element el = doc.select("div[id=\"gh-banner\"]").first().select("style").first(); - String cssContent = el.html(); - String url = "https:" + Parser.matchGroup1("url\\((.*)\\)", cssContent); - if(url.contains("s.ytimg.com")) { - return null; - } else { - return url; + if(!isAjaxPage) { + Element el = doc.select("div[id=\"gh-banner\"]").first().select("style").first(); + String cssContent = el.html(); + String url = "https:" + Parser.matchGroup1("url\\((.*)\\)", cssContent); + if (url.contains("s.ytimg.com")) { + bannerUrl = null; + } else { + bannerUrl = url; + } } + return bannerUrl; } catch(Exception e) { throw new ParsingException("Could not get Banner", e); } @@ -119,7 +149,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public StreamPreviewInfoCollector getStreams() throws ParsingException { StreamPreviewInfoCollector collector = getStreamPreviewInfoCollector(); - Element ul = doc.select("ul[id=\"browse-items-primary\"]").first(); + Element ul = null; + if(isAjaxPage) { + ul = doc.select("body").first(); + } else { + ul = doc.select("ul[id=\"browse-items-primary\"]").first(); + } for(final Element li : ul.children()) { if (li.select("div[class=\"feed-item-dismissable\"]").first() != null) { @@ -235,6 +270,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor { throw new ParsingException("Could not get thumbnail url", e); } } + + private boolean isLiveStream(Element item) { + Element bla = item.select("span[class*=\"yt-badge-live\"]").first(); + + if(bla == null) { + // sometimes livestreams dont have badges but sill are live streams + // if video time is not available we most likly have an offline livestream + if(item.select("span[class*=\"video-time\"]").first() == null) { + return true; + } + } + return bla != null; + } }); } } @@ -245,27 +293,49 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public String getFeedUrl() throws ParsingException { try { - return doc.select("link[title=\"RSS\"]").first().attr("abs:href"); + if(!isAjaxPage) { + feedUrl = doc.select("link[title=\"RSS\"]").first().attr("abs:href"); + } + return feedUrl; } catch(Exception e) { throw new ParsingException("Could not get feed url", e); } } + @Override + public boolean hasNextPage() throws ParsingException { + return !nextPageUrl.isEmpty(); + } + private String getUserUrl(Document d) throws ParsingException { return d.select("span[class=\"qualified-channel-title-text\"]").first() .select("a").first().attr("abs:href"); } - private boolean isLiveStream(Element item) { - Element bla = item.select("span[class*=\"yt-badge-live\"]").first(); + private boolean isUserUrl(String url) throws ParsingException { + return url.contains("/user/"); + } - if(bla == null) { - // sometimes livestreams dont have badges but sill are live streams - // if video time is not available we most likly have an offline livestream - if(item.select("span[class*=\"video-time\"]").first() == null) { - return true; - } + private String getNextPageUrl(Document d) throws ParsingException { + try { + Element button = d.select("button[class*=\"yt-uix-load-more\"]").first(); + return button.attr("abs:data-uix-load-more-href"); + } catch(Exception e) { + throw new ParsingException("could not load next page url", e); } - return bla != null; + } + + private String getNextPageUrl(JSONObject ajaxData) throws ParsingException { + Document doc = null; + try { + String docRaw = ajaxData.getString("load_more_widget_html"); + if(docRaw.isEmpty()) { + return ""; + } + doc = Jsoup.parse(docRaw); + } catch(JSONException je) { + throw new ParsingException("Could not get load_more_widget from ajax response", je); + } + return getNextPageUrl(doc); } } diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 812b74771..9c2198926 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -70,8 +70,8 @@ public class YoutubeService extends StreamingService { } @Override - public ChannelExtractor getChannelExtractorInstance(String url, Downloader downloader) + public ChannelExtractor getChannelExtractorInstance(String url, int page, Downloader downloader) throws ExtractionException, IOException { - return new YoutubeChannelExtractor(getChannelUrlIdHandlerInstance(), url, downloader, getServiceId()); + return new YoutubeChannelExtractor(getChannelUrlIdHandlerInstance(), url, page, downloader, getServiceId()); } } diff --git a/app/src/main/java/org/schabi/newpipe/info_list/InfoListAdapter.java b/app/src/main/java/org/schabi/newpipe/info_list/InfoListAdapter.java index fbb220b8d..365ed246e 100644 --- a/app/src/main/java/org/schabi/newpipe/info_list/InfoListAdapter.java +++ b/app/src/main/java/org/schabi/newpipe/info_list/InfoListAdapter.java @@ -46,8 +46,10 @@ public class InfoListAdapter extends RecyclerView.Adapter { } public void addStreamItemList(List videos) { - streamList.addAll(videos); - notifyDataSetChanged(); + if(videos!= null) { + streamList.addAll(videos); + notifyDataSetChanged(); + } } public void clearSteamItemList() { diff --git a/app/src/main/java/org/schabi/newpipe/search_fragment/SearchInfoItemFragment.java b/app/src/main/java/org/schabi/newpipe/search_fragment/SearchInfoItemFragment.java index 52659f29f..99d2a57dd 100644 --- a/app/src/main/java/org/schabi/newpipe/search_fragment/SearchInfoItemFragment.java +++ b/app/src/main/java/org/schabi/newpipe/search_fragment/SearchInfoItemFragment.java @@ -237,7 +237,7 @@ public class SearchInfoItemFragment extends Fragment { searchView.setSuggestionsAdapter(suggestionListAdapter); searchView.setOnSuggestionListener(new SearchSuggestionListener(searchView, suggestionListAdapter)); searchView.setOnQueryTextListener(new SearchQueryListener()); - if(!searchQuery.isEmpty()) { + if(searchQuery != null && !searchQuery.isEmpty()) { searchView.setQuery(searchQuery, false); searchView.setIconifiedByDefault(false); }