diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 18aedbe90..c448d98ca 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -87,7 +87,7 @@ public class YoutubeService extends StreamingService { // add kiosks here e.g.: YoutubeTrendingUrlIdHandler h = new YoutubeTrendingUrlIdHandler(); try { - list.addKioskEntry(new YoutubeTrendingExtractor(this, h.getUrl(""), h.getUrl("")), h); + list.addKioskEntry(new YoutubeTrendingExtractor(this, h.getUrl(""), null), h); } catch (Exception e) { throw new ExtractionException(e); } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java index 4e38e85a2..77cc2a135 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java @@ -20,30 +20,38 @@ package org.schabi.newpipe.extractor.services.youtube; * along with NewPipe. If not, see . */ -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.UrlIdHandler; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.*; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; + import java.io.IOException; public class YoutubeTrendingExtractor extends KioskExtractor { + private Document doc; + public YoutubeTrendingExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { super(service, url, nextStreamsUrl); } @Override - public void fetchPage() - throws IOException, ExtractionException { + public void fetchPage() throws IOException, ExtractionException { + Downloader downloader = NewPipe.getDownloader(); + String channelUrl = getCleanUrl(); + String pageContent = downloader.download(channelUrl); + doc = Jsoup.parse(pageContent, channelUrl); } @Override public String getType() { - return "Treinding"; + return "Trending"; } @Override @@ -57,7 +65,63 @@ public class YoutubeTrendingExtractor extends KioskExtractor { } @Override - public StreamInfoItemCollector getStreams() { - return null; + public StreamInfoItemCollector getStreams() throws ParsingException { + StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); + Element ul = doc.select("ul[class*=\"expanded-shelf-content-list\"]").first(); + for(final Element li : ul.children()) { + final Element el = li.select("div[class*=\"yt-lockup-dismissable\"]").first(); + collector.commit(new YoutubeStreamInfoItemExtractor(li) { + @Override + public String getUrl() throws ParsingException { + try { + Element dl = el.select("h3").first().select("a").first(); + return dl.attr("abs:href"); + } catch (Exception e) { + throw new ParsingException("Could not get web page url for the video", e); + } + } + + @Override + public String getName() throws ParsingException { + try { + Element dl = el.select("h3").first().select("a").first(); + return dl.text(); + } catch (Exception e) { + throw new ParsingException("Could not get web page url for the video", e); + } + } + + @Override + public String getUploaderName() throws ParsingException { + try { + Element uploaderEl = el.select("div[class*=\"yt-lockup-byline \"]").first(); + return uploaderEl.select("a").text(); + } catch (Exception e) { + throw new ParsingException("Could not get Uploader name"); + } + } + + @Override + public String getThumbnailUrl() throws ParsingException { + try { + String url; + Element te = li.select("span[class=\"yt-thumb-simple\"]").first() + .select("img").first(); + url = te.attr("abs:src"); + // Sometimes youtube sends links to gif files which somehow seem to not exist + // anymore. Items with such gif also offer a secondary image source. So we are going + // to use that if we've caught such an item. + if (url.contains(".gif")) { + url = te.attr("abs:data-thumb"); + } + return url; + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + }); + } + + return collector; } } diff --git a/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemCollector.java b/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemCollector.java index 8b1863ba2..b0757b886 100644 --- a/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemCollector.java +++ b/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemCollector.java @@ -1,9 +1,13 @@ package org.schabi.newpipe.extractor.stream; +import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItemCollector; import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import java.util.List; +import java.util.Vector; + /* * Created by Christian Schabesberger on 28.02.16. * @@ -80,4 +84,14 @@ public class StreamInfoItemCollector extends InfoItemCollector { addError(e); } } + + public List getStreamInfoItemList() { + List siiList = new Vector<>(); + for(InfoItem ii : super.getItemList()) { + if(ii instanceof StreamInfoItem) { + siiList.add((StreamInfoItem) ii); + } + } + return siiList; + } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java index 74ea20435..4dd9879f7 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java @@ -23,9 +23,13 @@ package org.schabi.newpipe.extractor.services.youtube; import org.junit.Before; import org.junit.Test; import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.InfoItemCollector; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import java.util.List; + import static junit.framework.TestCase.assertFalse; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -46,6 +50,7 @@ public class YoutubeTrendingExtractorTest { extractor = YouTube.getService() .getKioskList() .getExtractorByType("Trending"); + extractor.fetchPage(); } @Test @@ -65,7 +70,17 @@ public class YoutubeTrendingExtractorTest { @Test public void testGetStreams() throws Exception { - assertTrue("no streams are received", !extractor.getStreams().getItemList().isEmpty()); + InfoItemCollector collector = extractor.getStreams(); + if(!collector.getErrors().isEmpty()) { + System.err.println("----------"); + for(Throwable e : collector.getErrors()) { + e.printStackTrace(); + System.err.println("----------"); + } + } + assertTrue("no streams are received", + !collector.getItemList().isEmpty() + && collector.getErrors().isEmpty()); } @Test @@ -77,12 +92,17 @@ public class YoutubeTrendingExtractorTest { public void testHasMoreStreams() throws Exception { // Setup the streams extractor.getStreams(); - assertTrue("don't have more streams", extractor.hasMoreStreams()); + assertFalse("has more streams", extractor.hasMoreStreams()); } @Test public void testGetNextStreams() throws Exception { - assertFalse("extractor has next streams", !extractor.getNextStreams().nextItemsList.isEmpty()); - assertFalse("extractor has more streams after getNextStreams", extractor.hasMoreStreams()); + assertTrue("extractor has next streams", extractor.getNextStreams() == null + || extractor.getNextStreams().nextItemsList.isEmpty()); + } + + @Test + public void testGetCleanUrl() throws Exception { + assertEquals(extractor.getCleanUrl(), extractor.getCleanUrl(), "https://www.youtube.com/feed/trending"); } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java index feca28c4b..303d409d6 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java @@ -71,6 +71,7 @@ public class YoutubeTrendingUrlIdHandlerTest { assertFalse(urlIdHandler.acceptUrl("youtube.com/feed/trending askjkf")); assertFalse(urlIdHandler.acceptUrl("askdjfi youtube.com/feed/trending askjkf")); assertFalse(urlIdHandler.acceptUrl(" youtube.com/feed/trending")); + assertFalse(urlIdHandler.acceptUrl("https://www.youtube.com/feed/trending.html")); assertFalse(urlIdHandler.acceptUrl("")); } }