From 466d87ceb4f2b535cffdefad4436c7863ecbbd34 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Mon, 25 Sep 2017 12:43:40 +0200 Subject: [PATCH] remove type from kiosk and make getName() crawl the translated kiosk name --- .../extractor/kiosk/KioskExtractor.java | 30 +++++++------ .../newpipe/extractor/kiosk/KioskInfo.java | 3 +- .../newpipe/extractor/kiosk/KioskList.java | 30 ++++++------- .../soundcloud/SoundcloudChartsExtractor.java | 10 ++--- .../soundcloud/SoundcloudService.java | 42 +++++++------------ .../services/youtube/YoutubeService.java | 4 +- .../youtube/YoutubeTrendingExtractor.java | 16 ++++++- .../SoundcloudChartsExtractorTest.java | 2 +- .../services/youtube/YoutubeServiceTest.java | 2 +- .../YoutubeTreindingKioskInfoTest.java | 9 +++- .../youtube/YoutubeTrendingExtractorTest.java | 5 ++- 11 files changed, 79 insertions(+), 74 deletions(-) diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java index 102ed4bfd..e11e89b0c 100644 --- a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java @@ -29,16 +29,15 @@ import java.io.IOException; public abstract class KioskExtractor extends ListExtractor { private String contentCountry = null; - private String type = null; + private String id = null; public KioskExtractor(StreamingService streamingService, String url, String nextStreamsUrl, - String type) + String kioskId) throws IOException, ExtractionException { super(streamingService, url, nextStreamsUrl); - this.contentCountry = contentCountry; - this.type = type; + this.id = kioskId; } /** @@ -51,24 +50,23 @@ public abstract class KioskExtractor extends ListExtractor { this.contentCountry = contentCountry; } - /** - * Returns the type of the kiosk. - * eg. Trending, Top & Hot, Top last 24 hours - * @return type of kiosk - */ - public String getType() throws ParsingException { - return type; - } @Override public String getId() throws ParsingException { - return getType(); + return id; } + /** + * Id should be the name of the kiosk, tho Id is used for identifing it in the programm, + * so id should be kept in english. + * In order to get the name of the kiosk in the desired language we have to + * crawl if from the website. + * @return the tranlsated version of id + * @throws ParsingException + */ @Override - public String getName() throws ParsingException { - return getType(); - } + public abstract String getName() throws ParsingException; + public String getContentCountry() { return contentCountry; diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java index 121502c6c..8b82f9eca 100644 --- a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java @@ -27,7 +27,6 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import java.io.IOException; public class KioskInfo extends ListInfo { - public String type; public static ListExtractor.NextItemsResult getMoreItems(ServiceList serviceItem, String url, @@ -67,9 +66,9 @@ public class KioskInfo extends ListInfo { KioskInfo info = new KioskInfo(); extractor.setContentCountry(contentCountry); extractor.fetchPage(); - info.type = extractor.getType(); info.name = extractor.getName(); info.id = extractor.getId(); + info.url = extractor.getCleanUrl(); try { StreamInfoItemCollector c = extractor.getStreams(); diff --git a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java index 9960be4b7..de34c3f95 100644 --- a/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java +++ b/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java @@ -18,7 +18,7 @@ public class KioskList { KioskExtractor createNewKiosk(final StreamingService streamingService, final String url, final String nextStreamUrl, - final String type) + final String kioskId) throws ExtractionException, IOException; } @@ -39,12 +39,12 @@ public class KioskList { this.service_id = service_id; } - public void addKioskEntry(KioskExtractorFactory extractorFactory, UrlIdHandler handler, String type) + public void addKioskEntry(KioskExtractorFactory extractorFactory, UrlIdHandler handler, String id) throws Exception { - if(kioskList.get(type) != null) { - throw new Exception("Kiosk with type " + type + " already exists."); + if(kioskList.get(id) != null) { + throw new Exception("Kiosk with type " + id + " already exists."); } - kioskList.put(type, new KioskEntry(extractorFactory, handler)); + kioskList.put(id, new KioskEntry(extractorFactory, handler)); } public void setDefaultKiosk(String kioskType) { @@ -54,35 +54,35 @@ public class KioskList { public KioskExtractor getDefaultKioskExtractor(String nextStreamUrl) throws ExtractionException, IOException { if(defaultKiosk != null && !defaultKiosk.equals("")) { - return getExtractorByType(defaultKiosk, nextStreamUrl); + return getExtractorById(defaultKiosk, nextStreamUrl); } else { if(!kioskList.isEmpty()) { // if not set get any entry Object[] keySet = kioskList.keySet().toArray(); - return getExtractorByType(keySet[0].toString(), nextStreamUrl); + return getExtractorById(keySet[0].toString(), nextStreamUrl); } else { return null; } } } - public String getDefaultKioskType() { + public String getDefaultKioskId() { return defaultKiosk; } - public KioskExtractor getExtractorByType(String kioskType, String nextStreamsUrl) + public KioskExtractor getExtractorById(String kioskId, String nextStreamsUrl) throws ExtractionException, IOException { - KioskEntry ke = kioskList.get(kioskType); + KioskEntry ke = kioskList.get(kioskId); if(ke == null) { - throw new ExtractionException("No kiosk found with the type: " + kioskType); + throw new ExtractionException("No kiosk found with the type: " + kioskId); } else { return ke.extractorFactory.createNewKiosk(NewPipe.getService(service_id), - ke.handler.getUrl(kioskType), - nextStreamsUrl, kioskType); + ke.handler.getUrl(kioskId), + nextStreamsUrl, kioskId); } } - public Set getAvailableKisokTypes() { + public Set getAvailableKisoks() { return kioskList.keySet(); } @@ -91,7 +91,7 @@ public class KioskList { for(Map.Entry e : kioskList.entrySet()) { KioskEntry ke = e.getValue(); if(ke.handler.acceptUrl(url)) { - return getExtractorByType(e.getKey(), nextStreamsUrl); + return getExtractorById(e.getKey(), nextStreamsUrl); } } throw new ExtractionException("Could not find a kiosk that fits to the url: " + url); diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java index 47977756c..b1466e8f5 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -14,9 +14,9 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; public class SoundcloudChartsExtractor extends KioskExtractor { private String url; - public SoundcloudChartsExtractor(StreamingService service, String url, String nextStreamsUrl, String type) + public SoundcloudChartsExtractor(StreamingService service, String url, String nextStreamsUrl, String kioskId) throws IOException, ExtractionException { - super(service, url, nextStreamsUrl, type); + super(service, url, nextStreamsUrl, kioskId); this.url = url; } @@ -25,8 +25,8 @@ public class SoundcloudChartsExtractor extends KioskExtractor { } @Override - public String getType() throws ParsingException { - return getUrlIdHandler().getId(url); + public String getName() throws ParsingException { + return "< Implement me (♥_♥) >"; } @Override @@ -54,7 +54,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor { "?genre=soundcloud:genres:all-music" + "&client_id=" + SoundcloudParsingHelper.clientId(); - if (getType().equals("Top 50")) { + if (getId().equals("Top 50")) { apiUrl += "&kind=top"; } else { apiUrl += "&kind=new"; diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index 7a845f3e8..9e14c5698 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -62,37 +62,27 @@ public class SoundcloudService extends StreamingService { @Override public KioskList getKioskList() throws ExtractionException { + KioskList.KioskExtractorFactory chartsFactory = new KioskList.KioskExtractorFactory() { + @Override + public KioskExtractor createNewKiosk(StreamingService streamingService, + String url, + String nextStreamUrl, + String id) + throws ExtractionException, IOException { + return new SoundcloudChartsExtractor(SoundcloudService.this, + url, + nextStreamUrl, + id); + } + }; + KioskList list = new KioskList(getServiceId()); // add kiosks here e.g.: final SoundcloudChartsUrlIdHandler h = new SoundcloudChartsUrlIdHandler(); try { - list.addKioskEntry(new KioskList.KioskExtractorFactory() { - @Override - public KioskExtractor createNewKiosk(StreamingService streamingService, - String url, - String nextStreamUrl, - String type) - throws ExtractionException, IOException { - return new SoundcloudChartsExtractor(SoundcloudService.this, - h.getUrl(type), - nextStreamUrl, - type); - } - }, h, "Top 50"); - list.addKioskEntry(new KioskList.KioskExtractorFactory() { - @Override - public KioskExtractor createNewKiosk(StreamingService streamingService, - String url, - String nextStreamUrl, - String type) - throws ExtractionException, IOException { - return new SoundcloudChartsExtractor(SoundcloudService.this, - h.getUrl(type), - nextStreamUrl, - type); - } - }, h, "New & hot"); + list.addKioskEntry(chartsFactory, h, "Top 50"); + list.addKioskEntry(chartsFactory, h, "New & hot"); } catch (Exception e) { throw new ExtractionException(e); } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index d4c63b259..19cbc9d8f 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -90,9 +90,9 @@ public class YoutubeService extends StreamingService { try { list.addKioskEntry(new KioskList.KioskExtractorFactory() { @Override - public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String nextStreamUrl, String type) + public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String nextStreamUrl, String id) throws ExtractionException, IOException { - return new YoutubeTrendingExtractor(YoutubeService.this, url, nextStreamUrl, type); + return new YoutubeTrendingExtractor(YoutubeService.this, url, nextStreamUrl, id); } }, new YoutubeTrendingUrlIdHandler(), "Trending"); list.setDefaultKiosk("Trending"); diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java index 79baa516b..77754a120 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java @@ -35,9 +35,9 @@ public class YoutubeTrendingExtractor extends KioskExtractor { private Document doc; - public YoutubeTrendingExtractor(StreamingService service, String url, String nextStreamsUrl, String type) + public YoutubeTrendingExtractor(StreamingService service, String url, String nextStreamsUrl, String kioskId) throws IOException, ExtractionException { - super(service, url, nextStreamsUrl, type); + super(service, url, nextStreamsUrl, kioskId); } @Override @@ -64,6 +64,18 @@ public class YoutubeTrendingExtractor extends KioskExtractor { return null; } + @Override + public String getName() throws ParsingException { + try { + Element a = doc.select("a[href*=\"/feed/trending\"]").first(); + Element span = a.select("span[class*=\"display-name\"]").first(); + Element nameSpan = span.select("span").first(); + return nameSpan.text(); + } catch (Exception e) { + throw new ParsingException("Could not get Trending name", e); + } + } + @Override public StreamInfoItemCollector getStreams() throws ParsingException { StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java index 253577830..7d37824aa 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java @@ -25,7 +25,7 @@ public class SoundcloudChartsExtractorTest { NewPipe.init(Downloader.getInstance()); extractor = SoundCloud.getService() .getKioskList() - .getExtractorByType("Top 50", null); + .getExtractorById("Top 50", null); extractor.fetchPage(); } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java index a29b13df2..33fa788bf 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java @@ -48,7 +48,7 @@ public class YoutubeServiceTest { @Test public void testGetKioskAvailableKiosks() throws Exception { - assertFalse("No kiosk got returned", kioskList.getAvailableKisokTypes().isEmpty()); + assertFalse("No kiosk got returned", kioskList.getAvailableKisoks().isEmpty()); } @Test diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTreindingKioskInfoTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTreindingKioskInfoTest.java index 533ffac6d..a79015e9c 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTreindingKioskInfoTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTreindingKioskInfoTest.java @@ -54,7 +54,12 @@ public class YoutubeTreindingKioskInfoTest { } @Test - public void getType() { - assertEquals(kioskInfo.type, "Trending"); + public void getId() { + assertEquals(kioskInfo.id, "Trending"); + } + + @Test + public void getName() { + assertFalse(kioskInfo.name.isEmpty()); } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java index bdb82f7fe..c22283343 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java @@ -46,7 +46,7 @@ public class YoutubeTrendingExtractorTest { NewPipe.init(Downloader.getInstance()); extractor = YouTube.getService() .getKioskList() - .getExtractorByType("Trending", null); + .getExtractorById("Trending", null); } @Test @@ -56,7 +56,8 @@ public class YoutubeTrendingExtractorTest { @Test public void testGetName() throws Exception { - assertEquals(extractor.getName(), "Trending"); + System.out.println(extractor.getName()); + assertFalse(extractor.getName().isEmpty()); } @Test