From 1d94e7d477c35e3cbaf7b0851ed10100e10da179 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sat, 7 Apr 2018 16:45:07 +0200 Subject: [PATCH] add ability to get donation links from channel --- .../extractor/channel/ChannelExtractor.java | 1 + .../extractor/channel/ChannelInfo.java | 14 +++++++++++ .../SoundcloudChannelExtractor.java | 5 ++++ .../youtube/YoutubeChannelExtractor.java | 24 +++++++++++++++++++ .../youtube/YoutubeChannelExtractorTest.java | 15 ++++++++++++ 5 files changed, 59 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java index 6a920ad94..05a5bc587 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java @@ -45,4 +45,5 @@ public abstract class ChannelExtractor extends ListExtractor { public abstract String getFeedUrl() throws ParsingException; public abstract long getSubscriberCount() throws ParsingException; public abstract String getDescription() throws ParsingException; + public abstract String[] getDonationLinks() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java index 1691f9544..0d8a860bd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java @@ -92,6 +92,11 @@ public class ChannelInfo extends ListInfo { } catch (Exception e) { info.addError(e); } + try { + info.setDonationLinks(extractor.getDonationLinks()); + } catch (Exception e) { + info.addError(e); + } return info; } @@ -101,6 +106,7 @@ public class ChannelInfo extends ListInfo { private String feedUrl; private long subscriberCount = -1; private String description; + private String[] donationLinks; public String getAvatarUrl() { return avatarUrl; @@ -141,4 +147,12 @@ public class ChannelInfo extends ListInfo { public void setDescription(String description) { this.description = description; } + + public String[] getDonationLinks() { + return donationLinks; + } + + public void setDonationLinks(String[] donationLinks) { + this.donationLinks = donationLinks; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java index 0df15fa69..7232ac5ad 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java @@ -133,4 +133,9 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { return new InfoItemsPage<>(collector, nextPageUrl); } + + @Override + public String[] getDonationLinks() { + return new String[0]; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java index de0cb625f..91a89fd05 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java @@ -4,12 +4,14 @@ package org.schabi.newpipe.extractor.services.youtube; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import com.sun.org.apache.xerces.internal.xs.StringList; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -20,6 +22,7 @@ import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; import java.io.IOException; +import java.util.ArrayList; /* * Created by Christian Schabesberger on 25.07.16. @@ -181,6 +184,27 @@ public class YoutubeChannelExtractor extends ChannelExtractor { return new InfoItemsPage<>(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl)); } + @Override + public String[] getDonationLinks() throws ParsingException { + try { + ArrayList links = new ArrayList<>(); + Element linkHolder = doc.select("div[id=\"links-holder\"]").first(); + if(linkHolder == null) { + // this occures if no links are embeded into the channel + return new String[0]; + } + for(Element a : linkHolder.select("a")) { + System.err.println(a.attr("abs:href")); + links.add(a.attr("abs:href")); + } + String[] retLinks = new String[links.size()]; + retLinks = links.toArray(retLinks); + return retLinks; + } catch (Exception e) { + throw new ParsingException("Could not get donation links", e); + } + } + private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl) throws ParsingException { String loadMoreHtmlDataRaw = ajaxJson.getString("load_more_widget_html"); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 8fb520773..507f67255 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -103,6 +103,11 @@ public class YoutubeChannelExtractorTest { public void testSubscriberCount() throws Exception { assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0); } + + @Test + public void testChannelDonation() throws Exception { + assertTrue(extractor.getDonationLinks().length != 0); + } } public static class Kurzgesagt implements BaseChannelExtractorTest { @@ -205,6 +210,11 @@ public class YoutubeChannelExtractorTest { public void testSubscriberCount() throws Exception { assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5e6); } + + @Test + public void testChannelDonation() throws Exception { + assertTrue(extractor.getDonationLinks().length == 0); + } } public static class CaptainDisillusion implements BaseChannelExtractorTest { @@ -389,6 +399,11 @@ public class YoutubeChannelExtractorTest { public void testSubscriberCount() throws Exception { assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 50); } + + @Test + public void testChannelDonation() throws Exception { + assertTrue(extractor.getDonationLinks().length == 0); + } } };