From 8ecee8737cd1db5d770432ba6b154a25934a3ba4 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Wed, 22 Mar 2023 01:00:05 +0100 Subject: [PATCH] fix: channel extractor tests, docs --- .../youtube/YoutubeParsingHelper.java | 28 ++++++++++++++++ .../extractors/YoutubeChannelExtractor.java | 22 ++++++------- .../YoutubeChannelTabExtractor.java | 8 ++--- .../BandcampChannelExtractorTest.java | 32 +++++++++++-------- .../PeertubeAccountExtractorTest.java | 20 ++++++++---- .../PeertubeChannelExtractorTest.java | 20 ++++++++---- .../SoundcloudChannelExtractorTest.java | 22 +++++++++---- 7 files changed, 101 insertions(+), 51 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 0ddd44c5c..a0a329ac8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -1741,6 +1741,11 @@ public final class YoutubeParsingHelper { return false; } + /** + * Take a YouTube channel ID or URL path, resolve it if necessary and return a channel ID. + * @param idOrPath YouTube channel ID or URL path + * @return YouTube Channel ID + */ public static String resolveChannelId(final String idOrPath) throws ExtractionException, IOException { final String[] channelId = idOrPath.split("/"); @@ -1796,6 +1801,10 @@ public final class YoutubeParsingHelper { return channelId[1]; } + /** + * Response data object for + * {@link #getChannelResponse(String, String, Localization, ContentCountry)} + */ public static final class ChannelResponseData { public final JsonObject responseJson; public final String channelId; @@ -1806,6 +1815,25 @@ public final class YoutubeParsingHelper { } } + /** + * Fetch YouTube channel data. + *

Parameter list:

+ * + * + * @param channelId YouTube channel ID + * @param params Parameters to specify the YouTube channel tab + * @param loc YouTube localization + * @param country YouTube content country + * @return + * @throws ExtractionException + * @throws IOException + */ public static ChannelResponseData getChannelResponse(final String channelId, final String params, final Localization loc, diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index b34d44aa1..5df0de62d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -225,7 +225,6 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Nonnull @Override public List getTabs() throws ParsingException { - getVideoTab(); return tabs; } @@ -244,9 +243,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); Page nextPage = null; + extractTabs(); - if (getVideoTab() != null) { - final JsonObject tabContent = getVideoTab().getObject("content"); + if (videoTab != null) { + final JsonObject tabContent = videoTab.getObject("content"); JsonArray items = tabContent .getObject("sectionListRenderer") .getArray("contents").getObject(0).getObject("itemSectionRenderer") @@ -370,12 +370,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor { return continuation; } - @Nullable - private JsonObject getVideoTab() throws ParsingException { - if (videoTab != null) { - return videoTab; - } - + /** + * Collect a list of available tabs and get the video tab data. + */ + private void extractTabs() throws ParsingException { final JsonArray responseTabs = initialData.getObject("contents") .getObject("twoColumnBrowseResultsRenderer") .getArray("tabs"); @@ -426,8 +424,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { if (tabs.isEmpty()) { throw new ContentNotSupportedException("This channel has no supported tabs"); } - - return null; + return; } final String messageRendererText = getTextFromObject( @@ -442,10 +439,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor { .getObject("text")); if (messageRendererText != null && messageRendererText.equals("This channel has no videos.")) { - return null; + return; } videoTab = foundVideoTab; - return foundVideoTab; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java index f7c779a16..204fe9ddb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java @@ -117,8 +117,9 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor { final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); Page nextPage = null; + tabData = getTabData(); - if (getTabData() != null) { + if (tabData != null) { final JsonObject tabContent = tabData.getObject("content"); JsonArray items = tabContent .getObject("sectionListRenderer") @@ -171,10 +172,6 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor { @Nullable private JsonObject getTabData() throws ParsingException { - if (this.tabData != null) { - return this.tabData; - } - final String urlSuffix = YoutubeChannelTabLinkHandlerFactory.getUrlSuffix(getTab()); final JsonArray tabs = initialData.getObject("contents") @@ -206,7 +203,6 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor { return null; } - this.tabData = foundTab; return foundTab; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java index 501bbb1f9..c6bcca1b5 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java @@ -8,9 +8,12 @@ import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.linkhandler.ChannelTabs; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import java.io.IOException; +import java.util.Set; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.*; import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; @@ -31,40 +34,41 @@ public class BandcampChannelExtractorTest implements BaseChannelExtractorTest { assertTrue(extractor.getInitialPage().getItems().size() >= 0); } - @Override @Test public void testDescription() throws Exception { assertEquals("making music:)", extractor.getDescription()); } - @Override + @Test public void testAvatarUrl() throws Exception { assertTrue(extractor.getAvatarUrl().contains("://f4.bcbits.com/"), "unexpected avatar URL"); } - @Override + @Test public void testBannerUrl() throws Exception { assertTrue(extractor.getBannerUrl().contains("://f4.bcbits.com/"), "unexpected banner URL"); } - @Override + @Test public void testFeedUrl() throws Exception { assertNull(extractor.getFeedUrl()); } - @Override + @Test public void testSubscriberCount() throws Exception { assertEquals(-1, extractor.getSubscriberCount()); } - @Override + @Test public void testVerified() throws Exception { assertFalse(extractor.isVerified()); } - @Override + @Test public void testTabs() throws Exception { - assertTrue(extractor.getTabs().isEmpty()); + Set tabs = extractor.getTabs().stream() + .map(linkHandler -> linkHandler.getContentFilters().get(0)).collect(Collectors.toSet()); + assertTrue(tabs.contains(ChannelTabs.ALBUMS)); } @Override @@ -77,27 +81,27 @@ public class BandcampChannelExtractorTest implements BaseChannelExtractorTest { // not implemented } - @Override + @Test public void testServiceId() { assertEquals(Bandcamp.getServiceId(), extractor.getServiceId()); } - @Override + @Test public void testName() throws Exception { assertEquals("toupie", extractor.getName()); } - @Override + @Test public void testId() throws Exception { - assertEquals("https://toupie.bandcamp.com/", extractor.getId()); + assertEquals("2450875064", extractor.getId()); } - @Override + @Test public void testUrl() throws Exception { assertEquals("https://toupie.bandcamp.com", extractor.getUrl()); } - @Override + @Test public void testOriginalUrl() throws Exception { assertEquals("https://toupie.bandcamp.com", extractor.getUrl()); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java index d94ca18fa..a7c4eb5e1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java @@ -7,9 +7,13 @@ import org.schabi.newpipe.extractor.ExtractorAsserts; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ChannelTabs; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeAccountExtractor; +import java.util.Set; +import java.util.stream.Collectors; + import static org.junit.jupiter.api.Assertions.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.PeerTube; @@ -105,14 +109,16 @@ public class PeertubeAccountExtractorTest { ExtractorAsserts.assertGreaterOrEqual(700, extractor.getSubscriberCount()); } - @Override + @Test public void testVerified() throws Exception { assertFalse(extractor.isVerified()); } - @Override + @Test public void testTabs() throws Exception { - + Set tabs = extractor.getTabs().stream() + .map(linkHandler -> linkHandler.getContentFilters().get(0)).collect(Collectors.toSet()); + assertTrue(tabs.contains(ChannelTabs.CHANNELS)); } } @@ -211,14 +217,16 @@ public class PeertubeAccountExtractorTest { ExtractorAsserts.assertGreaterOrEqual(100, extractor.getSubscriberCount()); } - @Override + @Test public void testVerified() throws Exception { assertFalse(extractor.isVerified()); } - @Override + @Test public void testTabs() throws Exception { - // TODO: implement Peertube account channels tab + Set tabs = extractor.getTabs().stream() + .map(linkHandler -> linkHandler.getContentFilters().get(0)).collect(Collectors.toSet()); + assertTrue(tabs.contains(ChannelTabs.CHANNELS)); } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java index e2ae60f4b..d17b3b314 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java @@ -7,9 +7,13 @@ import org.schabi.newpipe.extractor.ExtractorAsserts; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ChannelTabs; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor; +import java.util.Set; +import java.util.stream.Collectors; + import static org.junit.jupiter.api.Assertions.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.PeerTube; @@ -120,14 +124,16 @@ public class PeertubeChannelExtractorTest { ExtractorAsserts.assertGreaterOrEqual(230, extractor.getSubscriberCount()); } - @Override + @Test public void testVerified() throws Exception { assertFalse(extractor.isVerified()); } - @Override + @Test public void testTabs() throws Exception { - + Set tabs = extractor.getTabs().stream() + .map(linkHandler -> linkHandler.getContentFilters().get(0)).collect(Collectors.toSet()); + assertTrue(tabs.contains(ChannelTabs.PLAYLISTS)); } } @@ -242,14 +248,16 @@ public class PeertubeChannelExtractorTest { ExtractorAsserts.assertGreaterOrEqual(700, extractor.getSubscriberCount()); } - @Override + @Test public void testVerified() throws Exception { assertFalse(extractor.isVerified()); } - @Override + @Test public void testTabs() throws Exception { - // TODO: implement Peertube channel playlists tab + Set tabs = extractor.getTabs().stream() + .map(linkHandler -> linkHandler.getContentFilters().get(0)).collect(Collectors.toSet()); + assertTrue(tabs.contains(ChannelTabs.PLAYLISTS)); } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java index 933335584..e520d3eb1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java @@ -6,9 +6,13 @@ import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ChannelTabs; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChannelExtractor; +import java.util.Set; +import java.util.stream.Collectors; + import static org.junit.jupiter.api.Assertions.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; @@ -102,14 +106,17 @@ public class SoundcloudChannelExtractorTest { assertTrue(extractor.getSubscriberCount() >= 1e6, "Wrong subscriber count"); } - @Override + @Test public void testVerified() throws Exception { assertTrue(extractor.isVerified()); } - @Override + @Test public void testTabs() throws Exception { - + Set tabs = extractor.getTabs().stream() + .map(linkHandler -> linkHandler.getContentFilters().get(0)).collect(Collectors.toSet()); + assertTrue(tabs.contains(ChannelTabs.PLAYLISTS)); + assertTrue(tabs.contains(ChannelTabs.ALBUMS)); } } @@ -206,14 +213,17 @@ public class SoundcloudChannelExtractorTest { assertTrue(extractor.getSubscriberCount() >= 2e6, "Wrong subscriber count"); } - @Override + @Test public void testVerified() throws Exception { assertTrue(extractor.isVerified()); } - @Override + @Test public void testTabs() throws Exception { - // TODO: implement soundcloud playlist tab + Set tabs = extractor.getTabs().stream() + .map(linkHandler -> linkHandler.getContentFilters().get(0)).collect(Collectors.toSet()); + assertTrue(tabs.contains(ChannelTabs.PLAYLISTS)); + assertTrue(tabs.contains(ChannelTabs.ALBUMS)); } } }