lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix: channel extractor tests, docs

This commit is contained in:
ThetaDev 2023-03-22 01:00:05 +01:00
parent 76052de72a
commit 8ecee8737c
7 changed files with 101 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -1741,6 +1741,11 @@ public final class YoutubeParsingHelper {
return false;
}
/**
* Take a YouTube channel ID or URL path, resolve it if necessary and return a channel ID.
* @param idOrPath YouTube channel ID or URL path
* @return YouTube Channel ID
*/
public static String resolveChannelId(final String idOrPath)
throws ExtractionException, IOException {
final String[] channelId = idOrPath.split("/");
@ -1796,6 +1801,10 @@ public final class YoutubeParsingHelper {
return channelId[1];
}
/**
* Response data object for
* {@link #getChannelResponse(String, String, Localization, ContentCountry)}
*/
public static final class ChannelResponseData {
public final JsonObject responseJson;
public final String channelId;
@ -1806,6 +1815,25 @@ public final class YoutubeParsingHelper {
}
}
/**
* Fetch YouTube channel data.
* <p>Parameter list:</p>
* <ul>
* <li>Videos: {@code EgZ2aWRlb3PyBgQKAjoA}</li>
* <li>Shorts: {@code EgZzaG9ydHPyBgUKA5oBAA%3D%3D}</li>
* <li>Livestreams: {@code EgdzdHJlYW1z8gYECgJ6AA%3D%3D}</li>
* <li>Playlists: {@code EglwbGF5bGlzdHMgAQ%3D%3D}</li>
* <li>Info: {@code EgVhYm91dPIGBAoCEgA%3D}</li>
* </ul>
*
* @param channelId YouTube channel ID
* @param params Parameters to specify the YouTube channel tab
* @param loc YouTube localization
* @param country YouTube content country
* @return
* @throws ExtractionException
* @throws IOException
*/
public static ChannelResponseData getChannelResponse(final String channelId,
final String params,
final Localization loc,

22
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -225,7 +225,6 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Nonnull
@Override
public List<ListLinkHandler> getTabs() throws ParsingException {
getVideoTab();
return tabs;
}
@ -244,9 +243,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Page nextPage = null;
extractTabs();
if (getVideoTab() != null) {
final JsonObject tabContent = getVideoTab().getObject("content");
if (videoTab != null) {
final JsonObject tabContent = videoTab.getObject("content");
JsonArray items = tabContent
.getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
@ -370,12 +370,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return continuation;
}
@Nullable
private JsonObject getVideoTab() throws ParsingException {
if (videoTab != null) {
return videoTab;
}
/**
* Collect a list of available tabs and get the video tab data.
*/
private void extractTabs() throws ParsingException {
final JsonArray responseTabs = initialData.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs");
@ -426,8 +424,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
if (tabs.isEmpty()) {
throw new ContentNotSupportedException("This channel has no supported tabs");
}
return null;
return;
}
final String messageRendererText = getTextFromObject(
@ -442,10 +439,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
.getObject("text"));
if (messageRendererText != null
&& messageRendererText.equals("This channel has no videos.")) {
return null;
return;
}
videoTab = foundVideoTab;
return foundVideoTab;
}
}

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java
View File

@ -117,8 +117,9 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
Page nextPage = null;
tabData = getTabData();
if (getTabData() != null) {
if (tabData != null) {
final JsonObject tabContent = tabData.getObject("content");
JsonArray items = tabContent
.getObject("sectionListRenderer")
@ -171,10 +172,6 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
@Nullable
private JsonObject getTabData() throws ParsingException {
if (this.tabData != null) {
return this.tabData;
}
final String urlSuffix = YoutubeChannelTabLinkHandlerFactory.getUrlSuffix(getTab());
final JsonArray tabs = initialData.getObject("contents")
@ -206,7 +203,6 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
return null;
}
this.tabData = foundTab;
return foundTab;
}

32
extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java
View File

@ -8,9 +8,12 @@ import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import java.io.IOException;
import java.util.Set;
import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.*;
import static org.schabi.newpipe.extractor.ServiceList.Bandcamp;
@ -31,40 +34,41 @@ public class BandcampChannelExtractorTest implements BaseChannelExtractorTest {
assertTrue(extractor.getInitialPage().getItems().size() >= 0);
}
@Override
@Test
public void testDescription() throws Exception {
assertEquals("making music:)", extractor.getDescription());
}
@Override
@Test
public void testAvatarUrl() throws Exception {
assertTrue(extractor.getAvatarUrl().contains("://f4.bcbits.com/"), "unexpected avatar URL");
}
@Override
@Test
public void testBannerUrl() throws Exception {
assertTrue(extractor.getBannerUrl().contains("://f4.bcbits.com/"), "unexpected banner URL");
}
@Override
@Test
public void testFeedUrl() throws Exception {
assertNull(extractor.getFeedUrl());
}
@Override
@Test
public void testSubscriberCount() throws Exception {
assertEquals(-1, extractor.getSubscriberCount());
}
@Override
@Test
public void testVerified() throws Exception {
assertFalse(extractor.isVerified());
}
@Override
@Test
public void testTabs() throws Exception {
assertTrue(extractor.getTabs().isEmpty());
Set<String> tabs = extractor.getTabs().stream()
.map(linkHandler -> linkHandler.getContentFilters().get(0)).collect(Collectors.toSet());
assertTrue(tabs.contains(ChannelTabs.ALBUMS));
}
@Override
@ -77,27 +81,27 @@ public class BandcampChannelExtractorTest implements BaseChannelExtractorTest {
// not implemented
}
@Override
@Test
public void testServiceId() {
assertEquals(Bandcamp.getServiceId(), extractor.getServiceId());
}
@Override
@Test
public void testName() throws Exception {
assertEquals("toupie", extractor.getName());
}
@Override
@Test
public void testId() throws Exception {
assertEquals("https://toupie.bandcamp.com/", extractor.getId());
assertEquals("2450875064", extractor.getId());
}
@Override
@Test
public void testUrl() throws Exception {
assertEquals("https://toupie.bandcamp.com", extractor.getUrl());
}
@Override
@Test
public void testOriginalUrl() throws Exception {
assertEquals("https://toupie.bandcamp.com", extractor.getUrl());
}

20
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java
View File

@ -7,9 +7,13 @@ import org.schabi.newpipe.extractor.ExtractorAsserts;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeAccountExtractor;
import java.util.Set;
import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
@ -105,14 +109,16 @@ public class PeertubeAccountExtractorTest {
ExtractorAsserts.assertGreaterOrEqual(700, extractor.getSubscriberCount());
}
@Override
@Test
public void testVerified() throws Exception {
assertFalse(extractor.isVerified());
}
@Override
@Test
public void testTabs() throws Exception {
Set<String> tabs = extractor.getTabs().stream()
.map(linkHandler -> linkHandler.getContentFilters().get(0)).collect(Collectors.toSet());
assertTrue(tabs.contains(ChannelTabs.CHANNELS));
}
}
@ -211,14 +217,16 @@ public class PeertubeAccountExtractorTest {
ExtractorAsserts.assertGreaterOrEqual(100, extractor.getSubscriberCount());
}
@Override
@Test
public void testVerified() throws Exception {
assertFalse(extractor.isVerified());
}
@Override
@Test
public void testTabs() throws Exception {
// TODO: implement Peertube account channels tab
Set<String> tabs = extractor.getTabs().stream()
.map(linkHandler -> linkHandler.getContentFilters().get(0)).collect(Collectors.toSet());
assertTrue(tabs.contains(ChannelTabs.CHANNELS));
}
}
}

20
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java
View File

@ -7,9 +7,13 @@ import org.schabi.newpipe.extractor.ExtractorAsserts;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor;
import java.util.Set;
import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
@ -120,14 +124,16 @@ public class PeertubeChannelExtractorTest {
ExtractorAsserts.assertGreaterOrEqual(230, extractor.getSubscriberCount());
}
@Override
@Test
public void testVerified() throws Exception {
assertFalse(extractor.isVerified());
}
@Override
@Test
public void testTabs() throws Exception {
Set<String> tabs = extractor.getTabs().stream()
.map(linkHandler -> linkHandler.getContentFilters().get(0)).collect(Collectors.toSet());
assertTrue(tabs.contains(ChannelTabs.PLAYLISTS));
}
}
@ -242,14 +248,16 @@ public class PeertubeChannelExtractorTest {
ExtractorAsserts.assertGreaterOrEqual(700, extractor.getSubscriberCount());
}
@Override
@Test
public void testVerified() throws Exception {
assertFalse(extractor.isVerified());
}
@Override
@Test
public void testTabs() throws Exception {
// TODO: implement Peertube channel playlists tab
Set<String> tabs = extractor.getTabs().stream()
.map(linkHandler -> linkHandler.getContentFilters().get(0)).collect(Collectors.toSet());
assertTrue(tabs.contains(ChannelTabs.PLAYLISTS));
}
}
}

22
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java
View File

@ -6,9 +6,13 @@ import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChannelExtractor;
import java.util.Set;
import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
@ -102,14 +106,17 @@ public class SoundcloudChannelExtractorTest {
assertTrue(extractor.getSubscriberCount() >= 1e6, "Wrong subscriber count");
}
@Override
@Test
public void testVerified() throws Exception {
assertTrue(extractor.isVerified());
}
@Override
@Test
public void testTabs() throws Exception {
Set<String> tabs = extractor.getTabs().stream()
.map(linkHandler -> linkHandler.getContentFilters().get(0)).collect(Collectors.toSet());
assertTrue(tabs.contains(ChannelTabs.PLAYLISTS));
assertTrue(tabs.contains(ChannelTabs.ALBUMS));
}
}
@ -206,14 +213,17 @@ public class SoundcloudChannelExtractorTest {
assertTrue(extractor.getSubscriberCount() >= 2e6, "Wrong subscriber count");
}
@Override
@Test
public void testVerified() throws Exception {
assertTrue(extractor.isVerified());
}
@Override
@Test
public void testTabs() throws Exception {
// TODO: implement soundcloud playlist tab
Set<String> tabs = extractor.getTabs().stream()
.map(linkHandler -> linkHandler.getContentFilters().get(0)).collect(Collectors.toSet());
assertTrue(tabs.contains(ChannelTabs.PLAYLISTS));
assertTrue(tabs.contains(ChannelTabs.ALBUMS));
}
}
}