diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java index 7bfa85662..db2e66c4b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java @@ -1,15 +1,15 @@ package org.schabi.newpipe.extractor.channel; -import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.Extractor; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import javax.annotation.Nonnull; import java.util.Collections; import java.util.List; +import javax.annotation.Nonnull; + /* * Created by Christian Schabesberger on 25.07.16. * @@ -30,7 +30,7 @@ import java.util.List; * along with NewPipe. If not, see . */ -public abstract class ChannelExtractor extends ListExtractor { +public abstract class ChannelExtractor extends Extractor { public static final long UNKNOWN_SUBSCRIBER_COUNT = -1; @@ -48,9 +48,7 @@ public abstract class ChannelExtractor extends ListExtractor { public abstract String getParentChannelAvatarUrl() throws ParsingException; public abstract boolean isVerified() throws ParsingException; @Nonnull - public List getTabs() throws ParsingException { - return Collections.emptyList(); - } + public abstract List getTabs() throws ParsingException; @Nonnull public List getTags() throws ParsingException { return Collections.emptyList(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java index 226bfc0d0..bdcce0c44 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java @@ -1,20 +1,17 @@ package org.schabi.newpipe.extractor.channel; -import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; -import org.schabi.newpipe.extractor.ListInfo; +import org.schabi.newpipe.extractor.Info; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import org.schabi.newpipe.extractor.utils.ExtractorHelper; -import javax.annotation.Nonnull; import java.io.IOException; import java.util.Collections; import java.util.List; +import javax.annotation.Nonnull; + /* * Created by Christian Schabesberger on 31.07.16. * @@ -35,16 +32,14 @@ import java.util.List; * along with NewPipe. If not, see . */ -public class ChannelInfo extends ListInfo { +public class ChannelInfo extends Info { public ChannelInfo(final int serviceId, final String id, final String url, final String originalUrl, - final String name, - final ListLinkHandler listLinkHandler) { - super(serviceId, id, url, originalUrl, name, listLinkHandler.getContentFilters(), - listLinkHandler.getSortFilter()); + final String name) { + super(serviceId, id, url, originalUrl, name); } public static ChannelInfo getInfo(final String url) throws IOException, ExtractionException { @@ -58,13 +53,6 @@ public class ChannelInfo extends ListInfo { return getInfo(extractor); } - public static InfoItemsPage getMoreItems(final StreamingService service, - final String url, - final Page page) - throws IOException, ExtractionException { - return service.getChannelExtractor(url).getPage(page); - } - public static ChannelInfo getInfo(final ChannelExtractor extractor) throws IOException, ExtractionException { @@ -74,8 +62,7 @@ public class ChannelInfo extends ListInfo { final String originalUrl = extractor.getOriginalUrl(); final String name = extractor.getName(); - final ChannelInfo info = - new ChannelInfo(serviceId, id, url, originalUrl, name, extractor.getLinkHandler()); + final ChannelInfo info = new ChannelInfo(serviceId, id, url, originalUrl, name); try { info.setAvatarUrl(extractor.getAvatarUrl()); @@ -93,11 +80,6 @@ public class ChannelInfo extends ListInfo { info.addError(e); } - final InfoItemsPage itemsPage = - ExtractorHelper.getItemsPageOrLogError(info, extractor); - info.setRelatedItems(itemsPage.getItems()); - info.setNextPage(itemsPage.getNextPage()); - try { info.setSubscriberCount(extractor.getSubscriberCount()); } catch (final Exception e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ChannelTabs.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ChannelTabs.java index 5ff0f06f8..91322acc2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ChannelTabs.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ChannelTabs.java @@ -1,6 +1,8 @@ package org.schabi.newpipe.extractor.linkhandler; public final class ChannelTabs { + public static final String VIDEOS = "videos"; + public static final String TRACKS = "tracks"; public static final String SHORTS = "shorts"; public static final String LIVESTREAMS = "livestreams"; public static final String CHANNELS = "channels"; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ReadyChannelTabListLinkHandler.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ReadyChannelTabListLinkHandler.java new file mode 100644 index 000000000..4a416adf6 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ReadyChannelTabListLinkHandler.java @@ -0,0 +1,28 @@ +package org.schabi.newpipe.extractor.linkhandler; + +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelTabExtractor; + +import java.util.Collections; +import java.util.List; + +public class ReadyChannelTabListLinkHandler extends ListLinkHandler { + + public interface ChannelTabExtractorBuilder { + ChannelTabExtractor build(StreamingService service, ListLinkHandler linkHandler); + } + + private final ChannelTabExtractorBuilder extractorBuilder; + + public ReadyChannelTabListLinkHandler(final String url, + final String channelId, + final String channelTab, + final ChannelTabExtractorBuilder extractorBuilder) { + super(url, url, channelId, Collections.singletonList(channelTab), ""); + this.extractorBuilder = extractorBuilder; + } + + public ChannelTabExtractor getChannelTabExtractor(final StreamingService service) { + return extractorBuilder.build(service, new ListLinkHandler(this)); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java index 33475f038..73306da4b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java @@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; @@ -146,7 +147,11 @@ public class BandcampService extends StreamingService { @Override public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler) { - return new BandcampChannelTabExtractor(this, linkHandler); + if (linkHandler instanceof ReadyChannelTabListLinkHandler) { + return ((ReadyChannelTabListLinkHandler) linkHandler).getChannelTabExtractor(this); + } else { + return new BandcampChannelTabExtractor(this, linkHandler); + } } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java index 22d68321f..a49c7a635 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java @@ -2,32 +2,37 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; + import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; + import org.jsoup.Jsoup; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.channel.ChannelTabExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.ChannelTabs; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler; import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.BandcampDiscographStreamInfoItemExtractor; import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampChannelTabHandler; import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampChannelTabLinkHandlerFactory; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import javax.annotation.Nonnull; import java.io.IOException; -import java.util.Collections; +import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.stream.Stream; -import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; +import javax.annotation.Nonnull; public class BandcampChannelExtractor extends ChannelExtractor { @@ -113,41 +118,18 @@ public class BandcampChannelExtractor extends ChannelExtractor { public List getTabs() throws ParsingException { final JsonArray discography = channelInfo.getArray("discography"); + final List tabs = new ArrayList<>(); + tabs.add(new ReadyChannelTabListLinkHandler(getUrl(), getId(), + ChannelTabs.TRACKS, this::buildTracksTabExtractor)); + if (discography.stream().anyMatch(o -> ( (JsonObject) o).getString("item_type").equals("album"))) { - return Collections.singletonList( - new BandcampChannelTabHandler(getUrl() - + BandcampChannelTabLinkHandlerFactory.URL_SUFFIX, - getId(), ChannelTabs.ALBUMS, discography)); - } - return Collections.emptyList(); - } - - @Nonnull - @Override - public InfoItemsPage getInitialPage() throws ParsingException { - - final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - - final JsonArray discography = channelInfo.getArray("discography"); - - for (int i = 0; i < discography.size(); i++) { - // A discograph is as an item appears in a discography - final JsonObject discograph = discography.getObject(i); - - if (!discograph.getString("item_type").equals("track")) { - continue; - } - - collector.commit(new BandcampDiscographStreamInfoItemExtractor(discograph, getUrl())); + tabs.add(new BandcampChannelTabHandler( + getUrl() + BandcampChannelTabLinkHandlerFactory.URL_SUFFIX, + getId(), ChannelTabs.ALBUMS, discography)); } - return new InfoItemsPage<>(collector, null); - } - - @Override - public InfoItemsPage getPage(final Page page) { - return null; + return tabs; } @Override @@ -161,4 +143,42 @@ public class BandcampChannelExtractor extends ChannelExtractor { public String getName() { return channelInfo.getString("name"); } + + private ChannelTabExtractor buildTracksTabExtractor(final StreamingService service, + final ListLinkHandler linkHandler) { + return new ChannelTabExtractor(service, linkHandler) { + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws ExtractionException { + final MultiInfoItemsCollector collector = + new MultiInfoItemsCollector(getServiceId()); + + final JsonArray discography = channelInfo.getArray("discography"); + + for (int i = 0; i < discography.size(); i++) { + // A discograph is as an item appears in a discography + final JsonObject discograph = discography.getObject(i); + + if (!discograph.getString("item_type").equals("track")) { + continue; + } + + collector.commit( + new BandcampDiscographStreamInfoItemExtractor(discograph, getUrl())); + } + + return new InfoItemsPage<>(collector, null); + } + + @Override + public InfoItemsPage getPage(final Page page) { + return ListExtractor.InfoItemsPage.emptyPage(); + } + + @Override + public void onFetchPage(@Nonnull final Downloader downloader) { + // nothing to do here, as data was already fetched + } + }; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java index d9016c178..733fce4db 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java @@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; @@ -85,9 +86,8 @@ public class MediaCCCService extends StreamingService { } @Override - public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler) - throws ExtractionException { - return null; + public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler) { + return ((ReadyChannelTabListLinkHandler) linkHandler).getChannelTabExtractor(this); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java index 6bd943598..f8d5fef80 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java @@ -4,23 +4,28 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.channel.ChannelTabExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ChannelTabs; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler; import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import javax.annotation.Nonnull; import java.io.IOException; import java.util.Collections; import java.util.List; +import javax.annotation.Nonnull; + public class MediaCCCConferenceExtractor extends ChannelExtractor { private JsonObject conferenceData; @@ -76,24 +81,9 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor { @Nonnull @Override - public List getTabs() { - return Collections.emptyList(); - } - - @Nonnull - @Override - public InfoItemsPage getInitialPage() { - final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final JsonArray events = conferenceData.getArray("events"); - for (int i = 0; i < events.size(); i++) { - collector.commit(new MediaCCCStreamInfoItemExtractor(events.getObject(i))); - } - return new InfoItemsPage<>(collector, null); - } - - @Override - public InfoItemsPage getPage(final Page page) { - return InfoItemsPage.emptyPage(); + public List getTabs() throws ParsingException { + return Collections.singletonList(new ReadyChannelTabListLinkHandler(getUrl(), getId(), + ChannelTabs.VIDEOS, this::buildEventsTabExtractor)); } @Override @@ -113,4 +103,31 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor { public String getName() throws ParsingException { return conferenceData.getString("title"); } + + private ChannelTabExtractor buildEventsTabExtractor(final StreamingService service, + final ListLinkHandler linkHandler) { + return new ChannelTabExtractor(service, linkHandler) { + @Nonnull + @Override + public InfoItemsPage getInitialPage() { + final MultiInfoItemsCollector collector = + new MultiInfoItemsCollector(getServiceId()); + final JsonArray events = conferenceData.getArray("events"); + for (int i = 0; i < events.size(); i++) { + collector.commit(new MediaCCCStreamInfoItemExtractor(events.getObject(i))); + } + return new InfoItemsPage<>(collector, null); + } + + @Override + public InfoItemsPage getPage(final Page page) { + return InfoItemsPage.emptyPage(); + } + + @Override + public void onFetchPage(@Nonnull final Downloader downloader) { + // nothing to do here, as data was already fetched + } + }; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index 260c3409a..8a5baf19a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -72,10 +72,10 @@ public final class PeertubeParsingHelper { } } - public static void collectStreamsFrom(final InfoItemsCollector collector, - final JsonObject json, - final String baseUrl) throws ParsingException { - collectStreamsFrom(collector, json, baseUrl, false); + public static void collectItemsFrom(final InfoItemsCollector collector, + final JsonObject json, + final String baseUrl) throws ParsingException { + collectItemsFrom(collector, json, baseUrl, false); } /** @@ -86,10 +86,10 @@ public final class PeertubeParsingHelper { * @param baseUrl the base Url of the instance * @param sepia if we should use PeertubeSepiaStreamInfoItemExtractor */ - public static void collectStreamsFrom(final InfoItemsCollector collector, - final JsonObject json, - final String baseUrl, - final boolean sepia) throws ParsingException { + public static void collectItemsFrom(final InfoItemsCollector collector, + final JsonObject json, + final String baseUrl, + final boolean sepia) throws ParsingException { final JsonArray contents; try { contents = (JsonArray) JsonUtils.getValue(json, "data"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index 6d6cfad39..5c3d2b2c2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -17,9 +17,8 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeAccountExtractor; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeAccountTabExtractor; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelTabExtractor; +import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubePlaylistExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor; @@ -117,14 +116,7 @@ public class PeertubeService extends StreamingService { @Override public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler) throws ExtractionException { - final String tab = linkHandler.getContentFilters().get(0); - switch (tab) { - case ChannelTabs.CHANNELS: - return new PeertubeAccountTabExtractor(this, linkHandler); - case ChannelTabs.PLAYLISTS: - return new PeertubeChannelTabExtractor(this, linkHandler); - } - throw new ParsingException("tab " + tab + " not supported"); + return new PeertubeChannelTabExtractor(this, linkHandler); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java index d163e6bb1..225f16324 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java @@ -4,7 +4,7 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.Page; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -14,24 +14,16 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.ChannelTabs; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelTabLinkHandlerFactory; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; -import org.schabi.newpipe.extractor.utils.Utils; -import javax.annotation.Nonnull; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.List; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import javax.annotation.Nonnull; public class PeertubeAccountExtractor extends ChannelExtractor { private JsonObject json; @@ -126,62 +118,19 @@ public class PeertubeAccountExtractor extends ChannelExtractor { @Nonnull @Override public List getTabs() throws ParsingException { - return Collections.singletonList( + return Arrays.asList( + PeertubeChannelTabLinkHandlerFactory.getInstance().fromQuery(getId(), + Collections.singletonList(ChannelTabs.VIDEOS), "", getBaseUrl()), PeertubeChannelTabLinkHandlerFactory.getInstance().fromQuery(getId(), Collections.singletonList(ChannelTabs.CHANNELS), "", getBaseUrl()) ); } - @Nonnull - @Override - public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - return getPage(new Page(baseUrl + "/api/v1/" + getId() + "/videos?" + START_KEY + "=0&" - + COUNT_KEY + "=" + ITEMS_PER_PAGE)); - } - - @Override - public InfoItemsPage getPage(final Page page) - throws IOException, ExtractionException { - if (page == null || isNullOrEmpty(page.getUrl())) { - throw new IllegalArgumentException("Page doesn't contain an URL"); - } - - final Response response = getDownloader().get(page.getUrl()); - - JsonObject pageJson = null; - if (response != null && !Utils.isBlank(response.responseBody())) { - try { - pageJson = JsonParser.object().from(response.responseBody()); - } catch (final Exception e) { - throw new ParsingException("Could not parse json data for account info", e); - } - } - - if (pageJson != null) { - PeertubeParsingHelper.validate(pageJson); - final long total = pageJson.getLong("total"); - - final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - collectStreamsFrom(collector, pageJson, getBaseUrl()); - - return new InfoItemsPage<>(collector, - PeertubeParsingHelper.getNextPage(page.getUrl(), total)); - } else { - throw new ExtractionException("Unable to get PeerTube account info"); - } - } - @Override public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { - String accountUrl = baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT; - if (getId().contains(ACCOUNTS)) { - accountUrl += getId(); - } else { - accountUrl += ACCOUNTS + getId(); - } - - final Response response = downloader.get(accountUrl); + final Response response = downloader.get(baseUrl + + PeertubeChannelLinkHandlerFactory.API_ENDPOINT + getId()); if (response != null) { setInitialData(response.responseBody()); } else { @@ -205,4 +154,14 @@ public class PeertubeAccountExtractor extends ChannelExtractor { public String getName() throws ParsingException { return JsonUtils.getString(json, "displayName"); } + + @Nonnull + @Override + public String getId() throws ParsingException { + if (super.getId().contains(ACCOUNTS)) { + return super.getId(); + } else { + return ACCOUNTS + super.getId(); + } + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountTabExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountTabExtractor.java deleted file mode 100644 index b4b1b53fb..000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountTabExtractor.java +++ /dev/null @@ -1,100 +0,0 @@ -package org.schabi.newpipe.extractor.services.peertube.extractors; - -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.MultiInfoItemsCollector; -import org.schabi.newpipe.extractor.Page; -import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.channel.ChannelTabExtractor; -import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.ChannelTabs; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; -import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; -import org.schabi.newpipe.extractor.utils.Utils; - -import javax.annotation.Nonnull; -import java.io.IOException; - -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; - -public class PeertubeAccountTabExtractor extends ChannelTabExtractor { - private final String baseUrl; - private static final String ACCOUNTS = "accounts/"; - - public PeertubeAccountTabExtractor(final StreamingService service, - final ListLinkHandler linkHandler) - throws ParsingException { - super(service, linkHandler); - baseUrl = getBaseUrl(); - } - - @Override - public void onFetchPage(final @Nonnull Downloader downloader) throws ParsingException { - if (!getTab().equals(ChannelTabs.CHANNELS)) { - throw new ParsingException("tab " + getTab() + " not supported"); - } - } - - @Nonnull - @Override - public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - String url = baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT; - if (getId().contains(ACCOUNTS)) { - url += getId(); - } else { - url += ACCOUNTS + getId(); - } - url += "/video-channels?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE; - return getPage(new Page(url)); - } - - @Override - public InfoItemsPage getPage(final Page page) - throws IOException, ExtractionException { - if (page == null || isNullOrEmpty(page.getUrl())) { - throw new IllegalArgumentException("Page doesn't contain an URL"); - } - - final Response response = getDownloader().get(page.getUrl()); - - JsonObject pageJson = null; - if (response != null && !Utils.isBlank(response.responseBody())) { - try { - pageJson = JsonParser.object().from(response.responseBody()); - } catch (final Exception e) { - throw new ParsingException("Could not parse json data for account info", e); - } - } - - if (pageJson == null) { - throw new ExtractionException("Unable to get account channel list"); - } - - PeertubeParsingHelper.validate(pageJson); - - final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); - final JsonArray contents = pageJson.getArray("data"); - if (contents == null) { - throw new ParsingException("Unable to extract account channel list"); - } - - contents.stream() - .filter(c -> c instanceof JsonObject) - .forEach(c -> collector.commit(new PeertubeChannelInfoItemExtractor( - (JsonObject) c, baseUrl)) - ); - - return new InfoItemsPage<>( - collector, PeertubeParsingHelper.getNextPage(page.getUrl(), - pageJson.getLong("total"))); - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java index 225c61ba6..3b1400ce5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java @@ -3,7 +3,7 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.Page; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -12,24 +12,16 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ChannelTabs; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelTabLinkHandlerFactory; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; -import org.schabi.newpipe.extractor.utils.Utils; -import javax.annotation.Nonnull; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.List; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import javax.annotation.Nonnull; public class PeertubeChannelExtractor extends ChannelExtractor { private JsonObject json; @@ -105,51 +97,14 @@ public class PeertubeChannelExtractor extends ChannelExtractor { @Nonnull @Override public List getTabs() throws ParsingException { - return Collections.singletonList( + return Arrays.asList( + PeertubeChannelTabLinkHandlerFactory.getInstance().fromQuery(getId(), + Collections.singletonList(ChannelTabs.VIDEOS), "", getBaseUrl()), PeertubeChannelTabLinkHandlerFactory.getInstance().fromQuery(getId(), Collections.singletonList(ChannelTabs.PLAYLISTS), "", getBaseUrl()) ); } - @Nonnull - @Override - public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - return getPage(new Page(baseUrl + "/api/v1/" + getId() + "/videos?" + START_KEY + "=0&" - + COUNT_KEY + "=" + ITEMS_PER_PAGE)); - } - - @Override - public InfoItemsPage getPage(final Page page) - throws IOException, ExtractionException { - if (page == null || isNullOrEmpty(page.getUrl())) { - throw new IllegalArgumentException("Page doesn't contain an URL"); - } - - final Response response = getDownloader().get(page.getUrl()); - - JsonObject pageJson = null; - if (response != null && !Utils.isBlank(response.responseBody())) { - try { - pageJson = JsonParser.object().from(response.responseBody()); - } catch (final Exception e) { - throw new ParsingException("Could not parse json data for channel info", e); - } - } - - if (pageJson != null) { - PeertubeParsingHelper.validate(pageJson); - final long total = pageJson.getLong("total"); - - final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - collectStreamsFrom(collector, pageJson, getBaseUrl()); - - return new InfoItemsPage<>(collector, - PeertubeParsingHelper.getNextPage(page.getUrl(), total)); - } else { - throw new ExtractionException("Unable to get PeerTube channel info"); - } - } - @Override public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelInfoItemExtractor.java index 3ab1be539..a82ec0a15 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelInfoItemExtractor.java @@ -1,6 +1,8 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -52,7 +54,7 @@ public class PeertubeChannelInfoItemExtractor implements ChannelInfoItemExtracto @Override public long getStreamCount() throws ParsingException { - return ChannelExtractor.ITEM_COUNT_UNKNOWN; + return ListExtractor.ITEM_COUNT_UNKNOWN; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelTabExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelTabExtractor.java index 1a3926bce..11e0f5f3c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelTabExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelTabExtractor.java @@ -1,6 +1,5 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; -import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import org.schabi.newpipe.extractor.InfoItem; @@ -16,6 +15,7 @@ import org.schabi.newpipe.extractor.linkhandler.ChannelTabs; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelTabLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; @@ -24,6 +24,8 @@ import java.io.IOException; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectItemsFrom; +import static org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelTabLinkHandlerFactory.getUrlSuffix; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class PeertubeChannelTabExtractor extends ChannelTabExtractor { @@ -37,18 +39,15 @@ public class PeertubeChannelTabExtractor extends ChannelTabExtractor { } @Override - public void onFetchPage(final @Nonnull Downloader downloader) throws ParsingException { - if (!getTab().equals(ChannelTabs.PLAYLISTS)) { - throw new ParsingException("tab " + getTab() + " not supported"); - } + public void onFetchPage(final @Nonnull Downloader downloader) { } @Nonnull @Override - public InfoItemsPage getInitialPage() - throws IOException, ExtractionException { - return getPage(new Page(baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT + getId() - + "/video-playlists?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE)); + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + return getPage(new Page(baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT + + getUrlSuffix(getTab()) + "?" + START_KEY + "=0&" + COUNT_KEY + "=" + + ITEMS_PER_PAGE)); } @Override @@ -70,25 +69,14 @@ public class PeertubeChannelTabExtractor extends ChannelTabExtractor { } if (pageJson == null) { - throw new ExtractionException("Unable to get channel playlist list"); + throw new ExtractionException("Unable to get account channel list"); } - PeertubeParsingHelper.validate(pageJson); final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); - final JsonArray contents = pageJson.getArray("data"); - if (contents == null) { - throw new ParsingException("Unable to extract channel playlist list"); - } + collectItemsFrom(collector, pageJson, getBaseUrl()); - for (final Object c : contents) { - if (c instanceof JsonObject) { - collector.commit(new PeertubePlaylistInfoItemExtractor((JsonObject) c, baseUrl)); - } - } - - return new InfoItemsPage<>( - collector, PeertubeParsingHelper.getNextPage(page.getUrl(), - pageJson.getLong("total"))); + return new InfoItemsPage<>(collector, + PeertubeParsingHelper.getNextPage(page.getUrl(), pageJson.getLong("total"))); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java index 71648b1ec..e2dd7fe66 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java @@ -22,7 +22,7 @@ import java.io.IOException; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectItemsFrom; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class PeertubePlaylistExtractor extends PlaylistExtractor { @@ -114,7 +114,7 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor { final long total = json.getLong("total"); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - collectStreamsFrom(collector, json, getBaseUrl()); + PeertubeParsingHelper.collectItemsFrom(collector, json, getBaseUrl()); return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPage(page.getUrl(), total)); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java index 8be07855d..5f7ac45d4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -26,7 +26,7 @@ import javax.annotation.Nonnull; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectItemsFrom; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class PeertubeSearchExtractor extends SearchExtractor { @@ -93,7 +93,7 @@ public class PeertubeSearchExtractor extends SearchExtractor { final long total = json.getLong("total"); final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); - collectStreamsFrom(collector, json, getBaseUrl(), sepia); + collectItemsFrom(collector, json, getBaseUrl(), sepia); return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPage(page.getUrl(), total)); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java index 2baca78ad..57f401058 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java @@ -23,7 +23,7 @@ import javax.annotation.Nonnull; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectItemsFrom; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class PeertubeTrendingExtractor extends KioskExtractor { @@ -69,7 +69,7 @@ public class PeertubeTrendingExtractor extends KioskExtractor { final long total = json.getLong("total"); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - collectStreamsFrom(collector, json, getBaseUrl()); + PeertubeParsingHelper.collectItemsFrom(collector, json, getBaseUrl()); return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPage(page.getUrl(), total)); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelTabLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelTabLinkHandlerFactory.java index 8cd6643c3..2450ecad5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelTabLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelTabLinkHandlerFactory.java @@ -17,12 +17,14 @@ public final class PeertubeChannelTabLinkHandlerFactory extends ListLinkHandlerF return INSTANCE; } - private static String getUrlSuffix(final String tab) throws ParsingException { + public static String getUrlSuffix(final String tab) throws ParsingException { switch (tab) { - case ChannelTabs.PLAYLISTS: - return "/video-playlists"; - case ChannelTabs.CHANNELS: + case ChannelTabs.VIDEOS: + return "/videos"; + case ChannelTabs.CHANNELS: // only available on accounts return "/video-channels"; + case ChannelTabs.PLAYLISTS: // only available on channels + return "/video-playlists"; } throw new ParsingException("tab " + tab + " not supported"); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java index d2a87ffa4..94ac17cbd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java @@ -1,9 +1,11 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; + import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.Page; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -13,17 +15,13 @@ import org.schabi.newpipe.extractor.linkhandler.ChannelTabs; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudChannelTabLinkHandlerFactory; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import javax.annotation.Nonnull; import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; -import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import javax.annotation.Nonnull; public class SoundcloudChannelExtractor extends ChannelExtractor { private String userId; @@ -112,49 +110,20 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { @Nonnull @Override public List getTabs() throws ParsingException { + final String urlTracks = getUrl() + + SoundcloudChannelTabLinkHandlerFactory.getUrlSuffix(ChannelTabs.TRACKS); final String urlPlaylists = getUrl() + SoundcloudChannelTabLinkHandlerFactory.getUrlSuffix(ChannelTabs.PLAYLISTS); final String urlAlbums = getUrl() + SoundcloudChannelTabLinkHandlerFactory.getUrlSuffix(ChannelTabs.ALBUMS); return Arrays.asList( + new ListLinkHandler(urlTracks, urlTracks, getId(), + Collections.singletonList(ChannelTabs.TRACKS), ""), new ListLinkHandler(urlPlaylists, urlPlaylists, getId(), Collections.singletonList(ChannelTabs.PLAYLISTS), ""), new ListLinkHandler(urlAlbums, urlAlbums, getId(), Collections.singletonList(ChannelTabs.ALBUMS), "") ); } - - @Nonnull - @Override - public InfoItemsPage getInitialPage() throws ExtractionException { - try { - final StreamInfoItemsCollector streamInfoItemsCollector = - new StreamInfoItemsCollector(getServiceId()); - - final String apiUrl = USERS_ENDPOINT + getId() + "/tracks" + "?client_id=" - + SoundcloudParsingHelper.clientId() + "&limit=20" + "&linked_partitioning=1"; - - final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, - streamInfoItemsCollector, apiUrl); - - return new InfoItemsPage<>(streamInfoItemsCollector, new Page(nextPageUrl)); - } catch (final Exception e) { - throw new ExtractionException("Could not get next page", e); - } - } - - @Override - public InfoItemsPage getPage(final Page page) throws IOException, - ExtractionException { - if (page == null || isNullOrEmpty(page.getUrl())) { - throw new IllegalArgumentException("Page doesn't contain an URL"); - } - - final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, - page.getUrl()); - - return new InfoItemsPage<>(collector, new Page(nextPageUrl)); - } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelTabExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelTabExtractor.java index 3b35d68e6..aef776703 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelTabExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelTabExtractor.java @@ -29,6 +29,8 @@ public class SoundcloudChannelTabExtractor extends ChannelTabExtractor { private String getEndpoint() { switch (getTab()) { + case ChannelTabs.TRACKS: + return "/tracks"; case ChannelTabs.PLAYLISTS: return "/playlists_without_albums"; case ChannelTabs.ALBUMS: @@ -38,8 +40,7 @@ public class SoundcloudChannelTabExtractor extends ChannelTabExtractor { } @Override - public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, - ExtractionException { + public void onFetchPage(@Nonnull final Downloader downloader) { } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelTabLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelTabLinkHandlerFactory.java index 09e4b60b9..0e073efdc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelTabLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelTabLinkHandlerFactory.java @@ -19,6 +19,8 @@ public final class SoundcloudChannelTabLinkHandlerFactory extends ListLinkHandle public static String getUrlSuffix(final String tab) throws ParsingException { switch (tab) { + case ChannelTabs.TRACKS: + return "/tracks"; case ChannelTabs.PLAYLISTS: return "/sets"; case ChannelTabs.ALBUMS: @@ -47,6 +49,7 @@ public final class SoundcloudChannelTabLinkHandlerFactory extends ListLinkHandle @Override public String[] getAvailableContentFilter() { return new String[] { + ChannelTabs.TRACKS, ChannelTabs.PLAYLISTS, ChannelTabs.ALBUMS, }; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 937dd4066..7ab951ae4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import org.schabi.newpipe.extractor.localization.ContentCountry; @@ -117,7 +118,11 @@ public class YoutubeService extends StreamingService { @Override public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler) { - return new YoutubeChannelTabExtractor(this, linkHandler); + if (linkHandler instanceof ReadyChannelTabListLinkHandler) { + return ((ReadyChannelTabListLinkHandler) linkHandler).getChannelTabExtractor(this); + } else { + return new YoutubeChannelTabExtractor(this, linkHandler); + } } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index d2fa072bd..3cde88b75 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -1,39 +1,29 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YouTubeChannelHelper.ChannelResponseData; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YouTubeChannelHelper.getChannelResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.services.youtube.YouTubeChannelHelper.resolveChannelId; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonWriter; -import org.schabi.newpipe.extractor.Page; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ChannelTabs; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -41,7 +31,6 @@ import java.util.function.Consumer; import java.util.stream.Collectors; import javax.annotation.Nonnull; -import javax.annotation.Nullable; /* * Created by Christian Schabesberger on 25.07.16. @@ -65,8 +54,6 @@ import javax.annotation.Nullable; public class YoutubeChannelExtractor extends ChannelExtractor { private JsonObject initialData; - private JsonObject videoTab; - private List tabs; /** * Some channels have response redirects and the only way to reliably get the id is by saving it @@ -226,162 +213,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Nonnull @Override public List getTabs() throws ParsingException { - return tabs; - } - - @Nonnull - @Override - public List getTags() throws ParsingException { - final JsonArray tags = initialData.getObject("microformat") - .getObject("microformatDataRenderer").getArray("tags"); - - return tags.stream().map(Object::toString).collect(Collectors.toList()); - } - - @Nonnull - @Override - public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - - Page nextPage = null; - extractTabs(); - - if (videoTab != null) { - final JsonObject tabContent = videoTab.getObject("content"); - JsonArray items = tabContent - .getObject("sectionListRenderer") - .getArray("contents").getObject(0).getObject("itemSectionRenderer") - .getArray("contents").getObject(0).getObject("gridRenderer").getArray("items"); - - if (items.isEmpty()) { - items = tabContent.getObject("richGridRenderer").getArray("contents"); - } - - final List channelIds = new ArrayList<>(); - channelIds.add(getName()); - channelIds.add(getUrl()); - final JsonObject continuation = collectStreamsFrom(collector, items, channelIds); - - nextPage = getNextPageFrom(continuation, channelIds); - } - - return new InfoItemsPage<>(collector, nextPage); - } - - @Override - public InfoItemsPage getPage(final Page page) - throws IOException, ExtractionException { - if (page == null || isNullOrEmpty(page.getUrl())) { - throw new IllegalArgumentException("Page doesn't contain an URL"); - } - - final List channelIds = page.getIds(); - - final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - - final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(), - getExtractorLocalization()); - - final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions") - .getObject(0) - .getObject("appendContinuationItemsAction"); - - final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation - .getArray("continuationItems"), channelIds); - - return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelIds)); - } - - @Nullable - private Page getNextPageFrom(final JsonObject continuations, - final List channelIds) - throws IOException, ExtractionException { - if (isNullOrEmpty(continuations)) { - return null; - } - - final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint"); - final String continuation = continuationEndpoint.getObject("continuationCommand") - .getString("token"); - - final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(), - getExtractorContentCountry()) - .value("continuation", continuation) - .done()) - .getBytes(StandardCharsets.UTF_8); - - return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey() - + DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body); - } - - /** - * Collect streams from an array of items - * - * @param collector the collector where videos will be committed - * @param videos the array to get videos from - * @param channelIds the ids of the channel, which are its name and its URL - * @return the continuation object - */ - private JsonObject collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector, - @Nonnull final JsonArray videos, - @Nonnull final List channelIds) { - collector.reset(); - - final String uploaderName = channelIds.get(0); - final String uploaderUrl = channelIds.get(1); - final TimeAgoParser timeAgoParser = getTimeAgoParser(); - - JsonObject continuation = null; - - for (final Object object : videos) { - final JsonObject video = (JsonObject) object; - if (video.has("gridVideoRenderer")) { - collector.commit(new YoutubeStreamInfoItemExtractor( - video.getObject("gridVideoRenderer"), timeAgoParser) { - @Override - public String getUploaderName() { - return uploaderName; - } - - @Override - public String getUploaderUrl() { - return uploaderUrl; - } - }); - } else if (video.has("richItemRenderer")) { - collector.commit(new YoutubeStreamInfoItemExtractor( - video.getObject("richItemRenderer") - .getObject("content").getObject("videoRenderer"), timeAgoParser) { - @Override - public String getUploaderName() { - return uploaderName; - } - - @Override - public String getUploaderUrl() { - return uploaderUrl; - } - }); - - } else if (video.has("continuationItemRenderer")) { - continuation = video.getObject("continuationItemRenderer"); - } - } - - return continuation; - } - - /** - * Collect a list of available tabs and get the video tab data. - */ - private void extractTabs() throws ParsingException { final JsonArray responseTabs = initialData.getObject("contents") .getObject("twoColumnBrowseResultsRenderer") .getArray("tabs"); - JsonObject foundVideoTab = null; - tabs = new ArrayList<>(); - + final List tabs = new ArrayList<>(); final Consumer addTab = tab -> { try { tabs.add(YoutubeChannelTabLinkHandlerFactory.getInstance().fromQuery( @@ -402,7 +238,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor { switch (urlSuffix) { case "videos": - foundVideoTab = tabRenderer; + // since the videos tab already has its contents fetched, make sure + // it is in the first position + String mutName = ""; + try { + mutName = getName(); + } catch (final ParsingException ignored) { + } + final String name = mutName; + final String url = getUrl(); + tabs.add(0, new ReadyChannelTabListLinkHandler(tabUrl, + redirectedChannelId, ChannelTabs.VIDEOS, + (service, linkHandler) -> new YoutubeChannelVideosTabExtractor( + service, linkHandler, tabRenderer, name, url))); break; case "playlists": addTab.accept(ChannelTabs.PLAYLISTS); @@ -421,28 +269,15 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } } - if (foundVideoTab == null) { - if (tabs.isEmpty()) { - throw new ContentNotSupportedException("This channel has no supported tabs"); - } - return; - } + return tabs; + } - final String messageRendererText = getTextFromObject( - foundVideoTab.getObject("content") - .getObject("sectionListRenderer") - .getArray("contents") - .getObject(0) - .getObject("itemSectionRenderer") - .getArray("contents") - .getObject(0) - .getObject("messageRenderer") - .getObject("text")); - if (messageRendererText != null - && messageRendererText.equals("This channel has no videos.")) { - return; - } + @Nonnull + @Override + public List getTags() throws ParsingException { + final JsonArray tags = initialData.getObject("microformat") + .getObject("microformatDataRenderer").getArray("tags"); - videoTab = foundVideoTab; + return tags.stream().map(Object::toString).collect(Collectors.toList()); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java index ce43922a2..5d4646174 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java @@ -71,8 +71,7 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor { initialData = data.responseJson; redirectedChannelId = data.channelId; - visitorData = - initialData.getObject("responseContext").getString("visitorData"); + visitorData = initialData.getObject("responseContext").getString("visitorData"); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelVideosTabExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelVideosTabExtractor.java new file mode 100644 index 000000000..757936985 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelVideosTabExtractor.java @@ -0,0 +1,180 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonWriter; + +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.MultiInfoItemsCollector; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelTabExtractor; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.localization.TimeAgoParser; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public class YoutubeChannelVideosTabExtractor extends ChannelTabExtractor { + private final JsonObject tabRenderer; + private final String channelName; + private final String channelUrl; + + public YoutubeChannelVideosTabExtractor(final StreamingService service, + final ListLinkHandler linkHandler, + final JsonObject tabRenderer, + final String channelName, + final String channelUrl) { + super(service, linkHandler); + this.tabRenderer = tabRenderer; + this.channelName = channelName; + this.channelUrl = channelUrl; + } + + @Override + public void onFetchPage(@Nonnull final Downloader downloader) + throws IOException, ExtractionException { + // nothing to do, all data was already fetched and is stored in the link handler + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); + + final JsonObject tabContent = tabRenderer.getObject("content"); + + JsonArray items = tabContent + .getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer") + .getArray("contents").getObject(0).getObject("gridRenderer").getArray("items"); + + if (items.isEmpty()) { + items = tabContent.getObject("richGridRenderer").getArray("contents"); + } + + final List channelIds = new ArrayList<>(); + channelIds.add(channelName); + channelIds.add(channelUrl); + final JsonObject continuation = collectStreamsFrom(collector, items, channelIds); + + return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelIds)); + } + + @Override + public InfoItemsPage getPage(final Page page) + throws IOException, ExtractionException { + if (page == null || isNullOrEmpty(page.getUrl())) { + throw new IllegalArgumentException("Page doesn't contain an URL"); + } + + final List channelIds = page.getIds(); + + final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); + + final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(), + getExtractorLocalization()); + + final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions") + .getObject(0) + .getObject("appendContinuationItemsAction"); + + final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation + .getArray("continuationItems"), channelIds); + + return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelIds)); + } + + @Nullable + private Page getNextPageFrom(final JsonObject continuations, + final List channelIds) + throws IOException, ExtractionException { + if (isNullOrEmpty(continuations)) { + return null; + } + + final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint"); + final String continuation = continuationEndpoint.getObject("continuationCommand") + .getString("token"); + + final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(), + getExtractorContentCountry()) + .value("continuation", continuation) + .done()) + .getBytes(StandardCharsets.UTF_8); + + return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey() + + DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body); + } + + /** + * Collect streams from an array of items + * + * @param collector the collector where videos will be committed + * @param videos the array to get videos from + * @param channelIds the ids of the channel, which are its name and its URL + * @return the continuation object + */ + private JsonObject collectStreamsFrom(@Nonnull final MultiInfoItemsCollector collector, + @Nonnull final JsonArray videos, + @Nonnull final List channelIds) { + collector.reset(); + + final String uploaderName = channelIds.get(0); + final String uploaderUrl = channelIds.get(1); + final TimeAgoParser timeAgoParser = getTimeAgoParser(); + + JsonObject continuation = null; + + for (final Object object : videos) { + final JsonObject video = (JsonObject) object; + if (video.has("gridVideoRenderer")) { + collector.commit(new YoutubeStreamInfoItemExtractor( + video.getObject("gridVideoRenderer"), timeAgoParser) { + @Override + public String getUploaderName() { + return uploaderName; + } + + @Override + public String getUploaderUrl() { + return uploaderUrl; + } + }); + } else if (video.has("richItemRenderer")) { + collector.commit(new YoutubeStreamInfoItemExtractor( + video.getObject("richItemRenderer") + .getObject("content").getObject("videoRenderer"), timeAgoParser) { + @Override + public String getUploaderName() { + return uploaderName; + } + + @Override + public String getUploaderUrl() { + return uploaderUrl; + } + }); + + } else if (video.has("continuationItemRenderer")) { + continuation = video.getObject("continuationItemRenderer"); + } + } + + return continuation; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelTabLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelTabLinkHandlerFactory.java index 2b34067c4..3ae6a89c7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelTabLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelTabLinkHandlerFactory.java @@ -19,6 +19,8 @@ public final class YoutubeChannelTabLinkHandlerFactory extends ListLinkHandlerFa public static String getUrlSuffix(final String tab) throws ParsingException { switch (tab) { + case ChannelTabs.VIDEOS: + return "/videos"; case ChannelTabs.PLAYLISTS: return "/playlists"; case ChannelTabs.LIVESTREAMS: @@ -55,6 +57,7 @@ public final class YoutubeChannelTabLinkHandlerFactory extends ListLinkHandlerFa @Override public String[] getAvailableContentFilter() { return new String[] { + ChannelTabs.VIDEOS, ChannelTabs.SHORTS, ChannelTabs.LIVESTREAMS, ChannelTabs.CHANNELS, diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelTabExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelTabExtractorTest.java index 2185fe458..5ac295b88 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelTabExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelTabExtractorTest.java @@ -7,7 +7,6 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ChannelTabs; -import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeAccountTabExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelTabExtractor; import java.io.IOException; @@ -56,12 +55,12 @@ public class PeertubeChannelTabExtractorTest { } public static class Channels { - private static PeertubeAccountTabExtractor extractor; + private static PeertubeChannelTabExtractor extractor; @BeforeAll public static void setUp() throws IOException, ExtractionException { NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = (PeertubeAccountTabExtractor) PeerTube + extractor = (PeertubeChannelTabExtractor) PeerTube .getChannelTabExtractorFromId("accounts/framasoft", ChannelTabs.CHANNELS, "https://framatube.org"); extractor.fetchPage();