lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Channels are now an Info 

The previous "main" tab is now just a normal tab returned in getTabs().
This is a breaking change.
This commit is contained in:
Stypox 2023-04-13 11:03:37 +02:00
parent e278a2d6d4
commit c3651bef5c
No known key found for this signature in database
GPG Key ID: 4BDF1B40A49FDD23
28 changed files with 438 additions and 594 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java
View File

@ -1,15 +1,15 @@
package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.Extractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import javax.annotation.Nonnull;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nonnull;
/*
* Created by Christian Schabesberger on 25.07.16.
*
@ -30,7 +30,7 @@ import java.util.List;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public abstract class ChannelExtractor extends ListExtractor<StreamInfoItem> {
public abstract class ChannelExtractor extends Extractor {
public static final long UNKNOWN_SUBSCRIBER_COUNT = -1;
@ -48,9 +48,7 @@ public abstract class ChannelExtractor extends ListExtractor<StreamInfoItem> {
public abstract String getParentChannelAvatarUrl() throws ParsingException;
public abstract boolean isVerified() throws ParsingException;
@Nonnull
public List<ListLinkHandler> getTabs() throws ParsingException {
return Collections.emptyList();
}
public abstract List<ListLinkHandler> getTabs() throws ParsingException;
@Nonnull
public List<String> getTags() throws ParsingException {
return Collections.emptyList();

32
extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java
View File

@ -1,20 +1,17 @@
package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.Info;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nonnull;
/*
* Created by Christian Schabesberger on 31.07.16.
*
@ -35,16 +32,14 @@ import java.util.List;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class ChannelInfo extends ListInfo<StreamInfoItem> {
public class ChannelInfo extends Info {
public ChannelInfo(final int serviceId,
final String id,
final String url,
final String originalUrl,
final String name,
final ListLinkHandler listLinkHandler) {
super(serviceId, id, url, originalUrl, name, listLinkHandler.getContentFilters(),
listLinkHandler.getSortFilter());
final String name) {
super(serviceId, id, url, originalUrl, name);
}
public static ChannelInfo getInfo(final String url) throws IOException, ExtractionException {
@ -58,13 +53,6 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
return getInfo(extractor);
}
public static InfoItemsPage<StreamInfoItem> getMoreItems(final StreamingService service,
final String url,
final Page page)
throws IOException, ExtractionException {
return service.getChannelExtractor(url).getPage(page);
}
public static ChannelInfo getInfo(final ChannelExtractor extractor)
throws IOException, ExtractionException {
@ -74,8 +62,7 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
final String originalUrl = extractor.getOriginalUrl();
final String name = extractor.getName();
final ChannelInfo info =
new ChannelInfo(serviceId, id, url, originalUrl, name, extractor.getLinkHandler());
final ChannelInfo info = new ChannelInfo(serviceId, id, url, originalUrl, name);
try {
info.setAvatarUrl(extractor.getAvatarUrl());
@ -93,11 +80,6 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
info.addError(e);
}
final InfoItemsPage<StreamInfoItem> itemsPage =
ExtractorHelper.getItemsPageOrLogError(info, extractor);
info.setRelatedItems(itemsPage.getItems());
info.setNextPage(itemsPage.getNextPage());
try {
info.setSubscriberCount(extractor.getSubscriberCount());
} catch (final Exception e) {

2
extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ChannelTabs.java
View File

@ -1,6 +1,8 @@
package org.schabi.newpipe.extractor.linkhandler;
public final class ChannelTabs {
public static final String VIDEOS = "videos";
public static final String TRACKS = "tracks";
public static final String SHORTS = "shorts";
public static final String LIVESTREAMS = "livestreams";
public static final String CHANNELS = "channels";

28
extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ReadyChannelTabListLinkHandler.java Normal file
View File

@ -0,0 +1,28 @@
package org.schabi.newpipe.extractor.linkhandler;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelTabExtractor;
import java.util.Collections;
import java.util.List;
public class ReadyChannelTabListLinkHandler extends ListLinkHandler {
public interface ChannelTabExtractorBuilder {
ChannelTabExtractor build(StreamingService service, ListLinkHandler linkHandler);
}
private final ChannelTabExtractorBuilder extractorBuilder;
public ReadyChannelTabListLinkHandler(final String url,
final String channelId,
final String channelTab,
final ChannelTabExtractorBuilder extractorBuilder) {
super(url, url, channelId, Collections.singletonList(channelTab), "");
this.extractorBuilder = extractorBuilder;
}
public ChannelTabExtractor getChannelTabExtractor(final StreamingService service) {
return extractorBuilder.build(service, new ListLinkHandler(this));
}
}

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java
View File

@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
@ -146,7 +147,11 @@ public class BandcampService extends StreamingService {
@Override
public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler) {
return new BandcampChannelTabExtractor(this, linkHandler);
if (linkHandler instanceof ReadyChannelTabListLinkHandler) {
return ((ReadyChannelTabListLinkHandler) linkHandler).getChannelTabExtractor(this);
} else {
return new BandcampChannelTabExtractor(this, linkHandler);
}
}
@Override

92
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java
View File

@ -2,32 +2,37 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.jsoup.Jsoup;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.ChannelTabExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.BandcampDiscographStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampChannelTabHandler;
import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Stream;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
import javax.annotation.Nonnull;
public class BandcampChannelExtractor extends ChannelExtractor {
@ -113,41 +118,18 @@ public class BandcampChannelExtractor extends ChannelExtractor {
public List<ListLinkHandler> getTabs() throws ParsingException {
final JsonArray discography = channelInfo.getArray("discography");
final List<ListLinkHandler> tabs = new ArrayList<>();
tabs.add(new ReadyChannelTabListLinkHandler(getUrl(), getId(),
ChannelTabs.TRACKS, this::buildTracksTabExtractor));
if (discography.stream().anyMatch(o -> (
(JsonObject) o).getString("item_type").equals("album"))) {
return Collections.singletonList(
new BandcampChannelTabHandler(getUrl()
+ BandcampChannelTabLinkHandlerFactory.URL_SUFFIX,
getId(), ChannelTabs.ALBUMS, discography));
}
return Collections.emptyList();
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonArray discography = channelInfo.getArray("discography");
for (int i = 0; i < discography.size(); i++) {
// A discograph is as an item appears in a discography
final JsonObject discograph = discography.getObject(i);
if (!discograph.getString("item_type").equals("track")) {
continue;
}
collector.commit(new BandcampDiscographStreamInfoItemExtractor(discograph, getUrl()));
tabs.add(new BandcampChannelTabHandler(
getUrl() + BandcampChannelTabLinkHandlerFactory.URL_SUFFIX,
getId(), ChannelTabs.ALBUMS, discography));
}
return new InfoItemsPage<>(collector, null);
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) {
return null;
return tabs;
}
@Override
@ -161,4 +143,42 @@ public class BandcampChannelExtractor extends ChannelExtractor {
public String getName() {
return channelInfo.getString("name");
}
private ChannelTabExtractor buildTracksTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
return new ChannelTabExtractor(service, linkHandler) {
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws ExtractionException {
final MultiInfoItemsCollector collector =
new MultiInfoItemsCollector(getServiceId());
final JsonArray discography = channelInfo.getArray("discography");
for (int i = 0; i < discography.size(); i++) {
// A discograph is as an item appears in a discography
final JsonObject discograph = discography.getObject(i);
if (!discograph.getString("item_type").equals("track")) {
continue;
}
collector.commit(
new BandcampDiscographStreamInfoItemExtractor(discograph, getUrl()));
}
return new InfoItemsPage<>(collector, null);
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page) {
return ListExtractor.InfoItemsPage.emptyPage();
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) {
// nothing to do here, as data was already fetched
}
};
}
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java
View File

@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
@ -85,9 +86,8 @@ public class MediaCCCService extends StreamingService {
}
@Override
public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler)
throws ExtractionException {
return null;
public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler) {
return ((ReadyChannelTabListLinkHandler) linkHandler).getChannelTabExtractor(this);
}
@Override

59
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java
View File

@ -4,23 +4,28 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.ChannelTabExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nonnull;
public class MediaCCCConferenceExtractor extends ChannelExtractor {
private JsonObject conferenceData;
@ -76,24 +81,9 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
@Nonnull
@Override
public List<ListLinkHandler> getTabs() {
return Collections.emptyList();
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonArray events = conferenceData.getArray("events");
for (int i = 0; i < events.size(); i++) {
collector.commit(new MediaCCCStreamInfoItemExtractor(events.getObject(i)));
}
return new InfoItemsPage<>(collector, null);
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) {
return InfoItemsPage.emptyPage();
public List<ListLinkHandler> getTabs() throws ParsingException {
return Collections.singletonList(new ReadyChannelTabListLinkHandler(getUrl(), getId(),
ChannelTabs.VIDEOS, this::buildEventsTabExtractor));
}
@Override
@ -113,4 +103,31 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
public String getName() throws ParsingException {
return conferenceData.getString("title");
}
private ChannelTabExtractor buildEventsTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
return new ChannelTabExtractor(service, linkHandler) {
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() {
final MultiInfoItemsCollector collector =
new MultiInfoItemsCollector(getServiceId());
final JsonArray events = conferenceData.getArray("events");
for (int i = 0; i < events.size(); i++) {
collector.commit(new MediaCCCStreamInfoItemExtractor(events.getObject(i)));
}
return new InfoItemsPage<>(collector, null);
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page) {
return InfoItemsPage.emptyPage();
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) {
// nothing to do here, as data was already fetched
}
};
}
}

16
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java
View File

@ -72,10 +72,10 @@ public final class PeertubeParsingHelper {
}
}
public static void collectStreamsFrom(final InfoItemsCollector collector,
final JsonObject json,
final String baseUrl) throws ParsingException {
collectStreamsFrom(collector, json, baseUrl, false);
public static void collectItemsFrom(final InfoItemsCollector collector,
final JsonObject json,
final String baseUrl) throws ParsingException {
collectItemsFrom(collector, json, baseUrl, false);
}
/**
@ -86,10 +86,10 @@ public final class PeertubeParsingHelper {
* @param baseUrl the base Url of the instance
* @param sepia if we should use PeertubeSepiaStreamInfoItemExtractor
*/
public static void collectStreamsFrom(final InfoItemsCollector collector,
final JsonObject json,
final String baseUrl,
final boolean sepia) throws ParsingException {
public static void collectItemsFrom(final InfoItemsCollector collector,
final JsonObject json,
final String baseUrl,
final boolean sepia) throws ParsingException {
final JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");

12
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java
View File

@ -17,9 +17,8 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeAccountExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeAccountTabExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelTabExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubePlaylistExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor;
@ -117,14 +116,7 @@ public class PeertubeService extends StreamingService {
@Override
public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler)
throws ExtractionException {
final String tab = linkHandler.getContentFilters().get(0);
switch (tab) {
case ChannelTabs.CHANNELS:
return new PeertubeAccountTabExtractor(this, linkHandler);
case ChannelTabs.PLAYLISTS:
return new PeertubeChannelTabExtractor(this, linkHandler);
}
throw new ParsingException("tab " + tab + " not supported");
return new PeertubeChannelTabExtractor(this, linkHandler);
}
@Override

77
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java
View File

@ -4,7 +4,7 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -14,24 +14,16 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import javax.annotation.Nonnull;
public class PeertubeAccountExtractor extends ChannelExtractor {
private JsonObject json;
@ -126,62 +118,19 @@ public class PeertubeAccountExtractor extends ChannelExtractor {
@Nonnull
@Override
public List<ListLinkHandler> getTabs() throws ParsingException {
return Collections.singletonList(
return Arrays.asList(
PeertubeChannelTabLinkHandlerFactory.getInstance().fromQuery(getId(),
Collections.singletonList(ChannelTabs.VIDEOS), "", getBaseUrl()),
PeertubeChannelTabLinkHandlerFactory.getInstance().fromQuery(getId(),
Collections.singletonList(ChannelTabs.CHANNELS), "", getBaseUrl())
);
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
return getPage(new Page(baseUrl + "/api/v1/" + getId() + "/videos?" + START_KEY + "=0&"
+ COUNT_KEY + "=" + ITEMS_PER_PAGE));
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final Response response = getDownloader().get(page.getUrl());
JsonObject pageJson = null;
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
pageJson = JsonParser.object().from(response.responseBody());
} catch (final Exception e) {
throw new ParsingException("Could not parse json data for account info", e);
}
}
if (pageJson != null) {
PeertubeParsingHelper.validate(pageJson);
final long total = pageJson.getLong("total");
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, pageJson, getBaseUrl());
return new InfoItemsPage<>(collector,
PeertubeParsingHelper.getNextPage(page.getUrl(), total));
} else {
throw new ExtractionException("Unable to get PeerTube account info");
}
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
String accountUrl = baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT;
if (getId().contains(ACCOUNTS)) {
accountUrl += getId();
} else {
accountUrl += ACCOUNTS + getId();
}
final Response response = downloader.get(accountUrl);
final Response response = downloader.get(baseUrl
+ PeertubeChannelLinkHandlerFactory.API_ENDPOINT + getId());
if (response != null) {
setInitialData(response.responseBody());
} else {
@ -205,4 +154,14 @@ public class PeertubeAccountExtractor extends ChannelExtractor {
public String getName() throws ParsingException {
return JsonUtils.getString(json, "displayName");
}
@Nonnull
@Override
public String getId() throws ParsingException {
if (super.getId().contains(ACCOUNTS)) {
return super.getId();
} else {
return ACCOUNTS + super.getId();
}
}
}

100
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountTabExtractor.java
View File

@ -1,100 +0,0 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelTabExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubeAccountTabExtractor extends ChannelTabExtractor {
private final String baseUrl;
private static final String ACCOUNTS = "accounts/";
public PeertubeAccountTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler)
throws ParsingException {
super(service, linkHandler);
baseUrl = getBaseUrl();
}
@Override
public void onFetchPage(final @Nonnull Downloader downloader) throws ParsingException {
if (!getTab().equals(ChannelTabs.CHANNELS)) {
throw new ParsingException("tab " + getTab() + " not supported");
}
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
String url = baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT;
if (getId().contains(ACCOUNTS)) {
url += getId();
} else {
url += ACCOUNTS + getId();
}
url += "/video-channels?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
return getPage(new Page(url));
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final Response response = getDownloader().get(page.getUrl());
JsonObject pageJson = null;
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
pageJson = JsonParser.object().from(response.responseBody());
} catch (final Exception e) {
throw new ParsingException("Could not parse json data for account info", e);
}
}
if (pageJson == null) {
throw new ExtractionException("Unable to get account channel list");
}
PeertubeParsingHelper.validate(pageJson);
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
final JsonArray contents = pageJson.getArray("data");
if (contents == null) {
throw new ParsingException("Unable to extract account channel list");
}
contents.stream()
.filter(c -> c instanceof JsonObject)
.forEach(c -> collector.commit(new PeertubeChannelInfoItemExtractor(
(JsonObject) c, baseUrl))
);
return new InfoItemsPage<>(
collector, PeertubeParsingHelper.getNextPage(page.getUrl(),
pageJson.getLong("total")));
}
}

57
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java
View File

@ -3,7 +3,7 @@ package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -12,24 +12,16 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import javax.annotation.Nonnull;
public class PeertubeChannelExtractor extends ChannelExtractor {
private JsonObject json;
@ -105,51 +97,14 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
@Nonnull
@Override
public List<ListLinkHandler> getTabs() throws ParsingException {
return Collections.singletonList(
return Arrays.asList(
PeertubeChannelTabLinkHandlerFactory.getInstance().fromQuery(getId(),
Collections.singletonList(ChannelTabs.VIDEOS), "", getBaseUrl()),
PeertubeChannelTabLinkHandlerFactory.getInstance().fromQuery(getId(),
Collections.singletonList(ChannelTabs.PLAYLISTS), "", getBaseUrl())
);
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
return getPage(new Page(baseUrl + "/api/v1/" + getId() + "/videos?" + START_KEY + "=0&"
+ COUNT_KEY + "=" + ITEMS_PER_PAGE));
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final Response response = getDownloader().get(page.getUrl());
JsonObject pageJson = null;
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
pageJson = JsonParser.object().from(response.responseBody());
} catch (final Exception e) {
throw new ParsingException("Could not parse json data for channel info", e);
}
}
if (pageJson != null) {
PeertubeParsingHelper.validate(pageJson);
final long total = pageJson.getLong("total");
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, pageJson, getBaseUrl());
return new InfoItemsPage<>(collector,
PeertubeParsingHelper.getNextPage(page.getUrl(), total));
} else {
throw new ExtractionException("Unable to get PeerTube channel info");
}
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelInfoItemExtractor.java
View File

@ -1,6 +1,8 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -52,7 +54,7 @@ public class PeertubeChannelInfoItemExtractor implements ChannelInfoItemExtracto
@Override
public long getStreamCount() throws ParsingException {
return ChannelExtractor.ITEM_COUNT_UNKNOWN;
return ListExtractor.ITEM_COUNT_UNKNOWN;
}
@Override

36
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelTabExtractor.java
View File

@ -1,6 +1,5 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.schabi.newpipe.extractor.InfoItem;
@ -16,6 +15,7 @@ import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
@ -24,6 +24,8 @@ import java.io.IOException;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectItemsFrom;
import static org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelTabLinkHandlerFactory.getUrlSuffix;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubeChannelTabExtractor extends ChannelTabExtractor {
@ -37,18 +39,15 @@ public class PeertubeChannelTabExtractor extends ChannelTabExtractor {
}
@Override
public void onFetchPage(final @Nonnull Downloader downloader) throws ParsingException {
if (!getTab().equals(ChannelTabs.PLAYLISTS)) {
throw new ParsingException("tab " + getTab() + " not supported");
}
public void onFetchPage(final @Nonnull Downloader downloader) {
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage()
throws IOException, ExtractionException {
return getPage(new Page(baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT + getId()
+ "/video-playlists?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE));
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
return getPage(new Page(baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT
+ getUrlSuffix(getTab()) + "?" + START_KEY + "=0&" + COUNT_KEY + "="
+ ITEMS_PER_PAGE));
}
@Override
@ -70,25 +69,14 @@ public class PeertubeChannelTabExtractor extends ChannelTabExtractor {
}
if (pageJson == null) {
throw new ExtractionException("Unable to get channel playlist list");
throw new ExtractionException("Unable to get account channel list");
}
PeertubeParsingHelper.validate(pageJson);
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
final JsonArray contents = pageJson.getArray("data");
if (contents == null) {
throw new ParsingException("Unable to extract channel playlist list");
}
collectItemsFrom(collector, pageJson, getBaseUrl());
for (final Object c : contents) {
if (c instanceof JsonObject) {
collector.commit(new PeertubePlaylistInfoItemExtractor((JsonObject) c, baseUrl));
}
}
return new InfoItemsPage<>(
collector, PeertubeParsingHelper.getNextPage(page.getUrl(),
pageJson.getLong("total")));
return new InfoItemsPage<>(collector,
PeertubeParsingHelper.getNextPage(page.getUrl(), pageJson.getLong("total")));
}
}

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java
View File

@ -22,7 +22,7 @@ import java.io.IOException;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectItemsFrom;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubePlaylistExtractor extends PlaylistExtractor {
@ -114,7 +114,7 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor {
final long total = json.getLong("total");
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, json, getBaseUrl());
PeertubeParsingHelper.collectItemsFrom(collector, json, getBaseUrl());
return new InfoItemsPage<>(collector,
PeertubeParsingHelper.getNextPage(page.getUrl(), total));

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java
View File

@ -26,7 +26,7 @@ import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectItemsFrom;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubeSearchExtractor extends SearchExtractor {
@ -93,7 +93,7 @@ public class PeertubeSearchExtractor extends SearchExtractor {
final long total = json.getLong("total");
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, json, getBaseUrl(), sepia);
collectItemsFrom(collector, json, getBaseUrl(), sepia);
return new InfoItemsPage<>(collector,
PeertubeParsingHelper.getNextPage(page.getUrl(), total));

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java
View File

@ -23,7 +23,7 @@ import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectItemsFrom;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
@ -69,7 +69,7 @@ public class PeertubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
final long total = json.getLong("total");
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, json, getBaseUrl());
PeertubeParsingHelper.collectItemsFrom(collector, json, getBaseUrl());
return new InfoItemsPage<>(collector,
PeertubeParsingHelper.getNextPage(page.getUrl(), total));

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelTabLinkHandlerFactory.java
View File

@ -17,12 +17,14 @@ public final class PeertubeChannelTabLinkHandlerFactory extends ListLinkHandlerF
return INSTANCE;
}
private static String getUrlSuffix(final String tab) throws ParsingException {
public static String getUrlSuffix(final String tab) throws ParsingException {
switch (tab) {
case ChannelTabs.PLAYLISTS:
return "/video-playlists";
case ChannelTabs.CHANNELS:
case ChannelTabs.VIDEOS:
return "/videos";
case ChannelTabs.CHANNELS: // only available on accounts
return "/video-channels";
case ChannelTabs.PLAYLISTS: // only available on channels
return "/video-playlists";
}
throw new ParsingException("tab " + tab + " not supported");
}

47
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java
View File

@ -1,9 +1,11 @@
package org.schabi.newpipe.extractor.services.soundcloud.extractors;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -13,17 +15,13 @@ import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import javax.annotation.Nonnull;
public class SoundcloudChannelExtractor extends ChannelExtractor {
private String userId;
@ -112,49 +110,20 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Nonnull
@Override
public List<ListLinkHandler> getTabs() throws ParsingException {
final String urlTracks = getUrl()
+ SoundcloudChannelTabLinkHandlerFactory.getUrlSuffix(ChannelTabs.TRACKS);
final String urlPlaylists = getUrl()
+ SoundcloudChannelTabLinkHandlerFactory.getUrlSuffix(ChannelTabs.PLAYLISTS);
final String urlAlbums = getUrl()
+ SoundcloudChannelTabLinkHandlerFactory.getUrlSuffix(ChannelTabs.ALBUMS);
return Arrays.asList(
new ListLinkHandler(urlTracks, urlTracks, getId(),
Collections.singletonList(ChannelTabs.TRACKS), ""),
new ListLinkHandler(urlPlaylists, urlPlaylists, getId(),
Collections.singletonList(ChannelTabs.PLAYLISTS), ""),
new ListLinkHandler(urlAlbums, urlAlbums, getId(),
Collections.singletonList(ChannelTabs.ALBUMS), "")
);
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
try {
final StreamInfoItemsCollector streamInfoItemsCollector =
new StreamInfoItemsCollector(getServiceId());
final String apiUrl = USERS_ENDPOINT + getId() + "/tracks" + "?client_id="
+ SoundcloudParsingHelper.clientId() + "&limit=20" + "&linked_partitioning=1";
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15,
streamInfoItemsCollector, apiUrl);
return new InfoItemsPage<>(streamInfoItemsCollector, new Page(nextPageUrl));
} catch (final Exception e) {
throw new ExtractionException("Could not get next page", e);
}
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector,
page.getUrl());
return new InfoItemsPage<>(collector, new Page(nextPageUrl));
}
}

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelTabExtractor.java
View File

@ -29,6 +29,8 @@ public class SoundcloudChannelTabExtractor extends ChannelTabExtractor {
private String getEndpoint() {
switch (getTab()) {
case ChannelTabs.TRACKS:
return "/tracks";
case ChannelTabs.PLAYLISTS:
return "/playlists_without_albums";
case ChannelTabs.ALBUMS:
@ -38,8 +40,7 @@ public class SoundcloudChannelTabExtractor extends ChannelTabExtractor {
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader) {
}
@Nonnull

3
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelTabLinkHandlerFactory.java
View File

@ -19,6 +19,8 @@ public final class SoundcloudChannelTabLinkHandlerFactory extends ListLinkHandle
public static String getUrlSuffix(final String tab) throws ParsingException {
switch (tab) {
case ChannelTabs.TRACKS:
return "/tracks";
case ChannelTabs.PLAYLISTS:
return "/sets";
case ChannelTabs.ALBUMS:
@ -47,6 +49,7 @@ public final class SoundcloudChannelTabLinkHandlerFactory extends ListLinkHandle
@Override
public String[] getAvailableContentFilter() {
return new String[] {
ChannelTabs.TRACKS,
ChannelTabs.PLAYLISTS,
ChannelTabs.ALBUMS,
};

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View File

@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.localization.ContentCountry;
@ -117,7 +118,11 @@ public class YoutubeService extends StreamingService {
@Override
public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler) {
return new YoutubeChannelTabExtractor(this, linkHandler);
if (linkHandler instanceof ReadyChannelTabListLinkHandler) {
return ((ReadyChannelTabListLinkHandler) linkHandler).getChannelTabExtractor(this);
} else {
return new YoutubeChannelTabExtractor(this, linkHandler);
}
}
@Override

217
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -1,39 +1,29 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YouTubeChannelHelper.ChannelResponseData;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YouTubeChannelHelper.getChannelResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.YouTubeChannelHelper.resolveChannelId;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@ -41,7 +31,6 @@ import java.util.function.Consumer;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
/*
* Created by Christian Schabesberger on 25.07.16.
@ -65,8 +54,6 @@ import javax.annotation.Nullable;
public class YoutubeChannelExtractor extends ChannelExtractor {
private JsonObject initialData;
private JsonObject videoTab;
private List<ListLinkHandler> tabs;
/**
* Some channels have response redirects and the only way to reliably get the id is by saving it
@ -226,162 +213,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Nonnull
@Override
public List<ListLinkHandler> getTabs() throws ParsingException {
return tabs;
}
@Nonnull
@Override
public List<String> getTags() throws ParsingException {
final JsonArray tags = initialData.getObject("microformat")
.getObject("microformatDataRenderer").getArray("tags");
return tags.stream().map(Object::toString).collect(Collectors.toList());
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Page nextPage = null;
extractTabs();
if (videoTab != null) {
final JsonObject tabContent = videoTab.getObject("content");
JsonArray items = tabContent
.getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("gridRenderer").getArray("items");
if (items.isEmpty()) {
items = tabContent.getObject("richGridRenderer").getArray("contents");
}
final List<String> channelIds = new ArrayList<>();
channelIds.add(getName());
channelIds.add(getUrl());
final JsonObject continuation = collectStreamsFrom(collector, items, channelIds);
nextPage = getNextPageFrom(continuation, channelIds);
}
return new InfoItemsPage<>(collector, nextPage);
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final List<String> channelIds = page.getIds();
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(),
getExtractorLocalization());
final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("appendContinuationItemsAction");
final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation
.getArray("continuationItems"), channelIds);
return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelIds));
}
@Nullable
private Page getNextPageFrom(final JsonObject continuations,
final List<String> channelIds)
throws IOException, ExtractionException {
if (isNullOrEmpty(continuations)) {
return null;
}
final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
final String continuation = continuationEndpoint.getObject("continuationCommand")
.getString("token");
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry())
.value("continuation", continuation)
.done())
.getBytes(StandardCharsets.UTF_8);
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+ DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
}
/**
* Collect streams from an array of items
*
* @param collector the collector where videos will be committed
* @param videos the array to get videos from
* @param channelIds the ids of the channel, which are its name and its URL
* @return the continuation object
*/
private JsonObject collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
@Nonnull final JsonArray videos,
@Nonnull final List<String> channelIds) {
collector.reset();
final String uploaderName = channelIds.get(0);
final String uploaderUrl = channelIds.get(1);
final TimeAgoParser timeAgoParser = getTimeAgoParser();
JsonObject continuation = null;
for (final Object object : videos) {
final JsonObject video = (JsonObject) object;
if (video.has("gridVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(
video.getObject("gridVideoRenderer"), timeAgoParser) {
@Override
public String getUploaderName() {
return uploaderName;
}
@Override
public String getUploaderUrl() {
return uploaderUrl;
}
});
} else if (video.has("richItemRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(
video.getObject("richItemRenderer")
.getObject("content").getObject("videoRenderer"), timeAgoParser) {
@Override
public String getUploaderName() {
return uploaderName;
}
@Override
public String getUploaderUrl() {
return uploaderUrl;
}
});
} else if (video.has("continuationItemRenderer")) {
continuation = video.getObject("continuationItemRenderer");
}
}
return continuation;
}
/**
* Collect a list of available tabs and get the video tab data.
*/
private void extractTabs() throws ParsingException {
final JsonArray responseTabs = initialData.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs");
JsonObject foundVideoTab = null;
tabs = new ArrayList<>();
final List<ListLinkHandler> tabs = new ArrayList<>();
final Consumer<String> addTab = tab -> {
try {
tabs.add(YoutubeChannelTabLinkHandlerFactory.getInstance().fromQuery(
@ -402,7 +238,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
switch (urlSuffix) {
case "videos":
foundVideoTab = tabRenderer;
// since the videos tab already has its contents fetched, make sure
// it is in the first position
String mutName = "";
try {
mutName = getName();
} catch (final ParsingException ignored) {
}
final String name = mutName;
final String url = getUrl();
tabs.add(0, new ReadyChannelTabListLinkHandler(tabUrl,
redirectedChannelId, ChannelTabs.VIDEOS,
(service, linkHandler) -> new YoutubeChannelVideosTabExtractor(
service, linkHandler, tabRenderer, name, url)));
break;
case "playlists":
addTab.accept(ChannelTabs.PLAYLISTS);
@ -421,28 +269,15 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
}
if (foundVideoTab == null) {
if (tabs.isEmpty()) {
throw new ContentNotSupportedException("This channel has no supported tabs");
}
return;
}
return tabs;
}
final String messageRendererText = getTextFromObject(
foundVideoTab.getObject("content")
.getObject("sectionListRenderer")
.getArray("contents")
.getObject(0)
.getObject("itemSectionRenderer")
.getArray("contents")
.getObject(0)
.getObject("messageRenderer")
.getObject("text"));
if (messageRendererText != null
&& messageRendererText.equals("This channel has no videos.")) {
return;
}
@Nonnull
@Override
public List<String> getTags() throws ParsingException {
final JsonArray tags = initialData.getObject("microformat")
.getObject("microformatDataRenderer").getArray("tags");
videoTab = foundVideoTab;
return tags.stream().map(Object::toString).collect(Collectors.toList());
}
}

3
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java
View File

@ -71,8 +71,7 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
initialData = data.responseJson;
redirectedChannelId = data.channelId;
visitorData =
initialData.getObject("responseContext").getString("visitorData");
visitorData = initialData.getObject("responseContext").getString("visitorData");
}
@Nonnull

180
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelVideosTabExtractor.java Normal file
View File

@ -0,0 +1,180 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelTabExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
public class YoutubeChannelVideosTabExtractor extends ChannelTabExtractor {
private final JsonObject tabRenderer;
private final String channelName;
private final String channelUrl;
public YoutubeChannelVideosTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler,
final JsonObject tabRenderer,
final String channelName,
final String channelUrl) {
super(service, linkHandler);
this.tabRenderer = tabRenderer;
this.channelName = channelName;
this.channelUrl = channelUrl;
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
// nothing to do, all data was already fetched and is stored in the link handler
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
final JsonObject tabContent = tabRenderer.getObject("content");
JsonArray items = tabContent
.getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("gridRenderer").getArray("items");
if (items.isEmpty()) {
items = tabContent.getObject("richGridRenderer").getArray("contents");
}
final List<String> channelIds = new ArrayList<>();
channelIds.add(channelName);
channelIds.add(channelUrl);
final JsonObject continuation = collectStreamsFrom(collector, items, channelIds);
return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelIds));
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final List<String> channelIds = page.getIds();
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(),
getExtractorLocalization());
final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("appendContinuationItemsAction");
final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation
.getArray("continuationItems"), channelIds);
return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelIds));
}
@Nullable
private Page getNextPageFrom(final JsonObject continuations,
final List<String> channelIds)
throws IOException, ExtractionException {
if (isNullOrEmpty(continuations)) {
return null;
}
final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
final String continuation = continuationEndpoint.getObject("continuationCommand")
.getString("token");
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry())
.value("continuation", continuation)
.done())
.getBytes(StandardCharsets.UTF_8);
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+ DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
}
/**
* Collect streams from an array of items
*
* @param collector the collector where videos will be committed
* @param videos the array to get videos from
* @param channelIds the ids of the channel, which are its name and its URL
* @return the continuation object
*/
private JsonObject collectStreamsFrom(@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final JsonArray videos,
@Nonnull final List<String> channelIds) {
collector.reset();
final String uploaderName = channelIds.get(0);
final String uploaderUrl = channelIds.get(1);
final TimeAgoParser timeAgoParser = getTimeAgoParser();
JsonObject continuation = null;
for (final Object object : videos) {
final JsonObject video = (JsonObject) object;
if (video.has("gridVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(
video.getObject("gridVideoRenderer"), timeAgoParser) {
@Override
public String getUploaderName() {
return uploaderName;
}
@Override
public String getUploaderUrl() {
return uploaderUrl;
}
});
} else if (video.has("richItemRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(
video.getObject("richItemRenderer")
.getObject("content").getObject("videoRenderer"), timeAgoParser) {
@Override
public String getUploaderName() {
return uploaderName;
}
@Override
public String getUploaderUrl() {
return uploaderUrl;
}
});
} else if (video.has("continuationItemRenderer")) {
continuation = video.getObject("continuationItemRenderer");
}
}
return continuation;
}
}

3
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelTabLinkHandlerFactory.java
View File

@ -19,6 +19,8 @@ public final class YoutubeChannelTabLinkHandlerFactory extends ListLinkHandlerFa
public static String getUrlSuffix(final String tab) throws ParsingException {
switch (tab) {
case ChannelTabs.VIDEOS:
return "/videos";
case ChannelTabs.PLAYLISTS:
return "/playlists";
case ChannelTabs.LIVESTREAMS:
@ -55,6 +57,7 @@ public final class YoutubeChannelTabLinkHandlerFactory extends ListLinkHandlerFa
@Override
public String[] getAvailableContentFilter() {
return new String[] {
ChannelTabs.VIDEOS,
ChannelTabs.SHORTS,
ChannelTabs.LIVESTREAMS,
ChannelTabs.CHANNELS,

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelTabExtractorTest.java
View File

@ -7,7 +7,6 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeAccountTabExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelTabExtractor;
import java.io.IOException;
@ -56,12 +55,12 @@ public class PeertubeChannelTabExtractorTest {
}
public static class Channels {
private static PeertubeAccountTabExtractor extractor;
private static PeertubeChannelTabExtractor extractor;
@BeforeAll
public static void setUp() throws IOException, ExtractionException {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (PeertubeAccountTabExtractor) PeerTube
extractor = (PeertubeChannelTabExtractor) PeerTube
.getChannelTabExtractorFromId("accounts/framasoft",
ChannelTabs.CHANNELS, "https://framatube.org");
extractor.fetchPage();