lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat: add tab support to channel extractor 

- extract YouTube channel tabs: playlists, channels, shorts, live
This commit is contained in:
ThetaDev 2022-10-22 15:28:48 +02:00
parent ed4559d4de
commit 8b4b4310ea
50 changed files with 6549 additions and 262 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java
View File

@ -1,11 +1,13 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.ChannelTabExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.feed.FeedExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@ -57,7 +59,8 @@ public abstract class StreamingService {
/**
* Creates a new instance of a ServiceInfo
* @param name the name of the service
*
* @param name the name of the service
* @param mediaCapabilities the type of media this service can handle
*/
public ServiceInfo(final String name, final List<MediaCapability> mediaCapabilities) {
@ -97,8 +100,9 @@ public abstract class StreamingService {
* If you Implement one do not set id within your implementation of this extractor, instead
* set the id when you put the extractor into {@link ServiceList}
* All other parameters can be set directly from the overriding constructor.
* @param id the number of the service to identify him within the NewPipe frontend
* @param name the name of the service
*
* @param id the number of the service to identify him within the NewPipe frontend
* @param name the name of the service
* @param capabilities the type of media this service can handle
*/
public StreamingService(final int id,
@ -129,6 +133,7 @@ public abstract class StreamingService {
/**
* Must return a new instance of an implementation of LinkHandlerFactory for streams.
*
* @return an instance of a LinkHandlerFactory for streams
*/
public abstract LinkHandlerFactory getStreamLHFactory();
@ -136,6 +141,7 @@ public abstract class StreamingService {
/**
* Must return a new instance of an implementation of ListLinkHandlerFactory for channels.
* If support for channels is not given null must be returned.
*
* @return an instance of a ListLinkHandlerFactory for channels or null
*/
public abstract ListLinkHandlerFactory getChannelLHFactory();
@ -143,15 +149,18 @@ public abstract class StreamingService {
/**
* Must return a new instance of an implementation of ListLinkHandlerFactory for playlists.
* If support for playlists is not given null must be returned.
*
* @return an instance of a ListLinkHandlerFactory for playlists or null
*/
public abstract ListLinkHandlerFactory getPlaylistLHFactory();
/**
* Must return an instance of an implementation of SearchQueryHandlerFactory.
*
* @return an instance of a SearchQueryHandlerFactory
*/
public abstract SearchQueryHandlerFactory getSearchQHFactory();
public abstract ListLinkHandlerFactory getCommentsLHFactory();
/*//////////////////////////////////////////////////////////////////////////
@ -160,6 +169,7 @@ public abstract class StreamingService {
/**
* Must create a new instance of a SearchExtractor implementation.
*
* @param queryHandler specifies the keyword lock for, and the filters which should be applied.
* @return a new SearchExtractor instance
*/
@ -167,12 +177,14 @@ public abstract class StreamingService {
/**
* Must create a new instance of a SuggestionExtractor implementation.
*
* @return a new SuggestionExtractor instance
*/
public abstract SuggestionExtractor getSuggestionExtractor();
/**
* Outdated or obsolete. null can be returned.
*
* @return just null
*/
public abstract SubscriptionExtractor getSubscriptionExtractor();
@ -192,20 +204,26 @@ public abstract class StreamingService {
/**
* Must create a new instance of a KioskList implementation.
*
* @return a new KioskList instance
*/
public abstract KioskList getKioskList() throws ExtractionException;
/**
* Must create a new instance of a ChannelExtractor implementation.
*
* @param linkHandler is pointing to the channel which should be handled by this new instance.
* @return a new ChannelExtractor
*/
public abstract ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler)
throws ExtractionException;
public abstract ChannelTabExtractor getChannelTabExtractor(ChannelTabHandler linkHandler)
throws ExtractionException;
/**
* Must crete a new instance of a PlaylistExtractor implementation.
*
* @param linkHandler is pointing to the playlist which should be handled by this new instance.
* @return a new PlaylistExtractor
*/
@ -214,6 +232,7 @@ public abstract class StreamingService {
/**
* Must create a new instance of a StreamExtractor implementation.
*
* @param linkHandler is pointing to the stream which should be handled by this new instance.
* @return a new StreamExtractor
*/
@ -242,6 +261,19 @@ public abstract class StreamingService {
.fromQuery(id, contentFilter, sortFilter));
}
public ChannelTabExtractor getChannelTabExtractorFromUrl(final String url,
final ChannelTabHandler.Tab tab)
throws ExtractionException {
return getChannelTabExtractor(
new ChannelTabHandler(getChannelLHFactory().fromUrl(url), tab));
}
public ChannelTabExtractor getChannelTabExtractorFromId(final String id,
final ChannelTabHandler.Tab tab)
throws ExtractionException {
return getChannelTabExtractor(new ChannelTabHandler(getChannelLHFactory().fromId(id), tab));
}
public PlaylistExtractor getPlaylistExtractor(final String id,
final List<String> contentFilter,
final String sortFilter)
@ -284,6 +316,7 @@ public abstract class StreamingService {
/**
* Figures out where the link is pointing to (a channel, a video, a playlist, etc.)
*
* @param url the url on which it should be decided of which link type it is
* @return the link type of url
*/

6
extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java
View File

@ -3,9 +3,13 @@ package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import javax.annotation.Nonnull;
import java.util.List;
/*
* Created by Christian Schabesberger on 25.07.16.
*
@ -43,5 +47,7 @@ public abstract class ChannelExtractor extends ListExtractor<StreamInfoItem> {
public abstract String getParentChannelUrl() throws ParsingException;
public abstract String getParentChannelAvatarUrl() throws ParsingException;
public abstract boolean isVerified() throws ParsingException;
@Nonnull
public abstract List<ChannelTabHandler> getTabs() throws ParsingException;
}

33
extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelTabExtractor.java Normal file
View File

@ -0,0 +1,33 @@
package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabHandler;
import javax.annotation.Nonnull;
public abstract class ChannelTabExtractor extends ListExtractor<InfoItem> {
public ChannelTabExtractor(final StreamingService service,
final ChannelTabHandler linkHandler) {
super(service, linkHandler);
}
@Nonnull
@Override
public ChannelTabHandler getLinkHandler() {
return (ChannelTabHandler) super.getLinkHandler();
}
@Nonnull
public ChannelTabHandler.Tab getTab() {
return getLinkHandler().getTab();
}
@Nonnull
@Override
public String getName() {
return getTab().name();
}
}

50
extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelTabInfo.java Normal file
View File

@ -0,0 +1,50 @@
package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabHandler;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import java.io.IOException;
public class ChannelTabInfo extends ListInfo<InfoItem> {
public ChannelTabInfo(final int serviceId, final ChannelTabHandler linkHandler) {
super(serviceId, linkHandler, linkHandler.getTab().name());
}
public static ChannelTabInfo getInfo(final StreamingService service,
final ChannelTabHandler linkHandler)
throws ExtractionException, IOException {
final ChannelTabExtractor extractor = service.getChannelTabExtractor(linkHandler);
extractor.fetchPage();
return getInfo(extractor);
}
public static ChannelTabInfo getInfo(final ChannelTabExtractor extractor) {
final ChannelTabInfo info =
new ChannelTabInfo(extractor.getServiceId(), extractor.getLinkHandler());
try {
info.setOriginalUrl(extractor.getOriginalUrl());
} catch (final Exception e) {
info.addError(e);
}
final ListExtractor.InfoItemsPage<InfoItem> page
= ExtractorHelper.getItemsPageOrLogError(info, extractor);
info.setRelatedItems(page.getItems());
info.setNextPage(page.getNextPage());
return info;
}
public static ListExtractor.InfoItemsPage<InfoItem> getMoreItems(
final StreamingService service, final ChannelTabHandler linkHandler, final Page page)
throws ExtractionException, IOException {
return service.getChannelTabExtractor(linkHandler).getPage(page);
}
}

21
extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ChannelTabHandler.java Normal file
View File

@ -0,0 +1,21 @@
package org.schabi.newpipe.extractor.linkhandler;
public class ChannelTabHandler extends ListLinkHandler {
public enum Tab {
Playlists,
Livestreams,
Shorts,
Channels,
}
private final Tab tab;
public ChannelTabHandler(final ListLinkHandler linkHandler, final Tab tab) {
super(linkHandler);
this.tab = tab;
}
public Tab getTab() {
return tab;
}
}

23
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java
View File

@ -2,19 +2,13 @@
package org.schabi.newpipe.extractor.services.bandcamp;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor.FEATURED_API_URL;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor.KIOSK_FEATURED;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor.KIOSK_RADIO;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor.RADIO_API_URL;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.ChannelTabExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@ -45,6 +39,14 @@ import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.util.Arrays;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor.FEATURED_API_URL;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor.KIOSK_FEATURED;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor.KIOSK_RADIO;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor.RADIO_API_URL;
public class BandcampService extends StreamingService {
public BandcampService(final int id) {
@ -136,6 +138,11 @@ public class BandcampService extends StreamingService {
return new BandcampChannelExtractor(this, linkHandler);
}
@Override
public ChannelTabExtractor getChannelTabExtractor(final ChannelTabHandler linkHandler) {
return null;
}
@Override
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
return new BandcampPlaylistExtractor(this, linkHandler);

15
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java
View File

@ -2,11 +2,8 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.jsoup.Jsoup;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
@ -15,16 +12,20 @@ import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.BandcampDiscographStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class BandcampChannelExtractor extends ChannelExtractor {
@ -105,6 +106,12 @@ public class BandcampChannelExtractor extends ChannelExtractor {
return false;
}
@Nonnull
@Override
public List<ChannelTabHandler> getTabs() throws ParsingException {
return Collections.emptyList();
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {

16
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java
View File

@ -1,14 +1,12 @@
package org.schabi.newpipe.extractor.services.media_ccc;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
import static java.util.Arrays.asList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.ChannelTabExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@ -35,6 +33,10 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
public class MediaCCCService extends StreamingService {
public MediaCCCService(final int id) {
super(id, "media.ccc.de", asList(AUDIO, VIDEO));
@ -78,6 +80,12 @@ public class MediaCCCService extends StreamingService {
return new MediaCCCConferenceExtractor(this, linkHandler);
}
@Override
public ChannelTabExtractor getChannelTabExtractor(final ChannelTabHandler linkHandler)
throws ExtractionException {
return null;
}
@Override
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
return null;

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java
View File

@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
@ -18,6 +19,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
public class MediaCCCConferenceExtractor extends ChannelExtractor {
private JsonObject conferenceData;
@ -72,6 +75,12 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
return false;
}
@Nonnull
@Override
public List<ChannelTabHandler> getTabs() {
return Collections.emptyList();
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() {

16
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java
View File

@ -1,14 +1,12 @@
package org.schabi.newpipe.extractor.services.peertube;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
import static java.util.Arrays.asList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.ChannelTabExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@ -37,6 +35,10 @@ import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.util.List;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
public class PeertubeService extends StreamingService {
private PeertubeInstance instance;
@ -103,6 +105,12 @@ public class PeertubeService extends StreamingService {
}
}
@Override
public ChannelTabExtractor getChannelTabExtractor(final ChannelTabHandler linkHandler)
throws ExtractionException {
return null;
}
@Override
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler)
throws ExtractionException {

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java
View File

@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
@ -22,6 +23,8 @@ import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
@ -119,6 +122,13 @@ public class PeertubeAccountExtractor extends ChannelExtractor {
return false;
}
@Nonnull
@Override
public List<ChannelTabHandler> getTabs() throws ParsingException {
// TODO: implement Peertube account channels tab
return Collections.emptyList();
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java
View File

@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
@ -20,6 +21,8 @@ import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
@ -98,6 +101,13 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
return false;
}
@Nonnull
@Override
public List<ChannelTabHandler> getTabs() throws ParsingException {
// TODO: implement Peertube channel playlists tab
return Collections.emptyList();
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {

16
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java
View File

@ -1,14 +1,12 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
import static java.util.Arrays.asList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.ChannelTabExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@ -37,6 +35,10 @@ import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import java.util.List;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
public class SoundcloudService extends StreamingService {
public SoundcloudService(final int id) {
@ -86,6 +88,12 @@ public class SoundcloudService extends StreamingService {
return new SoundcloudChannelExtractor(this, linkHandler);
}
@Override
public ChannelTabExtractor getChannelTabExtractor(final ChannelTabHandler linkHandler)
throws ExtractionException {
return null;
}
@Override
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
return new SoundcloudPlaylistExtractor(this, linkHandler);

20
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java
View File

@ -1,26 +1,27 @@
package org.schabi.newpipe.extractor.services.soundcloud.extractors;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import java.io.IOException;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudChannelExtractor extends ChannelExtractor {
private String userId;
@ -106,6 +107,13 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
return user.getBoolean("verified");
}
@Nonnull
@Override
public List<ChannelTabHandler> getTabs() throws ParsingException {
// TODO: implement soundcloud playlist tab
return Collections.emptyList();
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {

182
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -20,21 +20,12 @@
package org.schabi.newpipe.extractor.services.youtube;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static java.util.Collections.singletonList;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonBuilder;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.AccountTerminatedException;
@ -51,6 +42,8 @@ import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.RandomStringFromAlphabetGenerator;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
@ -73,8 +66,13 @@ import java.util.Random;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public final class YoutubeParsingHelper {
@ -258,6 +256,12 @@ public final class YoutubeParsingHelper {
*/
private static boolean consentAccepted = false;
/**
* Attach YouTube visitor data (session token) to all requests made with the desktop client.
* Used for testing to reproduce A/B tests.
*/
private static String visitorData = null;
private static boolean isGoogleURL(final String url) {
final String cachedUrl = extractCachedUrlIfNeeded(url);
try {
@ -1141,7 +1145,7 @@ public final class YoutubeParsingHelper {
@Nonnull final ContentCountry contentCountry)
throws IOException, ExtractionException {
// @formatter:off
return JsonObject.builder()
final JsonBuilder<JsonObject> builder = JsonObject.builder()
.object("context")
.object("client")
.value("hl", localization.getLocalizationCode())
@ -1149,8 +1153,13 @@ public final class YoutubeParsingHelper {
.value("clientName", "WEB")
.value("clientVersion", getClientVersion())
.value("originalUrl", "https://www.youtube.com")
.value("platform", "DESKTOP")
.end()
.value("platform", "DESKTOP");
if (visitorData != null) {
builder.value("visitorData", visitorData);
}
builder.end()
.object("request")
.array("internalExperimentFlags")
.end()
@ -1163,6 +1172,8 @@ public final class YoutubeParsingHelper {
.end()
.end();
// @formatter:on
return builder;
}
@Nonnull
@ -1337,6 +1348,7 @@ public final class YoutubeParsingHelper {
/**
* Add required headers and cookies to an existing headers Map.
*
* @see #addClientInfoHeaders(Map)
* @see #addCookieHeader(Map)
*/
@ -1349,6 +1361,7 @@ public final class YoutubeParsingHelper {
/**
* Add the <code>X-YouTube-Client-Name</code>, <code>X-YouTube-Client-Version</code>,
* <code>Origin</code>, and <code>Referer</code> headers.
*
* @param headers The headers which should be completed
*/
public static void addClientInfoHeaders(@Nonnull final Map<String, List<String>> headers)
@ -1363,6 +1376,7 @@ public final class YoutubeParsingHelper {
/**
* Create a map with the required cookie header.
*
* @return A singleton map containing the header.
*/
public static Map<String, List<String>> getCookieHeader() {
@ -1371,6 +1385,7 @@ public final class YoutubeParsingHelper {
/**
* Add the <code>CONSENT</code> cookie to prevent redirect to <code>consent.youtube.com</code>
*
* @param headers the headers which should be completed
*/
public static void addCookieHeader(@Nonnull final Map<String, List<String>> headers) {
@ -1609,6 +1624,138 @@ public final class YoutubeParsingHelper {
return false;
}
public static String resolveChannelId(final String idOrPath)
throws ExtractionException, IOException {
final String[] channelId = idOrPath.split("/");
if (channelId[0].startsWith("UC")) {
return channelId[0];
}
// If the url is an URL which is not a /channel URL, we need to use the
// navigation/resolve_url endpoint of the InnerTube API to get the channel id. Otherwise,
// we couldn't get information about the channel associated with this URL, if there is one.
if (!channelId[0].equals("channel")) {
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
Localization.DEFAULT, ContentCountry.DEFAULT)
.value("url", "https://www.youtube.com/" + idOrPath)
.done())
.getBytes(UTF_8);
final JsonObject jsonResponse = getJsonPostResponse("navigation/resolve_url",
body, Localization.DEFAULT);
if (!isNullOrEmpty(jsonResponse.getObject("error"))) {
final JsonObject errorJsonObject = jsonResponse.getObject("error");
final int errorCode = errorJsonObject.getInt("code");
if (errorCode == 404) {
throw new ContentNotAvailableException("This channel doesn't exist.");
} else {
throw new ContentNotAvailableException("Got error:\""
+ errorJsonObject.getString("status") + "\": "
+ errorJsonObject.getString("message"));
}
}
final JsonObject endpoint = jsonResponse.getObject("endpoint");
final String webPageType = endpoint.getObject("commandMetadata")
.getObject("webCommandMetadata")
.getString("webPageType", "");
final JsonObject browseEndpoint = endpoint.getObject("browseEndpoint");
final String browseId = browseEndpoint.getString("browseId", "");
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
&& !browseId.isEmpty()) {
if (!browseId.startsWith("UC")) {
throw new ExtractionException("Redirected id is not pointing to a channel");
}
return browseId;
}
}
return channelId[1];
}
public static final class ChannelResponseData {
public final JsonObject responseJson;
public final String channelId;
private ChannelResponseData(final JsonObject responseJson, final String channelId) {
this.responseJson = responseJson;
this.channelId = channelId;
}
}
public static ChannelResponseData getChannelResponse(final String channelId,
final String params,
final Localization loc,
final ContentCountry country)
throws ExtractionException, IOException {
String id = channelId;
JsonObject ajaxJson = null;
int level = 0;
while (level < 3) {
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
loc, country)
.value("browseId", id)
.value("params", params) // Equal to videos
.done())
.getBytes(UTF_8);
final JsonObject jsonResponse = getJsonPostResponse("browse", body, loc);
if (!isNullOrEmpty(jsonResponse.getObject("error"))) {
final JsonObject errorJsonObject = jsonResponse.getObject("error");
final int errorCode = errorJsonObject.getInt("code");
if (errorCode == 404) {
throw new ContentNotAvailableException("This channel doesn't exist.");
} else {
throw new ContentNotAvailableException("Got error:\""
+ errorJsonObject.getString("status") + "\": "
+ errorJsonObject.getString("message"));
}
}
final JsonObject endpoint = jsonResponse.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("navigateAction")
.getObject("endpoint");
final String webPageType = endpoint.getObject("commandMetadata")
.getObject("webCommandMetadata")
.getString("webPageType", "");
final String browseId = endpoint.getObject("browseEndpoint").getString("browseId",
"");
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
&& !browseId.isEmpty()) {
if (!browseId.startsWith("UC")) {
throw new ExtractionException("Redirected id is not pointing to a channel");
}
id = browseId;
level++;
} else {
ajaxJson = jsonResponse;
break;
}
}
if (ajaxJson == null) {
throw new ExtractionException("Got no channel response");
}
defaultAlertsCheck(ajaxJson);
return new ChannelResponseData(ajaxJson, id);
}
/**
* Generate a content playback nonce (also called {@code cpn}), sent by YouTube clients in
* playback requests (and also for some clients, in the player request body).
@ -1693,4 +1840,11 @@ public final class YoutubeParsingHelper {
public static boolean isConsentAccepted() {
return consentAccepted;
}
/**
* @see #visitorData
*/
public static void setVisitorData(@Nullable final String visitorData) {
YoutubeParsingHelper.visitorData = visitorData;
}
}

21
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View File

@ -1,17 +1,13 @@
package org.schabi.newpipe.extractor.services.youtube;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
import static java.util.Arrays.asList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.ChannelTabExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.feed.FeedExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@ -23,6 +19,7 @@ import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelTabExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
@ -43,9 +40,14 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import javax.annotation.Nonnull;
import java.util.List;
import javax.annotation.Nonnull;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
/*
* Created by Christian Schabesberger on 23.08.15.
@ -108,6 +110,11 @@ public class YoutubeService extends StreamingService {
return new YoutubeChannelExtractor(this, linkHandler);
}
@Override
public ChannelTabExtractor getChannelTabExtractor(final ChannelTabHandler linkHandler) {
return new YoutubeChannelTabExtractor(this, linkHandler);
}
@Override
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())

219
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -1,30 +1,17 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
@ -34,14 +21,28 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.ChannelResponseData;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getChannelResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.resolveChannelId;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*
* Created by Christian Schabesberger on 25.07.16.
@ -66,6 +67,7 @@ import javax.annotation.Nullable;
public class YoutubeChannelExtractor extends ChannelExtractor {
private JsonObject initialData;
private JsonObject videoTab;
private List<ChannelTabHandler> tabs;
/**
* Some channels have response redirects and the only way to reliably get the id is by saving it
@ -88,115 +90,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
final String channelPath = super.getId();
final String[] channelId = channelPath.split("/");
String id = "";
// If the url is an URL which is not a /channel URL, we need to use the
// navigation/resolve_url endpoint of the InnerTube API to get the channel id. Otherwise,
// we couldn't get information about the channel associated with this URL, if there is one.
if (!channelId[0].equals("channel")) {
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
getExtractorLocalization(), getExtractorContentCountry())
.value("url", "https://www.youtube.com/" + channelPath)
.done())
.getBytes(UTF_8);
final String id = resolveChannelId(channelPath);
final ChannelResponseData data = getChannelResponse(id, "EgZ2aWRlb3M%3D",
getExtractorLocalization(), getExtractorContentCountry());
final JsonObject jsonResponse = getJsonPostResponse("navigation/resolve_url",
body, getExtractorLocalization());
if (!isNullOrEmpty(jsonResponse.getObject("error"))) {
final JsonObject errorJsonObject = jsonResponse.getObject("error");
final int errorCode = errorJsonObject.getInt("code");
if (errorCode == 404) {
throw new ContentNotAvailableException("This channel doesn't exist.");
} else {
throw new ContentNotAvailableException("Got error:\""
+ errorJsonObject.getString("status") + "\": "
+ errorJsonObject.getString("message"));
}
}
final JsonObject endpoint = jsonResponse.getObject("endpoint");
final String webPageType = endpoint.getObject("commandMetadata")
.getObject("webCommandMetadata")
.getString("webPageType", "");
final JsonObject browseEndpoint = endpoint.getObject("browseEndpoint");
final String browseId = browseEndpoint.getString("browseId", "");
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
&& !browseId.isEmpty()) {
if (!browseId.startsWith("UC")) {
throw new ExtractionException("Redirected id is not pointing to a channel");
}
id = browseId;
redirectedChannelId = browseId;
}
} else {
id = channelId[1];
}
JsonObject ajaxJson = null;
int level = 0;
while (level < 3) {
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
getExtractorLocalization(), getExtractorContentCountry())
.value("browseId", id)
.value("params", "EgZ2aWRlb3M%3D") // Equal to videos
.done())
.getBytes(UTF_8);
final JsonObject jsonResponse = getJsonPostResponse("browse", body,
getExtractorLocalization());
if (!isNullOrEmpty(jsonResponse.getObject("error"))) {
final JsonObject errorJsonObject = jsonResponse.getObject("error");
final int errorCode = errorJsonObject.getInt("code");
if (errorCode == 404) {
throw new ContentNotAvailableException("This channel doesn't exist.");
} else {
throw new ContentNotAvailableException("Got error:\""
+ errorJsonObject.getString("status") + "\": "
+ errorJsonObject.getString("message"));
}
}
final JsonObject endpoint = jsonResponse.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("navigateAction")
.getObject("endpoint");
final String webPageType = endpoint.getObject("commandMetadata")
.getObject("webCommandMetadata")
.getString("webPageType", "");
final String browseId = endpoint.getObject("browseEndpoint").getString("browseId",
"");
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
&& !browseId.isEmpty()) {
if (!browseId.startsWith("UC")) {
throw new ExtractionException("Redirected id is not pointing to a channel");
}
id = browseId;
redirectedChannelId = browseId;
level++;
} else {
ajaxJson = jsonResponse;
break;
}
}
if (ajaxJson == null) {
throw new ExtractionException("Could not fetch initial JSON data");
}
initialData = ajaxJson;
YoutubeParsingHelper.defaultAlertsCheck(initialData);
initialData = data.responseJson;
redirectedChannelId = data.channelId;
}
@Nonnull
@ -324,6 +223,13 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return YoutubeParsingHelper.isVerified(badges);
}
@Nonnull
@Override
public List<ChannelTabHandler> getTabs() throws ParsingException {
getVideoTab();
return tabs;
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
@ -366,7 +272,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final Map<String, List<String>> headers = new HashMap<>();
addClientInfoHeaders(headers);
final Response response = getDownloader().post(page.getUrl(), null, page.getBody(),
final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(),
getExtractorLocalization());
final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
@ -397,7 +303,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
getExtractorContentCountry())
.value("continuation", continuation)
.done())
.getBytes(UTF_8);
.getBytes(StandardCharsets.UTF_8);
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+ DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
@ -466,32 +372,65 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return this.videoTab;
}
final JsonArray tabs = initialData.getObject("contents")
final JsonArray responseTabs = initialData.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs");
JsonObject foundVideoTab = null;
for (final Object tab : tabs) {
tabs = new ArrayList<>();
final Consumer<ChannelTabHandler.Tab> addTab = tab ->
tabs.add(new ChannelTabHandler(getLinkHandler(), tab));
for (final Object tab : responseTabs) {
if (((JsonObject) tab).has("tabRenderer")) {
if (((JsonObject) tab).getObject("tabRenderer").getString("title",
"").equals("Videos")) {
foundVideoTab = ((JsonObject) tab).getObject("tabRenderer");
break;
final JsonObject tabRenderer = ((JsonObject) tab).getObject("tabRenderer");
final String tabUrl = tabRenderer.getObject("endpoint")
.getObject("commandMetadata").getObject("webCommandMetadata")
.getString("url");
if (tabUrl != null) {
final String[] urlParts = tabUrl.split("/");
final String urlSuffix = urlParts[urlParts.length - 1];
switch (urlSuffix) {
case "videos":
foundVideoTab = tabRenderer;
break;
case "playlists":
addTab.accept(ChannelTabHandler.Tab.Playlists);
break;
case "streams":
addTab.accept(ChannelTabHandler.Tab.Livestreams);
break;
case "shorts":
addTab.accept(ChannelTabHandler.Tab.Shorts);
break;
case "channels":
addTab.accept(ChannelTabHandler.Tab.Channels);
break;
}
}
}
}
if (foundVideoTab == null) {
throw new ContentNotSupportedException("This channel has no Videos tab");
if (tabs.isEmpty()) {
throw new ContentNotSupportedException("This channel has no supported tabs");
}
return null;
}
final String messageRendererText = getTextFromObject(foundVideoTab.getObject("content")
.getObject("sectionListRenderer").getArray("contents").getObject(0)
.getObject("itemSectionRenderer").getArray("contents").getObject(0)
.getObject("messageRenderer").getObject("text"));
if (messageRendererText != null
&& messageRendererText.equals("This channel has no videos.")) {
return null;
try {
final String messageRendererText = getTextFromObject(foundVideoTab.getObject("content")
.getObject("sectionListRenderer").getArray("contents").getObject(0)
.getObject("itemSectionRenderer").getArray("contents").getObject(0)
.getObject("messageRenderer").getObject("text"));
if (messageRendererText != null
&& messageRendererText.equals("This channel has no videos.")) {
return null;
}
} catch (final ParsingException ignored) {
}
this.videoTab = foundVideoTab;

297
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java Normal file
View File

@ -0,0 +1,297 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelTabExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.ChannelResponseData;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getChannelResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.resolveChannelId;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
private JsonObject initialData;
private JsonObject tabData;
private String redirectedChannelId;
public YoutubeChannelTabExtractor(final StreamingService service,
final ChannelTabHandler linkHandler) {
super(service, linkHandler);
}
@Nullable
String getParams() {
switch (getTab()) {
case Playlists:
return "EglwbGF5bGlzdHMgAQ%3D%3D";
case Livestreams:
return "EgdzdHJlYW1z8gYECgJ6AA%3D%3D";
case Shorts:
return "EgZzaG9ydHPyBgUKA5oBAA%3D%3D";
case Channels:
return "EghjaGFubmVsc_IGBAoCUgA%3D";
}
return null;
}
String getUrlSuffix() {
switch (getTab()) {
case Playlists:
return "/playlists";
case Livestreams:
return "/streams";
case Shorts:
return "/shorts";
case Channels:
return "/channels";
}
return "";
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
final String params = getParams();
if (params == null) {
throw new ExtractionException("tab not supported");
}
final String id = resolveChannelId(super.getId());
final ChannelResponseData data = getChannelResponse(id, params,
getExtractorLocalization(), getExtractorContentCountry());
initialData = data.responseJson;
redirectedChannelId = data.channelId;
}
@Nonnull
@Override
public String getUrl() throws ParsingException {
try {
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(
"channel/" + getId() + getUrlSuffix());
} catch (final ParsingException e) {
return super.getUrl();
}
}
@Nonnull
@Override
public String getId() throws ParsingException {
final String channelId = initialData.getObject("header")
.getObject("c4TabbedHeaderRenderer")
.getString("channelId", "");
if (!channelId.isEmpty()) {
return channelId;
} else if (!isNullOrEmpty(redirectedChannelId)) {
return redirectedChannelId;
} else {
throw new ParsingException("Could not get channel id");
}
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
Page nextPage = null;
if (getTabData() != null) {
final JsonObject tabContent = tabData.getObject("content");
JsonArray items = tabContent
.getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("gridRenderer").getArray("items");
if (items.isEmpty()) {
items = tabContent.getObject("richGridRenderer").getArray("contents");
}
final List<String> channelIds = new ArrayList<>();
channelIds.add(getName());
channelIds.add(getUrl());
final JsonObject continuation = collectItemsFrom(collector, items, channelIds);
nextPage = getNextPageFrom(continuation, channelIds);
}
return new InfoItemsPage<>(collector, nextPage);
}
@Override
public InfoItemsPage<InfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final List<String> channelIds = page.getIds();
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
final Map<String, List<String>> headers = new HashMap<>();
addClientInfoHeaders(headers);
final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(),
getExtractorLocalization());
final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("appendContinuationItemsAction");
final JsonObject continuation = collectItemsFrom(collector, sectionListContinuation
.getArray("continuationItems"), channelIds);
return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelIds));
}
@Nullable
private JsonObject getTabData() {
if (this.tabData != null) {
return this.tabData;
}
final JsonArray tabs = initialData.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs");
JsonObject foundTab = null;
for (final Object tab : tabs) {
if (((JsonObject) tab).has("tabRenderer")) {
if (((JsonObject) tab).getObject("tabRenderer").getObject("endpoint")
.getObject("commandMetadata").getObject("webCommandMetadata")
.getString("url").endsWith(getUrlSuffix())) {
foundTab = ((JsonObject) tab).getObject("tabRenderer");
break;
}
}
}
// No tab
if (foundTab == null) {
return null;
}
// No content
final JsonArray tabContents = foundTab.getObject("content").getObject("sectionListRenderer")
.getArray("contents").getObject(0)
.getObject("itemSectionRenderer").getArray("contents");
if (tabContents.size() == 1 && tabContents.getObject(0).has("messageRenderer")) {
return null;
}
this.tabData = foundTab;
return foundTab;
}
private JsonObject collectItemsFrom(@Nonnull final MultiInfoItemsCollector collector,
@Nonnull final JsonArray items,
@Nonnull final List<String> channelIds) {
collector.reset();
final String uploaderName = channelIds.get(0);
final String uploaderUrl = channelIds.get(1);
final TimeAgoParser timeAgoParser = getTimeAgoParser();
JsonObject continuation = null;
final Consumer<JsonObject> commitVideo = videoRenderer -> collector.commit(
new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser) {
@Override
public String getUploaderName() {
return uploaderName;
}
@Override
public String getUploaderUrl() {
return uploaderUrl;
}
});
for (final Object object : items) {
final JsonObject item = (JsonObject) object;
if (item.has("gridVideoRenderer")) {
commitVideo.accept(item.getObject("gridVideoRenderer"));
} else if (item.has("richItemRenderer")) {
final JsonObject richItem = item.getObject("richItemRenderer").getObject("content");
if (richItem.has("videoRenderer")) {
commitVideo.accept(richItem.getObject("videoRenderer"));
} else if (richItem.has("reelItemRenderer")) {
commitVideo.accept(richItem.getObject("reelItemRenderer"));
}
} else if (item.has("gridPlaylistRenderer")) {
collector.commit(new YoutubePlaylistInfoItemExtractor(
item.getObject("gridPlaylistRenderer")) {
@Override
public String getUploaderName() {
return uploaderName;
}
});
} else if (item.has("gridChannelRenderer")) {
collector.commit(new YoutubeChannelInfoItemExtractor(
item.getObject("gridChannelRenderer")));
} else if (item.has("continuationItemRenderer")) {
continuation = item.getObject("continuationItemRenderer");
}
}
return continuation;
}
@Nullable
private Page getNextPageFrom(final JsonObject continuations,
final List<String> channelIds) throws IOException,
ExtractionException {
if (isNullOrEmpty(continuations)) {
return null;
}
final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
final String continuation = continuationEndpoint.getObject("continuationCommand")
.getString("token");
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry())
.value("continuation", continuation)
.done())
.getBytes(StandardCharsets.UTF_8);
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+ DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
}
}

18
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java
View File

@ -1,7 +1,7 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
@ -20,9 +20,13 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override
public String getThumbnailUrl() throws ParsingException {
try {
final String url = playlistInfoItem.getArray("thumbnails").getObject(0)
.getArray("thumbnails").getObject(0).getString("url");
JsonArray thumbnails = playlistInfoItem.getArray("thumbnails").getObject(0)
.getArray("thumbnails");
if (thumbnails.isEmpty()) {
thumbnails = playlistInfoItem.getObject("thumbnail").getArray("thumbnails");
}
final String url = thumbnails.getObject(0).getString("url");
return fixThumbnailUrl(url);
} catch (final Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
@ -59,9 +63,13 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override
public long getStreamCount() throws ParsingException {
String videoCountText = playlistInfoItem.getString("videoCount");
if (videoCountText == null) {
videoCountText = getTextFromObject(playlistInfoItem.getObject("videoCountShortText"));
}
try {
return Long.parseLong(Utils.removeNonDigitCharacters(
playlistInfoItem.getString("videoCount")));
return Long.parseLong(Utils.removeNonDigitCharacters(videoCountText));
} catch (final Exception e) {
throw new ParsingException("Could not get stream count", e);
}

32
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java
View File

@ -1,13 +1,7 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailUrlFromInfoItem;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
@ -18,12 +12,16 @@ import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailUrlFromInfoItem;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
@ -109,10 +107,16 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override
public String getName() throws ParsingException {
final String name = getTextFromObject(videoInfo.getObject("title"));
String name = getTextFromObject(videoInfo.getObject("title"));
if (!isNullOrEmpty(name)) {
return name;
}
name = getTextFromObject(videoInfo.getObject("headline"));
if (!isNullOrEmpty(name)) {
return name;
}
throw new ParsingException("Could not get name");
}
@ -133,7 +137,17 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
}
if (isNullOrEmpty(duration)) {
throw new ParsingException("Could not get duration");
// Duration of short videos in channel tab
// example: "simple is best - 49 seconds - play video"
final String accessibilityLabel = videoInfo.getObject("accessibility")
.getObject("accessibilityData").getString("label");
final String[] labelParts = accessibilityLabel.split(" \u2013 ");
if (labelParts.length > 2) {
duration = labelParts[labelParts.length - 2];
} else {
throw new ParsingException("Could not get duration");
}
}
}

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseChannelExtractorTest.java
View File

@ -8,4 +8,5 @@ public interface BaseChannelExtractorTest extends BaseListExtractorTest {
void testFeedUrl() throws Exception;
void testSubscriberCount() throws Exception;
void testVerified() throws Exception;
void testTabs() throws Exception;
}

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelExtractorTest.java
View File

@ -62,6 +62,11 @@ public class BandcampChannelExtractorTest implements BaseChannelExtractorTest {
assertFalse(extractor.isVerified());
}
@Override
public void testTabs() throws Exception {
assertTrue(extractor.getTabs().isEmpty());
}
@Override
public void testRelatedItems() throws Exception {
// not implemented

8
extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java
View File

@ -5,7 +5,11 @@ package org.schabi.newpipe.extractor.services.bandcamp;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
import org.schabi.newpipe.extractor.search.SearchExtractor;
@ -72,7 +76,7 @@ public class BandcampSearchExtractorTest {
*/
@Test
void testAlbumSearch() throws ExtractionException, IOException {
final SearchExtractor extractor = Bandcamp.getSearchExtractor("minecraft volume alpha");
final SearchExtractor extractor = Bandcamp.getSearchExtractor("minecraft volume alpha cover");
InfoItem minecraft = extractor.getInitialPage()
.getItems().get(0);

10
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java
View File

@ -109,6 +109,11 @@ public class PeertubeAccountExtractorTest {
public void testVerified() throws Exception {
assertFalse(extractor.isVerified());
}
@Override
public void testTabs() throws Exception {
}
}
public static class FreeSoftwareFoundation implements BaseChannelExtractorTest {
@ -210,5 +215,10 @@ public class PeertubeAccountExtractorTest {
public void testVerified() throws Exception {
assertFalse(extractor.isVerified());
}
@Override
public void testTabs() throws Exception {
// TODO: implement Peertube account channels tab
}
}
}

10
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java
View File

@ -124,6 +124,11 @@ public class PeertubeChannelExtractorTest {
public void testVerified() throws Exception {
assertFalse(extractor.isVerified());
}
@Override
public void testTabs() throws Exception {
}
}
public static class ChatSceptique implements BaseChannelExtractorTest {
@ -241,5 +246,10 @@ public class PeertubeChannelExtractorTest {
public void testVerified() throws Exception {
assertFalse(extractor.isVerified());
}
@Override
public void testTabs() throws Exception {
// TODO: implement Peertube channel playlists tab
}
}
}

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java
View File

@ -46,7 +46,7 @@ public class PeertubePlaylistExtractorTest {
@Test
void testGetUploaderAvatarUrl() throws ParsingException {
assertEquals(
"https://framatube.org/lazy-static/avatars/cd0f781d-0287-4be2-94f1-24cd732337b2.jpg",
"https://framatube.org/lazy-static/avatars/c6801ff9-cb49-42e6-b2db-3db623248115.jpg",
extractor.getUploaderAvatarUrl());
}
@ -73,7 +73,7 @@ public class PeertubePlaylistExtractorTest {
@Test
void testGetSubChannelAvatarUrl() throws ParsingException {
assertEquals(
"https://framatube.org/lazy-static/avatars/637753af-fcf2-4b61-88f9-b9857c953457.png",
"https://framatube.org/lazy-static/avatars/e801ccce-8694-4309-b0ab-e6f0e552ef77.png",
extractor.getSubChannelAvatarUrl());
}
}

10
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java
View File

@ -106,6 +106,11 @@ public class SoundcloudChannelExtractorTest {
public void testVerified() throws Exception {
assertTrue(extractor.isVerified());
}
@Override
public void testTabs() throws Exception {
}
}
public static class DubMatix implements BaseChannelExtractorTest {
@ -205,5 +210,10 @@ public class SoundcloudChannelExtractorTest {
public void testVerified() throws Exception {
assertTrue(extractor.isVerified());
}
@Override
public void testTabs() throws Exception {
// TODO: implement soundcloud playlist tab
}
}
}

109
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YouTubeChannelTabExtractorTest.java Normal file
View File

@ -0,0 +1,109 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabHandler;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelTabExtractor;
import java.io.IOException;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
public class YouTubeChannelTabExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/channelTab/";
public static class Playlists {
private static YoutubeChannelTabExtractor extractor;
@BeforeAll
public static void setUp() throws IOException, ExtractionException {
YoutubeTestsUtils.ensureStateless();
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "playlists"));
extractor = (YoutubeChannelTabExtractor) YouTube.getChannelTabExtractorFromId("UC2DjFE7Xf11URZqWBigcVOQ", ChannelTabHandler.Tab.Playlists);
extractor.fetchPage();
}
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
}
public static class Channels {
private static YoutubeChannelTabExtractor extractor;
@BeforeAll
public static void setUp() throws IOException, ExtractionException {
YoutubeTestsUtils.ensureStateless();
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "channels"));
extractor = (YoutubeChannelTabExtractor) YouTube.getChannelTabExtractorFromId("UC2DjFE7Xf11URZqWBigcVOQ", ChannelTabHandler.Tab.Channels);
extractor.fetchPage();
}
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
}
public static class Livestreams {
private static YoutubeChannelTabExtractor extractor;
@BeforeAll
public static void setUp() throws IOException, ExtractionException {
YoutubeTestsUtils.ensureStateless();
YoutubeParsingHelper.setVisitorData(YoutubeTestsUtils.VISITOR_DATA_NEW_CHANNEL_LAYOUT);
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "livestreams"));
extractor = (YoutubeChannelTabExtractor) YouTube.getChannelTabExtractorFromId("UCR-DXc1voovS8nhAvccRZhg", ChannelTabHandler.Tab.Livestreams);
extractor.fetchPage();
}
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
}
public static class Shorts {
private static YoutubeChannelTabExtractor extractor;
@BeforeAll
public static void setUp() throws IOException, ExtractionException {
YoutubeTestsUtils.ensureStateless();
YoutubeParsingHelper.setVisitorData(YoutubeTestsUtils.VISITOR_DATA_NEW_CHANNEL_LAYOUT);
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "shorts"));
extractor = (YoutubeChannelTabExtractor) YouTube.getChannelTabExtractorFromId("UCh8gHdtzO2tXd593_bjErWg", ChannelTabHandler.Tab.Shorts);
extractor.fetchPage();
}
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
}
}

113
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java
View File

@ -1,17 +1,5 @@
package org.schabi.newpipe.extractor.services.youtube;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.schabi.newpipe.downloader.DownloaderFactory;
@ -24,10 +12,19 @@ import org.schabi.newpipe.extractor.exceptions.AccountTerminatedException;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabHandler;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
import java.io.IOException;
import java.util.Set;
import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
/**
* Test for {@link ChannelExtractor}
@ -236,11 +233,17 @@ public class YoutubeChannelExtractorTest {
ExtractorAsserts.assertGreaterOrEqual(4_900_000, extractor.getSubscriberCount());
}
@Override
@Test
public void testVerified() throws Exception {
assertTrue(extractor.isVerified());
}
@Test
public void testTabs() throws Exception {
Set<ChannelTabHandler.Tab> tabs = extractor.getTabs().stream().map(ChannelTabHandler::getTab).collect(Collectors.toSet());
assertTrue(tabs.contains(ChannelTabHandler.Tab.Playlists));
assertTrue(tabs.contains(ChannelTabHandler.Tab.Channels));
}
}
// Youtube RED/Premium ad blocking test
@ -337,6 +340,12 @@ public class YoutubeChannelExtractorTest {
assertTrue(extractor.isVerified());
}
@Test
public void testTabs() throws Exception {
Set<ChannelTabHandler.Tab> tabs = extractor.getTabs().stream().map(ChannelTabHandler::getTab).collect(Collectors.toSet());
assertTrue(tabs.contains(ChannelTabHandler.Tab.Playlists));
assertTrue(tabs.contains(ChannelTabHandler.Tab.Channels));
}
}
public static class Kurzgesagt implements BaseChannelExtractorTest {
@ -434,6 +443,13 @@ public class YoutubeChannelExtractorTest {
public void testVerified() throws Exception {
assertTrue(extractor.isVerified());
}
@Test
public void testTabs() throws Exception {
Set<ChannelTabHandler.Tab> tabs = extractor.getTabs().stream().map(ChannelTabHandler::getTab).collect(Collectors.toSet());
assertTrue(tabs.contains(ChannelTabHandler.Tab.Playlists));
assertTrue(tabs.contains(ChannelTabHandler.Tab.Channels));
}
}
public static class KurzgesagtAdditional {
@ -548,6 +564,13 @@ public class YoutubeChannelExtractorTest {
public void testVerified() throws Exception {
assertTrue(extractor.isVerified());
}
@Test
public void testTabs() throws Exception {
Set<ChannelTabHandler.Tab> tabs = extractor.getTabs().stream().map(ChannelTabHandler::getTab).collect(Collectors.toSet());
assertTrue(tabs.contains(ChannelTabHandler.Tab.Playlists));
assertTrue(tabs.contains(ChannelTabHandler.Tab.Channels));
}
}
public static class RandomChannel implements BaseChannelExtractorTest {
@ -648,12 +671,19 @@ public class YoutubeChannelExtractorTest {
public void testVerified() throws Exception {
assertFalse(extractor.isVerified());
}
@Test
public void testTabs() throws Exception {
Set<ChannelTabHandler.Tab> tabs = extractor.getTabs().stream().map(ChannelTabHandler::getTab).collect(Collectors.toSet());
assertTrue(tabs.contains(ChannelTabHandler.Tab.Playlists));
assertTrue(tabs.contains(ChannelTabHandler.Tab.Channels));
}
}
/**
* Test the extraction of channel data from a RichGridRenderer (currently A/B tested, as of 11.10.2022)
*/
public static class RichGrid implements BaseChannelExtractorTest {
public static class RichGrid {
private static YoutubeChannelExtractor extractor;
@BeforeAll
@ -702,47 +732,60 @@ public class YoutubeChannelExtractorTest {
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
}
@Override
public void testMoreRelatedItems() throws Exception {
/**
* Test the extraction of the new channel tabs
*/
public static class ChannelTabs {
private static YoutubeChannelExtractor extractor;
@BeforeAll
public static void setUp() throws Exception {
YoutubeTestsUtils.ensureStateless();
YoutubeParsingHelper.setVisitorData(YoutubeTestsUtils.VISITOR_DATA_NEW_CHANNEL_LAYOUT);
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "tabs"));
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("https://www.youtube.com/channel/UCR-DXc1voovS8nhAvccRZhg");
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testDescription() throws Exception {
ExtractorAsserts.assertContains("NO SCRIPT, NO FEAR, ALL OPINION", extractor.getDescription());
public void testServiceId() {
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
}
@Test
public void testAvatarUrl() throws Exception {
String avatarUrl = extractor.getAvatarUrl();
assertIsSecureUrl(avatarUrl);
ExtractorAsserts.assertContains("yt3", avatarUrl);
public void testName() throws Exception {
assertEquals("Jeff Geerling", extractor.getName());
}
@Test
public void testBannerUrl() throws Exception {
String bannerUrl = extractor.getBannerUrl();
assertIsSecureUrl(bannerUrl);
ExtractorAsserts.assertContains("yt3", bannerUrl);
public void testId() throws Exception {
assertEquals("UCR-DXc1voovS8nhAvccRZhg", extractor.getId());
}
@Test
public void testFeedUrl() throws Exception {
assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UC2DjFE7Xf11URZqWBigcVOQ", extractor.getFeedUrl());
public void testUrl() throws ParsingException {
assertEquals("https://www.youtube.com/channel/UCR-DXc1voovS8nhAvccRZhg", extractor.getUrl());
}
@Test
public void testSubscriberCount() throws Exception {
ExtractorAsserts.assertGreaterOrEqual(800_000, extractor.getSubscriberCount());
public void testOriginalUrl() throws ParsingException {
assertEquals("https://www.youtube.com/channel/UCR-DXc1voovS8nhAvccRZhg", extractor.getOriginalUrl());
}
@Test
public void testVerified() throws Exception {
assertTrue(extractor.isVerified());
public void testTabs() throws Exception {
Set<ChannelTabHandler.Tab> tabs = extractor.getTabs().stream().map(ChannelTabHandler::getTab).collect(Collectors.toSet());
assertTrue(tabs.contains(ChannelTabHandler.Tab.Shorts));
assertTrue(tabs.contains(ChannelTabHandler.Tab.Livestreams));
assertTrue(tabs.contains(ChannelTabHandler.Tab.Playlists));
assertTrue(tabs.contains(ChannelTabHandler.Tab.Channels));
}
}
}

21
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View File

@ -1,18 +1,5 @@
package org.schabi.newpipe.extractor.services.youtube;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ListExtractor.ITEM_COUNT_UNKNOWN;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestListOfItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
@ -30,6 +17,11 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import java.io.IOException;
import static org.junit.jupiter.api.Assertions.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
/**
* Test for {@link YoutubePlaylistExtractor}
*/
@ -383,8 +375,7 @@ public class YoutubePlaylistExtractorTest {
@Test
public void testStreamCount() throws Exception {
// We are not able to extract the stream count of YouTube learning playlists
assertEquals(ITEM_COUNT_UNKNOWN, extractor.getStreamCount());
ExtractorAsserts.assertGreater(45, extractor.getStreamCount());
}
@Override

3
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTestsUtils.java
View File

@ -8,6 +8,8 @@ import java.util.Random;
* Utility class for keeping YouTube tests stateless.
*/
public final class YoutubeTestsUtils {
public static final String VISITOR_DATA_NEW_CHANNEL_LAYOUT = "CgtOa256ckVkcG5YVSi7-c6aBg%3D%3D";
private YoutubeTestsUtils() {
// No impl
}
@ -24,6 +26,7 @@ public final class YoutubeTestsUtils {
YoutubeParsingHelper.setConsentAccepted(false);
YoutubeParsingHelper.resetClientVersionAndKey();
YoutubeParsingHelper.setNumberGenerator(new Random(1));
YoutubeParsingHelper.setVisitorData(null);
YoutubeStreamExtractor.resetDeobfuscationCode();
}
}

82
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/tabs/generated_mock_0.json Normal file
View File

@ -0,0 +1,82 @@
{
"request": {
"httpMethod": "GET",
"url": "https://www.youtube.com/sw.js",
"headers": {
"Origin": [
"https://www.youtube.com"
],
"Referer": [
"https://www.youtube.com"
],
"Accept-Language": [
"en-GB, en;q\u003d0.9"
]
},
"localization": {
"languageCode": "en",
"countryCode": "GB"
}
},
"response": {
"responseCode": 200,
"responseMessage": "",
"responseHeaders": {
"access-control-allow-credentials": [
"true"
],
"access-control-allow-origin": [
"https://www.youtube.com"
],
"alt-svc": [
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
],
"cache-control": [
"private, max-age\u003d0"
],
"content-type": [
"text/javascript; charset\u003dutf-8"
],
"cross-origin-opener-policy-report-only": [
"same-origin; report-to\u003d\"youtube_main\""
],
"date": [
"Sat, 22 Oct 2022 11:38:25 GMT"
],
"expires": [
"Sat, 22 Oct 2022 11:38:25 GMT"
],
"p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
],
"permissions-policy": [
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
],
"report-to": [
"{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
],
"server": [
"ESF"
],
"set-cookie": [
"YSC\u003dpUJOIsVXl9I; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dSun, 26-Jan-2020 11:38:25 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+999; expires\u003dMon, 21-Oct-2024 11:38:25 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
],
"strict-transport-security": [
"max-age\u003d31536000"
],
"x-content-type-options": [
"nosniff"
],
"x-frame-options": [
"SAMEORIGIN"
],
"x-xss-protection": [
"0"
]
},
"responseBody": "\n self.addEventListener(\u0027install\u0027, event \u003d\u003e {\n event.waitUntil(self.skipWaiting());\n });\n self.addEventListener(\u0027activate\u0027, event \u003d\u003e {\n event.waitUntil(\n self.clients.claim().then(() \u003d\u003e self.registration.unregister()));\n });\n ",
"latestUrl": "https://www.youtube.com/sw.js"
}
}

76
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/tabs/generated_mock_1.json Normal file
View File

File diff suppressed because one or more lines are too long

442
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/tabs/generated_mock_2.json Normal file
View File

File diff suppressed because one or more lines are too long

82
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTab/channels/generated_mock_0.json Normal file
View File

@ -0,0 +1,82 @@
{
"request": {
"httpMethod": "GET",
"url": "https://www.youtube.com/sw.js",
"headers": {
"Origin": [
"https://www.youtube.com"
],
"Referer": [
"https://www.youtube.com"
],
"Accept-Language": [
"en-GB, en;q\u003d0.9"
]
},
"localization": {
"languageCode": "en",
"countryCode": "GB"
}
},
"response": {
"responseCode": 200,
"responseMessage": "",
"responseHeaders": {
"access-control-allow-credentials": [
"true"
],
"access-control-allow-origin": [
"https://www.youtube.com"
],
"alt-svc": [
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
],
"cache-control": [
"private, max-age\u003d0"
],
"content-type": [
"text/javascript; charset\u003dutf-8"
],
"cross-origin-opener-policy-report-only": [
"same-origin; report-to\u003d\"youtube_main\""
],
"date": [
"Sat, 22 Oct 2022 12:28:57 GMT"
],
"expires": [
"Sat, 22 Oct 2022 12:28:57 GMT"
],
"p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
],
"permissions-policy": [
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
],
"report-to": [
"{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
],
"server": [
"ESF"
],
"set-cookie": [
"YSC\u003dGguzpOaxDTM; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dSun, 26-Jan-2020 12:28:57 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+198; expires\u003dMon, 21-Oct-2024 12:28:57 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
],
"strict-transport-security": [
"max-age\u003d31536000"
],
"x-content-type-options": [
"nosniff"
],
"x-frame-options": [
"SAMEORIGIN"
],
"x-xss-protection": [
"0"
]
},
"responseBody": "\n self.addEventListener(\u0027install\u0027, event \u003d\u003e {\n event.waitUntil(self.skipWaiting());\n });\n self.addEventListener(\u0027activate\u0027, event \u003d\u003e {\n event.waitUntil(\n self.clients.claim().then(() \u003d\u003e self.registration.unregister()));\n });\n ",
"latestUrl": "https://www.youtube.com/sw.js"
}
}

76
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTab/channels/generated_mock_1.json Normal file
View File

File diff suppressed because one or more lines are too long

405
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTab/channels/generated_mock_2.json Normal file
View File

File diff suppressed because one or more lines are too long

572
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTab/channels/generated_mock_3.json Normal file
View File

File diff suppressed because one or more lines are too long

82
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTab/livestreams/generated_mock_0.json Normal file
View File

@ -0,0 +1,82 @@
{
"request": {
"httpMethod": "GET",
"url": "https://www.youtube.com/sw.js",
"headers": {
"Origin": [
"https://www.youtube.com"
],
"Referer": [
"https://www.youtube.com"
],
"Accept-Language": [
"en-GB, en;q\u003d0.9"
]
},
"localization": {
"languageCode": "en",
"countryCode": "GB"
}
},
"response": {
"responseCode": 200,
"responseMessage": "",
"responseHeaders": {
"access-control-allow-credentials": [
"true"
],
"access-control-allow-origin": [
"https://www.youtube.com"
],
"alt-svc": [
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
],
"cache-control": [
"private, max-age\u003d0"
],
"content-type": [
"text/javascript; charset\u003dutf-8"
],
"cross-origin-opener-policy-report-only": [
"same-origin; report-to\u003d\"youtube_main\""
],
"date": [
"Sat, 22 Oct 2022 12:35:49 GMT"
],
"expires": [
"Sat, 22 Oct 2022 12:35:49 GMT"
],
"p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
],
"permissions-policy": [
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
],
"report-to": [
"{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
],
"server": [
"ESF"
],
"set-cookie": [
"YSC\u003dqdVXyKBpMvg; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dSun, 26-Jan-2020 12:35:49 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+966; expires\u003dMon, 21-Oct-2024 12:35:49 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
],
"strict-transport-security": [
"max-age\u003d31536000"
],
"x-content-type-options": [
"nosniff"
],
"x-frame-options": [
"SAMEORIGIN"
],
"x-xss-protection": [
"0"
]
},
"responseBody": "\n self.addEventListener(\u0027install\u0027, event \u003d\u003e {\n event.waitUntil(self.skipWaiting());\n });\n self.addEventListener(\u0027activate\u0027, event \u003d\u003e {\n event.waitUntil(\n self.clients.claim().then(() \u003d\u003e self.registration.unregister()));\n });\n ",
"latestUrl": "https://www.youtube.com/sw.js"
}
}

76
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTab/livestreams/generated_mock_1.json Normal file
View File

File diff suppressed because one or more lines are too long

456
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTab/livestreams/generated_mock_2.json Normal file
View File

File diff suppressed because one or more lines are too long

617
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTab/livestreams/generated_mock_3.json Normal file
View File

File diff suppressed because one or more lines are too long

82
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTab/playlists/generated_mock_0.json Normal file
View File

@ -0,0 +1,82 @@
{
"request": {
"httpMethod": "GET",
"url": "https://www.youtube.com/sw.js",
"headers": {
"Origin": [
"https://www.youtube.com"
],
"Referer": [
"https://www.youtube.com"
],
"Accept-Language": [
"en-GB, en;q\u003d0.9"
]
},
"localization": {
"languageCode": "en",
"countryCode": "GB"
}
},
"response": {
"responseCode": 200,
"responseMessage": "",
"responseHeaders": {
"access-control-allow-credentials": [
"true"
],
"access-control-allow-origin": [
"https://www.youtube.com"
],
"alt-svc": [
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
],
"cache-control": [
"private, max-age\u003d0"
],
"content-type": [
"text/javascript; charset\u003dutf-8"
],
"cross-origin-opener-policy-report-only": [
"same-origin; report-to\u003d\"youtube_main\""
],
"date": [
"Sat, 22 Oct 2022 12:28:54 GMT"
],
"expires": [
"Sat, 22 Oct 2022 12:28:54 GMT"
],
"p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
],
"permissions-policy": [
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
],
"report-to": [
"{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
],
"server": [
"ESF"
],
"set-cookie": [
"YSC\u003d5H2kjM-PAmc; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dSun, 26-Jan-2020 12:28:54 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+749; expires\u003dMon, 21-Oct-2024 12:28:54 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
],
"strict-transport-security": [
"max-age\u003d31536000"
],
"x-content-type-options": [
"nosniff"
],
"x-frame-options": [
"SAMEORIGIN"
],
"x-xss-protection": [
"0"
]
},
"responseBody": "\n self.addEventListener(\u0027install\u0027, event \u003d\u003e {\n event.waitUntil(self.skipWaiting());\n });\n self.addEventListener(\u0027activate\u0027, event \u003d\u003e {\n event.waitUntil(\n self.clients.claim().then(() \u003d\u003e self.registration.unregister()));\n });\n ",
"latestUrl": "https://www.youtube.com/sw.js"
}
}

76
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTab/playlists/generated_mock_1.json Normal file
View File

File diff suppressed because one or more lines are too long

403
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTab/playlists/generated_mock_2.json Normal file
View File

File diff suppressed because one or more lines are too long

612
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTab/playlists/generated_mock_3.json Normal file
View File

File diff suppressed because one or more lines are too long

82
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTab/shorts/generated_mock_0.json Normal file
View File

@ -0,0 +1,82 @@
{
"request": {
"httpMethod": "GET",
"url": "https://www.youtube.com/sw.js",
"headers": {
"Origin": [
"https://www.youtube.com"
],
"Referer": [
"https://www.youtube.com"
],
"Accept-Language": [
"en-GB, en;q\u003d0.9"
]
},
"localization": {
"languageCode": "en",
"countryCode": "GB"
}
},
"response": {
"responseCode": 200,
"responseMessage": "",
"responseHeaders": {
"access-control-allow-credentials": [
"true"
],
"access-control-allow-origin": [
"https://www.youtube.com"
],
"alt-svc": [
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
],
"cache-control": [
"private, max-age\u003d0"
],
"content-type": [
"text/javascript; charset\u003dutf-8"
],
"cross-origin-opener-policy-report-only": [
"same-origin; report-to\u003d\"youtube_main\""
],
"date": [
"Sat, 22 Oct 2022 12:51:23 GMT"
],
"expires": [
"Sat, 22 Oct 2022 12:51:23 GMT"
],
"p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
],
"permissions-policy": [
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
],
"report-to": [
"{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
],
"server": [
"ESF"
],
"set-cookie": [
"YSC\u003dWjNB29jj0Ww; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dSun, 26-Jan-2020 12:51:23 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+068; expires\u003dMon, 21-Oct-2024 12:51:23 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
],
"strict-transport-security": [
"max-age\u003d31536000"
],
"x-content-type-options": [
"nosniff"
],
"x-frame-options": [
"SAMEORIGIN"
],
"x-xss-protection": [
"0"
]
},
"responseBody": "\n self.addEventListener(\u0027install\u0027, event \u003d\u003e {\n event.waitUntil(self.skipWaiting());\n });\n self.addEventListener(\u0027activate\u0027, event \u003d\u003e {\n event.waitUntil(\n self.clients.claim().then(() \u003d\u003e self.registration.unregister()));\n });\n ",
"latestUrl": "https://www.youtube.com/sw.js"
}
}

76
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTab/shorts/generated_mock_1.json Normal file
View File

File diff suppressed because one or more lines are too long

456
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTab/shorts/generated_mock_2.json Normal file
View File

File diff suppressed because one or more lines are too long

711
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTab/shorts/generated_mock_3.json Normal file
View File

File diff suppressed because one or more lines are too long