mirror of
https://github.com/TeamNewPipe/NewPipeExtractor
synced 2024-11-29 21:41:05 +01:00
[YouTube] Add support for channel tabs and tags and age-restricted channels
Support of tags and videos, shorts, live, playlists and channels tabs has been added for non-age restricted channels. Age-restricted channels are now also supported and always returned the videos, shorts and live tabs, accessible using system playlists. These tabs are the only ones which can be accessed using YouTube's desktop website without being logged-in. The videos channel tab parameter has been updated to the one used by the desktop website and when a channel extraction is fetched, this tab is returned in the list of tabs as a cached one in the corresponding link handler. Visitor data support per request has been added, as a valid visitor data is required to fetch continuations with contents on the shorts tab. It is only used in this case to enhance privacy. A dedicated shorts UI elements (reelItemRenderers) extractor has been added, YoutubeReelInfoItemExtractor. These elements do not provide the exact view count, any uploader info (name, URL, avatar, verified status) and the upload date. All service's LinkHandlers are now using the singleton pattern and some code has been also improved on the files changed. Co-authored-by: ThetaDev <t.testboy@gmail.com> Co-authored-by: Stypox <stypox@pm.me>
This commit is contained in:
AudricV
Stypox
parent
4586067934
commit
7366eab156
@ -0,0 +1,271 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube;
|
||||
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonWriter;
|
||||
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.localization.ContentCountry;
|
||||
import org.schabi.newpipe.extractor.localization.Localization;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.defaultAlertsCheck;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
/**
|
||||
* Shared functions for extracting YouTube channel pages and tabs.
|
||||
*/
|
||||
public final class YoutubeChannelHelper {
|
||||
private YoutubeChannelHelper() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Take a YouTube channel ID or URL path, resolve it if necessary and return a channel ID.
|
||||
*
|
||||
* @param idOrPath a YouTube channel ID or URL path
|
||||
* @return a YouTube channel ID
|
||||
* @throws IOException if a channel resolve request failed
|
||||
* @throws ExtractionException if a channel resolve request response could not be parsed or is
|
||||
* invalid
|
||||
*/
|
||||
@Nonnull
|
||||
public static String resolveChannelId(@Nonnull final String idOrPath)
|
||||
throws ExtractionException, IOException {
|
||||
final String[] channelId = idOrPath.split("/");
|
||||
|
||||
if (channelId[0].startsWith("UC")) {
|
||||
return channelId[0];
|
||||
}
|
||||
|
||||
// If the URL is not a /channel URL, we need to use the navigation/resolve_url endpoint of
|
||||
// the InnerTube API to get the channel id.
|
||||
// Otherwise, we couldn't get information about the channel associated with this URL, if
|
||||
// there is one.
|
||||
if (!channelId[0].equals("channel")) {
|
||||
final byte[] body = JsonWriter.string(
|
||||
prepareDesktopJsonBuilder(Localization.DEFAULT, ContentCountry.DEFAULT)
|
||||
.value("url", "https://www.youtube.com/" + idOrPath)
|
||||
.done())
|
||||
.getBytes(StandardCharsets.UTF_8);
|
||||
|
||||
final JsonObject jsonResponse = getJsonPostResponse(
|
||||
"navigation/resolve_url", body, Localization.DEFAULT);
|
||||
|
||||
checkIfChannelResponseIsValid(jsonResponse);
|
||||
|
||||
final JsonObject endpoint = jsonResponse.getObject("endpoint");
|
||||
|
||||
final String webPageType = endpoint.getObject("commandMetadata")
|
||||
.getObject("webCommandMetadata")
|
||||
.getString("webPageType", "");
|
||||
|
||||
final JsonObject browseEndpoint = endpoint.getObject("browseEndpoint");
|
||||
final String browseId = browseEndpoint.getString("browseId", "");
|
||||
|
||||
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|
||||
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
|
||||
&& !browseId.isEmpty()) {
|
||||
if (!browseId.startsWith("UC")) {
|
||||
throw new ExtractionException("Redirected id is not pointing to a channel");
|
||||
}
|
||||
|
||||
return browseId;
|
||||
}
|
||||
}
|
||||
|
||||
return channelId[1];
|
||||
}
|
||||
|
||||
/**
|
||||
* Response data object for {@link #getChannelResponse(String, String, Localization,
|
||||
* ContentCountry)}, after any redirection in the allowed redirects count ({@code 3}).
|
||||
*/
|
||||
public static final class ChannelResponseData {
|
||||
|
||||
/**
|
||||
* The channel response as a JSON object, after all redirects.
|
||||
*/
|
||||
@Nonnull
|
||||
public final JsonObject jsonResponse;
|
||||
|
||||
/**
|
||||
* The channel ID after all redirects.
|
||||
*/
|
||||
@Nonnull
|
||||
public final String channelId;
|
||||
|
||||
private ChannelResponseData(@Nonnull final JsonObject jsonResponse,
|
||||
@Nonnull final String channelId) {
|
||||
this.jsonResponse = jsonResponse;
|
||||
this.channelId = channelId;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a YouTube channel tab response, using the given channel ID and tab parameters.
|
||||
*
|
||||
* <p>
|
||||
* Redirections to other channels such as are supported to up to 3 redirects, which could
|
||||
* happen for instance for localized channels or auto-generated ones such as the {@code Movies
|
||||
* and Shows} (channel IDs {@code UCuJcl0Ju-gPDoksRjK1ya-w}, {@code UChBfWrfBXL9wS6tQtgjt_OQ}
|
||||
* and {@code UCok7UTQQEP1Rsctxiv3gwSQ} of this channel redirect to the
|
||||
* {@code UClgRkhTL3_hImCAmdLfDE4g} one).
|
||||
* </p>
|
||||
*
|
||||
* @param channelId a valid YouTube channel ID
|
||||
* @param parameters the parameters to specify the YouTube channel tab; if invalid ones are
|
||||
* specified, YouTube should return the {@code Home} tab
|
||||
* @param localization the {@link Localization} to use
|
||||
* @param country the {@link ContentCountry} to use
|
||||
* @return a {@link ChannelResponseData channel response data}
|
||||
* @throws IOException if a channel request failed
|
||||
* @throws ExtractionException if a channel request response could not be parsed or is invalid
|
||||
*/
|
||||
@Nonnull
|
||||
public static ChannelResponseData getChannelResponse(@Nonnull final String channelId,
|
||||
@Nonnull final String parameters,
|
||||
@Nonnull final Localization localization,
|
||||
@Nonnull final ContentCountry country)
|
||||
throws ExtractionException, IOException {
|
||||
String id = channelId;
|
||||
JsonObject ajaxJson = null;
|
||||
|
||||
int level = 0;
|
||||
while (level < 3) {
|
||||
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
|
||||
localization, country)
|
||||
.value("browseId", id)
|
||||
.value("params", parameters)
|
||||
.done())
|
||||
.getBytes(StandardCharsets.UTF_8);
|
||||
|
||||
final JsonObject jsonResponse = getJsonPostResponse(
|
||||
"browse", body, localization);
|
||||
|
||||
checkIfChannelResponseIsValid(jsonResponse);
|
||||
|
||||
final JsonObject endpoint = jsonResponse.getArray("onResponseReceivedActions")
|
||||
.getObject(0)
|
||||
.getObject("navigateAction")
|
||||
.getObject("endpoint");
|
||||
|
||||
final String webPageType = endpoint.getObject("commandMetadata")
|
||||
.getObject("webCommandMetadata")
|
||||
.getString("webPageType", "");
|
||||
|
||||
final String browseId = endpoint.getObject("browseEndpoint")
|
||||
.getString("browseId", "");
|
||||
|
||||
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|
||||
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
|
||||
&& !browseId.isEmpty()) {
|
||||
if (!browseId.startsWith("UC")) {
|
||||
throw new ExtractionException("Redirected id is not pointing to a channel");
|
||||
}
|
||||
|
||||
id = browseId;
|
||||
level++;
|
||||
} else {
|
||||
ajaxJson = jsonResponse;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (ajaxJson == null) {
|
||||
throw new ExtractionException("Got no channel response");
|
||||
}
|
||||
|
||||
defaultAlertsCheck(ajaxJson);
|
||||
|
||||
return new ChannelResponseData(ajaxJson, id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Assert that a channel JSON response does not contain an {@code error} JSON object.
|
||||
*
|
||||
* @param jsonResponse a channel JSON response
|
||||
* @throws ContentNotAvailableException if the channel was not found
|
||||
*/
|
||||
private static void checkIfChannelResponseIsValid(@Nonnull final JsonObject jsonResponse)
|
||||
throws ContentNotAvailableException {
|
||||
if (!isNullOrEmpty(jsonResponse.getObject("error"))) {
|
||||
final JsonObject errorJsonObject = jsonResponse.getObject("error");
|
||||
final int errorCode = errorJsonObject.getInt("code");
|
||||
if (errorCode == 404) {
|
||||
throw new ContentNotAvailableException("This channel doesn't exist.");
|
||||
} else {
|
||||
throw new ContentNotAvailableException("Got error:\""
|
||||
+ errorJsonObject.getString("status") + "\": "
|
||||
+ errorJsonObject.getString("message"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A channel header response.
|
||||
*
|
||||
* <p>
|
||||
* This class allows the distinction between a classic header and a carousel one, used for
|
||||
* auto-generated ones like the gaming or music topic channels and for big events such as the
|
||||
* Coachella music festival, which have a different data structure and do not return the same
|
||||
* properties.
|
||||
* </p>
|
||||
*/
|
||||
public static final class ChannelHeader {
|
||||
|
||||
/**
|
||||
* The channel header JSON response.
|
||||
*/
|
||||
@Nonnull
|
||||
public final JsonObject json;
|
||||
|
||||
/**
|
||||
* Whether the header is a {@code carouselHeaderRenderer}.
|
||||
*
|
||||
* <p>
|
||||
* See the class documentation for more details.
|
||||
* </p>
|
||||
*/
|
||||
public final boolean isCarouselHeader;
|
||||
|
||||
private ChannelHeader(@Nonnull final JsonObject json, final boolean isCarouselHeader) {
|
||||
this.json = json;
|
||||
this.isCarouselHeader = isCarouselHeader;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a channel header as an {@link Optional} it if exists.
|
||||
*
|
||||
* @param channelResponse a full channel JSON response
|
||||
* @return an {@link Optional} containing a {@link ChannelHeader} or an empty {@link Optional}
|
||||
* if no supported header has been found
|
||||
*/
|
||||
@Nonnull
|
||||
public static Optional<ChannelHeader> getChannelHeader(
|
||||
@Nonnull final JsonObject channelResponse) {
|
||||
final JsonObject header = channelResponse.getObject("header");
|
||||
|
||||
if (header.has("c4TabbedHeaderRenderer")) {
|
||||
return Optional.of(header.getObject("c4TabbedHeaderRenderer"))
|
||||
.map(json -> new ChannelHeader(json, false));
|
||||
} else if (header.has("carouselHeaderRenderer")) {
|
||||
return header.getObject("carouselHeaderRenderer")
|
||||
.getArray("contents")
|
||||
.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.filter(item -> item.has("topicChannelDetailsRenderer"))
|
||||
.findFirst()
|
||||
.map(item -> item.getObject("topicChannelDetailsRenderer"))
|
||||
.map(json -> new ChannelHeader(json, true));
|
||||
} else {
|
||||
return Optional.empty();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1230,8 +1230,17 @@ public final class YoutubeParsingHelper {
|
||||
@Nonnull final Localization localization,
|
||||
@Nonnull final ContentCountry contentCountry)
|
||||
throws IOException, ExtractionException {
|
||||
return prepareDesktopJsonBuilder(localization, contentCountry, null);
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
|
||||
@Nonnull final Localization localization,
|
||||
@Nonnull final ContentCountry contentCountry,
|
||||
@Nullable final String visitorData)
|
||||
throws IOException, ExtractionException {
|
||||
// @formatter:off
|
||||
return JsonObject.builder()
|
||||
final JsonBuilder<JsonObject> builder = JsonObject.builder()
|
||||
.object("context")
|
||||
.object("client")
|
||||
.value("hl", localization.getLocalizationCode())
|
||||
@ -1239,8 +1248,13 @@ public final class YoutubeParsingHelper {
|
||||
.value("clientName", "WEB")
|
||||
.value("clientVersion", getClientVersion())
|
||||
.value("originalUrl", "https://www.youtube.com")
|
||||
.value("platform", "DESKTOP")
|
||||
.end()
|
||||
.value("platform", "DESKTOP");
|
||||
|
||||
if (visitorData != null) {
|
||||
builder.value("visitorData", visitorData);
|
||||
}
|
||||
|
||||
return builder.end()
|
||||
.object("request")
|
||||
.array("internalExperimentFlags")
|
||||
.end()
|
||||
|
||||
@ -8,6 +8,7 @@ import static java.util.Arrays.asList;
|
||||
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
|
||||
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
|
||||
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.feed.FeedExtractor;
|
||||
@ -16,6 +17,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
|
||||
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
|
||||
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
|
||||
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
|
||||
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.localization.ContentCountry;
|
||||
@ -23,6 +25,7 @@ import org.schabi.newpipe.extractor.localization.Localization;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
||||
import org.schabi.newpipe.extractor.search.SearchExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelTabExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
|
||||
@ -34,6 +37,7 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscript
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSuggestionExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
|
||||
@ -88,6 +92,11 @@ public class YoutubeService extends StreamingService {
|
||||
return YoutubeChannelLinkHandlerFactory.getInstance();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ListLinkHandlerFactory getChannelTabLHFactory() {
|
||||
return YoutubeChannelTabLinkHandlerFactory.getInstance();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ListLinkHandlerFactory getPlaylistLHFactory() {
|
||||
return YoutubePlaylistLinkHandlerFactory.getInstance();
|
||||
@ -108,6 +117,15 @@ public class YoutubeService extends StreamingService {
|
||||
return new YoutubeChannelExtractor(this, linkHandler);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChannelTabExtractor getChannelTabExtractor(final ListLinkHandler linkHandler) {
|
||||
if (linkHandler instanceof ReadyChannelTabListLinkHandler) {
|
||||
return ((ReadyChannelTabListLinkHandler) linkHandler).getChannelTabExtractor(this);
|
||||
} else {
|
||||
return new YoutubeChannelTabExtractor(this, linkHandler);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
|
||||
if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) {
|
||||
@ -136,16 +154,17 @@ public class YoutubeService extends StreamingService {
|
||||
@Override
|
||||
public KioskList getKioskList() throws ExtractionException {
|
||||
final KioskList list = new KioskList(this);
|
||||
final ListLinkHandlerFactory h = YoutubeTrendingLinkHandlerFactory.getInstance();
|
||||
|
||||
// add kiosks here e.g.:
|
||||
try {
|
||||
list.addKioskEntry(
|
||||
(streamingService, url, id) -> new YoutubeTrendingExtractor(
|
||||
YoutubeService.this,
|
||||
new YoutubeTrendingLinkHandlerFactory().fromUrl(url),
|
||||
h.fromUrl(url),
|
||||
id
|
||||
),
|
||||
new YoutubeTrendingLinkHandlerFactory(),
|
||||
h,
|
||||
YoutubeTrendingExtractor.KIOSK_ID
|
||||
);
|
||||
list.setDefaultKiosk(YoutubeTrendingExtractor.KIOSK_ID);
|
||||
|
||||
@ -1,39 +1,35 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube.extractors;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.getChannelResponse;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.resolveChannelId;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
import com.grack.nanojson.JsonArray;
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonWriter;
|
||||
|
||||
import org.schabi.newpipe.extractor.Page;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
|
||||
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
|
||||
import org.schabi.newpipe.extractor.downloader.Downloader;
|
||||
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
|
||||
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
|
||||
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
|
||||
import org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper;
|
||||
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelTabExtractor.VideosTabExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
@ -59,22 +55,24 @@ import javax.annotation.Nullable;
|
||||
*/
|
||||
|
||||
public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
private JsonObject initialData;
|
||||
private Optional<JsonObject> channelHeader;
|
||||
private boolean isCarouselHeader = false;
|
||||
private JsonObject videoTab;
|
||||
|
||||
private JsonObject jsonResponse;
|
||||
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
|
||||
private Optional<YoutubeChannelHelper.ChannelHeader> channelHeader;
|
||||
|
||||
private String channelId;
|
||||
|
||||
/**
|
||||
* Some channels have response redirects and the only way to reliably get the id is by saving it
|
||||
* If a channel is age-restricted, its pages are only accessible to logged-in and
|
||||
* age-verified users, we get an {@code channelAgeGateRenderer} in this case, containing only
|
||||
* the following metadata: channel name and channel avatar.
|
||||
*
|
||||
* <p>
|
||||
* "Movies & Shows":
|
||||
* <pre>
|
||||
* UCuJcl0Ju-gPDoksRjK1ya-w ┐
|
||||
* UChBfWrfBXL9wS6tQtgjt_OQ ├ UClgRkhTL3_hImCAmdLfDE4g
|
||||
* UCok7UTQQEP1Rsctxiv3gwSQ ┘
|
||||
* </pre>
|
||||
* This restriction doesn't seem to apply to all countries.
|
||||
* </p>
|
||||
*/
|
||||
private String redirectedChannelId;
|
||||
@Nullable
|
||||
private JsonObject channelAgeGateRenderer;
|
||||
|
||||
public YoutubeChannelExtractor(final StreamingService service,
|
||||
final ListLinkHandler linkHandler) {
|
||||
@ -85,132 +83,42 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
public void onFetchPage(@Nonnull final Downloader downloader)
|
||||
throws IOException, ExtractionException {
|
||||
final String channelPath = super.getId();
|
||||
final String[] channelId = channelPath.split("/");
|
||||
String id = "";
|
||||
// If the url is an URL which is not a /channel URL, we need to use the
|
||||
// navigation/resolve_url endpoint of the InnerTube API to get the channel id. Otherwise,
|
||||
// we couldn't get information about the channel associated with this URL, if there is one.
|
||||
if (!channelId[0].equals("channel")) {
|
||||
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
|
||||
getExtractorLocalization(), getExtractorContentCountry())
|
||||
.value("url", "https://www.youtube.com/" + channelPath)
|
||||
.done())
|
||||
.getBytes(StandardCharsets.UTF_8);
|
||||
final String id = resolveChannelId(channelPath);
|
||||
// Fetch Videos tab
|
||||
final YoutubeChannelHelper.ChannelResponseData data = getChannelResponse(id,
|
||||
"EgZ2aWRlb3PyBgQKAjoA", getExtractorLocalization(), getExtractorContentCountry());
|
||||
|
||||
final JsonObject jsonResponse = getJsonPostResponse("navigation/resolve_url",
|
||||
body, getExtractorLocalization());
|
||||
|
||||
checkIfChannelResponseIsValid(jsonResponse);
|
||||
|
||||
final JsonObject endpoint = jsonResponse.getObject("endpoint");
|
||||
|
||||
final String webPageType = endpoint.getObject("commandMetadata")
|
||||
.getObject("webCommandMetadata")
|
||||
.getString("webPageType", "");
|
||||
|
||||
final JsonObject browseEndpoint = endpoint.getObject("browseEndpoint");
|
||||
final String browseId = browseEndpoint.getString("browseId", "");
|
||||
|
||||
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|
||||
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
|
||||
&& !browseId.isEmpty()) {
|
||||
if (!browseId.startsWith("UC")) {
|
||||
throw new ExtractionException("Redirected id is not pointing to a channel");
|
||||
}
|
||||
|
||||
id = browseId;
|
||||
redirectedChannelId = browseId;
|
||||
}
|
||||
} else {
|
||||
id = channelId[1];
|
||||
}
|
||||
JsonObject ajaxJson = null;
|
||||
|
||||
int level = 0;
|
||||
while (level < 3) {
|
||||
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
|
||||
getExtractorLocalization(), getExtractorContentCountry())
|
||||
.value("browseId", id)
|
||||
.value("params", "EgZ2aWRlb3M%3D") // Equal to videos
|
||||
.done())
|
||||
.getBytes(StandardCharsets.UTF_8);
|
||||
|
||||
final JsonObject jsonResponse = getJsonPostResponse("browse", body,
|
||||
getExtractorLocalization());
|
||||
|
||||
checkIfChannelResponseIsValid(jsonResponse);
|
||||
|
||||
final JsonObject endpoint = jsonResponse.getArray("onResponseReceivedActions")
|
||||
.getObject(0)
|
||||
.getObject("navigateAction")
|
||||
.getObject("endpoint");
|
||||
|
||||
final String webPageType = endpoint.getObject("commandMetadata")
|
||||
.getObject("webCommandMetadata")
|
||||
.getString("webPageType", "");
|
||||
|
||||
final String browseId = endpoint.getObject("browseEndpoint").getString("browseId",
|
||||
"");
|
||||
|
||||
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|
||||
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
|
||||
&& !browseId.isEmpty()) {
|
||||
if (!browseId.startsWith("UC")) {
|
||||
throw new ExtractionException("Redirected id is not pointing to a channel");
|
||||
}
|
||||
|
||||
id = browseId;
|
||||
redirectedChannelId = browseId;
|
||||
level++;
|
||||
} else {
|
||||
ajaxJson = jsonResponse;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (ajaxJson == null) {
|
||||
throw new ExtractionException("Could not fetch initial JSON data");
|
||||
}
|
||||
|
||||
initialData = ajaxJson;
|
||||
YoutubeParsingHelper.defaultAlertsCheck(initialData);
|
||||
jsonResponse = data.jsonResponse;
|
||||
channelId = data.channelId;
|
||||
channelAgeGateRenderer = getChannelAgeGateRenderer();
|
||||
}
|
||||
|
||||
private void checkIfChannelResponseIsValid(@Nonnull final JsonObject jsonResponse)
|
||||
throws ContentNotAvailableException {
|
||||
if (!isNullOrEmpty(jsonResponse.getObject("error"))) {
|
||||
final JsonObject errorJsonObject = jsonResponse.getObject("error");
|
||||
final int errorCode = errorJsonObject.getInt("code");
|
||||
if (errorCode == 404) {
|
||||
throw new ContentNotAvailableException("This channel doesn't exist.");
|
||||
} else {
|
||||
throw new ContentNotAvailableException("Got error:\""
|
||||
+ errorJsonObject.getString("status") + "\": "
|
||||
+ errorJsonObject.getString("message"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private Optional<JsonObject> getChannelHeader() {
|
||||
if (channelHeader == null) {
|
||||
final JsonObject h = initialData.getObject("header");
|
||||
|
||||
if (h.has("c4TabbedHeaderRenderer")) {
|
||||
channelHeader = Optional.of(h.getObject("c4TabbedHeaderRenderer"));
|
||||
} else if (h.has("carouselHeaderRenderer")) {
|
||||
isCarouselHeader = true;
|
||||
channelHeader = h.getObject("carouselHeaderRenderer")
|
||||
@Nullable
|
||||
private JsonObject getChannelAgeGateRenderer() {
|
||||
return jsonResponse.getObject("contents")
|
||||
.getObject("twoColumnBrowseResultsRenderer")
|
||||
.getArray("tabs")
|
||||
.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.flatMap(tab -> tab.getObject("tabRenderer")
|
||||
.getObject("content")
|
||||
.getObject("sectionListRenderer")
|
||||
.getArray("contents")
|
||||
.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.filter(itm -> itm.has("topicChannelDetailsRenderer"))
|
||||
.findFirst()
|
||||
.map(itm -> itm.getObject("topicChannelDetailsRenderer"));
|
||||
} else {
|
||||
channelHeader = Optional.empty();
|
||||
}
|
||||
.map(JsonObject.class::cast))
|
||||
.filter(content -> content.has("channelAgeGateRenderer"))
|
||||
.map(content -> content.getObject("channelAgeGateRenderer"))
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private Optional<YoutubeChannelHelper.ChannelHeader> getChannelHeader() {
|
||||
//noinspection OptionalAssignedToNull
|
||||
if (channelHeader == null) {
|
||||
channelHeader = YoutubeChannelHelper.getChannelHeader(jsonResponse);
|
||||
}
|
||||
return channelHeader;
|
||||
}
|
||||
@ -229,57 +137,70 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
@Override
|
||||
public String getId() throws ParsingException {
|
||||
return getChannelHeader()
|
||||
.flatMap(header -> Optional.ofNullable(header.getString("channelId")).or(
|
||||
() -> Optional.ofNullable(header.getObject("navigationEndpoint")
|
||||
.flatMap(header -> Optional.ofNullable(header.json.getString("channelId"))
|
||||
.or(() -> Optional.ofNullable(header.json.getObject("navigationEndpoint")
|
||||
.getObject("browseEndpoint")
|
||||
.getString("browseId"))
|
||||
))
|
||||
.or(() -> Optional.ofNullable(redirectedChannelId))
|
||||
.orElseThrow(() -> new ParsingException("Could not get channel id"));
|
||||
.or(() -> Optional.ofNullable(channelId))
|
||||
.orElseThrow(() -> new ParsingException("Could not get channel ID"));
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getName() throws ParsingException {
|
||||
final String mdName = initialData.getObject("metadata")
|
||||
.getObject("channelMetadataRenderer")
|
||||
.getString("title");
|
||||
if (!isNullOrEmpty(mdName)) {
|
||||
return mdName;
|
||||
if (channelAgeGateRenderer != null) {
|
||||
return channelAgeGateRenderer.getString("channelTitle");
|
||||
}
|
||||
|
||||
final Optional<JsonObject> header = getChannelHeader();
|
||||
if (header.isPresent()) {
|
||||
final Object title = header.get().get("title");
|
||||
final String metadataRendererTitle = jsonResponse.getObject("metadata")
|
||||
.getObject("channelMetadataRenderer")
|
||||
.getString("title");
|
||||
if (!isNullOrEmpty(metadataRendererTitle)) {
|
||||
return metadataRendererTitle;
|
||||
}
|
||||
|
||||
return getChannelHeader().flatMap(header -> {
|
||||
final Object title = header.json.get("title");
|
||||
if (title instanceof String) {
|
||||
return (String) title;
|
||||
return Optional.of((String) title);
|
||||
} else if (title instanceof JsonObject) {
|
||||
final String headerName = getTextFromObject((JsonObject) title);
|
||||
if (!isNullOrEmpty(headerName)) {
|
||||
return headerName;
|
||||
return Optional.of(headerName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw new ParsingException("Could not get channel name");
|
||||
return Optional.empty();
|
||||
}).orElseThrow(() -> new ParsingException("Could not get channel name"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getAvatarUrl() throws ParsingException {
|
||||
return getChannelHeader().flatMap(header -> Optional.ofNullable(
|
||||
header.getObject("avatar").getArray("thumbnails")
|
||||
.getObject(0).getString("url")
|
||||
))
|
||||
.map(YoutubeParsingHelper::fixThumbnailUrl)
|
||||
.orElseThrow(() -> new ParsingException("Could not get avatar"));
|
||||
final JsonObject avatarJsonObjectContainer;
|
||||
if (channelAgeGateRenderer != null) {
|
||||
avatarJsonObjectContainer = channelAgeGateRenderer;
|
||||
} else {
|
||||
avatarJsonObjectContainer = getChannelHeader().map(header -> header.json)
|
||||
.orElseThrow(() -> new ParsingException("Could not get avatar URL"));
|
||||
}
|
||||
|
||||
return YoutubeParsingHelper.fixThumbnailUrl(avatarJsonObjectContainer.getObject("avatar")
|
||||
.getArray("thumbnails")
|
||||
.getObject(0)
|
||||
.getString("url"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getBannerUrl() throws ParsingException {
|
||||
if (channelAgeGateRenderer != null) {
|
||||
return "";
|
||||
}
|
||||
|
||||
return getChannelHeader().flatMap(header -> Optional.ofNullable(
|
||||
header.getObject("banner").getArray("thumbnails")
|
||||
.getObject(0).getString("url")
|
||||
))
|
||||
header.json.getObject("banner")
|
||||
.getArray("thumbnails")
|
||||
.getObject(0)
|
||||
.getString("url")))
|
||||
.filter(url -> !url.contains("s.ytimg.com") && !url.contains("default_banner"))
|
||||
.map(YoutubeParsingHelper::fixThumbnailUrl)
|
||||
// Channels may not have a banner, so no exception should be thrown if no banner is
|
||||
@ -290,6 +211,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
|
||||
@Override
|
||||
public String getFeedUrl() throws ParsingException {
|
||||
// RSS feeds are accessible for age-restricted channels, no need to check whether a channel
|
||||
// has a channelAgeGateRenderer
|
||||
try {
|
||||
return YoutubeParsingHelper.getFeedUrlFrom(getId());
|
||||
} catch (final Exception e) {
|
||||
@ -299,14 +222,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
|
||||
@Override
|
||||
public long getSubscriberCount() throws ParsingException {
|
||||
final Optional<JsonObject> header = getChannelHeader();
|
||||
if (header.isPresent()) {
|
||||
if (channelAgeGateRenderer != null) {
|
||||
return UNKNOWN_SUBSCRIBER_COUNT;
|
||||
}
|
||||
|
||||
final Optional<YoutubeChannelHelper.ChannelHeader> headerOpt = getChannelHeader();
|
||||
if (headerOpt.isPresent()) {
|
||||
final JsonObject header = headerOpt.get().json;
|
||||
JsonObject textObject = null;
|
||||
|
||||
if (header.get().has("subscriberCountText")) {
|
||||
textObject = header.get().getObject("subscriberCountText");
|
||||
} else if (header.get().has("subtitle")) {
|
||||
textObject = header.get().getObject("subtitle");
|
||||
if (header.has("subscriberCountText")) {
|
||||
textObject = header.getObject("subscriberCountText");
|
||||
} else if (header.has("subtitle")) {
|
||||
textObject = header.getObject("subtitle");
|
||||
}
|
||||
|
||||
if (textObject != null) {
|
||||
@ -317,13 +245,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return UNKNOWN_SUBSCRIBER_COUNT;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() throws ParsingException {
|
||||
if (channelAgeGateRenderer != null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
return initialData.getObject("metadata").getObject("channelMetadataRenderer")
|
||||
return jsonResponse.getObject("metadata")
|
||||
.getObject("channelMetadataRenderer")
|
||||
.getString("description");
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get channel description", e);
|
||||
@ -347,190 +281,139 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||
|
||||
@Override
|
||||
public boolean isVerified() throws ParsingException {
|
||||
// The CarouselHeaderRenderer does not contain any verification badges.
|
||||
// Since it is only shown on YT-internal channels or on channels of large organizations
|
||||
// broadcasting live events, we can assume the channel to be verified.
|
||||
if (isCarouselHeader) {
|
||||
return true;
|
||||
if (channelAgeGateRenderer != null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return getChannelHeader()
|
||||
.map(header -> header.getArray("badges"))
|
||||
.map(YoutubeParsingHelper::isVerified)
|
||||
.orElse(false);
|
||||
final Optional<YoutubeChannelHelper.ChannelHeader> headerOpt = getChannelHeader();
|
||||
if (headerOpt.isPresent()) {
|
||||
final YoutubeChannelHelper.ChannelHeader header = headerOpt.get();
|
||||
|
||||
// The CarouselHeaderRenderer does not contain any verification badges.
|
||||
// Since it is only shown on YT-internal channels or on channels of large organizations
|
||||
// broadcasting live events, we can assume the channel to be verified.
|
||||
if (header.isCarouselHeader) {
|
||||
return true;
|
||||
}
|
||||
return YoutubeParsingHelper.isVerified(header.json.getArray("badges"));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
|
||||
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
|
||||
Page nextPage = null;
|
||||
|
||||
if (getVideoTab() != null) {
|
||||
final JsonObject tabContent = getVideoTab().getObject("content");
|
||||
JsonArray items = tabContent
|
||||
.getObject("sectionListRenderer")
|
||||
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
|
||||
.getArray("contents").getObject(0).getObject("gridRenderer").getArray("items");
|
||||
|
||||
if (items.isEmpty()) {
|
||||
items = tabContent.getObject("richGridRenderer").getArray("contents");
|
||||
}
|
||||
|
||||
final List<String> channelIds = new ArrayList<>();
|
||||
channelIds.add(getName());
|
||||
channelIds.add(getUrl());
|
||||
final JsonObject continuation = collectStreamsFrom(collector, items, channelIds);
|
||||
|
||||
nextPage = getNextPageFrom(continuation, channelIds);
|
||||
public List<ListLinkHandler> getTabs() throws ParsingException {
|
||||
if (channelAgeGateRenderer == null) {
|
||||
return getTabsForNonAgeRestrictedChannels();
|
||||
}
|
||||
|
||||
return new InfoItemsPage<>(collector, nextPage);
|
||||
return getTabsForAgeRestrictedChannels();
|
||||
}
|
||||
|
||||
@Override
|
||||
public InfoItemsPage<StreamInfoItem> getPage(final Page page)
|
||||
throws IOException, ExtractionException {
|
||||
if (page == null || isNullOrEmpty(page.getUrl())) {
|
||||
throw new IllegalArgumentException("Page doesn't contain an URL");
|
||||
}
|
||||
|
||||
final List<String> channelIds = page.getIds();
|
||||
|
||||
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
||||
|
||||
final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(),
|
||||
getExtractorLocalization());
|
||||
|
||||
final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
|
||||
.getObject(0)
|
||||
.getObject("appendContinuationItemsAction");
|
||||
|
||||
final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation
|
||||
.getArray("continuationItems"), channelIds);
|
||||
|
||||
return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelIds));
|
||||
}
|
||||
|
||||
@Nullable
|
||||
private Page getNextPageFrom(final JsonObject continuations,
|
||||
final List<String> channelIds)
|
||||
throws IOException, ExtractionException {
|
||||
if (isNullOrEmpty(continuations)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
|
||||
final String continuation = continuationEndpoint.getObject("continuationCommand")
|
||||
.getString("token");
|
||||
|
||||
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
|
||||
getExtractorContentCountry())
|
||||
.value("continuation", continuation)
|
||||
.done())
|
||||
.getBytes(StandardCharsets.UTF_8);
|
||||
|
||||
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
|
||||
+ DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect streams from an array of items
|
||||
*
|
||||
* @param collector the collector where videos will be committed
|
||||
* @param videos the array to get videos from
|
||||
* @param channelIds the ids of the channel, which are its name and its URL
|
||||
* @return the continuation object
|
||||
*/
|
||||
private JsonObject collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
|
||||
@Nonnull final JsonArray videos,
|
||||
@Nonnull final List<String> channelIds) {
|
||||
collector.reset();
|
||||
|
||||
final String uploaderName = channelIds.get(0);
|
||||
final String uploaderUrl = channelIds.get(1);
|
||||
final TimeAgoParser timeAgoParser = getTimeAgoParser();
|
||||
|
||||
JsonObject continuation = null;
|
||||
|
||||
for (final Object object : videos) {
|
||||
final JsonObject video = (JsonObject) object;
|
||||
if (video.has("gridVideoRenderer")) {
|
||||
collector.commit(new YoutubeStreamInfoItemExtractor(
|
||||
video.getObject("gridVideoRenderer"), timeAgoParser) {
|
||||
@Override
|
||||
public String getUploaderName() {
|
||||
return uploaderName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUploaderUrl() {
|
||||
return uploaderUrl;
|
||||
}
|
||||
});
|
||||
} else if (video.has("richItemRenderer")) {
|
||||
collector.commit(new YoutubeStreamInfoItemExtractor(
|
||||
video.getObject("richItemRenderer")
|
||||
.getObject("content").getObject("videoRenderer"), timeAgoParser) {
|
||||
@Override
|
||||
public String getUploaderName() {
|
||||
return uploaderName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUploaderUrl() {
|
||||
return uploaderUrl;
|
||||
}
|
||||
});
|
||||
|
||||
} else if (video.has("continuationItemRenderer")) {
|
||||
continuation = video.getObject("continuationItemRenderer");
|
||||
}
|
||||
}
|
||||
|
||||
return continuation;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
private JsonObject getVideoTab() throws ParsingException {
|
||||
if (videoTab != null) {
|
||||
return videoTab;
|
||||
}
|
||||
|
||||
final JsonArray tabs = initialData.getObject("contents")
|
||||
@Nonnull
|
||||
private List<ListLinkHandler> getTabsForNonAgeRestrictedChannels() throws ParsingException {
|
||||
final JsonArray responseTabs = jsonResponse.getObject("contents")
|
||||
.getObject("twoColumnBrowseResultsRenderer")
|
||||
.getArray("tabs");
|
||||
|
||||
final JsonObject foundVideoTab = tabs.stream()
|
||||
.filter(Objects::nonNull)
|
||||
final List<ListLinkHandler> tabs = new ArrayList<>();
|
||||
final Consumer<String> addNonVideosTab = tabName -> {
|
||||
try {
|
||||
tabs.add(YoutubeChannelTabLinkHandlerFactory.getInstance().fromQuery(
|
||||
channelId, List.of(tabName), ""));
|
||||
} catch (final ParsingException ignored) {
|
||||
// Do not add the tab if we couldn't create the LinkHandler
|
||||
}
|
||||
};
|
||||
|
||||
final String name = getName();
|
||||
final String url = getUrl();
|
||||
final String id = getId();
|
||||
|
||||
responseTabs.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.filter(tab -> tab.has("tabRenderer")
|
||||
&& tab.getObject("tabRenderer")
|
||||
.getString("title", "")
|
||||
.equals("Videos"))
|
||||
.findFirst()
|
||||
.filter(tab -> tab.has("tabRenderer"))
|
||||
.map(tab -> tab.getObject("tabRenderer"))
|
||||
.orElseThrow(
|
||||
() -> new ContentNotSupportedException("This channel has no Videos tab"));
|
||||
.forEach(tabRenderer -> {
|
||||
final String tabUrl = tabRenderer.getObject("endpoint")
|
||||
.getObject("commandMetadata")
|
||||
.getObject("webCommandMetadata")
|
||||
.getString("url");
|
||||
if (tabUrl != null) {
|
||||
final String[] urlParts = tabUrl.split("/");
|
||||
if (urlParts.length == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
final String messageRendererText = getTextFromObject(
|
||||
foundVideoTab.getObject("content")
|
||||
.getObject("sectionListRenderer")
|
||||
.getArray("contents")
|
||||
.getObject(0)
|
||||
.getObject("itemSectionRenderer")
|
||||
.getArray("contents")
|
||||
.getObject(0)
|
||||
.getObject("messageRenderer")
|
||||
.getObject("text"));
|
||||
if (messageRendererText != null
|
||||
&& messageRendererText.equals("This channel has no videos.")) {
|
||||
return null;
|
||||
final String urlSuffix = urlParts[urlParts.length - 1];
|
||||
|
||||
switch (urlSuffix) {
|
||||
case "videos":
|
||||
// Since the Videos tab has already its contents fetched, make
|
||||
// sure it is in the first position
|
||||
// YoutubeChannelTabExtractor still supports fetching this tab
|
||||
tabs.add(0, new ReadyChannelTabListLinkHandler(
|
||||
tabUrl,
|
||||
channelId,
|
||||
ChannelTabs.VIDEOS,
|
||||
(service, linkHandler) -> new VideosTabExtractor(
|
||||
service, linkHandler, tabRenderer, name, id, url)));
|
||||
|
||||
break;
|
||||
case "shorts":
|
||||
addNonVideosTab.accept(ChannelTabs.SHORTS);
|
||||
break;
|
||||
case "streams":
|
||||
addNonVideosTab.accept(ChannelTabs.LIVESTREAMS);
|
||||
break;
|
||||
case "playlists":
|
||||
addNonVideosTab.accept(ChannelTabs.PLAYLISTS);
|
||||
break;
|
||||
case "channels":
|
||||
addNonVideosTab.accept(ChannelTabs.CHANNELS);
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return Collections.unmodifiableList(tabs);
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private List<ListLinkHandler> getTabsForAgeRestrictedChannels() throws ParsingException {
|
||||
// As we don't have access to the channel tabs list, consider that the channel has videos,
|
||||
// shorts and livestreams, the data only accessible without login on YouTube's desktop
|
||||
// client using uploads system playlists
|
||||
// The playlists channel tab is still available on YouTube Music, but this is not
|
||||
// implemented in the extractor
|
||||
|
||||
final List<ListLinkHandler> tabs = new ArrayList<>();
|
||||
final String channelUrl = getUrl();
|
||||
|
||||
final Consumer<String> addTab = tabName ->
|
||||
tabs.add(new ReadyChannelTabListLinkHandler(channelUrl + "/" + tabName,
|
||||
channelId, tabName, YoutubeChannelTabPlaylistExtractor::new));
|
||||
|
||||
addTab.accept(ChannelTabs.VIDEOS);
|
||||
addTab.accept(ChannelTabs.SHORTS);
|
||||
addTab.accept(ChannelTabs.LIVESTREAMS);
|
||||
return Collections.unmodifiableList(tabs);
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public List<String> getTags() throws ParsingException {
|
||||
if (channelAgeGateRenderer != null) {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
videoTab = foundVideoTab;
|
||||
return foundVideoTab;
|
||||
return jsonResponse.getObject("microformat")
|
||||
.getObject("microformatDataRenderer")
|
||||
.getArray("tags")
|
||||
.stream()
|
||||
.filter(String.class::isInstance)
|
||||
.map(String.class::cast)
|
||||
.collect(Collectors.toUnmodifiableList());
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,486 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube.extractors;
|
||||
|
||||
import com.grack.nanojson.JsonArray;
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonWriter;
|
||||
import org.schabi.newpipe.extractor.InfoItem;
|
||||
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.Page;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
|
||||
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
|
||||
import org.schabi.newpipe.extractor.downloader.Downloader;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
|
||||
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
|
||||
import org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.getChannelResponse;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.resolveChannelId;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
/**
|
||||
* A {@link ChannelTabExtractor} implementation for the YouTube service.
|
||||
*
|
||||
* <p>
|
||||
* It currently supports {@code Videos}, {@code Shorts}, {@code Live}, {@code Playlists} and
|
||||
* {@code Channels} tabs.
|
||||
* </p>
|
||||
*/
|
||||
public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
|
||||
|
||||
/**
|
||||
* Whether the visitor data extracted from the initial channel response is required to be used
|
||||
* for continuations.
|
||||
*
|
||||
* <p>
|
||||
* A valid {@code visitorData} is required to get continuations of shorts in channels.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* It should be not used when it is not needed, in order to reduce YouTube's tracking.
|
||||
* </p>
|
||||
*/
|
||||
private final boolean useVisitorData;
|
||||
private JsonObject jsonResponse;
|
||||
private String channelId;
|
||||
@Nullable
|
||||
private String visitorData;
|
||||
|
||||
public YoutubeChannelTabExtractor(final StreamingService service,
|
||||
final ListLinkHandler linkHandler) {
|
||||
super(service, linkHandler);
|
||||
useVisitorData = getName().equals(ChannelTabs.SHORTS);
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private String getChannelTabsParameters() throws ParsingException {
|
||||
final String name = getName();
|
||||
switch (name) {
|
||||
case ChannelTabs.VIDEOS:
|
||||
return "EgZ2aWRlb3PyBgQKAjoA";
|
||||
case ChannelTabs.SHORTS:
|
||||
return "EgZzaG9ydHPyBgUKA5oBAA%3D%3D";
|
||||
case ChannelTabs.LIVESTREAMS:
|
||||
return "EgdzdHJlYW1z8gYECgJ6AA%3D%3D";
|
||||
case ChannelTabs.PLAYLISTS:
|
||||
return "EglwbGF5bGlzdHPyBgQKAkIA";
|
||||
case ChannelTabs.CHANNELS:
|
||||
return "EghjaGFubmVsc_IGBAoCUgA%3D";
|
||||
}
|
||||
throw new ParsingException("Unsupported channel tab: " + name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
|
||||
ExtractionException {
|
||||
channelId = resolveChannelId(super.getId());
|
||||
|
||||
final String params = getChannelTabsParameters();
|
||||
|
||||
final YoutubeChannelHelper.ChannelResponseData data = getChannelResponse(channelId,
|
||||
params, getExtractorLocalization(), getExtractorContentCountry());
|
||||
|
||||
jsonResponse = data.jsonResponse;
|
||||
channelId = data.channelId;
|
||||
if (useVisitorData) {
|
||||
visitorData = jsonResponse.getObject("responseContext").getString("visitorData");
|
||||
}
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getUrl() throws ParsingException {
|
||||
try {
|
||||
return YoutubeChannelTabLinkHandlerFactory.getInstance()
|
||||
.getUrl("channel/" + getId(), List.of(getName()), "");
|
||||
} catch (final ParsingException e) {
|
||||
return super.getUrl();
|
||||
}
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getId() throws ParsingException {
|
||||
final String id = jsonResponse.getObject("header")
|
||||
.getObject("c4TabbedHeaderRenderer")
|
||||
.getString("channelId", "");
|
||||
|
||||
if (!id.isEmpty()) {
|
||||
return id;
|
||||
}
|
||||
|
||||
final Optional<String> carouselHeaderId = jsonResponse.getObject("header")
|
||||
.getObject("carouselHeaderRenderer")
|
||||
.getArray("contents")
|
||||
.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.filter(item -> item.has("topicChannelDetailsRenderer"))
|
||||
.findFirst()
|
||||
.flatMap(item ->
|
||||
Optional.ofNullable(item.getObject("topicChannelDetailsRenderer")
|
||||
.getObject("navigationEndpoint")
|
||||
.getObject("browseEndpoint")
|
||||
.getString("browseId")));
|
||||
if (carouselHeaderId.isPresent()) {
|
||||
return carouselHeaderId.get();
|
||||
}
|
||||
|
||||
if (!isNullOrEmpty(channelId)) {
|
||||
return channelId;
|
||||
} else {
|
||||
throw new ParsingException("Could not get channel ID");
|
||||
}
|
||||
}
|
||||
|
||||
protected String getChannelName() {
|
||||
final String metadataName = jsonResponse.getObject("metadata")
|
||||
.getObject("channelMetadataRenderer")
|
||||
.getString("title");
|
||||
if (!isNullOrEmpty(metadataName)) {
|
||||
return metadataName;
|
||||
}
|
||||
|
||||
return YoutubeChannelHelper.getChannelHeader(jsonResponse)
|
||||
.map(header -> {
|
||||
final Object title = header.json.get("title");
|
||||
if (title instanceof String) {
|
||||
return (String) title;
|
||||
} else if (title instanceof JsonObject) {
|
||||
final String headerName = getTextFromObject((JsonObject) title);
|
||||
if (!isNullOrEmpty(headerName)) {
|
||||
return headerName;
|
||||
}
|
||||
}
|
||||
return "";
|
||||
})
|
||||
.orElse("");
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
|
||||
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
|
||||
|
||||
JsonArray items = new JsonArray();
|
||||
final Optional<JsonObject> tab = getTabData();
|
||||
|
||||
if (tab.isPresent()) {
|
||||
final JsonObject tabContent = tab.get().getObject("content");
|
||||
|
||||
items = tabContent.getObject("sectionListRenderer")
|
||||
.getArray("contents")
|
||||
.getObject(0)
|
||||
.getObject("itemSectionRenderer")
|
||||
.getArray("contents")
|
||||
.getObject(0)
|
||||
.getObject("gridRenderer")
|
||||
.getArray("items");
|
||||
|
||||
if (items.isEmpty()) {
|
||||
items = tabContent.getObject("richGridRenderer")
|
||||
.getArray("contents");
|
||||
|
||||
if (items.isEmpty()) {
|
||||
items = tabContent.getObject("sectionListRenderer")
|
||||
.getArray("contents");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If a channel tab is fetched, the next page requires channel ID and name, as channel
|
||||
// streams don't have their channel specified.
|
||||
// We also need to set the visitor data here when it should be enabled, as it is required
|
||||
// to get continuations on some channel tabs, and we need a way to pass it between pages
|
||||
final List<String> channelIds = useVisitorData && !isNullOrEmpty(visitorData)
|
||||
? List.of(getChannelName(), getUrl(), visitorData)
|
||||
: List.of(getChannelName(), getUrl());
|
||||
|
||||
final JsonObject continuation = collectItemsFrom(collector, items, channelIds)
|
||||
.orElse(null);
|
||||
|
||||
final Page nextPage = getNextPageFrom(continuation, channelIds);
|
||||
|
||||
return new InfoItemsPage<>(collector, nextPage);
|
||||
}
|
||||
|
||||
@Override
|
||||
public InfoItemsPage<InfoItem> getPage(final Page page)
|
||||
throws IOException, ExtractionException {
|
||||
if (page == null || isNullOrEmpty(page.getUrl())) {
|
||||
throw new IllegalArgumentException("Page doesn't contain an URL");
|
||||
}
|
||||
|
||||
final List<String> channelIds = page.getIds();
|
||||
|
||||
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
|
||||
|
||||
final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(),
|
||||
getExtractorLocalization());
|
||||
|
||||
final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
|
||||
.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.filter(jsonObject -> jsonObject.has("appendContinuationItemsAction"))
|
||||
.map(jsonObject -> jsonObject.getObject("appendContinuationItemsAction"))
|
||||
.findFirst()
|
||||
.orElse(new JsonObject());
|
||||
|
||||
final JsonObject continuation = collectItemsFrom(collector,
|
||||
sectionListContinuation.getArray("continuationItems"), channelIds)
|
||||
.orElse(null);
|
||||
|
||||
return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelIds));
|
||||
}
|
||||
|
||||
Optional<JsonObject> getTabData() {
|
||||
final String urlSuffix = YoutubeChannelTabLinkHandlerFactory.getUrlSuffix(getName());
|
||||
|
||||
return jsonResponse.getObject("contents")
|
||||
.getObject("twoColumnBrowseResultsRenderer")
|
||||
.getArray("tabs")
|
||||
.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.filter(tab -> tab.has("tabRenderer"))
|
||||
.map(tab -> tab.getObject("tabRenderer"))
|
||||
.filter(tabRenderer -> tabRenderer.getObject("endpoint")
|
||||
.getObject("commandMetadata").getObject("webCommandMetadata")
|
||||
.getString("url", "").endsWith(urlSuffix))
|
||||
.findFirst()
|
||||
// Check if tab has no content
|
||||
.filter(tabRenderer -> {
|
||||
final JsonArray tabContents = tabRenderer.getObject("content")
|
||||
.getObject("sectionListRenderer")
|
||||
.getArray("contents")
|
||||
.getObject(0)
|
||||
.getObject("itemSectionRenderer")
|
||||
.getArray("contents");
|
||||
return tabContents.size() != 1
|
||||
|| !tabContents.getObject(0).has("messageRenderer");
|
||||
});
|
||||
}
|
||||
|
||||
private Optional<JsonObject> collectItemsFrom(@Nonnull final MultiInfoItemsCollector collector,
|
||||
@Nonnull final JsonArray items,
|
||||
@Nonnull final List<String> channelIds) {
|
||||
return items.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.map(item -> collectItem(collector, item, channelIds))
|
||||
.reduce(Optional.empty(), (c1, c2) -> c1.or(() -> c2));
|
||||
}
|
||||
|
||||
private Optional<JsonObject> collectItem(@Nonnull final MultiInfoItemsCollector collector,
|
||||
@Nonnull final JsonObject item,
|
||||
@Nonnull final List<String> channelIds) {
|
||||
final TimeAgoParser timeAgoParser = getTimeAgoParser();
|
||||
|
||||
if (item.has("richItemRenderer")) {
|
||||
final JsonObject richItem = item.getObject("richItemRenderer")
|
||||
.getObject("content");
|
||||
|
||||
if (richItem.has("videoRenderer")) {
|
||||
getCommitVideoConsumer(collector, timeAgoParser, channelIds).accept(
|
||||
richItem.getObject("videoRenderer"));
|
||||
} else if (richItem.has("reelItemRenderer")) {
|
||||
getCommitReelItemConsumer(collector, timeAgoParser, channelIds).accept(
|
||||
richItem.getObject("reelItemRenderer"));
|
||||
} else if (richItem.has("playlistRenderer")) {
|
||||
getCommitPlaylistConsumer(collector, channelIds).accept(
|
||||
item.getObject("playlistRenderer"));
|
||||
}
|
||||
} else if (item.has("gridVideoRenderer")) {
|
||||
getCommitVideoConsumer(collector, timeAgoParser, channelIds).accept(
|
||||
item.getObject("gridVideoRenderer"));
|
||||
} else if (item.has("gridPlaylistRenderer")) {
|
||||
getCommitPlaylistConsumer(collector, channelIds).accept(
|
||||
item.getObject("gridPlaylistRenderer"));
|
||||
} else if (item.has("gridChannelRenderer")) {
|
||||
collector.commit(new YoutubeChannelInfoItemExtractor(
|
||||
item.getObject("gridChannelRenderer")));
|
||||
} else if (item.has("shelfRenderer")) {
|
||||
return collectItem(collector, item.getObject("shelfRenderer")
|
||||
.getObject("content"), channelIds);
|
||||
} else if (item.has("itemSectionRenderer")) {
|
||||
return collectItemsFrom(collector, item.getObject("itemSectionRenderer")
|
||||
.getArray("contents"), channelIds);
|
||||
} else if (item.has("horizontalListRenderer")) {
|
||||
return collectItemsFrom(collector, item.getObject("horizontalListRenderer")
|
||||
.getArray("items"), channelIds);
|
||||
} else if (item.has("expandedShelfContentsRenderer")) {
|
||||
return collectItemsFrom(collector, item.getObject("expandedShelfContentsRenderer")
|
||||
.getArray("items"), channelIds);
|
||||
} else if (item.has("continuationItemRenderer")) {
|
||||
return Optional.ofNullable(item.getObject("continuationItemRenderer"));
|
||||
}
|
||||
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private Consumer<JsonObject> getCommitVideoConsumer(
|
||||
@Nonnull final MultiInfoItemsCollector collector,
|
||||
@Nonnull final TimeAgoParser timeAgoParser,
|
||||
@Nonnull final List<String> channelIds) {
|
||||
return videoRenderer -> collector.commit(
|
||||
new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser) {
|
||||
@Override
|
||||
public String getUploaderName() throws ParsingException {
|
||||
if (channelIds.size() >= 2) {
|
||||
return channelIds.get(0);
|
||||
}
|
||||
return super.getUploaderName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUploaderUrl() throws ParsingException {
|
||||
if (channelIds.size() >= 2) {
|
||||
return channelIds.get(1);
|
||||
}
|
||||
return super.getUploaderUrl();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private Consumer<JsonObject> getCommitReelItemConsumer(
|
||||
@Nonnull final MultiInfoItemsCollector collector,
|
||||
@Nonnull final TimeAgoParser timeAgoParser,
|
||||
@Nonnull final List<String> channelIds) {
|
||||
return reelItemRenderer -> collector.commit(
|
||||
new YoutubeReelInfoItemExtractor(reelItemRenderer, timeAgoParser) {
|
||||
@Override
|
||||
public String getUploaderName() throws ParsingException {
|
||||
if (channelIds.size() >= 2) {
|
||||
return channelIds.get(0);
|
||||
}
|
||||
return super.getUploaderName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUploaderUrl() throws ParsingException {
|
||||
if (channelIds.size() >= 2) {
|
||||
return channelIds.get(1);
|
||||
}
|
||||
return super.getUploaderUrl();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private Consumer<JsonObject> getCommitPlaylistConsumer(
|
||||
@Nonnull final MultiInfoItemsCollector collector,
|
||||
@Nonnull final List<String> channelIds) {
|
||||
return playlistRenderer -> collector.commit(
|
||||
new YoutubePlaylistInfoItemExtractor(playlistRenderer) {
|
||||
@Override
|
||||
public String getUploaderName() throws ParsingException {
|
||||
if (channelIds.size() >= 2) {
|
||||
return channelIds.get(0);
|
||||
}
|
||||
return super.getUploaderName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUploaderUrl() throws ParsingException {
|
||||
if (channelIds.size() >= 2) {
|
||||
return channelIds.get(1);
|
||||
}
|
||||
return super.getUploaderUrl();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Nullable
|
||||
private Page getNextPageFrom(final JsonObject continuations,
|
||||
final List<String> channelIds) throws IOException,
|
||||
ExtractionException {
|
||||
if (isNullOrEmpty(continuations)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
|
||||
final String continuation = continuationEndpoint.getObject("continuationCommand")
|
||||
.getString("token");
|
||||
|
||||
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
|
||||
getExtractorContentCountry(),
|
||||
useVisitorData && channelIds.size() >= 3 ? channelIds.get(2) : null)
|
||||
.value("continuation", continuation)
|
||||
.done())
|
||||
.getBytes(StandardCharsets.UTF_8);
|
||||
|
||||
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
|
||||
+ DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
|
||||
}
|
||||
|
||||
/**
|
||||
* A {@link YoutubeChannelTabExtractor} for the {@code Videos} tab, if it has been already
|
||||
* fetched.
|
||||
*/
|
||||
public static final class VideosTabExtractor extends YoutubeChannelTabExtractor {
|
||||
private final JsonObject tabRenderer;
|
||||
private final String channelName;
|
||||
private final String channelId;
|
||||
private final String channelUrl;
|
||||
|
||||
VideosTabExtractor(final StreamingService service,
|
||||
final ListLinkHandler linkHandler,
|
||||
final JsonObject tabRenderer,
|
||||
final String channelName,
|
||||
final String channelId,
|
||||
final String channelUrl) {
|
||||
super(service, linkHandler);
|
||||
this.tabRenderer = tabRenderer;
|
||||
this.channelName = channelName;
|
||||
this.channelId = channelId;
|
||||
this.channelUrl = channelUrl;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onFetchPage(@Nonnull final Downloader downloader) {
|
||||
// Nothing to do, the initial data was already fetched and is stored in the link handler
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getId() throws ParsingException {
|
||||
return channelId;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getUrl() throws ParsingException {
|
||||
return channelUrl;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getChannelName() {
|
||||
return channelName;
|
||||
}
|
||||
|
||||
@Override
|
||||
Optional<JsonObject> getTabData() {
|
||||
return Optional.of(tabRenderer);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,191 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube.extractors;
|
||||
|
||||
import org.schabi.newpipe.extractor.InfoItem;
|
||||
import org.schabi.newpipe.extractor.Page;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
|
||||
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
|
||||
import org.schabi.newpipe.extractor.downloader.Downloader;
|
||||
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
/**
|
||||
* A {@link ChannelTabExtractor} for YouTube system playlists using a
|
||||
* {@link YoutubePlaylistExtractor} instance.
|
||||
*
|
||||
* <p>
|
||||
* It is currently used to bypass age-restrictions on channels marked as age-restricted by their
|
||||
* owner(s).
|
||||
* </p>
|
||||
*/
|
||||
public class YoutubeChannelTabPlaylistExtractor extends ChannelTabExtractor {
|
||||
|
||||
private final PlaylistExtractor playlistExtractorInstance;
|
||||
private boolean playlistExisting;
|
||||
|
||||
/**
|
||||
* Construct a {@link YoutubeChannelTabPlaylistExtractor} instance.
|
||||
*
|
||||
* @param service a {@link StreamingService} implementation, which must be the YouTube
|
||||
* one
|
||||
* @param linkHandler a {@link ListLinkHandler} which must have a valid channel ID (starting
|
||||
* with `UC`) and one of the given and supported content filters:
|
||||
* {@link ChannelTabs#VIDEOS}, {@link ChannelTabs#SHORTS},
|
||||
* {@link ChannelTabs#LIVESTREAMS}
|
||||
* @throws IllegalArgumentException if the given {@link ListLinkHandler} doesn't have the
|
||||
* required arguments
|
||||
* @throws SystemPlaylistUrlCreationException if the system playlist URL could not be created,
|
||||
* which should never happen
|
||||
*/
|
||||
public YoutubeChannelTabPlaylistExtractor(@Nonnull final StreamingService service,
|
||||
@Nonnull final ListLinkHandler linkHandler)
|
||||
throws IllegalArgumentException, SystemPlaylistUrlCreationException {
|
||||
super(service, linkHandler);
|
||||
final ListLinkHandler playlistLinkHandler = getPlaylistLinkHandler(linkHandler);
|
||||
this.playlistExtractorInstance = new YoutubePlaylistExtractor(service, playlistLinkHandler);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onFetchPage(@Nonnull final Downloader downloader)
|
||||
throws IOException, ExtractionException {
|
||||
try {
|
||||
playlistExtractorInstance.onFetchPage(downloader);
|
||||
if (!playlistExisting) {
|
||||
playlistExisting = true;
|
||||
}
|
||||
} catch (final ContentNotAvailableException e) {
|
||||
// If a channel has no content of the type requested, the corresponding system playlist
|
||||
// won't exist, so a ContentNotAvailableException would be thrown
|
||||
// Ignore such issues in this case
|
||||
}
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
|
||||
if (!playlistExisting) {
|
||||
return InfoItemsPage.emptyPage();
|
||||
}
|
||||
|
||||
final InfoItemsPage<StreamInfoItem> playlistInitialPage =
|
||||
playlistExtractorInstance.getInitialPage();
|
||||
|
||||
// We can't provide the playlist page as it is due to a type conflict, we need to wrap the
|
||||
// page items and provide a new InfoItemsPage
|
||||
final List<InfoItem> infoItems = new ArrayList<>(playlistInitialPage.getItems());
|
||||
return new InfoItemsPage<>(infoItems, playlistInitialPage.getNextPage(),
|
||||
playlistInitialPage.getErrors());
|
||||
}
|
||||
|
||||
@Override
|
||||
public InfoItemsPage<InfoItem> getPage(final Page page)
|
||||
throws IOException, ExtractionException {
|
||||
if (!playlistExisting) {
|
||||
return InfoItemsPage.emptyPage();
|
||||
}
|
||||
|
||||
final InfoItemsPage<StreamInfoItem> playlistPage = playlistExtractorInstance.getPage(page);
|
||||
|
||||
// We can't provide the playlist page as it is due to a type conflict, we need to wrap the
|
||||
// page items and provide a new InfoItemsPage
|
||||
final List<InfoItem> infoItems = new ArrayList<>(playlistPage.getItems());
|
||||
return new InfoItemsPage<>(infoItems, playlistPage.getNextPage(),
|
||||
playlistPage.getErrors());
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a playlist {@link ListLinkHandler} from a channel tab one.
|
||||
*
|
||||
* <p>
|
||||
* This method converts a channel ID without its {@code UC} prefix into a YouTube system
|
||||
* playlist, depending on the first content filter provided in the given
|
||||
* {@link ListLinkHandler}.
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* The first content filter must be a channel tabs one among the
|
||||
* {@link ChannelTabs#VIDEOS videos}, {@link ChannelTabs#SHORTS shorts} and
|
||||
* {@link ChannelTabs#LIVESTREAMS} ones, which would be converted respectively into playlists
|
||||
* with the ID {@code UULF}, {@code UUSH} and {@code UULV} on which the channel ID without the
|
||||
* {@code UC} part is appended.
|
||||
* </p>
|
||||
*
|
||||
* @param originalLinkHandler the original {@link ListLinkHandler} with which a
|
||||
* {@link YoutubeChannelTabPlaylistExtractor} instance is being constructed
|
||||
*
|
||||
* @return a {@link ListLinkHandler} to use for the {@link YoutubePlaylistExtractor} instance
|
||||
* needed to extract channel tabs data from a system playlist
|
||||
* @throws IllegalArgumentException if the original {@link ListLinkHandler} does not meet the
|
||||
* required criteria above
|
||||
* @throws SystemPlaylistUrlCreationException if the system playlist URL could not be created,
|
||||
* which should never happen
|
||||
*/
|
||||
@Nonnull
|
||||
private ListLinkHandler getPlaylistLinkHandler(
|
||||
@Nonnull final ListLinkHandler originalLinkHandler)
|
||||
throws IllegalArgumentException, SystemPlaylistUrlCreationException {
|
||||
final List<String> contentFilters = originalLinkHandler.getContentFilters();
|
||||
if (contentFilters.isEmpty()) {
|
||||
throw new IllegalArgumentException("A content filter is required");
|
||||
}
|
||||
|
||||
final String channelId = originalLinkHandler.getId();
|
||||
if (isNullOrEmpty(channelId) || !channelId.startsWith("UC")) {
|
||||
throw new IllegalArgumentException("Invalid channel ID");
|
||||
}
|
||||
|
||||
final String channelIdWithoutUc = channelId.substring(2);
|
||||
|
||||
final String playlistId;
|
||||
switch (contentFilters.get(0)) {
|
||||
case ChannelTabs.VIDEOS:
|
||||
playlistId = "UULF" + channelIdWithoutUc;
|
||||
break;
|
||||
case ChannelTabs.SHORTS:
|
||||
playlistId = "UUSH" + channelIdWithoutUc;
|
||||
break;
|
||||
case ChannelTabs.LIVESTREAMS:
|
||||
playlistId = "UULV" + channelIdWithoutUc;
|
||||
break;
|
||||
default:
|
||||
throw new IllegalArgumentException(
|
||||
"Only Videos, Shorts and Livestreams tabs can extracted as playlists");
|
||||
}
|
||||
|
||||
try {
|
||||
final String newUrl = YoutubePlaylistLinkHandlerFactory.getInstance()
|
||||
.getUrl(playlistId);
|
||||
return new ListLinkHandler(newUrl, newUrl, playlistId, List.of(), "");
|
||||
} catch (final ParsingException e) {
|
||||
// This should be not reachable, as the given playlist ID should be valid and
|
||||
// YoutubePlaylistLinkHandlerFactory doesn't throw any exception
|
||||
throw new SystemPlaylistUrlCreationException(
|
||||
"Could not create a YouTube playlist from a valid playlist ID", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Exception thrown when a YouTube system playlist URL could not be created.
|
||||
*
|
||||
* <p>
|
||||
* This exception should be never thrown, as given playlist IDs should be always valid.
|
||||
* </p>
|
||||
*/
|
||||
public static final class SystemPlaylistUrlCreationException extends RuntimeException {
|
||||
SystemPlaylistUrlCreationException(final String message, final Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,5 +1,6 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube.extractors;
|
||||
|
||||
import com.grack.nanojson.JsonArray;
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
|
||||
@ -22,10 +23,15 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
|
||||
@Override
|
||||
public String getThumbnailUrl() throws ParsingException {
|
||||
try {
|
||||
final String url = playlistInfoItem.getArray("thumbnails").getObject(0)
|
||||
.getArray("thumbnails").getObject(0).getString("url");
|
||||
JsonArray thumbnails = playlistInfoItem.getArray("thumbnails")
|
||||
.getObject(0)
|
||||
.getArray("thumbnails");
|
||||
if (thumbnails.isEmpty()) {
|
||||
thumbnails = playlistInfoItem.getObject("thumbnail")
|
||||
.getArray("thumbnails");
|
||||
}
|
||||
|
||||
return fixThumbnailUrl(url);
|
||||
return fixThumbnailUrl(thumbnails.getObject(0).getString("url"));
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get thumbnail url", e);
|
||||
}
|
||||
@ -79,9 +85,21 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
|
||||
|
||||
@Override
|
||||
public long getStreamCount() throws ParsingException {
|
||||
String videoCountText = playlistInfoItem.getString("videoCount");
|
||||
if (videoCountText == null) {
|
||||
videoCountText = getTextFromObject(playlistInfoItem.getObject("videoCountText"));
|
||||
}
|
||||
|
||||
if (videoCountText == null) {
|
||||
videoCountText = getTextFromObject(playlistInfoItem.getObject("videoCountShortText"));
|
||||
}
|
||||
|
||||
if (videoCountText == null) {
|
||||
throw new ParsingException("Could not get stream count");
|
||||
}
|
||||
|
||||
try {
|
||||
return Long.parseLong(Utils.removeNonDigitCharacters(
|
||||
playlistInfoItem.getString("videoCount")));
|
||||
return Long.parseLong(Utils.removeNonDigitCharacters(videoCountText));
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get stream count", e);
|
||||
}
|
||||
|
||||
@ -0,0 +1,147 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube.extractors;
|
||||
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.localization.DateWrapper;
|
||||
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getThumbnailUrlFromInfoItem;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
/**
|
||||
* A {@link StreamInfoItemExtractor} for YouTube's {@code reelItemRenderers}.
|
||||
*
|
||||
* <p>
|
||||
* {@code reelItemRenderers} are returned on YouTube for their short-form contents on almost every
|
||||
* place and every major client. They provide a limited amount of information and do not provide
|
||||
* the exact view count, any uploader info (name, URL, avatar, verified status) and the upload date.
|
||||
* </p>
|
||||
*/
|
||||
public class YoutubeReelInfoItemExtractor implements StreamInfoItemExtractor {
|
||||
|
||||
@Nonnull
|
||||
private final JsonObject reelInfo;
|
||||
@Nullable
|
||||
private final TimeAgoParser timeAgoParser;
|
||||
|
||||
public YoutubeReelInfoItemExtractor(@Nonnull final JsonObject reelInfo,
|
||||
@Nullable final TimeAgoParser timeAgoParser) {
|
||||
this.reelInfo = reelInfo;
|
||||
this.timeAgoParser = timeAgoParser;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() throws ParsingException {
|
||||
return getTextFromObject(reelInfo.getObject("headline"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUrl() throws ParsingException {
|
||||
try {
|
||||
final String videoId = reelInfo.getString("videoId");
|
||||
return YoutubeStreamLinkHandlerFactory.getInstance().getUrl(videoId);
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get URL", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getThumbnailUrl() throws ParsingException {
|
||||
return getThumbnailUrlFromInfoItem(reelInfo);
|
||||
}
|
||||
|
||||
@Override
|
||||
public StreamType getStreamType() throws ParsingException {
|
||||
return StreamType.VIDEO_STREAM;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getDuration() throws ParsingException {
|
||||
// Duration of reelItems is only provided in the accessibility data
|
||||
// example: "VIDEO TITLE - 49 seconds - play video"
|
||||
// "VIDEO TITLE - 1 minute, 1 second - play video"
|
||||
final String accessibilityLabel = reelInfo.getObject("accessibility")
|
||||
.getObject("accessibilityData").getString("label");
|
||||
if (accessibilityLabel == null || timeAgoParser == null) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// This approach may be language dependent
|
||||
final String[] labelParts = accessibilityLabel.split(" [\u2013-] ");
|
||||
|
||||
if (labelParts.length > 2) {
|
||||
final String textualDuration = labelParts[labelParts.length - 2];
|
||||
return timeAgoParser.parseDuration(textualDuration);
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getViewCount() throws ParsingException {
|
||||
final String viewCountText = getTextFromObject(reelInfo.getObject("viewCountText"));
|
||||
if (!isNullOrEmpty(viewCountText)) {
|
||||
// This approach is language dependent
|
||||
if (viewCountText.toLowerCase().contains("no views")) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return Utils.mixedNumberWordToLong(viewCountText);
|
||||
}
|
||||
|
||||
throw new ParsingException("Could not get short view count");
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isShortFormContent() throws ParsingException {
|
||||
return true;
|
||||
}
|
||||
|
||||
// All the following properties cannot be obtained from reelItemRenderers
|
||||
|
||||
@Override
|
||||
public boolean isAd() throws ParsingException {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUploaderName() throws ParsingException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUploaderUrl() throws ParsingException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public String getUploaderAvatarUrl() throws ParsingException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isUploaderVerified() throws ParsingException {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public String getTextualUploadDate() throws ParsingException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public DateWrapper getUploadDate() throws ParsingException {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@ -58,7 +58,8 @@ public final class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFacto
|
||||
@Override
|
||||
public String getUrl(final String id,
|
||||
final List<String> contentFilters,
|
||||
final String searchFilter) {
|
||||
final String searchFilter)
|
||||
throws ParsingException, UnsupportedOperationException {
|
||||
return "https://www.youtube.com/" + id;
|
||||
}
|
||||
|
||||
@ -84,7 +85,7 @@ public final class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFacto
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getId(final String url) throws ParsingException {
|
||||
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
|
||||
try {
|
||||
final URL urlObj = Utils.stringToURL(url);
|
||||
String path = urlObj.getPath();
|
||||
|
||||
@ -0,0 +1,73 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
|
||||
|
||||
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.UnsupportedTabException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import java.util.List;
|
||||
|
||||
public final class YoutubeChannelTabLinkHandlerFactory extends ListLinkHandlerFactory {
|
||||
private static final YoutubeChannelTabLinkHandlerFactory INSTANCE =
|
||||
new YoutubeChannelTabLinkHandlerFactory();
|
||||
|
||||
private YoutubeChannelTabLinkHandlerFactory() {
|
||||
}
|
||||
|
||||
public static YoutubeChannelTabLinkHandlerFactory getInstance() {
|
||||
return INSTANCE;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
public static String getUrlSuffix(@Nonnull final String tab)
|
||||
throws UnsupportedTabException {
|
||||
switch (tab) {
|
||||
case ChannelTabs.VIDEOS:
|
||||
return "/videos";
|
||||
case ChannelTabs.SHORTS:
|
||||
return "/shorts";
|
||||
case ChannelTabs.LIVESTREAMS:
|
||||
return "/streams";
|
||||
case ChannelTabs.PLAYLISTS:
|
||||
return "/playlists";
|
||||
case ChannelTabs.CHANNELS:
|
||||
return "/channels";
|
||||
}
|
||||
throw new UnsupportedTabException(tab);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUrl(final String id,
|
||||
final List<String> contentFilter,
|
||||
final String sortFilter)
|
||||
throws ParsingException, UnsupportedOperationException {
|
||||
return "https://www.youtube.com/" + id + getUrlSuffix(contentFilter.get(0));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
|
||||
return YoutubeChannelLinkHandlerFactory.getInstance().getId(url);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean onAcceptUrl(final String url) throws ParsingException {
|
||||
try {
|
||||
getId(url);
|
||||
} catch (final ParsingException e) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String[] getAvailableContentFilter() {
|
||||
return new String[] {
|
||||
ChannelTabs.VIDEOS,
|
||||
ChannelTabs.SHORTS,
|
||||
ChannelTabs.LIVESTREAMS,
|
||||
ChannelTabs.PLAYLISTS,
|
||||
ChannelTabs.CHANNELS
|
||||
};
|
||||
}
|
||||
}
|
||||
@ -19,13 +19,14 @@ public final class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFact
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUrl(final String id) {
|
||||
public String getUrl(final String id) throws ParsingException, UnsupportedOperationException {
|
||||
return "https://www.youtube.com/watch?v=" + id;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getId(final String urlString) throws ParsingException, IllegalArgumentException {
|
||||
// we need the same id, avoids duplicate code
|
||||
public String getId(final String urlString)
|
||||
throws ParsingException, UnsupportedOperationException {
|
||||
// We need the same id, avoids duplicate code
|
||||
return YoutubeStreamLinkHandlerFactory.getInstance().getId(urlString);
|
||||
}
|
||||
|
||||
@ -44,7 +45,8 @@ public final class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFact
|
||||
@Override
|
||||
public String getUrl(final String id,
|
||||
final List<String> contentFilter,
|
||||
final String sortFilter) throws ParsingException {
|
||||
final String sortFilter)
|
||||
throws ParsingException, UnsupportedOperationException {
|
||||
return getUrl(id);
|
||||
}
|
||||
}
|
||||
|
||||
@ -26,12 +26,13 @@ public final class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFact
|
||||
|
||||
@Override
|
||||
public String getUrl(final String id, final List<String> contentFilters,
|
||||
final String sortFilter) {
|
||||
final String sortFilter)
|
||||
throws ParsingException, UnsupportedOperationException {
|
||||
return "https://www.youtube.com/playlist?list=" + id;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getId(final String url) throws ParsingException {
|
||||
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
|
||||
try {
|
||||
final URL urlObj = Utils.stringToURL(url);
|
||||
|
||||
|
||||
@ -13,6 +13,9 @@ import javax.annotation.Nonnull;
|
||||
|
||||
public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
|
||||
|
||||
private static final YoutubeSearchQueryHandlerFactory INSTANCE =
|
||||
new YoutubeSearchQueryHandlerFactory();
|
||||
|
||||
public static final String ALL = "all";
|
||||
public static final String VIDEOS = "videos";
|
||||
public static final String CHANNELS = "channels";
|
||||
@ -29,20 +32,18 @@ public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFa
|
||||
|
||||
@Nonnull
|
||||
public static YoutubeSearchQueryHandlerFactory getInstance() {
|
||||
return new YoutubeSearchQueryHandlerFactory();
|
||||
return INSTANCE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUrl(final String searchString,
|
||||
@Nonnull final List<String> contentFilters,
|
||||
final String sortFilter) throws ParsingException {
|
||||
final String sortFilter)
|
||||
throws ParsingException, UnsupportedOperationException {
|
||||
try {
|
||||
if (!contentFilters.isEmpty()) {
|
||||
final String contentFilter = contentFilters.get(0);
|
||||
switch (contentFilter) {
|
||||
case ALL:
|
||||
default:
|
||||
break;
|
||||
case VIDEOS:
|
||||
return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQAQ%253D%253D";
|
||||
case CHANNELS:
|
||||
|
||||
@ -79,7 +79,7 @@ public final class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getUrl(final String id) {
|
||||
public String getUrl(final String id) throws ParsingException, UnsupportedOperationException {
|
||||
return "https://www.youtube.com/watch?v=" + id;
|
||||
}
|
||||
|
||||
@ -87,7 +87,7 @@ public final class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getId(final String theUrlString)
|
||||
throws ParsingException, IllegalArgumentException {
|
||||
throws ParsingException, UnsupportedOperationException {
|
||||
String urlString = theUrlString;
|
||||
try {
|
||||
final URI uri = new URI(urlString);
|
||||
|
||||
@ -1,28 +1,29 @@
|
||||
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 12.08.17.
|
||||
*
|
||||
* Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
|
||||
* YoutubeTrendingLinkHandlerFactory.java is part of NewPipe.
|
||||
* YoutubeTrendingLinkHandlerFactory.java is part of NewPipe Extractor.
|
||||
*
|
||||
* NewPipe is free software: you can redistribute it and/or modify
|
||||
* NewPipe Extractor is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* NewPipe is distributed in the hope that it will be useful,
|
||||
* NewPipe Extractor is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||
* along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidiousURL;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL;
|
||||
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
@ -30,16 +31,27 @@ import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.util.List;
|
||||
|
||||
public class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
|
||||
public final class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
|
||||
|
||||
private static final YoutubeTrendingLinkHandlerFactory INSTANCE =
|
||||
new YoutubeTrendingLinkHandlerFactory();
|
||||
|
||||
private YoutubeTrendingLinkHandlerFactory() {
|
||||
}
|
||||
|
||||
public static YoutubeTrendingLinkHandlerFactory getInstance() {
|
||||
return INSTANCE;
|
||||
}
|
||||
|
||||
public String getUrl(final String id,
|
||||
final List<String> contentFilters,
|
||||
final String sortFilter) {
|
||||
final String sortFilter)
|
||||
throws ParsingException, UnsupportedOperationException {
|
||||
return "https://www.youtube.com/feed/trending";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getId(final String url) {
|
||||
public String getId(final String url) throws ParsingException, UnsupportedOperationException {
|
||||
return "Trending";
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user