fix: add #1050 fix to channel tab name extraction

use shared method for channel header extraction
This commit is contained in:
ThetaDev 2023-05-07 21:30:13 +02:00
parent 2adc2caebc
commit e8fab3be9c
3 changed files with 87 additions and 55 deletions

View File

@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.localization.Localization;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.Optional;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -22,6 +23,7 @@ public final class YouTubeChannelHelper {
/** /**
* Take a YouTube channel ID or URL path, resolve it if necessary and return a channel ID. * Take a YouTube channel ID or URL path, resolve it if necessary and return a channel ID.
*
* @param idOrPath YouTube channel ID or URL path * @param idOrPath YouTube channel ID or URL path
* @return YouTube Channel ID * @return YouTube Channel ID
*/ */
@ -38,7 +40,7 @@ public final class YouTubeChannelHelper {
// we couldn't get information about the channel associated with this URL, if there is one. // we couldn't get information about the channel associated with this URL, if there is one.
if (!channelId[0].equals("channel")) { if (!channelId[0].equals("channel")) {
final byte[] body = JsonWriter.string(YoutubeParsingHelper.prepareDesktopJsonBuilder( final byte[] body = JsonWriter.string(YoutubeParsingHelper.prepareDesktopJsonBuilder(
Localization.DEFAULT, ContentCountry.DEFAULT) Localization.DEFAULT, ContentCountry.DEFAULT)
.value("url", "https://www.youtube.com/" + idOrPath) .value("url", "https://www.youtube.com/" + idOrPath)
.done()) .done())
.getBytes(StandardCharsets.UTF_8); .getBytes(StandardCharsets.UTF_8);
@ -96,9 +98,9 @@ public final class YouTubeChannelHelper {
* </ul> * </ul>
* *
* @param channelId YouTube channel ID * @param channelId YouTube channel ID
* @param params Parameters to specify the YouTube channel tab * @param params Parameters to specify the YouTube channel tab
* @param loc YouTube localization * @param loc YouTube localization
* @param country YouTube content country * @param country YouTube content country
* @return Channel response data * @return Channel response data
*/ */
public static ChannelResponseData getChannelResponse(final String channelId, public static ChannelResponseData getChannelResponse(final String channelId,
@ -112,7 +114,7 @@ public final class YouTubeChannelHelper {
int level = 0; int level = 0;
while (level < 3) { while (level < 3) {
final byte[] body = JsonWriter.string(YoutubeParsingHelper.prepareDesktopJsonBuilder( final byte[] body = JsonWriter.string(YoutubeParsingHelper.prepareDesktopJsonBuilder(
loc, country) loc, country)
.value("browseId", id) .value("browseId", id)
.value("params", params) // Equal to videos .value("params", params) // Equal to videos
.done()) .done())
@ -161,6 +163,7 @@ public final class YouTubeChannelHelper {
/** /**
* Assert that a channel JSON response does not contain a 404 error. * Assert that a channel JSON response does not contain a 404 error.
*
* @param jsonResponse channel JSON response * @param jsonResponse channel JSON response
* @throws ContentNotAvailableException if the channel was not found * @throws ContentNotAvailableException if the channel was not found
*/ */
@ -178,4 +181,35 @@ public final class YouTubeChannelHelper {
} }
} }
} }
public static final class ChannelHeader {
public final JsonObject json;
public final boolean isCarouselHeader;
private ChannelHeader(final JsonObject json, final boolean isCarouselHeader) {
this.json = json;
this.isCarouselHeader = isCarouselHeader;
}
}
public static Optional<ChannelHeader> getChannelHeader(final JsonObject initialData) {
final JsonObject h = initialData.getObject("header");
if (h.has("c4TabbedHeaderRenderer")) {
return Optional.of(h.getObject("c4TabbedHeaderRenderer"))
.map(json -> new ChannelHeader(json, false));
} else if (h.has("carouselHeaderRenderer")) {
return h.getObject("carouselHeaderRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(itm -> itm.has("topicChannelDetailsRenderer"))
.findFirst()
.map(itm -> itm.getObject("topicChannelDetailsRenderer"))
.map(json -> new ChannelHeader(json, true));
} else {
return Optional.empty();
}
}
} }

View File

@ -18,6 +18,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabs; import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.services.youtube.YouTubeChannelHelper;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
@ -55,8 +56,7 @@ import javax.annotation.Nonnull;
public class YoutubeChannelExtractor extends ChannelExtractor { public class YoutubeChannelExtractor extends ChannelExtractor {
private JsonObject initialData; private JsonObject initialData;
private Optional<JsonObject> channelHeader; private Optional<YouTubeChannelHelper.ChannelHeader> channelHeader;
private boolean isCarouselHeader = false;
private JsonObject videoTab; private JsonObject videoTab;
/** /**
@ -90,25 +90,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
} }
@Nonnull @Nonnull
private Optional<JsonObject> getChannelHeader() { private Optional<YouTubeChannelHelper.ChannelHeader> getChannelHeader() {
if (channelHeader == null) { if (channelHeader == null) {
final JsonObject h = initialData.getObject("header"); channelHeader = YouTubeChannelHelper.getChannelHeader(initialData);
if (h.has("c4TabbedHeaderRenderer")) {
channelHeader = Optional.of(h.getObject("c4TabbedHeaderRenderer"));
} else if (h.has("carouselHeaderRenderer")) {
isCarouselHeader = true;
channelHeader = h.getObject("carouselHeaderRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(itm -> itm.has("topicChannelDetailsRenderer"))
.findFirst()
.map(itm -> itm.getObject("topicChannelDetailsRenderer"));
} else {
channelHeader = Optional.empty();
}
} }
return channelHeader; return channelHeader;
} }
@ -127,8 +111,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override @Override
public String getId() throws ParsingException { public String getId() throws ParsingException {
return getChannelHeader() return getChannelHeader()
.flatMap(header -> Optional.ofNullable(header.getString("channelId")).or( .flatMap(header -> Optional.ofNullable(header.json.getString("channelId")).or(
() -> Optional.ofNullable(header.getObject("navigationEndpoint") () -> Optional.ofNullable(header.json.getObject("navigationEndpoint")
.getObject("browseEndpoint") .getObject("browseEndpoint")
.getString("browseId")) .getString("browseId"))
)) ))
@ -146,26 +130,24 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return mdName; return mdName;
} }
final Optional<JsonObject> header = getChannelHeader(); return getChannelHeader().flatMap(header -> {
if (header.isPresent()) { final Object title = header.json.get("title");
final Object title = header.get().get("title");
if (title instanceof String) { if (title instanceof String) {
return (String) title; return Optional.of((String) title);
} else if (title instanceof JsonObject) { } else if (title instanceof JsonObject) {
final String headerName = getTextFromObject((JsonObject) title); final String headerName = getTextFromObject((JsonObject) title);
if (!isNullOrEmpty(headerName)) { if (!isNullOrEmpty(headerName)) {
return headerName; return Optional.of(headerName);
} }
} }
} return Optional.empty();
}).orElseThrow(() -> new ParsingException("Could not get channel name"));
throw new ParsingException("Could not get channel name");
} }
@Override @Override
public String getAvatarUrl() throws ParsingException { public String getAvatarUrl() throws ParsingException {
return getChannelHeader().flatMap(header -> Optional.ofNullable( return getChannelHeader().flatMap(header -> Optional.ofNullable(
header.getObject("avatar").getArray("thumbnails") header.json.getObject("avatar").getArray("thumbnails")
.getObject(0).getString("url") .getObject(0).getString("url")
)) ))
.map(YoutubeParsingHelper::fixThumbnailUrl) .map(YoutubeParsingHelper::fixThumbnailUrl)
@ -175,7 +157,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override @Override
public String getBannerUrl() throws ParsingException { public String getBannerUrl() throws ParsingException {
return getChannelHeader().flatMap(header -> Optional.ofNullable( return getChannelHeader().flatMap(header -> Optional.ofNullable(
header.getObject("banner").getArray("thumbnails") header.json.getObject("banner").getArray("thumbnails")
.getObject(0).getString("url") .getObject(0).getString("url")
)) ))
.filter(url -> !url.contains("s.ytimg.com") && !url.contains("default_banner")) .filter(url -> !url.contains("s.ytimg.com") && !url.contains("default_banner"))
@ -194,14 +176,15 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override @Override
public long getSubscriberCount() throws ParsingException { public long getSubscriberCount() throws ParsingException {
final Optional<JsonObject> header = getChannelHeader(); final Optional<YouTubeChannelHelper.ChannelHeader> headerOpt = getChannelHeader();
if (header.isPresent()) { if (headerOpt.isPresent()) {
final JsonObject header = headerOpt.get().json;
JsonObject textObject = null; JsonObject textObject = null;
if (header.get().has("subscriberCountText")) { if (header.has("subscriberCountText")) {
textObject = header.get().getObject("subscriberCountText"); textObject = header.getObject("subscriberCountText");
} else if (header.get().has("subtitle")) { } else if (header.has("subtitle")) {
textObject = header.get().getObject("subtitle"); textObject = header.getObject("subtitle");
} }
if (textObject != null) { if (textObject != null) {
@ -242,17 +225,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override @Override
public boolean isVerified() throws ParsingException { public boolean isVerified() throws ParsingException {
// The CarouselHeaderRenderer does not contain any verification badges. final Optional<YouTubeChannelHelper.ChannelHeader> headerOpt = getChannelHeader();
// Since it is only shown on YT-internal channels or on channels of large organizations if (headerOpt.isPresent()) {
// broadcasting live events, we can assume the channel to be verified. final YouTubeChannelHelper.ChannelHeader header = headerOpt.get();
if (isCarouselHeader) {
return true;
}
return getChannelHeader() // The CarouselHeaderRenderer does not contain any verification badges.
.map(header -> header.getArray("badges")) // Since it is only shown on YT-internal channels or on channels of large organizations
.map(YoutubeParsingHelper::isVerified) // broadcasting live events, we can assume the channel to be verified.
.orElse(false); if (header.isCarouselHeader) {
return true;
}
return YoutubeParsingHelper.isVerified(header.json.getArray("badges"));
}
return false;
} }
@Nonnull @Nonnull

View File

@ -14,6 +14,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabs; import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.services.youtube.YouTubeChannelHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
@ -31,6 +32,7 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
import static org.schabi.newpipe.extractor.services.youtube.YouTubeChannelHelper.getChannelResponse; import static org.schabi.newpipe.extractor.services.youtube.YouTubeChannelHelper.getChannelResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.defaultAlertsCheck; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.defaultAlertsCheck;
import static org.schabi.newpipe.extractor.services.youtube.YouTubeChannelHelper.resolveChannelId; import static org.schabi.newpipe.extractor.services.youtube.YouTubeChannelHelper.resolveChannelId;
@ -155,8 +157,19 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
return mdName; return mdName;
} }
return initialData.getObject("header").getObject("c4TabbedHeaderRenderer") return YouTubeChannelHelper.getChannelHeader(initialData)
.getString("title", ""); .map(header -> {
final Object title = header.json.get("title");
if (title instanceof String) {
return (String) title;
} else if (title instanceof JsonObject) {
final String headerName = getTextFromObject((JsonObject) title);
if (!isNullOrEmpty(headerName)) {
return headerName;
}
}
return "";
}).orElse("");
} }
@Nonnull @Nonnull