lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix: add #1050 fix to channel tab name extraction 

use shared method for channel header extraction
This commit is contained in:
ThetaDev 2023-05-07 21:30:13 +02:00
parent 2adc2caebc
commit e8fab3be9c
3 changed files with 87 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YouTubeChannelHelper.java
View File

@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.localization.Localization;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -22,6 +23,7 @@ public final class YouTubeChannelHelper {
/**
* Take a YouTube channel ID or URL path, resolve it if necessary and return a channel ID.
*
* @param idOrPath YouTube channel ID or URL path
* @return YouTube Channel ID
*/
@ -38,7 +40,7 @@ public final class YouTubeChannelHelper {
// we couldn't get information about the channel associated with this URL, if there is one.
if (!channelId[0].equals("channel")) {
final byte[] body = JsonWriter.string(YoutubeParsingHelper.prepareDesktopJsonBuilder(
Localization.DEFAULT, ContentCountry.DEFAULT)
Localization.DEFAULT, ContentCountry.DEFAULT)
.value("url", "https://www.youtube.com/" + idOrPath)
.done())
.getBytes(StandardCharsets.UTF_8);
@ -96,9 +98,9 @@ public final class YouTubeChannelHelper {
* </ul>
*
* @param channelId YouTube channel ID
* @param params Parameters to specify the YouTube channel tab
* @param loc YouTube localization
* @param country YouTube content country
* @param params Parameters to specify the YouTube channel tab
* @param loc YouTube localization
* @param country YouTube content country
* @return Channel response data
*/
public static ChannelResponseData getChannelResponse(final String channelId,
@ -112,7 +114,7 @@ public final class YouTubeChannelHelper {
int level = 0;
while (level < 3) {
final byte[] body = JsonWriter.string(YoutubeParsingHelper.prepareDesktopJsonBuilder(
loc, country)
loc, country)
.value("browseId", id)
.value("params", params) // Equal to videos
.done())
@ -161,6 +163,7 @@ public final class YouTubeChannelHelper {
/**
* Assert that a channel JSON response does not contain a 404 error.
*
* @param jsonResponse channel JSON response
* @throws ContentNotAvailableException if the channel was not found
*/
@ -178,4 +181,35 @@ public final class YouTubeChannelHelper {
}
}
}
public static final class ChannelHeader {
public final JsonObject json;
public final boolean isCarouselHeader;
private ChannelHeader(final JsonObject json, final boolean isCarouselHeader) {
this.json = json;
this.isCarouselHeader = isCarouselHeader;
}
}
public static Optional<ChannelHeader> getChannelHeader(final JsonObject initialData) {
final JsonObject h = initialData.getObject("header");
if (h.has("c4TabbedHeaderRenderer")) {
return Optional.of(h.getObject("c4TabbedHeaderRenderer"))
.map(json -> new ChannelHeader(json, false));
} else if (h.has("carouselHeaderRenderer")) {
return h.getObject("carouselHeaderRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(itm -> itm.has("topicChannelDetailsRenderer"))
.findFirst()
.map(itm -> itm.getObject("topicChannelDetailsRenderer"))
.map(json -> new ChannelHeader(json, true));
} else {
return Optional.empty();
}
}
}

81
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -18,6 +18,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.services.youtube.YouTubeChannelHelper;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
@ -55,8 +56,7 @@ import javax.annotation.Nonnull;
public class YoutubeChannelExtractor extends ChannelExtractor {
private JsonObject initialData;
private Optional<JsonObject> channelHeader;
private boolean isCarouselHeader = false;
private Optional<YouTubeChannelHelper.ChannelHeader> channelHeader;
private JsonObject videoTab;
/**
@ -90,25 +90,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
@Nonnull
private Optional<JsonObject> getChannelHeader() {
private Optional<YouTubeChannelHelper.ChannelHeader> getChannelHeader() {
if (channelHeader == null) {
final JsonObject h = initialData.getObject("header");
if (h.has("c4TabbedHeaderRenderer")) {
channelHeader = Optional.of(h.getObject("c4TabbedHeaderRenderer"));
} else if (h.has("carouselHeaderRenderer")) {
isCarouselHeader = true;
channelHeader = h.getObject("carouselHeaderRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(itm -> itm.has("topicChannelDetailsRenderer"))
.findFirst()
.map(itm -> itm.getObject("topicChannelDetailsRenderer"));
} else {
channelHeader = Optional.empty();
}
channelHeader = YouTubeChannelHelper.getChannelHeader(initialData);
}
return channelHeader;
}
@ -127,8 +111,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getId() throws ParsingException {
return getChannelHeader()
.flatMap(header -> Optional.ofNullable(header.getString("channelId")).or(
() -> Optional.ofNullable(header.getObject("navigationEndpoint")
.flatMap(header -> Optional.ofNullable(header.json.getString("channelId")).or(
() -> Optional.ofNullable(header.json.getObject("navigationEndpoint")
.getObject("browseEndpoint")
.getString("browseId"))
))
@ -146,26 +130,24 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return mdName;
}
final Optional<JsonObject> header = getChannelHeader();
if (header.isPresent()) {
final Object title = header.get().get("title");
return getChannelHeader().flatMap(header -> {
final Object title = header.json.get("title");
if (title instanceof String) {
return (String) title;
return Optional.of((String) title);
} else if (title instanceof JsonObject) {
final String headerName = getTextFromObject((JsonObject) title);
if (!isNullOrEmpty(headerName)) {
return headerName;
return Optional.of(headerName);
}
}
}
throw new ParsingException("Could not get channel name");
return Optional.empty();
}).orElseThrow(() -> new ParsingException("Could not get channel name"));
}
@Override
public String getAvatarUrl() throws ParsingException {
return getChannelHeader().flatMap(header -> Optional.ofNullable(
header.getObject("avatar").getArray("thumbnails")
header.json.getObject("avatar").getArray("thumbnails")
.getObject(0).getString("url")
))
.map(YoutubeParsingHelper::fixThumbnailUrl)
@ -175,7 +157,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getBannerUrl() throws ParsingException {
return getChannelHeader().flatMap(header -> Optional.ofNullable(
header.getObject("banner").getArray("thumbnails")
header.json.getObject("banner").getArray("thumbnails")
.getObject(0).getString("url")
))
.filter(url -> !url.contains("s.ytimg.com") && !url.contains("default_banner"))
@ -194,14 +176,15 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public long getSubscriberCount() throws ParsingException {
final Optional<JsonObject> header = getChannelHeader();
if (header.isPresent()) {
final Optional<YouTubeChannelHelper.ChannelHeader> headerOpt = getChannelHeader();
if (headerOpt.isPresent()) {
final JsonObject header = headerOpt.get().json;
JsonObject textObject = null;
if (header.get().has("subscriberCountText")) {
textObject = header.get().getObject("subscriberCountText");
} else if (header.get().has("subtitle")) {
textObject = header.get().getObject("subtitle");
if (header.has("subscriberCountText")) {
textObject = header.getObject("subscriberCountText");
} else if (header.has("subtitle")) {
textObject = header.getObject("subtitle");
}
if (textObject != null) {
@ -242,17 +225,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public boolean isVerified() throws ParsingException {
// The CarouselHeaderRenderer does not contain any verification badges.
// Since it is only shown on YT-internal channels or on channels of large organizations
// broadcasting live events, we can assume the channel to be verified.
if (isCarouselHeader) {
return true;
}
final Optional<YouTubeChannelHelper.ChannelHeader> headerOpt = getChannelHeader();
if (headerOpt.isPresent()) {
final YouTubeChannelHelper.ChannelHeader header = headerOpt.get();
return getChannelHeader()
.map(header -> header.getArray("badges"))
.map(YoutubeParsingHelper::isVerified)
.orElse(false);
// The CarouselHeaderRenderer does not contain any verification badges.
// Since it is only shown on YT-internal channels or on channels of large organizations
// broadcasting live events, we can assume the channel to be verified.
if (header.isCarouselHeader) {
return true;
}
return YoutubeParsingHelper.isVerified(header.json.getArray("badges"));
}
return false;
}
@Nonnull

17
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java
View File

@ -14,6 +14,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.services.youtube.YouTubeChannelHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
import javax.annotation.Nonnull;
@ -31,6 +32,7 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
import static org.schabi.newpipe.extractor.services.youtube.YouTubeChannelHelper.getChannelResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.defaultAlertsCheck;
import static org.schabi.newpipe.extractor.services.youtube.YouTubeChannelHelper.resolveChannelId;
@ -155,8 +157,19 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
return mdName;
}
return initialData.getObject("header").getObject("c4TabbedHeaderRenderer")
.getString("title", "");
return YouTubeChannelHelper.getChannelHeader(initialData)
.map(header -> {
final Object title = header.json.get("title");
if (title instanceof String) {
return (String) title;
} else if (title instanceof JsonObject) {
final String headerName = getTextFromObject((JsonObject) title);
if (!isNullOrEmpty(headerName)) {
return headerName;
}
}
return "";
}).orElse("");
}
@Nonnull