Implement getTextFromObject() function

This commit is contained in:
wb9688 2020-02-27 17:39:23 +01:00
parent 0798bdd5cd
commit 365b0329f3
11 changed files with 185 additions and 172 deletions

View File

@ -25,6 +25,7 @@ import java.util.Map;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP; import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
@ -124,8 +125,20 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override @Override
public String getAvatarUrl() throws ParsingException { public String getAvatarUrl() throws ParsingException {
try { try {
return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar") String url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar")
.getArray("thumbnails").getObject(0).getString("url"); .getArray("thumbnails").getObject(0).getString("url");
// the first characters of the avatar URLs are different for each channel and some are not even valid URLs
if (url.startsWith("//")) {
url = url.substring(2);
}
if (url.startsWith(HTTP)) {
url = Utils.replaceHttpWithHttps(url);
} else if (!url.startsWith(HTTPS)) {
url = HTTPS + url;
}
return url;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get avatar", e); throw new ParsingException("Could not get avatar", e);
} }
@ -172,7 +185,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final JsonObject subscriberInfo = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText"); final JsonObject subscriberInfo = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText");
if (subscriberInfo != null) { if (subscriberInfo != null) {
try { try {
return Utils.mixedNumberWordToLong(subscriberInfo.getArray("runs").getObject(0).getString("text")); return Utils.mixedNumberWordToLong(getTextFromObject(subscriberInfo));
} catch (NumberFormatException e) { } catch (NumberFormatException e) {
throw new ParsingException("Could not get subscriber count", e); throw new ParsingException("Could not get subscriber count", e);
} }
@ -301,10 +314,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
} }
try { try {
if (videoTab.getObject("content").getObject("sectionListRenderer").getArray("contents") if (getTextFromObject(videoTab.getObject("content").getObject("sectionListRenderer")
.getObject(0).getObject("itemSectionRenderer").getArray("contents") .getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getObject(0).getObject("messageRenderer").getObject("text").getArray("runs") .getArray("contents").getObject(0).getObject("messageRenderer")
.getObject(0).getString("text").equals("This channel has no videos.")) .getObject("text")).equals("This channel has no videos."))
return null; return null;
} catch (Exception ignored) {} } catch (Exception ignored) {}

View File

@ -1,6 +1,5 @@
package org.schabi.newpipe.extractor.services.youtube.extractors; package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
@ -8,6 +7,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP; import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
@ -59,7 +59,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
try { try {
return channelInfoItem.getObject("title").getString("simpleText"); return getTextFromObject(channelInfoItem.getObject("title"));
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get name", e); throw new ParsingException("Could not get name", e);
} }
@ -68,7 +68,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override @Override
public String getUrl() throws ParsingException { public String getUrl() throws ParsingException {
try { try {
String id = "channel/" + channelInfoItem.getString("channelId"); // Does prepending 'channel/' always work? String id = "channel/" + channelInfoItem.getString("channelId");
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id); return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id);
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get url", e); throw new ParsingException("Could not get url", e);
@ -78,7 +78,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override @Override
public long getSubscriberCount() throws ParsingException { public long getSubscriberCount() throws ParsingException {
try { try {
String subscribers = channelInfoItem.getObject("subscriberCountText").getString("simpleText").split(" ")[0]; String subscribers = getTextFromObject(channelInfoItem.getObject("subscriberCountText"));
return Utils.mixedNumberWordToLong(subscribers); return Utils.mixedNumberWordToLong(subscribers);
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get subscriber count", e); throw new ParsingException("Could not get subscriber count", e);
@ -88,8 +88,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override @Override
public long getStreamCount() throws ParsingException { public long getStreamCount() throws ParsingException {
try { try {
return Long.parseLong(Utils.removeNonDigitCharacters(channelInfoItem.getObject("videoCountText") return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(channelInfoItem.getObject("videoCountText"))));
.getArray("runs").getObject(0).getString("text")));
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get stream count", e); throw new ParsingException("Could not get stream count", e);
} }
@ -98,11 +97,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override @Override
public String getDescription() throws ParsingException { public String getDescription() throws ParsingException {
try { try {
StringBuilder description = new StringBuilder(); return getTextFromObject(channelInfoItem.getObject("descriptionSnippet"));
JsonArray descriptionArray = channelInfoItem.getObject("descriptionSnippet").getArray("runs");
for (Object descriptionPart : descriptionArray)
description.append(((JsonObject) descriptionPart).getString("text"));
return description.toString();
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get description", e); throw new ParsingException("Could not get description", e);
} }

View File

@ -25,6 +25,11 @@ import java.util.Map;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
@SuppressWarnings("WeakerAccess") @SuppressWarnings("WeakerAccess")
public class YoutubePlaylistExtractor extends PlaylistExtractor { public class YoutubePlaylistExtractor extends PlaylistExtractor {
private JsonObject initialData; private JsonObject initialData;
@ -104,7 +109,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
try { try {
String name = playlistInfo.getObject("title").getArray("runs").getObject(0).getString("text"); String name = getTextFromObject(playlistInfo.getObject("title"));
if (name != null) return name; if (name != null) return name;
} catch (Exception ignored) {} } catch (Exception ignored) {}
try { try {
@ -137,8 +142,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getUploaderUrl() throws ParsingException { public String getUploaderUrl() throws ParsingException {
try { try {
return YoutubeChannelExtractor.CHANNEL_URL_BASE + return getUrlFromNavigationEndpoint(getUploaderInfo().getObject("navigationEndpoint"));
getUploaderInfo().getObject("navigationEndpoint").getObject("browseEndpoint").getString("browseId");
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get playlist uploader url", e); throw new ParsingException("Could not get playlist uploader url", e);
} }
@ -147,7 +151,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getUploaderName() throws ParsingException { public String getUploaderName() throws ParsingException {
try { try {
return getUploaderInfo().getObject("title").getArray("runs").getObject(0).getString("text"); return getTextFromObject(getUploaderInfo().getObject("title"));
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get playlist uploader name", e); throw new ParsingException("Could not get playlist uploader name", e);
} }
@ -156,7 +160,19 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getUploaderAvatarUrl() throws ParsingException { public String getUploaderAvatarUrl() throws ParsingException {
try { try {
return getUploaderInfo().getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); String url = getUploaderInfo().getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
// the first characters of the avatar URLs are different for each channel and some are not even valid URLs
if (url.startsWith("//")) {
url = url.substring(2);
}
if (url.startsWith(HTTP)) {
url = Utils.replaceHttpWithHttps(url);
} else if (!url.startsWith(HTTPS)) {
url = HTTPS + url;
}
return url;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get playlist uploader avatar", e); throw new ParsingException("Could not get playlist uploader avatar", e);
} }
@ -165,7 +181,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public long getStreamCount() throws ParsingException { public long getStreamCount() throws ParsingException {
try { try {
String viewsText = getPlaylistInfo().getArray("stats").getObject(0).getArray("runs").getObject(0).getString("text"); String viewsText = getTextFromObject(getPlaylistInfo().getArray("stats").getObject(0));
return Long.parseLong(Utils.removeNonDigitCharacters(viewsText)); return Long.parseLong(Utils.removeNonDigitCharacters(viewsText));
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get video count from playlist", e); throw new ParsingException("Could not get video count from playlist", e);

View File

@ -7,6 +7,8 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
private JsonObject playlistInfoItem; private JsonObject playlistInfoItem;
@ -27,7 +29,7 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
try { try {
return playlistInfoItem.getObject("title").getString("simpleText"); return getTextFromObject(playlistInfoItem.getObject("title"));
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get name", e); throw new ParsingException("Could not get name", e);
} }
@ -46,7 +48,7 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override @Override
public String getUploaderName() throws ParsingException { public String getUploaderName() throws ParsingException {
try { try {
return playlistInfoItem.getObject("longBylineText").getArray("runs").getObject(0).getString("text"); return getTextFromObject(playlistInfoItem.getObject("longBylineText"));
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get uploader name", e); throw new ParsingException("Could not get uploader name", e);
} }

View File

@ -24,6 +24,8 @@ import java.util.Map;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
/* /*
* Created by Christian Schabesberger on 22.07.2018 * Created by Christian Schabesberger on 22.07.2018
* *
@ -91,8 +93,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
if (showingResultsForRenderer == null) { if (showingResultsForRenderer == null) {
return ""; return "";
} else { } else {
return showingResultsForRenderer.getObject("correctedQuery").getArray("runs") return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery"));
.getObject(0).getString("text");
} }
} }
@ -155,8 +156,8 @@ public class YoutubeSearchExtractor extends SearchExtractor {
for (Object item : videos) { for (Object item : videos) {
if (((JsonObject) item).getObject("backgroundPromoRenderer") != null) { if (((JsonObject) item).getObject("backgroundPromoRenderer") != null) {
throw new NothingFoundException(((JsonObject) item).getObject("backgroundPromoRenderer") throw new NothingFoundException(getTextFromObject(((JsonObject) item)
.getObject("bodyText").getArray("runs").getObject(0).getString("text")); .getObject("backgroundPromoRenderer").getObject("bodyText")));
} else if (((JsonObject) item).getObject("videoRenderer") != null) { } else if (((JsonObject) item).getObject("videoRenderer") != null) {
collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) item).getObject("videoRenderer"), timeAgoParser)); collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) item).getObject("videoRenderer"), timeAgoParser));
} else if (((JsonObject) item).getObject("channelRenderer") != null) { } else if (((JsonObject) item).getObject("channelRenderer") != null) {

View File

@ -40,8 +40,6 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Calendar; import java.util.Calendar;
@ -56,6 +54,10 @@ import java.util.Map;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
/* /*
* Created by Christian Schabesberger on 06.08.15. * Created by Christian Schabesberger on 06.08.15.
* *
@ -114,11 +116,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
assertPageFetched(); assertPageFetched();
String title = null; String title = null;
try { try {
StringBuilder titleBuilder = new StringBuilder(); title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title"));
JsonArray titleArray = getVideoPrimaryInfoRenderer().getObject("title").getArray("runs");
for (Object titlePart : titleArray)
titleBuilder.append(((JsonObject) titlePart).getString("text"));
title = titleBuilder.toString();
} catch (Exception ignored) {} } catch (Exception ignored) {}
if (title == null) { if (title == null) {
try { try {
@ -146,8 +144,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} catch (Exception ignored) {} } catch (Exception ignored) {}
try { try {
if (getVideoPrimaryInfoRenderer().getObject("dateText").getString("simpleText").startsWith("Premiered")) { if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).startsWith("Premiered")) {
String time = getVideoPrimaryInfoRenderer().getObject("dateText").getString("simpleText").substring(10); String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).substring(10);
try { // Premiered 20 hours ago try { // Premiered 20 hours ago
TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en")); TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en"));
@ -165,7 +163,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try { try {
// TODO this parses English formatted dates only, we need a better approach to parse the textual date // TODO this parses English formatted dates only, we need a better approach to parse the textual date
Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse( Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse(
getVideoPrimaryInfoRenderer().getObject("dateText").getString("simpleText")); getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")));
return new SimpleDateFormat("yyyy-MM-dd").format(d); return new SimpleDateFormat("yyyy-MM-dd").format(d);
} catch (Exception ignored) {} } catch (Exception ignored) {}
throw new ParsingException("Could not get upload date"); throw new ParsingException("Could not get upload date");
@ -203,73 +201,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
assertPageFetched(); assertPageFetched();
// description with more info on links // description with more info on links
try { try {
boolean htmlConversionRequired = false; String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true);
JsonArray descriptions = getVideoSecondaryInfoRenderer().getObject("description").getArray("runs"); return new Description(description, Description.HTML);
StringBuilder descriptionBuilder = new StringBuilder(descriptions.size());
for (Object textObjectHolder : descriptions) {
JsonObject textHolder = (JsonObject) textObjectHolder;
String text = textHolder.getString("text");
if (textHolder.getObject("navigationEndpoint") != null) {
// The text is a link. Get the URL it points to and generate a HTML link of it
if (textHolder.getObject("navigationEndpoint").getObject("urlEndpoint") != null) {
String internUrl = textHolder.getObject("navigationEndpoint").getObject("urlEndpoint").getString("url");
if (internUrl.startsWith("/redirect?")) {
// q parameter can be the first parameter
internUrl = internUrl.substring(10);
String[] params = internUrl.split("&");
for (String param : params) {
if (param.split("=")[0].equals("q")) {
String url = URLDecoder.decode(param.split("=")[1], StandardCharsets.UTF_8.name());
if (url != null && !url.isEmpty()) {
descriptionBuilder.append("<a href=\"").append(url).append("\">").append(text).append("</a>");
htmlConversionRequired = true;
} else {
descriptionBuilder.append(text);
}
break;
}
}
} else if (internUrl.startsWith("http")) {
descriptionBuilder.append("<a href=\"").append(internUrl).append("\">").append(text).append("</a>");
htmlConversionRequired = true;
}
} else if (textHolder.getObject("navigationEndpoint").getObject("browseEndpoint") != null) {
descriptionBuilder.append("<a href=\"https://www.youtube.com").append(
textHolder.getObject("navigationEndpoint").getObject("browseEndpoint")
.getString("canonicalBaseUrl")).append("\">").append(text).append("</a>");
htmlConversionRequired = true;
} else if (textHolder.getObject("navigationEndpoint").getObject("watchEndpoint") != null) {
descriptionBuilder.append("<a href=\"https://www.youtube.com/watch?v=").append(
textHolder.getObject("navigationEndpoint").getObject("watchEndpoint")
.getString("videoId"));
if (textHolder.getObject("navigationEndpoint").getObject("watchEndpoint").getString("playlistId") != null) {
descriptionBuilder.append("&amp;list=").append(textHolder.getObject("navigationEndpoint")
.getObject("watchEndpoint").getString("playlistId"));
}
if (textHolder.getObject("navigationEndpoint").getObject("watchEndpoint").has("startTimeSeconds")) {
descriptionBuilder.append("&amp;t=").append(textHolder.getObject("navigationEndpoint")
.getObject("watchEndpoint").getInt("startTimeSeconds"));
}
descriptionBuilder.append("\">").append(text).append("</a>");
htmlConversionRequired = true;
}
continue;
}
if (text != null) {
descriptionBuilder.append(text);
}
}
String description = descriptionBuilder.toString();
if (!description.isEmpty()) {
if (htmlConversionRequired) {
description = description.replaceAll("\\n", "<br>");
description = description.replaceAll(" ", " &nbsp;");
return new Description(description, Description.HTML);
}
return new Description(description, Description.PLAIN_TEXT);
}
} catch (Exception ignored) { } } catch (Exception ignored) { }
// raw non-html description // raw non-html description
@ -329,16 +262,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
assertPageFetched(); assertPageFetched();
String views = null; String views = null;
try { try {
views = getVideoPrimaryInfoRenderer().getObject("viewCount") views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount")
.getObject("videoViewCountRenderer").getObject("viewCount") .getObject("videoViewCountRenderer").getObject("viewCount"));
.getArray("runs").getObject(0).getString("text");
} catch (Exception ignored) {} } catch (Exception ignored) {}
if (views == null) {
try {
views = getVideoPrimaryInfoRenderer().getObject("viewCount")
.getObject("videoViewCountRenderer").getObject("viewCount").getString("simpleText");
} catch (Exception ignored) {}
}
if (views == null) { if (views == null) {
try { try {
views = playerResponse.getObject("videoDetails").getString("viewCount"); views = playerResponse.getObject("videoDetails").getString("viewCount");
@ -398,17 +324,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override @Override
public String getUploaderUrl() throws ParsingException { public String getUploaderUrl() throws ParsingException {
assertPageFetched(); assertPageFetched();
String uploaderId = null;
try { try {
uploaderId = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") String uploaderUrl = getUrlFromNavigationEndpoint(getVideoSecondaryInfoRenderer()
.getObject("navigationEndpoint").getObject("browseEndpoint").getString("browseId"); .getObject("owner").getObject("videoOwnerRenderer").getObject("navigationEndpoint"));
if (uploaderUrl != null) return uploaderUrl;
} catch (Exception ignored) {}
try {
String uploaderId = playerResponse.getObject("videoDetails").getString("channelId");
if (uploaderId != null) return "https://www.youtube.com/channel/" + uploaderId;
} catch (Exception ignored) {} } catch (Exception ignored) {}
if (uploaderId == null) {
try {
uploaderId = playerResponse.getObject("videoDetails").getString("channelId");
} catch (Exception ignored) {}
}
if (uploaderId != null) return "https://www.youtube.com/channel/" + uploaderId;
throw new ParsingException("Could not get uploader url"); throw new ParsingException("Could not get uploader url");
} }
@ -418,8 +342,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
assertPageFetched(); assertPageFetched();
String uploaderName = null; String uploaderName = null;
try { try {
uploaderName = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner")
.getObject("title").getArray("runs").getObject(0).getString("text"); .getObject("videoOwnerRenderer").getObject("title"));
} catch (Exception ignored) {} } catch (Exception ignored) {}
if (uploaderName == null) { if (uploaderName == null) {
try { try {
@ -435,8 +359,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public String getUploaderAvatarUrl() throws ParsingException { public String getUploaderAvatarUrl() throws ParsingException {
assertPageFetched(); assertPageFetched();
try { try {
return getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") String url = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer")
.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); .getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
// the first characters of the avatar URLs are different for each channel and some are not even valid URLs
if (url.startsWith("//")) {
url = url.substring(2);
}
if (url.startsWith(HTTP)) {
url = Utils.replaceHttpWithHttps(url);
} else if (!url.startsWith(HTTPS)) {
url = HTTPS + url;
}
return url;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get uploader avatar url", e); throw new ParsingException("Could not get uploader avatar url", e);
} }

View File

@ -6,7 +6,6 @@ import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
@ -15,6 +14,9 @@ import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
/* /*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org> * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* YoutubeStreamInfoItemExtractor.java is part of NewPipe. * YoutubeStreamInfoItemExtractor.java is part of NewPipe.
@ -76,15 +78,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
String name = null; String name = getTextFromObject(videoInfo.getObject("title"));
try {
name = videoInfo.getObject("title").getString("simpleText");
} catch (Exception ignored) {}
if (name == null) {
try {
name = videoInfo.getObject("title").getArray("runs").getObject(0).getString("text");
} catch (Exception ignored) {}
}
if (name != null && !name.isEmpty()) return name; if (name != null && !name.isEmpty()) return name;
throw new ParsingException("Could not get name"); throw new ParsingException("Could not get name");
} }
@ -94,14 +88,14 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
if (getStreamType() == StreamType.LIVE_STREAM) return -1; if (getStreamType() == StreamType.LIVE_STREAM) return -1;
String duration = null; String duration = null;
try { try {
duration = videoInfo.getObject("lengthText").getString("simpleText"); duration = getTextFromObject(videoInfo.getObject("lengthText"));
} catch (Exception ignored) {} } catch (Exception ignored) {}
if (duration == null) { if (duration == null) {
try { try {
for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) { for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) {
if (((JsonObject) thumbnailOverlay).getObject("thumbnailOverlayTimeStatusRenderer") != null) { if (((JsonObject) thumbnailOverlay).getObject("thumbnailOverlayTimeStatusRenderer") != null) {
duration = ((JsonObject) thumbnailOverlay).getObject("thumbnailOverlayTimeStatusRenderer") duration = getTextFromObject(((JsonObject) thumbnailOverlay)
.getObject("text").getString("simpleText"); .getObject("thumbnailOverlayTimeStatusRenderer").getObject("text"));
} }
} }
} catch (Exception ignored) {} } catch (Exception ignored) {}
@ -114,19 +108,16 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
public String getUploaderName() throws ParsingException { public String getUploaderName() throws ParsingException {
String name = null; String name = null;
try { try {
name = videoInfo.getObject("longBylineText").getArray("runs") name = getTextFromObject(videoInfo.getObject("longBylineText"));
.getObject(0).getString("text");
} catch (Exception ignored) {} } catch (Exception ignored) {}
if (name == null) { if (name == null) {
try { try {
name = videoInfo.getObject("ownerText").getArray("runs") name = getTextFromObject(videoInfo.getObject("ownerText"));
.getObject(0).getString("text");
} catch (Exception ignored) {} } catch (Exception ignored) {}
} }
if (name == null) { if (name == null) {
try { try {
name = videoInfo.getObject("shortBylineText").getArray("runs") name = getTextFromObject(videoInfo.getObject("shortBylineText"));
.getObject(0).getString("text");
} catch (Exception ignored) {} } catch (Exception ignored) {}
} }
if (name != null && !name.isEmpty()) return name; if (name != null && !name.isEmpty()) return name;
@ -136,30 +127,27 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override @Override
public String getUploaderUrl() throws ParsingException { public String getUploaderUrl() throws ParsingException {
try { try {
String id = null; String url = null;
try { try {
id = videoInfo.getObject("longBylineText").getArray("runs") url = getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText")
.getObject(0).getObject("navigationEndpoint") .getArray("runs").getObject(0).getObject("navigationEndpoint"));
.getObject("browseEndpoint").getString("browseId");
} catch (Exception ignored) {} } catch (Exception ignored) {}
if (id == null) { if (url == null) {
try { try {
id = videoInfo.getObject("ownerText").getArray("runs") url = getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText")
.getObject(0).getObject("navigationEndpoint") .getArray("runs").getObject(0).getObject("navigationEndpoint"));
.getObject("browseEndpoint").getString("browseId");
} catch (Exception ignored) {} } catch (Exception ignored) {}
} }
if (id == null) { if (url == null) {
try { try {
id = videoInfo.getObject("shortBylineText").getArray("runs") url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText")
.getObject(0).getObject("navigationEndpoint") .getArray("runs").getObject(0).getObject("navigationEndpoint"));
.getObject("browseEndpoint").getString("browseId");
} catch (Exception ignored) {} } catch (Exception ignored) {}
} }
if (id == null || id.isEmpty()) { if (url == null || url.isEmpty()) {
throw new IllegalArgumentException("is empty"); throw new IllegalArgumentException("is empty");
} }
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + id); return url;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get uploader url"); throw new ParsingException("Could not get uploader url");
} }
@ -169,7 +157,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override @Override
public String getTextualUploadDate() { public String getTextualUploadDate() {
try { try {
return videoInfo.getObject("publishedTimeText").getString("simpleText"); return getTextFromObject(videoInfo.getObject("publishedTimeText"));
} catch (Exception e) { } catch (Exception e) {
// upload date is not always available, e.g. in playlists // upload date is not always available, e.g. in playlists
return null; return null;
@ -196,13 +184,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
if (videoInfo.getObject("topStandaloneBadge") != null || isPremium()) { if (videoInfo.getObject("topStandaloneBadge") != null || isPremium()) {
return -1; return -1;
} }
String viewCount; String viewCount = getTextFromObject(videoInfo.getObject("viewCountText"));
if (getStreamType() == StreamType.LIVE_STREAM) {
viewCount = videoInfo.getObject("viewCountText")
.getArray("runs").getObject(0).getString("text");
} else {
viewCount = videoInfo.getObject("viewCountText").getString("simpleText");
}
if (viewCount.equals("Recommended for you")) return -1; if (viewCount.equals("Recommended for you")) return -1;
return Long.parseLong(Utils.removeNonDigitCharacters(viewCount)); return Long.parseLong(Utils.removeNonDigitCharacters(viewCount));
} catch (Exception e) { } catch (Exception e) {

View File

@ -44,6 +44,8 @@ import java.util.Map;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> { public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
private JsonObject initialData; private JsonObject initialData;
@ -93,8 +95,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
public String getName() throws ParsingException { public String getName() throws ParsingException {
String name; String name;
try { try {
name = initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title") name = getTextFromObject(initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title"));
.getArray("runs").getObject(0).getString("text");
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get Trending name", e); throw new ParsingException("Could not get Trending name", e);
} }

View File

@ -13,7 +13,10 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import java.io.UnsupportedEncodingException;
import java.net.URL; import java.net.URL;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.text.ParseException; import java.text.ParseException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.Calendar; import java.util.Calendar;
@ -251,4 +254,69 @@ public class YoutubeParsingHelper {
throw new ParsingException("Could not get client version"); throw new ParsingException("Could not get client version");
} }
public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) {
if (navigationEndpoint.getObject("urlEndpoint") != null) {
String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url");
if (internUrl.startsWith("/redirect?")) {
// q parameter can be the first parameter
internUrl = internUrl.substring(10);
String[] params = internUrl.split("&");
for (String param : params) {
if (param.split("=")[0].equals("q")) {
String url;
try {
url = URLDecoder.decode(param.split("=")[1], StandardCharsets.UTF_8.name());
} catch (UnsupportedEncodingException e) {
return null;
}
return url;
}
}
} else if (internUrl.startsWith("http")) {
return internUrl;
}
} else if (navigationEndpoint.getObject("browseEndpoint") != null) {
return "https://www.youtube.com" + navigationEndpoint.getObject("browseEndpoint").getString("canonicalBaseUrl");
} else if (navigationEndpoint.getObject("watchEndpoint") != null) {
StringBuilder url = new StringBuilder();
url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId"));
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId"))
url.append("&amp;list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId"));
if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds"))
url.append("&amp;t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds"));
return url.toString();
}
return null;
}
public static String getTextFromObject(JsonObject textObject, boolean html) {
if (textObject.has("simpleText")) return textObject.getString("simpleText");
StringBuilder textBuilder = new StringBuilder();
for (Object textPart : textObject.getArray("runs")) {
String text = ((JsonObject) textPart).getString("text");
if (html && ((JsonObject) textPart).getObject("navigationEndpoint") != null) {
String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint"));
if (url != null && !url.isEmpty()) {
textBuilder.append("<a href=\"").append(url).append("\">").append(text).append("</a>");
continue;
}
}
textBuilder.append(text);
}
String text = textBuilder.toString();
if (html) {
text = text.replaceAll("\\n", "<br>");
text = text.replaceAll(" ", " &nbsp;");
}
return text;
}
public static String getTextFromObject(JsonObject textObject) {
return getTextFromObject(textObject, false);
}
} }

View File

@ -99,7 +99,7 @@ public class YoutubePlaylistExtractorTest {
@Test @Test
public void testUploaderUrl() throws Exception { public void testUploaderUrl() throws Exception {
assertEquals("https://www.youtube.com/channel/UCs72iRpTEuwV3y6pdWYLgiw", extractor.getUploaderUrl()); assertEquals("https://www.youtube.com/user/andre0y0you", extractor.getUploaderUrl());
} }
@Test @Test

View File

@ -245,7 +245,6 @@ public class YoutubeStreamExtractorDefaultTest {
@Test @Test
public void testGetDescription() throws ParsingException { public void testGetDescription() throws ParsingException {
System.out.println(extractor.getDescription().getContent());
assertNotNull(extractor.getDescription()); assertNotNull(extractor.getDescription());
assertFalse(extractor.getDescription().getContent().isEmpty()); assertFalse(extractor.getDescription().getContent().isEmpty());
} }