lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Use our fork of nanojson

This commit is contained in:
wb9688 2020-04-16 16:08:14 +02:00
parent 1b522304c1
commit 979c5a7502
22 changed files with 219 additions and 323 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
build.gradle
View File

@ -10,6 +10,7 @@ allprojects {
repositories {
jcenter()
maven { url "https://jitpack.io" }
}
}

4
extractor/build.gradle
View File

@ -1,11 +1,11 @@
dependencies {
implementation project(':timeago-parser')
implementation 'com.grack:nanojson:1.1'
implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751'
implementation 'org.jsoup:jsoup:1.9.2'
implementation 'org.mozilla:rhino:1.7.7.1'
implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0'
implementation 'org.nibor.autolink:autolink:0.8.0'
testImplementation 'junit:junit:4.12'
}
}

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java
View File

@ -109,7 +109,7 @@ public class PeertubeAccountExtractor extends ChannelExtractor {
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if (null != response && !StringUtil.isBlank(response.responseBody())) {
if (response != null && !StringUtil.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
@ -169,7 +169,7 @@ public class PeertubeAccountExtractor extends ChannelExtractor {
try {
json = JsonParser.object().from(responseBody);
} catch (JsonParserException e) {
throw new ExtractionException("Unable to extract peertube channel data", e);
throw new ExtractionException("Unable to extract PeerTube channel data", e);
}
if (json == null) throw new ExtractionException("Unable to extract PeerTube channel data");
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java
View File

@ -167,7 +167,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
assertPageFetched();
List<VideoStream> videoStreams = new ArrayList<>();
try {
JsonArray streams = json.getArray("files", new JsonArray());
JsonArray streams = json.getArray("files");
for (Object s : streams) {
if (!(s instanceof JsonObject)) continue;
JsonObject stream = (JsonObject) s;

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java
View File

@ -16,6 +16,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
@SuppressWarnings("WeakerAccess")
public class SoundcloudChannelExtractor extends ChannelExtractor {
private String userId;
@ -62,10 +64,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Override
public String getBannerUrl() {
return user.getObject("visuals", new JsonObject())
.getArray("visuals", new JsonArray())
.getObject(0, new JsonObject())
.getString("visual_url");
return user.getObject("visuals").getArray("visuals").getObject(0).getString("visual_url");
}
@Override
@ -80,7 +79,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Override
public String getDescription() {
return user.getString("description", "");
return user.getString("description", EMPTY_STRING);
}
@Nonnull

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java
View File

@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.soundcloud;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {
@ -24,7 +25,7 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
@Override
public String getThumbnailUrl() {
String avatarUrl = itemObject.getString("avatar_url", "");
String avatarUrl = itemObject.getString("avatar_url", EMPTY_STRING);
String avatarUrlBetterResolution = avatarUrl.replace("large.jpg", "crop.jpg");
return avatarUrlBetterResolution;
}
@ -41,6 +42,6 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
@Override
public String getDescription() {
return itemObject.getString("description", "");
return itemObject.getString("description", EMPTY_STRING);
}
}

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View File

@ -28,6 +28,7 @@ import java.util.*;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudParsingHelper {
@ -256,17 +257,17 @@ public class SoundcloudParsingHelper {
@Nonnull
static String getUploaderUrl(JsonObject object) {
String url = object.getObject("user").getString("permalink_url", "");
String url = object.getObject("user").getString("permalink_url", EMPTY_STRING);
return replaceHttpWithHttps(url);
}
@Nonnull
static String getAvatarUrl(JsonObject object) {
String url = object.getObject("user", new JsonObject()).getString("avatar_url", "");
String url = object.getObject("user").getString("avatar_url", EMPTY_STRING);
return replaceHttpWithHttps(url);
}
public static String getUploaderName(JsonObject object) {
return object.getObject("user").getString("username", "");
return object.getObject("user").getString("username", EMPTY_STRING);
}
}

11
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java
View File

@ -4,6 +4,7 @@ import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
@ -31,7 +32,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
public String getThumbnailUrl() throws ParsingException {
// Over-engineering at its finest
if (itemObject.isString(ARTWORK_URL_KEY)) {
final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, "");
final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, EMPTY_STRING);
if (!artworkUrl.isEmpty()) {
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
return artworkUrlBetterResolution;
@ -45,7 +46,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
// First look for track artwork url
if (trackObject.isString(ARTWORK_URL_KEY)) {
String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, "");
String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, EMPTY_STRING);
if (!artworkUrl.isEmpty()) {
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
return artworkUrlBetterResolution;
@ -53,8 +54,8 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
}
// Then look for track creator avatar url
final JsonObject creator = trackObject.getObject(USER_KEY, new JsonObject());
final String creatorAvatar = creator.getString(AVATAR_URL_KEY, "");
final JsonObject creator = trackObject.getObject(USER_KEY);
final String creatorAvatar = creator.getString(AVATAR_URL_KEY, EMPTY_STRING);
if (!creatorAvatar.isEmpty()) return creatorAvatar;
}
} catch (Exception ignored) {
@ -63,7 +64,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
try {
// Last resort, use user avatar url. If still not found, then throw exception.
return itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY, "");
return itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY, EMPTY_STRING);
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist thumbnail url", e);
}

3
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java
View File

@ -23,6 +23,7 @@ import java.net.MalformedURLException;
import java.net.URL;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
public class SoundcloudSearchExtractor extends SearchExtractor {
@ -84,7 +85,7 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
if (!(result instanceof JsonObject)) continue;
//noinspection ConstantConditions
JsonObject searchResult = (JsonObject) result;
String kind = searchResult.getString("kind", "");
String kind = searchResult.getString("kind", EMPTY_STRING);
switch (kind) {
case "user":
collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult));

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java
View File

@ -34,6 +34,8 @@ import java.util.Locale;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
public class SoundcloudStreamExtractor extends StreamExtractor {
private JsonObject track;
@ -45,7 +47,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
track = SoundcloudParsingHelper.resolveFor(downloader, getOriginalUrl());
String policy = track.getString("policy", "");
String policy = track.getString("policy", EMPTY_STRING);
if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) {
throw new ContentNotAvailableException("Content not available: policy " + policy);
}
@ -78,9 +80,9 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getThumbnailUrl() {
String artworkUrl = track.getString("artwork_url", "");
String artworkUrl = track.getString("artwork_url", EMPTY_STRING);
if (artworkUrl.isEmpty()) {
artworkUrl = track.getObject("user").getString("avatar_url", "");
artworkUrl = track.getObject("user").getString("avatar_url", EMPTY_STRING);
}
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
return artworkUrlBetterResolution;

3
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java
View File

@ -6,6 +6,7 @@ import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor {
@ -62,7 +63,7 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
@Override
public String getThumbnailUrl() {
String artworkUrl = itemObject.getString("artwork_url", "");
String artworkUrl = itemObject.getString("artwork_url", EMPTY_STRING);
if (artworkUrl.isEmpty()) {
artworkUrl = itemObject.getObject("user").getString("avatar_url");
}

62
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -72,22 +72,16 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
while (level < 3) {
final JsonArray jsonResponse = getJsonResponse(url, getExtractorLocalization());
final JsonObject endpoint = jsonResponse.getObject(1, EMPTY_OBJECT)
.getObject("response", EMPTY_OBJECT).getArray("onResponseReceivedActions", EMPTY_ARRAY)
.getObject(0, EMPTY_OBJECT).getObject("navigateAction", EMPTY_OBJECT)
.getObject("endpoint", EMPTY_OBJECT);
final JsonObject endpoint = jsonResponse.getObject(1).getObject("response")
.getArray("onResponseReceivedActions").getObject(0).getObject("navigateAction")
.getObject("endpoint");
final String webPageType = endpoint
.getObject("commandMetadata", EMPTY_OBJECT)
.getObject("webCommandMetadata", EMPTY_OBJECT)
final String webPageType = endpoint.getObject("commandMetadata").getObject("webCommandMetadata")
.getString("webPageType", EMPTY_STRING);
final String browseId = endpoint
.getObject("browseEndpoint", EMPTY_OBJECT)
.getString("browseId", EMPTY_STRING);
final String browseId = endpoint.getObject("browseEndpoint").getString("browseId", EMPTY_STRING);
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE") && !browseId.isEmpty()) {
if (!browseId.startsWith("UC")) {
throw new ExtractionException("Redirected id is not pointing to a channel");
}
@ -131,10 +125,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Nonnull
@Override
public String getId() throws ParsingException {
final String channelId = initialData
.getObject("header", EMPTY_OBJECT)
.getObject("c4TabbedHeaderRenderer", EMPTY_OBJECT)
.getString("channelId", EMPTY_STRING);
final String channelId = initialData.getObject("header").getObject("c4TabbedHeaderRenderer")
.getString("channelId", EMPTY_STRING);
if (!channelId.isEmpty()) {
return channelId;
@ -170,11 +162,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getBannerUrl() throws ParsingException {
try {
String url = null;
try {
url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner")
String url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner")
.getArray("thumbnails").getObject(0).getString("url");
} catch (Exception ignored) {}
if (url == null || url.contains("s.ytimg.com") || url.contains("default_banner")) {
return null;
}
@ -196,19 +186,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public long getSubscriberCount() throws ParsingException {
final JsonObject subscriberInfo = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText");
if (subscriberInfo != null) {
final JsonObject c4TabbedHeaderRenderer = initialData.getObject("header").getObject("c4TabbedHeaderRenderer");
if (c4TabbedHeaderRenderer.has("subscriberCountText")) {
try {
return Utils.mixedNumberWordToLong(getTextFromObject(subscriberInfo));
return Utils.mixedNumberWordToLong(getTextFromObject(c4TabbedHeaderRenderer.getObject("subscriberCountText")));
} catch (NumberFormatException e) {
throw new ParsingException("Could not get subscriber count", e);
}
} else {
// If there's no subscribe button, the channel has the subscriber count disabled
if (initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscribeButton") == null) {
return -1;
} else {
if (c4TabbedHeaderRenderer.has("subscribeButton")) {
return 0;
} else {
return -1;
}
}
}
@ -260,7 +250,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
private String getNextPageUrlFrom(JsonArray continuations) {
if (continuations == null) return "";
if (continuations == null || continuations.isEmpty()) {
return "";
}
JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
String continuation = nextContinuationData.getString("continuation");
@ -277,7 +269,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object video : videos) {
if (((JsonObject) video).getObject("gridVideoRenderer") != null) {
if (((JsonObject) video).has("gridVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(
((JsonObject) video).getObject("gridVideoRenderer"), timeAgoParser) {
@Override
@ -302,8 +294,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
JsonObject videoTab = null;
for (Object tab : tabs) {
if (((JsonObject) tab).getObject("tabRenderer") != null) {
if (((JsonObject) tab).getObject("tabRenderer").getString("title").equals("Videos")) {
if (((JsonObject) tab).has("tabRenderer")) {
if (((JsonObject) tab).getObject("tabRenderer").getString("title", EMPTY_STRING).equals("Videos")) {
videoTab = ((JsonObject) tab).getObject("tabRenderer");
break;
}
@ -314,13 +306,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
throw new ContentNotSupportedException("This channel has no Videos tab");
}
try {
if (getTextFromObject(videoTab.getObject("content").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("messageRenderer")
.getObject("text")).equals("This channel has no videos."))
return null;
} catch (Exception ignored) {}
if (getTextFromObject(videoTab.getObject("content").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("messageRenderer")
.getObject("text")).equals("This channel has no videos."))
return null;
this.videoTab = videoTab;
return videoTab;

19
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java
View File

@ -70,14 +70,12 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override
public long getSubscriberCount() throws ParsingException {
try {
final JsonObject subscriberCountObject = channelInfoItem.getObject("subscriberCountText");
if (subscriberCountObject == null) {
if (!channelInfoItem.has("subscriberCountText")) {
// Subscription count is not available for this channel item.
return -1;
}
return Utils.mixedNumberWordToLong(getTextFromObject(subscriberCountObject));
return Utils.mixedNumberWordToLong(getTextFromObject(channelInfoItem.getObject("subscriberCountText")));
} catch (Exception e) {
throw new ParsingException("Could not get subscriber count", e);
}
@ -86,14 +84,13 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override
public long getStreamCount() throws ParsingException {
try {
final JsonObject videoCountObject = channelInfoItem.getObject("videoCountText");
if (videoCountObject == null) {
if (!channelInfoItem.has("videoCountText")) {
// Video count is not available, channel probably has no public uploads.
return -1;
}
return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(videoCountObject)));
return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(
channelInfoItem.getObject("videoCountText"))));
} catch (Exception e) {
throw new ParsingException("Could not get stream count", e);
}
@ -102,14 +99,12 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override
public String getDescription() throws ParsingException {
try {
final JsonObject descriptionObject = channelInfoItem.getObject("descriptionSnippet");
if (descriptionObject == null) {
if (!channelInfoItem.has("descriptionSnippet")) {
// Channel have no description.
return null;
}
return getTextFromObject(descriptionObject);
return getTextFromObject(channelInfoItem.getObject("descriptionSnippet"));
} catch (Exception e) {
throw new ParsingException("Could not get description", e);
}

38
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
View File

@ -37,6 +37,7 @@ import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeS
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
public class YoutubeMusicSearchExtractor extends SearchExtractor {
private JsonObject initialData;
@ -128,14 +129,10 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override
public String getSearchSuggestion() throws ParsingException {
final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer");
if (itemSectionRenderer == null) {
return "";
}
final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents")
.getObject(0).getObject("didYouMeanRenderer");
if (didYouMeanRenderer == null) {
final JsonObject didYouMeanRenderer = initialData.getObject("contents").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("didYouMeanRenderer");
if (!didYouMeanRenderer.has("correctedQuery")) {
return "";
}
return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery"));
@ -149,7 +146,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents");
for (Object content : contents) {
if (((JsonObject) content).getObject("musicShelfRenderer") != null) {
if (((JsonObject) content).has("musicShelfRenderer")) {
collectMusicStreamsFrom(collector, ((JsonObject) content).getObject("musicShelfRenderer").getArray("contents"));
}
}
@ -162,7 +159,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents");
for (Object content : contents) {
if (((JsonObject) content).getObject("musicShelfRenderer") != null) {
if (((JsonObject) content).has("musicShelfRenderer")) {
return getNextPageUrlFrom(((JsonObject) content).getObject("musicShelfRenderer").getArray("continuations"));
}
}
@ -224,10 +221,6 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
throw new ParsingException("Could not parse JSON", e);
}
if (ajaxJson.getObject("continuationContents") == null) {
return InfoItemsPage.emptyPage();
}
final JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents").getObject("musicShelfContinuation");
collectMusicStreamsFrom(collector, musicShelfContinuation.getArray("contents"));
@ -240,7 +233,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object item : videos) {
final JsonObject info = ((JsonObject) item).getObject("musicResponsiveListItemRenderer");
final JsonObject info = ((JsonObject) item).getObject("musicResponsiveListItemRenderer", null);
if (info != null) {
final String searchType = getLinkHandler().getContentFilters().get(0);
if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) {
@ -290,7 +283,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
JsonArray items = info.getObject("menu").getObject("menuRenderer").getArray("items");
for (Object item : items) {
final JsonObject menuNavigationItemRenderer = ((JsonObject) item).getObject("menuNavigationItemRenderer");
if (menuNavigationItemRenderer != null && menuNavigationItemRenderer.getObject("icon").getString("iconType").equals("ARTIST")) {
if (menuNavigationItemRenderer.getObject("icon").getString("iconType", EMPTY_STRING).equals("ARTIST")) {
return getUrlFromNavigationEndpoint(menuNavigationItemRenderer.getObject("navigationEndpoint"));
}
}
@ -301,16 +294,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
.getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer")
.getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint");
if (navigationEndpoint == null) {
return null;
}
final String url = getUrlFromNavigationEndpoint(navigationEndpoint);
if (url != null && !url.isEmpty()) {
return url;
}
throw new ParsingException("Could not get uploader url");
return getUrlFromNavigationEndpoint(navigationEndpoint);
}
}
@ -480,7 +464,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
}
private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException, IOException, ReCaptchaException {
if (continuations == null) {
if (continuations == null || continuations.isEmpty()) {
return "";
}

61
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -47,23 +47,16 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
private JsonObject getUploaderInfo() throws ParsingException {
JsonArray items = initialData.getObject("sidebar").getObject("playlistSidebarRenderer").getArray("items");
try {
JsonObject uploaderInfo = items.getObject(1).getObject("playlistSidebarSecondaryInfoRenderer")
.getObject("videoOwner").getObject("videoOwnerRenderer");
if (uploaderInfo != null) {
return uploaderInfo;
}
} catch (Exception ignored) {}
JsonObject videoOwner = items.getObject(1).getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner");
if (videoOwner.has("videoOwnerRenderer")) {
return videoOwner.getObject("videoOwnerRenderer");
}
// we might want to create a loop here instead of using duplicated code
try {
JsonObject uploaderInfo = items.getObject(items.size()).getObject("playlistSidebarSecondaryInfoRenderer")
.getObject("videoOwner").getObject("videoOwnerRenderer");
if (uploaderInfo != null) {
return uploaderInfo;
}
} catch (Exception e) {
throw new ParsingException("Could not get uploader info", e);
videoOwner = items.getObject(items.size()).getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner");
if (videoOwner.has("videoOwnerRenderer")) {
return videoOwner.getObject("videoOwnerRenderer");
}
throw new ParsingException("Could not get uploader info");
}
@ -89,33 +82,22 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Nonnull
@Override
public String getName() throws ParsingException {
try {
String name = getTextFromObject(playlistInfo.getObject("title"));
if (name != null) return name;
} catch (Exception ignored) {}
try {
return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title");
} catch (Exception e) {
throw new ParsingException("Could not get playlist name", e);
}
String name = getTextFromObject(playlistInfo.getObject("title"));
if (name == null || !name.isEmpty()) return name;
return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title");
}
@Override
public String getThumbnailUrl() throws ParsingException {
String url = null;
String url = playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer")
.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
try {
url = playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer")
.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
} catch (Exception ignored) {}
if (url == null || url.isEmpty()) {
url = initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail")
.getArray("thumbnails").getObject(0).getString("url");
if (url == null) {
try {
url = initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail")
.getArray("thumbnails").getObject(0).getString("url");
} catch (Exception ignored) {}
if (url == null) throw new ParsingException("Could not get playlist thumbnail");
if (url == null || url.isEmpty()) throw new ParsingException("Could not get playlist thumbnail");
}
return fixThumbnailUrl(url);
@ -123,8 +105,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getBannerUrl() {
return ""; // Banner can't be handled by frontend right now.
// Banner can't be handled by frontend right now.
// Whoever is willing to implement this should also implement it in the frontend.
return "";
}
@Override
@ -199,7 +182,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
private String getNextPageUrlFrom(JsonArray continuations) {
if (continuations == null) {
if (continuations == null || continuations.isEmpty()) {
return "";
}
@ -216,7 +199,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object video : videos) {
if (((JsonObject) video).getObject("playlistVideoRenderer") != null) {
if (((JsonObject) video).has("playlistVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) video).getObject("playlistVideoRenderer"), timeAgoParser) {
@Override
public long getViewCount() {

12
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -69,7 +69,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
.getObject("sectionListRenderer").getArray("contents").getObject(0)
.getObject("itemSectionRenderer").getArray("contents").getObject(0)
.getObject("showingResultsForRenderer");
if (showingResultsForRenderer == null) {
if (!showingResultsForRenderer.has("correctedQuery")) {
return "";
}
return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery"));
@ -119,21 +119,21 @@ public class YoutubeSearchExtractor extends SearchExtractor {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object item : videos) {
if (((JsonObject) item).getObject("backgroundPromoRenderer") != null) {
if (((JsonObject) item).has("backgroundPromoRenderer")) {
throw new NothingFoundException(getTextFromObject(((JsonObject) item)
.getObject("backgroundPromoRenderer").getObject("bodyText")));
} else if (((JsonObject) item).getObject("videoRenderer") != null) {
} else if (((JsonObject) item).has("videoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) item).getObject("videoRenderer"), timeAgoParser));
} else if (((JsonObject) item).getObject("channelRenderer") != null) {
} else if (((JsonObject) item).has("channelRenderer")) {
collector.commit(new YoutubeChannelInfoItemExtractor(((JsonObject) item).getObject("channelRenderer")));
} else if (((JsonObject) item).getObject("playlistRenderer") != null) {
} else if (((JsonObject) item).has("playlistRenderer")) {
collector.commit(new YoutubePlaylistInfoItemExtractor(((JsonObject) item).getObject("playlistRenderer")));
}
}
}
private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException {
if (continuations == null) {
if (continuations == null || continuations.isEmpty()) {
return "";
}

145
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -57,6 +57,7 @@ import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeP
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
/*
* Created by Christian Schabesberger on 06.08.15.
@ -117,18 +118,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public String getName() throws ParsingException {
assertPageFetched();
String title = null;
String title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title"));
try {
title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title"));
} catch (Exception ignored) {}
if (title == null || title.isEmpty()) {
title = playerResponse.getObject("videoDetails").getString("title");
if (title == null) {
try {
title = playerResponse.getObject("videoDetails").getString("title");
} catch (Exception ignored) {}
if (title == null) throw new ParsingException("Could not get name");
if (title == null || title.isEmpty()) throw new ParsingException("Could not get name");
}
return title;
@ -140,35 +135,31 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return null;
}
try {
JsonObject micro = playerResponse.getObject("microformat").getObject("playerMicroformatRenderer");
if (micro.getString("uploadDate") != null && !micro.getString("uploadDate").isEmpty()) {
return micro.getString("uploadDate");
}
if (micro.getString("publishDate") != null && !micro.getString("publishDate").isEmpty()) {
return micro.getString("publishDate");
}
} catch (Exception ignored) {}
JsonObject micro = playerResponse.getObject("microformat").getObject("playerMicroformatRenderer");
if (micro.isString("uploadDate") && !micro.getString("uploadDate").isEmpty()) {
return micro.getString("uploadDate");
}
if (micro.isString("publishDate") && !micro.getString("publishDate").isEmpty()) {
return micro.getString("publishDate");
}
if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).startsWith("Premiered")) {
String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).substring(10);
try { // Premiered 20 hours ago
TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en"));
Calendar parsedTime = timeAgoParser.parse(time).date();
return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime());
} catch (Exception ignored) {}
try { // Premiered Feb 21, 2020
Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time);
return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime());
} catch (Exception ignored) {}
}
try {
if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).startsWith("Premiered")) {
String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).substring(10);
try { // Premiered 20 hours ago
TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en"));
Calendar parsedTime = timeAgoParser.parse(time).date();
return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime());
} catch (Exception ignored) {}
try { // Premiered Feb 21, 2020
Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time);
return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime());
} catch (Exception ignored) {}
}
} catch (Exception ignored) {}
try {
// TODO this parses English formatted dates only, we need a better approach to parse the textual date
// TODO: this parses English formatted dates only, we need a better approach to parse the textual date
Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse(
getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")));
return new SimpleDateFormat("yyyy-MM-dd").format(d);
@ -180,7 +171,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public DateWrapper getUploadDate() throws ParsingException {
final String textualUploadDate = getTextualUploadDate();
if (textualUploadDate == null) {
if (textualUploadDate == null || textualUploadDate.isEmpty()) {
return null;
}
@ -208,17 +199,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public Description getDescription() throws ParsingException {
assertPageFetched();
// description with more info on links
try {
String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true);
return new Description(description, Description.HTML);
} catch (Exception ignored) { }
String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true);
if (description != null && !description.isEmpty()) return new Description(description, Description.HTML);
// raw non-html description
try {
return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT);
} catch (Exception ignored) {
throw new ParsingException("Could not get description");
}
return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT);
}
@Override
@ -264,19 +249,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public long getViewCount() throws ParsingException {
assertPageFetched();
String views = null;
try {
views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount")
String views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount")
.getObject("videoViewCountRenderer").getObject("viewCount"));
} catch (Exception ignored) {}
if (views == null) {
try {
views = playerResponse.getObject("videoDetails").getString("viewCount");
} catch (Exception ignored) {}
if (views == null || views.isEmpty()) {
views = playerResponse.getObject("videoDetails").getString("viewCount");
if (views == null) throw new ParsingException("Could not get view count");
if (views == null || views.isEmpty()) throw new ParsingException("Could not get view count");
}
if (views.toLowerCase().contains("no views")) return 0;
@ -334,16 +313,16 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public String getUploaderUrl() throws ParsingException {
assertPageFetched();
try {
String uploaderUrl = getUrlFromNavigationEndpoint(getVideoSecondaryInfoRenderer()
.getObject("owner").getObject("videoOwnerRenderer").getObject("navigationEndpoint"));
if (uploaderUrl != null) return uploaderUrl;
} catch (Exception ignored) {}
try {
String uploaderId = playerResponse.getObject("videoDetails").getString("channelId");
if (uploaderId != null)
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + uploaderId);
} catch (Exception ignored) {}
if (uploaderUrl != null && !uploaderUrl.isEmpty()) return uploaderUrl;
String uploaderId = playerResponse.getObject("videoDetails").getString("channelId");
if (uploaderId != null && !uploaderId.isEmpty())
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + uploaderId);
throw new ParsingException("Could not get uploader url");
}
@ -351,19 +330,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public String getUploaderName() throws ParsingException {
assertPageFetched();
String uploaderName = null;
try {
uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner")
String uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner")
.getObject("videoOwnerRenderer").getObject("title"));
} catch (Exception ignored) {}
if (uploaderName == null) {
try {
uploaderName = playerResponse.getObject("videoDetails").getString("author");
} catch (Exception ignored) {}
if (uploaderName == null || uploaderName.isEmpty()) {
uploaderName = playerResponse.getObject("videoDetails").getString("author");
if (uploaderName == null) throw new ParsingException("Could not get uploader name");
if (uploaderName == null || uploaderName.isEmpty()) throw new ParsingException("Could not get uploader name");
}
return uploaderName;
@ -392,7 +365,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (videoInfoPage.containsKey("dashmpd")) {
dashManifestUrl = videoInfoPage.get("dashmpd");
} else if (playerArgs != null && playerArgs.isString("dashmpd")) {
dashManifestUrl = playerArgs.getString("dashmpd", "");
dashManifestUrl = playerArgs.getString("dashmpd", EMPTY_STRING);
} else {
return "";
}
@ -561,9 +534,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object ul : results) {
final JsonObject videoInfo = ((JsonObject) ul).getObject("compactVideoRenderer");
if (videoInfo != null) collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser));
if (((JsonObject) ul).has("compactVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) ul).getObject("compactVideoRenderer"), timeAgoParser));
}
}
return collector;
} catch (Exception e) {
@ -612,7 +585,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final String playerUrl;
if (initialAjaxJson.getObject(2).getObject("response") != null) { // age-restricted videos
if (initialAjaxJson.getObject(2).has("response")) { // age-restricted videos
initialData = initialAjaxJson.getObject(2).getObject("response");
ageLimit = 18;
@ -631,7 +604,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
playerResponse = getPlayerResponse();
final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus", JsonUtils.EMPTY_OBJECT);
final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus");
final String status = playabilityStatus.getString("status");
// If status exist, and is not "OK", throw a ContentNotAvailableException with the reason.
if (status != null && !status.toLowerCase().equals("ok")) {
@ -808,10 +781,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
captions = playerResponse.getObject("captions");
final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer", new JsonObject());
final JsonArray captionsArray = renderer.getArray("captionTracks", new JsonArray());
final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer");
final JsonArray captionsArray = renderer.getArray("captionTracks");
// todo: use this to apply auto translation to different language from a source language
// final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages", new JsonArray());
// final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages");
// This check is necessary since there may be cases where subtitles metadata do not contain caption track info
// e.g. https://www.youtube.com/watch?v=-Vpwatutnko
@ -876,7 +849,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
JsonObject videoPrimaryInfoRenderer = null;
for (Object content : contents) {
if (((JsonObject) content).getObject("videoPrimaryInfoRenderer") != null) {
if (((JsonObject) content).has("videoPrimaryInfoRenderer")) {
videoPrimaryInfoRenderer = ((JsonObject) content).getObject("videoPrimaryInfoRenderer");
break;
}
@ -898,7 +871,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
JsonObject videoSecondaryInfoRenderer = null;
for (Object content : contents) {
if (((JsonObject) content).getObject("videoSecondaryInfoRenderer") != null) {
if (((JsonObject) content).has("videoSecondaryInfoRenderer")) {
videoSecondaryInfoRenderer = ((JsonObject) content).getObject("videoSecondaryInfoRenderer");
break;
}

121
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java
View File

@ -17,6 +17,7 @@ import java.util.Calendar;
import java.util.Date;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
/*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
@ -37,7 +38,6 @@ import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeP
*/
public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
private JsonObject videoInfo;
private final TimeAgoParser timeAgoParser;
private StreamType cachedStreamType;
@ -59,23 +59,18 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
return cachedStreamType;
}
try {
JsonArray badges = videoInfo.getArray("badges");
for (Object badge : badges) {
if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label").equals("LIVE NOW")) {
return cachedStreamType = StreamType.LIVE_STREAM;
}
}
} catch (Exception ignored) {}
try {
final String style = videoInfo.getArray("thumbnailOverlays").getObject(0)
.getObject("thumbnailOverlayTimeStatusRenderer").getString("style");
if (style.equalsIgnoreCase("LIVE")) {
final JsonArray badges = videoInfo.getArray("badges");
for (Object badge : badges) {
if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label", EMPTY_STRING).equals("LIVE NOW")) {
return cachedStreamType = StreamType.LIVE_STREAM;
}
} catch (Exception ignored) {}
}
final String style = videoInfo.getArray("thumbnailOverlays").getObject(0)
.getObject("thumbnailOverlayTimeStatusRenderer").getString("style", EMPTY_STRING);
if (style.equalsIgnoreCase("LIVE")) {
return cachedStreamType = StreamType.LIVE_STREAM;
}
return cachedStreamType = StreamType.VIDEO_STREAM;
}
@ -98,7 +93,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override
public String getName() throws ParsingException {
String name = getTextFromObject(videoInfo.getObject("title"));
if (name != null && !name.isEmpty()) return name;
if (name == null || !name.isEmpty()) return name;
throw new ParsingException("Could not get name");
}
@ -108,23 +103,17 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
return -1;
}
String duration = null;
String duration = getTextFromObject(videoInfo.getObject("lengthText"));
try {
duration = getTextFromObject(videoInfo.getObject("lengthText"));
} catch (Exception ignored) {}
if (duration == null) {
try {
for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) {
if (((JsonObject) thumbnailOverlay).getObject("thumbnailOverlayTimeStatusRenderer") != null) {
duration = getTextFromObject(((JsonObject) thumbnailOverlay)
.getObject("thumbnailOverlayTimeStatusRenderer").getObject("text"));
}
if (duration == null || duration.isEmpty()) {
for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) {
if (((JsonObject) thumbnailOverlay).has("thumbnailOverlayTimeStatusRenderer")) {
duration = getTextFromObject(((JsonObject) thumbnailOverlay)
.getObject("thumbnailOverlayTimeStatusRenderer").getObject("text"));
}
} catch (Exception ignored) {}
}
if (duration == null) throw new ParsingException("Could not get duration");
if (duration == null || duration.isEmpty()) throw new ParsingException("Could not get duration");
}
return YoutubeParsingHelper.parseDurationString(duration);
@ -132,23 +121,15 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override
public String getUploaderName() throws ParsingException {
String name = null;
String name = getTextFromObject(videoInfo.getObject("longBylineText"));
try {
name = getTextFromObject(videoInfo.getObject("longBylineText"));
} catch (Exception ignored) {}
if (name == null || name.isEmpty()) {
name = getTextFromObject(videoInfo.getObject("ownerText"));
if (name == null) {
try {
name = getTextFromObject(videoInfo.getObject("ownerText"));
} catch (Exception ignored) {}
if (name == null || name.isEmpty()) {
name = getTextFromObject(videoInfo.getObject("shortBylineText"));
if (name == null) {
try {
name = getTextFromObject(videoInfo.getObject("shortBylineText"));
} catch (Exception ignored) {}
if (name == null) throw new ParsingException("Could not get uploader name");
if (name == null || name.isEmpty()) throw new ParsingException("Could not get uploader name");
}
}
@ -157,26 +138,18 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override
public String getUploaderUrl() throws ParsingException {
String url = null;
String url = getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText")
.getArray("runs").getObject(0).getObject("navigationEndpoint"));
try {
url = getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText")
if (url == null || url.isEmpty()) {
url = getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText")
.getArray("runs").getObject(0).getObject("navigationEndpoint"));
} catch (Exception ignored) {}
if (url == null) {
try {
url = getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText")
if (url == null || url.isEmpty()) {
url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText")
.getArray("runs").getObject(0).getObject("navigationEndpoint"));
} catch (Exception ignored) {}
if (url == null) {
try {
url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText")
.getArray("runs").getObject(0).getObject("navigationEndpoint"));
} catch (Exception ignored) {}
if (url == null) throw new ParsingException("Could not get uploader url");
if (url == null || url.isEmpty()) throw new ParsingException("Could not get uploader url");
}
}
@ -195,12 +168,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
return new SimpleDateFormat("yyyy-MM-dd HH:mm").format(date);
}
try {
return getTextFromObject(videoInfo.getObject("publishedTimeText"));
} catch (Exception e) {
// upload date is not always available, e.g. in playlists
return null;
}
final String publishedTimeText = getTextFromObject(videoInfo.getObject("publishedTimeText"));
if (publishedTimeText != null && !publishedTimeText.isEmpty()) return publishedTimeText;
return null;
}
@Nullable
@ -228,17 +199,16 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override
public long getViewCount() throws ParsingException {
try {
if (videoInfo.getObject("topStandaloneBadge") != null || isPremium()) {
if (videoInfo.has("topStandaloneBadge") || isPremium()) {
return -1;
}
final JsonObject viewCountObject = videoInfo.getObject("viewCountText");
if (viewCountObject == null) {
if (!videoInfo.has("viewCountText")) {
// This object is null when a video has its views hidden.
return -1;
}
final String viewCount = getTextFromObject(viewCountObject);
final String viewCount = getTextFromObject(videoInfo.getObject("viewCountText"));
if (viewCount.toLowerCase().contains("no views")) {
return 0;
@ -266,14 +236,11 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
}
private boolean isPremium() {
try {
JsonArray badges = videoInfo.getArray("badges");
for (Object badge : badges) {
if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label").equals("Premium")) {
return true;
}
JsonArray badges = videoInfo.getArray("badges");
for (Object badge : badges) {
if (((JsonObject) badge).getObject("metadataBadgeRenderer").getString("label", EMPTY_STRING).equals("Premium")) {
return true;
}
} catch (Exception ignored) {
}
return false;
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java
View File

@ -59,7 +59,7 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
// trim JSONP part "JP(...)"
response = response.substring(3, response.length() - 1);
try {
JsonArray collection = JsonParser.array().from(response).getArray(1, new JsonArray());
JsonArray collection = JsonParser.array().from(response).getArray(1);
for (Object suggestion : collection) {
if (!(suggestion instanceof JsonArray)) continue;
String suggestionStr = ((JsonArray) suggestion).getString(0);

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View File

@ -97,14 +97,11 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
JsonObject expandedShelfContentsRenderer = ((JsonObject) itemSectionRenderer).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("shelfRenderer").getObject("content")
.getObject("expandedShelfContentsRenderer");
if (expandedShelfContentsRenderer != null) {
for (Object ul : expandedShelfContentsRenderer.getArray("items")) {
final JsonObject videoInfo = ((JsonObject) ul).getObject("videoRenderer");
collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser));
}
for (Object ul : expandedShelfContentsRenderer.getArray("items")) {
final JsonObject videoInfo = ((JsonObject) ul).getObject("videoRenderer");
collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser));
}
}
return new InfoItemsPage<>(collector, getNextPageUrl());
}
}

12
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java
View File

@ -334,7 +334,7 @@ public class YoutubeParsingHelper {
}
public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) throws ParsingException {
if (navigationEndpoint.getObject("urlEndpoint") != null) {
if (navigationEndpoint.has("urlEndpoint")) {
String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url");
if (internUrl.startsWith("/redirect?")) {
// q parameter can be the first parameter
@ -354,7 +354,7 @@ public class YoutubeParsingHelper {
} else if (internUrl.startsWith("http")) {
return internUrl;
}
} else if (navigationEndpoint.getObject("browseEndpoint") != null) {
} else if (navigationEndpoint.has("browseEndpoint")) {
final JsonObject browseEndpoint = navigationEndpoint.getObject("browseEndpoint");
final String canonicalBaseUrl = browseEndpoint.getString("canonicalBaseUrl");
final String browseId = browseEndpoint.getString("browseId");
@ -369,7 +369,7 @@ public class YoutubeParsingHelper {
}
throw new ParsingException("canonicalBaseUrl is null and browseId is not a channel (\"" + browseEndpoint + "\")");
} else if (navigationEndpoint.getObject("watchEndpoint") != null) {
} else if (navigationEndpoint.has("watchEndpoint")) {
StringBuilder url = new StringBuilder();
url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId"));
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId"))
@ -377,7 +377,7 @@ public class YoutubeParsingHelper {
if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds"))
url.append("&amp;t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds"));
return url.toString();
} else if (navigationEndpoint.getObject("watchPlaylistEndpoint") != null) {
} else if (navigationEndpoint.has("watchPlaylistEndpoint")) {
return "https://www.youtube.com/playlist?list=" +
navigationEndpoint.getObject("watchPlaylistEndpoint").getString("playlistId");
}
@ -390,7 +390,7 @@ public class YoutubeParsingHelper {
StringBuilder textBuilder = new StringBuilder();
for (Object textPart : textObject.getArray("runs")) {
String text = ((JsonObject) textPart).getString("text");
if (html && ((JsonObject) textPart).getObject("navigationEndpoint") != null) {
if (html && ((JsonObject) textPart).has("navigationEndpoint")) {
String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint"));
if (url != null && !url.isEmpty()) {
textBuilder.append("<a href=\"").append(url).append("\">").append(text).append("</a>");
@ -486,7 +486,7 @@ public class YoutubeParsingHelper {
*/
public static void defaultAlertsCheck(JsonObject initialData) throws ContentNotAvailableException {
final JsonArray alerts = initialData.getArray("alerts");
if (alerts != null && !alerts.isEmpty()) {
if (!alerts.isEmpty()) {
final JsonObject alertRenderer = alerts.getObject(0).getObject("alertRenderer");
final String alertText = alertRenderer.getObject("text").getString("simpleText");
final String alertType = alertRenderer.getString("type");

4
timeago-parser/build.gradle
View File

@ -1,6 +1,6 @@
dependencies {
testImplementation 'junit:junit:4.12'
implementation 'com.grack:nanojson:1.1'
implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751'
implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0'
}
}