From 94efe86c7147792db661613b1b5804b8ee5de8c4 Mon Sep 17 00:00:00 2001 From: FireMasterK <20838718+FireMasterK@users.noreply.github.com> Date: Wed, 22 Sep 2021 16:19:44 +0100 Subject: [PATCH] Extract Video Short Description in YouTube. In Trending, and Search results. --- .../BandcampRadioInfoItemExtractor.java | 6 + ...campDiscographStreamInfoItemExtractor.java | 6 + ...ndcampPlaylistStreamInfoItemExtractor.java | 6 + ...BandcampSearchStreamInfoItemExtractor.java | 6 + .../MediaCCCLiveStreamKioskExtractor.java | 6 + .../MediaCCCRecentKioskExtractor.java | 6 + .../MediaCCCStreamInfoItemExtractor.java | 6 + .../PeertubeStreamInfoItemExtractor.java | 6 + .../SoundcloudStreamInfoItemExtractor.java | 6 + .../YoutubeFeedInfoItemExtractor.java | 6 + .../YoutubeStreamInfoItemExtractor.java | 13 + .../extractor/stream/StreamInfoItem.java | 9 + .../stream/StreamInfoItemExtractor.java | 10 + .../stream/StreamInfoItemsCollector.java | 5 + .../search/YoutubeSearchExtractorTest.java | 30 + .../video_description/generated_mock_0.json | 226 +++++ .../video_description/generated_mock_1.json | 259 ++++++ .../video_description/generated_mock_2.json | 786 ++++++++++++++++++ 18 files changed, 1398 insertions(+) create mode 100644 extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/search/video_description/generated_mock_0.json create mode 100644 extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/search/video_description/generated_mock_1.json create mode 100644 extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/search/video_description/generated_mock_2.json diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java index cca802e95..3699921be 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java @@ -43,6 +43,12 @@ public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor { return BandcampExtractorHelper.parseDate(getTextualUploadDate()); } + @Nullable + @Override + public String getShortDescription() { + return null; + } + @Override public String getName() throws ParsingException { return show.getString("subtitle"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java index 6a2549989..cb8db136b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java @@ -26,6 +26,12 @@ public class BandcampDiscographStreamInfoItemExtractor extends BandcampStreamInf return null; } + @Nullable + @Override + public String getShortDescription() { + return null; + } + @Override public String getName() { return discograph.getString("title"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java index c28c24ba2..922d95bd3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java @@ -60,6 +60,12 @@ public class BandcampPlaylistStreamInfoItemExtractor extends BandcampStreamInfoI return null; } + @Nullable + @Override + public String getShortDescription() { + return null; + } + /** * Each track can have its own cover art. Therefore, unless a substitute is provided, * the thumbnail is extracted using a stream extractor. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java index 87ba69d11..4288f2d91 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java @@ -32,6 +32,12 @@ public class BandcampSearchStreamInfoItemExtractor extends BandcampStreamInfoIte return null; } + @Nullable + @Override + public String getShortDescription() { + return null; + } + @Override public String getName() throws ParsingException { return resultInfo.getElementsByClass("heading").text(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java index e04be76f9..794531269 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java @@ -95,4 +95,10 @@ public class MediaCCCLiveStreamKioskExtractor implements StreamInfoItemExtractor public DateWrapper getUploadDate() throws ParsingException { return null; } + + @Nullable + @Override + public String getShortDescription() { + return null; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java index 4f272898c..8c9a8c736 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java @@ -92,4 +92,10 @@ public class MediaCCCRecentKioskExtractor implements StreamInfoItemExtractor { DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSzzzz")); return new DateWrapper(zonedDateTime.toOffsetDateTime(), false); } + + @Nullable + @Override + public String getShortDescription() { + return null; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java index 0f03788a4..cecaa15e9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java @@ -73,6 +73,12 @@ public class MediaCCCStreamInfoItemExtractor implements StreamInfoItemExtractor return new DateWrapper(MediaCCCParsingHelper.parseDateFrom(date)); } + @Nullable + @Override + public String getShortDescription() { + return null; + } + @Override public String getName() throws ParsingException { return event.getString("title"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java index b9c7f4b13..e2992915f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java @@ -92,6 +92,12 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); } + @Nullable + @Override + public String getShortDescription() { + return null; + } + @Override public StreamType getStreamType() { return item.getBoolean("isLive") ? StreamType.LIVE_STREAM : StreamType.VIDEO_STREAM; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java index 3647ff09e..f95eb9d92 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java @@ -66,6 +66,12 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(getTextualUploadDate())); } + @Nullable + @Override + public String getShortDescription() { + return null; + } + @Override public long getViewCount() { return itemObject.getLong("playback_count"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java index a0bd8917b..be135be02 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java @@ -77,6 +77,12 @@ public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor { } } + @Nullable + @Override + public String getShortDescription() { + return null; + } + @Override public String getName() { return entryElement.getElementsByTag("title").first().text(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index 8a4ecf05f..5c903aef7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -288,4 +288,17 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { throw new ParsingException("Could not parse date from premiere: \"" + startTime + "\""); } } + + @Nullable + @Override + public String getShortDescription() throws ParsingException { + + if(videoInfo.has("detailedMetadataSnippets")) + return getTextFromObject(videoInfo.getArray("detailedMetadataSnippets").getObject(0).getObject("snippetText")); + + if(videoInfo.has("descriptionSnippet")) + return getTextFromObject(videoInfo.getObject("descriptionSnippet")); + + return null; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java index 8f2d42437..d974ee2c0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java @@ -32,6 +32,7 @@ public class StreamInfoItem extends InfoItem { private final StreamType streamType; private String uploaderName; + private String shortDescription; private String textualUploadDate; @Nullable private DateWrapper uploadDate; @@ -92,6 +93,14 @@ public class StreamInfoItem extends InfoItem { this.uploaderAvatarUrl = uploaderAvatarUrl; } + public String getShortDescription() { + return shortDescription; + } + + public void setShortDescription(String shortDescription) { + this.shortDescription = shortDescription; + } + @Nullable public String getTextualUploadDate() { return textualUploadDate; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java index 6d61ee441..4a996dc04 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java @@ -116,4 +116,14 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor { @Nullable DateWrapper getUploadDate() throws ParsingException; + + /** + * Get the video's short description. + * + * @return The video's short description or {@code null} if not provided by the service. + * @throws ParsingException if there is an error in the extraction + */ + @Nullable + String getShortDescription() throws ParsingException; + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java index ae00a1b7f..01909c26e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java @@ -101,6 +101,11 @@ public class StreamInfoItemsCollector extends InfoItemsCollector