From 981aee40927ae03442f16856185ac5726ea836c8 Mon Sep 17 00:00:00 2001 From: FireMasterK <20838718+FireMasterK@users.noreply.github.com> Date: Tue, 8 Feb 2022 10:44:55 +0000 Subject: [PATCH] Add support to extract total comment count. --- .../extractor/comments/CommentsExtractor.java | 7 +++++ .../extractor/comments/CommentsInfo.java | 26 +++++++++++++++++++ .../extractors/YoutubeCommentsExtractor.java | 23 +++++++++++++--- .../youtube/YoutubeCommentsExtractorTest.java | 7 +++++ 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java index ac4792fc0..5f34ffbfd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java @@ -22,6 +22,13 @@ public abstract class CommentsExtractor extends ListExtractor return false; } + /** + * @return total number of comments. + */ + public int getCommentsCount() throws ExtractionException { + return -1; + } + @Nonnull @Override public String getName() throws ParsingException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java index 98ec136e5..73dfa8ab2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java @@ -48,6 +48,11 @@ public final class CommentsInfo extends ListInfo { ExtractorHelper.getItemsPageOrLogError(commentsInfo, commentsExtractor); commentsInfo.setCommentsDisabled(commentsExtractor.isCommentsDisabled()); commentsInfo.setRelatedItems(initialCommentsPage.getItems()); + try { + commentsInfo.setCommentsCount(commentsExtractor.getCommentsCount()); + } catch (Exception e) { + commentsInfo.addError(e); + } commentsInfo.setNextPage(initialCommentsPage.getNextPage()); return commentsInfo; @@ -76,6 +81,7 @@ public final class CommentsInfo extends ListInfo { private transient CommentsExtractor commentsExtractor; private boolean commentsDisabled = false; + private int commentsCount; public CommentsExtractor getCommentsExtractor() { return commentsExtractor; @@ -86,6 +92,7 @@ public final class CommentsInfo extends ListInfo { } /** + * @return true if the comments are disabled otherwise false (default) * @apiNote Warning: This method is experimental and may get removed in a future release. * @return {@code true} if the comments are disabled otherwise {@code false} (default) * @see CommentsExtractor#isCommentsDisabled() @@ -95,10 +102,29 @@ public final class CommentsInfo extends ListInfo { } /** + * @param commentsDisabled true if the comments are disabled otherwise false * @apiNote Warning: This method is experimental and may get removed in a future release. * @param commentsDisabled {@code true} if the comments are disabled otherwise {@code false} */ public void setCommentsDisabled(final boolean commentsDisabled) { this.commentsDisabled = commentsDisabled; } + + /** + * Returns the total number of comments. + * + * @return totalComments + */ + public int getCommentsCount() { + return commentsCount; + } + + /** + * Sets the total number of comments. + * + * @param commentsCount + */ + public void setCommentsCount(int commentsCount) { + this.commentsCount = commentsCount; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index e24819526..6d0c38cb4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -28,6 +29,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonWriter; +import org.schabi.newpipe.extractor.utils.Utils; public class YoutubeCommentsExtractor extends CommentsExtractor { @@ -44,6 +46,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { */ @SuppressWarnings("OptionalUsedAsFieldOrParameterType") private Optional optCommentsDisabled = Optional.empty(); + private JsonObject ajaxJson; public YoutubeCommentsExtractor( final StreamingService service, @@ -187,16 +190,15 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { .done()) .getBytes(StandardCharsets.UTF_8); - final JsonObject ajaxJson = getJsonPostResponse("next", body, localization); + this.ajaxJson = getJsonPostResponse("next", body, localization); final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector( getServiceId()); - collectCommentsFrom(collector, ajaxJson); + collectCommentsFrom(collector); return new InfoItemsPage<>(collector, getNextPage(ajaxJson)); } - private void collectCommentsFrom(final CommentsInfoItemsCollector collector, - @Nonnull final JsonObject ajaxJson) throws ParsingException { + private void collectCommentsFrom(final CommentsInfoItemsCollector collector) throws ParsingException { final JsonArray onResponseReceivedEndpoints = ajaxJson.getArray("onResponseReceivedEndpoints"); @@ -274,4 +276,17 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { return optCommentsDisabled.get(); } + + @Override + public int getCommentsCount() throws ExtractionException { + final JsonObject commentsHeaderRenderer = ajaxJson + .getArray("onResponseReceivedEndpoints").getObject(0) + .getObject("reloadContinuationItemsCommand") + .getArray("continuationItems").getObject(0) + .getObject("commentsHeaderRenderer"); + + final String text = getTextFromObject(commentsHeaderRenderer.getObject("countText")); + + return Integer.parseInt(Utils.removeNonDigitCharacters(text)); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java index 7f7a85512..951fe9584 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java @@ -89,6 +89,7 @@ public class YoutubeCommentsExtractorTest { @Test public void testGetCommentsAllData() throws IOException, ExtractionException { InfoItemsPage comments = extractor.getInitialPage(); + assertTrue(extractor.getCommentsCount() > 5); // at least 5 comments DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors()); for (CommentsInfoItem c : comments.getItems()) { @@ -344,6 +345,12 @@ public class YoutubeCommentsExtractorTest { assertNotEquals(UNKNOWN_REPLY_COUNT, firstComment.getReplyCount(), "Could not get the reply count of the first comment"); assertGreater(300, firstComment.getReplyCount()); } + + @Test + public void testCommentsCount() throws IOException, ExtractionException { + extractor.getInitialPage(); // Needs to be called first + assertTrue(extractor.getCommentsCount() > 18800); + } } public static class FormattingTest {