lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add support to extract total comment count.

This commit is contained in:
FireMasterK 2022-02-08 10:44:55 +00:00 committed by Kavin
parent 45636b0d00
commit 981aee4092
No known key found for this signature in database
GPG Key ID: 49451E4482CC5BCD
4 changed files with 59 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java
View File

@ -22,6 +22,13 @@ public abstract class CommentsExtractor extends ListExtractor<CommentsInfoItem>
return false;
}
/**
* @return total number of comments.
*/
public int getCommentsCount() throws ExtractionException {
return -1;
}
@Nonnull
@Override
public String getName() throws ParsingException {

26
extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java
View File

@ -48,6 +48,11 @@ public final class CommentsInfo extends ListInfo<CommentsInfoItem> {
ExtractorHelper.getItemsPageOrLogError(commentsInfo, commentsExtractor);
commentsInfo.setCommentsDisabled(commentsExtractor.isCommentsDisabled());
commentsInfo.setRelatedItems(initialCommentsPage.getItems());
try {
commentsInfo.setCommentsCount(commentsExtractor.getCommentsCount());
} catch (Exception e) {
commentsInfo.addError(e);
}
commentsInfo.setNextPage(initialCommentsPage.getNextPage());
return commentsInfo;
@ -76,6 +81,7 @@ public final class CommentsInfo extends ListInfo<CommentsInfoItem> {
private transient CommentsExtractor commentsExtractor;
private boolean commentsDisabled = false;
private int commentsCount;
public CommentsExtractor getCommentsExtractor() {
return commentsExtractor;
@ -86,6 +92,7 @@ public final class CommentsInfo extends ListInfo<CommentsInfoItem> {
}
/**
* @return <code>true</code> if the comments are disabled otherwise <code>false</code> (default)
* @apiNote Warning: This method is experimental and may get removed in a future release.
* @return {@code true} if the comments are disabled otherwise {@code false} (default)
* @see CommentsExtractor#isCommentsDisabled()
@ -95,10 +102,29 @@ public final class CommentsInfo extends ListInfo<CommentsInfoItem> {
}
/**
* @param commentsDisabled <code>true</code> if the comments are disabled otherwise <code>false</code>
* @apiNote Warning: This method is experimental and may get removed in a future release.
* @param commentsDisabled {@code true} if the comments are disabled otherwise {@code false}
*/
public void setCommentsDisabled(final boolean commentsDisabled) {
this.commentsDisabled = commentsDisabled;
}
/**
* Returns the total number of comments.
*
* @return totalComments
*/
public int getCommentsCount() {
return commentsCount;
}
/**
* Sets the total number of comments.
*
* @param commentsCount
*/
public void setCommentsCount(int commentsCount) {
this.commentsCount = commentsCount;
}
}

23
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java
View File

@ -1,6 +1,7 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -28,6 +29,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.utils.Utils;
public class YoutubeCommentsExtractor extends CommentsExtractor {
@ -44,6 +46,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
*/
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
private Optional<Boolean> optCommentsDisabled = Optional.empty();
private JsonObject ajaxJson;
public YoutubeCommentsExtractor(
final StreamingService service,
@ -187,16 +190,15 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
.done())
.getBytes(StandardCharsets.UTF_8);
final JsonObject ajaxJson = getJsonPostResponse("next", body, localization);
this.ajaxJson = getJsonPostResponse("next", body, localization);
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(
getServiceId());
collectCommentsFrom(collector, ajaxJson);
collectCommentsFrom(collector);
return new InfoItemsPage<>(collector, getNextPage(ajaxJson));
}
private void collectCommentsFrom(final CommentsInfoItemsCollector collector,
@Nonnull final JsonObject ajaxJson) throws ParsingException {
private void collectCommentsFrom(final CommentsInfoItemsCollector collector) throws ParsingException {
final JsonArray onResponseReceivedEndpoints =
ajaxJson.getArray("onResponseReceivedEndpoints");
@ -274,4 +276,17 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
return optCommentsDisabled.get();
}
@Override
public int getCommentsCount() throws ExtractionException {
final JsonObject commentsHeaderRenderer = ajaxJson
.getArray("onResponseReceivedEndpoints").getObject(0)
.getObject("reloadContinuationItemsCommand")
.getArray("continuationItems").getObject(0)
.getObject("commentsHeaderRenderer");
final String text = getTextFromObject(commentsHeaderRenderer.getObject("countText"));
return Integer.parseInt(Utils.removeNonDigitCharacters(text));
}
}

7
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java
View File

@ -89,6 +89,7 @@ public class YoutubeCommentsExtractorTest {
@Test
public void testGetCommentsAllData() throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
assertTrue(extractor.getCommentsCount() > 5); // at least 5 comments
DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors());
for (CommentsInfoItem c : comments.getItems()) {
@ -344,6 +345,12 @@ public class YoutubeCommentsExtractorTest {
assertNotEquals(UNKNOWN_REPLY_COUNT, firstComment.getReplyCount(), "Could not get the reply count of the first comment");
assertGreater(300, firstComment.getReplyCount());
}
@Test
public void testCommentsCount() throws IOException, ExtractionException {
extractor.getInitialPage(); // Needs to be called first
assertTrue(extractor.getCommentsCount() > 18800);
}
}
public static class FormattingTest {