From efe2b964f00f27a29b1c942eda0770ebb1cd3c6e Mon Sep 17 00:00:00 2001 From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com> Date: Fri, 5 Mar 2021 19:12:06 +0100 Subject: [PATCH 1/4] Extract real subscribers of a Peertube account Apply fix provided in the PR and move all accounts/ strings to a constant, ACCOUNTS. --- .../extractors/PeertubeAccountExtractor.java | 32 ++++++++++++++++--- .../PeertubeAccountExtractorTest.java | 2 +- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java index 282e6d9cc..831a99880 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.peertube.extractors; +import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; @@ -10,6 +11,7 @@ import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory; @@ -27,6 +29,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class PeertubeAccountExtractor extends ChannelExtractor { private JsonObject json; private final String baseUrl; + private static final String ACCOUNTS = "accounts/"; public PeertubeAccountExtractor(final StreamingService service, final ListLinkHandler linkHandler) throws ParsingException { super(service, linkHandler); @@ -55,8 +58,29 @@ public class PeertubeAccountExtractor extends ChannelExtractor { } @Override - public long getSubscriberCount() { - return json.getLong("followersCount"); + public long getSubscriberCount() throws ParsingException { + long subscribersCount = json.getLong("followersCount"); + String accountVideoChannelUrl = baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT; + if (getId().contains(ACCOUNTS)) { + accountVideoChannelUrl += getId(); + } else { + accountVideoChannelUrl += ACCOUNTS + getId(); + } + accountVideoChannelUrl += "/video-channels"; + + try { + final String responseBody = getDownloader().get(accountVideoChannelUrl).responseBody(); + final JsonObject jsonResponse = JsonParser.object().from(responseBody); + final JsonArray videoChannels = jsonResponse.getArray("data"); + for (final Object videoChannel : videoChannels) { + final JsonObject videoChannelJsonObject = (JsonObject) videoChannel; + subscribersCount += videoChannelJsonObject.getInt("followersCount"); + } + } catch (final IOException | JsonParserException | ReCaptchaException ignored) { + // something went wrong during video channels extraction, only return subscribers of ownerAccount + } + System.out.println(subscribersCount); + return subscribersCount; } @Override @@ -130,10 +154,10 @@ public class PeertubeAccountExtractor extends ChannelExtractor { public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { String accountUrl = baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT; - if (getId().contains("accounts/")) { + if (getId().contains(ACCOUNTS)) { accountUrl += getId(); } else { - accountUrl += "accounts/" + getId(); + accountUrl += ACCOUNTS + getId(); } final Response response = downloader.get(accountUrl); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java index 2bff01042..cc3cf7162 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java @@ -102,7 +102,7 @@ public class PeertubeAccountExtractorTest { @Test public void testSubscriberCount() throws ParsingException { - assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 500); + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 425); } @Override From 772de53a663c2deee4264b1e088b70138797f704 Mon Sep 17 00:00:00 2001 From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com> Date: Sun, 14 Mar 2021 18:41:07 +0100 Subject: [PATCH 2/4] Increase the number of subscribers in PeertubeAccountExtractorTest --- .../services/peertube/PeertubeAccountExtractorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java index cc3cf7162..92cf0b8c8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java @@ -102,7 +102,7 @@ public class PeertubeAccountExtractorTest { @Test public void testSubscriberCount() throws ParsingException { - assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 425); + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 750); } @Override From abb790f46563bef95d0c82b60cdfca9e76d1ce7c Mon Sep 17 00:00:00 2001 From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com> Date: Thu, 18 Mar 2021 19:02:25 +0100 Subject: [PATCH 3/4] Remove println in getSubscriberCount from PeertubeAccountExtractor class --- .../services/peertube/extractors/PeertubeAccountExtractor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java index 831a99880..73dda98c2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java @@ -79,7 +79,6 @@ public class PeertubeAccountExtractor extends ChannelExtractor { } catch (final IOException | JsonParserException | ReCaptchaException ignored) { // something went wrong during video channels extraction, only return subscribers of ownerAccount } - System.out.println(subscribersCount); return subscribersCount; } From ae283314dabbbd742a289e81519e7d6d1d05baaa Mon Sep 17 00:00:00 2001 From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com> Date: Thu, 25 Mar 2021 17:54:46 +0100 Subject: [PATCH 4/4] Add a comment explaining subscribers count calculation --- .../services/peertube/extractors/PeertubeAccountExtractor.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java index 73dda98c2..c8d6258c8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java @@ -59,6 +59,9 @@ public class PeertubeAccountExtractor extends ChannelExtractor { @Override public long getSubscriberCount() throws ParsingException { + // The subscriber count cannot be retrieved directly. It needs to be calculated. + // An accounts subscriber count is the number of the channel owner's subscriptions + // plus the sum of all sub channels subscriptions. long subscribersCount = json.getLong("followersCount"); String accountVideoChannelUrl = baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT; if (getId().contains(ACCOUNTS)) {