From 2fb9922a15da62916085d1ac854edd4af61dadbf Mon Sep 17 00:00:00 2001 From: TobiGr Date: Fri, 14 Jul 2023 17:43:09 +0200 Subject: [PATCH] [SoundCloud] Detect whether there are any more search results Add test for this edge case. --- .../extractors/SoundcloudSearchExtractor.java | 48 ++++++++++++++----- .../search/SoundcloudSearchExtractorTest.java | 24 ++++++++++ 2 files changed, 60 insertions(+), 12 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java index 5ec7fb72d..c0d1e1bf7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java @@ -33,7 +33,9 @@ import java.util.function.IntUnaryOperator; import javax.annotation.Nonnull; public class SoundcloudSearchExtractor extends SearchExtractor { - private JsonArray initialSearchCollection; + private JsonObject initialSearchObject; + private static final String COLLECTION = "collection"; + private static final String TOTAL_RESULTS = "total_results"; public SoundcloudSearchExtractor(final StreamingService service, final SearchQueryHandler linkHandler) { @@ -60,9 +62,15 @@ public class SoundcloudSearchExtractor extends SearchExtractor { @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - return new InfoItemsPage<>( - collectItems(initialSearchCollection), - getNextPageFromCurrentUrl(getUrl(), currentOffset -> ITEMS_PER_PAGE)); + if (initialSearchObject.getInt(TOTAL_RESULTS) > ITEMS_PER_PAGE) { + return new InfoItemsPage<>( + collectItems(initialSearchObject.getArray(COLLECTION)), + getNextPageFromCurrentUrl(getUrl(), currentOffset -> ITEMS_PER_PAGE)); + } else { + return new InfoItemsPage<>( + collectItems(initialSearchObject.getArray(COLLECTION)), null); + } + } @Override @@ -74,17 +82,29 @@ public class SoundcloudSearchExtractor extends SearchExtractor { final Downloader dl = getDownloader(); final JsonArray searchCollection; + final int totalResults; try { final String response = dl.get(page.getUrl(), getExtractorLocalization()) .responseBody(); - searchCollection = JsonParser.object().from(response).getArray("collection"); + final JsonObject result = JsonParser.object().from(response); + searchCollection = result.getArray(COLLECTION); + totalResults = result.getInt(TOTAL_RESULTS); } catch (final JsonParserException e) { throw new ParsingException("Could not parse json response", e); } + final boolean hasNextPage; + try { + hasNextPage = getOffsetFromUrl(page.getUrl()) + ITEMS_PER_PAGE <= totalResults; + } catch (MalformedURLException | UnsupportedEncodingException e) { + throw new ParsingException("Could not get offset from page URL", e); + } + if (hasNextPage) { + return new InfoItemsPage<>(collectItems(searchCollection), + getNextPageFromCurrentUrl(page.getUrl(), + currentOffset -> currentOffset + ITEMS_PER_PAGE)); + } + return new InfoItemsPage<>(collectItems(searchCollection), null); - return new InfoItemsPage<>(collectItems(searchCollection), - getNextPageFromCurrentUrl(page.getUrl(), - currentOffset -> currentOffset + ITEMS_PER_PAGE)); } @Override @@ -94,12 +114,12 @@ public class SoundcloudSearchExtractor extends SearchExtractor { final String url = getUrl(); try { final String response = dl.get(url, getExtractorLocalization()).responseBody(); - initialSearchCollection = JsonParser.object().from(response).getArray("collection"); + initialSearchObject = JsonParser.object().from(response); } catch (final JsonParserException e) { throw new ParsingException("Could not parse json response", e); } - if (initialSearchCollection.isEmpty()) { + if (initialSearchObject.getArray(COLLECTION).isEmpty()) { throw new SearchExtractor.NothingFoundException("Nothing found"); } } @@ -134,12 +154,16 @@ public class SoundcloudSearchExtractor extends SearchExtractor { private Page getNextPageFromCurrentUrl(final String currentUrl, final IntUnaryOperator newPageOffsetCalculator) throws MalformedURLException, UnsupportedEncodingException { - final int currentPageOffset = Integer.parseInt( - Parser.compatParseMap(new URL(currentUrl).getQuery()).get("offset")); + final int currentPageOffset = getOffsetFromUrl(currentUrl); return new Page( currentUrl.replace( "&offset=" + currentPageOffset, "&offset=" + newPageOffsetCalculator.applyAsInt(currentPageOffset))); } + + private int getOffsetFromUrl(final String url) + throws MalformedURLException, UnsupportedEncodingException { + return Integer.parseInt(Parser.compatParseMap(new URL(url).getQuery()).get("offset")); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java index c79a4505e..173cf1b0b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.soundcloud.search; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems; @@ -181,4 +182,27 @@ public class SoundcloudSearchExtractorTest { assertTrue(verified); } } + + public static class NoNextPage extends DefaultSearchExtractorTest { + + private static SearchExtractor extractor; + private static final String QUERY = "Dan at hor#berlgbd"; + + @BeforeAll + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = SoundCloud.getSearchExtractor(QUERY); + extractor.fetchPage(); + } + + @Override public boolean expectedHasMoreItems() { return false; } + @Override public SearchExtractor extractor() throws Exception { return extractor; } + @Override public StreamingService expectedService() throws Exception { return SoundCloud; } + @Override public String expectedName() throws Exception { return QUERY; } + @Override public String expectedId() throws Exception { return QUERY; } + @Override public String expectedUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); } + @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + } }