diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java index a7efea962..0101b96f0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java @@ -37,4 +37,15 @@ public abstract class SearchExtractor extends ListExtractor { public String getName() { return getLinkHandler().getSearchString(); } + + /** + * When you search on some service, it can give you another and corrected request. + * This method says if it's the case. + *

+ * Example: on YouTube, if you search for "pewdeipie", + * it will give you results for "pewdiepie", then isCorrectedSearch should return true. + * + * @return whether the results comes from a corrected query or not. + */ + public abstract boolean isCorrectedSearch() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java index eedce719e..547162bd1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java @@ -15,6 +15,7 @@ public class SearchInfo extends ListInfo { private String searchString; private String searchSuggestion; + private boolean isCorrectedSearch; public SearchInfo(int serviceId, SearchQueryHandler qIHandler, @@ -46,6 +47,11 @@ public class SearchInfo extends ListInfo { } catch (Exception e) { info.addError(e); } + try { + info.isCorrectedSearch = extractor.isCorrectedSearch(); + } catch (Exception e) { + info.addError(e); + } ListExtractor.InfoItemsPage page = ExtractorHelper.getItemsPageOrLogError(info, extractor); info.setRelatedItems(page.getItems()); @@ -70,4 +76,8 @@ public class SearchInfo extends ListInfo { public String getSearchSuggestion() { return searchSuggestion; } + + public boolean isCorrectedSearch() { + return this.isCorrectedSearch; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index d5ced534b..67f19a838 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -47,6 +47,11 @@ public class MediaCCCSearchExtractor extends SearchExtractor { return null; } + @Override + public boolean isCorrectedSearch() { + return false; + } + @Nonnull @Override public InfoItemsPage getInitialPage() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java index 61fcdd6df..958f53619 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -40,6 +40,11 @@ public class PeertubeSearchExtractor extends SearchExtractor { return null; } + @Override + public boolean isCorrectedSearch() { + return false; + } + @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { super.fetchPage(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java index 92730ec46..b121d56c3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java @@ -38,6 +38,11 @@ public class SoundcloudSearchExtractor extends SearchExtractor { return null; } + @Override + public boolean isCorrectedSearch() { + return false; + } + @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index 7e952c94b..99f6ecd2a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -135,6 +135,19 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); } + @Override + public boolean isCorrectedSearch() { + final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer"); + if (itemSectionRenderer == null) { + return false; + } + + JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) + .getObject("showingResultsForRenderer"); + return showingResultsForRenderer != null; + } + @Nonnull @Override public InfoItemsPage getInitialPage() throws ExtractionException, IOException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 732ef09ad..69e27e2f5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; - import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -12,10 +11,10 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; - -import java.io.IOException; +import org.schabi.newpipe.extractor.utils.JsonUtils; import javax.annotation.Nonnull; +import java.io.IOException; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; @@ -64,15 +63,25 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getSearchSuggestion() throws ParsingException { + final JsonObject didYouMeanRenderer = initialData.getObject("contents") + .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") + .getObject("sectionListRenderer").getArray("contents").getObject(0) + .getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("didYouMeanRenderer"); + if (didYouMeanRenderer == null) { + return ""; + } + return JsonUtils.getString(didYouMeanRenderer, "correctedQueryEndpoint.searchEndpoint.query"); + } + + @Override + public boolean isCorrectedSearch() { final JsonObject showingResultsForRenderer = initialData.getObject("contents") .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") .getObject("sectionListRenderer").getArray("contents").getObject(0) .getObject("itemSectionRenderer").getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - if (!showingResultsForRenderer.has("correctedQuery")) { - return ""; - } - return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); + return showingResultsForRenderer != null; } @Nonnull diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java index e82ad7d0a..ddfa27fb0 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java @@ -4,4 +4,5 @@ package org.schabi.newpipe.extractor.services; public interface BaseSearchExtractorTest extends BaseListExtractorTest { void testSearchString() throws Exception; void testSearchSuggestion() throws Exception; + void testSearchCorrected() throws Exception; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java index 93c4eac70..b363031d7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java @@ -15,6 +15,10 @@ public abstract class DefaultSearchExtractorTest extends DefaultListExtractorTes public abstract String expectedSearchString(); @Nullable public abstract String expectedSearchSuggestion(); + public boolean isCorrectedSearch() { + return false; + } + @Test @Override public void testSearchString() throws Exception { @@ -31,4 +35,9 @@ public abstract class DefaultSearchExtractorTest extends DefaultListExtractorTes assertEquals(expectedSearchSuggestion, extractor().getSearchSuggestion()); } } + + @Test + public void testSearchCorrected() throws Exception { + assertEquals(isCorrectedSearch(), extractor().isCorrectedSearch()); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java index 420db0adb..2dc4c2814 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java @@ -150,4 +150,27 @@ public class YoutubeMusicSearchExtractorTest { @Nullable @Override public String expectedSearchSuggestion() { return "mega man x3"; } @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } } + + public static class CorrectedSearch extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "duo lipa"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + @Override public boolean isCorrectedSearch() { return true; } + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java index d5f2f1af2..9cc4ad1ff 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java @@ -114,8 +114,8 @@ public class YoutubeSearchExtractorTest { public static class Suggestion extends DefaultSearchExtractorTest { private static SearchExtractor extractor; - private static final String QUERY = "pewdeipie"; - private static final String EXPECTED_SUGGESTION = "pewdiepie"; + private static final String QUERY = "newpip"; + private static final String EXPECTED_SUGGESTION = "newpipe"; @BeforeClass public static void setUp() throws Exception { @@ -132,10 +132,32 @@ public class YoutubeSearchExtractorTest { @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } @Override public String expectedSearchString() { return QUERY; } @Nullable @Override public String expectedSearchSuggestion() { return EXPECTED_SUGGESTION; } - @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } } + public static class CorrectedSearch extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "pewdeipie"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + @Override public boolean isCorrectedSearch() { return true; } + } + public static class RandomQueryNoMorePages extends DefaultSearchExtractorTest { private static SearchExtractor extractor; private static final String QUERY = "UCO6AK";