diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java index a7efea962..d1c481bb4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java @@ -25,6 +25,16 @@ public abstract class SearchExtractor extends ListExtractor { return getLinkHandler().getSearchString(); } + /** + * The search suggestion provided by the service. + *

+ * This method also returns the corrected query if + * {@link SearchExtractor#isCorrectedSearch()} is true. + * + * @return a suggestion to another query, the corrected query, or an empty String. + * @throws ParsingException + */ + @Nonnull public abstract String getSearchSuggestion() throws ParsingException; @Override @@ -37,4 +47,14 @@ public abstract class SearchExtractor extends ListExtractor { public String getName() { return getLinkHandler().getSearchString(); } + + /** + * Tell if the search was corrected by the service (if it's not exactly the search you typed). + *

+ * Example: on YouTube, if you search for "pewdeipie", + * it will give you results for "pewdiepie", then isCorrectedSearch should return true. + * + * @return whether the results comes from a corrected query or not. + */ + public abstract boolean isCorrectedSearch() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java index eedce719e..b2e072cce 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java @@ -15,6 +15,7 @@ public class SearchInfo extends ListInfo { private String searchString; private String searchSuggestion; + private boolean isCorrectedSearch; public SearchInfo(int serviceId, SearchQueryHandler qIHandler, @@ -42,7 +43,12 @@ public class SearchInfo extends ListInfo { info.addError(e); } try { - info.searchSuggestion = extractor.getSearchSuggestion(); + info.setSearchSuggestion(extractor.getSearchSuggestion()); + } catch (Exception e) { + info.addError(e); + } + try { + info.setIsCorrectedSearch(extractor.isCorrectedSearch()); } catch (Exception e) { info.addError(e); } @@ -64,10 +70,22 @@ public class SearchInfo extends ListInfo { // Getter public String getSearchString() { - return searchString; + return this.searchString; } public String getSearchSuggestion() { - return searchSuggestion; + return this.searchSuggestion; + } + + public boolean isCorrectedSearch() { + return this.isCorrectedSearch; + } + + public void setIsCorrectedSearch(boolean isCorrectedSearch) { + this.isCorrectedSearch = isCorrectedSearch; + } + + public void setSearchSuggestion(String searchSuggestion) { + this.searchSuggestion = searchSuggestion; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index d5ced534b..914c77497 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; @@ -42,9 +43,15 @@ public class MediaCCCSearchExtractor extends SearchExtractor { } } + @Nonnull @Override public String getSearchSuggestion() { - return null; + return ""; + } + + @Override + public boolean isCorrectedSearch() { + return false; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java index 8313d08ed..6ea78b378 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -20,6 +20,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; +import javax.annotation.Nonnull; import java.io.IOException; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*; @@ -33,9 +34,15 @@ public class PeertubeSearchExtractor extends SearchExtractor { super(service, linkHandler); } + @Nonnull @Override public String getSearchSuggestion() throws ParsingException { - return null; + return ""; + } + + @Override + public boolean isCorrectedSearch() { + return false; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java index 92730ec46..24f5987ec 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java @@ -33,9 +33,15 @@ public class SoundcloudSearchExtractor extends SearchExtractor { super(service, linkHandler); } + @Nonnull @Override public String getSearchSuggestion() { - return null; + return ""; + } + + @Override + public boolean isCorrectedSearch() { + return false; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index 20fe8a087..6d26e8d6a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -18,6 +18,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; @@ -125,15 +126,40 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { return super.getUrl(); } + @Nonnull @Override public String getSearchSuggestion() throws ParsingException { - final JsonObject didYouMeanRenderer = initialData.getObject("contents").getObject("sectionListRenderer") - .getArray("contents").getObject(0).getObject("itemSectionRenderer") - .getArray("contents").getObject(0).getObject("didYouMeanRenderer"); - if (!didYouMeanRenderer.has("correctedQuery")) { + final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer"); + if (itemSectionRenderer.isEmpty()) { return ""; } - return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); + + final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents") + .getObject(0).getObject("didYouMeanRenderer"); + final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) + .getObject("showingResultsForRenderer"); + + if (!didYouMeanRenderer.isEmpty()) { + return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); + } else if (!showingResultsForRenderer.isEmpty()) { + return JsonUtils.getString(showingResultsForRenderer, "correctedQueryEndpoint.searchEndpoint.query"); + } else { + return ""; + } + } + + @Override + public boolean isCorrectedSearch() { + final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer"); + if (itemSectionRenderer.isEmpty()) { + return false; + } + + final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) + .getObject("showingResultsForRenderer"); + return !showingResultsForRenderer.isEmpty(); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 1ef19b820..560943628 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; - import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -12,10 +11,10 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; - -import java.io.IOException; +import org.schabi.newpipe.extractor.utils.JsonUtils; import javax.annotation.Nonnull; +import java.io.IOException; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; @@ -63,17 +62,35 @@ public class YoutubeSearchExtractor extends SearchExtractor { return super.getUrl() + "&gl=" + getExtractorContentCountry().getCountryCode(); } + @Nonnull @Override public String getSearchSuggestion() throws ParsingException { + final JsonObject itemSectionRenderer = initialData.getObject("contents") + .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") + .getObject("sectionListRenderer").getArray("contents").getObject(0) + .getObject("itemSectionRenderer"); + final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents").getObject(0) + .getObject("didYouMeanRenderer"); + final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) + .getObject("showingResultsForRenderer"); + + if (!didYouMeanRenderer.isEmpty()) { + return JsonUtils.getString(didYouMeanRenderer, "correctedQueryEndpoint.searchEndpoint.query"); + } else if (showingResultsForRenderer != null) { + return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); + } else { + return ""; + } + } + + @Override + public boolean isCorrectedSearch() { final JsonObject showingResultsForRenderer = initialData.getObject("contents") .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") .getObject("sectionListRenderer").getArray("contents").getObject(0) .getObject("itemSectionRenderer").getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - if (!showingResultsForRenderer.has("correctedQuery")) { - return ""; - } - return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); + return !showingResultsForRenderer.isEmpty(); } @Nonnull diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java index e82ad7d0a..ddfa27fb0 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java @@ -4,4 +4,5 @@ package org.schabi.newpipe.extractor.services; public interface BaseSearchExtractorTest extends BaseListExtractorTest { void testSearchString() throws Exception; void testSearchSuggestion() throws Exception; + void testSearchCorrected() throws Exception; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java index 250df78f8..8a8a4e53d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java @@ -16,6 +16,10 @@ public abstract class DefaultSearchExtractorTest extends DefaultListExtractorTes public abstract String expectedSearchString(); @Nullable public abstract String expectedSearchSuggestion(); + public boolean isCorrectedSearch() { + return false; + } + @Test @Override public void testSearchString() throws Exception { @@ -32,4 +36,9 @@ public abstract class DefaultSearchExtractorTest extends DefaultListExtractorTes assertEquals(expectedSearchSuggestion, extractor().getSearchSuggestion()); } } + + @Test + public void testSearchCorrected() throws Exception { + assertEquals(isCorrectedSearch(), extractor().isCorrectedSearch()); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java index 420db0adb..eeac8c49b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java @@ -150,4 +150,28 @@ public class YoutubeMusicSearchExtractorTest { @Nullable @Override public String expectedSearchSuggestion() { return "mega man x3"; } @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } } + + public static class CorrectedSearch extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "duo lipa"; + private static final String EXPECTED_SUGGESTION = "dua lipa"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return EXPECTED_SUGGESTION; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + @Override public boolean isCorrectedSearch() { return true; } + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java index 29fab054d..518796fb2 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java @@ -114,6 +114,29 @@ public class YoutubeSearchExtractorTest { } public static class Suggestion extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "newpip"; + private static final String EXPECTED_SUGGESTION = "newpipe"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return EXPECTED_SUGGESTION; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + } + + public static class CorrectedSearch extends DefaultSearchExtractorTest { private static SearchExtractor extractor; private static final String QUERY = "pewdeipie"; private static final String EXPECTED_SUGGESTION = "pewdiepie"; @@ -133,8 +156,8 @@ public class YoutubeSearchExtractorTest { @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } @Override public String expectedSearchString() { return QUERY; } @Nullable @Override public String expectedSearchSuggestion() { return EXPECTED_SUGGESTION; } - @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + @Override public boolean isCorrectedSearch() { return true; } } public static class RandomQueryNoMorePages extends DefaultSearchExtractorTest {