Merge pull request #311 from B0pol/fixSearchSuggestion

Search: add isCorrectedSearch() and fix YoutubeSearchExtractor#getSea…

extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java

@@ -25,6 +25,16 @@ public abstract class SearchExtractor extends ListExtractor<InfoItem> {
         return getLinkHandler().getSearchString();
     }
 
+    /**
+     * The search suggestion provided by the service.
+     * <p>
+     * This method also returns the corrected query if
+     * {@link SearchExtractor#isCorrectedSearch()} is true.
+     *
+     * @return a suggestion to another query, the corrected query, or an empty String.
+     * @throws ParsingException
+     */
+    @Nonnull
     public abstract String getSearchSuggestion() throws ParsingException;
 
     @Override
@@ -37,4 +47,14 @@ public abstract class SearchExtractor extends ListExtractor<InfoItem> {
     public String getName() {
         return getLinkHandler().getSearchString();
     }
+
+    /**
+     * Tell if the search was corrected by the service (if it's not exactly the search you typed).
+     * <p>
+     * Example: on YouTube, if you search for "pewdeipie",
+     * it will give you results for "pewdiepie", then isCorrectedSearch should return true.
+     *
+     * @return whether the results comes from a corrected query or not.
+     */
+    public abstract boolean isCorrectedSearch() throws ParsingException;
 }

extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java

@@ -15,6 +15,7 @@ public class SearchInfo extends ListInfo<InfoItem> {
 
     private String searchString;
     private String searchSuggestion;
+    private boolean isCorrectedSearch;
 
     public SearchInfo(int serviceId,
                       SearchQueryHandler qIHandler,
@@ -42,7 +43,12 @@ public class SearchInfo extends ListInfo<InfoItem> {
             info.addError(e);
         }
         try {
-            info.searchSuggestion = extractor.getSearchSuggestion();
+            info.setSearchSuggestion(extractor.getSearchSuggestion());
+        } catch (Exception e) {
+            info.addError(e);
+        }
+        try {
+            info.setIsCorrectedSearch(extractor.isCorrectedSearch());
         } catch (Exception e) {
             info.addError(e);
         }
@@ -64,10 +70,22 @@ public class SearchInfo extends ListInfo<InfoItem> {
 
     // Getter
     public String getSearchString() {
-        return searchString;
+        return this.searchString;
     }
 
     public String getSearchSuggestion() {
-        return searchSuggestion;
+        return this.searchSuggestion;
+    }
+
+    public boolean isCorrectedSearch() {
+        return this.isCorrectedSearch;
+    }
+
+    public void setIsCorrectedSearch(boolean isCorrectedSearch) {
+        this.isCorrectedSearch = isCorrectedSearch;
+    }
+
+    public void setSearchSuggestion(String searchSuggestion) {
+        this.searchSuggestion = searchSuggestion;
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java

@@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
 import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
 import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
@@ -42,9 +43,15 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
         }
     }
 
+    @Nonnull
     @Override
     public String getSearchSuggestion() {
-        return null;
+        return "";
+    }
+
+    @Override
+    public boolean isCorrectedSearch() {
+        return false;
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java

@@ -20,6 +20,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Parser.RegexException;
 
+import javax.annotation.Nonnull;
 import java.io.IOException;
 
 import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*;
@@ -33,9 +34,15 @@ public class PeertubeSearchExtractor extends SearchExtractor {
         super(service, linkHandler);
     }
 
+    @Nonnull
     @Override
     public String getSearchSuggestion() throws ParsingException {
-        return null;
+        return "";
+    }
+
+    @Override
+    public boolean isCorrectedSearch() {
+        return false;
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java

@@ -33,9 +33,15 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
         super(service, linkHandler);
     }
 
+    @Nonnull
     @Override
     public String getSearchSuggestion() {
-        return null;
+        return "";
+    }
+
+    @Override
+    public boolean isCorrectedSearch() {
+        return false;
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java

@@ -18,6 +18,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
@@ -125,15 +126,40 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
         return super.getUrl();
     }
 
+    @Nonnull
     @Override
     public String getSearchSuggestion() throws ParsingException {
-        final JsonObject didYouMeanRenderer = initialData.getObject("contents").getObject("sectionListRenderer")
+        final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer")
-                .getArray("contents").getObject(0).getObject("itemSectionRenderer")
+                .getArray("contents").getObject(0).getObject("itemSectionRenderer");
-                .getArray("contents").getObject(0).getObject("didYouMeanRenderer");
+        if (itemSectionRenderer.isEmpty()) {
-        if (!didYouMeanRenderer.has("correctedQuery")) {
             return "";
         }
-        return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery"));
+
+        final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents")
+                .getObject(0).getObject("didYouMeanRenderer");
+        final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0)
+                .getObject("showingResultsForRenderer");
+
+        if (!didYouMeanRenderer.isEmpty()) {
+            return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery"));
+        } else if (!showingResultsForRenderer.isEmpty()) {
+            return JsonUtils.getString(showingResultsForRenderer, "correctedQueryEndpoint.searchEndpoint.query");
+        } else {
+            return "";
+        }
+    }
+
+    @Override
+    public boolean isCorrectedSearch() {
+        final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer")
+                .getArray("contents").getObject(0).getObject("itemSectionRenderer");
+        if (itemSectionRenderer.isEmpty()) {
+            return false;
+        }
+
+        final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0)
+                .getObject("showingResultsForRenderer");
+        return !showingResultsForRenderer.isEmpty();
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java

@@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
-
 import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
@@ -12,10 +11,10 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
 import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
-
+import org.schabi.newpipe.extractor.utils.JsonUtils;
-import java.io.IOException;
 
 import javax.annotation.Nonnull;
+import java.io.IOException;
 
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
@@ -63,17 +62,35 @@ public class YoutubeSearchExtractor extends SearchExtractor {
         return super.getUrl() + "&gl=" + getExtractorContentCountry().getCountryCode();
     }
 
+    @Nonnull
     @Override
     public String getSearchSuggestion() throws ParsingException {
+        final JsonObject itemSectionRenderer = initialData.getObject("contents")
+                .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents")
+                .getObject("sectionListRenderer").getArray("contents").getObject(0)
+                .getObject("itemSectionRenderer");
+        final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents").getObject(0)
+                .getObject("didYouMeanRenderer");
+        final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0)
+                .getObject("showingResultsForRenderer");
+
+        if (!didYouMeanRenderer.isEmpty()) {
+            return JsonUtils.getString(didYouMeanRenderer, "correctedQueryEndpoint.searchEndpoint.query");
+        } else if (showingResultsForRenderer != null) {
+            return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery"));
+        } else {
+            return "";
+        }
+    }
+
+    @Override
+    public boolean isCorrectedSearch() {
         final JsonObject showingResultsForRenderer = initialData.getObject("contents")
                 .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents")
                 .getObject("sectionListRenderer").getArray("contents").getObject(0)
                 .getObject("itemSectionRenderer").getArray("contents").getObject(0)
                 .getObject("showingResultsForRenderer");
-        if (!showingResultsForRenderer.has("correctedQuery")) {
+        return !showingResultsForRenderer.isEmpty();
-            return "";
-        }
-        return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery"));
     }
 
     @Nonnull

extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java

@@ -4,4 +4,5 @@ package org.schabi.newpipe.extractor.services;
 public interface BaseSearchExtractorTest extends BaseListExtractorTest {
     void testSearchString() throws Exception;
     void testSearchSuggestion() throws Exception;
+    void testSearchCorrected() throws Exception;
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java

@@ -16,6 +16,10 @@ public abstract class DefaultSearchExtractorTest extends DefaultListExtractorTes
     public abstract String expectedSearchString();
     @Nullable public abstract String expectedSearchSuggestion();
 
+    public boolean isCorrectedSearch() {
+        return false;
+    }
+
     @Test
     @Override
     public void testSearchString() throws Exception {
@@ -32,4 +36,9 @@ public abstract class DefaultSearchExtractorTest extends DefaultListExtractorTes
             assertEquals(expectedSearchSuggestion, extractor().getSearchSuggestion());
         }
     }
+
+    @Test
+    public void testSearchCorrected() throws Exception {
+        assertEquals(isCorrectedSearch(), extractor().isCorrectedSearch());
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java

@@ -150,4 +150,28 @@ public class YoutubeMusicSearchExtractorTest {
         @Nullable @Override public String expectedSearchSuggestion() { return "mega man x3"; }
         @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
     }
+
+    public static class CorrectedSearch extends DefaultSearchExtractorTest {
+        private static SearchExtractor extractor;
+        private static final String QUERY = "duo lipa";
+        private static final String EXPECTED_SUGGESTION = "dua lipa";
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+            extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), "");
+            extractor.fetchPage();
+        }
+
+        @Override public SearchExtractor extractor() { return extractor; }
+        @Override public StreamingService expectedService() { return YouTube; }
+        @Override public String expectedName() { return QUERY; }
+        @Override public String expectedId() { return QUERY; }
+        @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); }
+        @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); }
+        @Override public String expectedSearchString() { return QUERY; }
+        @Nullable @Override public String expectedSearchSuggestion() { return EXPECTED_SUGGESTION; }
+        @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
+        @Override public boolean isCorrectedSearch() { return true; }
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java

@@ -114,6 +114,29 @@ public class YoutubeSearchExtractorTest {
     }
 
     public static class Suggestion extends DefaultSearchExtractorTest {
+        private static SearchExtractor extractor;
+        private static final String QUERY = "newpip";
+        private static final String EXPECTED_SUGGESTION = "newpipe";
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+            extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
+            extractor.fetchPage();
+        }
+
+        @Override public SearchExtractor extractor() { return extractor; }
+        @Override public StreamingService expectedService() { return YouTube; }
+        @Override public String expectedName() { return QUERY; }
+        @Override public String expectedId() { return QUERY; }
+        @Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; }
+        @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; }
+        @Override public String expectedSearchString() { return QUERY; }
+        @Nullable @Override public String expectedSearchSuggestion() { return EXPECTED_SUGGESTION; }
+        @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
+    }
+
+    public static class CorrectedSearch extends DefaultSearchExtractorTest {
         private static SearchExtractor extractor;
         private static final String QUERY = "pewdeipie";
         private static final String EXPECTED_SUGGESTION = "pewdiepie";
@@ -133,8 +156,8 @@ public class YoutubeSearchExtractorTest {
         @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; }
         @Override public String expectedSearchString() { return QUERY; }
         @Nullable @Override public String expectedSearchSuggestion() { return EXPECTED_SUGGESTION; }
-
         @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
+        @Override public boolean isCorrectedSearch() { return true; }
     }
 
     public static class RandomQueryNoMorePages extends DefaultSearchExtractorTest {