From 29b639b454ce38f29d9dcc4c9fe054d581005bbe Mon Sep 17 00:00:00 2001 From: bopol Date: Sat, 11 Apr 2020 19:18:08 +0200 Subject: [PATCH 1/8] Search: add isCorrectedSearch() and fix YoutubeSearchExtractor#getSearchSuggestion() isCorrectedSearch: see the javadoc getSearchSuggestion: in YoutubeSearchExtractor, it was giving the corrected search query. It now gives the suggested query, as it should --- .../extractor/search/SearchExtractor.java | 11 ++++++++ .../newpipe/extractor/search/SearchInfo.java | 10 +++++++ .../extractors/MediaCCCSearchExtractor.java | 5 ++++ .../extractors/PeertubeSearchExtractor.java | 5 ++++ .../extractors/SoundcloudSearchExtractor.java | 5 ++++ .../YoutubeMusicSearchExtractor.java | 13 +++++++++ .../extractors/YoutubeSearchExtractor.java | 23 ++++++++++----- .../services/BaseSearchExtractorTest.java | 1 + .../services/DefaultSearchExtractorTest.java | 9 ++++++ .../YoutubeMusicSearchExtractorTest.java | 23 +++++++++++++++ .../search/YoutubeSearchExtractorTest.java | 28 +++++++++++++++++-- 11 files changed, 123 insertions(+), 10 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java index a7efea962..0101b96f0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java @@ -37,4 +37,15 @@ public abstract class SearchExtractor extends ListExtractor { public String getName() { return getLinkHandler().getSearchString(); } + + /** + * When you search on some service, it can give you another and corrected request. + * This method says if it's the case. + *

+ * Example: on YouTube, if you search for "pewdeipie", + * it will give you results for "pewdiepie", then isCorrectedSearch should return true. + * + * @return whether the results comes from a corrected query or not. + */ + public abstract boolean isCorrectedSearch() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java index eedce719e..547162bd1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java @@ -15,6 +15,7 @@ public class SearchInfo extends ListInfo { private String searchString; private String searchSuggestion; + private boolean isCorrectedSearch; public SearchInfo(int serviceId, SearchQueryHandler qIHandler, @@ -46,6 +47,11 @@ public class SearchInfo extends ListInfo { } catch (Exception e) { info.addError(e); } + try { + info.isCorrectedSearch = extractor.isCorrectedSearch(); + } catch (Exception e) { + info.addError(e); + } ListExtractor.InfoItemsPage page = ExtractorHelper.getItemsPageOrLogError(info, extractor); info.setRelatedItems(page.getItems()); @@ -70,4 +76,8 @@ public class SearchInfo extends ListInfo { public String getSearchSuggestion() { return searchSuggestion; } + + public boolean isCorrectedSearch() { + return this.isCorrectedSearch; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index d5ced534b..67f19a838 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -47,6 +47,11 @@ public class MediaCCCSearchExtractor extends SearchExtractor { return null; } + @Override + public boolean isCorrectedSearch() { + return false; + } + @Nonnull @Override public InfoItemsPage getInitialPage() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java index 61fcdd6df..958f53619 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -40,6 +40,11 @@ public class PeertubeSearchExtractor extends SearchExtractor { return null; } + @Override + public boolean isCorrectedSearch() { + return false; + } + @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { super.fetchPage(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java index 92730ec46..b121d56c3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java @@ -38,6 +38,11 @@ public class SoundcloudSearchExtractor extends SearchExtractor { return null; } + @Override + public boolean isCorrectedSearch() { + return false; + } + @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index 7e952c94b..99f6ecd2a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -135,6 +135,19 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); } + @Override + public boolean isCorrectedSearch() { + final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer"); + if (itemSectionRenderer == null) { + return false; + } + + JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) + .getObject("showingResultsForRenderer"); + return showingResultsForRenderer != null; + } + @Nonnull @Override public InfoItemsPage getInitialPage() throws ExtractionException, IOException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 732ef09ad..69e27e2f5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; - import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -12,10 +11,10 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; - -import java.io.IOException; +import org.schabi.newpipe.extractor.utils.JsonUtils; import javax.annotation.Nonnull; +import java.io.IOException; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; @@ -64,15 +63,25 @@ public class YoutubeSearchExtractor extends SearchExtractor { @Override public String getSearchSuggestion() throws ParsingException { + final JsonObject didYouMeanRenderer = initialData.getObject("contents") + .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") + .getObject("sectionListRenderer").getArray("contents").getObject(0) + .getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("didYouMeanRenderer"); + if (didYouMeanRenderer == null) { + return ""; + } + return JsonUtils.getString(didYouMeanRenderer, "correctedQueryEndpoint.searchEndpoint.query"); + } + + @Override + public boolean isCorrectedSearch() { final JsonObject showingResultsForRenderer = initialData.getObject("contents") .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") .getObject("sectionListRenderer").getArray("contents").getObject(0) .getObject("itemSectionRenderer").getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - if (!showingResultsForRenderer.has("correctedQuery")) { - return ""; - } - return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); + return showingResultsForRenderer != null; } @Nonnull diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java index e82ad7d0a..ddfa27fb0 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java @@ -4,4 +4,5 @@ package org.schabi.newpipe.extractor.services; public interface BaseSearchExtractorTest extends BaseListExtractorTest { void testSearchString() throws Exception; void testSearchSuggestion() throws Exception; + void testSearchCorrected() throws Exception; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java index 93c4eac70..b363031d7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java @@ -15,6 +15,10 @@ public abstract class DefaultSearchExtractorTest extends DefaultListExtractorTes public abstract String expectedSearchString(); @Nullable public abstract String expectedSearchSuggestion(); + public boolean isCorrectedSearch() { + return false; + } + @Test @Override public void testSearchString() throws Exception { @@ -31,4 +35,9 @@ public abstract class DefaultSearchExtractorTest extends DefaultListExtractorTes assertEquals(expectedSearchSuggestion, extractor().getSearchSuggestion()); } } + + @Test + public void testSearchCorrected() throws Exception { + assertEquals(isCorrectedSearch(), extractor().isCorrectedSearch()); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java index 420db0adb..2dc4c2814 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java @@ -150,4 +150,27 @@ public class YoutubeMusicSearchExtractorTest { @Nullable @Override public String expectedSearchSuggestion() { return "mega man x3"; } @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } } + + public static class CorrectedSearch extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "duo lipa"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + @Override public boolean isCorrectedSearch() { return true; } + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java index d5f2f1af2..9cc4ad1ff 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java @@ -114,8 +114,8 @@ public class YoutubeSearchExtractorTest { public static class Suggestion extends DefaultSearchExtractorTest { private static SearchExtractor extractor; - private static final String QUERY = "pewdeipie"; - private static final String EXPECTED_SUGGESTION = "pewdiepie"; + private static final String QUERY = "newpip"; + private static final String EXPECTED_SUGGESTION = "newpipe"; @BeforeClass public static void setUp() throws Exception { @@ -132,10 +132,32 @@ public class YoutubeSearchExtractorTest { @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } @Override public String expectedSearchString() { return QUERY; } @Nullable @Override public String expectedSearchSuggestion() { return EXPECTED_SUGGESTION; } - @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } } + public static class CorrectedSearch extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "pewdeipie"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + @Override public boolean isCorrectedSearch() { return true; } + } + public static class RandomQueryNoMorePages extends DefaultSearchExtractorTest { private static SearchExtractor extractor; private static final String QUERY = "UCO6AK"; From 5d1c3b3fa2f9a28e6caf91621b3a88eff0e81a3c Mon Sep 17 00:00:00 2001 From: bopol Date: Sun, 12 Apr 2020 15:27:51 +0200 Subject: [PATCH 2/8] Use setters in SearchInfo --- .../schabi/newpipe/extractor/search/SearchInfo.java | 12 ++++++++++-- .../extractors/YoutubeMusicSearchExtractor.java | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java index 547162bd1..e5c054571 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java @@ -43,12 +43,12 @@ public class SearchInfo extends ListInfo { info.addError(e); } try { - info.searchSuggestion = extractor.getSearchSuggestion(); + info.setSearchSuggestion(extractor.getSearchSuggestion()); } catch (Exception e) { info.addError(e); } try { - info.isCorrectedSearch = extractor.isCorrectedSearch(); + info.setCorrectedSearch(extractor.isCorrectedSearch()); } catch (Exception e) { info.addError(e); } @@ -80,4 +80,12 @@ public class SearchInfo extends ListInfo { public boolean isCorrectedSearch() { return this.isCorrectedSearch; } + + public void setCorrectedSearch(boolean correctedSearch) { + isCorrectedSearch = correctedSearch; + } + + public void setSearchSuggestion(String searchSuggestion) { + this.searchSuggestion = searchSuggestion; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index 99f6ecd2a..a6ea6531f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -143,7 +143,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { return false; } - JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) + final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); return showingResultsForRenderer != null; } From e8d58e09c9be249bfa430357c059443cebda6989 Mon Sep 17 00:00:00 2001 From: bopol Date: Sun, 12 Apr 2020 16:06:52 +0200 Subject: [PATCH 3/8] getSearchSuggestion now returns the corrected query if isCorrectedSearch() is true --- .../extractor/search/SearchExtractor.java | 10 ++++++++ .../extractors/MediaCCCSearchExtractor.java | 6 +++-- .../extractors/PeertubeSearchExtractor.java | 4 +++- .../extractors/SoundcloudSearchExtractor.java | 3 ++- .../YoutubeMusicSearchExtractor.java | 23 +++++++++++++++---- .../extractors/YoutubeSearchExtractor.java | 16 +++++++++---- .../YoutubeMusicSearchExtractorTest.java | 3 ++- .../search/YoutubeSearchExtractorTest.java | 4 +++- 8 files changed, 54 insertions(+), 15 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java index 0101b96f0..4290d6948 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java @@ -25,6 +25,16 @@ public abstract class SearchExtractor extends ListExtractor { return getLinkHandler().getSearchString(); } + /** + * The search suggestion provided by the service. + *

+ * This method may also return the corrected query, + * see {@link SearchExtractor#isCorrectedSearch()}. + * + * @return a suggestion to another query, the corrected query, or an empty String. + * @throws ParsingException + */ + @Nonnull public abstract String getSearchSuggestion() throws ParsingException; @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index 67f19a838..c4b368923 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; @@ -42,9 +43,10 @@ public class MediaCCCSearchExtractor extends SearchExtractor { } } + @Nonnull @Override - public String getSearchSuggestion() { - return null; + public String getSearchSuggestion() throws ParsingException { + return ""; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java index 958f53619..352164de3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -19,6 +19,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; +import javax.annotation.Nonnull; import java.io.IOException; public class PeertubeSearchExtractor extends SearchExtractor { @@ -35,9 +36,10 @@ public class PeertubeSearchExtractor extends SearchExtractor { super(service, linkHandler); } + @Nonnull @Override public String getSearchSuggestion() throws ParsingException { - return null; + return ""; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java index b121d56c3..24f5987ec 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java @@ -33,9 +33,10 @@ public class SoundcloudSearchExtractor extends SearchExtractor { super(service, linkHandler); } + @Nonnull @Override public String getSearchSuggestion() { - return null; + return ""; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index a6ea6531f..95b8c9f85 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -18,6 +18,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; @@ -124,15 +125,27 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { return super.getUrl(); } + @Nonnull @Override public String getSearchSuggestion() throws ParsingException { - final JsonObject didYouMeanRenderer = initialData.getObject("contents").getObject("sectionListRenderer") - .getArray("contents").getObject(0).getObject("itemSectionRenderer") - .getArray("contents").getObject(0).getObject("didYouMeanRenderer"); - if (!didYouMeanRenderer.has("correctedQuery")) { + final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer"); + if (itemSectionRenderer == null) { + return ""; + } + + final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents") + .getObject(0).getObject("didYouMeanRenderer"); + final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) + .getObject("showingResultsForRenderer"); + + if (didYouMeanRenderer != null) { + return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); + } else if (showingResultsForRenderer != null) { + return JsonUtils.getString(showingResultsForRenderer, "correctedQueryEndpoint.searchEndpoint.query"); + } else { return ""; } - return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 69e27e2f5..8c18c6024 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -61,17 +61,25 @@ public class YoutubeSearchExtractor extends SearchExtractor { return super.getUrl() + "&gl=" + getExtractorContentCountry().getCountryCode(); } + @Nonnull @Override public String getSearchSuggestion() throws ParsingException { - final JsonObject didYouMeanRenderer = initialData.getObject("contents") + final JsonObject itemSectionRenderer = initialData.getObject("contents") .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") .getObject("sectionListRenderer").getArray("contents").getObject(0) - .getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("itemSectionRenderer"); + final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents").getObject(0) .getObject("didYouMeanRenderer"); - if (didYouMeanRenderer == null) { + final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) + .getObject("showingResultsForRenderer"); + + if (didYouMeanRenderer != null) { + return JsonUtils.getString(didYouMeanRenderer, "correctedQueryEndpoint.searchEndpoint.query"); + } else if (showingResultsForRenderer != null) { + return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); + } else { return ""; } - return JsonUtils.getString(didYouMeanRenderer, "correctedQueryEndpoint.searchEndpoint.query"); } @Override diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java index 2dc4c2814..eeac8c49b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java @@ -154,6 +154,7 @@ public class YoutubeMusicSearchExtractorTest { public static class CorrectedSearch extends DefaultSearchExtractorTest { private static SearchExtractor extractor; private static final String QUERY = "duo lipa"; + private static final String EXPECTED_SUGGESTION = "dua lipa"; @BeforeClass public static void setUp() throws Exception { @@ -169,7 +170,7 @@ public class YoutubeMusicSearchExtractorTest { @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } @Override public String expectedSearchString() { return QUERY; } - @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Nullable @Override public String expectedSearchSuggestion() { return EXPECTED_SUGGESTION; } @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } @Override public boolean isCorrectedSearch() { return true; } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java index 9cc4ad1ff..4f7d46d52 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java @@ -7,6 +7,7 @@ import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest; @@ -138,6 +139,7 @@ public class YoutubeSearchExtractorTest { public static class CorrectedSearch extends DefaultSearchExtractorTest { private static SearchExtractor extractor; private static final String QUERY = "pewdeipie"; + private static final String EXPECTED_SUGGESTION = "pewdiepie"; @BeforeClass public static void setUp() throws Exception { @@ -153,7 +155,7 @@ public class YoutubeSearchExtractorTest { @Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; } @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } @Override public String expectedSearchString() { return QUERY; } - @Nullable @Override public String expectedSearchSuggestion() { return null; } + @Nullable @Override public String expectedSearchSuggestion() { return EXPECTED_SUGGESTION; } @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } @Override public boolean isCorrectedSearch() { return true; } } From 79483a6dc04a1d37e1e9e661b113632ddb420bf7 Mon Sep 17 00:00:00 2001 From: bopol Date: Wed, 15 Apr 2020 21:05:31 +0200 Subject: [PATCH 4/8] =?UTF-8?q?=C2=ABUse=20this=20in=20SearchInfo,=20for?= =?UTF-8?q?=20consistency's=20sake=C2=BB=20@Stypox=20order=20xD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../org/schabi/newpipe/extractor/search/SearchInfo.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java index e5c054571..9c6dd54de 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java @@ -70,11 +70,11 @@ public class SearchInfo extends ListInfo { // Getter public String getSearchString() { - return searchString; + return this.searchString; } public String getSearchSuggestion() { - return searchSuggestion; + return this.searchSuggestion; } public boolean isCorrectedSearch() { @@ -82,7 +82,7 @@ public class SearchInfo extends ListInfo { } public void setCorrectedSearch(boolean correctedSearch) { - isCorrectedSearch = correctedSearch; + this.isCorrectedSearch = correctedSearch; } public void setSearchSuggestion(String searchSuggestion) { From 3757541a9c8223d4ffce261faddee95112f125b5 Mon Sep 17 00:00:00 2001 From: bopol Date: Tue, 5 May 2020 10:02:56 +0200 Subject: [PATCH 5/8] adapt new nanojson version --- .../media_ccc/extractors/MediaCCCSearchExtractor.java | 2 +- .../extractors/YoutubeMusicSearchExtractor.java | 10 +++++----- .../youtube/extractors/YoutubeSearchExtractor.java | 4 ++-- .../youtube/search/YoutubeSearchExtractorTest.java | 1 - 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index c4b368923..914c77497 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -45,7 +45,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor { @Nonnull @Override - public String getSearchSuggestion() throws ParsingException { + public String getSearchSuggestion() { return ""; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index 95b8c9f85..0731f3041 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -130,7 +130,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { public String getSearchSuggestion() throws ParsingException { final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") .getArray("contents").getObject(0).getObject("itemSectionRenderer"); - if (itemSectionRenderer == null) { + if (itemSectionRenderer.equals(JsonUtils.EMPTY_OBJECT)) { return ""; } @@ -139,9 +139,9 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - if (didYouMeanRenderer != null) { + if (!didYouMeanRenderer.equals(JsonUtils.EMPTY_OBJECT)) { return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); - } else if (showingResultsForRenderer != null) { + } else if (!showingResultsForRenderer.equals(JsonUtils.EMPTY_OBJECT)) { return JsonUtils.getString(showingResultsForRenderer, "correctedQueryEndpoint.searchEndpoint.query"); } else { return ""; @@ -152,13 +152,13 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { public boolean isCorrectedSearch() { final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") .getArray("contents").getObject(0).getObject("itemSectionRenderer"); - if (itemSectionRenderer == null) { + if (itemSectionRenderer.equals(JsonUtils.EMPTY_OBJECT)) { return false; } final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - return showingResultsForRenderer != null; + return !showingResultsForRenderer.equals(JsonUtils.EMPTY_OBJECT); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 8c18c6024..f35bc6c06 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -73,7 +73,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - if (didYouMeanRenderer != null) { + if (!didYouMeanRenderer.equals(JsonUtils.EMPTY_OBJECT)) { return JsonUtils.getString(didYouMeanRenderer, "correctedQueryEndpoint.searchEndpoint.query"); } else if (showingResultsForRenderer != null) { return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); @@ -89,7 +89,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { .getObject("sectionListRenderer").getArray("contents").getObject(0) .getObject("itemSectionRenderer").getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - return showingResultsForRenderer != null; + return !showingResultsForRenderer.equals(JsonUtils.EMPTY_OBJECT); } @Nonnull diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java index 4f7d46d52..84e1501b8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java @@ -7,7 +7,6 @@ import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest; From 93e868634e1f3ab0fbfaf3724555c3c496108c21 Mon Sep 17 00:00:00 2001 From: bopol Date: Tue, 5 May 2020 10:09:35 +0200 Subject: [PATCH 6/8] clarified javadoc for isCorrectedSearch and getSearchSuggestions in SearchExtractor --- .../schabi/newpipe/extractor/search/SearchExtractor.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java index 4290d6948..d1c481bb4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java @@ -28,8 +28,8 @@ public abstract class SearchExtractor extends ListExtractor { /** * The search suggestion provided by the service. *

- * This method may also return the corrected query, - * see {@link SearchExtractor#isCorrectedSearch()}. + * This method also returns the corrected query if + * {@link SearchExtractor#isCorrectedSearch()} is true. * * @return a suggestion to another query, the corrected query, or an empty String. * @throws ParsingException @@ -49,8 +49,7 @@ public abstract class SearchExtractor extends ListExtractor { } /** - * When you search on some service, it can give you another and corrected request. - * This method says if it's the case. + * Tell if the search was corrected by the service (if it's not exactly the search you typed). *

* Example: on YouTube, if you search for "pewdeipie", * it will give you results for "pewdiepie", then isCorrectedSearch should return true. From 5760366d70ca5c6413f759af5b9be2c7e36c919a Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 7 May 2020 15:32:25 +0200 Subject: [PATCH 7/8] use JsonObject#isEmpty instead of JsonObject#equals(JsonUtils.EMPTY_OBJECT) --- .../extractors/YoutubeMusicSearchExtractor.java | 10 +++++----- .../youtube/extractors/YoutubeSearchExtractor.java | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index 0731f3041..39a6e94d4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -130,7 +130,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { public String getSearchSuggestion() throws ParsingException { final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") .getArray("contents").getObject(0).getObject("itemSectionRenderer"); - if (itemSectionRenderer.equals(JsonUtils.EMPTY_OBJECT)) { + if (itemSectionRenderer.isEmpty()) { return ""; } @@ -139,9 +139,9 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - if (!didYouMeanRenderer.equals(JsonUtils.EMPTY_OBJECT)) { + if (!didYouMeanRenderer.isEmpty()) { return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); - } else if (!showingResultsForRenderer.equals(JsonUtils.EMPTY_OBJECT)) { + } else if (!showingResultsForRenderer.isEmpty()) { return JsonUtils.getString(showingResultsForRenderer, "correctedQueryEndpoint.searchEndpoint.query"); } else { return ""; @@ -152,13 +152,13 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { public boolean isCorrectedSearch() { final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") .getArray("contents").getObject(0).getObject("itemSectionRenderer"); - if (itemSectionRenderer.equals(JsonUtils.EMPTY_OBJECT)) { + if (itemSectionRenderer.isEmpty()) { return false; } final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - return !showingResultsForRenderer.equals(JsonUtils.EMPTY_OBJECT); + return !showingResultsForRenderer.isEmpty(); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index f35bc6c06..05c27d53d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -73,7 +73,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - if (!didYouMeanRenderer.equals(JsonUtils.EMPTY_OBJECT)) { + if (!didYouMeanRenderer.isEmpty()) { return JsonUtils.getString(didYouMeanRenderer, "correctedQueryEndpoint.searchEndpoint.query"); } else if (showingResultsForRenderer != null) { return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); @@ -89,7 +89,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { .getObject("sectionListRenderer").getArray("contents").getObject(0) .getObject("itemSectionRenderer").getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - return !showingResultsForRenderer.equals(JsonUtils.EMPTY_OBJECT); + return !showingResultsForRenderer.isEmpty(); } @Nonnull From 6cff5de400f33f9e93c79efbf60cc0b7011b3027 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 7 May 2020 15:33:58 +0200 Subject: [PATCH 8/8] renaming methods for consistency --- .../org/schabi/newpipe/extractor/search/SearchInfo.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java index 9c6dd54de..b2e072cce 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java @@ -48,7 +48,7 @@ public class SearchInfo extends ListInfo { info.addError(e); } try { - info.setCorrectedSearch(extractor.isCorrectedSearch()); + info.setIsCorrectedSearch(extractor.isCorrectedSearch()); } catch (Exception e) { info.addError(e); } @@ -81,8 +81,8 @@ public class SearchInfo extends ListInfo { return this.isCorrectedSearch; } - public void setCorrectedSearch(boolean correctedSearch) { - this.isCorrectedSearch = correctedSearch; + public void setIsCorrectedSearch(boolean isCorrectedSearch) { + this.isCorrectedSearch = isCorrectedSearch; } public void setSearchSuggestion(String searchSuggestion) {