lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #311 from B0pol/fixSearchSuggestion 

Search: add isCorrectedSearch() and fix YoutubeSearchExtractor#getSea…
This commit is contained in:
wb9688 2020-05-12 09:29:15 +02:00 committed by GitHub
parent b40ccb5075 6cff5de400
commit baf5dd3e8f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 177 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java
View File

@ -25,6 +25,16 @@ public abstract class SearchExtractor extends ListExtractor<InfoItem> {
return getLinkHandler().getSearchString();
}
/**
* The search suggestion provided by the service.
* <p>
* This method also returns the corrected query if
* {@link SearchExtractor#isCorrectedSearch()} is true.
*
* @return a suggestion to another query, the corrected query, or an empty String.
* @throws ParsingException
*/
@Nonnull
public abstract String getSearchSuggestion() throws ParsingException;
@Override
@ -37,4 +47,14 @@ public abstract class SearchExtractor extends ListExtractor<InfoItem> {
public String getName() {
return getLinkHandler().getSearchString();
}
/**
* Tell if the search was corrected by the service (if it's not exactly the search you typed).
* <p>
* Example: on YouTube, if you search for "pewdeipie",
* it will give you results for "pewdiepie", then isCorrectedSearch should return true.
*
* @return whether the results comes from a corrected query or not.
*/
public abstract boolean isCorrectedSearch() throws ParsingException;
}

24
extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java
View File

@ -15,6 +15,7 @@ public class SearchInfo extends ListInfo<InfoItem> {
private String searchString;
private String searchSuggestion;
private boolean isCorrectedSearch;
public SearchInfo(int serviceId,
SearchQueryHandler qIHandler,
@ -42,7 +43,12 @@ public class SearchInfo extends ListInfo<InfoItem> {
info.addError(e);
}
try {
info.searchSuggestion = extractor.getSearchSuggestion();
info.setSearchSuggestion(extractor.getSearchSuggestion());
} catch (Exception e) {
info.addError(e);
}
try {
info.setIsCorrectedSearch(extractor.isCorrectedSearch());
} catch (Exception e) {
info.addError(e);
}
@ -64,10 +70,22 @@ public class SearchInfo extends ListInfo<InfoItem> {
// Getter
public String getSearchString() {
return searchString;
return this.searchString;
}
public String getSearchSuggestion() {
return searchSuggestion;
return this.searchSuggestion;
}
public boolean isCorrectedSearch() {
return this.isCorrectedSearch;
}
public void setIsCorrectedSearch(boolean isCorrectedSearch) {
this.isCorrectedSearch = isCorrectedSearch;
}
public void setSearchSuggestion(String searchSuggestion) {
this.searchSuggestion = searchSuggestion;
}
}

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java
View File

@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
@ -42,9 +43,15 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
}
}
@Nonnull
@Override
public String getSearchSuggestion() {
return null;
return "";
}
@Override
public boolean isCorrectedSearch() {
return false;
}
@Nonnull

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java
View File

@ -20,6 +20,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*;
@ -33,9 +34,15 @@ public class PeertubeSearchExtractor extends SearchExtractor {
super(service, linkHandler);
}
@Nonnull
@Override
public String getSearchSuggestion() throws ParsingException {
return null;
return "";
}
@Override
public boolean isCorrectedSearch() {
return false;
}
@Override

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java
View File

@ -33,9 +33,15 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
super(service, linkHandler);
}
@Nonnull
@Override
public String getSearchSuggestion() {
return null;
return "";
}
@Override
public boolean isCorrectedSearch() {
return false;
}
@Nonnull

36
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
View File

@ -18,6 +18,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
@ -125,15 +126,40 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
return super.getUrl();
}
@Nonnull
@Override
public String getSearchSuggestion() throws ParsingException {
final JsonObject didYouMeanRenderer = initialData.getObject("contents").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("didYouMeanRenderer");
if (!didYouMeanRenderer.has("correctedQuery")) {
final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer");
if (itemSectionRenderer.isEmpty()) {
return "";
}
return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery"));
final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents")
.getObject(0).getObject("didYouMeanRenderer");
final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0)
.getObject("showingResultsForRenderer");
if (!didYouMeanRenderer.isEmpty()) {
return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery"));
} else if (!showingResultsForRenderer.isEmpty()) {
return JsonUtils.getString(showingResultsForRenderer, "correctedQueryEndpoint.searchEndpoint.query");
} else {
return "";
}
}
@Override
public boolean isCorrectedSearch() {
final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer");
if (itemSectionRenderer.isEmpty()) {
return false;
}
final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0)
.getObject("showingResultsForRenderer");
return !showingResultsForRenderer.isEmpty();
}
@Nonnull

31
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -12,10 +11,10 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import java.io.IOException;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
@ -63,17 +62,35 @@ public class YoutubeSearchExtractor extends SearchExtractor {
return super.getUrl() + "&gl=" + getExtractorContentCountry().getCountryCode();
}
@Nonnull
@Override
public String getSearchSuggestion() throws ParsingException {
final JsonObject itemSectionRenderer = initialData.getObject("contents")
.getObject("twoColumnSearchResultsRenderer").getObject("primaryContents")
.getObject("sectionListRenderer").getArray("contents").getObject(0)
.getObject("itemSectionRenderer");
final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents").getObject(0)
.getObject("didYouMeanRenderer");
final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0)
.getObject("showingResultsForRenderer");
if (!didYouMeanRenderer.isEmpty()) {
return JsonUtils.getString(didYouMeanRenderer, "correctedQueryEndpoint.searchEndpoint.query");
} else if (showingResultsForRenderer != null) {
return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery"));
} else {
return "";
}
}
@Override
public boolean isCorrectedSearch() {
final JsonObject showingResultsForRenderer = initialData.getObject("contents")
.getObject("twoColumnSearchResultsRenderer").getObject("primaryContents")
.getObject("sectionListRenderer").getArray("contents").getObject(0)
.getObject("itemSectionRenderer").getArray("contents").getObject(0)
.getObject("showingResultsForRenderer");
if (!showingResultsForRenderer.has("correctedQuery")) {
return "";
}
return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery"));
return !showingResultsForRenderer.isEmpty();
}
@Nonnull

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java
View File

@ -4,4 +4,5 @@ package org.schabi.newpipe.extractor.services;
public interface BaseSearchExtractorTest extends BaseListExtractorTest {
void testSearchString() throws Exception;
void testSearchSuggestion() throws Exception;
void testSearchCorrected() throws Exception;
}

9
extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java
View File

@ -16,6 +16,10 @@ public abstract class DefaultSearchExtractorTest extends DefaultListExtractorTes
public abstract String expectedSearchString();
@Nullable public abstract String expectedSearchSuggestion();
public boolean isCorrectedSearch() {
return false;
}
@Test
@Override
public void testSearchString() throws Exception {
@ -32,4 +36,9 @@ public abstract class DefaultSearchExtractorTest extends DefaultListExtractorTes
assertEquals(expectedSearchSuggestion, extractor().getSearchSuggestion());
}
}
@Test
public void testSearchCorrected() throws Exception {
assertEquals(isCorrectedSearch(), extractor().isCorrectedSearch());
}
}

24
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java
View File

@ -150,4 +150,28 @@ public class YoutubeMusicSearchExtractorTest {
@Nullable @Override public String expectedSearchSuggestion() { return "mega man x3"; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
}
public static class CorrectedSearch extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "duo lipa";
private static final String EXPECTED_SUGGESTION = "dua lipa";
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), "");
extractor.fetchPage();
}
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); }
@Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return EXPECTED_SUGGESTION; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
@Override public boolean isCorrectedSearch() { return true; }
}
}

25
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java
View File

@ -114,6 +114,29 @@ public class YoutubeSearchExtractorTest {
}
public static class Suggestion extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "newpip";
private static final String EXPECTED_SUGGESTION = "newpipe";
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
extractor.fetchPage();
}
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; }
@Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return EXPECTED_SUGGESTION; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
}
public static class CorrectedSearch extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "pewdeipie";
private static final String EXPECTED_SUGGESTION = "pewdiepie";
@ -133,8 +156,8 @@ public class YoutubeSearchExtractorTest {
@Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return EXPECTED_SUGGESTION; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
@Override public boolean isCorrectedSearch() { return true; }
}
public static class RandomQueryNoMorePages extends DefaultSearchExtractorTest {