lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

remove searchengine

This commit is contained in:
Christian Schabesberger 2018-07-02 13:47:39 +02:00
parent 291de18418
commit 812a785811
8 changed files with 10 additions and 344 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
extractor/src/main/java/org/schabi/newpipe/extractor/ListInfo.java
View File

@ -1,16 +1,14 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.uih.ListUIHandler;
import java.util.ArrayList;
import java.util.List;
public abstract class ListInfo<T extends InfoItem> extends Info {
private List<T> relatedItems;
private String nextPageUrl = null;
private List<String> contentFilters = new ArrayList<>();
private String sortFilter = "";
private final List<String> contentFilters;
private final String sortFilter;
public ListInfo(int serviceId,
String id,
@ -24,7 +22,7 @@ public abstract class ListInfo<T extends InfoItem> extends Info {
this.sortFilter = sortFilter;
}
public ListInfo(int serviceId, ListUIHandler listUrlIdHandler, String name) throws ParsingException {
public ListInfo(int serviceId, ListUIHandler listUrlIdHandler, String name) {
super(serviceId, listUrlIdHandler, name);
this.contentFilters = listUrlIdHandler.getContentFilters();
this.sortFilter = listUrlIdHandler.getSortFilter();

2
extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java
View File

@ -15,7 +15,7 @@ import java.io.IOException;
public class PlaylistInfo extends ListInfo<StreamInfoItem> {
public PlaylistInfo(int serviceId, ListUIHandler urlIdHandler, String name) throws ParsingException {
private PlaylistInfo(int serviceId, ListUIHandler urlIdHandler, String name) throws ParsingException {
super(serviceId, urlIdHandler, name);
}

50
extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java
View File

@ -1,50 +0,0 @@
package org.schabi.newpipe.extractor.search;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.io.IOException;
/*
* Created by Christian Schabesberger on 10.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* SearchEngine.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public abstract class SearchEngine {
public enum Filter {
ANY, STREAM, CHANNEL, PLAYLIST
}
public static class NothingFoundException extends ExtractionException {
public NothingFoundException(String message) {
super(message);
}
}
private final InfoItemsSearchCollector collector;
public SearchEngine(int serviceId) {
collector = new InfoItemsSearchCollector(serviceId);
}
protected InfoItemsSearchCollector getInfoItemSearchCollector() {
return collector;
}
public abstract InfoItemsSearchCollector search(String query, int page, String contentCountry, Filter filter)
throws IOException, ExtractionException;
}

12
extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java
View File

@ -2,21 +2,19 @@ package org.schabi.newpipe.extractor.search;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.uih.ListUIHFactory;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.uih.ListUIHandler;
import org.schabi.newpipe.extractor.uih.SearchQIHandler;
public class SearchInfo extends ListInfo<InfoItem> {
private String searchString = "";
private String searchSuggestion = "";
private String searchString;
private String searchSuggestion;
public SearchInfo(int serviceId,
ListUIHandler urlIdHandler,
SearchQIHandler qIHandler,
String searchString) throws ParsingException {
super(serviceId, urlIdHandler, "Search");
super(serviceId, qIHandler, "Search");
this.searchString = searchString;
}

71
extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java
View File

@ -1,71 +0,0 @@
package org.schabi.newpipe.extractor.search;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/*
* Created by Christian Schabesberger on 29.02.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* SearchResult.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class SearchResult {
private final int serviceId;
public final String suggestion;
@Nonnull
public final List<InfoItem> resultList;
@Nonnull
public final List<Throwable> errors;
public SearchResult(int serviceId, String suggestion, List<InfoItem> results, List<Throwable> errors) {
this.serviceId = serviceId;
this.suggestion = suggestion;
this.resultList = Collections.unmodifiableList(new ArrayList<>(results));
this.errors = Collections.unmodifiableList(new ArrayList<>(errors));
}
public static SearchResult getSearchResult(@Nonnull final SearchEngine engine, final String query, final int page,
final String languageCode, final SearchEngine.Filter filter)
throws IOException, ExtractionException {
return null;
}
public String getSuggestion() {
return suggestion;
}
@Nonnull
public List<InfoItem> getResults() {
return Collections.unmodifiableList(resultList);
}
@Nonnull
public List<Throwable> getErrors() {
return errors;
}
public int getServiceId() {
return serviceId;
}
}

84
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java
View File

@ -1,84 +0,0 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchEngine;
import java.io.IOException;
import java.net.URLEncoder;
public class SoundcloudSearchEngine extends SearchEngine {
public static final String CHARSET_UTF_8 = "UTF-8";
public SoundcloudSearchEngine(int serviceId) {
super(serviceId);
}
@Override
public InfoItemsSearchCollector search(String query, int page, String languageCode, Filter filter) throws IOException, ExtractionException {
InfoItemsSearchCollector collector = getInfoItemSearchCollector();
Downloader dl = NewPipe.getDownloader();
String url = "https://api-v2.soundcloud.com/search";
switch (filter) {
case STREAM:
url += "/tracks";
break;
case CHANNEL:
url += "/users";
break;
case PLAYLIST:
url += "/playlists";
break;
case ANY:
// Don't append any parameter to search for everything
default:
break;
}
url += "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=10"
+ "&offset=" + Integer.toString(page * 10);
JsonArray searchCollection;
try {
searchCollection = JsonParser.object().from(dl.download(url)).getArray("collection");
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
if (searchCollection.size() == 0) {
throw new NothingFoundException("Nothing found");
}
for (Object result : searchCollection) {
if (!(result instanceof JsonObject)) continue;
//noinspection ConstantConditions
JsonObject searchResult = (JsonObject) result;
String kind = searchResult.getString("kind", "");
switch (kind) {
case "user":
collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult));
break;
case "track":
collector.commit(new SoundcloudStreamInfoItemExtractor(searchResult));
break;
case "playlist":
collector.commit(new SoundcloudPlaylistInfoItemExtractor(searchResult));
break;
}
}
return collector;
}
}

3
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java
View File

@ -8,7 +8,6 @@ import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.uih.SearchQIHandler;
import org.schabi.newpipe.extractor.utils.Parser;
@ -70,7 +69,7 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
}
if (searchCollection.size() == 0) {
throw new SearchEngine.NothingFoundException("Nothing found");
throw new SearchExtractor.NothingFoundException("Nothing found");
}
}

124
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchEngine.java
View File

@ -1,124 +0,0 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchEngine;
import java.io.IOException;
import java.net.URLEncoder;
/*
* Created by Christian Schabesberger on 09.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* YoutubeSearchEngine.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeSearchEngine extends SearchEngine {
private static final String TAG = YoutubeSearchEngine.class.toString();
public static final String CHARSET_UTF_8 = "UTF-8";
public YoutubeSearchEngine(int serviceId) {
super(serviceId);
}
@Override
public InfoItemsSearchCollector search(String query, int page, String languageCode, Filter filter)
throws IOException, ExtractionException {
InfoItemsSearchCollector collector = getInfoItemSearchCollector();
Downloader downloader = NewPipe.getDownloader();
String url = "https://www.youtube.com/results"
+ "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
+ "&page=" + Integer.toString(page + 1);
switch (filter) {
case STREAM:
url += "&sp=EgIQAVAU";
break;
case CHANNEL:
url += "&sp=EgIQAlAU"; //EgIQA( lowercase L )AU
break;
case PLAYLIST:
url += "&sp=EgIQA1AU"; //EgIQA( one )AU
break;
case ANY:
// Don't append any parameter to search for everything
default:
break;
}
String site;
//String url = builder.build().toString();
//if we've been passed a valid language code, append it to the URL
if (!languageCode.isEmpty()) {
//assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode);
site = downloader.download(url, languageCode);
} else {
site = downloader.download(url);
}
Document doc = Jsoup.parse(site, url);
Element list = doc.select("ol[class=\"item-section\"]").first();
for (Element item : list.children()) {
/* First we need to determine which kind of item we are working with.
Youtube depicts five different kinds of items on its search result page. These are
regular videos, playlists, channels, two types of video suggestions, and a "no video
found" item. Since we only want videos, we need to filter out all the others.
An example for this can be seen here:
https://www.youtube.com/results?search_query=asdf&page=1
We already applied a filter to the url, so we don't need to care about channels and
playlists now.
*/
Element el;
// both types of spell correction item
if ((el = item.select("div[class*=\"spell-correction\"]").first()) != null) {
if (list.children().size() == 1) {
throw new NothingFoundException("Did you mean: " + el.select("a").first().text());
}
// search message item
} else if ((el = item.select("div[class*=\"search-message\"]").first()) != null) {
throw new NothingFoundException(el.text());
// video item type
} else if ((el = item.select("div[class*=\"yt-lockup-video\"]").first()) != null) {
collector.commit(new YoutubeStreamInfoItemExtractor(el));
} else if ((el = item.select("div[class*=\"yt-lockup-channel\"]").first()) != null) {
collector.commit(new YoutubeChannelInfoItemExtractor(el));
} else if ((el = item.select("div[class*=\"yt-lockup-playlist\"]").first()) != null &&
item.select(".yt-pl-icon-mix").isEmpty()) {
collector.commit(new YoutubePlaylistInfoItemExtractor(el));
} else {
// noinspection ConstantConditions
// simply ignore not known items
// throw new ExtractionException("unexpected element found: \"" + item + "\"");
}
}
return collector;
}
}