diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java index 4be05c5ed..3fa9472c7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java @@ -1,6 +1,27 @@ +/* + * Created by Christian Schabesberger on 28.09.16. + * + * Copyright (C) Christian Schabesberger 2015 + * YoutubeSuggestionExtractor.java is part of NewPipe Extractor. + * + * NewPipe Extractor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe Extractor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe Extractor. If not, see . + */ + package org.schabi.newpipe.extractor.services.youtube.extractors; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getCookieHeader; +import static org.schabi.newpipe.extractor.utils.Utils.isBlank; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonParser; @@ -8,35 +29,18 @@ import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; - -/* - * Created by Christian Schabesberger on 28.09.16. - * - * Copyright (C) Christian Schabesberger 2015 - * YoutubeSuggestionExtractor.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ +import java.util.Map; +import java.util.stream.Collectors; public class YoutubeSuggestionExtractor extends SuggestionExtractor { @@ -46,35 +50,45 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor { @Override public List suggestionList(final String query) throws IOException, ExtractionException { - final Downloader dl = NewPipe.getDownloader(); - final List suggestions = new ArrayList<>(); - - final String url = "https://suggestqueries.google.com/complete/search" - + "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml - + "&jsonp=" + "JP" + final String url = "https://suggestqueries-clients6.youtube.com/complete/search" + + "?client=" + "youtube" + "&ds=" + "yt" + "&gl=" + Utils.encodeUrlUtf8(getExtractorContentCountry().getCountryCode()) - + "&q=" + Utils.encodeUrlUtf8(query); + + "&q=" + Utils.encodeUrlUtf8(query) + + "&xhr=t"; + + final Map> headers = new HashMap<>(); + headers.put("Origin", Collections.singletonList("https://www.youtube.com")); + headers.put("Referer", Collections.singletonList("https://www.youtube.com")); + + final Response response = NewPipe.getDownloader() + .get(url, headers, getExtractorLocalization()); + + final String contentTypeHeader = response.getHeader("Content-Type"); + if (isNullOrEmpty(contentTypeHeader) || !contentTypeHeader.contains("application/json")) { + throw new ExtractionException("Invalid response type (got \"" + contentTypeHeader + + "\", excepted a JSON response) (response code " + + response.responseCode() + ")"); + } + + final String responseBody = response.responseBody(); + + if (responseBody.isEmpty()) { + throw new ExtractionException("Empty response received"); + } - String response = dl.get(url, getCookieHeader(), getExtractorLocalization()).responseBody(); - // trim JSONP part "JP(...)" - response = response.substring(3, response.length() - 1); try { - final JsonArray collection = JsonParser.array().from(response).getArray(1); - for (final Object suggestion : collection) { - if (!(suggestion instanceof JsonArray)) { - continue; - } - final String suggestionStr = ((JsonArray) suggestion).getString(0); - if (suggestionStr == null) { - continue; - } - suggestions.add(suggestionStr); - } - - return suggestions; + final JsonArray suggestions = JsonParser.array() + .from(responseBody) + .getArray(1); // 0: search query, 1: search suggestions, 2: tracking data? + return suggestions.stream() + .filter(JsonArray.class::isInstance) + .map(JsonArray.class::cast) + .map(suggestion -> suggestion.getString(0)) // 0 is the search suggestion + .filter(suggestion -> !isBlank(suggestion)) // Filter blank suggestions + .collect(Collectors.toUnmodifiableList()); } catch (final JsonParserException e) { - throw new ParsingException("Could not parse json response", e); + throw new ParsingException("Could not parse JSON response", e); } } }