lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Support YouTube's new continuations for search

This commit is contained in:
wb9688 2020-07-26 10:01:03 +02:00
parent df28a087de
commit 4c987a5302
2 changed files with 98 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
extractor/src/main/java/org/schabi/newpipe/extractor/Page.java
View File

@ -8,35 +8,45 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class Page implements Serializable {
private final String url;
private final String id;
private final List<String> ids;
private final Map<String, String> cookies;
public Page(final String url, final List<String> ids, final Map<String, String> cookies) {
public Page(final String url, final String id, final List<String> ids, final Map<String, String> cookies) {
this.url = url;
this.id = id;
this.ids = ids;
this.cookies = cookies;
}
public Page(final String url) {
this(url, null, null);
this(url, null, null, null);
}
public Page(final String url, final String id) {
this(url, id, null, null);
}
public Page(final String url, final Map<String, String> cookies) {
this(url, null, cookies);
this(url, null, null, cookies);
}
public Page(final List<String> ids) {
this(null, ids, null);
this(null, null, ids, null);
}
public Page(final List<String> ids, final Map<String, String> cookies) {
this(null, ids, cookies);
this(null, null, ids, cookies);
}
public String getUrl() {
return url;
}
public String getId() {
return id;
}
public List<String> getIds() {
return ids;
}

94
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -2,6 +2,10 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
@ -14,11 +18,18 @@ import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*
@ -104,12 +115,16 @@ public class YoutubeSearchExtractor extends SearchExtractor {
Page nextPage = null;
for (Object section : sections) {
final JsonObject itemSectionRenderer = ((JsonObject) section).getObject("itemSectionRenderer");
for (final Object section : sections) {
if (((JsonObject) section).has("itemSectionRenderer")) {
final JsonObject itemSectionRenderer = ((JsonObject) section).getObject("itemSectionRenderer");
collectStreamsFrom(collector, itemSectionRenderer.getArray("contents"));
collectStreamsFrom(collector, itemSectionRenderer.getArray("contents"));
nextPage = getNextPageFrom(itemSectionRenderer.getArray("continuations"));
nextPage = getNextPageFrom(itemSectionRenderer.getArray("continuations"));
} else if (((JsonObject) section).has("continuationItemRenderer")) {
nextPage = getNewNextPageFrom(((JsonObject) section).getObject("continuationItemRenderer"));
}
}
return new InfoItemsPage<>(collector, nextPage);
@ -122,15 +137,58 @@ public class YoutubeSearchExtractor extends SearchExtractor {
}
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
final JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response")
.getObject("continuationContents").getObject("itemSectionContinuation");
if (page.getId() == null) {
final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
collectStreamsFrom(collector, itemSectionRenderer.getArray("contents"));
final JsonArray continuations = itemSectionRenderer.getArray("continuations");
final JsonObject itemSectionContinuation = ajaxJson.getObject(1).getObject("response")
.getObject("continuationContents").getObject("itemSectionContinuation");
return new InfoItemsPage<>(collector, getNextPageFrom(continuations));
collectStreamsFrom(collector, itemSectionContinuation.getArray("contents"));
final JsonArray continuations = itemSectionContinuation.getArray("continuations");
return new InfoItemsPage<>(collector, getNextPageFrom(continuations));
} else {
// @formatter:off
final byte[] json = JsonWriter.string()
.object()
.object("context")
.object("client")
.value("hl", "en")
.value("gl", getExtractorContentCountry().getCountryCode())
.value("clientName", "WEB")
.value("clientVersion", getClientVersion())
.value("utcOffsetMinutes", 0)
.end()
.object("request").end()
.object("user").end()
.end()
.value("continuation", page.getId())
.end().done().getBytes("UTF-8");
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();
headers.put("Origin", Collections.singletonList("https://www.youtube.com"));
headers.put("Referer", Collections.singletonList(this.getUrl()));
headers.put("Content-Type", Collections.singletonList("application/json"));
final String responseBody = getValidJsonResponseBody(getDownloader().post(page.getUrl(), headers, json));
final JsonObject ajaxJson;
try {
ajaxJson = JsonParser.object().from(responseBody);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse JSON", e);
}
final JsonArray continuationItems = ajaxJson.getArray("onResponseReceivedCommands")
.getObject(0).getObject("appendContinuationItemsAction").getArray("continuationItems");
final JsonArray contents = continuationItems.getObject(0).getObject("itemSectionRenderer").getArray("contents");
collectStreamsFrom(collector, contents);
return new InfoItemsPage<>(collector, getNewNextPageFrom(continuationItems.getObject(1).getObject("continuationItemRenderer")));
}
}
private void collectStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray videos) throws NothingFoundException, ParsingException {
@ -162,4 +220,18 @@ public class YoutubeSearchExtractor extends SearchExtractor {
return new Page(getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation
+ "&itct=" + clickTrackingParams);
}
private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) {
if (isNullOrEmpty(continuationItemRenderer)) {
return null;
}
final String token = continuationItemRenderer.getObject("continuationEndpoint")
.getObject("continuationCommand").getString("token");
// FIXME: Key needs to be extracted
final String url = "https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
return new Page(url, token);
}
}