lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

moved cookie logic outside

This commit is contained in:
Ritvik Saraf 2018-09-27 02:04:12 +05:30
parent fb1419608a
commit 6b620914b6
5 changed files with 76 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
extractor/src/main/java/org/schabi/newpipe/extractor/DownloadRequest.java Normal file
View File

@ -0,0 +1,44 @@
package org.schabi.newpipe.extractor;
import java.util.Collections;
import java.util.List;
import java.util.Map;
public class DownloadRequest {
private final String requestBody;
private final Map<String, List<String>> requestHeaders;
public static final DownloadRequest emptyRequest = new DownloadRequest(null, null);
public DownloadRequest(String requestBody, Map<String, List<String>> headers) {
super();
this.requestBody = requestBody;
if(null != headers) {
this.requestHeaders = headers;
}else {
this.requestHeaders = Collections.emptyMap();
}
}
public String getRequestBody() {
return requestBody;
}
public Map<String, List<String>> getRequestHeaders() {
return requestHeaders;
}
public void setRequestCookies(List<String> cookies){
requestHeaders.put("Cookie", cookies);
}
public List<String> getRequestCookies(){
if(null == requestHeaders) return Collections.emptyList();
List<String> cookies = requestHeaders.get("Cookie");
if(null == cookies)
return Collections.emptyList();
else
return cookies;
}
}

13
extractor/src/main/java/org/schabi/newpipe/extractor/DownloadResponse.java
View File

@ -1,8 +1,11 @@
package org.schabi.newpipe.extractor;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
public class DownloadResponse {
private final String responseBody;
private final Map<String, List<String>> responseHeaders;
@ -20,5 +23,15 @@ public class DownloadResponse {
public Map<String, List<String>> getResponseHeaders() {
return responseHeaders;
}
@Nonnull
public List<String> getResponseCookies(){
if(null == responseHeaders) return Collections.emptyList();
List<String> cookies = responseHeaders.get("Set-Cookie");
if(null == cookies)
return Collections.emptyList();
else
return cookies;
}
}

4
extractor/src/main/java/org/schabi/newpipe/extractor/Downloader.java
View File

@ -61,11 +61,11 @@ public interface Downloader {
*/
String download(String siteUrl) throws IOException, ReCaptchaException;
DownloadResponse get(String siteUrl, Map<String, List<String>> requestHeaders)
DownloadResponse get(String siteUrl, DownloadRequest request)
throws IOException, ReCaptchaException;
DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException;
DownloadResponse post(String siteUrl, String requestBody, Map<String, List<String>> requestHeaders)
DownloadResponse post(String siteUrl, DownloadRequest request)
throws IOException, ReCaptchaException;
}

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java
View File

@ -8,6 +8,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.schabi.newpipe.extractor.DownloadRequest;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
@ -145,7 +146,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
DownloadResponse response = downloader.get(getUrl());
String responseBody = response.getResponseBody();
cookies = response.getResponseHeaders().get("Set-Cookie");
cookies = response.getResponseCookies();
sessionToken = findValue(responseBody, "XSRF_TOKEN");
String commentsToken = findValue(responseBody, "COMMENTS_TOKEN");
initPage = getPage(getNextPageUrl(commentsToken));
@ -168,9 +169,10 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
requestHeaders.put("User-Agent", Arrays.asList(USER_AGENT));
requestHeaders.put("X-YouTube-Client-Version", Arrays.asList("2.20180815"));
requestHeaders.put("X-YouTube-Client-Name", Arrays.asList("1"));
requestHeaders.put("Cookie", cookies);
DownloadRequest request = new DownloadRequest(postData, requestHeaders);
request.setRequestCookies(cookies);
return NewPipe.getDownloader().post(siteUrl, postData, requestHeaders).getResponseBody();
return NewPipe.getDownloader().post(siteUrl, request).getResponseBody();
}
private String getDataString(Map<String, String> params) throws UnsupportedEncodingException {

22
extractor/src/test/java/org/schabi/newpipe/Downloader.java
View File

@ -12,6 +12,7 @@ import java.util.Map;
import javax.net.ssl.HttpsURLConnection;
import org.schabi.newpipe.extractor.DownloadRequest;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
@ -172,11 +173,11 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
}
@Override
public DownloadResponse get(String siteUrl, Map<String, List<String>> requestHeaders)
public DownloadResponse get(String siteUrl, DownloadRequest request)
throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
for (Map.Entry<String, List<String>> pair : requestHeaders.entrySet()) {
for (Map.Entry<String, List<String>> pair : request.getRequestHeaders().entrySet()) {
for(String value: pair.getValue()) {
con.addRequestProperty(pair.getKey(), value);
}
@ -187,16 +188,16 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
@Override
public DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException {
return get(siteUrl, Collections.EMPTY_MAP);
return get(siteUrl, DownloadRequest.emptyRequest);
}
@Override
public DownloadResponse post(String siteUrl, String requestBody, Map<String, List<String>> requestHeaders)
public DownloadResponse post(String siteUrl, DownloadRequest request)
throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
con.setRequestMethod("POST");
for (Map.Entry<String, List<String>> pair : requestHeaders.entrySet()) {
for (Map.Entry<String, List<String>> pair : request.getRequestHeaders().entrySet()) {
for(String value: pair.getValue()) {
con.addRequestProperty(pair.getKey(), value);
}
@ -204,11 +205,12 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
// set fields to default if not set already
setDefaults(con);
byte[] postDataBytes = requestBody.toString().getBytes("UTF-8");
con.setRequestProperty("Content-Length", String.valueOf(postDataBytes.length));
con.setDoOutput(true);
con.getOutputStream().write(postDataBytes);
if(null != request.getRequestBody()) {
byte[] postDataBytes = request.getRequestBody().getBytes("UTF-8");
con.setRequestProperty("Content-Length", String.valueOf(postDataBytes.length));
con.setDoOutput(true);
con.getOutputStream().write(postDataBytes);
}
StringBuilder sb = new StringBuilder();
try (BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()))) {