moved cookie logic outside

This commit is contained in:
Ritvik Saraf 2018-09-27 02:04:12 +05:30
parent fb1419608a
commit 6b620914b6
5 changed files with 76 additions and 15 deletions

View File

@ -0,0 +1,44 @@
package org.schabi.newpipe.extractor;
import java.util.Collections;
import java.util.List;
import java.util.Map;
public class DownloadRequest {
private final String requestBody;
private final Map<String, List<String>> requestHeaders;
public static final DownloadRequest emptyRequest = new DownloadRequest(null, null);
public DownloadRequest(String requestBody, Map<String, List<String>> headers) {
super();
this.requestBody = requestBody;
if(null != headers) {
this.requestHeaders = headers;
}else {
this.requestHeaders = Collections.emptyMap();
}
}
public String getRequestBody() {
return requestBody;
}
public Map<String, List<String>> getRequestHeaders() {
return requestHeaders;
}
public void setRequestCookies(List<String> cookies){
requestHeaders.put("Cookie", cookies);
}
public List<String> getRequestCookies(){
if(null == requestHeaders) return Collections.emptyList();
List<String> cookies = requestHeaders.get("Cookie");
if(null == cookies)
return Collections.emptyList();
else
return cookies;
}
}

View File

@ -1,8 +1,11 @@
package org.schabi.newpipe.extractor; package org.schabi.newpipe.extractor;
import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import javax.annotation.Nonnull;
public class DownloadResponse { public class DownloadResponse {
private final String responseBody; private final String responseBody;
private final Map<String, List<String>> responseHeaders; private final Map<String, List<String>> responseHeaders;
@ -20,5 +23,15 @@ public class DownloadResponse {
public Map<String, List<String>> getResponseHeaders() { public Map<String, List<String>> getResponseHeaders() {
return responseHeaders; return responseHeaders;
} }
@Nonnull
public List<String> getResponseCookies(){
if(null == responseHeaders) return Collections.emptyList();
List<String> cookies = responseHeaders.get("Set-Cookie");
if(null == cookies)
return Collections.emptyList();
else
return cookies;
}
} }

View File

@ -61,11 +61,11 @@ public interface Downloader {
*/ */
String download(String siteUrl) throws IOException, ReCaptchaException; String download(String siteUrl) throws IOException, ReCaptchaException;
DownloadResponse get(String siteUrl, Map<String, List<String>> requestHeaders) DownloadResponse get(String siteUrl, DownloadRequest request)
throws IOException, ReCaptchaException; throws IOException, ReCaptchaException;
DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException; DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException;
DownloadResponse post(String siteUrl, String requestBody, Map<String, List<String>> requestHeaders) DownloadResponse post(String siteUrl, DownloadRequest request)
throws IOException, ReCaptchaException; throws IOException, ReCaptchaException;
} }

View File

@ -8,6 +8,7 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import org.schabi.newpipe.extractor.DownloadRequest;
import org.schabi.newpipe.extractor.DownloadResponse; import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
@ -145,7 +146,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
DownloadResponse response = downloader.get(getUrl()); DownloadResponse response = downloader.get(getUrl());
String responseBody = response.getResponseBody(); String responseBody = response.getResponseBody();
cookies = response.getResponseHeaders().get("Set-Cookie"); cookies = response.getResponseCookies();
sessionToken = findValue(responseBody, "XSRF_TOKEN"); sessionToken = findValue(responseBody, "XSRF_TOKEN");
String commentsToken = findValue(responseBody, "COMMENTS_TOKEN"); String commentsToken = findValue(responseBody, "COMMENTS_TOKEN");
initPage = getPage(getNextPageUrl(commentsToken)); initPage = getPage(getNextPageUrl(commentsToken));
@ -168,9 +169,10 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
requestHeaders.put("User-Agent", Arrays.asList(USER_AGENT)); requestHeaders.put("User-Agent", Arrays.asList(USER_AGENT));
requestHeaders.put("X-YouTube-Client-Version", Arrays.asList("2.20180815")); requestHeaders.put("X-YouTube-Client-Version", Arrays.asList("2.20180815"));
requestHeaders.put("X-YouTube-Client-Name", Arrays.asList("1")); requestHeaders.put("X-YouTube-Client-Name", Arrays.asList("1"));
requestHeaders.put("Cookie", cookies); DownloadRequest request = new DownloadRequest(postData, requestHeaders);
request.setRequestCookies(cookies);
return NewPipe.getDownloader().post(siteUrl, postData, requestHeaders).getResponseBody(); return NewPipe.getDownloader().post(siteUrl, request).getResponseBody();
} }
private String getDataString(Map<String, String> params) throws UnsupportedEncodingException { private String getDataString(Map<String, String> params) throws UnsupportedEncodingException {

View File

@ -12,6 +12,7 @@ import java.util.Map;
import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.HttpsURLConnection;
import org.schabi.newpipe.extractor.DownloadRequest;
import org.schabi.newpipe.extractor.DownloadResponse; import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
@ -172,11 +173,11 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
} }
@Override @Override
public DownloadResponse get(String siteUrl, Map<String, List<String>> requestHeaders) public DownloadResponse get(String siteUrl, DownloadRequest request)
throws IOException, ReCaptchaException { throws IOException, ReCaptchaException {
URL url = new URL(siteUrl); URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
for (Map.Entry<String, List<String>> pair : requestHeaders.entrySet()) { for (Map.Entry<String, List<String>> pair : request.getRequestHeaders().entrySet()) {
for(String value: pair.getValue()) { for(String value: pair.getValue()) {
con.addRequestProperty(pair.getKey(), value); con.addRequestProperty(pair.getKey(), value);
} }
@ -187,16 +188,16 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
@Override @Override
public DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException { public DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException {
return get(siteUrl, Collections.EMPTY_MAP); return get(siteUrl, DownloadRequest.emptyRequest);
} }
@Override @Override
public DownloadResponse post(String siteUrl, String requestBody, Map<String, List<String>> requestHeaders) public DownloadResponse post(String siteUrl, DownloadRequest request)
throws IOException, ReCaptchaException { throws IOException, ReCaptchaException {
URL url = new URL(siteUrl); URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
con.setRequestMethod("POST"); con.setRequestMethod("POST");
for (Map.Entry<String, List<String>> pair : requestHeaders.entrySet()) { for (Map.Entry<String, List<String>> pair : request.getRequestHeaders().entrySet()) {
for(String value: pair.getValue()) { for(String value: pair.getValue()) {
con.addRequestProperty(pair.getKey(), value); con.addRequestProperty(pair.getKey(), value);
} }
@ -204,11 +205,12 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
// set fields to default if not set already // set fields to default if not set already
setDefaults(con); setDefaults(con);
byte[] postDataBytes = requestBody.toString().getBytes("UTF-8"); if(null != request.getRequestBody()) {
con.setRequestProperty("Content-Length", String.valueOf(postDataBytes.length)); byte[] postDataBytes = request.getRequestBody().getBytes("UTF-8");
con.setRequestProperty("Content-Length", String.valueOf(postDataBytes.length));
con.setDoOutput(true); con.setDoOutput(true);
con.getOutputStream().write(postDataBytes); con.getOutputStream().write(postDataBytes);
}
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
try (BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()))) { try (BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()))) {