package org.schabi.newpipe; import android.util.Log; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.util.ExtractorHelper; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.InterruptedIOException; import java.net.URL; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import javax.net.ssl.HttpsURLConnection; /* * Created by Christian Schabesberger on 28.01.16. * * Copyright (C) Christian Schabesberger 2016 * Downloader.java is part of NewPipe. * * NewPipe is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * NewPipe is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with NewPipe. If not, see . */ public class Downloader implements org.schabi.newpipe.extractor.Downloader { public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0"; private static String mCookies = ""; private static Downloader instance = null; private Downloader() { } public static Downloader getInstance() { if (instance == null) { synchronized (Downloader.class) { if (instance == null) { instance = new Downloader(); } } } return instance; } public static synchronized void setCookies(String cookies) { Downloader.mCookies = cookies; } public static synchronized String getCookies() { return Downloader.mCookies; } /** * Download the text file at the supplied URL as in download(String), * but set the HTTP header field "Accept-Language" to the supplied string. * * @param siteUrl the URL of the text file to return the contents of * @param language the language (usually a 2-character code) to set as the preferred language * @return the contents of the specified text file */ @Override public String download(String siteUrl, String language) throws IOException, ReCaptchaException { Map requestProperties = new HashMap<>(); requestProperties.put("Accept-Language", language); return download(siteUrl, requestProperties); } /** * Download the text file at the supplied URL as in download(String), * but set the HTTP headers included in the customProperties map. * * @param siteUrl the URL of the text file to return the contents of * @param customProperties set request header properties * @return the contents of the specified text file * @throws IOException */ @Override public String download(String siteUrl, Map customProperties) throws IOException, ReCaptchaException { URL url = new URL(siteUrl); HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); Iterator it = customProperties.entrySet().iterator(); while (it.hasNext()) { Map.Entry pair = (Map.Entry) it.next(); con.setRequestProperty((String) pair.getKey(), (String) pair.getValue()); } return dl(con); } /** * Download (via HTTP) the text file located at the supplied URL, and return its contents. * Primarily intended for downloading web pages. * * @param siteUrl the URL of the text file to download * @return the contents of the specified text file */ @Override public String download(String siteUrl) throws IOException, ReCaptchaException { URL url = new URL(siteUrl); HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); //HttpsURLConnection con = NetCipher.getHttpsURLConnection(url); return dl(con); } /** * Common functionality between download(String url) and download(String url, String language) */ private static String dl(HttpsURLConnection con) throws IOException, ReCaptchaException { StringBuilder response = new StringBuilder(); BufferedReader in = null; try { con.setReadTimeout(30 * 1000);// 30s con.setRequestMethod("GET"); con.setRequestProperty("User-Agent", USER_AGENT); if (getCookies().length() > 0) { con.setRequestProperty("Cookie", getCookies()); } in = new BufferedReader(new InputStreamReader(con.getInputStream())); for (Map.Entry> entry : con.getHeaderFields().entrySet()) { System.err.println(entry.getKey() + ": " + entry.getValue()); } String inputLine; while ((inputLine = in.readLine()) != null) { response.append(inputLine); } } catch (Exception e) { Log.e("Downloader", "dl() ----- Exception thrown → " + e.getClass().getName()); if (ExtractorHelper.isInterruptedCaused(e)) { throw new InterruptedIOException(e.getMessage()); } /* * HTTP 429 == Too Many Request * Receive from Youtube.com = ReCaptcha challenge request * See : https://github.com/rg3/youtube-dl/issues/5138 */ if (con.getResponseCode() == 429) { throw new ReCaptchaException("reCaptcha Challenge requested"); } throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e); } finally { if (in != null) { in.close(); } } return response.toString(); } }