2017-08-04 16:21:45 +02:00
|
|
|
package org.schabi.newpipe.extractor.services.soundcloud;
|
|
|
|
|
2017-08-16 04:40:03 +02:00
|
|
|
import com.grack.nanojson.JsonArray;
|
|
|
|
import com.grack.nanojson.JsonObject;
|
|
|
|
import com.grack.nanojson.JsonParser;
|
|
|
|
import com.grack.nanojson.JsonParserException;
|
2017-08-04 16:21:45 +02:00
|
|
|
import org.jsoup.Jsoup;
|
|
|
|
import org.jsoup.nodes.Document;
|
|
|
|
import org.jsoup.nodes.Element;
|
2019-10-22 18:21:24 +02:00
|
|
|
import org.jsoup.select.Elements;
|
|
|
|
import org.schabi.newpipe.extractor.DownloadResponse;
|
2017-08-04 16:21:45 +02:00
|
|
|
import org.schabi.newpipe.extractor.Downloader;
|
|
|
|
import org.schabi.newpipe.extractor.NewPipe;
|
2018-02-24 22:20:50 +01:00
|
|
|
import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
|
2019-10-22 18:21:24 +02:00
|
|
|
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
2017-08-04 16:21:45 +02:00
|
|
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
|
|
|
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
2018-02-24 22:20:50 +01:00
|
|
|
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
2017-08-04 16:21:45 +02:00
|
|
|
import org.schabi.newpipe.extractor.utils.Parser;
|
|
|
|
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
|
|
|
|
|
2018-02-22 15:52:38 +01:00
|
|
|
import javax.annotation.Nonnull;
|
2017-08-06 22:20:15 +02:00
|
|
|
import java.io.IOException;
|
|
|
|
import java.net.URLEncoder;
|
|
|
|
import java.text.ParseException;
|
|
|
|
import java.text.SimpleDateFormat;
|
2019-10-22 18:21:24 +02:00
|
|
|
import java.util.Collections;
|
2017-08-06 22:20:15 +02:00
|
|
|
import java.util.Date;
|
2018-04-08 03:05:16 +02:00
|
|
|
import java.util.HashMap;
|
2017-08-06 22:20:15 +02:00
|
|
|
|
2018-03-04 21:30:31 +01:00
|
|
|
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
|
|
|
|
|
2017-08-04 16:21:45 +02:00
|
|
|
public class SoundcloudParsingHelper {
|
2017-08-06 22:20:15 +02:00
|
|
|
private static String clientId;
|
2018-02-21 09:23:57 +01:00
|
|
|
|
2017-08-04 16:21:45 +02:00
|
|
|
private SoundcloudParsingHelper() {
|
|
|
|
}
|
|
|
|
|
2019-10-22 18:21:24 +02:00
|
|
|
public static String clientId() throws ExtractionException, IOException {
|
2017-08-06 22:20:15 +02:00
|
|
|
if (clientId != null && !clientId.isEmpty()) return clientId;
|
|
|
|
|
2017-08-04 16:21:45 +02:00
|
|
|
Downloader dl = NewPipe.getDownloader();
|
2019-10-22 18:21:24 +02:00
|
|
|
clientId = "LHzSAKe8eP9Yy3FgBugfBapRPLncO6Ng"; // Updated on 22/10/19
|
|
|
|
final String apiUrl = "https://api.soundcloud.com/connect?client_id=" + clientId;
|
|
|
|
// Should return 200 to indicate that the client id is valid, a 401 is returned otherwise.
|
|
|
|
// In that case, the fallback method is used.
|
|
|
|
if (dl.head(apiUrl).getResponseCode() == 200) {
|
|
|
|
return clientId;
|
|
|
|
}
|
2017-08-04 16:21:45 +02:00
|
|
|
|
2019-10-22 18:21:24 +02:00
|
|
|
final DownloadResponse download = dl.get("https://soundcloud.com");
|
|
|
|
String response = download.getResponseBody();
|
2018-04-08 03:05:16 +02:00
|
|
|
final String clientIdPattern = ",client_id:\"(.*?)\"";
|
|
|
|
|
2019-10-22 18:21:24 +02:00
|
|
|
Document doc = Jsoup.parse(response);
|
|
|
|
final Elements possibleScripts = doc.select("script[src*=\"sndcdn.com/assets/\"][src$=\".js\"]");
|
|
|
|
// The one containing the client id will likely be the last one
|
|
|
|
Collections.reverse(possibleScripts);
|
|
|
|
|
|
|
|
final HashMap<String, String> headers = new HashMap<>();
|
|
|
|
headers.put("Range", "bytes=0-16384");
|
|
|
|
|
|
|
|
for (Element element : possibleScripts) {
|
|
|
|
final String srcUrl = element.attr("src");
|
|
|
|
if (srcUrl != null && !srcUrl.isEmpty()) {
|
|
|
|
try {
|
|
|
|
return clientId = Parser.matchGroup1(clientIdPattern, dl.download(srcUrl, headers));
|
|
|
|
} catch (RegexException ignored) {
|
|
|
|
// Ignore it and proceed to try searching other script
|
|
|
|
}
|
|
|
|
}
|
2018-04-08 03:05:16 +02:00
|
|
|
}
|
|
|
|
|
2019-10-22 18:21:24 +02:00
|
|
|
// Officially give up
|
|
|
|
throw new ExtractionException("Couldn't extract client id");
|
2017-08-04 16:21:45 +02:00
|
|
|
}
|
|
|
|
|
2017-08-06 22:20:15 +02:00
|
|
|
public static String toDateString(String time) throws ParsingException {
|
2017-08-04 16:21:45 +02:00
|
|
|
try {
|
2017-08-06 22:20:15 +02:00
|
|
|
Date date;
|
|
|
|
// Have two date formats, one for the 'api.soundc...' and the other 'api-v2.soundc...'.
|
|
|
|
try {
|
|
|
|
date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").parse(time);
|
|
|
|
} catch (Exception e) {
|
|
|
|
date = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse(time);
|
|
|
|
}
|
2017-08-04 16:21:45 +02:00
|
|
|
|
2017-08-06 22:20:15 +02:00
|
|
|
SimpleDateFormat newDateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
|
|
|
return newDateFormat.format(date);
|
|
|
|
} catch (ParseException e) {
|
|
|
|
throw new ParsingException(e.getMessage(), e);
|
|
|
|
}
|
|
|
|
}
|
2017-08-04 16:21:45 +02:00
|
|
|
|
2017-08-06 22:20:15 +02:00
|
|
|
/**
|
2017-11-30 11:57:09 +01:00
|
|
|
* Call the endpoint "/resolve" of the api.<p>
|
|
|
|
*
|
2017-08-06 22:20:15 +02:00
|
|
|
* See https://developers.soundcloud.com/docs/api/reference#resolve
|
|
|
|
*/
|
2019-10-22 18:21:24 +02:00
|
|
|
public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException {
|
2017-08-06 22:20:15 +02:00
|
|
|
String apiUrl = "https://api.soundcloud.com/resolve"
|
|
|
|
+ "?url=" + URLEncoder.encode(url, "UTF-8")
|
|
|
|
+ "&client_id=" + clientId();
|
2017-08-04 16:21:45 +02:00
|
|
|
|
2017-08-16 04:40:03 +02:00
|
|
|
try {
|
2017-11-28 13:37:01 +01:00
|
|
|
return JsonParser.object().from(downloader.download(apiUrl));
|
2017-08-16 04:40:03 +02:00
|
|
|
} catch (JsonParserException e) {
|
|
|
|
throw new ParsingException("Could not parse json response", e);
|
|
|
|
}
|
2017-08-06 22:20:15 +02:00
|
|
|
}
|
2017-08-04 16:21:45 +02:00
|
|
|
|
2017-08-06 22:20:15 +02:00
|
|
|
/**
|
2017-11-30 11:57:09 +01:00
|
|
|
* Fetch the embed player with the apiUrl and return the canonical url (like the permalink_url from the json api).
|
2017-08-06 22:20:15 +02:00
|
|
|
*
|
|
|
|
* @return the url resolved
|
|
|
|
*/
|
|
|
|
public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException, ReCaptchaException, ParsingException {
|
|
|
|
|
|
|
|
String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url="
|
|
|
|
+ URLEncoder.encode(apiUrl, "UTF-8"));
|
|
|
|
|
|
|
|
return Jsoup.parse(response).select("link[rel=\"canonical\"]").first().attr("abs:href");
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2017-11-30 11:57:09 +01:00
|
|
|
* Fetch the embed player with the url and return the id (like the id from the json api).
|
2017-08-06 22:20:15 +02:00
|
|
|
*
|
2018-02-22 15:52:38 +01:00
|
|
|
* @return the resolved id
|
2017-08-06 22:20:15 +02:00
|
|
|
*/
|
|
|
|
public static String resolveIdWithEmbedPlayer(String url) throws IOException, ReCaptchaException, ParsingException {
|
|
|
|
|
|
|
|
String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url="
|
|
|
|
+ URLEncoder.encode(url, "UTF-8"));
|
2019-06-26 00:56:03 +02:00
|
|
|
// handle playlists / sets different and get playlist id via uir field in JSON
|
|
|
|
if (url.contains("sets") && !url.endsWith("sets") && !url.endsWith("sets/"))
|
|
|
|
return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response);
|
2019-05-31 20:15:36 +02:00
|
|
|
return Parser.matchGroup1(",\"id\":(([^}\\n])*?),", response);
|
2017-08-06 22:20:15 +02:00
|
|
|
}
|
|
|
|
|
2018-02-22 15:52:38 +01:00
|
|
|
/**
|
|
|
|
* Fetch the users from the given api and commit each of them to the collector.
|
|
|
|
* <p>
|
2018-02-24 22:20:50 +01:00
|
|
|
* This differ from {@link #getUsersFromApi(ChannelInfoItemsCollector, String)} in the sense that they will always
|
2018-02-22 15:52:38 +01:00
|
|
|
* get MIN_ITEMS or more.
|
|
|
|
*
|
|
|
|
* @param minItems the method will return only when it have extracted that many items (equal or more)
|
|
|
|
*/
|
2018-02-24 22:20:50 +01:00
|
|
|
public static String getUsersFromApiMinItems(int minItems, ChannelInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
|
|
|
|
String nextPageUrl = SoundcloudParsingHelper.getUsersFromApi(collector, apiUrl);
|
2018-02-22 15:52:38 +01:00
|
|
|
|
2018-03-11 21:50:40 +01:00
|
|
|
while (!nextPageUrl.isEmpty() && collector.getItems().size() < minItems) {
|
2018-02-24 22:20:50 +01:00
|
|
|
nextPageUrl = SoundcloudParsingHelper.getUsersFromApi(collector, nextPageUrl);
|
2018-02-22 15:52:38 +01:00
|
|
|
}
|
|
|
|
|
2018-02-24 22:20:50 +01:00
|
|
|
return nextPageUrl;
|
2018-02-22 15:52:38 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Fetch the user items from the given api and commit each of them to the collector.
|
|
|
|
*
|
|
|
|
* @return the next streams url, empty if don't have
|
|
|
|
*/
|
2018-02-24 22:20:50 +01:00
|
|
|
public static String getUsersFromApi(ChannelInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
|
2018-02-22 15:52:38 +01:00
|
|
|
String response = NewPipe.getDownloader().download(apiUrl);
|
|
|
|
JsonObject responseObject;
|
|
|
|
try {
|
|
|
|
responseObject = JsonParser.object().from(response);
|
|
|
|
} catch (JsonParserException e) {
|
|
|
|
throw new ParsingException("Could not parse json response", e);
|
|
|
|
}
|
|
|
|
|
|
|
|
JsonArray responseCollection = responseObject.getArray("collection");
|
|
|
|
for (Object o : responseCollection) {
|
|
|
|
if (o instanceof JsonObject) {
|
|
|
|
JsonObject object = (JsonObject) o;
|
|
|
|
collector.commit(new SoundcloudChannelInfoItemExtractor(object));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-02-24 22:20:50 +01:00
|
|
|
String nextPageUrl;
|
2018-02-22 15:52:38 +01:00
|
|
|
try {
|
2018-02-24 22:20:50 +01:00
|
|
|
nextPageUrl = responseObject.getString("next_href");
|
|
|
|
if (!nextPageUrl.contains("client_id=")) nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
|
2018-02-22 15:52:38 +01:00
|
|
|
} catch (Exception ignored) {
|
2018-02-24 22:20:50 +01:00
|
|
|
nextPageUrl = "";
|
2018-02-22 15:52:38 +01:00
|
|
|
}
|
|
|
|
|
2018-02-24 22:20:50 +01:00
|
|
|
return nextPageUrl;
|
2018-02-22 15:52:38 +01:00
|
|
|
}
|
|
|
|
|
2017-08-06 22:20:15 +02:00
|
|
|
/**
|
|
|
|
* Fetch the streams from the given api and commit each of them to the collector.
|
|
|
|
* <p>
|
2018-02-24 22:20:50 +01:00
|
|
|
* This differ from {@link #getStreamsFromApi(StreamInfoItemsCollector, String)} in the sense that they will always
|
2017-08-06 22:20:15 +02:00
|
|
|
* get MIN_ITEMS or more items.
|
|
|
|
*
|
|
|
|
* @param minItems the method will return only when it have extracted that many items (equal or more)
|
|
|
|
*/
|
2018-02-24 22:20:50 +01:00
|
|
|
public static String getStreamsFromApiMinItems(int minItems, StreamInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
|
|
|
|
String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
|
2017-08-06 22:20:15 +02:00
|
|
|
|
2018-03-11 21:50:40 +01:00
|
|
|
while (!nextPageUrl.isEmpty() && collector.getItems().size() < minItems) {
|
2018-02-24 22:20:50 +01:00
|
|
|
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextPageUrl);
|
2017-08-04 16:21:45 +02:00
|
|
|
}
|
2017-08-06 22:20:15 +02:00
|
|
|
|
2018-02-24 22:20:50 +01:00
|
|
|
return nextPageUrl;
|
2017-08-04 16:21:45 +02:00
|
|
|
}
|
|
|
|
|
2017-08-06 22:20:15 +02:00
|
|
|
/**
|
|
|
|
* Fetch the streams from the given api and commit each of them to the collector.
|
|
|
|
*
|
|
|
|
* @return the next streams url, empty if don't have
|
|
|
|
*/
|
2018-02-24 22:20:50 +01:00
|
|
|
public static String getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException {
|
2017-08-06 22:20:15 +02:00
|
|
|
String response = NewPipe.getDownloader().download(apiUrl);
|
2017-08-16 04:40:03 +02:00
|
|
|
JsonObject responseObject;
|
|
|
|
try {
|
|
|
|
responseObject = JsonParser.object().from(response);
|
|
|
|
} catch (JsonParserException e) {
|
|
|
|
throw new ParsingException("Could not parse json response", e);
|
|
|
|
}
|
2017-08-06 22:20:15 +02:00
|
|
|
|
2017-08-16 04:40:03 +02:00
|
|
|
JsonArray responseCollection = responseObject.getArray("collection");
|
|
|
|
for (Object o : responseCollection) {
|
2017-09-11 18:18:17 +02:00
|
|
|
if (o instanceof JsonObject) {
|
|
|
|
JsonObject object = (JsonObject) o;
|
|
|
|
collector.commit(new SoundcloudStreamInfoItemExtractor(charts ? object.getObject("track") : object));
|
|
|
|
}
|
2017-08-06 22:20:15 +02:00
|
|
|
}
|
|
|
|
|
2018-02-24 22:20:50 +01:00
|
|
|
String nextPageUrl;
|
2017-08-04 16:21:45 +02:00
|
|
|
try {
|
2018-02-24 22:20:50 +01:00
|
|
|
nextPageUrl = responseObject.getString("next_href");
|
|
|
|
if (!nextPageUrl.contains("client_id=")) nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
|
2017-08-06 22:20:15 +02:00
|
|
|
} catch (Exception ignored) {
|
2018-02-24 22:20:50 +01:00
|
|
|
nextPageUrl = "";
|
2017-08-04 16:21:45 +02:00
|
|
|
}
|
|
|
|
|
2018-02-24 22:20:50 +01:00
|
|
|
return nextPageUrl;
|
2017-08-06 22:20:15 +02:00
|
|
|
}
|
2017-08-20 10:03:41 +02:00
|
|
|
|
2018-02-24 22:20:50 +01:00
|
|
|
public static String getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, ParsingException, IOException {
|
2017-08-20 10:03:41 +02:00
|
|
|
return getStreamsFromApi(collector, apiUrl, false);
|
|
|
|
}
|
2018-02-21 09:23:57 +01:00
|
|
|
|
|
|
|
@Nonnull
|
|
|
|
static String getUploaderUrl(JsonObject object) {
|
|
|
|
String url = object.getObject("user").getString("permalink_url", "");
|
|
|
|
return replaceHttpWithHttps(url);
|
|
|
|
}
|
|
|
|
|
|
|
|
@Nonnull
|
|
|
|
static String getAvatarUrl(JsonObject object) {
|
|
|
|
String url = object.getObject("user", new JsonObject()).getString("avatar_url", "");
|
|
|
|
return replaceHttpWithHttps(url);
|
|
|
|
}
|
|
|
|
|
|
|
|
public static String getUploaderName(JsonObject object) {
|
|
|
|
return object.getObject("user").getString("username", "");
|
|
|
|
}
|
2017-08-04 16:21:45 +02:00
|
|
|
}
|