lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #627 from TiA4f8R/use-snd-api-v2-everywhere 

[SoundCloud] Use a lightweight request to check if the hardcoded client_id is valid, fix the extraction of mobile URLs and more
This commit is contained in:
Tobi 2021-05-23 22:52:40 +02:00 committed by GitHub
parent 636e27333b a00fdcbd3d
commit ff005122bf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 407 additions and 309 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

193
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View File

@ -16,7 +16,6 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChannelInfoItemExtractor; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamInfoItemExtractor; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.JsonUtils;
@ -41,8 +40,10 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.Utils.*; import static org.schabi.newpipe.extractor.utils.Utils.*;
public class SoundcloudParsingHelper { public class SoundcloudParsingHelper {
private static final String HARDCODED_CLIENT_ID = "NcIaRZItQCNQp3Vq9Plvzf7tvjmVJnF6"; // Updated on 26/04/21 private static final String HARDCODED_CLIENT_ID =
"TT9Uj7PkasKPYxBlhLNxg2nFm9cLcKmv"; // Updated on 15/05/21
private static String clientId; private static String clientId;
public static final String SOUNDCLOUD_API_V2_URL = "https://api-v2.soundcloud.com/";
private SoundcloudParsingHelper() { private SoundcloudParsingHelper() {
} }
@ -50,7 +51,7 @@ public class SoundcloudParsingHelper {
public static synchronized String clientId() throws ExtractionException, IOException { public static synchronized String clientId() throws ExtractionException, IOException {
if (!isNullOrEmpty(clientId)) return clientId; if (!isNullOrEmpty(clientId)) return clientId;
Downloader dl = NewPipe.getDownloader(); final Downloader dl = NewPipe.getDownloader();
clientId = HARDCODED_CLIENT_ID; clientId = HARDCODED_CLIENT_ID;
if (checkIfHardcodedClientIdIsValid()) { if (checkIfHardcodedClientIdIsValid()) {
return clientId; return clientId;
@ -62,20 +63,23 @@ public class SoundcloudParsingHelper {
final String responseBody = download.responseBody(); final String responseBody = download.responseBody();
final String clientIdPattern = ",client_id:\"(.*?)\""; final String clientIdPattern = ",client_id:\"(.*?)\"";
Document doc = Jsoup.parse(responseBody); final Document doc = Jsoup.parse(responseBody);
final Elements possibleScripts = doc.select("script[src*=\"sndcdn.com/assets/\"][src$=\".js\"]"); final Elements possibleScripts = doc.select(
"script[src*=\"sndcdn.com/assets/\"][src$=\".js\"]");
// The one containing the client id will likely be the last one // The one containing the client id will likely be the last one
Collections.reverse(possibleScripts); Collections.reverse(possibleScripts);
final HashMap<String, List<String>> headers = new HashMap<>(); final HashMap<String, List<String>> headers = new HashMap<>();
headers.put("Range", singletonList("bytes=0-50000")); headers.put("Range", singletonList("bytes=0-50000"));
for (Element element : possibleScripts) { for (final Element element : possibleScripts) {
final String srcUrl = element.attr("src"); final String srcUrl = element.attr("src");
if (!isNullOrEmpty(srcUrl)) { if (!isNullOrEmpty(srcUrl)) {
try { try {
return clientId = Parser.matchGroup1(clientIdPattern, dl.get(srcUrl, headers).responseBody()); clientId = Parser.matchGroup1(clientIdPattern, dl.get(srcUrl, headers)
} catch (RegexException ignored) { .responseBody());
return clientId;
} catch (final RegexException ignored) {
// Ignore it and proceed to try searching other script // Ignore it and proceed to try searching other script
} }
} }
@ -85,77 +89,83 @@ public class SoundcloudParsingHelper {
throw new ExtractionException("Couldn't extract client id"); throw new ExtractionException("Couldn't extract client id");
} }
static boolean checkIfHardcodedClientIdIsValid() { static boolean checkIfHardcodedClientIdIsValid() throws IOException, ReCaptchaException {
try { final int responseCode = NewPipe.getDownloader().get(SOUNDCLOUD_API_V2_URL + "?client_id="
SoundcloudStreamExtractor e = (SoundcloudStreamExtractor) SoundCloud + HARDCODED_CLIENT_ID).responseCode();
.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon"); // If the response code is 404, it means that the client_id is valid; otherwise,
e.fetchPage(); // it should be not valid
return !e.getAudioStreams().isEmpty(); return responseCode == 404;
} catch (Exception ignored) {
// No need to throw an exception here. If something went wrong, the client_id is wrong
return false;
}
} }
public static OffsetDateTime parseDateFrom(String textualUploadDate) throws ParsingException { public static OffsetDateTime parseDateFrom(final String textualUploadDate)
throws ParsingException {
try { try {
return OffsetDateTime.parse(textualUploadDate); return OffsetDateTime.parse(textualUploadDate);
} catch (DateTimeParseException e1) { } catch (final DateTimeParseException e1) {
try { try {
return OffsetDateTime.parse(textualUploadDate, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss +0000")); return OffsetDateTime.parse(textualUploadDate, DateTimeFormatter
} catch (DateTimeParseException e2) { .ofPattern("yyyy/MM/dd HH:mm:ss +0000"));
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"" + ", " + e1.getMessage(), e2); } catch (final DateTimeParseException e2) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\""
+ ", " + e1.getMessage(), e2);
} }
} }
} }
/** /**
* Call the endpoint "/resolve" of the api.<p> * Call the endpoint "/resolve" of the API.<p>
* <p> * <p>
* See https://developers.soundcloud.com/docs/api/reference#resolve * See https://developers.soundcloud.com/docs/api/reference#resolve
*/ */
public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException { public static JsonObject resolveFor(@Nonnull final Downloader downloader, final String url)
String apiUrl = "https://api-v2.soundcloud.com/resolve" throws IOException, ExtractionException {
+ "?url=" + URLEncoder.encode(url, UTF_8) final String apiUrl = SOUNDCLOUD_API_V2_URL + "resolve"
+ "?url=" + URLEncoder.encode(url, UTF_8)
+ "&client_id=" + clientId(); + "&client_id=" + clientId();
try { try {
final String response = downloader.get(apiUrl, SoundCloud.getLocalization()).responseBody(); final String response = downloader.get(apiUrl, SoundCloud.getLocalization())
.responseBody();
return JsonParser.object().from(response); return JsonParser.object().from(response);
} catch (JsonParserException e) { } catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e); throw new ParsingException("Could not parse json response", e);
} }
} }
/** /**
* Fetch the embed player with the apiUrl and return the canonical url (like the permalink_url from the json api). * Fetch the embed player with the apiUrl and return the canonical url (like the permalink_url
* from the json API).
* *
* @return the url resolved * @return the url resolved
*/ */
public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException, ReCaptchaException { public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOException,
ReCaptchaException {
String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url=" final String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(apiUrl, UTF_8), SoundCloud.getLocalization()).responseBody(); + URLEncoder.encode(apiUrl, UTF_8), SoundCloud.getLocalization()).responseBody();
return Jsoup.parse(response).select("link[rel=\"canonical\"]").first().attr("abs:href"); return Jsoup.parse(response).select("link[rel=\"canonical\"]").first()
.attr("abs:href");
} }
/** /**
* Fetch the widget API with the url and return the id (like the id from the json api). * Fetch the widget API with the url and return the id (like the id from the json API).
* *
* @return the resolved id * @return the resolved id
*/ */
public static String resolveIdWithWidgetApi(String urlString) throws IOException, ReCaptchaException, ParsingException { public static String resolveIdWithWidgetApi(String urlString) throws IOException,
ParsingException {
// Remove the tailing slash from URLs due to issues with the SoundCloud API // Remove the tailing slash from URLs due to issues with the SoundCloud API
if (urlString.charAt(urlString.length() - 1) == '/') urlString = urlString.substring(0, urlString.length() - 1); if (urlString.charAt(urlString.length() - 1) == '/') urlString = urlString.substring(0,
// Make URL lower case and remove www. if it exists. urlString.length() - 1);
// Make URL lower case and remove m. and www. if it exists.
// Without doing this, the widget API does not recognize the URL. // Without doing this, the widget API does not recognize the URL.
urlString = Utils.removeWWWFromUrl(urlString.toLowerCase()); urlString = Utils.removeMAndWWWFromUrl(urlString.toLowerCase());
final URL url; final URL url;
try { try {
url = Utils.stringToURL(urlString); url = Utils.stringToURL(urlString);
} catch (MalformedURLException e) { } catch (final MalformedURLException e) {
throw new IllegalArgumentException("The given URL is not valid"); throw new IllegalArgumentException("The given URL is not valid");
} }
@ -167,22 +177,27 @@ public class SoundcloudParsingHelper {
SoundCloud.getLocalization()).responseBody(); SoundCloud.getLocalization()).responseBody();
final JsonObject o = JsonParser.object().from(response); final JsonObject o = JsonParser.object().from(response);
return String.valueOf(JsonUtils.getValue(o, "id")); return String.valueOf(JsonUtils.getValue(o, "id"));
} catch (JsonParserException e) { } catch (final JsonParserException e) {
throw new ParsingException("Could not parse JSON response", e); throw new ParsingException("Could not parse JSON response", e);
} catch (ExtractionException e) { } catch (final ExtractionException e) {
throw new ParsingException("Could not resolve id with embedded player. ClientId not extracted", e); throw new ParsingException(
"Could not resolve id with embedded player. ClientId not extracted", e);
} }
} }
/** /**
* Fetch the users from the given api and commit each of them to the collector. * Fetch the users from the given API and commit each of them to the collector.
* <p> * <p>
* This differ from {@link #getUsersFromApi(ChannelInfoItemsCollector, String)} in the sense that they will always * This differ from {@link #getUsersFromApi(ChannelInfoItemsCollector, String)} in the sense
* get MIN_ITEMS or more. * that they will always get MIN_ITEMS or more.
* *
* @param minItems the method will return only when it have extracted that many items (equal or more) * @param minItems the method will return only when it have extracted that many items
* (equal or more)
*/ */
public static String getUsersFromApiMinItems(int minItems, ChannelInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException { public static String getUsersFromApiMinItems(final int minItems,
final ChannelInfoItemsCollector collector,
final String apiUrl) throws IOException,
ReCaptchaException, ParsingException {
String nextPageUrl = SoundcloudParsingHelper.getUsersFromApi(collector, apiUrl); String nextPageUrl = SoundcloudParsingHelper.getUsersFromApi(collector, apiUrl);
while (!nextPageUrl.isEmpty() && collector.getItems().size() < minItems) { while (!nextPageUrl.isEmpty() && collector.getItems().size() < minItems) {
@ -193,23 +208,27 @@ public class SoundcloudParsingHelper {
} }
/** /**
* Fetch the user items from the given api and commit each of them to the collector. * Fetch the user items from the given API and commit each of them to the collector.
* *
* @return the next streams url, empty if don't have * @return the next streams url, empty if don't have
*/ */
public static String getUsersFromApi(ChannelInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException { public static String getUsersFromApi(final ChannelInfoItemsCollector collector,
String response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization()).responseBody(); final String apiUrl) throws IOException,
JsonObject responseObject; ReCaptchaException, ParsingException {
final String response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization())
.responseBody();
final JsonObject responseObject;
try { try {
responseObject = JsonParser.object().from(response); responseObject = JsonParser.object().from(response);
} catch (JsonParserException e) { } catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e); throw new ParsingException("Could not parse json response", e);
} }
JsonArray responseCollection = responseObject.getArray("collection"); final JsonArray responseCollection = responseObject.getArray("collection");
for (Object o : responseCollection) { for (final Object o : responseCollection) {
if (o instanceof JsonObject) { if (o instanceof JsonObject) {
JsonObject object = (JsonObject) o; final JsonObject object = (JsonObject) o;
collector.commit(new SoundcloudChannelInfoItemExtractor(object)); collector.commit(new SoundcloudChannelInfoItemExtractor(object));
} }
} }
@ -217,8 +236,9 @@ public class SoundcloudParsingHelper {
String nextPageUrl; String nextPageUrl;
try { try {
nextPageUrl = responseObject.getString("next_href"); nextPageUrl = responseObject.getString("next_href");
if (!nextPageUrl.contains("client_id=")) nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId(); if (!nextPageUrl.contains("client_id=")) nextPageUrl += "&client_id="
} catch (Exception ignored) { + SoundcloudParsingHelper.clientId();
} catch (final Exception ignored) {
nextPageUrl = ""; nextPageUrl = "";
} }
@ -226,14 +246,18 @@ public class SoundcloudParsingHelper {
} }
/** /**
* Fetch the streams from the given api and commit each of them to the collector. * Fetch the streams from the given API and commit each of them to the collector.
* <p> * <p>
* This differ from {@link #getStreamsFromApi(StreamInfoItemsCollector, String)} in the sense that they will always * This differ from {@link #getStreamsFromApi(StreamInfoItemsCollector, String)} in the sense
* get MIN_ITEMS or more items. * that they will always get MIN_ITEMS or more items.
* *
* @param minItems the method will return only when it have extracted that many items (equal or more) * @param minItems the method will return only when it have extracted that many items
* (equal or more)
*/ */
public static String getStreamsFromApiMinItems(int minItems, StreamInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException { public static String getStreamsFromApiMinItems(final int minItems,
final StreamInfoItemsCollector collector,
final String apiUrl) throws IOException,
ReCaptchaException, ParsingException {
String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl); String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
while (!nextPageUrl.isEmpty() && collector.getItems().size() < minItems) { while (!nextPageUrl.isEmpty() && collector.getItems().size() < minItems) {
@ -244,59 +268,68 @@ public class SoundcloudParsingHelper {
} }
/** /**
* Fetch the streams from the given api and commit each of them to the collector. * Fetch the streams from the given API and commit each of them to the collector.
* *
* @return the next streams url, empty if don't have * @return the next streams url, empty if don't have
*/ */
public static String getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException { public static String getStreamsFromApi(final StreamInfoItemsCollector collector,
final Response response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization()); final String apiUrl,
final boolean charts) throws IOException,
ReCaptchaException, ParsingException {
final Response response = NewPipe.getDownloader().get(apiUrl, SoundCloud
.getLocalization());
if (response.responseCode() >= 400) { if (response.responseCode() >= 400) {
throw new IOException("Could not get streams from API, HTTP " + response.responseCode()); throw new IOException("Could not get streams from API, HTTP " + response
.responseCode());
} }
JsonObject responseObject; final JsonObject responseObject;
try { try {
responseObject = JsonParser.object().from(response.responseBody()); responseObject = JsonParser.object().from(response.responseBody());
} catch (JsonParserException e) { } catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e); throw new ParsingException("Could not parse json response", e);
} }
JsonArray responseCollection = responseObject.getArray("collection"); final JsonArray responseCollection = responseObject.getArray("collection");
for (Object o : responseCollection) { for (final Object o : responseCollection) {
if (o instanceof JsonObject) { if (o instanceof JsonObject) {
JsonObject object = (JsonObject) o; final JsonObject object = (JsonObject) o;
collector.commit(new SoundcloudStreamInfoItemExtractor(charts ? object.getObject("track") : object)); collector.commit(new SoundcloudStreamInfoItemExtractor(charts
? object.getObject("track") : object));
} }
} }
String nextPageUrl; String nextPageUrl;
try { try {
nextPageUrl = responseObject.getString("next_href"); nextPageUrl = responseObject.getString("next_href");
if (!nextPageUrl.contains("client_id=")) nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId(); if (!nextPageUrl.contains("client_id=")) nextPageUrl += "&client_id="
} catch (Exception ignored) { + SoundcloudParsingHelper.clientId();
} catch (final Exception ignored) {
nextPageUrl = ""; nextPageUrl = "";
} }
return nextPageUrl; return nextPageUrl;
} }
public static String getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, ParsingException, IOException { public static String getStreamsFromApi(final StreamInfoItemsCollector collector,
final String apiUrl) throws ReCaptchaException,
ParsingException, IOException {
return getStreamsFromApi(collector, apiUrl, false); return getStreamsFromApi(collector, apiUrl, false);
} }
@Nonnull @Nonnull
public static String getUploaderUrl(JsonObject object) { public static String getUploaderUrl(final JsonObject object) {
String url = object.getObject("user").getString("permalink_url", EMPTY_STRING); final String url = object.getObject("user").getString("permalink_url", EMPTY_STRING);
return replaceHttpWithHttps(url); return replaceHttpWithHttps(url);
} }
@Nonnull @Nonnull
public static String getAvatarUrl(JsonObject object) { public static String getAvatarUrl(final JsonObject object) {
String url = object.getObject("user").getString("avatar_url", EMPTY_STRING); final String url = object.getObject("user").getString("avatar_url", EMPTY_STRING);
return replaceHttpWithHttps(url); return replaceHttpWithHttps(url);
} }
public static String getUploaderName(JsonObject object) { public static String getUploaderName(final JsonObject object) {
return object.getObject("user").getString("username", EMPTY_STRING); return object.getObject("user").getString("username", EMPTY_STRING);
} }
} }

33
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java
View File

@ -4,7 +4,6 @@ import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.linkhandler.*;
import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.ContentCountry;
@ -23,7 +22,7 @@ import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCap
public class SoundcloudService extends StreamingService { public class SoundcloudService extends StreamingService {
public SoundcloudService(int id) { public SoundcloudService(final int id) {
super(id, "SoundCloud", asList(AUDIO, COMMENTS)); super(id, "SoundCloud", asList(AUDIO, COMMENTS));
} }
@ -54,29 +53,29 @@ public class SoundcloudService extends StreamingService {
@Override @Override
public List<ContentCountry> getSupportedCountries() { public List<ContentCountry> getSupportedCountries() {
//Country selector here https://soundcloud.com/charts/top?genre=all-music // Country selector here: https://soundcloud.com/charts/top?genre=all-music
return ContentCountry.listFrom( return ContentCountry.listFrom(
"AU", "CA", "DE", "FR", "GB", "IE", "NL", "NZ", "US" "AU", "CA", "DE", "FR", "GB", "IE", "NL", "NZ", "US"
); );
} }
@Override @Override
public StreamExtractor getStreamExtractor(LinkHandler LinkHandler) { public StreamExtractor getStreamExtractor(final LinkHandler linkHandler) {
return new SoundcloudStreamExtractor(this, LinkHandler); return new SoundcloudStreamExtractor(this, linkHandler);
} }
@Override @Override
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) { public ChannelExtractor getChannelExtractor(final ListLinkHandler linkHandler) {
return new SoundcloudChannelExtractor(this, linkHandler); return new SoundcloudChannelExtractor(this, linkHandler);
} }
@Override @Override
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) { public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
return new SoundcloudPlaylistExtractor(this, linkHandler); return new SoundcloudPlaylistExtractor(this, linkHandler);
} }
@Override @Override
public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) { public SearchExtractor getSearchExtractor(final SearchQueryHandler queryHandler) {
return new SoundcloudSearchExtractor(this, queryHandler); return new SoundcloudSearchExtractor(this, queryHandler);
} }
@ -87,18 +86,11 @@ public class SoundcloudService extends StreamingService {
@Override @Override
public KioskList getKioskList() throws ExtractionException { public KioskList getKioskList() throws ExtractionException {
KioskList.KioskExtractorFactory chartsFactory = new KioskList.KioskExtractorFactory() { final KioskList.KioskExtractorFactory chartsFactory = (streamingService, url, id) ->
@Override new SoundcloudChartsExtractor(SoundcloudService.this,
public KioskExtractor createNewKiosk(StreamingService streamingService,
String url,
String id)
throws ExtractionException {
return new SoundcloudChartsExtractor(SoundcloudService.this,
new SoundcloudChartsLinkHandlerFactory().fromUrl(url), id); new SoundcloudChartsLinkHandlerFactory().fromUrl(url), id);
}
};
KioskList list = new KioskList(this); final KioskList list = new KioskList(this);
// add kiosks here e.g.: // add kiosks here e.g.:
final SoundcloudChartsLinkHandlerFactory h = new SoundcloudChartsLinkHandlerFactory(); final SoundcloudChartsLinkHandlerFactory h = new SoundcloudChartsLinkHandlerFactory();
@ -106,7 +98,7 @@ public class SoundcloudService extends StreamingService {
list.addKioskEntry(chartsFactory, h, "Top 50"); list.addKioskEntry(chartsFactory, h, "Top 50");
list.addKioskEntry(chartsFactory, h, "New & hot"); list.addKioskEntry(chartsFactory, h, "New & hot");
list.setDefaultKiosk("New & hot"); list.setDefaultKiosk("New & hot");
} catch (Exception e) { } catch (final Exception e) {
throw new ExtractionException(e); throw new ExtractionException(e);
} }
@ -124,9 +116,8 @@ public class SoundcloudService extends StreamingService {
} }
@Override @Override
public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler) public CommentsExtractor getCommentsExtractor(final ListLinkHandler linkHandler)
throws ExtractionException { throws ExtractionException {
return new SoundcloudCommentsExtractor(this, linkHandler); return new SoundcloudCommentsExtractor(this, linkHandler);
} }
} }

37
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java
View File

@ -17,6 +17,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -24,22 +25,25 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudChannelExtractor extends ChannelExtractor { public class SoundcloudChannelExtractor extends ChannelExtractor {
private String userId; private String userId;
private JsonObject user; private JsonObject user;
private static final String USERS_ENDPOINT = SOUNDCLOUD_API_V2_URL + "users/";
public SoundcloudChannelExtractor(final StreamingService service, final ListLinkHandler linkHandler) { public SoundcloudChannelExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
super(service, linkHandler); super(service, linkHandler);
} }
@Override @Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
userId = getLinkHandler().getId(); userId = getLinkHandler().getId();
final String apiUrl = "https://api-v2.soundcloud.com/users/" + userId + final String apiUrl = USERS_ENDPOINT + userId + "?client_id="
"?client_id=" + SoundcloudParsingHelper.clientId(); + SoundcloudParsingHelper.clientId();
final String response = downloader.get(apiUrl, getExtractorLocalization()).responseBody(); final String response = downloader.get(apiUrl, getExtractorLocalization()).responseBody();
try { try {
user = JsonParser.object().from(response); user = JsonParser.object().from(response);
} catch (JsonParserException e) { } catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e); throw new ParsingException("Could not parse json response", e);
} }
} }
@ -63,7 +67,8 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Override @Override
public String getBannerUrl() { public String getBannerUrl() {
return user.getObject("visuals").getArray("visuals").getObject(0).getString("visual_url"); return user.getObject("visuals").getArray("visuals").getObject(0)
.getString("visual_url");
} }
@Override @Override
@ -105,29 +110,31 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Override @Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException { public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
try { try {
final StreamInfoItemsCollector streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId()); final StreamInfoItemsCollector streamInfoItemsCollector =
new StreamInfoItemsCollector(getServiceId());
final String apiUrl = "https://api-v2.soundcloud.com/users/" + getId() + "/tracks" final String apiUrl = USERS_ENDPOINT + getId() + "/tracks" + "?client_id="
+ "?client_id=" + SoundcloudParsingHelper.clientId() + SoundcloudParsingHelper.clientId() + "&limit=20" + "&linked_partitioning=1";
+ "&limit=20"
+ "&linked_partitioning=1";
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl); final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15,
streamInfoItemsCollector, apiUrl);
return new InfoItemsPage<>(streamInfoItemsCollector, new Page(nextPageUrl)); return new InfoItemsPage<>(streamInfoItemsCollector, new Page(nextPageUrl));
} catch (Exception e) { } catch (final Exception e) {
throw new ExtractionException("Could not get next page", e); throw new ExtractionException("Could not get next page", e);
} }
} }
@Override @Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException { public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) { if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL"); throw new IllegalArgumentException("Page doesn't contain an URL");
} }
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, page.getUrl()); final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector,
page.getUrl());
return new InfoItemsPage<>(collector, new Page(nextPageUrl)); return new InfoItemsPage<>(collector, new Page(nextPageUrl));
} }

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java
View File

@ -9,7 +9,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor { public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {
private final JsonObject itemObject; private final JsonObject itemObject;
public SoundcloudChannelInfoItemExtractor(JsonObject itemObject) { public SoundcloudChannelInfoItemExtractor(final JsonObject itemObject) {
this.itemObject = itemObject; this.itemObject = itemObject;
} }
@ -26,8 +26,8 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
@Override @Override
public String getThumbnailUrl() { public String getThumbnailUrl() {
String avatarUrl = itemObject.getString("avatar_url", EMPTY_STRING); String avatarUrl = itemObject.getString("avatar_url", EMPTY_STRING);
String avatarUrlBetterResolution = avatarUrl.replace("large.jpg", "crop.jpg"); // An avatar URL with a better resolution
return avatarUrlBetterResolution; return avatarUrl.replace("large.jpg", "crop.jpg");
} }
@Override @Override

33
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java
View File

@ -15,17 +15,18 @@ import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> { public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
public SoundcloudChartsExtractor(StreamingService service, public SoundcloudChartsExtractor(final StreamingService service,
ListLinkHandler linkHandler, final ListLinkHandler linkHandler,
String kioskId) { final String kioskId) {
super(service, linkHandler, kioskId); super(service, linkHandler, kioskId);
} }
@Override @Override
public void onFetchPage(@Nonnull Downloader downloader) { public void onFetchPage(@Nonnull final Downloader downloader) {
} }
@Nonnull @Nonnull
@ -35,13 +36,15 @@ public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
} }
@Override @Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException { public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) { if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL"); throw new IllegalArgumentException("Page doesn't contain an URL");
} }
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, page.getUrl(), true); final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector,
page.getUrl(), true);
return new InfoItemsPage<>(collector, new Page(nextPageUrl)); return new InfoItemsPage<>(collector, new Page(nextPageUrl));
} }
@ -51,9 +54,8 @@ public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException { public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
String apiUrl = "https://api-v2.soundcloud.com/charts" + String apiUrl = SOUNDCLOUD_API_V2_URL + "charts" + "?genre=soundcloud:genres:all-music"
"?genre=soundcloud:genres:all-music" + + "&client_id=" + SoundcloudParsingHelper.clientId();
"&client_id=" + SoundcloudParsingHelper.clientId();
if (getId().equals("Top 50")) { if (getId().equals("Top 50")) {
apiUrl += "&kind=top"; apiUrl += "&kind=top";
@ -64,15 +66,18 @@ public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
final ContentCountry contentCountry = SoundCloud.getContentCountry(); final ContentCountry contentCountry = SoundCloud.getContentCountry();
String apiUrlWithRegion = null; String apiUrlWithRegion = null;
if (getService().getSupportedCountries().contains(contentCountry)) { if (getService().getSupportedCountries().contains(contentCountry)) {
apiUrlWithRegion = apiUrl + "&region=soundcloud:regions:" + contentCountry.getCountryCode(); apiUrlWithRegion = apiUrl + "&region=soundcloud:regions:"
+ contentCountry.getCountryCode();
} }
String nextPageUrl; String nextPageUrl;
try { try {
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true); nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector,
} catch (IOException e) { apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true);
// Request to other region may be geo-restricted. See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537 } catch (final IOException e) {
// we retry without the specified region. // Request to other region may be geo-restricted.
// See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537.
// We retry without the specified region.
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true); nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
} }

24
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java
View File

@ -24,24 +24,27 @@ import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudCommentsExtractor extends CommentsExtractor { public class SoundcloudCommentsExtractor extends CommentsExtractor {
public SoundcloudCommentsExtractor(final StreamingService service, final ListLinkHandler uiHandler) { public SoundcloudCommentsExtractor(final StreamingService service,
final ListLinkHandler uiHandler) {
super(service, uiHandler); super(service, uiHandler);
} }
@Nonnull @Nonnull
@Override @Override
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws ExtractionException, IOException { public InfoItemsPage<CommentsInfoItem> getInitialPage() throws ExtractionException,
IOException {
final Downloader downloader = NewPipe.getDownloader(); final Downloader downloader = NewPipe.getDownloader();
final Response response = downloader.get(getUrl()); final Response response = downloader.get(getUrl());
final JsonObject json; final JsonObject json;
try { try {
json = JsonParser.object().from(response.responseBody()); json = JsonParser.object().from(response.responseBody());
} catch (JsonParserException e) { } catch (final JsonParserException e) {
throw new ParsingException("Could not parse json", e); throw new ParsingException("Could not parse json", e);
} }
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(
getServiceId());
collectStreamsFrom(collector, json.getArray("collection")); collectStreamsFrom(collector, json.getArray("collection"));
@ -49,7 +52,8 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
} }
@Override @Override
public InfoItemsPage<CommentsInfoItem> getPage(final Page page) throws ExtractionException, IOException { public InfoItemsPage<CommentsInfoItem> getPage(final Page page) throws ExtractionException,
IOException {
if (page == null || isNullOrEmpty(page.getUrl())) { if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL"); throw new IllegalArgumentException("Page doesn't contain an URL");
} }
@ -60,11 +64,12 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
final JsonObject json; final JsonObject json;
try { try {
json = JsonParser.object().from(response.responseBody()); json = JsonParser.object().from(response.responseBody());
} catch (JsonParserException e) { } catch (final JsonParserException e) {
throw new ParsingException("Could not parse json", e); throw new ParsingException("Could not parse json", e);
} }
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(
getServiceId());
collectStreamsFrom(collector, json.getArray("collection")); collectStreamsFrom(collector, json.getArray("collection"));
@ -74,9 +79,10 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor {
@Override @Override
public void onFetchPage(@Nonnull final Downloader downloader) { } public void onFetchPage(@Nonnull final Downloader downloader) { }
private void collectStreamsFrom(final CommentsInfoItemsCollector collector, final JsonArray entries) throws ParsingException { private void collectStreamsFrom(final CommentsInfoItemsCollector collector,
final JsonArray entries) throws ParsingException {
final String url = getUrl(); final String url = getUrl();
for (Object comment : entries) { for (final Object comment : entries) {
collector.commit(new SoundcloudCommentsInfoItemExtractor((JsonObject) comment, url)); collector.commit(new SoundcloudCommentsInfoItemExtractor((JsonObject) comment, url));
} }
} }

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java
View File

@ -10,10 +10,10 @@ import javax.annotation.Nullable;
import java.util.Objects; import java.util.Objects;
public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtractor { public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
private JsonObject json; private final JsonObject json;
private String url; private final String url;
public SoundcloudCommentsInfoItemExtractor(JsonObject json, String url) { public SoundcloudCommentsInfoItemExtractor(final JsonObject json, final String url) {
this.json = json; this.json = json;
this.url = url; this.url = url;
} }

45
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java
View File

@ -25,6 +25,7 @@ import java.util.List;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudPlaylistExtractor extends PlaylistExtractor { public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
@ -33,22 +34,23 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
private String playlistId; private String playlistId;
private JsonObject playlist; private JsonObject playlist;
public SoundcloudPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { public SoundcloudPlaylistExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
super(service, linkHandler); super(service, linkHandler);
} }
@Override @Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
playlistId = getLinkHandler().getId(); playlistId = getLinkHandler().getId();
String apiUrl = "https://api-v2.soundcloud.com/playlists/" + playlistId + final String apiUrl = SOUNDCLOUD_API_V2_URL + "playlists/" + playlistId + "?client_id="
"?client_id=" + SoundcloudParsingHelper.clientId() + + SoundcloudParsingHelper.clientId() + "&representation=compact";
"&representation=compact";
String response = downloader.get(apiUrl, getExtractorLocalization()).responseBody(); final String response = downloader.get(apiUrl, getExtractorLocalization()).responseBody();
try { try {
playlist = JsonParser.object().from(response); playlist = JsonParser.object().from(response);
} catch (JsonParserException e) { } catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e); throw new ParsingException("Could not parse json response", e);
} }
} }
@ -76,11 +78,11 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
try { try {
final InfoItemsPage<StreamInfoItem> infoItems = getInitialPage(); final InfoItemsPage<StreamInfoItem> infoItems = getInitialPage();
for (StreamInfoItem item : infoItems.getItems()) { for (final StreamInfoItem item : infoItems.getItems()) {
artworkUrl = item.getThumbnailUrl(); artworkUrl = item.getThumbnailUrl();
if (!isNullOrEmpty(artworkUrl)) break; if (!isNullOrEmpty(artworkUrl)) break;
} }
} catch (Exception ignored) { } catch (final Exception ignored) {
} }
if (artworkUrl == null) { if (artworkUrl == null) {
@ -139,18 +141,22 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
return ""; return "";
} }
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() { public InfoItemsPage<StreamInfoItem> getInitialPage() {
final StreamInfoItemsCollector streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId()); final StreamInfoItemsCollector streamInfoItemsCollector =
new StreamInfoItemsCollector(getServiceId());
final List<String> ids = new ArrayList<>(); final List<String> ids = new ArrayList<>();
final JsonArray tracks = playlist.getArray("tracks"); final JsonArray tracks = playlist.getArray("tracks");
for (Object o : tracks) { for (final Object o : tracks) {
if (o instanceof JsonObject) { if (o instanceof JsonObject) {
final JsonObject track = (JsonObject) o; final JsonObject track = (JsonObject) o;
if (track.has("title")) { // i.e. if full info is available if (track.has("title")) { // i.e. if full info is available
streamInfoItemsCollector.commit(new SoundcloudStreamInfoItemExtractor(track)); streamInfoItemsCollector.commit(new SoundcloudStreamInfoItemExtractor(track));
} else { } else {
// %09d would be enough, but a 0 before the number does not create problems, so let's be sure // %09d would be enough, but a 0 before the number does not create problems, so
// let's be sure
ids.add(String.format("%010d", track.getInt("id"))); ids.add(String.format("%010d", track.getInt("id")));
} }
} }
@ -160,7 +166,8 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
} }
@Override @Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException { public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getIds())) { if (page == null || isNullOrEmpty(page.getIds())) {
throw new IllegalArgumentException("Page doesn't contain IDs"); throw new IllegalArgumentException("Page doesn't contain IDs");
} }
@ -176,21 +183,21 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
nextIds = page.getIds().subList(STREAMS_PER_REQUESTED_PAGE, page.getIds().size()); nextIds = page.getIds().subList(STREAMS_PER_REQUESTED_PAGE, page.getIds().size());
} }
final String currentPageUrl = "https://api-v2.soundcloud.com/tracks?client_id=" final String currentPageUrl = SOUNDCLOUD_API_V2_URL + "tracks?client_id="
+ SoundcloudParsingHelper.clientId() + SoundcloudParsingHelper.clientId() + "&ids=" + Utils.join(",", currentIds);
+ "&ids=" + Utils.join(",", currentIds);
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final String response = NewPipe.getDownloader().get(currentPageUrl, getExtractorLocalization()).responseBody(); final String response = NewPipe.getDownloader().get(currentPageUrl,
getExtractorLocalization()).responseBody();
try { try {
final JsonArray tracks = JsonParser.array().from(response); final JsonArray tracks = JsonParser.array().from(response);
for (Object track : tracks) { for (final Object track : tracks) {
if (track instanceof JsonObject) { if (track instanceof JsonObject) {
collector.commit(new SoundcloudStreamInfoItemExtractor((JsonObject) track)); collector.commit(new SoundcloudStreamInfoItemExtractor((JsonObject) track));
} }
} }
} catch (JsonParserException e) { } catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e); throw new ParsingException("Could not parse json response", e);
} }

18
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java
View File

@ -14,7 +14,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
private final JsonObject itemObject; private final JsonObject itemObject;
public SoundcloudPlaylistInfoItemExtractor(JsonObject itemObject) { public SoundcloudPlaylistInfoItemExtractor(final JsonObject itemObject) {
this.itemObject = itemObject; this.itemObject = itemObject;
} }
@ -34,22 +34,22 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
if (itemObject.isString(ARTWORK_URL_KEY)) { if (itemObject.isString(ARTWORK_URL_KEY)) {
final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, EMPTY_STRING); final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, EMPTY_STRING);
if (!artworkUrl.isEmpty()) { if (!artworkUrl.isEmpty()) {
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg"); // An artwork URL with a better resolution
return artworkUrlBetterResolution; return artworkUrl.replace("large.jpg", "crop.jpg");
} }
} }
try { try {
// Look for artwork url inside the track list // Look for artwork url inside the track list
for (Object track : itemObject.getArray("tracks")) { for (final Object track : itemObject.getArray("tracks")) {
final JsonObject trackObject = (JsonObject) track; final JsonObject trackObject = (JsonObject) track;
// First look for track artwork url // First look for track artwork url
if (trackObject.isString(ARTWORK_URL_KEY)) { if (trackObject.isString(ARTWORK_URL_KEY)) {
String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, EMPTY_STRING); String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, EMPTY_STRING);
if (!artworkUrl.isEmpty()) { if (!artworkUrl.isEmpty()) {
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg"); // An artwork URL with a better resolution
return artworkUrlBetterResolution; return artworkUrl.replace("large.jpg", "crop.jpg");
} }
} }
@ -58,14 +58,14 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
final String creatorAvatar = creator.getString(AVATAR_URL_KEY, EMPTY_STRING); final String creatorAvatar = creator.getString(AVATAR_URL_KEY, EMPTY_STRING);
if (!creatorAvatar.isEmpty()) return creatorAvatar; if (!creatorAvatar.isEmpty()) return creatorAvatar;
} }
} catch (Exception ignored) { } catch (final Exception ignored) {
// Try other method // Try other method
} }
try { try {
// Last resort, use user avatar url. If still not found, then throw exception. // Last resort, use user avatar url. If still not found, then throw exception.
return itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY, EMPTY_STRING); return itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY, EMPTY_STRING);
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Failed to extract playlist thumbnail url", e); throw new ParsingException("Failed to extract playlist thumbnail url", e);
} }
} }
@ -74,7 +74,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
public String getUploaderName() throws ParsingException { public String getUploaderName() throws ParsingException {
try { try {
return itemObject.getObject(USER_KEY).getString("username"); return itemObject.getObject(USER_KEY).getString("username");
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Failed to extract playlist uploader", e); throw new ParsingException("Failed to extract playlist uploader", e);
} }
} }

43
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java
View File

@ -28,7 +28,8 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudSearchExtractor extends SearchExtractor { public class SoundcloudSearchExtractor extends SearchExtractor {
private JsonArray searchCollection; private JsonArray searchCollection;
public SoundcloudSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { public SoundcloudSearchExtractor(final StreamingService service,
final SearchQueryHandler linkHandler) {
super(service, linkHandler); super(service, linkHandler);
} }
@ -52,34 +53,39 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
@Nonnull @Nonnull
@Override @Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException { public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
return new InfoItemsPage<>(collectItems(searchCollection), getNextPageFromCurrentUrl(getUrl())); return new InfoItemsPage<>(collectItems(searchCollection), getNextPageFromCurrentUrl(
getUrl()));
} }
@Override @Override
public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException, ExtractionException { public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) { if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL"); throw new IllegalArgumentException("Page doesn't contain an URL");
} }
final Downloader dl = getDownloader(); final Downloader dl = getDownloader();
try { try {
final String response = dl.get(page.getUrl(), getExtractorLocalization()).responseBody(); final String response = dl.get(page.getUrl(), getExtractorLocalization())
.responseBody();
searchCollection = JsonParser.object().from(response).getArray("collection"); searchCollection = JsonParser.object().from(response).getArray("collection");
} catch (JsonParserException e) { } catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e); throw new ParsingException("Could not parse json response", e);
} }
return new InfoItemsPage<>(collectItems(searchCollection), getNextPageFromCurrentUrl(page.getUrl())); return new InfoItemsPage<>(collectItems(searchCollection), getNextPageFromCurrentUrl(page
.getUrl()));
} }
@Override @Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
final Downloader dl = getDownloader(); final Downloader dl = getDownloader();
final String url = getUrl(); final String url = getUrl();
try { try {
final String response = dl.get(url, getExtractorLocalization()).responseBody(); final String response = dl.get(url, getExtractorLocalization()).responseBody();
searchCollection = JsonParser.object().from(response).getArray("collection"); searchCollection = JsonParser.object().from(response).getArray("collection");
} catch (JsonParserException e) { } catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e); throw new ParsingException("Could not parse json response", e);
} }
@ -88,14 +94,14 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
} }
} }
private InfoItemsCollector<InfoItem, InfoItemExtractor> collectItems(JsonArray searchCollection) { private InfoItemsCollector<InfoItem, InfoItemExtractor> collectItems(
final JsonArray searchCollection) {
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
for (Object result : searchCollection) { for (final Object result : searchCollection) {
if (!(result instanceof JsonObject)) continue; if (!(result instanceof JsonObject)) continue;
//noinspection ConstantConditions final JsonObject searchResult = (JsonObject) result;
JsonObject searchResult = (JsonObject) result; final String kind = searchResult.getString("kind", EMPTY_STRING);
String kind = searchResult.getString("kind", EMPTY_STRING);
switch (kind) { switch (kind) {
case "user": case "user":
collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult)); collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult));
@ -112,15 +118,12 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
return collector; return collector;
} }
private Page getNextPageFromCurrentUrl(String currentUrl) private Page getNextPageFromCurrentUrl(final String currentUrl)
throws MalformedURLException, UnsupportedEncodingException { throws MalformedURLException, UnsupportedEncodingException {
final int pageOffset = Integer.parseInt( final int pageOffset = Integer.parseInt(
Parser.compatParseMap( Parser.compatParseMap(new URL(currentUrl).getQuery()).get("offset"));
new URL(currentUrl)
.getQuery())
.get("offset"));
return new Page(currentUrl.replace("&offset=" + pageOffset, return new Page(currentUrl.replace("&offset=" + pageOffset, "&offset="
"&offset=" + (pageOffset + ITEMS_PER_PAGE))); + (pageOffset + ITEMS_PER_PAGE)));
} }
} }

49
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java
View File

@ -30,18 +30,21 @@ import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.*; import static org.schabi.newpipe.extractor.utils.Utils.*;
public class SoundcloudStreamExtractor extends StreamExtractor { public class SoundcloudStreamExtractor extends StreamExtractor {
private JsonObject track; private JsonObject track;
private boolean isAvailable = true; private boolean isAvailable = true;
public SoundcloudStreamExtractor(StreamingService service, LinkHandler linkHandler) { public SoundcloudStreamExtractor(final StreamingService service,
final LinkHandler linkHandler) {
super(service, linkHandler); super(service, linkHandler);
} }
@Override @Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
track = SoundcloudParsingHelper.resolveFor(downloader, getUrl()); track = SoundcloudParsingHelper.resolveFor(downloader, getUrl());
final String policy = track.getString("policy", EMPTY_STRING); final String policy = track.getString("policy", EMPTY_STRING);
@ -50,9 +53,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
if (policy.equals("SNIP")) { if (policy.equals("SNIP")) {
throw new SoundCloudGoPlusContentException(); throw new SoundCloudGoPlusContentException();
} }
if (policy.equals("BLOCK")) { if (policy.equals("BLOCK")) throw new GeographicRestrictionException(
throw new GeographicRestrictionException("This track is not available in user's country"); "This track is not available in user's country");
}
throw new ContentNotAvailableException("Content not available: policy " + policy); throw new ContentNotAvailableException("Content not available: policy " + policy);
} }
} }
@ -80,7 +82,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull @Nonnull
@Override @Override
public DateWrapper getUploadDate() throws ParsingException { public DateWrapper getUploadDate() throws ParsingException {
return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(track.getString("created_at"))); return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(track.getString(
"created_at")));
} }
@Nonnull @Nonnull
@ -220,9 +223,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
} }
@Nonnull @Nonnull
private static String getTranscodingUrl(final String endpointUrl, final String protocol) throws IOException, ExtractionException { private static String getTranscodingUrl(final String endpointUrl,
final String protocol)
throws IOException, ExtractionException {
final Downloader downloader = NewPipe.getDownloader(); final Downloader downloader = NewPipe.getDownloader();
final String apiStreamUrl = endpointUrl + "?client_id=" + SoundcloudParsingHelper.clientId(); final String apiStreamUrl = endpointUrl + "?client_id="
+ SoundcloudParsingHelper.clientId();
final String response = downloader.get(apiStreamUrl).responseBody(); final String response = downloader.get(apiStreamUrl).responseBody();
final JsonObject urlObject; final JsonObject urlObject;
try { try {
@ -255,7 +261,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
} }
final String mediaUrl; final String mediaUrl;
final String preset = transcodingJsonObject.getString("preset"); final String preset = transcodingJsonObject.getString("preset");
final String protocol = transcodingJsonObject.getObject("format").getString("protocol"); final String protocol = transcodingJsonObject.getObject("format")
.getString("protocol");
MediaFormat mediaFormat = null; MediaFormat mediaFormat = null;
int bitrate = 0; int bitrate = 0;
if (preset.contains("mp3")) { if (preset.contains("mp3")) {
@ -285,7 +292,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
} }
} }
/** Parses a SoundCloud HLS manifest to get a single URL of HLS streams. /**
* Parses a SoundCloud HLS manifest to get a single URL of HLS streams.
* <p> * <p>
* This method downloads the provided manifest URL, find all web occurrences in the manifest, * This method downloads the provided manifest URL, find all web occurrences in the manifest,
* get the last segment URL, changes its segment range to {@code 0/track-length} and return * get the last segment URL, changes its segment range to {@code 0/track-length} and return
@ -293,7 +301,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
* @param hlsManifestUrl the URL of the manifest to be parsed * @param hlsManifestUrl the URL of the manifest to be parsed
* @return a single URL that contains a range equal to the length of the track * @return a single URL that contains a range equal to the length of the track
*/ */
private static String getSingleUrlFromHlsManifest(final String hlsManifestUrl) throws ParsingException { private static String getSingleUrlFromHlsManifest(final String hlsManifestUrl)
throws ParsingException {
final Downloader dl = NewPipe.getDownloader(); final Downloader dl = NewPipe.getDownloader();
final String hlsManifestResponse; final String hlsManifestResponse;
@ -306,11 +315,11 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
final String[] lines = hlsManifestResponse.split("\\r?\\n"); final String[] lines = hlsManifestResponse.split("\\r?\\n");
for (int l = lines.length - 1; l >= 0; l--) { for (int l = lines.length - 1; l >= 0; l--) {
final String line = lines[l]; final String line = lines[l];
// get the last URL from manifest, because it contains the range of the stream // Get the last URL from manifest, because it contains the range of the stream
if (line.trim().length() != 0 && !line.startsWith("#") && line.startsWith("https")) { if (line.trim().length() != 0 && !line.startsWith("#") && line.startsWith("https")) {
final String[] hlsLastRangeUrlArray = line.split("/"); final String[] hlsLastRangeUrlArray = line.split("/");
return HTTPS + hlsLastRangeUrlArray[2] + "/media/0/" + hlsLastRangeUrlArray[5] + "/" return HTTPS + hlsLastRangeUrlArray[2] + "/media/0/" + hlsLastRangeUrlArray[5]
+ hlsLastRangeUrlArray[6]; + "/" + hlsLastRangeUrlArray[6];
} }
} }
throw new ParsingException("Could not get any URL from HLS manifest"); throw new ParsingException("Could not get any URL from HLS manifest");
@ -356,7 +365,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
public StreamInfoItemsCollector getRelatedItems() throws IOException, ExtractionException { public StreamInfoItemsCollector getRelatedItems() throws IOException, ExtractionException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final String apiUrl = "https://api-v2.soundcloud.com/tracks/" + urlEncode(getId()) final String apiUrl = SOUNDCLOUD_API_V2_URL + "tracks/" + urlEncode(getId())
+ "/related?client_id=" + urlEncode(SoundcloudParsingHelper.clientId()); + "/related?client_id=" + urlEncode(SoundcloudParsingHelper.clientId());
SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl); SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
@ -399,15 +408,15 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull @Nonnull
@Override @Override
public List<String> getTags() { public List<String> getTags() {
// tags are separated by spaces, but they can be multiple words escaped by quotes " // Tags are separated by spaces, but they can be multiple words escaped by quotes "
final String[] tag_list = track.getString("tag_list").split(" "); final String[] tagList = track.getString("tag_list").split(" ");
final List<String> tags = new ArrayList<>(); final List<String> tags = new ArrayList<>();
String escapedTag = ""; String escapedTag = "";
boolean isEscaped = false; boolean isEscaped = false;
for (int i = 0; i < tag_list.length; i++) { for (int i = 0; i < tagList.length; i++) {
String tag = tag_list[i]; String tag = tagList[i];
if (tag.startsWith("\"")) { if (tag.startsWith("\"")) {
escapedTag += tag_list[i].replace("\"", ""); escapedTag += tagList[i].replace("\"", "");
isEscaped = true; isEscaped = true;
} else if (isEscaped) { } else if (isEscaped) {
if (tag.endsWith("\"")) { if (tag.endsWith("\"")) {

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java
View File

@ -14,7 +14,7 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
protected final JsonObject itemObject; protected final JsonObject itemObject;
public SoundcloudStreamInfoItemExtractor(JsonObject itemObject) { public SoundcloudStreamInfoItemExtractor(final JsonObject itemObject) {
this.itemObject = itemObject; this.itemObject = itemObject;
} }

35
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSubscriptionExtractor.java
View File

@ -13,12 +13,16 @@ import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
/** /**
* Extract the "followings" from a user in SoundCloud. * Extract the "followings" from a user in SoundCloud.
*/ */
public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor { public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor {
public SoundcloudSubscriptionExtractor(SoundcloudService service) { public SoundcloudSubscriptionExtractor(final SoundcloudService service) {
super(service, Collections.singletonList(ContentSource.CHANNEL_URL)); super(service, Collections.singletonList(ContentSource.CHANNEL_URL));
} }
@ -28,20 +32,21 @@ public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor {
} }
@Override @Override
public List<SubscriptionItem> fromChannelUrl(String channelUrl) throws IOException, ExtractionException { public List<SubscriptionItem> fromChannelUrl(final String channelUrl) throws IOException,
if (channelUrl == null) throw new InvalidSourceException("channel url is null"); ExtractionException {
if (channelUrl == null) throw new InvalidSourceException("Channel url is null");
String id; final String id;
try { try {
id = service.getChannelLHFactory().fromUrl(getUrlFrom(channelUrl)).getId(); id = service.getChannelLHFactory().fromUrl(getUrlFrom(channelUrl)).getId();
} catch (ExtractionException e) { } catch (final ExtractionException e) {
throw new InvalidSourceException(e); throw new InvalidSourceException(e);
} }
String apiUrl = "https://api-v2.soundcloud.com/users/" + id + "/followings" final String apiUrl = SOUNDCLOUD_API_V2_URL + "users/" + id + "/followings" + "?client_id="
+ "?client_id=" + SoundcloudParsingHelper.clientId() + SoundcloudParsingHelper.clientId() + "&limit=200";
+ "&limit=200"; final ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(service
ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(service.getServiceId()); .getServiceId());
// ± 2000 is the limit of followings on SoundCloud, so this minimum should be enough // ± 2000 is the limit of followings on SoundCloud, so this minimum should be enough
SoundcloudParsingHelper.getUsersFromApiMinItems(2500, collector, apiUrl); SoundcloudParsingHelper.getUsersFromApiMinItems(2500, collector, apiUrl);
@ -49,13 +54,13 @@ public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor {
} }
private String getUrlFrom(String channelUrl) { private String getUrlFrom(String channelUrl) {
channelUrl = channelUrl.replace("http://", "https://").trim(); channelUrl = replaceHttpWithHttps(channelUrl);
if (!channelUrl.startsWith("https://")) { if (!channelUrl.startsWith(HTTPS)) {
if (!channelUrl.contains("soundcloud.com/")) { if (!channelUrl.contains("soundcloud.com/")) {
channelUrl = "https://soundcloud.com/" + channelUrl; channelUrl = "https://soundcloud.com/" + channelUrl;
} else { } else {
channelUrl = "https://" + channelUrl; channelUrl = HTTPS + channelUrl;
} }
} }
@ -66,9 +71,9 @@ public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor {
// Utils // Utils
//////////////////////////////////////////////////////////////////////////*/ //////////////////////////////////////////////////////////////////////////*/
private List<SubscriptionItem> toSubscriptionItems(List<ChannelInfoItem> items) { private List<SubscriptionItem> toSubscriptionItems(final List<ChannelInfoItem> items) {
List<SubscriptionItem> result = new ArrayList<>(items.size()); final List<SubscriptionItem> result = new ArrayList<>(items.size());
for (ChannelInfoItem item : items) { for (final ChannelInfoItem item : items) {
result.add(new SubscriptionItem(item.getServiceId(), item.getUrl(), item.getName())); result.add(new SubscriptionItem(item.getServiceId(), item.getUrl(), item.getName()));
} }
return result; return result;

30
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java
View File

@ -17,34 +17,34 @@ import java.net.URLEncoder;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class SoundcloudSuggestionExtractor extends SuggestionExtractor { public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
public SoundcloudSuggestionExtractor(StreamingService service) { public SoundcloudSuggestionExtractor(final StreamingService service) {
super(service); super(service);
} }
@Override @Override
public List<String> suggestionList(String query) throws IOException, ExtractionException { public List<String> suggestionList(final String query) throws IOException,
List<String> suggestions = new ArrayList<>(); ExtractionException {
final List<String> suggestions = new ArrayList<>();
final Downloader dl = NewPipe.getDownloader();
final String url = SOUNDCLOUD_API_V2_URL + "search/queries" + "?q="
+ URLEncoder.encode(query, UTF_8) + "&client_id="
+ SoundcloudParsingHelper.clientId() + "&limit=10";
final String response = dl.get(url, getExtractorLocalization()).responseBody();
Downloader dl = NewPipe.getDownloader();
String url = "https://api-v2.soundcloud.com/search/queries"
+ "?q=" + URLEncoder.encode(query, UTF_8)
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=10";
String response = dl.get(url, getExtractorLocalization()).responseBody();
try { try {
JsonArray collection = JsonParser.object().from(response).getArray("collection"); final JsonArray collection = JsonParser.object().from(response).getArray("collection");
for (Object suggestion : collection) { for (final Object suggestion : collection) {
if (suggestion instanceof JsonObject) suggestions.add(((JsonObject) suggestion).getString("query")); if (suggestion instanceof JsonObject) suggestions.add(((JsonObject) suggestion)
.getString("query"));
} }
return suggestions; return suggestions;
} catch (JsonParserException e) { } catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e); throw new ParsingException("Could not parse json response", e);
} }
} }

20
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelLinkHandlerFactory.java
View File

@ -9,9 +9,10 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.util.List; import java.util.List;
public class SoundcloudChannelLinkHandlerFactory extends ListLinkHandlerFactory { public class SoundcloudChannelLinkHandlerFactory extends ListLinkHandlerFactory {
private static final SoundcloudChannelLinkHandlerFactory instance = new SoundcloudChannelLinkHandlerFactory(); private static final SoundcloudChannelLinkHandlerFactory instance =
private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" + new SoundcloudChannelLinkHandlerFactory();
"(/((tracks|albums|sets|reposts|followers|following)/?)?)?([#?].*)?$"; private static final String URL_PATTERN ="^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+"
+ "(/((tracks|albums|sets|reposts|followers|following)/?)?)?([#?].*)?$";
public static SoundcloudChannelLinkHandlerFactory getInstance() { public static SoundcloudChannelLinkHandlerFactory getInstance() {
return instance; return instance;
@ -19,21 +20,24 @@ public class SoundcloudChannelLinkHandlerFactory extends ListLinkHandlerFactory
@Override @Override
public String getId(String url) throws ParsingException { public String getId(final String url) throws ParsingException {
Utils.checkUrl(URL_PATTERN, url); Utils.checkUrl(URL_PATTERN, url);
try { try {
return SoundcloudParsingHelper.resolveIdWithWidgetApi(url); return SoundcloudParsingHelper.resolveIdWithWidgetApi(url);
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException(e.getMessage(), e); throw new ParsingException(e.getMessage(), e);
} }
} }
@Override @Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException { public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) throws ParsingException {
try { try {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/" + id); return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer(
} catch (Exception e) { "https://api.soundcloud.com/users/" + id);
} catch (final Exception e) {
throw new ParsingException(e.getMessage(), e); throw new ParsingException(e.getMessage(), e);
} }
} }

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChartsLinkHandlerFactory.java
View File

@ -6,12 +6,13 @@ import org.schabi.newpipe.extractor.utils.Parser;
import java.util.List; import java.util.List;
public class SoundcloudChartsLinkHandlerFactory extends ListLinkHandlerFactory { public class SoundcloudChartsLinkHandlerFactory extends ListLinkHandlerFactory {
private static final String TOP_URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$"; private static final String TOP_URL_PATTERN =
private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$"; "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$";
private static final String URL_PATTERN =
"^https?://(www\\.|m\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$";
@Override @Override
public String getId(String url) { public String getId(final String url) {
if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) { if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) {
return "Top 50"; return "Top 50";
} else { } else {
@ -20,7 +21,9 @@ public class SoundcloudChartsLinkHandlerFactory extends ListLinkHandlerFactory {
} }
@Override @Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) { public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) {
if (id.equals("Top 50")) { if (id.equals("Top 50")) {
return "https://soundcloud.com/charts/top"; return "https://soundcloud.com/charts/top";
} else { } else {

22
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java
View File

@ -11,36 +11,40 @@ import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsing
public class SoundcloudCommentsLinkHandlerFactory extends ListLinkHandlerFactory { public class SoundcloudCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
private static final SoundcloudCommentsLinkHandlerFactory instance = new SoundcloudCommentsLinkHandlerFactory(); private static final SoundcloudCommentsLinkHandlerFactory instance =
new SoundcloudCommentsLinkHandlerFactory();
public static SoundcloudCommentsLinkHandlerFactory getInstance() { public static SoundcloudCommentsLinkHandlerFactory getInstance() {
return instance; return instance;
} }
@Override @Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException { public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) throws ParsingException {
try { try {
return "https://api-v2.soundcloud.com/tracks/" + id + "/comments" + "?client_id=" + clientId() + return "https://api-v2.soundcloud.com/tracks/" + id + "/comments" + "?client_id="
"&threaded=0" + "&filter_replies=1"; // anything but 1 = sort by new + clientId() + "&threaded=0" + "&filter_replies=1";
// Anything but 1 = sort by new
// + "&limit=NUMBER_OF_ITEMS_PER_REQUEST". We let the API control (default = 10) // + "&limit=NUMBER_OF_ITEMS_PER_REQUEST". We let the API control (default = 10)
// + "&offset=OFFSET". We let the API control (default = 0, then we use nextPageUrl) // + "&offset=OFFSET". We let the API control (default = 0, then we use nextPageUrl)
} catch (ExtractionException | IOException e) { } catch (final ExtractionException | IOException e) {
throw new ParsingException("Could not get comments"); throw new ParsingException("Could not get comments");
} }
} }
@Override @Override
public String getId(String url) throws ParsingException { public String getId(final String url) throws ParsingException {
// delagation to avoid duplicate code, as we need the same id // Delegation to avoid duplicate code, as we need the same id
return SoundcloudStreamLinkHandlerFactory.getInstance().getId(url); return SoundcloudStreamLinkHandlerFactory.getInstance().getId(url);
} }
@Override @Override
public boolean onAcceptUrl(String url) { public boolean onAcceptUrl(final String url) {
try { try {
getId(url); getId(url);
return true; return true;
} catch (ParsingException e) { } catch (final ParsingException e) {
return false; return false;
} }
} }

24
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudPlaylistLinkHandlerFactory.java
View File

@ -9,30 +9,36 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.util.List; import java.util.List;
public class SoundcloudPlaylistLinkHandlerFactory extends ListLinkHandlerFactory { public class SoundcloudPlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
private static final SoundcloudPlaylistLinkHandlerFactory instance = new SoundcloudPlaylistLinkHandlerFactory(); private static final SoundcloudPlaylistLinkHandlerFactory instance =
private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" + new SoundcloudPlaylistLinkHandlerFactory();
"/sets/[0-9a-z_-]+/?([#?].*)?$"; private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+"
+ "/sets/[0-9a-z_-]+/?([#?].*)?$";
public static SoundcloudPlaylistLinkHandlerFactory getInstance() { public static SoundcloudPlaylistLinkHandlerFactory getInstance() {
return instance; return instance;
} }
@Override @Override
public String getId(String url) throws ParsingException { public String getId(final String url) throws ParsingException {
Utils.checkUrl(URL_PATTERN, url); Utils.checkUrl(URL_PATTERN, url);
try { try {
return SoundcloudParsingHelper.resolveIdWithWidgetApi(url); return SoundcloudParsingHelper.resolveIdWithWidgetApi(url);
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get id of url: " + url + " " + e.getMessage(), e); throw new ParsingException("Could not get id of url: " + url + " " + e.getMessage(),
e);
} }
} }
@Override @Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException { public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter)
throws ParsingException {
try { try {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/playlists/" + id); return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer(
} catch (Exception e) { "https://api.soundcloud.com/playlists/" + id);
} catch (final Exception e) {
throw new ParsingException(e.getMessage(), e); throw new ParsingException(e.getMessage(), e);
} }
} }

21
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java
View File

@ -11,6 +11,7 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.List; import java.util.List;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFactory { public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
@ -23,11 +24,14 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto
public static final int ITEMS_PER_PAGE = 10; public static final int ITEMS_PER_PAGE = 10;
@Override @Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException { public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter)
throws ParsingException {
try { try {
String url = "https://api-v2.soundcloud.com/search"; String url = SOUNDCLOUD_API_V2_URL + "search";
if (contentFilter.size() > 0) { if (!contentFilter.isEmpty()) {
switch (contentFilter.get(0)) { switch (contentFilter.get(0)) {
case TRACKS: case TRACKS:
url += "/tracks"; url += "/tracks";
@ -44,16 +48,15 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto
} }
} }
return url + "?q=" + URLEncoder.encode(id, UTF_8) return url + "?q=" + URLEncoder.encode(id, UTF_8) + "&client_id="
+ "&client_id=" + SoundcloudParsingHelper.clientId() + SoundcloudParsingHelper.clientId() + "&limit=" + ITEMS_PER_PAGE
+ "&limit=" + ITEMS_PER_PAGE
+ "&offset=0"; + "&offset=0";
} catch (UnsupportedEncodingException e) { } catch (final UnsupportedEncodingException e) {
throw new ParsingException("Could not encode query", e); throw new ParsingException("Could not encode query", e);
} catch (ReCaptchaException e) { } catch (final ReCaptchaException e) {
throw new ParsingException("ReCaptcha required", e); throw new ParsingException("ReCaptcha required", e);
} catch (IOException | ExtractionException e) { } catch (final IOException | ExtractionException e) {
throw new ParsingException("Could not get client id", e); throw new ParsingException("Could not get client id", e);
} }
} }

18
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java
View File

@ -7,9 +7,10 @@ import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
public class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory { public class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory {
private static final SoundcloudStreamLinkHandlerFactory instance = new SoundcloudStreamLinkHandlerFactory(); private static final SoundcloudStreamLinkHandlerFactory instance =
private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" + new SoundcloudStreamLinkHandlerFactory();
"/(?!(tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?([#?].*)?$"; private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+"
+ "/(?!(tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?([#?].*)?$";
private SoundcloudStreamLinkHandlerFactory() { private SoundcloudStreamLinkHandlerFactory() {
} }
@ -19,21 +20,22 @@ public class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory {
} }
@Override @Override
public String getUrl(String id) throws ParsingException { public String getUrl(final String id) throws ParsingException {
try { try {
return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/tracks/" + id); return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer(
} catch (Exception e) { "https://api.soundcloud.com/tracks/" + id);
} catch (final Exception e) {
throw new ParsingException(e.getMessage(), e); throw new ParsingException(e.getMessage(), e);
} }
} }
@Override @Override
public String getId(String url) throws ParsingException { public String getId(final String url) throws ParsingException {
Utils.checkUrl(URL_PATTERN, url); Utils.checkUrl(URL_PATTERN, url);
try { try {
return SoundcloudParsingHelper.resolveIdWithWidgetApi(url); return SoundcloudParsingHelper.resolveIdWithWidgetApi(url);
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException(e.getMessage(), e); throw new ParsingException(e.getMessage(), e);
} }
} }

44
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java
View File

@ -15,10 +15,11 @@ public class Utils {
public static final String HTTPS = "https://"; public static final String HTTPS = "https://";
public static final String UTF_8 = "UTF-8"; public static final String UTF_8 = "UTF-8";
public static final String EMPTY_STRING = ""; public static final String EMPTY_STRING = "";
private static final Pattern M_PATTERN = Pattern.compile("(https?)?:\\/\\/m\\.");
private static final Pattern WWW_PATTERN = Pattern.compile("(https?)?:\\/\\/www\\."); private static final Pattern WWW_PATTERN = Pattern.compile("(https?)?:\\/\\/www\\.");
private Utils() { private Utils() {
//no instance // no instance
} }
/** /**
@ -30,7 +31,7 @@ public class Utils {
* @param toRemove string to remove non-digit chars * @param toRemove string to remove non-digit chars
* @return a string that contains only digits * @return a string that contains only digits
*/ */
public static String removeNonDigitCharacters(String toRemove) { public static String removeNonDigitCharacters(final String toRemove) {
return toRemove.replaceAll("\\D+", ""); return toRemove.replaceAll("\\D+", "");
} }
@ -48,7 +49,8 @@ public class Utils {
* @throws NumberFormatException * @throws NumberFormatException
* @throws ParsingException * @throws ParsingException
*/ */
public static long mixedNumberWordToLong(String numberWord) throws NumberFormatException, ParsingException { public static long mixedNumberWordToLong(final String numberWord) throws NumberFormatException,
ParsingException {
String multiplier = ""; String multiplier = "";
try { try {
multiplier = Parser.matchGroup("[\\d]+([\\.,][\\d]+)?([KMBkmb])+", numberWord, 2); multiplier = Parser.matchGroup("[\\d]+([\\.,][\\d]+)?([KMBkmb])+", numberWord, 2);
@ -74,7 +76,7 @@ public class Utils {
* @param pattern the pattern that will be used to check the url * @param pattern the pattern that will be used to check the url
* @param url the url to be tested * @param url the url to be tested
*/ */
public static void checkUrl(String pattern, String url) throws ParsingException { public static void checkUrl(final String pattern, final String url) throws ParsingException {
if (isNullOrEmpty(url)) { if (isNullOrEmpty(url)) {
throw new IllegalArgumentException("Url can't be null or empty"); throw new IllegalArgumentException("Url can't be null or empty");
} }
@ -101,14 +103,14 @@ public class Utils {
} }
/** /**
* get the value of a URL-query by name. * Get the value of a URL-query by name.
* if a url-query is give multiple times, only the value of the first query is returned * If a url-query is give multiple times, only the value of the first query is returned
* *
* @param url the url to be used * @param url the url to be used
* @param parameterName the pattern that will be used to check the url * @param parameterName the pattern that will be used to check the url
* @return a string that contains the value of the query parameter or null if nothing was found * @return a string that contains the value of the query parameter or null if nothing was found
*/ */
public static String getQueryValue(URL url, String parameterName) { public static String getQueryValue(final URL url, final String parameterName) {
String urlQuery = url.getQuery(); String urlQuery = url.getQuery();
if (urlQuery != null) { if (urlQuery != null) {
@ -118,8 +120,9 @@ public class Utils {
String query; String query;
try { try {
query = URLDecoder.decode(params[0], UTF_8); query = URLDecoder.decode(params[0], UTF_8);
} catch (UnsupportedEncodingException e) { } catch (final UnsupportedEncodingException e) {
System.err.println("Cannot decode string with UTF-8. using the string without decoding"); System.err.println(
"Cannot decode string with UTF-8. using the string without decoding");
e.printStackTrace(); e.printStackTrace();
query = params[0]; query = params[0];
} }
@ -127,8 +130,9 @@ public class Utils {
if (query.equals(parameterName)) { if (query.equals(parameterName)) {
try { try {
return URLDecoder.decode(params[1], UTF_8); return URLDecoder.decode(params[1], UTF_8);
} catch (UnsupportedEncodingException e) { } catch (final UnsupportedEncodingException e) {
System.err.println("Cannot decode string with UTF-8. using the string without decoding"); System.err.println(
"Cannot decode string with UTF-8. using the string without decoding");
e.printStackTrace(); e.printStackTrace();
return params[1]; return params[1];
} }
@ -146,7 +150,7 @@ public class Utils {
* @param url the string to be converted to a URL-Object * @param url the string to be converted to a URL-Object
* @return a URL-Object containing the url * @return a URL-Object containing the url
*/ */
public static URL stringToURL(String url) throws MalformedURLException { public static URL stringToURL(final String url) throws MalformedURLException {
try { try {
return new URL(url); return new URL(url);
} catch (MalformedURLException e) { } catch (MalformedURLException e) {
@ -159,7 +163,7 @@ public class Utils {
} }
} }
public static boolean isHTTP(URL url) { public static boolean isHTTP(final URL url) {
// make sure its http or https // make sure its http or https
String protocol = url.getProtocol(); String protocol = url.getProtocol();
if (!protocol.equals("http") && !protocol.equals("https")) { if (!protocol.equals("http") && !protocol.equals("https")) {
@ -172,7 +176,10 @@ public class Utils {
return setsNoPort || usesDefaultPort; return setsNoPort || usesDefaultPort;
} }
public static String removeWWWFromUrl(String url) { public static String removeMAndWWWFromUrl(final String url) {
if (M_PATTERN.matcher(url).find()) {
return url.replace("m.", "");
}
if (WWW_PATTERN.matcher(url).find()) { if (WWW_PATTERN.matcher(url).find()) {
return url.replace("www.", ""); return url.replace("www.", "");
} }
@ -216,7 +223,8 @@ public class Utils {
try { try {
final URL decoded = Utils.stringToURL(url); final URL decoded = Utils.stringToURL(url);
if (decoded.getHost().contains("google") && decoded.getPath().equals("/url")) { if (decoded.getHost().contains("google") && decoded.getPath().equals("/url")) {
return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), UTF_8); return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url),
UTF_8);
} }
} catch (final Exception ignored) { } catch (final Exception ignored) {
} }
@ -258,7 +266,8 @@ public class Utils {
return true; return true;
} }
public static String join(final CharSequence delimiter, final Iterable<? extends CharSequence> elements) { public static String join(final CharSequence delimiter,
final Iterable<? extends CharSequence> elements) {
final StringBuilder stringBuilder = new StringBuilder(); final StringBuilder stringBuilder = new StringBuilder();
final Iterator<? extends CharSequence> iterator = elements.iterator(); final Iterator<? extends CharSequence> iterator = elements.iterator();
while (iterator.hasNext()) { while (iterator.hasNext()) {
@ -283,7 +292,8 @@ public class Utils {
/** /**
* Concatenate all non-null, non-empty and strings which are not equal to <code>"null"</code>. * Concatenate all non-null, non-empty and strings which are not equal to <code>"null"</code>.
*/ */
public static String nonEmptyAndNullJoin(final CharSequence delimiter, final String[] elements) { public static String nonEmptyAndNullJoin(final CharSequence delimiter,
final String[] elements) {
final List<String> list = new java.util.ArrayList<>(Arrays.asList(elements)); final List<String> list = new java.util.ArrayList<>(Arrays.asList(elements));
list.removeIf(s -> isNullOrEmpty(s) || s.equals("null")); list.removeIf(s -> isNullOrEmpty(s) || s.equals("null"));
return join(delimiter, list); return join(delimiter, list);