2020-04-10 10:51:05 +02:00
|
|
|
package org.schabi.newpipe.extractor.services.soundcloud.extractors;
|
2017-08-04 16:21:45 +02:00
|
|
|
|
2019-12-22 00:40:32 +01:00
|
|
|
import com.grack.nanojson.JsonArray;
|
2017-08-16 04:40:03 +02:00
|
|
|
import com.grack.nanojson.JsonObject;
|
|
|
|
import com.grack.nanojson.JsonParser;
|
|
|
|
import com.grack.nanojson.JsonParserException;
|
2020-03-17 20:31:01 +01:00
|
|
|
|
2020-02-08 23:58:46 +01:00
|
|
|
import org.schabi.newpipe.extractor.MediaFormat;
|
|
|
|
import org.schabi.newpipe.extractor.NewPipe;
|
|
|
|
import org.schabi.newpipe.extractor.StreamingService;
|
2019-04-28 22:03:16 +02:00
|
|
|
import org.schabi.newpipe.extractor.downloader.Downloader;
|
2017-08-04 16:21:45 +02:00
|
|
|
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
|
2020-03-30 11:48:00 +02:00
|
|
|
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
|
2017-08-04 16:21:45 +02:00
|
|
|
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
|
|
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
2018-07-13 18:02:40 +02:00
|
|
|
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
|
2019-11-03 19:45:25 +01:00
|
|
|
import org.schabi.newpipe.extractor.localization.DateWrapper;
|
2020-04-10 10:51:05 +02:00
|
|
|
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
|
2020-03-18 11:01:46 +01:00
|
|
|
import org.schabi.newpipe.extractor.stream.AudioStream;
|
|
|
|
import org.schabi.newpipe.extractor.stream.Description;
|
|
|
|
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
|
|
|
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
|
|
|
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
|
|
|
import org.schabi.newpipe.extractor.stream.StreamType;
|
|
|
|
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
|
|
|
|
import org.schabi.newpipe.extractor.stream.VideoStream;
|
2017-08-06 22:20:15 +02:00
|
|
|
|
|
|
|
import java.io.IOException;
|
2017-11-25 02:20:16 +01:00
|
|
|
import java.io.UnsupportedEncodingException;
|
|
|
|
import java.net.URLEncoder;
|
2018-03-04 21:30:31 +01:00
|
|
|
import java.util.ArrayList;
|
|
|
|
import java.util.Collections;
|
|
|
|
import java.util.List;
|
2020-01-25 13:16:42 +01:00
|
|
|
import java.util.Locale;
|
2018-03-04 21:30:31 +01:00
|
|
|
|
2020-03-17 20:31:01 +01:00
|
|
|
import javax.annotation.Nonnull;
|
|
|
|
|
2020-04-16 16:08:14 +02:00
|
|
|
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
|
|
|
|
|
2017-08-04 16:21:45 +02:00
|
|
|
public class SoundcloudStreamExtractor extends StreamExtractor {
|
2017-08-16 04:40:03 +02:00
|
|
|
private JsonObject track;
|
2017-08-04 16:21:45 +02:00
|
|
|
|
2019-04-28 22:03:16 +02:00
|
|
|
public SoundcloudStreamExtractor(StreamingService service, LinkHandler linkHandler) {
|
|
|
|
super(service, linkHandler);
|
2017-08-06 22:20:15 +02:00
|
|
|
}
|
2017-08-04 16:21:45 +02:00
|
|
|
|
2017-08-06 22:20:15 +02:00
|
|
|
@Override
|
2017-11-28 13:37:01 +01:00
|
|
|
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
|
|
|
|
track = SoundcloudParsingHelper.resolveFor(downloader, getOriginalUrl());
|
2017-08-04 16:21:45 +02:00
|
|
|
|
2020-04-16 16:08:14 +02:00
|
|
|
String policy = track.getString("policy", EMPTY_STRING);
|
2017-08-10 19:50:59 +02:00
|
|
|
if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) {
|
|
|
|
throw new ContentNotAvailableException("Content not available: policy " + policy);
|
2017-08-04 16:21:45 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-11-25 01:10:04 +01:00
|
|
|
@Nonnull
|
2017-08-04 16:21:45 +02:00
|
|
|
@Override
|
|
|
|
public String getId() {
|
2017-08-06 22:20:15 +02:00
|
|
|
return track.getInt("id") + "";
|
2017-08-04 16:21:45 +02:00
|
|
|
}
|
|
|
|
|
2017-11-25 01:10:04 +01:00
|
|
|
@Nonnull
|
2017-08-04 16:21:45 +02:00
|
|
|
@Override
|
2017-08-11 03:23:09 +02:00
|
|
|
public String getName() {
|
2017-08-16 04:40:03 +02:00
|
|
|
return track.getString("title");
|
2017-08-04 16:21:45 +02:00
|
|
|
}
|
|
|
|
|
2017-11-25 01:10:04 +01:00
|
|
|
@Nonnull
|
2017-08-10 19:50:59 +02:00
|
|
|
@Override
|
2020-03-17 18:06:13 +01:00
|
|
|
public String getTextualUploadDate() throws ParsingException {
|
|
|
|
return track.getString("created_at").replace("T"," ").replace("Z", "");
|
2019-04-28 22:03:16 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Nonnull
|
|
|
|
@Override
|
2019-11-03 19:45:25 +01:00
|
|
|
public DateWrapper getUploadDate() throws ParsingException {
|
2020-04-10 14:48:11 +02:00
|
|
|
return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(track.getString("created_at")));
|
2017-08-04 16:21:45 +02:00
|
|
|
}
|
|
|
|
|
2017-11-25 01:10:04 +01:00
|
|
|
@Nonnull
|
2017-08-04 16:21:45 +02:00
|
|
|
@Override
|
2017-08-11 03:23:09 +02:00
|
|
|
public String getThumbnailUrl() {
|
2020-04-16 16:08:14 +02:00
|
|
|
String artworkUrl = track.getString("artwork_url", EMPTY_STRING);
|
2019-04-25 23:29:15 +02:00
|
|
|
if (artworkUrl.isEmpty()) {
|
2020-04-16 16:08:14 +02:00
|
|
|
artworkUrl = track.getObject("user").getString("avatar_url", EMPTY_STRING);
|
2019-04-25 23:29:15 +02:00
|
|
|
}
|
|
|
|
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
|
|
|
|
return artworkUrlBetterResolution;
|
2017-08-04 16:21:45 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2020-02-06 23:35:46 +01:00
|
|
|
public Description getDescription() {
|
2020-02-07 13:28:27 +01:00
|
|
|
return new Description(track.getString("description"), Description.PLAIN_TEXT);
|
2017-08-04 16:21:45 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2017-08-11 03:23:09 +02:00
|
|
|
public int getAgeLimit() {
|
2017-11-25 02:03:30 +01:00
|
|
|
return NO_AGE_LIMIT;
|
2017-08-04 16:21:45 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2017-08-11 03:23:09 +02:00
|
|
|
public long getLength() {
|
2017-08-16 04:40:03 +02:00
|
|
|
return track.getNumber("duration", 0).longValue() / 1000L;
|
2017-08-04 16:21:45 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2017-08-11 03:23:09 +02:00
|
|
|
public long getTimeStamp() throws ParsingException {
|
2017-11-22 18:45:49 +01:00
|
|
|
return getTimestampSeconds("(#t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)");
|
2017-08-04 16:21:45 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2017-08-11 03:23:09 +02:00
|
|
|
public long getViewCount() {
|
2017-08-16 04:40:03 +02:00
|
|
|
return track.getNumber("playback_count", 0).longValue();
|
2017-08-04 16:21:45 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2017-08-10 19:50:59 +02:00
|
|
|
public long getLikeCount() {
|
2017-09-16 11:08:08 +02:00
|
|
|
return track.getNumber("favoritings_count", -1).longValue();
|
2017-08-04 16:21:45 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2017-08-10 19:50:59 +02:00
|
|
|
public long getDislikeCount() {
|
2017-09-15 19:35:43 +02:00
|
|
|
return -1;
|
2017-08-04 16:21:45 +02:00
|
|
|
}
|
|
|
|
|
2017-11-25 01:10:04 +01:00
|
|
|
@Nonnull
|
2017-08-11 03:23:09 +02:00
|
|
|
@Override
|
|
|
|
public String getUploaderUrl() {
|
2018-02-21 09:23:57 +01:00
|
|
|
return SoundcloudParsingHelper.getUploaderUrl(track);
|
2017-08-11 03:23:09 +02:00
|
|
|
}
|
|
|
|
|
2017-11-25 01:10:04 +01:00
|
|
|
@Nonnull
|
2017-08-11 03:23:09 +02:00
|
|
|
@Override
|
|
|
|
public String getUploaderName() {
|
2018-02-21 09:23:57 +01:00
|
|
|
return SoundcloudParsingHelper.getUploaderName(track);
|
2017-08-11 03:23:09 +02:00
|
|
|
}
|
|
|
|
|
2017-11-25 01:10:04 +01:00
|
|
|
@Nonnull
|
2017-08-11 03:23:09 +02:00
|
|
|
@Override
|
|
|
|
public String getUploaderAvatarUrl() {
|
2018-02-21 09:23:57 +01:00
|
|
|
return SoundcloudParsingHelper.getAvatarUrl(track);
|
2017-08-11 03:23:09 +02:00
|
|
|
}
|
|
|
|
|
2020-04-13 22:34:20 +02:00
|
|
|
@Nonnull
|
|
|
|
@Override
|
|
|
|
public String getParentChannelUrl() throws ParsingException {
|
|
|
|
return "";
|
|
|
|
}
|
|
|
|
|
|
|
|
@Nonnull
|
|
|
|
@Override
|
|
|
|
public String getParentChannelName() throws ParsingException {
|
|
|
|
return "";
|
|
|
|
}
|
|
|
|
|
|
|
|
@Nonnull
|
|
|
|
@Override
|
|
|
|
public String getParentChannelAvatarUrl() throws ParsingException {
|
|
|
|
return "";
|
|
|
|
}
|
|
|
|
|
2018-02-25 22:03:32 +01:00
|
|
|
@Nonnull
|
2017-08-11 03:23:09 +02:00
|
|
|
@Override
|
|
|
|
public String getDashMpdUrl() {
|
2018-02-25 22:03:32 +01:00
|
|
|
return "";
|
|
|
|
}
|
|
|
|
|
|
|
|
@Nonnull
|
|
|
|
@Override
|
|
|
|
public String getHlsUrl() throws ParsingException {
|
|
|
|
return "";
|
2017-08-11 03:23:09 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
|
|
|
|
List<AudioStream> audioStreams = new ArrayList<>();
|
|
|
|
Downloader dl = NewPipe.getDownloader();
|
|
|
|
|
2019-12-22 00:40:32 +01:00
|
|
|
// Streams can be streamable and downloadable - or explicitly not.
|
|
|
|
// For playing the track, it is only necessary to have a streamable track.
|
|
|
|
// If this is not the case, this track might not be published yet.
|
2020-03-18 11:01:46 +01:00
|
|
|
if (!track.getBoolean("streamable")) return audioStreams;
|
2019-12-22 00:40:32 +01:00
|
|
|
|
|
|
|
try {
|
2020-03-18 11:01:46 +01:00
|
|
|
JsonArray transcodings = track.getObject("media").getArray("transcodings");
|
2019-12-22 00:40:32 +01:00
|
|
|
|
|
|
|
// get information about what stream formats are available
|
|
|
|
for (Object transcoding : transcodings) {
|
|
|
|
|
|
|
|
JsonObject t = (JsonObject) transcoding;
|
|
|
|
String url = t.getString("url");
|
|
|
|
|
|
|
|
if (url != null && !url.isEmpty()) {
|
|
|
|
|
|
|
|
// We can only play the mp3 format, but not handle m3u playlists / streams.
|
|
|
|
// what about Opus?
|
|
|
|
if (t.getString("preset").contains("mp3")
|
|
|
|
&& t.getObject("format").getString("protocol").equals("progressive")) {
|
|
|
|
// This url points to the endpoint which generates a unique and short living url to the stream.
|
|
|
|
// TODO: move this to a separate method to generate valid urls when needed (e.g. resuming a paused stream)
|
|
|
|
url += "?client_id=" + SoundcloudParsingHelper.clientId();
|
|
|
|
String res = dl.get(url).responseBody();
|
|
|
|
|
|
|
|
try {
|
|
|
|
JsonObject mp3UrlObject = JsonParser.object().from(res);
|
|
|
|
// Links in this file are also only valid for a short period.
|
2020-02-08 23:58:46 +01:00
|
|
|
audioStreams.add(new AudioStream(mp3UrlObject.getString("url"),
|
2019-12-22 00:40:32 +01:00
|
|
|
MediaFormat.MP3, 128));
|
|
|
|
} catch (JsonParserException e) {
|
|
|
|
throw new ParsingException("Could not parse streamable url", e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
} catch (NullPointerException e) {
|
|
|
|
throw new ExtractionException("Could not get SoundCloud's track audio url", e);
|
2017-08-16 04:40:03 +02:00
|
|
|
}
|
2017-08-11 03:23:09 +02:00
|
|
|
|
2020-03-30 11:48:00 +02:00
|
|
|
if (audioStreams.isEmpty()) {
|
2020-04-08 15:30:39 +02:00
|
|
|
throw new ContentNotSupportedException("HLS audio streams are not yet supported");
|
2020-03-30 11:48:00 +02:00
|
|
|
}
|
|
|
|
|
2017-08-11 03:23:09 +02:00
|
|
|
return audioStreams;
|
|
|
|
}
|
|
|
|
|
2017-11-25 02:20:16 +01:00
|
|
|
private static String urlEncode(String value) {
|
|
|
|
try {
|
|
|
|
return URLEncoder.encode(value, "UTF-8");
|
|
|
|
} catch (UnsupportedEncodingException e) {
|
|
|
|
throw new IllegalStateException(e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-08-11 03:23:09 +02:00
|
|
|
@Override
|
|
|
|
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException {
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
|
2017-11-22 18:39:38 +01:00
|
|
|
@Override
|
2018-02-01 22:27:14 +01:00
|
|
|
@Nonnull
|
2018-09-24 21:04:22 +02:00
|
|
|
public List<SubtitlesStream> getSubtitlesDefault() throws IOException, ExtractionException {
|
2018-02-01 22:27:14 +01:00
|
|
|
return Collections.emptyList();
|
2017-11-23 16:33:03 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2018-02-01 22:27:14 +01:00
|
|
|
@Nonnull
|
2018-09-24 21:04:22 +02:00
|
|
|
public List<SubtitlesStream> getSubtitles(MediaFormat format) throws IOException, ExtractionException {
|
2018-02-01 22:27:14 +01:00
|
|
|
return Collections.emptyList();
|
2017-11-23 00:10:12 +01:00
|
|
|
}
|
|
|
|
|
2017-08-11 03:23:09 +02:00
|
|
|
@Override
|
|
|
|
public StreamType getStreamType() {
|
|
|
|
return StreamType.AUDIO_STREAM;
|
|
|
|
}
|
|
|
|
|
2017-08-04 16:21:45 +02:00
|
|
|
@Override
|
2018-11-07 18:28:44 +01:00
|
|
|
public StreamInfoItem getNextStream() throws IOException, ExtractionException {
|
2017-08-04 16:21:45 +02:00
|
|
|
return null;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2018-11-07 18:28:44 +01:00
|
|
|
public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException {
|
2018-02-24 22:20:50 +01:00
|
|
|
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
2017-08-04 16:21:45 +02:00
|
|
|
|
2017-11-25 02:20:16 +01:00
|
|
|
String apiUrl = "https://api-v2.soundcloud.com/tracks/" + urlEncode(getId()) + "/related"
|
|
|
|
+ "?client_id=" + urlEncode(SoundcloudParsingHelper.clientId());
|
2017-08-04 16:21:45 +02:00
|
|
|
|
2017-08-10 19:50:59 +02:00
|
|
|
SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
|
2017-08-04 16:21:45 +02:00
|
|
|
return collector;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public String getErrorMessage() {
|
|
|
|
return null;
|
|
|
|
}
|
added metadata, fix descriptions, fix thumbnail, update tests
thumbnail: quality before: https://peertube.cpy.re/static/thumbnails/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
quality after: https://peertube.cpy.re/static/previews/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
description: we were getting about the first 260 characters, we now get full description (with fallback to first 260 chars if the get request for full description fails)
test: updated tests to match description, also changed some test: it was assertEquals(extracted, expected), but the proper way to do it is assertEquals(expected, extracted)
metadata: got host, privacy (public, private, unlisted), licence, language, tags
2020-01-19 12:45:52 +01:00
|
|
|
|
|
|
|
@Override
|
|
|
|
public String getHost() throws ParsingException {
|
|
|
|
return "";
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public String getPrivacy() throws ParsingException {
|
|
|
|
return "";
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public String getCategory() throws ParsingException {
|
|
|
|
return "";
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public String getLicence() throws ParsingException {
|
|
|
|
return "";
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2020-01-25 13:16:42 +01:00
|
|
|
public Locale getLanguageInfo() throws ParsingException {
|
|
|
|
return null;
|
added metadata, fix descriptions, fix thumbnail, update tests
thumbnail: quality before: https://peertube.cpy.re/static/thumbnails/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
quality after: https://peertube.cpy.re/static/previews/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
description: we were getting about the first 260 characters, we now get full description (with fallback to first 260 chars if the get request for full description fails)
test: updated tests to match description, also changed some test: it was assertEquals(extracted, expected), but the proper way to do it is assertEquals(expected, extracted)
metadata: got host, privacy (public, private, unlisted), licence, language, tags
2020-01-19 12:45:52 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Nonnull
|
|
|
|
@Override
|
|
|
|
public List<String> getTags() throws ParsingException {
|
|
|
|
return new ArrayList<>();
|
|
|
|
}
|
2020-01-23 14:19:22 +01:00
|
|
|
|
|
|
|
@Nonnull
|
|
|
|
@Override
|
|
|
|
public String getSupportInfo() throws ParsingException {
|
|
|
|
return "";
|
|
|
|
}
|
2017-08-04 16:21:45 +02:00
|
|
|
}
|