lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Apply changes in all StreamExtractors except YouTube's one and fix extraction of PeerTube audio streams as video streams 

Some code in these classes has been also refactored/improved/optimized.
Also fix the extraction of PeerTube audio streams as video streams, which are now returned as audio streams.
This commit is contained in:
TiA4f8R 2022-03-03 20:46:53 +01:00
parent d5f3637fc3
commit 881969f1da
No known key found for this signature in database
GPG Key ID: E6D3E7F5949450DD
6 changed files with 629 additions and 235 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java
View File

@ -5,7 +5,6 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
@ -30,9 +29,12 @@ import java.util.List;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_API_URL;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class BandcampRadioStreamExtractor extends BandcampStreamExtractor {
private static final String OPUS_LO = "opus_lo";
private static final String MP3_128 = "mp3-128";
private JsonObject showInfo;
public BandcampRadioStreamExtractor(final StreamingService service,
@ -78,11 +80,9 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor {
@Nonnull
@Override
public String getUploaderName() throws ParsingException {
return Jsoup.parse(showInfo.getString("image_caption")).getElementsByTag("a").stream()
.map(Element::text)
.findFirst()
.orElseThrow(() -> new ParsingException("Could not get uploader name"));
public String getUploaderName() {
return Jsoup.parse(showInfo.getString("image_caption"))
.getElementsByTag("a").first().text();
}
@Nullable
@ -116,23 +116,25 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor {
@Override
public List<AudioStream> getAudioStreams() {
final ArrayList<AudioStream> list = new ArrayList<>();
final List<AudioStream> audioStreams = new ArrayList<>();
final JsonObject streams = showInfo.getObject("audio_stream");
if (streams.has("opus-lo")) {
list.add(new AudioStream(
streams.getString("opus-lo"),
MediaFormat.OPUS, 100
));
}
if (streams.has("mp3-128")) {
list.add(new AudioStream(
streams.getString("mp3-128"),
MediaFormat.MP3, 128
));
if (streams.has(MP3_128)) {
audioStreams.add(new AudioStream.Builder()
.setId(MP3_128)
.setContent(streams.getString(MP3_128), true)
.setMediaFormat(MediaFormat.MP3)
.setAverageBitrate(128)
.build());
} else if (streams.has(OPUS_LO)) {
audioStreams.add(new AudioStream.Builder()
.setId(OPUS_LO)
.setContent(streams.getString(OPUS_LO), true)
.setMediaFormat(MediaFormat.OPUS)
.setAverageBitrate(100).build());
}
return list;
return audioStreams;
}
@Nonnull
@ -156,14 +158,14 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor {
@Override
public String getLicence() {
// Contrary to other Bandcamp streams, radio streams don't have a license
return "";
return EMPTY_STRING;
}
@Nonnull
@Override
public String getCategory() {
// Contrary to other Bandcamp streams, radio streams don't have categories
return "";
return EMPTY_STRING;
}
@Nonnull

63
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java
View File

@ -3,6 +3,8 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParserException;
@ -10,7 +12,6 @@ import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -27,16 +28,15 @@ import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.stream.Collectors;
public class BandcampStreamExtractor extends StreamExtractor {
private JsonObject albumJson;
private JsonObject current;
private Document document;
@ -88,7 +88,7 @@ public class BandcampStreamExtractor extends StreamExtractor {
public String getUploaderUrl() throws ParsingException {
final String[] parts = getUrl().split("/");
// https: (/) (/) * .bandcamp.com (/) and leave out the rest
return "https://" + parts[2] + "/";
return HTTPS + parts[2] + "/";
}
@Nonnull
@ -119,7 +119,7 @@ public class BandcampStreamExtractor extends StreamExtractor {
@Override
public String getThumbnailUrl() throws ParsingException {
if (albumJson.isNull("art_id")) {
return Utils.EMPTY_STRING;
return EMPTY_STRING;
} else {
return getImageUrl(albumJson.getLong("art_id"), true);
}
@ -139,24 +139,26 @@ public class BandcampStreamExtractor extends StreamExtractor {
public Description getDescription() {
final String s = Utils.nonEmptyAndNullJoin(
"\n\n",
new String[]{
new String[] {
current.getString("about"),
current.getString("lyrics"),
current.getString("credits")
}
);
});
return new Description(s, Description.PLAIN_TEXT);
}
@Override
public List<AudioStream> getAudioStreams() {
final List<AudioStream> audioStreams = new ArrayList<>();
audioStreams.add(new AudioStream(
albumJson.getArray("trackinfo").getObject(0)
.getObject("file").getString("mp3-128"),
MediaFormat.MP3, 128
));
audioStreams.add(new AudioStream.Builder()
.setId("mp3-128")
.setContent(albumJson.getArray("trackinfo")
.getObject(0)
.getObject("file")
.getString("mp3-128"), true)
.setMediaFormat(MediaFormat.MP3)
.setAverageBitrate(128)
.build());
return audioStreams;
}
@ -184,11 +186,11 @@ public class BandcampStreamExtractor extends StreamExtractor {
@Override
public PlaylistInfoItemsCollector getRelatedItems() {
final PlaylistInfoItemsCollector collector = new PlaylistInfoItemsCollector(getServiceId());
final Elements recommendedAlbums = document.getElementsByClass("recommended-album");
document.getElementsByClass("recommended-album")
.stream()
.map(BandcampRelatedPlaylistInfoItemExtractor::new)
.forEach(collector::commit);
for (final Element album : recommendedAlbums) {
collector.commit(new BandcampRelatedPlaylistInfoItemExtractor(album));
}
return collector;
}
@ -200,15 +202,17 @@ public class BandcampStreamExtractor extends StreamExtractor {
.flatMap(element -> element.getElementsByClass("tag").stream())
.map(Element::text)
.findFirst()
.orElse("");
.orElse(EMPTY_STRING);
}
@Nonnull
@Override
public String getLicence() {
/* Tests resulted in this mapping of ints to licence:
/*
Tests resulted in this mapping of ints to licence:
https://cloud.disroot.org/s/ZTWBxbQ9fKRmRWJ/preview (screenshot from a Bandcamp artist's
account) */
account)
*/
switch (current.getInt("license_type")) {
case 1:
@ -233,14 +237,9 @@ public class BandcampStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public List<String> getTags() {
final Elements tagElements = document.getElementsByAttributeValue("itemprop", "keywords");
final List<String> tags = new ArrayList<>();
for (final Element e : tagElements) {
tags.add(e.text());
}
return tags;
return document.getElementsByAttributeValue("itemprop", "keywords")
.stream()
.map(Element::text)
.collect(Collectors.toList());
}
}

175
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java
View File

@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.DeliveryMethod;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
@ -17,11 +18,21 @@ import org.schabi.newpipe.extractor.stream.VideoStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.stream.IntStream;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE;
import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class MediaCCCLiveStreamExtractor extends StreamExtractor {
private static final String STREAMS = "streams";
private static final String URLS = "urls";
private static final String URL = "url";
private JsonObject conference = null;
private String group = "";
private JsonObject room = null;
@ -34,19 +45,22 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
@Override
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
final JsonArray doc =
MediaCCCParsingHelper.getLiveStreams(downloader, getExtractorLocalization());
// find correct room
final JsonArray doc = MediaCCCParsingHelper.getLiveStreams(downloader,
getExtractorLocalization());
// Find the correct room
for (int c = 0; c < doc.size(); c++) {
conference = doc.getObject(c);
final JsonArray groups = conference.getArray("groups");
final JsonObject conferenceObject = doc.getObject(c);
final JsonArray groups = conferenceObject.getArray("groups");
for (int g = 0; g < groups.size(); g++) {
group = groups.getObject(g).getString("group");
final String groupObject = groups.getObject(g).getString("group");
final JsonArray rooms = groups.getObject(g).getArray("rooms");
for (int r = 0; r < rooms.size(); r++) {
room = rooms.getObject(r);
if (getId().equals(
conference.getString("slug") + "/" + room.getString("slug"))) {
final JsonObject roomObject = rooms.getObject(r);
if (getId().equals(conferenceObject.getString("slug") + "/"
+ roomObject.getString("slug"))) {
this.conference = conferenceObject;
this.group = groupObject;
this.room = roomObject;
return;
}
}
@ -91,69 +105,136 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
return conference.getString("conference");
}
/**
* Get the URL of the first DASH stream found.
*
* <p>
* There can be several DASH streams, so the URL of the first found is returned by this method.
* </p>
*
* <p>
* You can find the other video DASH streams by using {@link #getVideoStreams()}
* </p>
*/
@Nonnull
@Override
public String getDashMpdUrl() throws ParsingException {
for (int s = 0; s < room.getArray(STREAMS).size(); s++) {
final JsonObject stream = room.getArray(STREAMS).getObject(s);
final JsonObject urls = stream.getObject(URLS);
if (urls.has("dash")) {
return urls.getObject("dash").getString(URL, EMPTY_STRING);
}
}
return EMPTY_STRING;
}
/**
* Get the URL of the first HLS stream found.
*
* <p>
* There can be several HLS streams, so the URL of the first found is returned by this method.
* </p>
*
* <p>
* You can find the other video HLS streams by using {@link #getVideoStreams()}
* </p>
*/
@Nonnull
@Override
public String getHlsUrl() {
// TODO: There are multiple HLS streams.
// Make getHlsUrl() and getDashMpdUrl() return lists of VideoStreams,
// so the user can choose a resolution.
for (int s = 0; s < room.getArray("streams").size(); s++) {
final JsonObject stream = room.getArray("streams").getObject(s);
if (stream.getString("type").equals("video")) {
if (stream.has("hls")) {
return stream.getObject("urls").getObject("hls").getString("url");
}
for (int s = 0; s < room.getArray(STREAMS).size(); s++) {
final JsonObject stream = room.getArray(STREAMS).getObject(s);
final JsonObject urls = stream.getObject(URLS);
if (urls.has("hls")) {
return urls.getObject("hls").getString(URL, EMPTY_STRING);
}
}
return "";
return EMPTY_STRING;
}
@Override
public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
final List<AudioStream> audioStreams = new ArrayList<>();
for (int s = 0; s < room.getArray("streams").size(); s++) {
final JsonObject stream = room.getArray("streams").getObject(s);
if (stream.getString("type").equals("audio")) {
for (final String type : stream.getObject("urls").keySet()) {
final JsonObject url = stream.getObject("urls").getObject(type);
audioStreams.add(new AudioStream(url.getString("url"),
MediaFormat.getFromSuffix(type), -1));
}
}
}
IntStream.range(0, room.getArray(STREAMS).size())
.mapToObj(s -> room.getArray(STREAMS).getObject(s))
.filter(streamJsonObject -> streamJsonObject.getString("type").equals("audio"))
.forEachOrdered(streamJsonObject -> streamJsonObject.getObject(URLS).keySet()
.forEach(type -> {
final JsonObject urlObject = streamJsonObject.getObject(URLS)
.getObject(type);
// The DASH manifest will be extracted with getDashMpdUrl
if (!type.equals("dash")) {
final AudioStream.Builder builder = new AudioStream.Builder()
.setId(urlObject.getString("tech", ID_UNKNOWN))
.setContent(urlObject.getString(URL), true)
.setAverageBitrate(UNKNOWN_BITRATE);
if (type.equals("hls")) {
// We don't know with the type string what media format will
// have HLS streams.
// However, the tech string may contain some information
// about the media format used.
builder.setDeliveryMethod(DeliveryMethod.HLS);
} else {
builder.setMediaFormat(MediaFormat.getFromSuffix(type));
}
audioStreams.add(builder.build());
}
}));
return audioStreams;
}
@Override
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
final List<VideoStream> videoStreams = new ArrayList<>();
for (int s = 0; s < room.getArray("streams").size(); s++) {
final JsonObject stream = room.getArray("streams").getObject(s);
if (stream.getString("type").equals("video")) {
final String resolution = stream.getArray("videoSize").getInt(0) + "x"
+ stream.getArray("videoSize").getInt(1);
for (final String type : stream.getObject("urls").keySet()) {
if (!type.equals("hls")) {
final JsonObject url = stream.getObject("urls").getObject(type);
videoStreams.add(new VideoStream(
url.getString("url"),
MediaFormat.getFromSuffix(type),
resolution));
}
}
}
}
IntStream.range(0, room.getArray(STREAMS).size())
.mapToObj(s -> room.getArray(STREAMS).getObject(s))
.filter(stream -> stream.getString("type").equals("video"))
.forEachOrdered(streamJsonObject -> streamJsonObject.getObject(URLS).keySet()
.forEach(type -> {
final String resolution =
streamJsonObject.getArray("videoSize").getInt(0)
+ "x"
+ streamJsonObject.getArray("videoSize").getInt(1);
final JsonObject urlObject = streamJsonObject.getObject(URLS)
.getObject(type);
// The DASH manifest will be extracted with getDashMpdUrl
if (!type.equals("dash")) {
final VideoStream.Builder builder = new VideoStream.Builder()
.setId(urlObject.getString("tech", ID_UNKNOWN))
.setContent(urlObject.getString(URL), true)
.setIsVideoOnly(false)
.setResolution(resolution);
if (type.equals("hls")) {
// We don't know with the type string what media format will
// have HLS streams.
// However, the tech string may contain some information
// about the media format used.
builder.setDeliveryMethod(DeliveryMethod.HLS);
} else {
builder.setMediaFormat(MediaFormat.getFromSuffix(type));
}
videoStreams.add(builder.build());
}
}));
return videoStreams;
}
@Override
public List<VideoStream> getVideoOnlyStreams() {
return null;
return Collections.emptyList();
}
@Override
public StreamType getStreamType() throws ParsingException {
return StreamType.LIVE_STREAM; // TODO: video and audio only streams are both available
return StreamType.LIVE_STREAM;
}
@Nonnull

37
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java
View File

@ -1,5 +1,8 @@
package org.schabi.newpipe.extractor.services.media_ccc.extractors;
import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE;
import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
@ -99,7 +102,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
final JsonObject recording = recordings.getObject(i);
final String mimeType = recording.getString("mime_type");
if (mimeType.startsWith("audio")) {
//first we need to resolve the actual video data from CDN
// First we need to resolve the actual video data from CDN
final MediaFormat mediaFormat;
if (mimeType.endsWith("opus")) {
mediaFormat = MediaFormat.OPUS;
@ -108,11 +111,18 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
} else if (mimeType.endsWith("ogg")) {
mediaFormat = MediaFormat.OGG;
} else {
throw new ExtractionException("Unknown media format: " + mimeType);
mediaFormat = null;
}
audioStreams.add(new AudioStream(recording.getString("recording_url"),
mediaFormat, -1));
// Don't use the containsSimilarStream method because it will always return
// false so if there are multiples audio streams available, only the first will
// be extracted in this case.
audioStreams.add(new AudioStream.Builder()
.setId(recording.getString("filename", ID_UNKNOWN))
.setContent(recording.getString("recording_url"), true)
.setMediaFormat(mediaFormat)
.setAverageBitrate(UNKNOWN_BITRATE)
.build());
}
}
return audioStreams;
@ -126,7 +136,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
final JsonObject recording = recordings.getObject(i);
final String mimeType = recording.getString("mime_type");
if (mimeType.startsWith("video")) {
//first we need to resolve the actual video data from CDN
// First we need to resolve the actual video data from CDN
final MediaFormat mediaFormat;
if (mimeType.endsWith("webm")) {
@ -134,13 +144,21 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
} else if (mimeType.endsWith("mp4")) {
mediaFormat = MediaFormat.MPEG_4;
} else {
throw new ExtractionException("Unknown media format: " + mimeType);
mediaFormat = null;
}
videoStreams.add(new VideoStream(recording.getString("recording_url"),
mediaFormat, recording.getInt("height") + "p"));
// Don't use the containsSimilarStream method because it will remove the
// extraction of some video versions (mostly languages)
videoStreams.add(new VideoStream.Builder()
.setId(recording.getString("filename", ID_UNKNOWN))
.setContent(recording.getString("recording_url"), true)
.setIsVideoOnly(false)
.setMediaFormat(mediaFormat)
.setResolution(recording.getInt("height") + "p")
.build());
}
}
return videoStreams;
}
@ -163,7 +181,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
conferenceData = JsonParser.object()
.from(downloader.get(data.getString("conference_url")).responseBody());
} catch (final JsonParserException jpe) {
throw new ExtractionException("Could not parse json returned by url: " + videoUrl, jpe);
throw new ExtractionException("Could not parse json returned by URL: " + videoUrl,
jpe);
}
}

409
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java
View File

@ -22,6 +22,7 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.DeliveryMethod;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
@ -39,14 +40,30 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class PeertubeStreamExtractor extends StreamExtractor {
private static final String ACCOUNT_HOST = "account.host";
private static final String ACCOUNT_NAME = "account.name";
private static final String FILES = "files";
private static final String FILE_DOWNLOAD_URL = "fileDownloadUrl";
private static final String FILE_URL = "fileUrl";
private static final String PLAYLIST_URL = "playlistUrl";
private static final String RESOLUTION_ID = "resolution.id";
private static final String STREAMING_PLAYLISTS = "streamingPlaylists";
private final String baseUrl;
private JsonObject json;
private final List<SubtitlesStream> subtitles = new ArrayList<>();
private final List<AudioStream> audioStreams = new ArrayList<>();
private final List<VideoStream> videoStreams = new ArrayList<>();
public PeertubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler)
throws ParsingException {
@ -85,9 +102,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
} catch (final ParsingException e) {
return Description.EMPTY_DESCRIPTION;
}
if (text.length() == 250 && text.substring(247).equals("...")) {
//if description is shortened, get full description
// If description is shortened, get full description
final Downloader dl = NewPipe.getDownloader();
try {
final Response response = dl.get(baseUrl
@ -95,8 +111,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
+ getId() + "/description");
final JsonObject jsonObject = JsonParser.object().from(response.responseBody());
text = JsonUtils.getString(jsonObject, "description");
} catch (ReCaptchaException | IOException | JsonParserException e) {
e.printStackTrace();
} catch (final IOException | ReCaptchaException | JsonParserException ignored) {
// Something went wrong when getting the full description, use the shortened one
}
}
return new Description(text, Description.MARKDOWN);
@ -119,8 +135,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
@Override
public long getTimeStamp() throws ParsingException {
final long timestamp =
getTimestampSeconds("((#|&|\\?)start=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)");
final long timestamp = getTimestampSeconds(
"((#|&|\\?)start=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)");
if (timestamp == -2) {
// regex for timestamp was not found
@ -148,10 +164,10 @@ public class PeertubeStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getUploaderUrl() throws ParsingException {
final String name = JsonUtils.getString(json, "account.name");
final String host = JsonUtils.getString(json, "account.host");
return getService().getChannelLHFactory()
.fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
final String name = JsonUtils.getString(json, ACCOUNT_NAME);
final String host = JsonUtils.getString(json, ACCOUNT_HOST);
return getService().getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl)
.getUrl();
}
@Nonnull
@ -199,77 +215,51 @@ public class PeertubeStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getHlsUrl() {
return json.getArray("streamingPlaylists").getObject(0).getString("playlistUrl");
assertPageFetched();
if (getStreamType() == StreamType.VIDEO_STREAM
&& !isNullOrEmpty(json.getObject(FILES))) {
return json.getObject(FILES).getString(PLAYLIST_URL, EMPTY_STRING);
} else {
return json.getArray(STREAMING_PLAYLISTS).getObject(0).getString(PLAYLIST_URL,
EMPTY_STRING);
}
}
@Override
public List<AudioStream> getAudioStreams() {
return Collections.emptyList();
public List<AudioStream> getAudioStreams() throws ParsingException {
assertPageFetched();
/*
Some videos have audio streams, some videos don't have audio streams.
So an audio stream may be available if a video stream is available.
Audio streams are also not returned as separated streams for livestreams.
That's why the extraction of audio streams is only run when there are video streams
extracted and when the content is not a livestream.
*/
if (audioStreams.isEmpty() && videoStreams.isEmpty()
&& getStreamType() == StreamType.VIDEO_STREAM) {
getStreams();
}
return audioStreams;
}
@Override
public List<VideoStream> getVideoStreams() throws ExtractionException {
assertPageFetched();
final List<VideoStream> videoStreams = new ArrayList<>();
// mp4
try {
videoStreams.addAll(getVideoStreamsFromArray(json.getArray("files")));
} catch (final Exception ignored) { }
// HLS
try {
final JsonArray streamingPlaylists = json.getArray("streamingPlaylists");
for (final Object p : streamingPlaylists) {
if (!(p instanceof JsonObject)) {
continue;
}
final JsonObject playlist = (JsonObject) p;
videoStreams.addAll(getVideoStreamsFromArray(playlist.getArray("files")));
if (videoStreams.isEmpty()) {
if (getStreamType() == StreamType.VIDEO_STREAM) {
getStreams();
} else {
extractLiveVideoStreams();
}
} catch (final Exception e) {
throw new ParsingException("Could not get video streams", e);
}
if (getStreamType() == StreamType.LIVE_STREAM) {
videoStreams.add(new VideoStream(getHlsUrl(), MediaFormat.MPEG_4, "720p"));
}
return videoStreams;
}
private List<VideoStream> getVideoStreamsFromArray(final JsonArray streams)
throws ParsingException {
try {
final List<VideoStream> videoStreams = new ArrayList<>();
for (final Object s : streams) {
if (!(s instanceof JsonObject)) {
continue;
}
final JsonObject stream = (JsonObject) s;
final String url;
if (stream.has("fileDownloadUrl")) {
url = JsonUtils.getString(stream, "fileDownloadUrl");
} else {
url = JsonUtils.getString(stream, "fileUrl");
}
final String torrentUrl = JsonUtils.getString(stream, "torrentUrl");
final String resolution = JsonUtils.getString(stream, "resolution.label");
final String extension = url.substring(url.lastIndexOf(".") + 1);
final MediaFormat format = MediaFormat.getFromSuffix(extension);
final VideoStream videoStream
= new VideoStream(url, torrentUrl, format, resolution);
if (!Stream.containSimilarStream(videoStream, videoStreams)) {
videoStreams.add(videoStream);
}
}
return videoStreams;
} catch (final Exception e) {
throw new ParsingException("Could not get video streams from array");
}
}
@Override
public List<VideoStream> getVideoOnlyStreams() {
return Collections.emptyList();
@ -284,13 +274,9 @@ public class PeertubeStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public List<SubtitlesStream> getSubtitles(final MediaFormat format) {
final List<SubtitlesStream> filteredSubs = new ArrayList<>();
for (final SubtitlesStream sub : subtitles) {
if (sub.getFormat() == format) {
filteredSubs.add(sub);
}
}
return filteredSubs;
return subtitles.stream()
.filter(sub -> sub.getFormat() == format)
.collect(Collectors.toList());
}
@Override
@ -304,8 +290,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
final List<String> tags = getTags();
final String apiUrl;
if (tags.isEmpty()) {
apiUrl = baseUrl + "/api/v1/accounts/" + JsonUtils.getString(json, "account.name")
+ "@" + JsonUtils.getString(json, "account.host")
apiUrl = baseUrl + "/api/v1/accounts/" + JsonUtils.getString(json, ACCOUNT_NAME)
+ "@" + JsonUtils.getString(json, ACCOUNT_HOST)
+ "/videos?start=0&count=8";
} else {
apiUrl = getRelatedItemsUrl(tags);
@ -314,7 +300,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
if (Utils.isBlank(apiUrl)) {
return null;
} else {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(
getServiceId());
getStreamsFromApi(collector, apiUrl);
return collector;
}
@ -332,11 +319,13 @@ public class PeertubeStreamExtractor extends StreamExtractor {
try {
return JsonUtils.getString(json, "support");
} catch (final ParsingException e) {
return "";
return EMPTY_STRING;
}
}
private String getRelatedItemsUrl(final List<String> tags) throws UnsupportedEncodingException {
@Nonnull
private String getRelatedItemsUrl(@Nonnull final List<String> tags)
throws UnsupportedEncodingException {
final String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT;
final StringBuilder params = new StringBuilder();
params.append("start=0&count=8&sort=-createdAt");
@ -348,7 +337,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
}
private void getStreamsFromApi(final StreamInfoItemsCollector collector, final String apiUrl)
throws ReCaptchaException, IOException, ParsingException {
throws IOException, ReCaptchaException, ParsingException {
final Response response = getDownloader().get(apiUrl);
JsonObject relatedVideosJson = null;
if (response != null && !Utils.isBlank(response.responseBody())) {
@ -365,21 +354,20 @@ public class PeertubeStreamExtractor extends StreamExtractor {
}
private void collectStreamsFrom(final StreamInfoItemsCollector collector,
final JsonObject jsonObject)
throws ParsingException {
final JsonObject jsonObject) throws ParsingException {
final JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(jsonObject, "data");
} catch (final Exception e) {
throw new ParsingException("unable to extract related videos", e);
throw new ParsingException("Could not extract related videos", e);
}
for (final Object c : contents) {
if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
final PeertubeStreamInfoItemExtractor extractor
= new PeertubeStreamInfoItemExtractor(item, baseUrl);
//do not add the same stream in related streams
final PeertubeStreamInfoItemExtractor extractor =
new PeertubeStreamInfoItemExtractor(item, baseUrl);
// Do not add the same stream in related streams
if (!extractor.getUrl().equals(getUrl())) {
collector.commit(extractor);
}
@ -395,7 +383,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
if (response != null) {
setInitialData(response.responseBody());
} else {
throw new ExtractionException("Unable to extract PeerTube channel data");
throw new ExtractionException("Could not extract PeerTube channel data");
}
loadSubtitles();
@ -405,10 +393,10 @@ public class PeertubeStreamExtractor extends StreamExtractor {
try {
json = JsonParser.object().from(responseBody);
} catch (final JsonParserException e) {
throw new ExtractionException("Unable to extract PeerTube stream data", e);
throw new ExtractionException("Could not extract PeerTube stream data", e);
}
if (json == null) {
throw new ExtractionException("Unable to extract PeerTube stream data");
throw new ExtractionException("Could not extract PeerTube stream data");
}
PeertubeParsingHelper.validate(json);
}
@ -429,16 +417,253 @@ public class PeertubeStreamExtractor extends StreamExtractor {
final String ext = url.substring(url.lastIndexOf(".") + 1);
final MediaFormat fmt = MediaFormat.getFromSuffix(ext);
if (fmt != null && !isNullOrEmpty(languageCode)) {
subtitles.add(new SubtitlesStream(fmt, languageCode, url, false));
subtitles.add(new SubtitlesStream.Builder()
.setContent(url, true)
.setMediaFormat(fmt)
.setLanguageCode(languageCode)
.setAutoGenerated(false)
.build());
}
}
}
} catch (final Exception e) {
// ignore all exceptions
} catch (final Exception ignored) {
// Ignore all exceptions
}
}
}
private void extractLiveVideoStreams() throws ParsingException {
try {
final JsonArray streamingPlaylists = json.getArray(STREAMING_PLAYLISTS);
for (final Object s : streamingPlaylists) {
if (!(s instanceof JsonObject)) {
continue;
}
final JsonObject stream = (JsonObject) s;
// Don't use the containsSimilarStream method because it will always return false
// so if there are multiples HLS URLs returned, only the first will be extracted in
// this case.
videoStreams.add(new VideoStream.Builder()
.setId(String.valueOf(stream.getInt("id", -1)))
.setContent(stream.getString(PLAYLIST_URL, EMPTY_STRING), true)
.setIsVideoOnly(false)
.setResolution(EMPTY_STRING)
.setMediaFormat(MediaFormat.MPEG_4)
.setDeliveryMethod(DeliveryMethod.HLS)
.build());
}
} catch (final Exception e) {
throw new ParsingException("Could not get video streams", e);
}
}
private void getStreams() throws ParsingException {
// Progressive streams
getStreamsFromArray(json.getArray(FILES), EMPTY_STRING);
// HLS streams
try {
final JsonArray streamingPlaylists = json.getArray(STREAMING_PLAYLISTS);
for (final Object p : streamingPlaylists) {
if (!(p instanceof JsonObject)) {
continue;
}
final JsonObject playlist = (JsonObject) p;
final String playlistUrl = playlist.getString(PLAYLIST_URL);
getStreamsFromArray(playlist.getArray(FILES), playlistUrl);
}
} catch (final Exception e) {
throw new ParsingException("Could not get streams", e);
}
}
private void getStreamsFromArray(@Nonnull final JsonArray streams,
final String playlistUrl) throws ParsingException {
try {
/*
Starting with version 3.4.0 of PeerTube, HLS playlist of stream resolutions contain the
UUID of the stream, so we can't use the same system to get HLS playlist URL of streams
without fetching the master playlist.
These UUIDs are the same that the ones returned into the fileUrl and fileDownloadUrl
strings.
*/
final boolean isInstanceUsingRandomUuidsForHlsStreams = !isNullOrEmpty(playlistUrl)
&& playlistUrl.endsWith("-master.m3u8");
for (final Object s : streams) {
if (!(s instanceof JsonObject)) {
continue;
}
final JsonObject stream = (JsonObject) s;
final String resolution = JsonUtils.getString(stream, "resolution.label");
final String url;
final String idSuffix;
// Extract stream version of streams first
if (stream.has(FILE_URL)) {
url = JsonUtils.getString(stream, FILE_URL);
idSuffix = FILE_URL;
} else {
url = JsonUtils.getString(stream, FILE_DOWNLOAD_URL);
idSuffix = FILE_DOWNLOAD_URL;
}
if (isNullOrEmpty(url)) {
// Not a valid stream URL
return;
}
if (resolution.toLowerCase().contains("audio")) {
// An audio stream
addNewAudioStream(stream, isInstanceUsingRandomUuidsForHlsStreams, resolution,
idSuffix, url, playlistUrl);
} else {
// A video stream
addNewVideoStream(stream, isInstanceUsingRandomUuidsForHlsStreams, resolution,
idSuffix, url, playlistUrl);
}
}
} catch (final Exception e) {
throw new ParsingException("Could not get streams from array", e);
}
}
@Nonnull
private String getHlsPlaylistUrlFromFragmentedFileUrl(
@Nonnull final JsonObject streamJsonObject,
@Nonnull final String idSuffix,
@Nonnull final String format,
@Nonnull final String url) throws ParsingException {
final String streamUrl;
if (FILE_DOWNLOAD_URL.equals(idSuffix)) {
streamUrl = JsonUtils.getString(streamJsonObject, FILE_URL);
} else {
streamUrl = url;
}
return streamUrl.replace("-fragmented." + format, ".m3u8");
}
@Nonnull
private String getHlsPlaylistUrlFromMasterPlaylist(@Nonnull final JsonObject streamJsonObject,
@Nonnull final String playlistUrl)
throws ParsingException {
return playlistUrl.replace("master", JsonUtils.getNumber(streamJsonObject,
RESOLUTION_ID).toString());
}
private void addNewAudioStream(@Nonnull final JsonObject streamJsonObject,
final boolean isInstanceUsingRandomUuidsForHlsStreams,
@Nonnull final String resolution,
@Nonnull final String idSuffix,
@Nonnull final String url,
@Nullable final String playlistUrl) throws ParsingException {
final String extension = url.substring(url.lastIndexOf(".") + 1);
final MediaFormat format = MediaFormat.getFromSuffix(extension);
final String id = resolution + "-" + extension;
// Add progressive HTTP streams first
audioStreams.add(new AudioStream.Builder()
.setId(id + "-" + idSuffix + "-" + DeliveryMethod.PROGRESSIVE_HTTP)
.setContent(url, true)
.setMediaFormat(format)
.setAverageBitrate(UNKNOWN_BITRATE)
.build());
// Then add HLS streams
if (!isNullOrEmpty(playlistUrl)) {
final String hlsStreamUrl;
if (isInstanceUsingRandomUuidsForHlsStreams) {
hlsStreamUrl = getHlsPlaylistUrlFromFragmentedFileUrl(streamJsonObject, idSuffix,
extension, url);
} else {
hlsStreamUrl = getHlsPlaylistUrlFromMasterPlaylist(streamJsonObject, playlistUrl);
}
final AudioStream audioStream = new AudioStream.Builder()
.setId(id + "-" + DeliveryMethod.HLS)
.setContent(hlsStreamUrl, true)
.setDeliveryMethod(DeliveryMethod.HLS)
.setMediaFormat(format)
.setAverageBitrate(UNKNOWN_BITRATE)
.setBaseUrl(playlistUrl)
.build();
if (!Stream.containSimilarStream(audioStream, audioStreams)) {
audioStreams.add(audioStream);
}
}
// Add finally torrent URLs
final String torrentUrl = JsonUtils.getString(streamJsonObject, "torrentUrl");
if (!isNullOrEmpty(torrentUrl)) {
audioStreams.add(new AudioStream.Builder()
.setId(id + "-" + idSuffix + "-" + DeliveryMethod.TORRENT)
.setContent(torrentUrl, true)
.setDeliveryMethod(DeliveryMethod.TORRENT)
.setMediaFormat(format)
.setAverageBitrate(UNKNOWN_BITRATE)
.build());
}
}
private void addNewVideoStream(@Nonnull final JsonObject streamJsonObject,
final boolean isInstanceUsingRandomUuidsForHlsStreams,
@Nonnull final String resolution,
@Nonnull final String idSuffix,
@Nonnull final String url,
@Nullable final String playlistUrl) throws ParsingException {
final String extension = url.substring(url.lastIndexOf(".") + 1);
final MediaFormat format = MediaFormat.getFromSuffix(extension);
final String id = resolution + "-" + extension;
// Add progressive HTTP streams first
videoStreams.add(new VideoStream.Builder()
.setId(id + "-" + idSuffix + "-" + DeliveryMethod.PROGRESSIVE_HTTP)
.setContent(url, true)
.setIsVideoOnly(false)
.setResolution(resolution)
.setMediaFormat(format)
.build());
// Then add HLS streams
if (!isNullOrEmpty(playlistUrl)) {
final String hlsStreamUrl;
if (isInstanceUsingRandomUuidsForHlsStreams) {
hlsStreamUrl = getHlsPlaylistUrlFromFragmentedFileUrl(streamJsonObject, idSuffix,
extension, url);
} else {
hlsStreamUrl = playlistUrl.replace("master", JsonUtils.getNumber(
streamJsonObject, RESOLUTION_ID).toString());
}
final VideoStream videoStream = new VideoStream.Builder()
.setId(id + "-" + DeliveryMethod.HLS)
.setContent(hlsStreamUrl, true)
.setIsVideoOnly(false)
.setDeliveryMethod(DeliveryMethod.HLS)
.setResolution(resolution)
.setMediaFormat(format)
.setBaseUrl(playlistUrl)
.build();
if (!Stream.containSimilarStream(videoStream, videoStreams)) {
videoStreams.add(videoStream);
}
}
// Add finally torrent URLs
final String torrentUrl = JsonUtils.getString(streamJsonObject, "torrentUrl");
if (!isNullOrEmpty(torrentUrl)) {
videoStreams.add(new VideoStream.Builder()
.setId(id + "-" + idSuffix + "-" + DeliveryMethod.TORRENT)
.setContent(torrentUrl, true)
.setIsVideoOnly(false)
.setDeliveryMethod(DeliveryMethod.TORRENT)
.setResolution(resolution)
.setMediaFormat(format)
.build());
}
}
@Nonnull
@Override
public String getName() throws ParsingException {
@ -448,7 +673,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getHost() throws ParsingException {
return JsonUtils.getString(json, "account.host");
return JsonUtils.getString(json, ACCOUNT_HOST);
}
@Nonnull

136
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java
View File

@ -1,6 +1,9 @@
package org.schabi.newpipe.extractor.services.soundcloud.extractors;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.clientId;
import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE;
import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
@ -26,6 +29,7 @@ import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
@ -58,13 +62,16 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
final String policy = track.getString("policy", EMPTY_STRING);
if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) {
isAvailable = false;
if (policy.equals("SNIP")) {
throw new SoundCloudGoPlusContentException();
}
if (policy.equals("BLOCK")) {
throw new GeographicRestrictionException(
"This track is not available in user's country");
}
throw new ContentNotAvailableException("Content not available: policy " + policy);
}
}
@ -72,7 +79,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getId() {
return track.getInt("id") + EMPTY_STRING;
return String.valueOf(track.getInt("id"));
}
@Nonnull
@ -162,17 +169,19 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
// Streams can be streamable and downloadable - or explicitly not.
// For playing the track, it is only necessary to have a streamable track.
// If this is not the case, this track might not be published yet.
// If audio streams were calculated, return the calculated result
if (!track.getBoolean("streamable") || !isAvailable) {
return audioStreams;
}
try {
final JsonArray transcodings = track.getObject("media").getArray("transcodings");
if (transcodings != null) {
if (!isNullOrEmpty(transcodings)) {
// Get information about what stream formats are available
extractAudioStreams(transcodings, checkMp3ProgressivePresence(transcodings),
audioStreams);
}
extractDownloadableFileIfAvailable(audioStreams);
} catch (final NullPointerException e) {
throw new ExtractionException("Could not get SoundCloud's tracks audio URL", e);
}
@ -180,7 +189,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
return audioStreams;
}
private static boolean checkMp3ProgressivePresence(final JsonArray transcodings) {
private static boolean checkMp3ProgressivePresence(@Nonnull final JsonArray transcodings) {
boolean presence = false;
for (final Object transcoding : transcodings) {
final JsonObject transcodingJsonObject = (JsonObject) transcoding;
@ -195,48 +204,68 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
}
@Nonnull
private static String getTranscodingUrl(final String endpointUrl,
final String protocol)
private String getTranscodingUrl(final String endpointUrl,
final String protocol)
throws IOException, ExtractionException {
final Downloader downloader = NewPipe.getDownloader();
final String apiStreamUrl = endpointUrl + "?client_id="
+ SoundcloudParsingHelper.clientId();
+ clientId();
final String response = downloader.get(apiStreamUrl).responseBody();
final JsonObject urlObject;
try {
urlObject = JsonParser.object().from(response);
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse streamable url", e);
throw new ParsingException("Could not parse streamable URL", e);
}
final String urlString = urlObject.getString("url");
if (protocol.equals("progressive")) {
return urlString;
} else if (protocol.equals("hls")) {
try {
return getSingleUrlFromHlsManifest(urlString);
} catch (final ParsingException ignored) {
}
return getSingleUrlFromHlsManifest(urlString);
}
// else, unknown protocol
return "";
return EMPTY_STRING;
}
private static void extractAudioStreams(final JsonArray transcodings,
final boolean mp3ProgressiveInStreams,
final List<AudioStream> audioStreams) {
@Nullable
private String getDownloadUrl(@Nonnull final String trackId)
throws IOException, ExtractionException {
final Downloader dl = NewPipe.getDownloader();
final JsonObject downloadJsonObject;
final String response = dl.get(SOUNDCLOUD_API_V2_URL + "tracks/" + trackId
+ "/download" + "?client_id=" + clientId()).responseBody();
try {
downloadJsonObject = JsonParser.object().from(response);
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse download URL", e);
}
final String redirectUri = downloadJsonObject.getString("redirectUri");
if (!isNullOrEmpty(redirectUri)) {
return redirectUri;
}
return null;
}
private void extractAudioStreams(@Nonnull final JsonArray transcodings,
final boolean mp3ProgressiveInStreams,
final List<AudioStream> audioStreams) {
for (final Object transcoding : transcodings) {
final JsonObject transcodingJsonObject = (JsonObject) transcoding;
final String url = transcodingJsonObject.getString("url");
if (isNullOrEmpty(url)) {
continue;
}
final String mediaUrl;
final String preset = transcodingJsonObject.getString("preset");
final String preset = transcodingJsonObject.getString("preset", ID_UNKNOWN);
final String protocol = transcodingJsonObject.getObject("format")
.getString("protocol");
MediaFormat mediaFormat = null;
int bitrate = 0;
int averageBitrate = UNKNOWN_BITRATE;
if (preset.contains("mp3")) {
// Don't add the MP3 HLS stream if there is a progressive stream present
// because the two have the same bitrate
@ -244,36 +273,75 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
continue;
}
mediaFormat = MediaFormat.MP3;
bitrate = 128;
averageBitrate = 128;
} else if (preset.contains("opus")) {
mediaFormat = MediaFormat.OPUS;
bitrate = 64;
averageBitrate = 64;
}
if (mediaFormat != null) {
try {
mediaUrl = getTranscodingUrl(url, protocol);
if (!mediaUrl.isEmpty()) {
audioStreams.add(new AudioStream(mediaUrl, mediaFormat, bitrate));
try {
mediaUrl = getTranscodingUrl(url, protocol);
if (!mediaUrl.isEmpty()) {
final AudioStream audioStream = new AudioStream.Builder()
.setId(preset)
.setContent(mediaUrl, true)
.setMediaFormat(mediaFormat)
.setAverageBitrate(averageBitrate)
.build();
if (!Stream.containSimilarStream(audioStream, audioStreams)) {
audioStreams.add(audioStream);
}
} catch (final Exception ignored) {
// something went wrong when parsing this transcoding, don't add it to
// audioStreams
}
} catch (final Exception ignored) {
// Something went wrong when parsing this transcoding, don't add it to the
// audioStreams
}
}
}
/**
* Add the downloadable format if it is available.
*
* <p>
* A track can have the {@code downloadable} boolean set to {@code true}, but it doesn't mean
* we can download it: if the value of the {@code has_download_left} boolean is true, the track
* can be downloaded; otherwise not.
* </p>
*
* @param audioStreams the audio streams to which add the downloadable file
*/
public void extractDownloadableFileIfAvailable(final List<AudioStream> audioStreams) {
if (track.getBoolean("downloadable") && track.getBoolean("has_downloads_left")) {
try {
final String downloadUrl = getDownloadUrl(getId());
if (!isNullOrEmpty(downloadUrl)) {
audioStreams.add(new AudioStream.Builder()
.setId("original-format")
.setContent(downloadUrl, true)
.setAverageBitrate(UNKNOWN_BITRATE)
.build());
}
} catch (final Exception ignored) {
// If something went wrong when trying to get the download URL, ignore the
// exception throw because this "stream" is not necessary to play the track
}
}
}
/**
* Parses a SoundCloud HLS manifest to get a single URL of HLS streams.
*
* <p>
* This method downloads the provided manifest URL, find all web occurrences in the manifest,
* get the last segment URL, changes its segment range to {@code 0/track-length} and return
* this string.
* </p>
*
* @param hlsManifestUrl the URL of the manifest to be parsed
* @return a single URL that contains a range equal to the length of the track
*/
private static String getSingleUrlFromHlsManifest(final String hlsManifestUrl)
@Nonnull
private static String getSingleUrlFromHlsManifest(@Nonnull final String hlsManifestUrl)
throws ParsingException {
final Downloader dl = NewPipe.getDownloader();
final String hlsManifestResponse;
@ -326,7 +394,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final String apiUrl = SOUNDCLOUD_API_V2_URL + "tracks/" + urlEncode(getId())
+ "/related?client_id=" + urlEncode(SoundcloudParsingHelper.clientId());
+ "/related?client_id=" + urlEncode(clientId());
SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
return collector;
@ -355,19 +423,19 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
// Tags are separated by spaces, but they can be multiple words escaped by quotes "
final String[] tagList = track.getString("tag_list").split(" ");
final List<String> tags = new ArrayList<>();
String escapedTag = "";
final StringBuilder escapedTag = new StringBuilder();
boolean isEscaped = false;
for (final String tag : tagList) {
if (tag.startsWith("\"")) {
escapedTag += tag.replace("\"", "");
escapedTag.append(tag.replace("\"", ""));
isEscaped = true;
} else if (isEscaped) {
if (tag.endsWith("\"")) {
escapedTag += " " + tag.replace("\"", "");
escapedTag.append(" ").append(tag.replace("\"", ""));
isEscaped = false;
tags.add(escapedTag);
tags.add(escapedTag.toString());
} else {
escapedTag += " " + tag;
escapedTag.append(" ").append(tag);
}
} else if (!tag.isEmpty()) {
tags.add(tag);