[SoundCloud] Use api-v2 in PlaylistExtractor

Rewrote methods to calculate next page url and to get items from it. `api-v2` is different from `api` since the initial playlist page contains (usually) the full info of the first 3 streams and only the id of the other. Then the single tracks can be requested in batch using `/tracks?ids=id1,id2,...`.
This commit is contained in:
Stypox 2020-03-17 15:12:13 +01:00
parent 9eca7df947
commit f3095713f9
No known key found for this signature in database
GPG Key ID: 4BDF1B40A49FDD23
1 changed files with 64 additions and 13 deletions

View File

@ -1,8 +1,11 @@
package org.schabi.newpipe.extractor.services.soundcloud; package org.schabi.newpipe.extractor.services.soundcloud;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -13,15 +16,23 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@SuppressWarnings("WeakerAccess") @SuppressWarnings("WeakerAccess")
public class SoundcloudPlaylistExtractor extends PlaylistExtractor { public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
private static final int streamsPerRequestedPage = 15;
private String playlistId; private String playlistId;
private JsonObject playlist; private JsonObject playlist;
private StreamInfoItemsCollector streamInfoItemsCollector = null; private StreamInfoItemsCollector streamInfoItemsCollector;
private String nextPageUrl = null; private List<Integer> nextTrackIds;
private int nextTrackIdsIndex;
private String nextPageUrl;
public SoundcloudPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { public SoundcloudPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler); super(service, linkHandler);
@ -31,7 +42,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
playlistId = getLinkHandler().getId(); playlistId = getLinkHandler().getId();
String apiUrl = "https://api.soundcloud.com/playlists/" + playlistId + String apiUrl = "https://api-v2.soundcloud.com/playlists/" + playlistId +
"?client_id=" + SoundcloudParsingHelper.clientId() + "?client_id=" + SoundcloudParsingHelper.clientId() +
"&representation=compact"; "&representation=compact";
@ -110,27 +121,55 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
@Override @Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException { public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
if (streamInfoItemsCollector == null) { if (streamInfoItemsCollector == null) {
computeStreamsAndNextPageUrl(); computeInitialTracksAndNextIds();
} }
return new InfoItemsPage<>(streamInfoItemsCollector, getNextPageUrl()); return new InfoItemsPage<>(streamInfoItemsCollector, getNextPageUrl());
} }
private void computeStreamsAndNextPageUrl() throws ExtractionException, IOException { private void computeInitialTracksAndNextIds() {
streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId()); streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId());
nextTrackIds = new ArrayList<>();
nextTrackIdsIndex = 0;
// Note the "api", NOT "api-v2" JsonArray tracks = playlist.getArray("tracks");
String apiUrl = "https://api.soundcloud.com/playlists/" + getId() + "/tracks" for (Object o : tracks) {
+ "?client_id=" + SoundcloudParsingHelper.clientId() if (o instanceof JsonObject) {
+ "&limit=20" JsonObject track = (JsonObject) o;
+ "&linked_partitioning=1"; if (track.has("title")) { // i.e. if full info is available
streamInfoItemsCollector.commit(new SoundcloudStreamInfoItemExtractor(track));
} else {
nextTrackIds.add(track.getInt("id"));
}
}
}
}
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl); private void computeAnotherNextPageUrl() throws IOException, ExtractionException {
if (nextTrackIdsIndex >= nextTrackIds.size()) {
nextPageUrl = ""; // there are no more tracks
return;
}
StringBuilder urlBuilder = new StringBuilder("https://api-v2.soundcloud.com/tracks?client_id=");
urlBuilder.append(SoundcloudParsingHelper.clientId());
urlBuilder.append("&ids=");
int upperIndex = Math.min(nextTrackIdsIndex + streamsPerRequestedPage, nextTrackIds.size());
for (int i = nextTrackIdsIndex; i < upperIndex; ++i) {
urlBuilder.append(nextTrackIds.get(i));
urlBuilder.append(","); // a , at the end is ok
}
nextPageUrl = urlBuilder.toString();
} }
@Override @Override
public String getNextPageUrl() throws IOException, ExtractionException { public String getNextPageUrl() throws IOException, ExtractionException {
if (nextPageUrl == null) { if (nextPageUrl == null) {
computeStreamsAndNextPageUrl(); if (nextTrackIds == null) {
computeInitialTracksAndNextIds();
}
computeAnotherNextPageUrl();
} }
return nextPageUrl; return nextPageUrl;
} }
@ -142,8 +181,20 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
} }
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl); String response = NewPipe.getDownloader().get(pageUrl, getExtractorLocalization()).responseBody();
try {
JsonArray tracks = JsonParser.array().from(response);
for (Object track : tracks) {
if (track instanceof JsonObject) {
collector.commit(new SoundcloudStreamInfoItemExtractor((JsonObject) track));
}
}
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
computeAnotherNextPageUrl();
return new InfoItemsPage<>(collector, nextPageUrl); return new InfoItemsPage<>(collector, nextPageUrl);
} }
} }