From 5f39bc4e9907e9af013bb7c5f241942ecb06221d Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 14 Mar 2020 12:41:44 +0100 Subject: [PATCH 01/21] [SoundCloud] Update hardcoded client id --- .../extractor/services/soundcloud/SoundcloudParsingHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index ed50ea544..b8f6af246 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -31,7 +31,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudParsingHelper { - private static final String HARDCODED_CLIENT_ID = "cZQKaMjH39KNADF4y2aeFtVqNSpgoKVj"; // Updated on 08/02/20 + private static final String HARDCODED_CLIENT_ID = "t0h1jzYMsaZXy6ggnZO71gHK3Ms6CFwE"; // Updated on 14/03/20 private static String clientId; private SoundcloudParsingHelper() { From dc20f2aa0cb674918a3ddfbf654ebde55a969d62 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 16 Mar 2020 17:27:26 +0100 Subject: [PATCH 02/21] version 0.18.7 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 977594438..8e9045652 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ allprojects { sourceCompatibility = 1.7 targetCompatibility = 1.7 - version 'v0.18.6' + version 'v0.18.7' group 'com.github.TeamNewPipe' repositories { From efad0be99ffaeff71900bb206a1216ece7c0a14e Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 16 Mar 2020 19:42:35 +0100 Subject: [PATCH 03/21] Test if the extractor can get AudiStreams with the hard-coded cliend_id SoundCloud client_id are changed on a regular basis now. We might want to remove the hardcoded id completely on a later point of time when client_id is changed to often --- .../services/soundcloud/SoundcloudParsingHelper.java | 10 +++++----- .../soundcloud/SoundcloudParsingHelperTest.java | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index b8f6af246..11cd609c6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -42,7 +42,7 @@ public class SoundcloudParsingHelper { Downloader dl = NewPipe.getDownloader(); clientId = HARDCODED_CLIENT_ID; - if (checkIfHardcodedClientIdIsValid(dl)) { + if (checkIfHardcodedClientIdIsValid()) { return clientId; } @@ -73,11 +73,11 @@ public class SoundcloudParsingHelper { throw new ExtractionException("Couldn't extract client id"); } - static boolean checkIfHardcodedClientIdIsValid(Downloader dl) { - final String apiUrl = "https://api.soundcloud.com/connect?client_id=" + HARDCODED_CLIENT_ID; + static boolean checkIfHardcodedClientIdIsValid() { try { - // Should return 200 to indicate that the client id is valid, a 401 is returned otherwise. - return dl.head(apiUrl).responseCode() == 200; + SoundcloudStreamExtractor e = (SoundcloudStreamExtractor) SoundCloud + .getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon"); + return e.getAudioStreams().size() >= 1; } catch (Exception ignored) { // No need to throw an exception here. If something went wrong, the client_id is wrong return false; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java index 16089aab1..bbbcb8618 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java @@ -17,7 +17,7 @@ public class SoundcloudParsingHelperTest { @Test public void assertThatHardcodedClientIdIsValid() throws Exception { assertTrue("Hardcoded client id is not valid anymore", - SoundcloudParsingHelper.checkIfHardcodedClientIdIsValid(DownloaderTestImpl.getInstance())); + SoundcloudParsingHelper.checkIfHardcodedClientIdIsValid()); } @Test From edb57840d72c919c032b21e54e359d22a7f5b0ef Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 16 Mar 2020 19:43:36 +0100 Subject: [PATCH 04/21] [SoundCloud] Update hard-coded client_id --- .../extractor/services/soundcloud/SoundcloudParsingHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 11cd609c6..977d01070 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -31,7 +31,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudParsingHelper { - private static final String HARDCODED_CLIENT_ID = "t0h1jzYMsaZXy6ggnZO71gHK3Ms6CFwE"; // Updated on 14/03/20 + private static final String HARDCODED_CLIENT_ID = "Uz4aPhG7GAl1VYGOnvOPW1wQ0M6xKtA9"; // Updated on 16/03/20 private static String clientId; private SoundcloudParsingHelper() { From 9eca7df947cd10eb1bf68c20d55a4aa04b0cc85b Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 16 Mar 2020 20:50:08 +0100 Subject: [PATCH 05/21] Forget to fetch the page --- .../extractor/services/soundcloud/SoundcloudParsingHelper.java | 1 + 1 file changed, 1 insertion(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 977d01070..6362f13cb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -77,6 +77,7 @@ public class SoundcloudParsingHelper { try { SoundcloudStreamExtractor e = (SoundcloudStreamExtractor) SoundCloud .getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon"); + e.fetchPage(); return e.getAudioStreams().size() >= 1; } catch (Exception ignored) { // No need to throw an exception here. If something went wrong, the client_id is wrong From f3095713f9cc158a092eb24d3338c0c8fafd4a8d Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 15:12:13 +0100 Subject: [PATCH 06/21] [SoundCloud] Use api-v2 in PlaylistExtractor Rewrote methods to calculate next page url and to get items from it. `api-v2` is different from `api` since the initial playlist page contains (usually) the full info of the first 3 streams and only the id of the other. Then the single tracks can be requested in batch using `/tracks?ids=id1,id2,...`. --- .../SoundcloudPlaylistExtractor.java | 77 +++++++++++++++---- 1 file changed, 64 insertions(+), 13 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index ca08f66ce..363a223a2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -1,8 +1,11 @@ package org.schabi.newpipe.extractor.services.soundcloud; +import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + +import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; @@ -13,15 +16,23 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; +import javax.annotation.Nullable; + import java.io.IOException; +import java.util.ArrayList; +import java.util.List; @SuppressWarnings("WeakerAccess") public class SoundcloudPlaylistExtractor extends PlaylistExtractor { + private static final int streamsPerRequestedPage = 15; + private String playlistId; private JsonObject playlist; - private StreamInfoItemsCollector streamInfoItemsCollector = null; - private String nextPageUrl = null; + private StreamInfoItemsCollector streamInfoItemsCollector; + private List nextTrackIds; + private int nextTrackIdsIndex; + private String nextPageUrl; public SoundcloudPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { super(service, linkHandler); @@ -31,7 +42,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { playlistId = getLinkHandler().getId(); - String apiUrl = "https://api.soundcloud.com/playlists/" + playlistId + + String apiUrl = "https://api-v2.soundcloud.com/playlists/" + playlistId + "?client_id=" + SoundcloudParsingHelper.clientId() + "&representation=compact"; @@ -110,27 +121,55 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { if (streamInfoItemsCollector == null) { - computeStreamsAndNextPageUrl(); + computeInitialTracksAndNextIds(); } return new InfoItemsPage<>(streamInfoItemsCollector, getNextPageUrl()); } - private void computeStreamsAndNextPageUrl() throws ExtractionException, IOException { + private void computeInitialTracksAndNextIds() { streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId()); + nextTrackIds = new ArrayList<>(); + nextTrackIdsIndex = 0; - // Note the "api", NOT "api-v2" - String apiUrl = "https://api.soundcloud.com/playlists/" + getId() + "/tracks" - + "?client_id=" + SoundcloudParsingHelper.clientId() - + "&limit=20" - + "&linked_partitioning=1"; + JsonArray tracks = playlist.getArray("tracks"); + for (Object o : tracks) { + if (o instanceof JsonObject) { + JsonObject track = (JsonObject) o; + if (track.has("title")) { // i.e. if full info is available + streamInfoItemsCollector.commit(new SoundcloudStreamInfoItemExtractor(track)); + } else { + nextTrackIds.add(track.getInt("id")); + } + } + } + } - nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl); + private void computeAnotherNextPageUrl() throws IOException, ExtractionException { + if (nextTrackIdsIndex >= nextTrackIds.size()) { + nextPageUrl = ""; // there are no more tracks + return; + } + + StringBuilder urlBuilder = new StringBuilder("https://api-v2.soundcloud.com/tracks?client_id="); + urlBuilder.append(SoundcloudParsingHelper.clientId()); + urlBuilder.append("&ids="); + + int upperIndex = Math.min(nextTrackIdsIndex + streamsPerRequestedPage, nextTrackIds.size()); + for (int i = nextTrackIdsIndex; i < upperIndex; ++i) { + urlBuilder.append(nextTrackIds.get(i)); + urlBuilder.append(","); // a , at the end is ok + } + + nextPageUrl = urlBuilder.toString(); } @Override public String getNextPageUrl() throws IOException, ExtractionException { if (nextPageUrl == null) { - computeStreamsAndNextPageUrl(); + if (nextTrackIds == null) { + computeInitialTracksAndNextIds(); + } + computeAnotherNextPageUrl(); } return nextPageUrl; } @@ -142,8 +181,20 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { } StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl); + String response = NewPipe.getDownloader().get(pageUrl, getExtractorLocalization()).responseBody(); + try { + JsonArray tracks = JsonParser.array().from(response); + for (Object track : tracks) { + if (track instanceof JsonObject) { + collector.commit(new SoundcloudStreamInfoItemExtractor((JsonObject) track)); + } + } + } catch (JsonParserException e) { + throw new ParsingException("Could not parse json response", e); + } + + computeAnotherNextPageUrl(); return new InfoItemsPage<>(collector, nextPageUrl); } } From d0e66cc600ad43a48e17c65e29e897f4b549a3d0 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 15:13:28 +0100 Subject: [PATCH 07/21] [SoundCloud] Improve thumbnail url extraction in playlists Prevent NullPointerExceptions and remove duplicate code --- .../soundcloud/SoundcloudPlaylistExtractor.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index 363a223a2..f503ac0fe 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -66,6 +66,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { return playlist.getString("title"); } + @Nullable @Override public String getThumbnailUrl() { String artworkUrl = playlist.getString("artwork_url"); @@ -75,21 +76,20 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { // if it also fails, return null try { final InfoItemsPage infoItems = getInitialPage(); - if (infoItems.getItems().isEmpty()) return null; for (StreamInfoItem item : infoItems.getItems()) { - final String thumbnailUrl = item.getThumbnailUrl(); - if (thumbnailUrl == null || thumbnailUrl.isEmpty()) continue; - - String thumbnailUrlBetterResolution = thumbnailUrl.replace("large.jpg", "crop.jpg"); - return thumbnailUrlBetterResolution; + artworkUrl = item.getThumbnailUrl(); + if (artworkUrl != null && !artworkUrl.isEmpty()) break; } } catch (Exception ignored) { } + + if (artworkUrl == null) { + return null; + } } - String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg"); - return artworkUrlBetterResolution; + return artworkUrl.replace("large.jpg", "crop.jpg"); } @Override From c3d811fde5ced6518381173b0d4c0e73a13d0b08 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 15:22:25 +0100 Subject: [PATCH 08/21] [SoundCloud] Use api-v2 in SubscriptionExtractor Also added --- .../services/soundcloud/SoundcloudSubscriptionExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java index f65bf98b7..cf01b0c93 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java @@ -36,7 +36,7 @@ public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor { throw new InvalidSourceException(e); } - String apiUrl = "https://api.soundcloud.com/users/" + id + "/followings" + String apiUrl = "https://api-v2.soundcloud.com/users/" + id + "/followings" + "?client_id=" + SoundcloudParsingHelper.clientId() + "&limit=200"; ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(service.getServiceId()); From 4b1121aac7d48e84f97048028aa6005f2acc1d5f Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 15:23:13 +0100 Subject: [PATCH 09/21] [SoundCloud] Add tests for api-v2 channel urls --- .../services/soundcloud/SoundcloudParsingHelperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java index bbbcb8618..130b0de30 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java @@ -24,6 +24,8 @@ public class SoundcloudParsingHelperTest { public void resolveUrlWithEmbedPlayerTest() throws Exception { Assert.assertEquals("https://soundcloud.com/trapcity", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/26057743")); Assert.assertEquals("https://soundcloud.com/nocopyrightsounds", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/16069159")); + Assert.assertEquals("https://soundcloud.com/trapcity", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api-v2.soundcloud.com/users/26057743")); + Assert.assertEquals("https://soundcloud.com/nocopyrightsounds", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api-v2.soundcloud.com/users/16069159")); } @Test From d4aa4a076375af22256fb622547bab71a3eb4a3e Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 15:25:11 +0100 Subject: [PATCH 10/21] [SoundCloud] Fix typo in ChartsExtractor --- .../services/soundcloud/SoundcloudChartsExtractor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java index b9f4894d3..a08502d70 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -44,7 +44,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor { } - private void computNextPageAndStreams() throws IOException, ExtractionException { + private void computeNextPageAndStreams() throws IOException, ExtractionException { collector = new StreamInfoItemsCollector(getServiceId()); String apiUrl = "https://api-v2.soundcloud.com/charts" + @@ -69,7 +69,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor { @Override public String getNextPageUrl() throws IOException, ExtractionException { if (nextPageUrl == null) { - computNextPageAndStreams(); + computeNextPageAndStreams(); } return nextPageUrl; } @@ -78,7 +78,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor { @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { if (collector == null) { - computNextPageAndStreams(); + computeNextPageAndStreams(); } return new InfoItemsPage<>(collector, getNextPageUrl()); } From ca8bf53b61131a8849789e76156f490d12a15751 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 15:35:33 +0100 Subject: [PATCH 11/21] [SoundCloud] Fix playlist test: number of streams changed --- .../services/soundcloud/SoundcloudPlaylistExtractorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index b93c2cfb9..067633bbe 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -317,7 +317,7 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testStreamCount() { - assertTrue("Error in the streams count", extractor.getStreamCount() >= 3900); + assertTrue("Stream count does not fit: " + extractor.getStreamCount(), extractor.getStreamCount() >= 370); } } } From 1558da6f6b2aa4613c6af45f302104b1464d28a0 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 15:41:16 +0100 Subject: [PATCH 12/21] [SoundCloud] Fix playlist next page generation --- .../services/soundcloud/SoundcloudPlaylistExtractor.java | 3 ++- .../soundcloud/SoundcloudPlaylistExtractorTest.java | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index f503ac0fe..a57abea8b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -145,7 +145,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { } private void computeAnotherNextPageUrl() throws IOException, ExtractionException { - if (nextTrackIdsIndex >= nextTrackIds.size()) { + if (nextTrackIds == null || nextTrackIdsIndex >= nextTrackIds.size()) { nextPageUrl = ""; // there are no more tracks return; } @@ -160,6 +160,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { urlBuilder.append(","); // a , at the end is ok } + nextTrackIdsIndex = upperIndex; nextPageUrl = urlBuilder.toString(); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index 067633bbe..f0832ef28 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -7,10 +7,13 @@ import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import java.io.IOException; + import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; @@ -70,7 +73,9 @@ public class SoundcloudPlaylistExtractorTest { } @Test - public void testMoreRelatedItems() { + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor); + try { defaultTestMoreItems(extractor); } catch (Throwable ignored) { From 65bdb3bc9dd45c8d8fd198064497e9473be6e16f Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 15:49:58 +0100 Subject: [PATCH 13/21] [SoundCloud] Unignore ignored playlist tests --- .../services/soundcloud/SoundcloudPlaylistExtractorTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index f0832ef28..d4ab882fa 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -269,14 +269,11 @@ public class SoundcloudPlaylistExtractorTest { // ListExtractor //////////////////////////////////////////////////////////////////////////*/ - @Ignore @Test public void testRelatedItems() throws Exception { defaultTestRelatedItems(extractor); } - //TODO: FUCK THIS: This triggers a 500 at sever - @Ignore @Test public void testMoreRelatedItems() throws Exception { ListExtractor.InfoItemsPage currentPage = defaultTestMoreItems(extractor); @@ -291,7 +288,6 @@ public class SoundcloudPlaylistExtractorTest { // PlaylistExtractor //////////////////////////////////////////////////////////////////////////*/ - @Ignore @Test public void testThumbnailUrl() { assertIsSecureUrl(extractor.getThumbnailUrl()); From 0e1b4bbf17d460b081b409c35d276d4ba0eed15f Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 15:53:25 +0100 Subject: [PATCH 14/21] [SoundCloud] Test playlists banner: it should not exist --- .../soundcloud/SoundcloudPlaylistExtractorTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index d4ab882fa..6f1af547f 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -94,10 +94,10 @@ public class SoundcloudPlaylistExtractorTest { assertIsSecureUrl(extractor.getThumbnailUrl()); } - @Ignore @Test public void testBannerUrl() { - assertIsSecureUrl(extractor.getBannerUrl()); + // SoundCloud playlists do not have a banner + assertNull(extractor.getBannerUrl()); } @Test @@ -186,10 +186,10 @@ public class SoundcloudPlaylistExtractorTest { assertIsSecureUrl(extractor.getThumbnailUrl()); } - @Ignore("not implemented") @Test public void testBannerUrl() { - assertIsSecureUrl(extractor.getBannerUrl()); + // SoundCloud playlists do not have a banner + assertNull(extractor.getBannerUrl()); } @Test @@ -293,10 +293,10 @@ public class SoundcloudPlaylistExtractorTest { assertIsSecureUrl(extractor.getThumbnailUrl()); } - @Ignore @Test public void testBannerUrl() { - assertIsSecureUrl(extractor.getBannerUrl()); + // SoundCloud playlists do not have a banner + assertNull(extractor.getBannerUrl()); } @Test From 5e4ddb368fe25b809e28428119b032f93b4ce28d Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 18:04:40 +0100 Subject: [PATCH 15/21] [SoundCloud] Fix extractors built from next playlist pages They didn't have the information to calculate another next page url. So now `nextPageUrl` contains a full link with all video ids, and `getPage` takes the first part of the url (containing 15 streams) and produces another `nextPageUrl` with the remaining streams. Also add a test for this. --- .../SoundcloudPlaylistExtractor.java | 62 +++++++++---------- .../SoundcloudPlaylistExtractorTest.java | 9 +++ 2 files changed, 39 insertions(+), 32 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index a57abea8b..c82180773 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -30,8 +30,6 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { private JsonObject playlist; private StreamInfoItemsCollector streamInfoItemsCollector; - private List nextTrackIds; - private int nextTrackIdsIndex; private String nextPageUrl; public SoundcloudPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { @@ -121,15 +119,16 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { if (streamInfoItemsCollector == null) { - computeInitialTracksAndNextIds(); + computeInitialTracksAndNextPageUrl(); } - return new InfoItemsPage<>(streamInfoItemsCollector, getNextPageUrl()); + return new InfoItemsPage<>(streamInfoItemsCollector, nextPageUrl); } - private void computeInitialTracksAndNextIds() { + private void computeInitialTracksAndNextPageUrl() throws IOException, ExtractionException { streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId()); - nextTrackIds = new ArrayList<>(); - nextTrackIdsIndex = 0; + StringBuilder nextPageUrlBuilder = new StringBuilder("https://api-v2.soundcloud.com/tracks?client_id="); + nextPageUrlBuilder.append(SoundcloudParsingHelper.clientId()); + nextPageUrlBuilder.append("&ids="); JsonArray tracks = playlist.getArray("tracks"); for (Object o : tracks) { @@ -138,39 +137,23 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { if (track.has("title")) { // i.e. if full info is available streamInfoItemsCollector.commit(new SoundcloudStreamInfoItemExtractor(track)); } else { - nextTrackIds.add(track.getInt("id")); + // %09d would be enough, but a 0 before the number does not create problems, so let's be sure + nextPageUrlBuilder.append(String.format("%010d,", track.getInt("id"))); } } } - } - private void computeAnotherNextPageUrl() throws IOException, ExtractionException { - if (nextTrackIds == null || nextTrackIdsIndex >= nextTrackIds.size()) { - nextPageUrl = ""; // there are no more tracks - return; + nextPageUrl = nextPageUrlBuilder.toString(); + if (nextPageUrl.endsWith("&ids=")) { + // there are no other videos + nextPageUrl = ""; } - - StringBuilder urlBuilder = new StringBuilder("https://api-v2.soundcloud.com/tracks?client_id="); - urlBuilder.append(SoundcloudParsingHelper.clientId()); - urlBuilder.append("&ids="); - - int upperIndex = Math.min(nextTrackIdsIndex + streamsPerRequestedPage, nextTrackIds.size()); - for (int i = nextTrackIdsIndex; i < upperIndex; ++i) { - urlBuilder.append(nextTrackIds.get(i)); - urlBuilder.append(","); // a , at the end is ok - } - - nextTrackIdsIndex = upperIndex; - nextPageUrl = urlBuilder.toString(); } @Override public String getNextPageUrl() throws IOException, ExtractionException { if (nextPageUrl == null) { - if (nextTrackIds == null) { - computeInitialTracksAndNextIds(); - } - computeAnotherNextPageUrl(); + computeInitialTracksAndNextPageUrl(); } return nextPageUrl; } @@ -181,8 +164,24 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { throw new ExtractionException(new IllegalArgumentException("Page url is empty or null")); } + // see computeInitialTracksAndNextPageUrl + final int lengthFirstPartOfUrl = ("https://api-v2.soundcloud.com/tracks?client_id=" + + SoundcloudParsingHelper.clientId() + + "&ids=").length(); + final int lengthOfEveryStream = 11; + + String currentPageUrl; + int lengthMaxStreams = lengthFirstPartOfUrl + lengthOfEveryStream * streamsPerRequestedPage; + if (pageUrl.length() <= lengthMaxStreams) { + currentPageUrl = pageUrl; // fetch every remaining video, there are less than the max + nextPageUrl = ""; // afterwards the list is complete + } else { + currentPageUrl = pageUrl.substring(0, lengthMaxStreams); + nextPageUrl = pageUrl.substring(0, lengthFirstPartOfUrl) + pageUrl.substring(lengthMaxStreams); + } + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); - String response = NewPipe.getDownloader().get(pageUrl, getExtractorLocalization()).responseBody(); + String response = NewPipe.getDownloader().get(currentPageUrl, getExtractorLocalization()).responseBody(); try { JsonArray tracks = JsonParser.array().from(response); @@ -195,7 +194,6 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { throw new ParsingException("Could not parse json response", e); } - computeAnotherNextPageUrl(); return new InfoItemsPage<>(collector, nextPageUrl); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index 6f1af547f..700d2e6dc 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -234,6 +234,15 @@ public class SoundcloudPlaylistExtractorTest { public void testGetPageInNewExtractor() throws Exception { final PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getUrl()); defaultTestGetPageInNewExtractor(extractor, newExtractor); + String page1 = newExtractor.getNextPageUrl(); + defaultTestMoreItems(newExtractor); // there has to be another page + String page2 = newExtractor.getNextPageUrl(); + defaultTestMoreItems(newExtractor); // and another one + String page3 = newExtractor.getNextPageUrl(); + + assertNotEquals("Same pages", page1, page2); + assertNotEquals("Same pages", page2, page3); + assertNotEquals("Same pages", page3, page1); } /*////////////////////////////////////////////////////////////////////////// From 4389fd3b7b37608be495a17f93e63cf1b6feba8b Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 18:06:13 +0100 Subject: [PATCH 16/21] [SoundCloud] Migrate StreamExtractor to api-v2 --- .../services/soundcloud/SoundcloudParsingHelper.java | 2 +- .../services/soundcloud/SoundcloudStreamExtractor.java | 8 +++++--- .../soundcloud/SoundcloudStreamExtractorDefaultTest.java | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 6362f13cb..a957ed810 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -108,7 +108,7 @@ public class SoundcloudParsingHelper { * See https://developers.soundcloud.com/docs/api/reference#resolve */ public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException { - String apiUrl = "https://api.soundcloud.com/resolve" + String apiUrl = "https://api-v2.soundcloud.com/resolve" + "?url=" + URLEncoder.encode(url, "UTF-8") + "&client_id=" + clientId(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 2919c89be..805db216b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -19,6 +19,8 @@ import javax.annotation.Nonnull; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.text.SimpleDateFormat; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -55,14 +57,14 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Nonnull @Override - public String getTextualUploadDate() { - return track.getString("created_at"); + public String getTextualUploadDate() throws ParsingException { + return track.getString("created_at").replace("T"," ").replace("Z", ""); } @Nonnull @Override public DateWrapper getUploadDate() throws ParsingException { - return new DateWrapper(SoundcloudParsingHelper.parseDate(getTextualUploadDate())); + return new DateWrapper(SoundcloudParsingHelper.parseDate(track.getString("created_at"))); } @Nonnull diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java index 0ffe06223..86203f4c3 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java @@ -74,7 +74,7 @@ public class SoundcloudStreamExtractorDefaultTest { @Test public void testGetTextualUploadDate() throws ParsingException { - Assert.assertEquals("2016/07/31 18:18:07 +0000", extractor.getTextualUploadDate()); + Assert.assertEquals("2016-07-31 18:18:07", extractor.getTextualUploadDate()); } @Test From ae47c9587c186013bb278d9aaafd0a9a1d8c0946 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 20:31:01 +0100 Subject: [PATCH 17/21] [SoundCloud] Optimize imports in edited files --- .../services/soundcloud/SoundcloudPlaylistExtractor.java | 6 ++---- .../services/soundcloud/SoundcloudStreamExtractor.java | 6 +++--- .../soundcloud/SoundcloudPlaylistExtractorTest.java | 4 ---- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index c82180773..a121e4db0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -15,13 +15,11 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import java.io.IOException; + import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - @SuppressWarnings("WeakerAccess") public class SoundcloudPlaylistExtractor extends PlaylistExtractor { private static final int streamsPerRequestedPage = 15; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 805db216b..5c176720c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -4,6 +4,7 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; + import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; @@ -15,17 +16,16 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.*; -import javax.annotation.Nonnull; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; -import java.text.SimpleDateFormat; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Locale; +import javax.annotation.Nonnull; + public class SoundcloudStreamExtractor extends StreamExtractor { private JsonObject track; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index 700d2e6dc..28634207f 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -2,18 +2,14 @@ package org.schabi.newpipe.extractor.services.soundcloud; import org.hamcrest.CoreMatchers; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import java.io.IOException; - import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; From c505d4e2b77b12e4d2d259121d66937ded756de1 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 20:36:59 +0100 Subject: [PATCH 18/21] [SoundCloud] Remove trailing , in playlist page urls --- .../services/soundcloud/SoundcloudPlaylistExtractor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index a121e4db0..051bfc006 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -141,8 +141,9 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { } } + nextPageUrlBuilder.setLength(nextPageUrlBuilder.length() - 1); // remove trailing , nextPageUrl = nextPageUrlBuilder.toString(); - if (nextPageUrl.endsWith("&ids=")) { + if (nextPageUrl.endsWith("&ids")) { // there are no other videos nextPageUrl = ""; } From 45bb646480b13c59d5c6f9aef53b257c51fbf0d5 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 20:56:47 +0100 Subject: [PATCH 19/21] [SoundCloud] Do not overwrite nextPageUrl in PlaylistExtractor Consistent with YouTube and the documentation --- .../services/soundcloud/SoundcloudPlaylistExtractor.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index 051bfc006..060b45caf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -169,14 +169,14 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { + "&ids=").length(); final int lengthOfEveryStream = 11; - String currentPageUrl; + String currentPageUrl, nextUrl; int lengthMaxStreams = lengthFirstPartOfUrl + lengthOfEveryStream * streamsPerRequestedPage; if (pageUrl.length() <= lengthMaxStreams) { currentPageUrl = pageUrl; // fetch every remaining video, there are less than the max - nextPageUrl = ""; // afterwards the list is complete + nextUrl = ""; // afterwards the list is complete } else { currentPageUrl = pageUrl.substring(0, lengthMaxStreams); - nextPageUrl = pageUrl.substring(0, lengthFirstPartOfUrl) + pageUrl.substring(lengthMaxStreams); + nextUrl = pageUrl.substring(0, lengthFirstPartOfUrl) + pageUrl.substring(lengthMaxStreams); } StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); @@ -193,6 +193,6 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { throw new ParsingException("Could not parse json response", e); } - return new InfoItemsPage<>(collector, nextPageUrl); + return new InfoItemsPage<>(collector, nextUrl); } } From 265cfb61f37c6a626b56f552e5619c95b6a3a34c Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 17 Mar 2020 21:03:50 +0100 Subject: [PATCH 20/21] [SoundCloud] Add test for small playlist, use defaultTestGetPageInNewExtractor --- .../SoundcloudPlaylistExtractorTest.java | 127 ++++++++++++++---- 1 file changed, 104 insertions(+), 23 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index 28634207f..59ecff492 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -1,6 +1,5 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.hamcrest.CoreMatchers; import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.DownloaderTestImpl; @@ -10,6 +9,7 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import static org.hamcrest.CoreMatchers.*; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; @@ -71,14 +71,6 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testMoreRelatedItems() throws Exception { defaultTestMoreItems(extractor); - - try { - defaultTestMoreItems(extractor); - } catch (Throwable ignored) { - return; - } - - fail("This playlist doesn't have more items, it should throw an error"); } /*////////////////////////////////////////////////////////////////////////// @@ -100,7 +92,7 @@ public class SoundcloudPlaylistExtractorTest { public void testUploaderUrl() { final String uploaderUrl = extractor.getUploaderUrl(); assertIsSecureUrl(uploaderUrl); - assertTrue(uploaderUrl, uploaderUrl.contains("liluzivert")); + assertThat(uploaderUrl, containsString("liluzivert")); } @Test @@ -115,7 +107,7 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testStreamCount() { - assertTrue("Error in the streams count", extractor.getStreamCount() >= 10); + assertTrue("Stream count does not fit: " + extractor.getStreamCount(), extractor.getStreamCount() >= 10); } } @@ -192,7 +184,7 @@ public class SoundcloudPlaylistExtractorTest { public void testUploaderUrl() { final String uploaderUrl = extractor.getUploaderUrl(); assertIsSecureUrl(uploaderUrl); - assertThat(uploaderUrl, CoreMatchers.containsString("micky96")); + assertThat(uploaderUrl, containsString("micky96")); } @Test @@ -207,7 +199,7 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testStreamCount() { - assertTrue("Error in the streams count", extractor.getStreamCount() >= 10); + assertTrue("Stream count does not fit: " + extractor.getStreamCount(), extractor.getStreamCount() >= 10); } } @@ -228,17 +220,8 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testGetPageInNewExtractor() throws Exception { - final PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getUrl()); + PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getUrl()); defaultTestGetPageInNewExtractor(extractor, newExtractor); - String page1 = newExtractor.getNextPageUrl(); - defaultTestMoreItems(newExtractor); // there has to be another page - String page2 = newExtractor.getNextPageUrl(); - defaultTestMoreItems(newExtractor); // and another one - String page3 = newExtractor.getNextPageUrl(); - - assertNotEquals("Same pages", page1, page2); - assertNotEquals("Same pages", page2, page3); - assertNotEquals("Same pages", page3, page1); } /*////////////////////////////////////////////////////////////////////////// @@ -326,4 +309,102 @@ public class SoundcloudPlaylistExtractorTest { assertTrue("Stream count does not fit: " + extractor.getStreamCount(), extractor.getStreamCount() >= 370); } } + + public static class SmallPlaylist implements BasePlaylistExtractorTest { + private static SoundcloudPlaylistExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (SoundcloudPlaylistExtractor) SoundCloud + .getPlaylistExtractor("https://soundcloud.com/breezy-123/sets/empty-playlist?test=123"); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(SoundCloud.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() { + assertEquals("EMPTY PLAYLIST", extractor.getName()); + } + + @Test + public void testId() { + assertEquals("23483459", extractor.getId()); + } + + @Test + public void testUrl() throws Exception { + assertEquals("https://soundcloud.com/breezy-123/sets/empty-playlist", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws Exception { + assertEquals("https://soundcloud.com/breezy-123/sets/empty-playlist?test=123", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor); + } + + @Test + public void testMoreRelatedItems() throws Exception { + try { + defaultTestMoreItems(extractor); + } catch (Throwable ignored) { + return; + } + + fail("This playlist doesn't have more items, it should throw an error"); + } + + /*////////////////////////////////////////////////////////////////////////// + // PlaylistExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testThumbnailUrl() { + assertIsSecureUrl(extractor.getThumbnailUrl()); + } + + @Test + public void testBannerUrl() { + // SoundCloud playlists do not have a banner + assertNull(extractor.getBannerUrl()); + } + + @Test + public void testUploaderUrl() { + final String uploaderUrl = extractor.getUploaderUrl(); + assertIsSecureUrl(uploaderUrl); + assertThat(uploaderUrl, containsString("breezy-123")); + } + + @Test + public void testUploaderName() { + assertEquals("breezy-123", extractor.getUploaderName()); + } + + @Test + public void testUploaderAvatarUrl() { + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + } + + @Test + public void testStreamCount() { + assertEquals(2, extractor.getStreamCount()); + } + } } From 222d659d9e6ebebed5ea9a690518b1a7c1ac0c2d Mon Sep 17 00:00:00 2001 From: wb9688 Date: Wed, 18 Mar 2020 11:01:46 +0100 Subject: [PATCH 21/21] [SoundCloud] Don't make separate request for getAudioStreams() in StreamExtractor Signed-off-by: Stypox --- .../soundcloud/SoundcloudStreamExtractor.java | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 5c176720c..16eaad012 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -14,7 +14,14 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.Description; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.SubtitlesStream; +import org.schabi.newpipe.extractor.stream.VideoStream; import java.io.IOException; import java.io.UnsupportedEncodingException; @@ -148,24 +155,13 @@ public class SoundcloudStreamExtractor extends StreamExtractor { List audioStreams = new ArrayList<>(); Downloader dl = NewPipe.getDownloader(); - String apiUrl = "https://api-v2.soundcloud.com/tracks/" + urlEncode(getId()) - + "?client_id=" + urlEncode(SoundcloudParsingHelper.clientId()); - - String response = dl.get(apiUrl, getExtractorLocalization()).responseBody(); - JsonObject responseObject; - try { - responseObject = JsonParser.object().from(response); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse json response", e); - } - // Streams can be streamable and downloadable - or explicitly not. // For playing the track, it is only necessary to have a streamable track. // If this is not the case, this track might not be published yet. - if (!responseObject.getBoolean("streamable")) return audioStreams; + if (!track.getBoolean("streamable")) return audioStreams; try { - JsonArray transcodings = responseObject.getObject("media").getArray("transcodings"); + JsonArray transcodings = track.getObject("media").getArray("transcodings"); // get information about what stream formats are available for (Object transcoding : transcodings) {