From ba967f1a1520f7fadded3b6bfd67c4da0f188c2b Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Fri, 3 Jan 2020 13:13:42 +0100 Subject: [PATCH] Workaround enourmous load times for long bandcamp playlists Additionally, get cover art from json instead of html --- .../extractors/BandcampPlaylistExtractor.java | 42 +++++++++++++------ .../extractors/BandcampStreamExtractor.java | 9 ++-- .../BandcampPlaylistExtractorTest.java | 16 +++++++ .../bandcamp/BandcampRadioExtractorTest.java | 1 - 4 files changed, 50 insertions(+), 18 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java index 1886d4a92..64f26b2e9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java @@ -18,11 +18,19 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import javax.annotation.Nonnull; import java.io.IOException; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getJSONFromJavaScriptVariables; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor.getAlbumInfoJson; public class BandcampPlaylistExtractor extends PlaylistExtractor { + /** + * An arbitrarily chosen number above which cover arts won't be fetched individually for each track; + * instead, it will be assumed that every track has the same cover art as the album, which is not + * always the case. + */ + private static final int MAXIMUM_INDIVIDUAL_COVER_ARTS = 10; + private Document document; private JSONObject albumJson; private JSONArray trackInfo; @@ -57,11 +65,8 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { @Override public String getThumbnailUrl() throws ParsingException { - try { - return document.getElementsByAttributeValue("property", "og:image").get(0).attr("content"); - } catch (NullPointerException e) { - return ""; - } + if (albumJson.isNull("art_id")) return ""; + else return getImageUrl(albumJson.getLong("art_id"), true); } @Override @@ -104,13 +109,26 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor { for (int i = 0; i < trackInfo.length(); i++) { JSONObject track = trackInfo.getJSONObject(i); - collector.commit(new BandcampStreamInfoItemExtractor( - track.getString("title"), - getUploaderUrl() + track.getString("title_link"), - "", - track.getLong("duration"), - getService() - )); + if (trackInfo.length() < MAXIMUM_INDIVIDUAL_COVER_ARTS) { + // Load cover art of every track individually + collector.commit(new BandcampStreamInfoItemExtractor( + track.getString("title"), + getUploaderUrl() + track.getString("title_link"), + "", + track.getLong("duration"), + getService() + )); + } else { + // Pretend every track has the same cover art as the album + collector.commit(new BandcampStreamInfoItemExtractor( + track.getString("title"), + getUploaderUrl() + track.getString("title_link"), + getThumbnailUrl(), + "", + track.getLong("duration") + )); + } + } return new InfoItemsPage<>(collector, null); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index 72c3a551c..c76cb474b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl; + public class BandcampStreamExtractor extends StreamExtractor { private JSONObject albumJson; @@ -103,11 +105,8 @@ public class BandcampStreamExtractor extends StreamExtractor { @Nonnull @Override public String getThumbnailUrl() throws ParsingException { - try { - return document.getElementsByAttributeValue("property", "og:image").get(0).attr("content"); - } catch (NullPointerException e) { - return ""; - } + if (albumJson.isNull("art_id")) return ""; + else return getImageUrl(albumJson.getLong("art_id"), true); } @Nonnull diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java index 7400b1821..7d930ca8f 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java @@ -52,6 +52,22 @@ public class BandcampPlaylistExtractorTest { assertNotEquals(extractor.getThumbnailUrl(), l.get(5).getThumbnailUrl()); } + /** + * Tests that no attempt to load every track's cover individually is made + */ + @Test(timeout = 10000L) + public void testDifferentTrackCoversDuration() throws ExtractionException, IOException { + PlaylistExtractor extractor = bandcamp.getPlaylistExtractor("https://infiniteammo.bandcamp.com/album/night-in-the-woods-vol-1-at-the-end-of-everything"); + extractor.fetchPage(); + + /* All tracks in this album have the same cover art, but I don't know any albums with more than 10 tracks + * that has at least one track with a cover art different from the rest. + */ + List l = extractor.getInitialPage().getItems(); + assertEquals(extractor.getThumbnailUrl(), l.get(0).getThumbnailUrl()); + assertEquals(extractor.getThumbnailUrl(), l.get(5).getThumbnailUrl()); + } + /** * Test playlists with locked content */ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java index 4ec3f66a9..03ea7e3b1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java @@ -34,7 +34,6 @@ public class BandcampRadioExtractorTest { @Test public void testRadioCount() throws ExtractionException, IOException { List list = bandcamp.getKioskList().getExtractorById("Radio", null).getInitialPage().getItems(); - System.out.println(list.size()); assertTrue(list.size() > 300); } }