lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Bandcamp: read info item data in place and not in advance

This commit is contained in:
Fynn Godau 2020-05-25 18:51:31 +02:00
parent fff21caa87
commit 692b2d06f4
16 changed files with 303 additions and 226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java
View File

@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.BandcampDiscographStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
@ -89,18 +90,7 @@ public class BandcampChannelExtractor extends ChannelExtractor {
if (!discograph.getString("item_type").equals("track")) continue;
collector.commit(new BandcampStreamInfoItemExtractor(
discograph.getString("title"),
BandcampExtractorHelper.getStreamUrlFromIds(
discograph.getLong("band_id"),
discograph.getLong("item_id"),
discograph.getString("item_type")
),
BandcampExtractorHelper.getImageUrl(
discograph.getLong("art_id"), true
),
discograph.getString("band_name")
));
collector.commit(new BandcampDiscographStreamInfoItemExtractor(discograph, getUrl()));
}
return new InfoItemsPage<>(collector, null);

30
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java
View File

@ -8,43 +8,35 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
public class BandcampChannelInfoItemExtractor implements ChannelInfoItemExtractor {
private String name, url, image, location;
private final Element resultInfo, searchResult;
public BandcampChannelInfoItemExtractor(Element searchResult) {
Element resultInfo = searchResult.getElementsByClass("result-info").first();
Element img = searchResult.getElementsByClass("art").first()
.getElementsByTag("img").first();
if (img != null) {
image = img.attr("src");
}
name = resultInfo.getElementsByClass("heading").text();
location = resultInfo.getElementsByClass("subhead").text();
url = resultInfo.getElementsByClass("itemurl").text();
this.searchResult = searchResult;
resultInfo = searchResult.getElementsByClass("result-info").first();
}
@Override
public String getName() throws ParsingException {
return name;
return resultInfo.getElementsByClass("heading").text();
}
@Override
public String getUrl() throws ParsingException {
return url;
return resultInfo.getElementsByClass("itemurl").text();
}
@Override
public String getThumbnailUrl() throws ParsingException {
return image;
Element img = searchResult.getElementsByClass("art").first()
.getElementsByTag("img").first();
if (img != null) {
return img.attr("src");
} else return null;
}
@Override
public String getDescription() {
return location;
return resultInfo.getElementsByClass("subhead").text();
}
@Override

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java
View File

@ -138,7 +138,7 @@ public class BandcampExtractorHelper {
* @return Url of image with this ID in size 10 which is 1200x1200 (we could also choose size 0
* but we don't want something as large as 3460x3460 here, do we?)
*/
static String getImageUrl(long id, boolean album) {
public static String getImageUrl(long id, boolean album) {
return "https://f4.bcbits.com/img/" + (album ? 'a' : "") + id + "_10.jpg";
}
}

11
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java
View File

@ -14,12 +14,13 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
public class BandcampFeaturedExtractor extends KioskExtractor<InfoItem> {
public class BandcampFeaturedExtractor extends KioskExtractor<PlaylistInfoItem> {
public static final String KIOSK_FEATURED = "Featured";
public static final String FEATURED_API_URL = "https://bandcamp.com/api/mobile/24/bootstrap_data";
@ -41,9 +42,9 @@ public class BandcampFeaturedExtractor extends KioskExtractor<InfoItem> {
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
public InfoItemsPage<PlaylistInfoItem> getInitialPage() throws IOException, ExtractionException {
InfoItemsCollector c = new PlaylistInfoItemsCollector(getServiceId());
PlaylistInfoItemsCollector c = new PlaylistInfoItemsCollector(getServiceId());
try {
@ -66,10 +67,10 @@ public class BandcampFeaturedExtractor extends KioskExtractor<InfoItem> {
continue;
}
c.commit(new BandcampPlaylistInfoItemExtractor(featuredStory));
c.commit(new BandcampPlaylistInfoItemFeaturedExtractor(featuredStory));
}
return new InfoItemsPage<InfoItem>(c, null);
return new InfoItemsPage<>(c, null);
} catch (JsonParserException e) {
e.printStackTrace();
throw new ParsingException("JSON error", e);

19
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java
View File

@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.BandcampPlaylistStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
@ -111,22 +112,12 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor {
if (trackInfo.size() < MAXIMUM_INDIVIDUAL_COVER_ARTS) {
// Load cover art of every track individually
collector.commit(new BandcampStreamInfoItemExtractor(
track.getString("title"),
getUploaderUrl() + track.getString("title_link"),
"",
track.getLong("duration"),
getService()
));
collector.commit(new BandcampPlaylistStreamInfoItemExtractor(
track, getUploaderUrl(), getService()));
} else {
// Pretend every track has the same cover art as the album
collector.commit(new BandcampStreamInfoItemExtractor(
track.getString("title"),
getUploaderUrl() + track.getString("title_link"),
getThumbnailUrl(),
"",
track.getLong("duration")
));
collector.commit(new BandcampPlaylistStreamInfoItemExtractor(
track, getUploaderUrl(), getThumbnailUrl()));
}
}

50
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java
View File

@ -4,64 +4,42 @@ import com.grack.nanojson.JsonObject;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl;
public class BandcampPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
private String title, artist, url, cover;
private int trackCount;
private final Element searchResult, resultInfo;
public BandcampPlaylistInfoItemExtractor(Element searchResult) {
Element resultInfo = searchResult.getElementsByClass("result-info").first();
Element img = searchResult.getElementsByClass("art").first()
.getElementsByTag("img").first();
if (img != null) {
cover = img.attr("src");
}
title = resultInfo.getElementsByClass("heading").text();
url = resultInfo.getElementsByClass("itemurl").text();
artist = resultInfo.getElementsByClass("subhead").text()
.split(" by")[0];
String length = resultInfo.getElementsByClass("length").text();
trackCount = Integer.parseInt(length.split(" track")[0]);
}
public BandcampPlaylistInfoItemExtractor(JsonObject featuredStory) {
title = featuredStory.getString("album_title");
artist = featuredStory.getString("band_name");
url = featuredStory.getString("item_url");
cover = featuredStory.has("art_id") ? getImageUrl(featuredStory.getLong("art_id"), true) : "";
trackCount = featuredStory.getInt("num_streamable_tracks");
this.searchResult = searchResult;
resultInfo = searchResult.getElementsByClass("result-info").first();
}
@Override
public String getUploaderName() {
return artist;
return resultInfo.getElementsByClass("subhead").text()
.split(" by")[0];
}
@Override
public long getStreamCount() {
return trackCount;
String length = resultInfo.getElementsByClass("length").text();
return Integer.parseInt(length.split(" track")[0]);
}
@Override
public String getName() {
return title;
return resultInfo.getElementsByClass("heading").text();
}
@Override
public String getUrl() {
return url;
return resultInfo.getElementsByClass("itemurl").text();
}
@Override
public String getThumbnailUrl() {
return cover;
Element img = searchResult.getElementsByClass("art").first()
.getElementsByTag("img").first();
if (img != null) {
return img.attr("src");
} else return null;
}
}

41
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java Normal file
View File

@ -0,0 +1,41 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl;
public class BandcampPlaylistInfoItemFeaturedExtractor implements PlaylistInfoItemExtractor {
private final JsonObject featuredStory;
public BandcampPlaylistInfoItemFeaturedExtractor(JsonObject featuredStory) {
this.featuredStory = featuredStory;
}
@Override
public String getUploaderName() {
return featuredStory.getString("band_name");
}
@Override
public long getStreamCount() {
return featuredStory.getInt("num_streamable_tracks");
}
@Override
public String getName() {
return featuredStory.getString("album_title");
}
@Override
public String getUrl() {
return featuredStory.getString("item_url");
}
@Override
public String getThumbnailUrl() {
return featuredStory.has("art_id") ? getImageUrl(featuredStory.getLong("art_id"), true) : "";
}
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java
View File

@ -22,7 +22,7 @@ public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor {
@Override
public long getDuration() {
/* Duration is only present in the more detailed information that has to be queried seperately.
/* Duration is only present in the more detailed information that has to be queried separately.
* Because the servers would probably not like over 300 queries every time someone opens the kiosk,
* we're just providing 0 here.
*/

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java
View File

@ -13,6 +13,8 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.BandcampPlaylistStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.BandcampSearchStreamInfoItemExtractor;
import javax.annotation.Nonnull;
import java.io.IOException;
@ -62,7 +64,7 @@ public class BandcampSearchExtractor extends SearchExtractor {
break;
case "TRACK":
collector.commit(new BandcampStreamInfoItemExtractor(searchResult));
collector.commit(new BandcampSearchStreamInfoItemExtractor(searchResult, null));
break;
}

134
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java
View File

@ -1,134 +0,0 @@
// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import javax.annotation.Nullable;
import java.io.IOException;
public class BandcampStreamInfoItemExtractor implements StreamInfoItemExtractor {
private String title;
private String url;
private String cover;
private String artist;
private long duration;
private StreamingService service;
public BandcampStreamInfoItemExtractor(String title, String url, String artist, long duration, StreamingService service) {
this(title, url, null, artist, duration);
this.service = service;
}
public BandcampStreamInfoItemExtractor(String title, String url, String cover, String artist) {
this(title, url, cover, artist, -1);
}
public BandcampStreamInfoItemExtractor(Element searchResult) {
Element resultInfo = searchResult.getElementsByClass("result-info").first();
Element img = searchResult.getElementsByClass("art").first()
.getElementsByTag("img").first();
if (img != null) {
cover = img.attr("src");
}
title = resultInfo.getElementsByClass("heading").text();
url = resultInfo.getElementsByClass("itemurl").text();
String subhead = resultInfo.getElementsByClass("subhead").text();
String[] splitBy = subhead.split(" by");
if (splitBy.length > 1) {
artist = subhead.split(" by")[1];
}
}
public BandcampStreamInfoItemExtractor(String title, String url, String cover, String artist, long duration) {
this.title = title;
this.url = url;
this.cover = cover;
this.artist = artist;
this.duration = duration;
}
@Override
public StreamType getStreamType() {
return StreamType.AUDIO_STREAM;
}
@Override
public long getDuration() {
return duration;
}
@Override
public long getViewCount() {
return -1;
}
@Override
public String getUploaderName() {
return artist;
}
@Override
public String getUploaderUrl() {
return null;
}
@Nullable
@Override
public String getTextualUploadDate() {
return null; // TODO
}
@Nullable
@Override
public DateWrapper getUploadDate() {
return null;
}
@Override
public String getName() throws ParsingException {
return title;
}
@Override
public String getUrl() throws ParsingException {
return url;
}
/**
* There is no guarantee that every track of an album has the same cover art, so it needs to be fetched
* per-track if in playlist view
*/
@Override
public String getThumbnailUrl() throws ParsingException {
if (cover != null) return cover;
else {
try {
StreamExtractor extractor = service.getStreamExtractor(getUrl());
extractor.fetchPage();
return extractor.getThumbnailUrl();
} catch (ExtractionException | IOException e) {
throw new ParsingException("could not download cover art location", e);
}
}
}
/**
* There are no ads just like that, duh
*/
@Override
public boolean isAd() {
return false;
}
}

46
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampDiscographStreamInfoItemExtractor.java Normal file
View File

@ -0,0 +1,46 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper;
public class BandcampDiscographStreamInfoItemExtractor extends BandcampStreamInfoItemExtractor {
private final JsonObject discograph;
public BandcampDiscographStreamInfoItemExtractor(JsonObject discograph, String uploaderUrl) {
super(uploaderUrl);
this.discograph = discograph;
}
@Override
public String getUploaderName() {
return discograph.getString("band_name");
}
@Override
public String getName() {
return discograph.getString("title");
}
@Override
public String getUrl() throws ParsingException {
return BandcampExtractorHelper.getStreamUrlFromIds(
discograph.getLong("band_id"),
discograph.getLong("item_id"),
discograph.getString("item_type")
);
}
@Override
public String getThumbnailUrl() throws ParsingException {
return BandcampExtractorHelper.getImageUrl(
discograph.getLong("art_id"), true
);
}
@Override
public long getDuration() {
return -1;
}
}

69
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java Normal file
View File

@ -0,0 +1,69 @@
// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later
package org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import java.io.IOException;
public class BandcampPlaylistStreamInfoItemExtractor extends BandcampStreamInfoItemExtractor {
private final JsonObject track;
private String substituteCoverUrl;
private StreamingService service;
public BandcampPlaylistStreamInfoItemExtractor(JsonObject track, String uploaderUrl, StreamingService service) {
super(uploaderUrl);
this.track = track;
this.service = service;
}
public BandcampPlaylistStreamInfoItemExtractor(JsonObject track, String uploaderUrl, String substituteCoverUrl) {
this(track, uploaderUrl, (StreamingService) null);
this.substituteCoverUrl = substituteCoverUrl;
}
@Override
public String getName() {
return track.getString("title");
}
@Override
public String getUrl() {
return getUploaderUrl() + track.getString("title_link");
}
@Override
public long getDuration() {
return track.getLong("duration");
}
@Override
public String getUploaderName() {
return "";
}
/**
* Each track can have its own cover art. Therefore, unless a substitute is provided,
* the thumbnail is extracted using a stream extractor.
*/
@Override
public String getThumbnailUrl() throws ParsingException {
if (substituteCoverUrl != null) {
return substituteCoverUrl;
} else {
try {
StreamExtractor extractor = service.getStreamExtractor(getUrl());
extractor.fetchPage();
return extractor.getThumbnailUrl();
} catch (ExtractionException | IOException e) {
throw new ParsingException("could not download cover art location", e);
}
}
}
}

48
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java Normal file
View File

@ -0,0 +1,48 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
public class BandcampSearchStreamInfoItemExtractor extends BandcampStreamInfoItemExtractor {
private final Element resultInfo, searchResult;
public BandcampSearchStreamInfoItemExtractor(Element searchResult, String uploaderUrl) {
super(uploaderUrl);
this.searchResult = searchResult;
resultInfo = searchResult.getElementsByClass("result-info").first();
}
@Override
public String getUploaderName() throws ParsingException {
String subhead = resultInfo.getElementsByClass("subhead").text();
String[] splitBy = subhead.split(" by");
if (splitBy.length > 1) {
return splitBy[1];
} else throw new ParsingException("Uploader name was not found as expected");
}
@Override
public String getName() throws ParsingException {
return resultInfo.getElementsByClass("heading").text();
}
@Override
public String getUrl() throws ParsingException {
return resultInfo.getElementsByClass("itemurl").text();
}
@Override
public String getThumbnailUrl() throws ParsingException {
Element img = searchResult.getElementsByClass("art").first()
.getElementsByTag("img").first();
if (img != null) {
return img.attr("src");
} else return null;
}
@Override
public long getDuration() {
return -1;
}
}

54
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampStreamInfoItemExtractor.java Normal file
View File

@ -0,0 +1,54 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import javax.annotation.Nullable;
/**
* Implements methods that return a constant value for better readability in
* subclasses.
*/
public abstract class BandcampStreamInfoItemExtractor implements StreamInfoItemExtractor {
private final String uploaderUrl;
public BandcampStreamInfoItemExtractor(String uploaderUrl) {
this.uploaderUrl = uploaderUrl;
}
@Override
public StreamType getStreamType() {
return StreamType.AUDIO_STREAM;
}
@Override
public long getViewCount() {
return -1;
}
@Override
public String getUploaderUrl() {
return uploaderUrl;
}
@Nullable
@Override
public String getTextualUploadDate() {
return null;
}
@Nullable
@Override
public DateWrapper getUploadDate() {
return null;
}
/**
* There are no ads just like that, duh
*/
@Override
public boolean isAd() {
return false;
}
}

3
extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java
View File

@ -8,6 +8,7 @@ import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor;
import java.io.IOException;
@ -32,7 +33,7 @@ public class BandcampFeaturedExtractorTest {
@Test
public void testFeaturedCount() throws ExtractionException, IOException {
List<InfoItem> list = extractor.getInitialPage().getItems();
List<PlaylistInfoItem> list = extractor.getInitialPage().getItems();
assertTrue(list.size() > 1);
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java
View File

@ -11,8 +11,6 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampPlaylistExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import java.io.IOException;