lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add bandcamp weekly kiosk 

Unfortunately a little messy
This commit is contained in:
Fynn Godau 2019-12-22 17:30:22 +01:00
parent 13ef11e9c8
commit 808c8aa087
12 changed files with 477 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java
View File

@ -21,8 +21,10 @@ import java.io.IOException;
import java.util.Collections;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor.FEATURED_API_URL;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor.KIOSK_FEATURED;
import static org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampFeaturedLinkHandlerFactory.FEATURED_API_URL;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor.KIOSK_RADIO;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor.RADIO_API_URL;
public class BandcampService extends StreamingService {
@ -79,14 +81,22 @@ public class BandcampService extends StreamingService {
public KioskList getKioskList() throws ExtractionException {
KioskList kioskList = new KioskList(this);
try {
kioskList
.addKioskEntry(new KioskList.KioskExtractorFactory() {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String kioskId) throws ExtractionException {
return new BandcampFeaturedExtractor(BandcampService.this, new BandcampFeaturedLinkHandlerFactory().fromUrl(FEATURED_API_URL), kioskId);
}
}, new BandcampFeaturedLinkHandlerFactory(), KIOSK_FEATURED);
kioskList.addKioskEntry(new KioskList.KioskExtractorFactory() {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String kioskId) throws ExtractionException {
return new BandcampFeaturedExtractor(BandcampService.this, new BandcampFeaturedLinkHandlerFactory().fromUrl(FEATURED_API_URL), kioskId);
}
}, new BandcampFeaturedLinkHandlerFactory(), KIOSK_FEATURED);
kioskList.addKioskEntry(new KioskList.KioskExtractorFactory() {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService, String url, String kioskId) throws ExtractionException {
return new BandcampRadioExtractor(BandcampService.this, new BandcampFeaturedLinkHandlerFactory().fromUrl(RADIO_API_URL), kioskId);
}
}, new BandcampFeaturedLinkHandlerFactory(), KIOSK_RADIO);
kioskList.setDefaultKiosk(KIOSK_FEATURED);
@ -109,7 +119,10 @@ public class BandcampService extends StreamingService {
@Override
public StreamExtractor getStreamExtractor(LinkHandler linkHandler) {
return new BandcampStreamExtractor(this, linkHandler);
if (linkHandler.getUrl().matches("https?://bandcamp\\.com/\\?show=\\d+"))
return new BandcampRadioStreamExtractor(this, linkHandler);
else
return new BandcampStreamExtractor(this, linkHandler);
}
@Override

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java
View File

@ -22,11 +22,11 @@ import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl;
import static org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampFeaturedLinkHandlerFactory.FEATURED_API_URL;
public class BandcampFeaturedExtractor extends KioskExtractor<InfoItem> {
public static final String KIOSK_FEATURED = "Featured";
public static final String FEATURED_API_URL = "https://bandcamp.com/api/mobile/24/bootstrap_data";
public BandcampFeaturedExtractor(StreamingService streamingService, ListLinkHandler listLinkHandler, String kioskId) {
super(streamingService, listLinkHandler, kioskId);
@ -40,7 +40,7 @@ public class BandcampFeaturedExtractor extends KioskExtractor<InfoItem> {
@Nonnull
@Override
public String getName() throws ParsingException {
return null;
return KIOSK_FEATURED;
}
@Nonnull

74
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java Normal file
View File

@ -0,0 +1,74 @@
// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import org.json.JSONArray;
import org.json.JSONObject;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
public class BandcampRadioExtractor extends KioskExtractor<InfoItem> {
public static final String KIOSK_RADIO = "Radio";
public static final String RADIO_API_URL = "https://bandcamp.com/api/bcweekly/1/list";
public BandcampRadioExtractor(StreamingService streamingService, ListLinkHandler linkHandler, String kioskId) {
super(streamingService, linkHandler, kioskId);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
}
@Nonnull
@Override
public String getName() throws ParsingException {
return KIOSK_RADIO;
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
InfoItemsCollector c = new StreamInfoItemsCollector(getServiceId());
JSONObject json = new JSONObject(
getDownloader().get(
RADIO_API_URL
).responseBody()
);
JSONArray radioShows = json.getJSONArray("results");
for (int i = 0; i < radioShows.length(); i++) {
JSONObject radioShow = radioShows.getJSONObject(i);
c.commit(
new BandcampRadioInfoItemExtractor(radioShow)
);
}
return new InfoItemsPage<InfoItem>(c, null);
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
return null;
}
@Override
public InfoItemsPage<InfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
return null;
}
}

88
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java Normal file
View File

@ -0,0 +1,88 @@
// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import org.json.JSONObject;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import javax.annotation.Nullable;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioStreamExtractor.query;
public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor {
private JSONObject show;
public BandcampRadioInfoItemExtractor(JSONObject radioShow) {
show = radioShow;
}
@Override
public long getDuration() {
/* Duration is only present in the more detailed information that has to be queried seperately.
* Because the servers would probably not like over 300 queries every time someone opens the kiosk,
* we're just providing 0 here.
*/
//return query(show.getInt("id")).getLong("audio_duration");
return 0;
}
@Nullable
@Override
public String getTextualUploadDate() {
return show.getString("date");
}
@Nullable
@Override
public DateWrapper getUploadDate() {
return null;
}
@Override
public String getName() throws ParsingException {
return show.getString("date");
}
@Override
public String getUrl() {
return "https://bandcamp.com/?show=" + show.getInt("id");
}
@Override
public String getThumbnailUrl() {
return getImageUrl(show.getLong("image_id"), false);
}
@Override
public StreamType getStreamType() {
return StreamType.AUDIO_STREAM;
}
@Override
public long getViewCount() {
return -1;
}
@Override
public String getUploaderName() {
return "";
}
@Override
public String getUploaderUrl() {
return "";
}
@Override
public boolean isAd() {
return false;
}
}

125
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java Normal file
View File

@ -0,0 +1,125 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.stream.AudioStream;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampChannelExtractor.getImageUrl;
public class BandcampRadioStreamExtractor extends BandcampStreamExtractor {
private JSONObject showInfo;
private LinkHandler linkHandler;
public BandcampRadioStreamExtractor(StreamingService service, LinkHandler linkHandler) {
super(service, linkHandler);
this.linkHandler = linkHandler;
}
static JSONObject query(int id) throws ParsingException {
try {
return new JSONObject(
NewPipe.getDownloader().get("https://bandcamp.com/api/bcweekly/1/get?id=" + id).responseBody()
);
} catch (IOException | ReCaptchaException e) {
throw new ParsingException("could not get show data", e);
}
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
showInfo = query(Integer.parseInt(getId()));
}
@Nonnull
@Override
public String getName() throws ParsingException {
return showInfo.getString("subtitle"); // "audio_title" is a boring title
}
@Nonnull
@Override
public String getUploaderUrl() {
return Jsoup.parse(showInfo.getString("image_caption"))
.getElementsByTag("a").first().attr("href");
}
@Nonnull
@Override
public String getUrl() throws ParsingException {
return linkHandler.getUrl();
}
@Nonnull
@Override
public String getUploaderName() {
return Jsoup.parse(showInfo.getString("image_caption"))
.getElementsByTag("a").first().text();
}
@Nullable
@Override
public String getTextualUploadDate() {
return showInfo.getString("published_date");
}
@Nonnull
@Override
public String getThumbnailUrl() throws ParsingException {
return getImageUrl(showInfo.getLong("show_image_id"), false);
}
@Nonnull
@Override
public String getUploaderAvatarUrl() {
return "https://bandcamp.com/img/buttons/bandcamp-button-circle-whitecolor-512.png";
}
@Nonnull
@Override
public String getDescription() {
return showInfo.getString("desc");
}
@Override
public long getLength() {
return showInfo.getLong("audio_duration");
}
@Override
public List<AudioStream> getAudioStreams() {
ArrayList<AudioStream> list = new ArrayList<>();
JSONObject streams = showInfo.getJSONObject("audio_stream");
if (streams.has("opus-lo")) {
list.add(new AudioStream(
streams.getString("opus-lo"),
MediaFormat.OPUS, 100
));
}
if (streams.has("mp3-128")) {
list.add(new AudioStream(
streams.getString("mp3-128"),
MediaFormat.MP3, 128
));
}
return list;
}
}

20
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampFeaturedLinkHandlerFactory.java
View File

@ -2,33 +2,35 @@
package org.schabi.newpipe.extractor.services.bandcamp.linkHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import java.util.List;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor.FEATURED_API_URL;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor.KIOSK_FEATURED;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor.KIOSK_RADIO;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor.RADIO_API_URL;
public class BandcampFeaturedLinkHandlerFactory extends ListLinkHandlerFactory {
/**
* This kiosk doesn't have a corresponding website
*/
public static final String FEATURED_API_URL = "https://bandcamp.com/api/mobile/24/bootstrap_data";
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) {
if (id.equals(KIOSK_FEATURED)) return FEATURED_API_URL;
if (id.equals(KIOSK_FEATURED)) return FEATURED_API_URL; // doesn't have a website
else if (id.equals(KIOSK_RADIO)) return RADIO_API_URL; // doesn't have its own website
else return null;
}
@Override
public String getId(String url) {
return KIOSK_FEATURED;
if (url.matches("https?://bandcamp\\.com/\\?show=\\d+") || url.equals(RADIO_API_URL))
return KIOSK_RADIO;
else if (url.equals(FEATURED_API_URL))
return KIOSK_FEATURED;
else return null;
}
@Override
public boolean onAcceptUrl(String url) {
return url.equals(FEATURED_API_URL);
return url.equals(FEATURED_API_URL) || (url.equals(RADIO_API_URL) || url.matches("https?://bandcamp\\.com/\\?show=\\d+"));
}
}

21
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java
View File

@ -8,6 +8,8 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
/**
* Tracks don't have standalone ids, they are always in combination with the band id.
* That's why id = url. Instead, URLs are cleaned up so that they always look the same.
* <br/><br/>
* Radio (bandcamp weekly) shows do have ids.
*/
public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory {
@ -17,7 +19,10 @@ public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory {
*/
@Override
public String getId(String url) throws ParsingException {
return getUrl(url);
if (url.matches("https?://bandcamp\\.com/\\?show=\\d+")) {
return url.split("bandcamp.com/\\?show=")[1];
} else
return getUrl(url);
}
/**
@ -25,11 +30,13 @@ public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory {
* @see BandcampStreamLinkHandlerFactory
*/
@Override
public String getUrl(String url) {
if (url.endsWith("/"))
url = url.substring(0, url.length() - 1);
url = url.replace("http://", "https://").toLowerCase();
return url;
public String getUrl(String input) {
if (input.matches("\\d+"))
return "https://bandcamp.com/?show=" + input;
if (input.endsWith("/"))
input = input.substring(0, input.length() - 1);
input = input.replace("http://", "https://").toLowerCase();
return input;
}
/**
@ -42,6 +49,6 @@ public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory {
*/
@Override
public boolean onAcceptUrl(String url) {
return getUrl(url).matches("https?://.+\\..+/track/.+");
return getUrl(url).matches("https?://.+\\..+/track/.+") || getUrl(url).matches("https?://bandcamp\\.com/\\?show=\\d+");
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java
View File

@ -31,7 +31,7 @@ public class BandcampFeaturedExtractorTest {
}
@Test
public void testKioskItems() throws ExtractionException, IOException {
public void testFeaturedCount() throws ExtractionException, IOException {
List<InfoItem> list = extractor.getInitialPage().getItems();
assertTrue(list.size() > 1);
}

52
extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedLinkHandlerFactoryTest.java Normal file
View File

@ -0,0 +1,52 @@
// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later
package org.schabi.newpipe.extractor.services.bandcamp;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampFeaturedLinkHandlerFactory;
import static org.junit.Assert.*;
/**
* Tests for {@link BandcampFeaturedLinkHandlerFactory}
*/
public class BandcampFeaturedLinkHandlerFactoryTest {
private static BandcampFeaturedLinkHandlerFactory linkHandler;
@BeforeClass
public static void setUp() {
linkHandler = new BandcampFeaturedLinkHandlerFactory();
}
@Test
public void testAcceptUrl() throws ParsingException {
// Tests expecting true
assertTrue(linkHandler.acceptUrl("https://bandcamp.com/?show=1"));
assertTrue(linkHandler.acceptUrl("http://bandcamp.com/?show=2"));
assertTrue(linkHandler.acceptUrl("https://bandcamp.com/api/mobile/24/bootstrap_data"));
assertTrue(linkHandler.acceptUrl("https://bandcamp.com/api/bcweekly/1/list"));
// Tests expecting false
assertFalse(linkHandler.acceptUrl("https://bandcamp.com/?show="));
assertFalse(linkHandler.acceptUrl("https://bandcamp.com/?show=a"));
assertFalse(linkHandler.acceptUrl("https://bandcamp.com/"));
}
@Test
public void testGetUrl() throws ParsingException {
assertEquals("https://bandcamp.com/api/mobile/24/bootstrap_data", linkHandler.getUrl("Featured"));
assertEquals("https://bandcamp.com/api/bcweekly/1/list", linkHandler.getUrl("Radio"));
}
@Test
public void testGetId() {
assertEquals("Featured", linkHandler.getId("https://bandcamp.com/api/mobile/24/bootstrap_data"));
assertEquals("Radio", linkHandler.getId("http://bandcamp.com/?show=1"));
assertEquals("Radio", linkHandler.getId("https://bandcamp.com/api/bcweekly/1/list"));
}
}

40
extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioExtractorTest.java Normal file
View File

@ -0,0 +1,40 @@
// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later
package org.schabi.newpipe.extractor.services.bandcamp;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioExtractor;
import java.io.IOException;
import java.util.List;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.bandcamp;
/**
* Tests for {@link BandcampRadioExtractor}
*/
public class BandcampRadioExtractorTest {
private static BandcampRadioExtractor extractor;
@BeforeClass
public static void setUp() throws ExtractionException, IOException {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (BandcampRadioExtractor) bandcamp
.getKioskList()
.getExtractorById("Radio", null);
}
@Test
public void testRadioCount() throws ExtractionException, IOException {
List<InfoItem> list = bandcamp.getKioskList().getExtractorById("Radio", null).getInitialPage().getItems();
System.out.println(list.size());
assertTrue(list.size() > 300);
}
}

37
extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java Normal file
View File

@ -0,0 +1,37 @@
package org.schabi.newpipe.extractor.services.bandcamp;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.Extractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampRadioStreamExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor;
import java.io.IOException;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.bandcamp;
public class BandcampRadioStreamExtractorTest {
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test
public void testGettingCorrectStreamExtractor() throws ExtractionException {
assertTrue(bandcamp.getStreamExtractor("https://bandcamp.com/?show=3") instanceof BandcampRadioStreamExtractor);
assertFalse(bandcamp.getStreamExtractor("https://zachbenson.bandcamp.com/track/deflated") instanceof BandcampRadioStreamExtractor);
}
@Test
public void testExtracting() throws ExtractionException, IOException {
BandcampRadioStreamExtractor e = (BandcampRadioStreamExtractor) bandcamp.getStreamExtractor("https://bandcamp.com/?show=230");
e.fetchPage();
assertEquals("Sound Movements", e.getName());
assertEquals("Andrew Jervis", e.getUploaderName());
}
}

11
extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java
View File

@ -30,6 +30,16 @@ public class BandcampStreamLinkHandlerFactoryTest {
assertEquals("https://zachbenson.bandcamp.com/track/u-i-tonite", linkHandler.getUrl("http://ZachBenson.Bandcamp.COM/Track/U-I-Tonite/"));
}
@Test
public void testGetRadioUrl() {
assertEquals("https://bandcamp.com/?show=1", linkHandler.getUrl("1"));
}
@Test
public void testGetRadioId() throws ParsingException {
assertEquals("2", linkHandler.getId("https://bandcamp.com/?show=2"));
}
@Test
public void testAcceptUrl() throws ParsingException {
// Tests expecting false
@ -44,5 +54,6 @@ public class BandcampStreamLinkHandlerFactoryTest {
assertTrue(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/track/kitchen"));
assertTrue(linkHandler.acceptUrl("http://ZachBenson.Bandcamp.COM/Track/U-I-Tonite/"));
assertTrue(linkHandler.acceptUrl("https://interovgm.com/track/title"));
assertTrue(linkHandler.acceptUrl("http://bandcamP.com/?show=38"));
}
}