lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #620 from TeamNewPipe/bandcamp-comments 

Add Bandcamp comments (first page)
This commit is contained in:
Tobi 2021-05-02 13:43:53 +02:00 committed by GitHub
parent 284362fc15 85a4270bd9
commit 3a3ade20f4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 280 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java
View File

@ -16,9 +16,11 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.util.Arrays;
import java.util.Collections;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor.FEATURED_API_URL;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampFeaturedExtractor.KIOSK_FEATURED;
@ -28,7 +30,7 @@ import static org.schabi.newpipe.extractor.services.bandcamp.extractors.Bandcamp
public class BandcampService extends StreamingService {
public BandcampService(final int id) {
super(id, "Bandcamp", Collections.singletonList(AUDIO));
super(id, "Bandcamp", Arrays.asList(AUDIO, COMMENTS));
}
@Override
@ -58,7 +60,7 @@ public class BandcampService extends StreamingService {
@Override
public ListLinkHandlerFactory getCommentsLHFactory() {
return null;
return new BandcampCommentsLinkHandlerFactory();
}
@Override
@ -122,6 +124,6 @@ public class BandcampService extends StreamingService {
@Override
public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler) {
return null;
return new BandcampCommentsExtractor(this, linkHandler);
}
}

53
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampCommentsExtractor.java Normal file
View File

@ -0,0 +1,53 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemsCollector;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import javax.annotation.Nonnull;
import java.io.IOException;
public class BandcampCommentsExtractor extends CommentsExtractor {
private Document document;
public BandcampCommentsExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
String html = downloader.get(getLinkHandler().getUrl()).responseBody();
document = Jsoup.parse(html);
}
@Nonnull
@Override
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws IOException, ExtractionException {
CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
Elements writings = document.getElementsByClass("writing");
for (Element writing : writings) {
collector.commit(new BandcampCommentsInfoItemExtractor(writing, getUrl()));
}
return new InfoItemsPage<>(collector, null);
}
@Override
public InfoItemsPage<CommentsInfoItem> getPage(Page page) throws IOException, ExtractionException {
return null;
}
}

92
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampCommentsInfoItemExtractor.java Normal file
View File

@ -0,0 +1,92 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import javax.annotation.Nullable;
public class BandcampCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
private final Element writing;
private final String url;
public BandcampCommentsInfoItemExtractor(Element writing, String url) {
this.writing = writing;
this.url = url;
}
@Override
public String getName() throws ParsingException {
return writing.getElementsByClass("text").first().ownText();
}
@Override
public String getUrl() {
return url;
}
@Override
public String getThumbnailUrl() throws ParsingException {
return writing.getElementsByClass("thumb").attr("src");
}
@Override
public int getLikeCount() {
return -1;
}
@Override
public String getCommentText() {
return writing.getElementsByClass("text").first().ownText();
}
@Override
public String getTextualUploadDate() {
return "";
}
@Nullable
@Override
public DateWrapper getUploadDate() {
return null;
}
@Override
public String getCommentId() {
return "";
}
@Override
public String getUploaderUrl() {
//return writing.getElementsByClass("name").attr("href");
// Fan links cannot be opened
return "";
}
@Override
public String getUploaderName() throws ParsingException {
return writing.getElementsByClass("name").first().text();
}
@Override
public String getUploaderAvatarUrl() {
return writing.getElementsByClass("thumb").attr("src");
}
@Override
public boolean isHeartedByUploader() {
return false;
}
@Override
public boolean isPinned() {
return false;
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return false;
}
}

33
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampCommentsLinkHandlerFactory.java Normal file
View File

@ -0,0 +1,33 @@
package org.schabi.newpipe.extractor.services.bandcamp.linkHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper;
import java.util.List;
/**
* Like in {@link BandcampStreamLinkHandlerFactory}, tracks have no meaningful IDs except for
* their URLs
*/
public class BandcampCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
@Override
public String getId(String url) throws ParsingException {
return url;
}
@Override
public boolean onAcceptUrl(String url) throws ParsingException {
// Don't accept URLs that don't point to a track
if (!url.toLowerCase().matches("https?://.+\\..+/(track|album)/.+")) return false;
// Test whether domain is supported
return BandcampExtractorHelper.isSupportedDomain(url);
}
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException {
return id;
}
}

52
extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampCommentsExtractorTest.java Normal file
View File

@ -0,0 +1,52 @@
package org.schabi.newpipe.extractor.services.bandcamp;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.services.DefaultTests;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import static org.junit.Assert.*;
import static org.junit.Assert.assertFalse;
import static org.schabi.newpipe.extractor.ServiceList.Bandcamp;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class BandcampCommentsExtractorTest {
private static CommentsExtractor extractor;
@BeforeClass
public static void setUp() throws ExtractionException, IOException {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = Bandcamp.getCommentsExtractor("https://floatingpoints.bandcamp.com/album/promises");
extractor.fetchPage();
}
@Test
public void hasComments() throws IOException, ExtractionException {
assertTrue(extractor.getInitialPage().getItems().size() >= 3);
}
@Test
public void testGetCommentsAllData() throws IOException, ExtractionException {
ListExtractor.InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
DefaultTests.defaultTestListOfItems(Bandcamp, comments.getItems(), comments.getErrors());
for (CommentsInfoItem c : comments.getItems()) {
assertFalse(Utils.isBlank(c.getUploaderName()));
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
assertFalse(Utils.isBlank(c.getCommentText()));
assertFalse(Utils.isBlank(c.getName()));
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl()));
assertEquals(-1, c.getLikeCount());
}
}
}

45
extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampCommentsLinkHandlerFactoryTest.java Normal file
View File

@ -0,0 +1,45 @@
// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later
package org.schabi.newpipe.extractor.services.bandcamp;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampCommentsLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampStreamLinkHandlerFactory;
import static org.junit.Assert.*;
/**
* Test for {@link BandcampCommentsLinkHandlerFactory}
*/
public class BandcampCommentsLinkHandlerFactoryTest {
private static BandcampCommentsLinkHandlerFactory linkHandler;
@BeforeClass
public static void setUp() {
linkHandler = new BandcampCommentsLinkHandlerFactory();
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test
public void testAcceptUrl() throws ParsingException {
assertFalse(linkHandler.acceptUrl("http://interovgm.com/releases/"));
assertFalse(linkHandler.acceptUrl("https://interovgm.com/releases"));
assertFalse(linkHandler.acceptUrl("http://zachbenson.bandcamp.com"));
assertFalse(linkHandler.acceptUrl("https://bandcamp.com"));
assertFalse(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/"));
assertFalse(linkHandler.acceptUrl("https://example.com/track/sampletrack"));
assertFalse(linkHandler.acceptUrl("http://bandcamP.com/?show=38"));
assertTrue(linkHandler.acceptUrl("https://powertothequeerkids.bandcamp.com/album/power-to-the-queer-kids"));
assertTrue(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/track/kitchen"));
assertTrue(linkHandler.acceptUrl("http://ZachBenson.Bandcamp.COM/Track/U-I-Tonite/"));
assertTrue(linkHandler.acceptUrl("https://interovgm.bandcamp.com/track/title"));
assertTrue(linkHandler.acceptUrl("https://goodgoodblood-tl.bandcamp.com/track/when-it-all-wakes-up"));
assertTrue(linkHandler.acceptUrl("https://lobstertheremin.com/track/unfinished"));
}
}