add support for PeerTube channels extraction

This commit is contained in:
Roy Yosef 2020-04-09 20:37:49 +03:00
parent a5155fb562
commit b6e6f403a8
10 changed files with 425 additions and 30 deletions

View File

@ -77,7 +77,10 @@ public class PeertubeService extends StreamingService {
@Override
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler)
throws ExtractionException {
return new PeertubeChannelExtractor(this, linkHandler);
return linkHandler.getUrl().matches("^.*\\/accounts\\/[^\\/]*$") ?
new PeertubeUserExtractor(this, linkHandler) :
new PeertubeChannelExtractor(this, linkHandler);
}
@Override

View File

@ -57,7 +57,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
@Override
public String getFeedUrl() throws ParsingException {
return getBaseUrl() + "/feeds/videos.xml?accountId=" + json.get("id");
return getBaseUrl() + "/feeds/videos.xml?videoChannelId=" + json.get("id");
}
@Override
@ -181,7 +181,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
@Override
public String getOriginalUrl() throws ParsingException {
return baseUrl + "/accounts/" + getId();
return baseUrl + "/" + getId();
}
}

View File

@ -97,7 +97,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
public String getAuthorEndpoint() throws ParsingException {
String name = JsonUtils.getString(item, "account.name");
String host = JsonUtils.getString(item, "account.host");
return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl();
return ServiceList.PeerTube.getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
}
}

View File

@ -128,7 +128,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
public String getUploaderUrl() throws ParsingException {
String name = JsonUtils.getString(json, "account.name");
String host = JsonUtils.getString(json, "account.host");
return getService().getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl();
return getService().getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
}
@Override

View File

@ -51,7 +51,8 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor
public String getUploaderUrl() throws ParsingException {
String name = JsonUtils.getString(item, "account.name");
String host = JsonUtils.getString(item, "account.host");
return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl();
return ServiceList.PeerTube.getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
}
@Override

View File

@ -0,0 +1,187 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import java.io.IOException;
public class PeertubeUserExtractor extends ChannelExtractor {
private static final String START_KEY = "start";
private static final String COUNT_KEY = "count";
private static final int ITEMS_PER_PAGE = 12;
private static final String START_PATTERN = "start=(\\d*)";
private InfoItemsPage<StreamInfoItem> initPage;
private long total;
private JsonObject json;
private final String baseUrl;
public PeertubeUserExtractor(StreamingService service, ListLinkHandler linkHandler) throws ParsingException {
super(service, linkHandler);
this.baseUrl = getBaseUrl();
}
@Override
public String getAvatarUrl() throws ParsingException {
String value;
try {
value = JsonUtils.getString(json, "avatar.path");
} catch (Exception e) {
value = "/client/assets/images/default-avatar.png";
}
return baseUrl + value;
}
@Override
public String getBannerUrl() throws ParsingException {
return null;
}
@Override
public String getFeedUrl() throws ParsingException {
return getBaseUrl() + "/feeds/videos.xml?accountId=" + json.get("id");
}
@Override
public long getSubscriberCount() throws ParsingException {
Number number = JsonUtils.getNumber(json, "followersCount");
return number.longValue();
}
@Override
public String getDescription() throws ParsingException {
try {
return JsonUtils.getString(json, "description");
} catch (ParsingException e) {
return "No description";
}
}
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
super.fetchPage();
return initPage;
}
private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException {
JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
} catch (Exception e) {
throw new ParsingException("unable to extract channel streams", e);
}
for (Object c : contents) {
if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
collector.commit(extractor);
}
}
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
super.fetchPage();
return initPage.getNextPageUrl();
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if (null != response && !StringUtil.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
throw new ParsingException("Could not parse json data for kiosk info", e);
}
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
if (json != null) {
PeertubeParsingHelper.validate(json);
Number number = JsonUtils.getNumber(json, "total");
if (number != null) this.total = number.longValue();
collectStreamsFrom(collector, json, pageUrl);
} else {
throw new ExtractionException("Unable to get PeerTube kiosk info");
}
return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl));
}
private String getNextPageUrl(String prevPageUrl) {
String prevStart;
try {
prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl);
} catch (RegexException e) {
return "";
}
if (StringUtil.isBlank(prevStart)) return "";
long nextStart = 0;
try {
nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE;
} catch (NumberFormatException e) {
return "";
}
if (nextStart >= total) {
return "";
} else {
return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart));
}
}
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
Response response = downloader.get(getUrl());
if (null != response && null != response.responseBody()) {
setInitialData(response.responseBody());
} else {
throw new ExtractionException("Unable to extract PeerTube channel data");
}
String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
this.initPage = getPage(pageUrl);
}
private void setInitialData(String responseBody) throws ExtractionException {
try {
json = JsonParser.object().from(responseBody);
} catch (JsonParserException e) {
throw new ExtractionException("Unable to extract peertube channel data", e);
}
if (json == null) throw new ExtractionException("Unable to extract PeerTube channel data");
}
@Override
public String getName() throws ParsingException {
return JsonUtils.getString(json, "displayName");
}
@Override
public String getOriginalUrl() throws ParsingException {
return baseUrl + "/" + getId();
}
}

View File

@ -10,8 +10,8 @@ import java.util.List;
public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
private static final PeertubeChannelLinkHandlerFactory instance = new PeertubeChannelLinkHandlerFactory();
private static final String ID_PATTERN = "/accounts/([^/?&#]*)";
private static final String ACCOUNTS_ENDPOINT = "/api/v1/accounts/";
private static final String ID_PATTERN = "(accounts|video-channels)/([^/?&#]*)";
private static final String API_ENDPOINT = "/api/v1/";
public static PeertubeChannelLinkHandlerFactory getInstance() {
return instance;
@ -19,7 +19,7 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
@Override
public String getId(String url) throws ParsingException {
return Parser.matchGroup1(ID_PATTERN, url);
return Parser.matchGroup(ID_PATTERN, url, 0);
}
@Override
@ -31,11 +31,11 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter, String baseUrl)
throws ParsingException {
return baseUrl + ACCOUNTS_ENDPOINT + id;
return baseUrl + API_ENDPOINT + id;
}
@Override
public boolean onAcceptUrl(String url) {
return url.contains("/accounts/");
return url.contains("/accounts/") || url.contains("/video-channels/");
}
}

View File

@ -11,7 +11,6 @@ import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
@ -20,7 +19,7 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*;
* Test for {@link PeertubeChannelExtractor}
*/
public class PeertubeChannelExtractorTest {
public static class KDE implements BaseChannelExtractorTest {
public static class DanDAugeTutoriels implements BaseChannelExtractorTest {
private static PeertubeChannelExtractor extractor;
@BeforeClass
@ -29,7 +28,7 @@ public class PeertubeChannelExtractorTest {
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
extractor = (PeertubeChannelExtractor) PeerTube
.getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/kde");
.getChannelExtractor("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa");
extractor.fetchPage();
}
@ -44,22 +43,22 @@ public class PeertubeChannelExtractorTest {
@Test
public void testName() throws ParsingException {
assertEquals("The KDE Community", extractor.getName());
assertEquals("Dan d'Auge tutoriels", extractor.getName());
}
@Test
public void testId() throws ParsingException {
assertEquals("kde", extractor.getId());
assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getId());
}
@Test
public void testUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/accounts/kde", extractor.getOriginalUrl());
assertEquals("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
@ -98,16 +97,16 @@ public class PeertubeChannelExtractorTest {
@Test
public void testFeedUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=32465", extractor.getFeedUrl());
assertEquals("https://peertube.mastodon.host/feeds/videos.xml?videoChannelId=1361", extractor.getFeedUrl());
}
@Test
public void testSubscriberCount() throws ParsingException {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5);
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 4);
}
}
public static class Booteille implements BaseChannelExtractorTest {
public static class Divers implements BaseChannelExtractorTest {
private static PeertubeChannelExtractor extractor;
@BeforeClass
@ -116,7 +115,7 @@ public class PeertubeChannelExtractorTest {
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
extractor = (PeertubeChannelExtractor) PeerTube
.getChannelExtractor("https://peertube.mastodon.host/accounts/booteille");
.getChannelExtractor("https://peertube.mastodon.host/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457");
extractor.fetchPage();
}
@ -141,22 +140,22 @@ public class PeertubeChannelExtractorTest {
@Test
public void testName() throws ParsingException {
assertEquals("booteille", extractor.getName());
assertEquals("Divers", extractor.getName());
}
@Test
public void testId() throws ParsingException {
assertEquals("booteille", extractor.getId());
assertEquals("video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getId());
}
@Test
public void testUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getOriginalUrl());
assertEquals("https://peertube.mastodon.host/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
@ -195,12 +194,12 @@ public class PeertubeChannelExtractorTest {
@Test
public void testFeedUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=1753", extractor.getFeedUrl());
assertEquals("https://peertube.mastodon.host/feeds/videos.xml?videoChannelId=1227", extractor.getFeedUrl());
}
@Test
public void testSubscriberCount() throws ParsingException {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 1);
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 2);
}
}
}

View File

@ -30,7 +30,7 @@ public class PeertubeChannelLinkHandlerFactoryTest {
@Test
public void getIdFromUrl() throws ParsingException {
assertEquals("kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net").getId());
assertEquals("kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId());
assertEquals("accounts/kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net").getId());
assertEquals("accounts/kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId());
}
}

View File

@ -0,0 +1,205 @@
package org.schabi.newpipe.extractor.services.peertube;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeUserExtractor;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
/**
* Test for {@link PeertubeUserExtractor}
*/
public class PeertubeUserExtractorTest {
public static class KDE implements BaseChannelExtractorTest {
private static PeertubeUserExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
extractor = (PeertubeUserExtractor) PeerTube
.getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/kde");
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testServiceId() {
assertEquals(PeerTube.getServiceId(), extractor.getServiceId());
}
@Test
public void testName() throws ParsingException {
assertEquals("The KDE Community", extractor.getName());
}
@Test
public void testId() throws ParsingException {
assertEquals("accounts/kde", extractor.getId());
}
@Test
public void testUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/accounts/kde", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testDescription() throws ParsingException {
assertNotNull(extractor.getDescription());
}
@Test
public void testAvatarUrl() throws ParsingException {
assertIsSecureUrl(extractor.getAvatarUrl());
}
@Ignore
@Test
public void testBannerUrl() throws ParsingException {
assertIsSecureUrl(extractor.getBannerUrl());
}
@Test
public void testFeedUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=32465", extractor.getFeedUrl());
}
@Test
public void testSubscriberCount() throws ParsingException {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5);
}
}
public static class Booteille implements BaseChannelExtractorTest {
private static PeertubeUserExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
extractor = (PeertubeUserExtractor) PeerTube
.getChannelExtractor("https://peertube.mastodon.host/accounts/booteille");
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Additional Testing
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testGetPageInNewExtractor() throws Exception {
final ChannelExtractor newExtractor = PeerTube.getChannelExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor);
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testServiceId() {
assertEquals(PeerTube.getServiceId(), extractor.getServiceId());
}
@Test
public void testName() throws ParsingException {
assertEquals("booteille", extractor.getName());
}
@Test
public void testId() throws ParsingException {
assertEquals("accounts/booteille", extractor.getId());
}
@Test
public void testUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testDescription() throws ParsingException {
assertNotNull(extractor.getDescription());
}
@Test
public void testAvatarUrl() throws ParsingException {
assertIsSecureUrl(extractor.getAvatarUrl());
}
@Ignore
@Test
public void testBannerUrl() throws ParsingException {
assertIsSecureUrl(extractor.getBannerUrl());
}
@Test
public void testFeedUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=1753", extractor.getFeedUrl());
}
@Test
public void testSubscriberCount() throws ParsingException {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 1);
}
}
}