Implement SubscriptionExtractor for services

- YouTube supports extracting from an export (from subscriptions manager)
- SoundCloud supports extracting the "followings" from an user
This commit is contained in:
Mauricio Colli 2018-02-22 11:52:38 -03:00
parent a99f466c28
commit 9dfcb3be06
13 changed files with 582 additions and 50 deletions

View File

@ -1,36 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<module external.linked.project.id="NewPipeExtractor" external.linked.project.path="$MODULE_DIR$" external.root.project.path="$MODULE_DIR$" external.system.id="GRADLE" external.system.module.group="" external.system.module.version="unspecified" type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_7">
<output url="file://$MODULE_DIR$/out/production/classes" />
<output-test url="file://$MODULE_DIR$/out/test/classes" />
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/src/test/resources" type="java-test-resource" />
<excludeFolder url="file://$MODULE_DIR$/.gradle" />
<excludeFolder url="file://$MODULE_DIR$/build" />
<excludeFolder url="file://$MODULE_DIR$/out" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" scope="PROVIDED" name="Gradle: com.grack:nanojson:1.1" level="project" />
<orderEntry type="library" scope="PROVIDED" name="Gradle: org.jsoup:jsoup:1.9.2" level="project" />
<orderEntry type="library" scope="PROVIDED" name="Gradle: org.mozilla:rhino:1.7.7.1" level="project" />
<orderEntry type="library" scope="PROVIDED" name="Gradle: com.github.spotbugs:spotbugs-annotations:3.1.0" level="project" />
<orderEntry type="library" scope="PROVIDED" name="Gradle: com.google.code.findbugs:jsr305:3.0.2" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Gradle: com.grack:nanojson:1.1" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Gradle: org.jsoup:jsoup:1.9.2" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Gradle: org.mozilla:rhino:1.7.7.1" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Gradle: com.github.spotbugs:spotbugs-annotations:3.1.0" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Gradle: com.google.code.findbugs:jsr305:3.0.2" level="project" />
<orderEntry type="library" scope="TEST" name="Gradle: com.grack:nanojson:1.1" level="project" />
<orderEntry type="library" scope="TEST" name="Gradle: org.jsoup:jsoup:1.9.2" level="project" />
<orderEntry type="library" scope="TEST" name="Gradle: org.mozilla:rhino:1.7.7.1" level="project" />
<orderEntry type="library" scope="TEST" name="Gradle: com.github.spotbugs:spotbugs-annotations:3.1.0" level="project" />
<orderEntry type="library" scope="TEST" name="Gradle: junit:junit:4.12" level="project" />
<orderEntry type="library" scope="TEST" name="Gradle: com.google.code.findbugs:jsr305:3.0.2" level="project" />
<orderEntry type="library" scope="TEST" name="Gradle: org.hamcrest:hamcrest-core:1.3" level="project" />
</component>
</module>

View File

@ -6,6 +6,7 @@ import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import java.io.IOException; import java.io.IOException;
import java.util.Collections; import java.util.Collections;
@ -71,6 +72,7 @@ public abstract class StreamingService {
public abstract ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException; public abstract ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException;
public abstract PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException; public abstract PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException;
public abstract KioskList getKioskList() throws ExtractionException; public abstract KioskList getKioskList() throws ExtractionException;
public abstract SubscriptionExtractor getSubscriptionExtractor();
public ChannelExtractor getChannelExtractor(String url) throws IOException, ExtractionException { public ChannelExtractor getChannelExtractor(String url) throws IOException, ExtractionException {
return getChannelExtractor(url, null); return getChannelExtractor(url, null);

View File

@ -4,39 +4,39 @@ import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor { public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {
private final JsonObject searchResult; private final JsonObject itemObject;
public SoundcloudChannelInfoItemExtractor(JsonObject searchResult) { public SoundcloudChannelInfoItemExtractor(JsonObject itemObject) {
this.searchResult = searchResult; this.itemObject = itemObject;
} }
@Override @Override
public String getName() { public String getName() {
return searchResult.getString("username"); return itemObject.getString("username");
} }
@Override @Override
public String getUrl() { public String getUrl() {
return searchResult.getString("permalink_url"); return itemObject.getString("permalink_url");
} }
@Override @Override
public String getThumbnailUrl() { public String getThumbnailUrl() {
return searchResult.getString("avatar_url", ""); return itemObject.getString("avatar_url", "");
} }
@Override @Override
public long getSubscriberCount() { public long getSubscriberCount() {
return searchResult.getNumber("followers_count", 0).longValue(); return itemObject.getNumber("followers_count", 0).longValue();
} }
@Override @Override
public long getStreamCount() { public long getStreamCount() {
return searchResult.getNumber("track_count", 0).longValue(); return itemObject.getNumber("track_count", 0).longValue();
} }
@Override @Override
public String getDescription() { public String getDescription() {
return searchResult.getString("description", ""); return itemObject.getString("description", "");
} }
} }

View File

@ -9,20 +9,20 @@ import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException; import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.text.ParseException; import java.text.ParseException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.Date; import java.util.Date;
import javax.annotation.Nonnull;
public class SoundcloudParsingHelper { public class SoundcloudParsingHelper {
private static String clientId; private static String clientId;
@ -100,7 +100,7 @@ public class SoundcloudParsingHelper {
/** /**
* Fetch the embed player with the url and return the id (like the id from the json api). * Fetch the embed player with the url and return the id (like the id from the json api).
* *
* @return the id resolved * @return the resolved id
*/ */
public static String resolveIdWithEmbedPlayer(String url) throws IOException, ReCaptchaException, ParsingException { public static String resolveIdWithEmbedPlayer(String url) throws IOException, ReCaptchaException, ParsingException {
@ -109,6 +109,57 @@ public class SoundcloudParsingHelper {
return Parser.matchGroup1(",\"id\":(.*?),", response); return Parser.matchGroup1(",\"id\":(.*?),", response);
} }
/**
* Fetch the users from the given api and commit each of them to the collector.
* <p>
* This differ from {@link #getUsersFromApi(ChannelInfoItemCollector, String)} in the sense that they will always
* get MIN_ITEMS or more.
*
* @param minItems the method will return only when it have extracted that many items (equal or more)
*/
public static String getUsersFromApiMinItems(int minItems, ChannelInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
String nextStreamsUrl = SoundcloudParsingHelper.getUsersFromApi(collector, apiUrl);
while (!nextStreamsUrl.isEmpty() && collector.getItemList().size() < minItems) {
nextStreamsUrl = SoundcloudParsingHelper.getUsersFromApi(collector, nextStreamsUrl);
}
return nextStreamsUrl;
}
/**
* Fetch the user items from the given api and commit each of them to the collector.
*
* @return the next streams url, empty if don't have
*/
public static String getUsersFromApi(ChannelInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().download(apiUrl);
JsonObject responseObject;
try {
responseObject = JsonParser.object().from(response);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
JsonArray responseCollection = responseObject.getArray("collection");
for (Object o : responseCollection) {
if (o instanceof JsonObject) {
JsonObject object = (JsonObject) o;
collector.commit(new SoundcloudChannelInfoItemExtractor(object));
}
}
String nextStreamsUrl;
try {
nextStreamsUrl = responseObject.getString("next_href");
if (!nextStreamsUrl.contains("client_id=")) nextStreamsUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
} catch (Exception ignored) {
nextStreamsUrl = "";
}
return nextStreamsUrl;
}
/** /**
* Fetch the streams from the given api and commit each of them to the collector. * Fetch the streams from the given api and commit each of them to the collector.
* <p> * <p>

View File

@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import java.io.IOException; import java.io.IOException;
@ -92,4 +93,10 @@ public class SoundcloudService extends StreamingService {
return list; return list;
} }
@Override
public SubscriptionExtractor getSubscriptionExtractor() {
return new SoundcloudSubscriptionExtractor(this);
}
} }

View File

@ -0,0 +1,74 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
* Extract the "followings" from a user in SoundCloud.
*/
public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor {
public SoundcloudSubscriptionExtractor(SoundcloudService service) {
super(service, Collections.singletonList(ContentSource.CHANNEL_URL));
}
@Override
public String getRelatedUrl() {
return "https://soundcloud.com/you";
}
@Override
public List<SubscriptionItem> fromChannelUrl(String channelUrl) throws IOException, ExtractionException {
if (channelUrl == null) throw new InvalidSourceException("channel url is null");
String id;
try {
id = service.getChannelUrlIdHandler().getId(getUrlFrom(channelUrl));
} catch (ExtractionException e) {
throw new InvalidSourceException(e);
}
String apiUrl = "https://api.soundcloud.com/users/" + id + "/followings"
+ "?client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=200";
ChannelInfoItemCollector collector = new ChannelInfoItemCollector(service.getServiceId());
// ± 2000 is the limit of followings on SoundCloud, so this minimum should be enough
SoundcloudParsingHelper.getUsersFromApiMinItems(2500, collector, apiUrl);
return toSubscriptionItems(collector.getItemList());
}
private String getUrlFrom(String channelUrl) {
channelUrl = channelUrl.replace("http://", "https://").trim();
if (!channelUrl.startsWith("https://")) {
if (!channelUrl.contains("soundcloud.com/")) {
channelUrl = "https://soundcloud.com/" + channelUrl;
} else {
channelUrl = "https://" + channelUrl;
}
}
return channelUrl;
}
/*//////////////////////////////////////////////////////////////////////////
// Utils
//////////////////////////////////////////////////////////////////////////*/
private List<SubscriptionItem> toSubscriptionItems(List<ChannelInfoItem> items) {
List<SubscriptionItem> result = new ArrayList<>(items.size());
for (ChannelInfoItem item : items) {
result.add(new SubscriptionItem(item.getServiceId(), item.getUrl(), item.getName()));
}
return result;
}
}

View File

@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import java.io.IOException; import java.io.IOException;
@ -84,8 +85,7 @@ public class YoutubeService extends StreamingService {
} }
@Override @Override
public KioskList getKioskList() public KioskList getKioskList() throws ExtractionException {
throws ExtractionException {
KioskList list = new KioskList(getServiceId()); KioskList list = new KioskList(getServiceId());
// add kiosks here e.g.: // add kiosks here e.g.:
@ -104,4 +104,10 @@ public class YoutubeService extends StreamingService {
return list; return list;
} }
@Override
public SubscriptionExtractor getSubscriptionExtractor() {
return new YoutubeSubscriptionExtractor(this);
}
} }

View File

@ -0,0 +1,131 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
import org.schabi.newpipe.extractor.utils.Parser;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import static org.schabi.newpipe.extractor.subscription.SubscriptionExtractor.ContentSource.INPUT_STREAM;
/**
* Extract subscriptions from a YouTube export (OPML format supported)
*/
public class YoutubeSubscriptionExtractor extends SubscriptionExtractor {
public YoutubeSubscriptionExtractor(YoutubeService service) {
super(service, Collections.singletonList(INPUT_STREAM));
}
@Override
public String getRelatedUrl() {
return "https://www.youtube.com/subscription_manager?action_takeout=1";
}
@Override
public List<SubscriptionItem> fromInputStream(InputStream contentInputStream) throws ExtractionException {
if (contentInputStream == null) throw new InvalidSourceException("input stream is null");
return getItemsFromOPML(contentInputStream);
}
/*//////////////////////////////////////////////////////////////////////////
// OPML implementation
//////////////////////////////////////////////////////////////////////////*/
private static final String ID_PATTERN = "/videos.xml\\?channel_id=([A-Za-z0-9_-]*)";
private static final String BASE_CHANNEL_URL = "https://www.youtube.com/channel/";
private List<SubscriptionItem> getItemsFromOPML(InputStream contentInputStream) throws ExtractionException {
final List<SubscriptionItem> result = new ArrayList<>();
final String contentString = readFromInputStream(contentInputStream);
Document document = Jsoup.parse(contentString, "", org.jsoup.parser.Parser.xmlParser());
if (document.select("opml").isEmpty()) {
throw new InvalidSourceException("document does not have OPML tag");
}
if (document.select("outline").isEmpty()) {
throw new InvalidSourceException("document does not have at least one outline tag");
}
for (Element outline : document.select("outline[type=rss]")) {
String title = outline.attr("title");
String xmlUrl = outline.attr("abs:xmlUrl");
if (title.isEmpty() || xmlUrl.isEmpty()) {
throw new InvalidSourceException("document has invalid entries");
}
try {
String id = Parser.matchGroup1(ID_PATTERN, xmlUrl);
result.add(new SubscriptionItem(service.getServiceId(), BASE_CHANNEL_URL + id, title));
} catch (Parser.RegexException e) {
throw new InvalidSourceException("document has invalid entries", e);
}
}
return result;
}
/*//////////////////////////////////////////////////////////////////////////
// Utils
//////////////////////////////////////////////////////////////////////////*/
/**
* Throws an exception if the string does not have the right tag/string from a valid export.
*/
private void throwIfTagIsNotFound(String content) throws InvalidSourceException {
if (!content.trim().contains("<opml")) {
throw new InvalidSourceException("input stream does not have OPML tag");
}
}
private String readFromInputStream(InputStream inputStream) throws InvalidSourceException {
StringBuilder contentBuilder = new StringBuilder();
boolean hasTag = false;
try {
byte[] buffer = new byte[16 * 1024];
int read;
while ((read = inputStream.read(buffer)) != -1) {
String currentPartOfContent = new String(buffer, 0, read, "UTF-8");
contentBuilder.append(currentPartOfContent);
// Fail-fast in case of reading a long unsupported input stream
if (!hasTag && contentBuilder.length() > 128) {
throwIfTagIsNotFound(contentBuilder.toString());
hasTag = true;
}
}
} catch (InvalidSourceException e) {
throw e;
} catch (Throwable e) {
throw new InvalidSourceException(e);
} finally {
try {
inputStream.close();
} catch (IOException ignored) {
}
}
final String fileContent = contentBuilder.toString().trim();
if (fileContent.isEmpty()) {
throw new InvalidSourceException("Empty input stream");
}
if (!hasTag) {
throwIfTagIsNotFound(fileContent);
}
return fileContent;
}
}

View File

@ -0,0 +1,78 @@
package org.schabi.newpipe.extractor.subscription;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
import java.util.List;
public abstract class SubscriptionExtractor {
/**
* Exception that should be thrown when the input <b>do not</b> contain valid content that the
* extractor can parse (e.g. nonexistent user in case of a url extraction).
*/
public static class InvalidSourceException extends ParsingException {
public InvalidSourceException() {
this(null, null);
}
public InvalidSourceException(String detailMessage) {
this(detailMessage, null);
}
public InvalidSourceException(Throwable cause) {
this(null, cause);
}
public InvalidSourceException(String detailMessage, Throwable cause) {
super(detailMessage == null ? "Not a valid source" : "Not a valid source (" + detailMessage + ")", cause);
}
}
public enum ContentSource {
CHANNEL_URL, INPUT_STREAM
}
private final List<ContentSource> supportedSources;
protected final StreamingService service;
public SubscriptionExtractor(StreamingService service, List<ContentSource> supportedSources) {
this.service = service;
this.supportedSources = Collections.unmodifiableList(supportedSources);
}
public List<ContentSource> getSupportedSources() {
return supportedSources;
}
/**
* Returns an url that can help/guide the user to the file (or channel url) to extract the subscriptions.
* <p>For example, in YouTube, the export subscriptions url is a good choice to return here.</p>
*/
@Nullable
public abstract String getRelatedUrl();
/**
* Reads and parse a list of {@link SubscriptionItem} from the given channel url.
*
* @throws InvalidSourceException when the channelUrl doesn't exist or is invalid
*/
public List<SubscriptionItem> fromChannelUrl(String channelUrl) throws IOException, ExtractionException {
throw new UnsupportedOperationException("Service " + service.getServiceInfo().getName() + " doesn't support extracting from a channel url");
}
/**
* Reads and parse a list of {@link SubscriptionItem} from the given InputStream.
*
* @throws InvalidSourceException when the content read from the InputStream is invalid and can not be parsed
*/
@SuppressWarnings("RedundantThrows")
public List<SubscriptionItem> fromInputStream(InputStream contentInputStream) throws IOException, ExtractionException {
throw new UnsupportedOperationException("Service " + service.getServiceInfo().getName() + " doesn't support extracting from an InputStream");
}
}

View File

@ -0,0 +1,32 @@
package org.schabi.newpipe.extractor.subscription;
import java.io.Serializable;
public class SubscriptionItem implements Serializable {
private final int serviceId;
private final String url, name;
public SubscriptionItem(int serviceId, String url, String name) {
this.serviceId = serviceId;
this.url = url;
this.name = name;
}
public int getServiceId() {
return serviceId;
}
public String getUrl() {
return url;
}
public String getName() {
return name;
}
@Override
public String toString() {
return getClass().getSimpleName() + "@" + Integer.toHexString(hashCode()) +
"[name=" + name + " > " + serviceId + ":" + url + "]";
}
}

View File

@ -0,0 +1,75 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import static org.junit.Assert.*;
/**
* Test for {@link SoundcloudSubscriptionExtractor}
*/
public class SoundcloudSubscriptionExtractorTest {
private static SoundcloudSubscriptionExtractor subscriptionExtractor;
private static UrlIdHandler urlHandler;
@BeforeClass
public static void setupClass() {
NewPipe.init(Downloader.getInstance());
subscriptionExtractor = new SoundcloudSubscriptionExtractor(ServiceList.SoundCloud);
urlHandler = ServiceList.SoundCloud.getChannelUrlIdHandler();
}
@Test
public void testFromChannelUrl() throws Exception {
testList(subscriptionExtractor.fromChannelUrl("https://soundcloud.com/monstercat"));
testList(subscriptionExtractor.fromChannelUrl("http://soundcloud.com/monstercat"));
testList(subscriptionExtractor.fromChannelUrl("soundcloud.com/monstercat"));
testList(subscriptionExtractor.fromChannelUrl("monstercat"));
//Empty followings user
testList(subscriptionExtractor.fromChannelUrl("some-random-user-184047028"));
}
@Test
public void testInvalidSourceException() {
List<String> invalidList = Arrays.asList(
"httttps://invalid.com/user",
".com/monstercat",
"ithinkthatthisuserdontexist",
"",
null
);
for (String invalidUser : invalidList) {
try {
subscriptionExtractor.fromChannelUrl(invalidUser);
fail("didn't throw exception");
} catch (IOException e) {
// Ignore it, could be an unstable network on the CI server
} catch (Exception e) {
boolean isExpectedException = e instanceof SubscriptionExtractor.InvalidSourceException;
assertTrue(e.getClass().getSimpleName() + " is not the expected exception", isExpectedException);
}
}
}
private void testList(List<SubscriptionItem> subscriptionItems) {
for (SubscriptionItem item : subscriptionItems) {
assertNotNull(item.getName());
assertNotNull(item.getUrl());
assertTrue(urlHandler.acceptUrl(item.getUrl()));
assertFalse(item.getServiceId() == -1);
}
}
}

View File

@ -0,0 +1,89 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.util.Arrays;
import java.util.List;
import static org.junit.Assert.*;
/**
* Test for {@link YoutubeSubscriptionExtractor}
*/
public class YoutubeSubscriptionExtractorTest {
private static YoutubeSubscriptionExtractor subscriptionExtractor;
private static UrlIdHandler urlHandler;
@BeforeClass
public static void setupClass() {
NewPipe.init(Downloader.getInstance());
subscriptionExtractor = new YoutubeSubscriptionExtractor(ServiceList.YouTube);
urlHandler = ServiceList.YouTube.getChannelUrlIdHandler();
}
@Test
public void testFromInputStream() throws Exception {
File testFile = new File("src/test/resources/youtube_export_test.xml");
List<SubscriptionItem> subscriptionItems = subscriptionExtractor.fromInputStream(new FileInputStream(testFile));
assertTrue("List doesn't have exactly 8 items (had " + subscriptionItems.size() + ")", subscriptionItems.size() == 8);
for (SubscriptionItem item : subscriptionItems) {
assertNotNull(item.getName());
assertNotNull(item.getUrl());
assertTrue(urlHandler.acceptUrl(item.getUrl()));
assertFalse(item.getServiceId() == -1);
}
}
@Test
public void testEmptySourceException() throws Exception {
String emptySource = "<opml version=\"1.1\"><body>" +
"<outline text=\"Testing\" title=\"123\" />" +
"</body></opml>";
List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(emptySource.getBytes("UTF-8")));
assertTrue(items.isEmpty());
}
@Test
public void testInvalidSourceException() {
List<String> invalidList = Arrays.asList(
"<xml><notvalid></notvalid></xml>",
"<opml><notvalid></notvalid></opml>",
"<opml><body></body></opml>",
"<opml><body><outline text=\"fail\" title=\"fail\" type=\"rss\" xmlUgrl=\"invalidTag\"/></outline></body></opml>",
"<opml><body><outline><outline text=\"invalid\" title=\"url\" type=\"rss\"" +
" xmlUrl=\"https://www.youtube.com/feeds/videos.xml?channel_not_id=|||||||\"/></outline></body></opml>",
"",
null,
"\uD83D\uDC28\uD83D\uDC28\uD83D\uDC28",
"gibberish");
for (String invalidContent : invalidList) {
try {
if (invalidContent != null) {
byte[] bytes = invalidContent.getBytes("UTF-8");
subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes));
} else {
subscriptionExtractor.fromInputStream(null);
}
fail("didn't throw exception");
} catch (Exception e) {
// System.out.println(" -> " + e);
boolean isExpectedException = e instanceof SubscriptionExtractor.InvalidSourceException;
assertTrue("\"" + e.getClass().getSimpleName() + "\" is not the expected exception", isExpectedException);
}
}
}
}

View File

@ -0,0 +1,23 @@
<opml version="1.1">
<body>
<outline text="YouTube Subscriptions" title="YouTube Subscriptions">
<outline text="Kurzgesagt In a Nutshell" title="Kurzgesagt In a Nutshell" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCsXVk37bltHxD1rDPwtNM8Q"/>
<outline text="CaptainDisillusion" title="CaptainDisillusion" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCEOXxzW2vU0P-0THehuIIeg"/>
<outline text="TED" title="TED" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCAuUUnT6oDeKwE6v1NGQxug"/>
<outline text="Gorillaz" title="Gorillaz" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCfIXdjDQH9Fau7y99_Orpjw"/>
<outline text="ElectroBOOM" title="ElectroBOOM" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCJ0-OtVpF0wOKEqT2Z1HEtA"/>
<outline text="ⓤⓝⓘⓒⓞⓓⓔ" title="ⓤⓝⓘⓒⓞⓓⓔ" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCsXVk37bltHxD1rDPwtNM8Q"/>
<outline text="中文" title="中文" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCsXVk37bltHxD1rDPwtNM8Q"/>
<outline text="हिंदी" title="हिंदी" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCsXVk37bltHxD1rDPwtNM8Q"/>
</outline>
</body>
</opml>