lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

merge with changes from master

This commit is contained in:
Christian Schabesberger 2017-09-10 15:49:32 +02:00
parent e2b7cb9c69 f308755359
commit 725b35ef20
44 changed files with 736 additions and 272 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
NewPipeExtractor.iml Normal file
View File

@ -0,0 +1,30 @@
<?xml version="1.0" encoding="UTF-8"?>
<module external.linked.project.id=":NewPipeExtractor" external.linked.project.path="$MODULE_DIR$" external.root.project.path="$MODULE_DIR$/.." external.system.id="GRADLE" type="JAVA_MODULE" version="4">
<component name="FacetManager">
<facet type="java-gradle" name="Java-Gradle">
<configuration>
<option name="BUILD_FOLDER_PATH" value="$MODULE_DIR$/build" />
<option name="BUILDABLE" value="true" />
</configuration>
</facet>
</component>
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_7" inherit-compiler-output="false">
<output url="file://$MODULE_DIR$/build/classes/java/main" />
<output-test url="file://$MODULE_DIR$/build/classes/java/test" />
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/src/test/resources" type="java-test-resource" />
<excludeFolder url="file://$MODULE_DIR$/.gradle" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" exported="" scope="PROVIDED" name="nanojson-1.1" level="project" />
<orderEntry type="library" exported="" scope="PROVIDED" name="jsoup-1.9.2" level="project" />
<orderEntry type="library" exported="" scope="PROVIDED" name="rhino-1.7.7.1" level="project" />
<orderEntry type="library" exported="" scope="TEST" name="junit-4.12" level="project" />
<orderEntry type="library" exported="" scope="TEST" name="hamcrest-core-1.3" level="project" />
</component>
</module>

10
build.gradle
View File

@ -1,18 +1,20 @@
apply plugin: 'java-library' apply plugin: 'java-library'
allprojects {
sourceCompatibility = 1.7
targetCompatibility = 1.7
}
repositories { repositories {
jcenter() jcenter()
} }
dependencies { dependencies {
implementation 'com.github.openjson:openjson:1.0.8' implementation 'com.grack:nanojson:1.1'
implementation 'org.jsoup:jsoup:1.9.2' implementation 'org.jsoup:jsoup:1.9.2'
implementation 'org.mozilla:rhino:1.7.7.1' implementation 'org.mozilla:rhino:1.7.7.1'
testImplementation 'junit:junit:4.12' testImplementation 'junit:junit:4.12'
sourceCompatibility = 1.7
targetCompatibility = 1.7
} }
task sourcesJar(type: Jar, dependsOn: classes) { task sourcesJar(type: Jar, dependsOn: classes) {

1
settings.gradle
View File

@ -1 +1,2 @@
include ':'
rootProject.name = 'NewPipeExtractor' rootProject.name = 'NewPipeExtractor'

12
src/main/java/org/schabi/newpipe/extractor/ListExtractor.java
View File

@ -69,9 +69,19 @@ public abstract class ListExtractor extends Extractor {
*/ */
public final String nextItemsUrl; public final String nextItemsUrl;
public NextItemsResult(List<InfoItem> nextItemsList, String nextItemsUrl) { /**
* Errors that happened during the extraction
*/
public final List<Throwable> errors;
public NextItemsResult(InfoItemCollector collector, String nextItemsUrl) {
this(collector.getItemList(), nextItemsUrl, collector.getErrors());
}
public NextItemsResult(List<InfoItem> nextItemsList, String nextItemsUrl, List<Throwable> errors) {
this.nextItemsList = nextItemsList; this.nextItemsList = nextItemsList;
this.nextItemsUrl = nextItemsUrl; this.nextItemsUrl = nextItemsUrl;
this.errors = errors;
} }
public boolean hasMoreStreams() { public boolean hasMoreStreams() {

15
src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java
View File

@ -5,6 +5,8 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemCollector;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
@ -32,6 +34,7 @@ public class InfoItemSearchCollector extends InfoItemCollector {
private String suggestion = ""; private String suggestion = "";
private StreamInfoItemCollector streamCollector; private StreamInfoItemCollector streamCollector;
private ChannelInfoItemCollector userCollector; private ChannelInfoItemCollector userCollector;
private PlaylistInfoItemCollector playlistCollector;
private SearchResult result = new SearchResult(); private SearchResult result = new SearchResult();
@ -39,6 +42,7 @@ public class InfoItemSearchCollector extends InfoItemCollector {
super(serviceId); super(serviceId);
streamCollector = new StreamInfoItemCollector(serviceId); streamCollector = new StreamInfoItemCollector(serviceId);
userCollector = new ChannelInfoItemCollector(serviceId); userCollector = new ChannelInfoItemCollector(serviceId);
playlistCollector = new PlaylistInfoItemCollector(serviceId);
} }
public void setSuggestion(String suggestion) { public void setSuggestion(String suggestion) {
@ -49,6 +53,7 @@ public class InfoItemSearchCollector extends InfoItemCollector {
addFromCollector(userCollector); addFromCollector(userCollector);
addFromCollector(streamCollector); addFromCollector(streamCollector);
addFromCollector(playlistCollector);
result.suggestion = suggestion; result.suggestion = suggestion;
result.errors = getErrors(); result.errors = getErrors();
@ -74,4 +79,14 @@ public class InfoItemSearchCollector extends InfoItemCollector {
addError(e); addError(e);
} }
} }
public void commit(PlaylistInfoItemExtractor extractor) {
try {
result.resultList.add(playlistCollector.extract(extractor));
} catch (FoundAdException ae) {
System.err.println("Found ad");
} catch (Exception e) {
addError(e);
}
}
} }

7
src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java
View File

@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.search;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.io.IOException; import java.io.IOException;
import java.util.EnumSet;
/* /*
* Created by Christian Schabesberger on 10.08.15. * Created by Christian Schabesberger on 10.08.15.
@ -27,7 +26,7 @@ import java.util.EnumSet;
public abstract class SearchEngine { public abstract class SearchEngine {
public enum Filter { public enum Filter {
STREAM, CHANNEL, PLAYLIST ANY, STREAM, CHANNEL, PLAYLIST
} }
public static class NothingFoundException extends ExtractionException { public static class NothingFoundException extends ExtractionException {
@ -46,8 +45,6 @@ public abstract class SearchEngine {
return collector; return collector;
} }
//Result search(String query, int page); public abstract InfoItemSearchCollector search(String query, int page, String contentCountry, Filter filter)
public abstract InfoItemSearchCollector search(
String query, int page, String contentCountry, EnumSet<Filter> filter)
throws IOException, ExtractionException; throws IOException, ExtractionException;
} }

4
src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java
View File

@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List; import java.util.List;
/* /*
@ -29,8 +28,7 @@ import java.util.List;
*/ */
public class SearchResult { public class SearchResult {
public static SearchResult getSearchResult(SearchEngine engine, String query, public static SearchResult getSearchResult(SearchEngine engine, String query, int page, String languageCode, SearchEngine.Filter filter)
int page, String languageCode, EnumSet<SearchEngine.Filter> filter)
throws IOException, ExtractionException { throws IOException, ExtractionException {
SearchResult result = engine SearchResult result = engine

32
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java
View File

@ -1,6 +1,8 @@
package org.schabi.newpipe.extractor.services.soundcloud; package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
@ -14,7 +16,7 @@ import java.io.IOException;
@SuppressWarnings("WeakerAccess") @SuppressWarnings("WeakerAccess")
public class SoundcloudChannelExtractor extends ChannelExtractor { public class SoundcloudChannelExtractor extends ChannelExtractor {
private String userId; private String userId;
private JSONObject user; private JsonObject user;
public SoundcloudChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { public SoundcloudChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
super(service, url, nextStreamsUrl); super(service, url, nextStreamsUrl);
@ -29,16 +31,16 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
"?client_id=" + SoundcloudParsingHelper.clientId(); "?client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl); String response = dl.download(apiUrl);
user = new JSONObject(response); try {
user = JsonParser.object().from(response);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
} }
@Override @Override
public String getCleanUrl() { public String getCleanUrl() {
try { return user.isString("permalink_url") ? user.getString("permalink_url") : getOriginalUrl();
return user.getString("permalink_url");
} catch (Exception e) {
return getOriginalUrl();
}
} }
@Override @Override
@ -53,16 +55,12 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Override @Override
public String getAvatarUrl() { public String getAvatarUrl() {
return user.optString("avatar_url"); return user.getString("avatar_url");
} }
@Override @Override
public String getBannerUrl() throws ParsingException { public String getBannerUrl() throws ParsingException {
try { return user.getObject("visuals").getArray("visuals").getObject(0).getString("visual_url", "");
return user.getJSONObject("visuals").getJSONArray("visuals").getJSONObject(0).getString("visual_url");
} catch (Exception e) {
throw new ParsingException("Could not get Banner", e);
}
} }
@Override @Override
@ -72,12 +70,12 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Override @Override
public long getSubscriberCount() { public long getSubscriberCount() {
return user.optLong("followers_count"); return user.getNumber("followers_count", 0).longValue();
} }
@Override @Override
public String getDescription() throws ParsingException { public String getDescription() throws ParsingException {
return user.optString("description"); return user.getString("description", "");
} }
@Override @Override
@ -102,6 +100,6 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl); nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl);
return new NextItemsResult(collector.getItemList(), nextStreamsUrl); return new NextItemsResult(collector, nextStreamsUrl);
} }
} }

14
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java
View File

@ -1,12 +1,12 @@
package org.schabi.newpipe.extractor.services.soundcloud; package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONObject; import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor { public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {
private JSONObject searchResult; private JsonObject searchResult;
public SoundcloudChannelInfoItemExtractor(JSONObject searchResult) { public SoundcloudChannelInfoItemExtractor(JsonObject searchResult) {
this.searchResult = searchResult; this.searchResult = searchResult;
} }
@ -22,21 +22,21 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
@Override @Override
public String getThumbnailUrl() { public String getThumbnailUrl() {
return searchResult.optString("avatar_url"); return searchResult.getString("avatar_url", "");
} }
@Override @Override
public long getSubscriberCount() { public long getSubscriberCount() {
return searchResult.optLong("followers_count"); return searchResult.getNumber("followers_count", 0).longValue();
} }
@Override @Override
public long getStreamCount() { public long getStreamCount() {
return searchResult.optLong("track_count"); return searchResult.getNumber("track_count", 0).longValue();
} }
@Override @Override
public String getDescription() { public String getDescription() {
return searchResult.optString("description"); return searchResult.getString("description", "");
} }
} }

2
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java
View File

@ -40,7 +40,7 @@ public class SoundcloudChartsExtractor extends KioskExtractor {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextStreamsUrl, true); nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextStreamsUrl, true);
return new NextItemsResult(collector.getItemList(), nextStreamsUrl); return new NextItemsResult(collector, nextStreamsUrl);
} }
@Override @Override

31
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View File

@ -1,7 +1,9 @@
package org.schabi.newpipe.extractor.services.soundcloud; package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONArray; import com.grack.nanojson.JsonArray;
import com.github.openjson.JSONObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
@ -65,12 +67,16 @@ public class SoundcloudParsingHelper {
* Call the endpoint "/resolve" of the api.<br/> * Call the endpoint "/resolve" of the api.<br/>
* See https://developers.soundcloud.com/docs/api/reference#resolve * See https://developers.soundcloud.com/docs/api/reference#resolve
*/ */
public static JSONObject resolveFor(String url) throws IOException, ReCaptchaException, ParsingException { public static JsonObject resolveFor(String url) throws IOException, ReCaptchaException, ParsingException {
String apiUrl = "https://api.soundcloud.com/resolve" String apiUrl = "https://api.soundcloud.com/resolve"
+ "?url=" + URLEncoder.encode(url, "UTF-8") + "?url=" + URLEncoder.encode(url, "UTF-8")
+ "&client_id=" + clientId(); + "&client_id=" + clientId();
return new JSONObject(NewPipe.getDownloader().download(apiUrl)); try {
return JsonParser.object().from(NewPipe.getDownloader().download(apiUrl));
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
} }
/** /**
@ -123,15 +129,16 @@ public class SoundcloudParsingHelper {
*/ */
public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException { public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().download(apiUrl); String response = NewPipe.getDownloader().download(apiUrl);
JSONObject responseObject = new JSONObject(response); JsonObject responseObject;
try {
responseObject = JsonParser.object().from(response);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
JSONArray responseCollection = responseObject.getJSONArray("collection"); JsonArray responseCollection = responseObject.getArray("collection");
for (int i = 0; i < responseCollection.length(); i++) { for (Object o : responseCollection) {
if (charts) { if (o instanceof JsonObject) collector.commit(new SoundcloudStreamInfoItemExtractor((JsonObject) o));
collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i).getJSONObject("track")));
} else {
collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i)));
}
} }
String nextStreamsUrl; String nextStreamsUrl;

33
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java
View File

@ -1,10 +1,13 @@
package org.schabi.newpipe.extractor.services.soundcloud; package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
@ -13,7 +16,7 @@ import java.io.IOException;
@SuppressWarnings("WeakerAccess") @SuppressWarnings("WeakerAccess")
public class SoundcloudPlaylistExtractor extends PlaylistExtractor { public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
private String playlistId; private String playlistId;
private JSONObject playlist; private JsonObject playlist;
public SoundcloudPlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException { public SoundcloudPlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
super(service, url, nextStreamsUrl); super(service, url, nextStreamsUrl);
@ -29,16 +32,16 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
"&representation=compact"; "&representation=compact";
String response = dl.download(apiUrl); String response = dl.download(apiUrl);
playlist = new JSONObject(response); try {
playlist = JsonParser.object().from(response);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
} }
@Override @Override
public String getCleanUrl() { public String getCleanUrl() {
try { return playlist.isString("permalink_url") ? playlist.getString("permalink_url") : getOriginalUrl();
return playlist.getString("permalink_url");
} catch (Exception e) {
return getOriginalUrl();
}
} }
@Override @Override
@ -48,12 +51,12 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getName() { public String getName() {
return playlist.optString("title"); return playlist.getString("title");
} }
@Override @Override
public String getThumbnailUrl() { public String getThumbnailUrl() {
return playlist.optString("artwork_url"); return playlist.getString("artwork_url");
} }
@Override @Override
@ -63,22 +66,22 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getUploaderUrl() { public String getUploaderUrl() {
return playlist.getJSONObject("user").getString("permalink_url"); return playlist.getObject("user").getString("permalink_url", "");
} }
@Override @Override
public String getUploaderName() { public String getUploaderName() {
return playlist.getJSONObject("user").getString("username"); return playlist.getObject("user").getString("username", "");
} }
@Override @Override
public String getUploaderAvatarUrl() { public String getUploaderAvatarUrl() {
return playlist.getJSONObject("user").getString("avatar_url"); return playlist.getObject("user", new JsonObject()).getString("avatar_url", "");
} }
@Override @Override
public long getStreamCount() { public long getStreamCount() {
return playlist.getLong("track_count"); return playlist.getNumber("track_count", 0).longValue();
} }
@Override @Override
@ -104,6 +107,6 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId()); StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl); nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl);
return new NextItemsResult(collector.getItemList(), nextStreamsUrl); return new NextItemsResult(collector, nextStreamsUrl);
} }
} }

77
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java Normal file
View File

@ -0,0 +1,77 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
private static final String USER_KEY = "user";
private static final String AVATAR_URL_KEY = "avatar_url";
private static final String ARTWORK_URL_KEY = "artwork_url";
private JsonObject searchResult;
public SoundcloudPlaylistInfoItemExtractor(JsonObject searchResult) {
this.searchResult = searchResult;
}
@Override
public String getName() throws ParsingException {
return searchResult.getString("title");
}
@Override
public String getUrl() throws ParsingException {
return searchResult.getString("permalink_url");
}
@Override
public String getThumbnailUrl() throws ParsingException {
// Over-engineering at its finest
if (searchResult.isString(ARTWORK_URL_KEY)) {
final String artworkUrl = searchResult.getString(ARTWORK_URL_KEY, "");
if (!artworkUrl.isEmpty()) return artworkUrl;
}
try {
// Look for artwork url inside the track list
for (Object track : searchResult.getArray("tracks")) {
final JsonObject trackObject = (JsonObject) track;
// First look for track artwork url
if (trackObject.isString(ARTWORK_URL_KEY)) {
final String url = trackObject.getString(ARTWORK_URL_KEY, "");
if (!url.isEmpty()) return url;
}
// Then look for track creator avatar url
final JsonObject creator = trackObject.getObject(USER_KEY, new JsonObject());
final String creatorAvatar = creator.getString(AVATAR_URL_KEY, "");
if (!creatorAvatar.isEmpty()) return creatorAvatar;
}
} catch (Exception ignored) {
// Try other method
}
try {
// Last resort, use user avatar url. If still not found, then throw exception.
return searchResult.getObject(USER_KEY).getString(AVATAR_URL_KEY, "");
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist thumbnail url", e);
}
}
@Override
public String getUploaderName() throws ParsingException {
try {
return searchResult.getObject(USER_KEY).getString("username");
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist uploader", e);
}
}
@Override
public long getStreamCount() throws ParsingException {
return searchResult.getNumber("track_count", 0).longValue();
}
}

63
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java
View File

@ -1,16 +1,18 @@
package org.schabi.newpipe.extractor.services.soundcloud; package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONArray; import com.grack.nanojson.JsonArray;
import com.github.openjson.JSONObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.search.InfoItemSearchCollector; import org.schabi.newpipe.extractor.search.InfoItemSearchCollector;
import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchEngine;
import java.io.IOException; import java.io.IOException;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.EnumSet;
public class SoundcloudSearchEngine extends SearchEngine { public class SoundcloudSearchEngine extends SearchEngine {
public static final String CHARSET_UTF_8 = "UTF-8"; public static final String CHARSET_UTF_8 = "UTF-8";
@ -20,17 +22,27 @@ public class SoundcloudSearchEngine extends SearchEngine {
} }
@Override @Override
public InfoItemSearchCollector search(String query, int page, String languageCode, EnumSet<Filter> filter) throws IOException, ExtractionException { public InfoItemSearchCollector search(String query, int page, String languageCode, Filter filter) throws IOException, ExtractionException {
InfoItemSearchCollector collector = getInfoItemSearchCollector(); InfoItemSearchCollector collector = getInfoItemSearchCollector();
Downloader downloader = NewPipe.getDownloader(); Downloader dl = NewPipe.getDownloader();
String url = "https://api-v2.soundcloud.com/search"; String url = "https://api-v2.soundcloud.com/search";
if (filter.contains(Filter.STREAM) && !filter.contains(Filter.CHANNEL)) { switch (filter) {
url += "/tracks"; case STREAM:
} else if (!filter.contains(Filter.STREAM) && filter.contains(Filter.CHANNEL)) { url += "/tracks";
url += "/users"; break;
case CHANNEL:
url += "/users";
break;
case PLAYLIST:
url += "/playlists";
break;
case ANY:
// Don't append any parameter to search for everything
default:
break;
} }
url += "?q=" + URLEncoder.encode(query, CHARSET_UTF_8) url += "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
@ -38,21 +50,32 @@ public class SoundcloudSearchEngine extends SearchEngine {
+ "&limit=10" + "&limit=10"
+ "&offset=" + Integer.toString(page * 10); + "&offset=" + Integer.toString(page * 10);
String searchJson = downloader.download(url); JsonArray searchCollection;
JSONObject search = new JSONObject(searchJson); try {
JSONArray searchCollection = search.getJSONArray("collection"); searchCollection = JsonParser.object().from(dl.download(url)).getArray("collection");
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
if (searchCollection.length() == 0) { if (searchCollection.size() == 0) {
throw new NothingFoundException("Nothing found"); throw new NothingFoundException("Nothing found");
} }
for (int i = 0; i < searchCollection.length(); i++) { for (Object result : searchCollection) {
JSONObject searchResult = searchCollection.getJSONObject(i); if (!(result instanceof JsonObject)) continue;
String kind = searchResult.getString("kind"); //noinspection ConstantConditions
if (kind.equals("user")) { JsonObject searchResult = (JsonObject) result;
collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult)); String kind = searchResult.getString("kind", "");
} else if (kind.equals("track")) { switch (kind) {
collector.commit(new SoundcloudStreamInfoItemExtractor(searchResult)); case "user":
collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult));
break;
case "track":
collector.commit(new SoundcloudStreamInfoItemExtractor(searchResult));
break;
case "playlist":
collector.commit(new SoundcloudPlaylistInfoItemExtractor(searchResult));
break;
} }
} }

47
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java
View File

@ -1,6 +1,8 @@
package org.schabi.newpipe.extractor.services.soundcloud; package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
@ -16,7 +18,7 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
public class SoundcloudStreamExtractor extends StreamExtractor { public class SoundcloudStreamExtractor extends StreamExtractor {
private JSONObject track; private JsonObject track;
public SoundcloudStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException { public SoundcloudStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
super(service, url); super(service, url);
@ -26,7 +28,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
public void fetchPage() throws IOException, ExtractionException { public void fetchPage() throws IOException, ExtractionException {
track = SoundcloudParsingHelper.resolveFor(getOriginalUrl()); track = SoundcloudParsingHelper.resolveFor(getOriginalUrl());
String policy = track.getString("policy"); String policy = track.getString("policy", "");
if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) { if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) {
throw new ContentNotAvailableException("Content not available: policy " + policy); throw new ContentNotAvailableException("Content not available: policy " + policy);
} }
@ -34,11 +36,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Override @Override
public String getCleanUrl() { public String getCleanUrl() {
try { return track.isString("permalink_url") ? track.getString("permalink_url") : getOriginalUrl();
return track.getString("permalink_url");
} catch (Exception e) {
return getOriginalUrl();
}
} }
@Override @Override
@ -48,7 +46,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Override @Override
public String getName() { public String getName() {
return track.optString("title"); return track.getString("title");
} }
@Override @Override
@ -58,12 +56,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Override @Override
public String getThumbnailUrl() { public String getThumbnailUrl() {
return track.optString("artwork_url"); return track.getString("artwork_url", "");
} }
@Override @Override
public String getDescription() { public String getDescription() {
return track.optString("description"); return track.getString("description");
} }
@Override @Override
@ -73,7 +71,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Override @Override
public long getLength() { public long getLength() {
return track.getLong("duration") / 1000L; return track.getNumber("duration", 0).longValue() / 1000L;
} }
@Override @Override
@ -125,12 +123,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Override @Override
public long getViewCount() { public long getViewCount() {
return track.getLong("playback_count"); return track.getNumber("playback_count", 0).longValue();
} }
@Override @Override
public long getLikeCount() { public long getLikeCount() {
return track.getLong("likes_count"); return track.getNumber("likes_count", 0).longValue();
} }
@Override @Override
@ -140,17 +138,17 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Override @Override
public String getUploaderUrl() { public String getUploaderUrl() {
return track.getJSONObject("user").getString("permalink_url"); return track.getObject("user").getString("permalink_url", "");
} }
@Override @Override
public String getUploaderName() { public String getUploaderName() {
return track.getJSONObject("user").getString("username"); return track.getObject("user").getString("username", "");
} }
@Override @Override
public String getUploaderAvatarUrl() { public String getUploaderAvatarUrl() {
return track.getJSONObject("user").optString("avatar_url"); return track.getObject("user", new JsonObject()).getString("avatar_url", "");
} }
@Override @Override
@ -167,10 +165,19 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
+ "?client_id=" + SoundcloudParsingHelper.clientId(); + "?client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl); String response = dl.download(apiUrl);
JSONObject responseObject = new JSONObject(response); JsonObject responseObject;
try {
responseObject = JsonParser.object().from(response);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
AudioStream audioStream = new AudioStream(responseObject.getString("http_mp3_128_url"), MediaFormat.MP3.id, 128); String mp3Url = responseObject.getString("http_mp3_128_url");
audioStreams.add(audioStream); if (mp3Url != null && !mp3Url.isEmpty()) {
audioStreams.add(new AudioStream(mp3Url, MediaFormat.MP3.id, 128));
} else {
throw new ExtractionException("Could not get SoundCloud's track audio url");
}
return audioStreams; return audioStreams;
} }

17
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java
View File

@ -1,15 +1,15 @@
package org.schabi.newpipe.extractor.services.soundcloud; package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONObject; import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.StreamType;
public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor { public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor {
private final JSONObject searchResult; private final JsonObject searchResult;
public SoundcloudStreamInfoItemExtractor(JSONObject searchResult) { public SoundcloudStreamInfoItemExtractor(JsonObject searchResult) {
this.searchResult = searchResult; this.searchResult = searchResult;
} }
@ -25,27 +25,28 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
@Override @Override
public long getDuration() { public long getDuration() {
return searchResult.getLong("duration") / 1000L; return searchResult.getNumber("duration", 0).longValue() / 1000L;
} }
@Override @Override
public String getUploaderName() { public String getUploaderName() {
return searchResult.getJSONObject("user").getString("username"); //return searchResult.getObject("user").getString("username");
return searchResult.getObject("track").getObject("user").getString("username");
} }
@Override @Override
public String getUploadDate() throws ParsingException { public String getUploadDate() throws ParsingException {
return SoundcloudParsingHelper.toDateString(searchResult.getString("created_at")); return SoundcloudParsingHelper.toDateString(searchResult.getObject("track").getString("created_at"));
} }
@Override @Override
public long getViewCount() { public long getViewCount() {
return searchResult.getLong("playback_count"); return searchResult.getNumber("playback_count", 0).longValue();
} }
@Override @Override
public String getThumbnailUrl() { public String getThumbnailUrl() {
return searchResult.optString("artwork_url"); return searchResult.getString("artwork_url");
} }
@Override @Override

27
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java
View File

@ -1,12 +1,14 @@
package org.schabi.newpipe.extractor.services.soundcloud; package org.schabi.newpipe.extractor.services.soundcloud;
import com.github.openjson.JSONArray; import com.grack.nanojson.JsonArray;
import com.github.openjson.JSONObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.SuggestionExtractor; import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.utils.Parser.RegexException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.io.IOException; import java.io.IOException;
import java.net.URLEncoder; import java.net.URLEncoder;
@ -22,7 +24,7 @@ public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
} }
@Override @Override
public List<String> suggestionList(String query, String contentCountry) throws RegexException, ReCaptchaException, IOException { public List<String> suggestionList(String query, String contentCountry) throws IOException, ExtractionException {
List<String> suggestions = new ArrayList<>(); List<String> suggestions = new ArrayList<>();
Downloader dl = NewPipe.getDownloader(); Downloader dl = NewPipe.getDownloader();
@ -33,14 +35,15 @@ public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
+ "&limit=10"; + "&limit=10";
String response = dl.download(url); String response = dl.download(url);
JSONObject responseObject = new JSONObject(response); try {
JSONArray responseCollection = responseObject.getJSONArray("collection"); JsonArray collection = JsonParser.object().from(response).getArray("collection");
for (Object suggestion : collection) {
if (suggestion instanceof JsonObject) suggestions.add(((JsonObject) suggestion).getString("query"));
}
for (int i = 0; i < responseCollection.length(); i++) { return suggestions;
JSONObject suggestion = responseCollection.getJSONObject(i); } catch (JsonParserException e) {
suggestions.add(suggestion.getString("query")); throw new ParsingException("Could not parse json response", e);
} }
return suggestions;
} }
} }

48
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java
View File

@ -1,8 +1,9 @@
package org.schabi.newpipe.extractor.services.youtube; package org.schabi.newpipe.extractor.services.youtube;
import com.github.openjson.JSONException; import com.grack.nanojson.JsonObject;
import com.github.openjson.JSONObject; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
@ -44,6 +45,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000"; private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000";
private String channelName = ""; //Small hack used to make the channelName available to NextStreams
private Document doc; private Document doc;
/** /**
* It's lazily initialized (when getNextStreams is called) * It's lazily initialized (when getNextStreams is called)
@ -58,7 +61,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
public void fetchPage() throws IOException, ExtractionException { public void fetchPage() throws IOException, ExtractionException {
Downloader downloader = NewPipe.getDownloader(); Downloader downloader = NewPipe.getDownloader();
String channelUrl = getCleanUrl() + CHANNEL_URL_PARAMETERS; String channelUrl = super.getCleanUrl() + CHANNEL_URL_PARAMETERS;
String pageContent = downloader.download(channelUrl); String pageContent = downloader.download(channelUrl);
doc = Jsoup.parse(pageContent, channelUrl); doc = Jsoup.parse(pageContent, channelUrl);
@ -66,21 +69,34 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
nextStreamsAjax = null; nextStreamsAjax = null;
} }
@Override
public String getCleanUrl() {
try {
return "https://www.youtube.com/channel/" + getId();
} catch (ParsingException e) {
return super.getCleanUrl();
}
}
@Override @Override
public String getId() throws ParsingException { public String getId() throws ParsingException {
try { try {
return getUrlIdHandler().getId(getCleanUrl()); Element element = doc.getElementsByClass("yt-uix-subscription-button").first();
if (element == null) element = doc.getElementsByClass("yt-uix-subscription-preferences-button").first();
return element.attr("data-channel-external-id");
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get channel id"); throw new ParsingException("Could not get channel id", e);
} }
} }
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
try { try {
return doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text(); channelName = doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
return channelName;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get channel name"); throw new ParsingException("Could not get channel name", e);
} }
} }
@ -109,8 +125,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override @Override
public String getFeedUrl() throws ParsingException { public String getFeedUrl() throws ParsingException {
try { try {
String channelId = doc.getElementsByClass("yt-uix-subscription-button").first().attr("data-channel-external-id"); return CHANNEL_FEED_BASE + getId();
return channelId == null ? "" : CHANNEL_FEED_BASE + channelId;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get feed url", e); throw new ParsingException("Could not get feed url", e);
} }
@ -153,13 +168,13 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
setupNextStreamsAjax(NewPipe.getDownloader()); setupNextStreamsAjax(NewPipe.getDownloader());
collectStreamsFrom(collector, nextStreamsAjax.select("body").first()); collectStreamsFrom(collector, nextStreamsAjax.select("body").first());
return new NextItemsResult(collector.getItemList(), nextStreamsUrl); return new NextItemsResult(collector, nextStreamsUrl);
} }
private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException { private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
String ajaxDataRaw = downloader.download(nextStreamsUrl); String ajaxDataRaw = downloader.download(nextStreamsUrl);
try { try {
JSONObject ajaxData = new JSONObject(ajaxDataRaw); JsonObject ajaxData = JsonParser.object().from(ajaxDataRaw);
String htmlDataRaw = ajaxData.getString("content_html"); String htmlDataRaw = ajaxData.getString("content_html");
nextStreamsAjax = Jsoup.parse(htmlDataRaw, nextStreamsUrl); nextStreamsAjax = Jsoup.parse(htmlDataRaw, nextStreamsUrl);
@ -170,7 +185,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
} else { } else {
nextStreamsUrl = ""; nextStreamsUrl = "";
} }
} catch (JSONException e) { } catch (JsonParserException e) {
throw new ParsingException("Could not parse json data for next streams", e); throw new ParsingException("Could not parse json data for next streams", e);
} }
} }
@ -189,7 +204,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
} }
} }
private void collectStreamsFrom(StreamInfoItemCollector collector, Element element) throws ParsingException { private void collectStreamsFrom(StreamInfoItemCollector collector,
Element element) throws ParsingException {
collector.getItemList().clear(); collector.getItemList().clear();
for (final Element li : element.children()) { for (final Element li : element.children()) {
@ -219,7 +235,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override @Override
public String getUploaderName() throws ParsingException { public String getUploaderName() throws ParsingException {
return YoutubeChannelExtractor.this.getName(); if(channelName.isEmpty()) {
return "";
} else {
return channelName;
}
} }
@Override @Override

11
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java
View File

@ -1,7 +1,8 @@
package org.schabi.newpipe.extractor.services.youtube; package org.schabi.newpipe.extractor.services.youtube;
import com.github.openjson.JSONException; import com.grack.nanojson.JsonObject;
import com.github.openjson.JSONObject; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
@ -157,13 +158,13 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
setupNextStreamsAjax(NewPipe.getDownloader()); setupNextStreamsAjax(NewPipe.getDownloader());
collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first()); collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first());
return new NextItemsResult(collector.getItemList(), nextStreamsUrl); return new NextItemsResult(collector, nextStreamsUrl);
} }
private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException { private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
String ajaxDataRaw = downloader.download(nextStreamsUrl); String ajaxDataRaw = downloader.download(nextStreamsUrl);
try { try {
JSONObject ajaxData = new JSONObject(ajaxDataRaw); JsonObject ajaxData = JsonParser.object().from(ajaxDataRaw);
String htmlDataRaw = "<table><tbody id=\"pl-load-more-destination\">" + ajaxData.getString("content_html") + "</tbody></table>"; String htmlDataRaw = "<table><tbody id=\"pl-load-more-destination\">" + ajaxData.getString("content_html") + "</tbody></table>";
nextStreamsAjax = Jsoup.parse(htmlDataRaw, nextStreamsUrl); nextStreamsAjax = Jsoup.parse(htmlDataRaw, nextStreamsUrl);
@ -174,7 +175,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
} else { } else {
nextStreamsUrl = ""; nextStreamsUrl = "";
} }
} catch (JSONException e) { } catch (JsonParserException e) {
throw new ParsingException("Could not parse json data for next streams", e); throw new ParsingException("Could not parse json data for next streams", e);
} }
} }

92
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java Normal file
View File

@ -0,0 +1,92 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import org.schabi.newpipe.extractor.utils.Utils;
public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
private Element el;
public YoutubePlaylistInfoItemExtractor(Element el) {
this.el = el;
}
@Override
public String getThumbnailUrl() throws ParsingException {
String url;
try {
Element te = el.select("div[class=\"yt-thumb video-thumb\"]").first()
.select("img").first();
url = te.attr("abs:src");
if (url.contains(".gif")) {
url = te.attr("abs:data-thumb");
}
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist thumbnail url", e);
}
return url;
}
@Override
public String getName() throws ParsingException {
String name;
try {
final Element title = el.select("[class=\"yt-lockup-title\"]").first()
.select("a").first();
name = title == null ? "" : title.text();
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist name", e);
}
return name;
}
@Override
public String getUrl() throws ParsingException {
String url;
try {
final Element href = el.select("div[class=\"yt-lockup-meta\"]").first()
.select("a").first();
url = href.attr("abs:href");
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist url", e);
}
return url;
}
@Override
public String getUploaderName() throws ParsingException {
String name;
try {
final Element div = el.select("div[class=\"yt-lockup-byline\"]").first()
.select("a").first();
name = div.text();
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist uploader", e);
}
return name;
}
@Override
public long getStreamCount() throws ParsingException {
try {
final Element count = el.select("span[class=\"formatted-video-count-label\"]").first()
.select("b").first();
return count == null ? 0 : Long.parseLong(Utils.removeNonDigitCharacters(count.text()));
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist stream count", e);
}
}
}

2
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistUrlIdHandler.java
View File

@ -8,7 +8,7 @@ import org.schabi.newpipe.extractor.utils.Parser;
public class YoutubePlaylistUrlIdHandler implements UrlIdHandler { public class YoutubePlaylistUrlIdHandler implements UrlIdHandler {
private static final YoutubePlaylistUrlIdHandler instance = new YoutubePlaylistUrlIdHandler(); private static final YoutubePlaylistUrlIdHandler instance = new YoutubePlaylistUrlIdHandler();
private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{34})"; private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{10,})";
public static YoutubePlaylistUrlIdHandler getInstance() { public static YoutubePlaylistUrlIdHandler getInstance() {
return instance; return instance;

29
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java
View File

@ -11,7 +11,6 @@ import org.schabi.newpipe.extractor.search.SearchEngine;
import java.io.IOException; import java.io.IOException;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.EnumSet;
/* /*
@ -44,23 +43,29 @@ public class YoutubeSearchEngine extends SearchEngine {
} }
@Override @Override
public InfoItemSearchCollector search(String query, public InfoItemSearchCollector search(String query, int page, String languageCode, Filter filter)
int page,
String languageCode,
EnumSet<Filter> filter)
throws IOException, ExtractionException { throws IOException, ExtractionException {
InfoItemSearchCollector collector = getInfoItemSearchCollector(); InfoItemSearchCollector collector = getInfoItemSearchCollector();
Downloader downloader = NewPipe.getDownloader(); Downloader downloader = NewPipe.getDownloader();
String url = "https://www.youtube.com/results" String url = "https://www.youtube.com/results"
+ "?q=" + URLEncoder.encode(query, CHARSET_UTF_8) + "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
+ "&page=" + Integer.toString(page + 1); + "&page=" + Integer.toString(page + 1);
if (filter.contains(Filter.STREAM) && !filter.contains(Filter.CHANNEL)) {
url += "&sp=EgIQAQ%253D%253D"; switch (filter) {
} else if (!filter.contains(Filter.STREAM) && filter.contains(Filter.CHANNEL)) { case STREAM:
url += "&sp=EgIQAg%253D%253D"; url += "&sp=EgIQAVAU";
break;
case CHANNEL:
url += "&sp=EgIQAlAU"; //EgIQA( lowercase L )AU
break;
case PLAYLIST:
url += "&sp=EgIQA1AU"; //EgIQA( one )AU
break;
case ANY:
// Don't append any parameter to search for everything
default:
break;
} }
String site; String site;
@ -105,6 +110,8 @@ public class YoutubeSearchEngine extends SearchEngine {
collector.commit(new YoutubeStreamInfoItemExtractor(el)); collector.commit(new YoutubeStreamInfoItemExtractor(el));
} else if ((el = item.select("div[class*=\"yt-lockup-channel\"]").first()) != null) { } else if ((el = item.select("div[class*=\"yt-lockup-channel\"]").first()) != null) {
collector.commit(new YoutubeChannelInfoItemExtractor(el)); collector.commit(new YoutubeChannelInfoItemExtractor(el));
} else if ((el = item.select("div[class*=\"yt-lockup-playlist\"]").first()) != null) {
collector.commit(new YoutubePlaylistInfoItemExtractor(el));
} else { } else {
// noinspection ConstantConditions // noinspection ConstantConditions
// simply ignore not known items // simply ignore not known items

42
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java
View File

@ -1,6 +1,7 @@
package org.schabi.newpipe.extractor.services.youtube; package org.schabi.newpipe.extractor.services.youtube;
import com.github.openjson.JSONObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
@ -74,7 +75,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
/*//////////////////////////////////////////////////////////////////////////*/ /*//////////////////////////////////////////////////////////////////////////*/
private Document doc; private Document doc;
private JSONObject playerArgs; private JsonObject playerArgs;
private Map<String, String> videoInfoPage; private Map<String, String> videoInfoPage;
private boolean isAgeRestricted; private boolean isAgeRestricted;
@ -137,7 +138,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
try { try {
return playerArgs.getString("thumbnail_url"); if (playerArgs.isString("thumbnail_url")) return playerArgs.getString("thumbnail_url");
} catch (Exception ignored) { } catch (Exception ignored) {
// Try other method... // Try other method...
} }
@ -174,7 +175,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override @Override
public long getLength() throws ParsingException { public long getLength() throws ParsingException {
try { try {
return playerArgs.getLong("length_seconds"); long returnValue = playerArgs.getNumber("length_seconds", -1).longValue();
if (returnValue >= 0) return returnValue;
} catch (Exception ignored) { } catch (Exception ignored) {
// Try other method... // Try other method...
} }
@ -342,8 +344,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
String dashManifestUrl; String dashManifestUrl;
if (videoInfoPage != null && videoInfoPage.containsKey("dashmpd")) { if (videoInfoPage != null && videoInfoPage.containsKey("dashmpd")) {
dashManifestUrl = videoInfoPage.get("dashmpd"); dashManifestUrl = videoInfoPage.get("dashmpd");
} else if (playerArgs.get("dashmpd") != null) { } else if (playerArgs.isString("dashmpd")) {
dashManifestUrl = playerArgs.optString("dashmpd"); dashManifestUrl = playerArgs.getString("dashmpd", "");
} else { } else {
return ""; return "";
} }
@ -513,7 +515,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
playerUrl = getPlayerUrlFromRestrictedVideo(); playerUrl = getPlayerUrlFromRestrictedVideo();
isAgeRestricted = true; isAgeRestricted = true;
} else { } else {
JSONObject ytPlayerConfig = getPlayerConfig(pageContent); JsonObject ytPlayerConfig = getPlayerConfig(pageContent);
playerArgs = getPlayerArgs(ytPlayerConfig); playerArgs = getPlayerArgs(ytPlayerConfig);
playerUrl = getPlayerUrl(ytPlayerConfig); playerUrl = getPlayerUrl(ytPlayerConfig);
isAgeRestricted = false; isAgeRestricted = false;
@ -524,11 +526,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
} }
private JSONObject getPlayerConfig(String pageContent) throws ParsingException { private JsonObject getPlayerConfig(String pageContent) throws ParsingException {
try { try {
String ytPlayerConfigRaw = String ytPlayerConfigRaw = Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent); return JsonParser.object().from(ytPlayerConfigRaw);
return new JSONObject(ytPlayerConfigRaw);
} catch (Parser.RegexException e) { } catch (Parser.RegexException e) {
String errorReason = getErrorMessage(); String errorReason = getErrorMessage();
switch (errorReason) { switch (errorReason) {
@ -544,13 +545,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
} }
private JSONObject getPlayerArgs(JSONObject playerConfig) throws ParsingException { private JsonObject getPlayerArgs(JsonObject playerConfig) throws ParsingException {
JSONObject playerArgs; JsonObject playerArgs;
//attempt to load the youtube js player JSON arguments //attempt to load the youtube js player JSON arguments
boolean isLiveStream = false; //used to determine if this is a livestream or not boolean isLiveStream = false; //used to determine if this is a livestream or not
try { try {
playerArgs = playerConfig.getJSONObject("args"); playerArgs = playerConfig.getObject("args");
// check if we have a live stream. We need to filter it, since its not yet supported. // check if we have a live stream. We need to filter it, since its not yet supported.
if ((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live")) if ((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))
@ -567,14 +568,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return playerArgs; return playerArgs;
} }
private String getPlayerUrl(JSONObject playerConfig) throws ParsingException { private String getPlayerUrl(JsonObject playerConfig) throws ParsingException {
try { try {
// The Youtube service needs to be initialized by downloading the // The Youtube service needs to be initialized by downloading the
// js-Youtube-player. This is done in order to get the algorithm // js-Youtube-player. This is done in order to get the algorithm
// for decrypting cryptic signatures inside certain stream urls. // for decrypting cryptic signatures inside certain stream urls.
String playerUrl; String playerUrl;
JSONObject ytAssets = playerConfig.getJSONObject("assets"); JsonObject ytAssets = playerConfig.getObject("assets");
playerUrl = ytAssets.getString("js"); playerUrl = ytAssets.getString("js");
if (playerUrl.startsWith("//")) { if (playerUrl.startsWith("//")) {
@ -582,8 +583,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
return playerUrl; return playerUrl;
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException( throw new ParsingException("Could not load decryption code for the Youtube service.", e);
"Could not load decryption code for the Youtube service.", e);
} }
} }
@ -682,10 +682,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
Map<String, ItagItem> urlAndItags = new LinkedHashMap<>(); Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();
String encodedUrlMap = ""; String encodedUrlMap = "";
if (videoInfoPage != null && videoInfoPage.containsKey(encodedUrlMapKey)) { if (playerArgs != null && playerArgs.isString(encodedUrlMapKey)) {
encodedUrlMap = playerArgs.getString(encodedUrlMapKey, "");
} else if (videoInfoPage != null && videoInfoPage.containsKey(encodedUrlMapKey)) {
encodedUrlMap = videoInfoPage.get(encodedUrlMapKey); encodedUrlMap = videoInfoPage.get(encodedUrlMapKey);
} else if (playerArgs != null && playerArgs.get(encodedUrlMapKey) != null) {
encodedUrlMap = playerArgs.optString(encodedUrlMapKey);
} }
for (String url_data_str : encodedUrlMap.split(",")) { for (String url_data_str : encodedUrlMap.split(",")) {

5
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java
View File

@ -74,7 +74,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
try { try {
if (getStreamType() == StreamType.LIVE_STREAM) return -1; if (getStreamType() == StreamType.LIVE_STREAM) return -1;
return YoutubeParsingHelper.parseDurationString(item.select("span[class*=\"video-time\"]").first().text()); final Element duration = item.select("span[class*=\"video-time\"]").first();
// apparently on youtube, video-time element will not show up if the video has a duration of 00:00
// see: https://www.youtube.com/results?sp=EgIQAVAU&q=asdfgf
return duration == null ? 0 : YoutubeParsingHelper.parseDurationString(duration.text());
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get Duration: " + getUrl(), e); throw new ParsingException("Could not get Duration: " + getUrl(), e);
} }

18
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java
View File

@ -1,6 +1,8 @@
package org.schabi.newpipe.extractor.services.youtube; package org.schabi.newpipe.extractor.services.youtube;
import com.github.openjson.JSONArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.SuggestionExtractor; import org.schabi.newpipe.extractor.SuggestionExtractor;
@ -53,14 +55,12 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
String response = dl.download(url); String response = dl.download(url);
try { try {
JSONArray suggestionsArray = new JSONArray(response).getJSONArray(1); JsonArray collection = JsonParser.array().from(response).getArray(1);
for (int i = 0; i < suggestionsArray.length(); i++) { for (Object suggestion : collection) suggestions.add(suggestion.toString());
suggestions.add(suggestionsArray.get(i).toString());
}
} catch (Exception e) {
throw new ParsingException("Could not parse suggestions response.", e);
}
return suggestions; return suggestions;
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
} }
} }

28
src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java
View File

@ -4,9 +4,7 @@ import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.utils.DashMpdParser; import org.schabi.newpipe.extractor.utils.DashMpdParser;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@ -144,26 +142,28 @@ public class StreamInfo extends Info {
if (streamInfo.video_only_streams == null) streamInfo.video_only_streams = new ArrayList<>(); if (streamInfo.video_only_streams == null) streamInfo.video_only_streams = new ArrayList<>();
if (streamInfo.audio_streams == null) streamInfo.audio_streams = new ArrayList<>(); if (streamInfo.audio_streams == null) streamInfo.audio_streams = new ArrayList<>();
Exception dashMpdError = null;
if (streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) { if (streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) {
try { try {
DashMpdParser.getStreams(streamInfo); DashMpdParser.getStreams(streamInfo);
} catch (Exception e) { } catch (Exception e) {
// Sometimes we receive 403 (forbidden) error when trying to download the manifest, // Sometimes we receive 403 (forbidden) error when trying to download the manifest (similar to what happens with youtube-dl),
// (similar to https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L1888) // just skip the exception (but store it somewhere), as we later check if we have streams anyway.
// just skip the exception, as we later check if we have any streams dashMpdError = e;
if (!Utils.hasCauseThrowable(e, FileNotFoundException.class)) {
streamInfo.addException(new ExtractionException("Couldn't get streams from dash mpd", e));
}
} }
} }
// either dash_mpd audio_only or video has to be available, otherwise we didn't get a stream, // Either audio or video has to be available, otherwise we didn't get a stream (since videoOnly are optional, they don't count).
// and therefore failed. (Since video_only_streams are just optional they don't caunt).
if ((streamInfo.video_streams == null || streamInfo.video_streams.isEmpty()) if ((streamInfo.video_streams == null || streamInfo.video_streams.isEmpty())
&& (streamInfo.audio_streams == null || streamInfo.audio_streams.isEmpty()) && (streamInfo.audio_streams == null || streamInfo.audio_streams.isEmpty())) {
&& (streamInfo.dashMpdUrl == null || streamInfo.dashMpdUrl.isEmpty())) {
throw new StreamExtractException( if (dashMpdError != null) {
"Could not get any stream. See error variable to get further details."); // If we don't have any video or audio and the dashMpd 'errored', add it to the error list
// (it's optional and it don't get added automatically, but it's good to have some additional error context)
streamInfo.addException(dashMpdError);
}
throw new StreamExtractException("Could not get any stream. See error variable to get further details.");
} }
return streamInfo; return streamInfo;

25
src/main/java/org/schabi/newpipe/extractor/utils/Utils.java
View File

@ -20,31 +20,6 @@ public class Utils {
return toRemove.replaceAll("\\D+", ""); return toRemove.replaceAll("\\D+", "");
} }
/**
* Check if throwable have the cause
*/
public static boolean hasCauseThrowable(Throwable throwable, Class<?>... causesToCheck) {
// Check if getCause is not the same as cause (the getCause is already the root),
// as it will cause a infinite loop if it is
Throwable cause, getCause = throwable;
for (Class<?> causesEl : causesToCheck) {
if (throwable.getClass().isAssignableFrom(causesEl)) {
return true;
}
}
while ((cause = throwable.getCause()) != null && getCause != cause) {
getCause = cause;
for (Class<?> causesEl : causesToCheck) {
if (cause.getClass().isAssignableFrom(causesEl)) {
return true;
}
}
}
return false;
}
/** /**
* Check if the url matches the pattern. * Check if the url matches the pattern.
* *

6
src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java
View File

@ -4,12 +4,10 @@ import org.junit.Test;
import java.util.HashSet; import java.util.HashSet;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.*;
import static org.junit.Assert.assertNotEquals; import static org.schabi.newpipe.extractor.NewPipe.getServiceByUrl;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.NewPipe.getServiceByUrl;
public class NewPipeTest { public class NewPipeTest {
@Test @Test

11
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java
View File

@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.schabi.newpipe.Downloader; import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
@ -31,17 +32,17 @@ public class SoundcloudChannelExtractorTest {
@Test @Test
public void testGetName() throws Exception { public void testGetName() throws Exception {
assertEquals(extractor.getName(), "LIL UZI VERT"); assertEquals("LIL UZI VERT", extractor.getName());
} }
@Test @Test
public void testGetDescription() throws Exception { public void testGetDescription() throws Exception {
assertEquals(extractor.getDescription(), ""); assertTrue(extractor.getDescription() != null);
} }
@Test @Test
public void testGetAvatarUrl() throws Exception { public void testGetAvatarUrl() throws Exception {
assertEquals(extractor.getAvatarUrl(), "https://a1.sndcdn.com/images/default_avatar_large.png"); assertTrue(extractor.getAvatarUrl().contains("https://"));
} }
@Test @Test
@ -70,7 +71,9 @@ public class SoundcloudChannelExtractorTest {
public void testGetNextStreams() throws Exception { public void testGetNextStreams() throws Exception {
// Setup the streams // Setup the streams
extractor.getStreams(); extractor.getStreams();
assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty()); ListExtractor.NextItemsResult nextItemsResult = extractor.getNextStreams();
assertTrue("extractor didn't have next streams", !nextItemsResult.nextItemsList.isEmpty());
assertTrue("errors occurred during extraction of the next streams", nextItemsResult.errors.isEmpty());
assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams()); assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams());
} }

28
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java Normal file
View File

@ -0,0 +1,28 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
public class SoundcloudParsingHelperTest {
@BeforeClass
public static void setUp() {
NewPipe.init(Downloader.getInstance());
}
@Test
public void resolveUrlWithEmbedPlayerTest() throws Exception {
Assert.assertEquals("https://soundcloud.com/trapcity", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/26057743"));
Assert.assertEquals("https://soundcloud.com/nocopyrightsounds", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/16069159"));
}
@Test
public void resolveIdWithEmbedPlayerTest() throws Exception {
Assert.assertEquals("26057743", SoundcloudParsingHelper.resolveIdWithEmbedPlayer("https://soundcloud.com/trapcity"));
Assert.assertEquals("16069159", SoundcloudParsingHelper.resolveIdWithEmbedPlayer("https://soundcloud.com/nocopyrightsounds"));
}
}

16
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java
View File

@ -4,6 +4,7 @@ import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.schabi.newpipe.Downloader; import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import static org.junit.Assert.*; import static org.junit.Assert.*;
@ -40,7 +41,7 @@ public class SoundcloudPlaylistExtractorTest {
@Test @Test
public void testGetThumbnailUrl() throws Exception { public void testGetThumbnailUrl() throws Exception {
assertEquals(extractor.getThumbnailUrl(), "https://i1.sndcdn.com/artworks-000174203688-bweu12-large.jpg"); assertTrue(extractor.getThumbnailUrl(), extractor.getThumbnailUrl().contains("https://"));
} }
@Test @Test
@ -55,7 +56,7 @@ public class SoundcloudPlaylistExtractorTest {
@Test @Test
public void testGetUploaderAvatarUrl() throws Exception { public void testGetUploaderAvatarUrl() throws Exception {
assertEquals(extractor.getUploaderAvatarUrl(), "https://a1.sndcdn.com/images/default_avatar_large.png"); assertTrue(extractor.getUploaderAvatarUrl(), extractor.getUploaderAvatarUrl().contains("https://"));
} }
@Test @Test
@ -79,4 +80,15 @@ public class SoundcloudPlaylistExtractorTest {
extractor.getStreams(); extractor.getStreams();
assertTrue("extractor didn't have more streams", !extractor.hasMoreStreams()); assertTrue("extractor didn't have more streams", !extractor.hasMoreStreams());
} }
@Test(expected = ExtractionException.class)
public void testGetNextStreamsNonExistent() throws Exception {
// Setup the streams
extractor.getStreams();
// This playlist don't have more streams, it should throw an error
extractor.getNextStreams();
fail("Expected exception wasn't thrown");
}
} }

7
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineAllTest.java
View File

@ -8,8 +8,6 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchResult; import org.schabi.newpipe.extractor.search.SearchResult;
import java.util.EnumSet;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
@ -27,9 +25,8 @@ public class SoundcloudSearchEngineAllTest {
// SoundCloud will suggest "lil uzi vert" instead of "lill uzi vert" // SoundCloud will suggest "lil uzi vert" instead of "lill uzi vert"
// keep in mind that the suggestions can NOT change by country (the parameter "de") // keep in mind that the suggestions can NOT change by country (the parameter "de")
result = engine.search("lill uzi vert", 0, "de", result = engine.search("lill uzi vert", 0, "de", SearchEngine.Filter.ANY)
EnumSet.of(SearchEngine.Filter.CHANNEL, .getSearchResult();
SearchEngine.Filter.STREAM)).getSearchResult();
} }
@Test @Test

6
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineChannelTest.java
View File

@ -9,8 +9,6 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchResult; import org.schabi.newpipe.extractor.search.SearchResult;
import java.util.EnumSet;
import static org.junit.Assert.*; import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
@ -27,8 +25,8 @@ public class SoundcloudSearchEngineChannelTest {
// SoundCloud will suggest "lil uzi vert" instead of "lill uzi vert" // SoundCloud will suggest "lil uzi vert" instead of "lill uzi vert"
// keep in mind that the suggestions can NOT change by country (the parameter "de") // keep in mind that the suggestions can NOT change by country (the parameter "de")
result = engine.search("lill uzi vert", 0, "de", result = engine.search("lill uzi vert", 0, "de", SearchEngine.Filter.CHANNEL)
EnumSet.of(SearchEngine.Filter.CHANNEL)).getSearchResult(); .getSearchResult();
} }
@Test @Test

76
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEnginePlaylistTest.java Normal file
View File

@ -0,0 +1,76 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchResult;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
/*
* Created by Christian Schabesberger on 29.12.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* YoutubeSearchEngineStreamTest.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* Test for {@link SearchEngine}
*/
public class SoundcloudSearchEnginePlaylistTest {
private SearchResult result;
@Before
public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
SearchEngine engine = SoundCloud.getService().getSearchEngine();
// Search by country not yet implemented
result = engine.search("parkmemme", 0, "", SearchEngine.Filter.PLAYLIST)
.getSearchResult();
}
@Test
public void testResultList() {
assertFalse(result.resultList.isEmpty());
}
@Test
public void testUserItemType() {
for (InfoItem infoItem : result.resultList) {
assertEquals(InfoItem.InfoType.PLAYLIST, infoItem.info_type);
}
}
@Test
public void testResultErrors() {
if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace();
assertTrue(result.errors == null || result.errors.isEmpty());
}
@Ignore
@Test
public void testSuggestion() {
//todo write a real test
assertTrue(result.suggestion != null);
}
}

6
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineStreamTest.java
View File

@ -9,8 +9,6 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchResult; import org.schabi.newpipe.extractor.search.SearchResult;
import java.util.EnumSet;
import static org.junit.Assert.*; import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
@ -27,8 +25,8 @@ public class SoundcloudSearchEngineStreamTest {
// SoundCloud will suggest "lil uzi vert" instead of "lil uzi vert", // SoundCloud will suggest "lil uzi vert" instead of "lil uzi vert",
// keep in mind that the suggestions can NOT change by country (the parameter "de") // keep in mind that the suggestions can NOT change by country (the parameter "de")
result = engine.search("lill uzi vert", 0, "de", result = engine.search("lill uzi vert", 0, "de", SearchEngine.Filter.STREAM)
EnumSet.of(SearchEngine.Filter.STREAM)).getSearchResult(); .getSearchResult();
} }
@Test @Test

4
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java
View File

@ -77,12 +77,12 @@ public class SoundcloudStreamExtractorDefaultTest {
@Test @Test
public void testGetThumbnailUrl() throws ParsingException { public void testGetThumbnailUrl() throws ParsingException {
assertEquals(extractor.getThumbnailUrl(), "https://i1.sndcdn.com/artworks-000174195399-iw6seg-large.jpg"); assertTrue(extractor.getThumbnailUrl(), extractor.getThumbnailUrl().contains("https://"));
} }
@Test @Test
public void testGetUploaderAvatarUrl() throws ParsingException { public void testGetUploaderAvatarUrl() throws ParsingException {
assertEquals(extractor.getUploaderAvatarUrl(), "https://a1.sndcdn.com/images/default_avatar_large.png"); assertTrue(extractor.getUploaderAvatarUrl(), extractor.getUploaderAvatarUrl().contains("https://"));
} }
@Test @Test

19
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java
View File

@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.schabi.newpipe.Downloader; import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
@ -40,7 +41,7 @@ public class YoutubeChannelExtractorTest {
public void setUp() throws Exception { public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance()); NewPipe.init(Downloader.getInstance());
extractor = YouTube.getService() extractor = YouTube.getService()
.getChannelExtractor("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw"); .getChannelExtractor("https://www.youtube.com/user/Gronkh");
} }
@Test @Test
@ -53,6 +54,16 @@ public class YoutubeChannelExtractorTest {
assertEquals(extractor.getName(), "Gronkh"); assertEquals(extractor.getName(), "Gronkh");
} }
@Test
public void testGetId() throws Exception {
assertEquals(extractor.getId(), "UCYJ61XIK64sp6ZFFS8sctxw");
}
@Test
public void testGetUrl() throws Exception {
assertEquals(extractor.getCleanUrl(), "https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw");
}
@Test @Test
public void testGetDescription() throws Exception { public void testGetDescription() throws Exception {
assertEquals(extractor.getDescription(), "★ ★ ★ KLICK MICH HART, DU SAU! :D ★ ★ ★ Zart im Schmelz und süffig im Abgang. Ungebremster Spieltrieb seit 1896. Tägliche Folgen nonstop seit dem 01.04.2010!..."); assertEquals(extractor.getDescription(), "★ ★ ★ KLICK MICH HART, DU SAU! :D ★ ★ ★ Zart im Schmelz und süffig im Abgang. Ungebremster Spieltrieb seit 1896. Tägliche Folgen nonstop seit dem 01.04.2010!...");
@ -70,7 +81,7 @@ public class YoutubeChannelExtractorTest {
@Test @Test
public void testGetFeedUrl() throws Exception { public void testGetFeedUrl() throws Exception {
assertTrue(extractor.getFeedUrl(), extractor.getFeedUrl().contains("feed")); assertEquals(extractor.getFeedUrl(), "https://www.youtube.com/feeds/videos.xml?channel_id=UCYJ61XIK64sp6ZFFS8sctxw");
} }
@Test @Test
@ -99,7 +110,9 @@ public class YoutubeChannelExtractorTest {
public void testGetNextStreams() throws Exception { public void testGetNextStreams() throws Exception {
// Setup the streams // Setup the streams
extractor.getStreams(); extractor.getStreams();
assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty()); ListExtractor.NextItemsResult nextItemsResult = extractor.getNextStreams();
assertTrue("extractor didn't have next streams", !nextItemsResult.nextItemsList.isEmpty());
assertTrue("errors occurred during extraction of the next streams", nextItemsResult.errors.isEmpty());
assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams()); assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams());
} }
} }

5
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View File

@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.schabi.newpipe.Downloader; import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
@ -89,7 +90,9 @@ public class YoutubePlaylistExtractorTest {
public void testGetNextStreams() throws Exception { public void testGetNextStreams() throws Exception {
// Setup the streams // Setup the streams
extractor.getStreams(); extractor.getStreams();
assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty()); ListExtractor.NextItemsResult nextItemsResult = extractor.getNextStreams();
assertTrue("extractor didn't have next streams", !nextItemsResult.nextItemsList.isEmpty());
assertTrue("errors occurred during extraction of the next streams", nextItemsResult.errors.isEmpty());
assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams()); assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams());
} }

7
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java
View File

@ -8,8 +8,6 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchResult; import org.schabi.newpipe.extractor.search.SearchResult;
import java.util.EnumSet;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@ -48,9 +46,8 @@ public class YoutubeSearchEngineAllTest {
// Youtube will suggest "asdf" instead of "asdgff" // Youtube will suggest "asdf" instead of "asdgff"
// keep in mind that the suggestions can change by country (the parameter "de") // keep in mind that the suggestions can change by country (the parameter "de")
result = engine.search("asdgff", 0, "de", result = engine.search("asdgff", 0, "de", SearchEngine.Filter.ANY)
EnumSet.of(SearchEngine.Filter.CHANNEL, .getSearchResult();
SearchEngine.Filter.STREAM)).getSearchResult();
} }
@Test @Test

6
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineChannelTest.java
View File

@ -9,8 +9,6 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchResult; import org.schabi.newpipe.extractor.search.SearchResult;
import java.util.EnumSet;
import static org.junit.Assert.*; import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@ -48,8 +46,8 @@ public class YoutubeSearchEngineChannelTest {
// Youtube will suggest "gronkh" instead of "grrunkh" // Youtube will suggest "gronkh" instead of "grrunkh"
// keep in mind that the suggestions can change by country (the parameter "de") // keep in mind that the suggestions can change by country (the parameter "de")
result = engine.search("grrunkh", 0, "de", result = engine.search("grrunkh", 0, "de", SearchEngine.Filter.CHANNEL)
EnumSet.of(SearchEngine.Filter.CHANNEL)).getSearchResult(); .getSearchResult();
} }
@Test @Test

77
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEnginePlaylistTest.java Normal file
View File

@ -0,0 +1,77 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchResult;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
/*
* Created by Christian Schabesberger on 29.12.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* YoutubeSearchEngineStreamTest.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* Test for {@link SearchEngine}
*/
public class YoutubeSearchEnginePlaylistTest {
private SearchResult result;
@Before
public void setUp() throws Exception {
NewPipe.init(Downloader.getInstance());
SearchEngine engine = YouTube.getService().getSearchEngine();
// Youtube will suggest "gronkh" instead of "grrunkh"
// keep in mind that the suggestions can change by country (the parameter "de")
result = engine.search("grrunkh", 0, "de", SearchEngine.Filter.PLAYLIST)
.getSearchResult();
}
@Test
public void testResultList() {
assertFalse(result.resultList.isEmpty());
}
@Test
public void testUserItemType() {
for (InfoItem infoItem : result.resultList) {
assertEquals(InfoItem.InfoType.PLAYLIST, infoItem.info_type);
}
}
@Test
public void testResultErrors() {
if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace();
assertTrue(result.errors == null || result.errors.isEmpty());
}
@Ignore
@Test
public void testSuggestion() {
//todo write a real test
assertTrue(result.suggestion != null);
}
}

6
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineStreamTest.java
View File

@ -9,8 +9,6 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchResult; import org.schabi.newpipe.extractor.search.SearchResult;
import java.util.EnumSet;
import static org.junit.Assert.*; import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@ -48,8 +46,8 @@ public class YoutubeSearchEngineStreamTest {
// Youtube will suggest "results" instead of "rsults", // Youtube will suggest "results" instead of "rsults",
// keep in mind that the suggestions can change by country (the parameter "de") // keep in mind that the suggestions can change by country (the parameter "de")
result = engine.search("rsults", 0, "de", result = engine.search("rsults", 0, "de", SearchEngine.Filter.STREAM)
EnumSet.of(SearchEngine.Filter.STREAM)).getSearchResult(); .getSearchResult();
} }
@Test @Test

4
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java
View File

@ -13,9 +13,7 @@ import org.schabi.newpipe.extractor.stream.VideoStream;
import java.io.IOException; import java.io.IOException;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.*;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.ServiceList.YouTube;
/* /*

2
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java
View File

@ -67,7 +67,7 @@ public class YoutubeStreamExtractorRestrictedTest {
@Test @Test
public void testGetViews() throws ParsingException { public void testGetViews() throws ParsingException {
assertTrue(extractor.getLength() > 0); assertTrue(extractor.getViewCount() > 0);
} }
@Test @Test