lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add SoundCloud service (#15) 

Add SoundCloud service
This commit is contained in:
wb9688 2017-08-04 16:21:45 +02:00 committed by Mauricio Colli
parent 12bfdf5234
commit f314bec396
19 changed files with 1093 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
MediaFormat.java
View File

@ -34,7 +34,8 @@ public enum MediaFormat {
WEBM (0x2, "WebM", "webm", "video/webm"),
// audio formats
M4A (0x3, "m4a", "m4a", "audio/mp4"),
WEBMA (0x4, "WebM", "webm", "audio/webm");
WEBMA (0x4, "WebM", "webm", "audio/webm"),
MP3 (0x5, "MP3", "mp3", "audio/mpeg");
public final int id;
@SuppressWarnings("WeakerAccess")

4
ServiceList.java
View File

@ -1,6 +1,7 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService;
/*
* Created by the-scrabi on 18.02.17.
@ -8,6 +9,7 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
class ServiceList {
public static final StreamingService[] serviceList = {
new YoutubeService(0)
new YoutubeService(0),
new SoundcloudService(1)
};
}

5
UrlIdHandler.java
View File

@ -1,6 +1,9 @@
package org.schabi.newpipe.extractor;
import java.io.IOException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
/*
* Created by Christian Schabesberger on 26.07.16.
@ -24,7 +27,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
public interface UrlIdHandler {
String getUrl(String videoId);
String getUrl(String videoId) throws ParsingException;
String getId(String siteUrl) throws ParsingException;
String cleanUrl(String siteUrl) throws ParsingException;

3
channel/ChannelExtractor.java
View File

@ -4,6 +4,7 @@ import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import java.io.IOException;
@ -39,7 +40,7 @@ public abstract class ChannelExtractor extends ListExtractor {
public abstract String getAvatarUrl() throws ParsingException;
public abstract String getBannerUrl() throws ParsingException;
public abstract String getFeedUrl() throws ParsingException;
public abstract StreamInfoItemCollector getStreams() throws ParsingException;
public abstract StreamInfoItemCollector getStreams() throws ParsingException, ReCaptchaException, IOException;
public abstract long getSubscriberCount() throws ParsingException;
}

3
playlist/PlaylistExtractor.java
View File

@ -4,6 +4,7 @@ import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import java.io.IOException;
@ -21,6 +22,6 @@ public abstract class PlaylistExtractor extends ListExtractor {
public abstract String getUploaderUrl() throws ParsingException;
public abstract String getUploaderName() throws ParsingException;
public abstract String getUploaderAvatarUrl() throws ParsingException;
public abstract StreamInfoItemCollector getStreams() throws ParsingException;
public abstract StreamInfoItemCollector getStreams() throws ParsingException, ReCaptchaException, IOException;
public abstract long getStreamsCount() throws ParsingException;
}

118
services/soundcloud/SoundcloudChannelExtractor.java Normal file
View File

@ -0,0 +1,118 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import java.io.IOException;
import org.json.JSONArray;
import org.json.JSONObject;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
@SuppressWarnings("WeakerAccess")
public class SoundcloudChannelExtractor extends ChannelExtractor {
private String channelId;
private JSONObject channel;
private String nextUrl;
public SoundcloudChannelExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException {
super(urlIdHandler, url, serviceId);
Downloader dl = NewPipe.getDownloader();
channelId = urlIdHandler.getId(url);
String apiUrl = "https://api-v2.soundcloud.com/users/" + channelId
+ "?client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl);
channel = new JSONObject(response);
}
@Override
public String getChannelId() {
return channelId;
}
@Override
public String getChannelName() {
return channel.getString("username");
}
@Override
public String getAvatarUrl() {
return channel.getString("avatar_url");
}
@Override
public String getBannerUrl() throws ParsingException {
try {
return channel.getJSONObject("visuals").getJSONArray("visuals").getJSONObject(0).getString("visual_url");
} catch (Exception e) {
throw new ParsingException("Could not get Banner", e);
}
}
@Override
public StreamInfoItemCollector getStreams() throws ReCaptchaException, IOException, ParsingException {
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
Downloader dl = NewPipe.getDownloader();
String apiUrl = "https://api-v2.soundcloud.com/users/" + channelId + "/tracks"
+ "?client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=10"
+ "&offset=0"
+ "&linked_partitioning=1";
String response = dl.download(apiUrl);
JSONObject responseObject = new JSONObject(response);
nextUrl = responseObject.getString("next_href")
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&linked_partitioning=1";
JSONArray responseCollection = responseObject.getJSONArray("collection");
for (int i = 0; i < responseCollection.length(); i++) {
JSONObject track = responseCollection.getJSONObject(i);
collector.commit(new SoundcloudStreamInfoItemExtractor(track));
}
return collector;
}
@Override
public long getSubscriberCount() {
return channel.getLong("followers_count");
}
@Override
public String getFeedUrl() throws ParsingException {
return null;
}
@Override
public StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException {
if (nextUrl.equals("")) {
throw new ExtractionException("Channel doesn't have more streams");
}
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
Downloader dl = NewPipe.getDownloader();
String response = dl.download(nextUrl);
JSONObject responseObject = new JSONObject(response);
nextUrl = responseObject.getString("next_href")
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&linked_partitioning=1";
JSONArray responseCollection = responseObject.getJSONArray("collection");
for (int i = 0; i < responseCollection.length(); i++) {
JSONObject track = responseCollection.getJSONObject(i);
collector.commit(new SoundcloudStreamInfoItemExtractor(track));
}
return collector;
}
}

42
services/soundcloud/SoundcloudChannelInfoItemExtractor.java Normal file
View File

@ -0,0 +1,42 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.json.JSONObject;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {
private JSONObject searchResult;
public SoundcloudChannelInfoItemExtractor(JSONObject searchResult) {
this.searchResult = searchResult;
}
@Override
public String getThumbnailUrl() {
return searchResult.getString("avatar_url");
}
@Override
public String getChannelName() {
return searchResult.getString("username");
}
@Override
public String getWebPageUrl() {
return searchResult.getString("permalink_url");
}
@Override
public long getSubscriberCount() {
return searchResult.getLong("followers_count");
}
@Override
public long getViewCount() {
return searchResult.getLong("track_count");
}
@Override
public String getDescription() {
return searchResult.getString("description");
}
}

76
services/soundcloud/SoundcloudChannelUrlIdHandler.java Normal file
View File

@ -0,0 +1,76 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Parser;
public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
private static final SoundcloudChannelUrlIdHandler instance = new SoundcloudChannelUrlIdHandler();
public static SoundcloudChannelUrlIdHandler getInstance() {
return instance;
}
@Override
public String getUrl(String channelId) throws ParsingException {
try {
Downloader dl = NewPipe.getDownloader();
String response = dl.download("https://api-v2.soundcloud.com/user/" + channelId
+ "?client_id=" + SoundcloudParsingHelper.clientId());
JSONObject responseObject = new JSONObject(response);
return responseObject.getString("permalink_url");
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}
@Override
public String getId(String siteUrl) throws ParsingException {
try {
Downloader dl = NewPipe.getDownloader();
String response = dl.download(siteUrl);
Document doc = Jsoup.parse(response);
Element androidElement = doc.select("meta[property=al:android:url]").first();
String id = androidElement.attr("content").substring(19);
return id;
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}
@Override
public String cleanUrl(String siteUrl) throws ParsingException {
try {
Downloader dl = NewPipe.getDownloader();
String response = dl.download(siteUrl);
Document doc = Jsoup.parse(response);
Element ogElement = doc.select("meta[property=og:url]").first();
String url = ogElement.attr("content");
return url;
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}
@Override
public boolean acceptUrl(String channelUrl) {
String regex = "^https?://(www\\.)?soundcloud.com/[0-9a-z_-]+(/((tracks|albums|sets|reposts|followers|following)/?)?)?([#?].*)?$";
return Parser.isMatch(regex, channelUrl.toLowerCase());
}
}

80
services/soundcloud/SoundcloudParsingHelper.java Normal file
View File

@ -0,0 +1,80 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
public class SoundcloudParsingHelper {
private SoundcloudParsingHelper() {
}
public static final String clientId() throws ReCaptchaException, IOException, RegexException {
Downloader dl = NewPipe.getDownloader();
String response = dl.download("https://soundcloud.com");
Document doc = Jsoup.parse(response);
Element jsElement = doc.select("script[src^=https://a-v2.sndcdn.com/assets/app]").first();
String js = dl.download(jsElement.attr("src"));
String clientId = Parser.matchGroup1(",client_id:\"(.*?)\"", js);
return clientId;
}
public static String toTimeAgoString(String time) throws ParsingException {
try {
List<Long> times = Arrays.asList(TimeUnit.DAYS.toMillis(365), TimeUnit.DAYS.toMillis(30),
TimeUnit.DAYS.toMillis(7), TimeUnit.HOURS.toMillis(1), TimeUnit.MINUTES.toMillis(1),
TimeUnit.SECONDS.toMillis(1));
List<String> timesString = Arrays.asList("year", "month", "week", "day", "hour", "minute", "second");
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
long timeAgo = System.currentTimeMillis() - dateFormat.parse(time).getTime();
StringBuilder timeAgoString = new StringBuilder();
for (int i = 0; i < times.size(); i++) {
Long current = times.get(i);
long currentAmount = timeAgo / current;
if (currentAmount > 0) {
timeAgoString.append(currentAmount).append(" ").append(timesString.get(i))
.append(currentAmount != 1 ? "s ago" : " ago");
break;
}
}
if (timeAgoString.toString().equals("")) {
timeAgoString.append("Just now");
}
return timeAgoString.toString();
} catch (ParseException e) {
throw new ParsingException(e.getMessage(), e);
}
}
public static String toDateString(String time) throws ParsingException {
try {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
Date date = dateFormat.parse(time);
SimpleDateFormat newDateFormat = new SimpleDateFormat("yyyy-MM-dd");
return newDateFormat.format(date);
} catch (ParseException e) {
throw new ParsingException(e.getMessage(), e);
}
}
}

129
services/soundcloud/SoundcloudPlaylistExtractor.java Normal file
View File

@ -0,0 +1,129 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import java.io.IOException;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
@SuppressWarnings("WeakerAccess")
public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
private String playlistId;
private JSONObject playlist;
private List<String> nextTracks;
public SoundcloudPlaylistExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws IOException, ExtractionException {
super(urlIdHandler, url, serviceId);
Downloader dl = NewPipe.getDownloader();
playlistId = urlIdHandler.getId(url);
String apiUrl = "https://api-v2.soundcloud.com/users/" + playlistId
+ "?client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl);
playlist = new JSONObject(response);
}
@Override
public String getPlaylistId() {
return playlistId;
}
@Override
public String getPlaylistName() {
return playlist.getString("title");
}
@Override
public String getAvatarUrl() {
return playlist.getString("artwork_url");
}
@Override
public String getBannerUrl() {
return null;
}
@Override
public String getUploaderUrl() {
return playlist.getJSONObject("user").getString("permalink_url");
}
@Override
public String getUploaderName() {
return playlist.getJSONObject("user").getString("username");
}
@Override
public String getUploaderAvatarUrl() {
return playlist.getJSONObject("user").getString("avatar_url");
}
@Override
public long getStreamsCount() {
return playlist.getLong("track_count");
}
@Override
public StreamInfoItemCollector getStreams() throws ParsingException, ReCaptchaException, IOException {
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
Downloader dl = NewPipe.getDownloader();
String apiUrl = "https://api-v2.soundcloud.com/playlists/" + playlistId
+ "?client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl);
JSONObject responseObject = new JSONObject(response);
JSONArray responseCollection = responseObject.getJSONArray("collection");
for (int i = 0; i < responseCollection.length(); i++) {
JSONObject track = responseCollection.getJSONObject(i);
try {
collector.commit(new SoundcloudStreamInfoItemExtractor(track));
} catch (Exception e) {
nextTracks.add(track.getString("id"));
}
}
return collector;
}
@Override
public StreamInfoItemCollector getNextStreams() throws ReCaptchaException, IOException, ParsingException {
if (nextTracks.equals(null)) {
return null;
}
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
Downloader dl = NewPipe.getDownloader();
// TODO: Do this per 10 tracks, instead of all tracks at once
String apiUrl = "https://api-v2.soundcloud.com/tracks?ids=";
for (String id : nextTracks) {
apiUrl += id;
if (!id.equals(nextTracks.get(nextTracks.size() - 1))) {
apiUrl += ",";
}
}
apiUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl);
JSONObject responseObject = new JSONObject(response);
JSONArray responseCollection = responseObject.getJSONArray("collection");
for (int i = 0; i < responseCollection.length(); i++) {
JSONObject track = responseCollection.getJSONObject(i);
collector.commit(new SoundcloudStreamInfoItemExtractor(track));
}
nextTracks = null;
return collector;
}
}

75
services/soundcloud/SoundcloudPlaylistUrlIdHandler.java Normal file
View File

@ -0,0 +1,75 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Parser;
public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
private static final SoundcloudPlaylistUrlIdHandler instance = new SoundcloudPlaylistUrlIdHandler();
public static SoundcloudPlaylistUrlIdHandler getInstance() {
return instance;
}
@Override
public String getUrl(String listId) throws ParsingException {
try {
Downloader dl = NewPipe.getDownloader();
String response = dl.download("https://api-v2.soundcloud.com/playlists/" + listId
+ "?client_id=" + SoundcloudParsingHelper.clientId());
JSONObject responseObject = new JSONObject(response);
return responseObject.getString("permalink_url");
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}
@Override
public String getId(String url) throws ParsingException {
try {
Downloader dl = NewPipe.getDownloader();
String response = dl.download(url);
Document doc = Jsoup.parse(response);
Element androidElement = doc.select("meta[property=al:android:url]").first();
String id = androidElement.attr("content").substring(23);
return id;
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}
@Override
public String cleanUrl(String complexUrl) throws ParsingException {
try {
Downloader dl = NewPipe.getDownloader();
String response = dl.download(complexUrl);
Document doc = Jsoup.parse(response);
Element ogElement = doc.select("meta[property=og:url]").first();
String url = ogElement.attr("content");
return url;
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}
@Override
public boolean acceptUrl(String videoUrl) {
String regex = "^https?://(www\\.)?soundcloud.com/[0-9a-z_-]+/sets/[0-9a-z_-]+/?([#?].*)?$";
return Parser.isMatch(regex, videoUrl.toLowerCase());
}
}

61
services/soundcloud/SoundcloudSearchEngine.java Normal file
View File

@ -0,0 +1,61 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.EnumSet;
import org.json.JSONArray;
import org.json.JSONObject;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.search.InfoItemSearchCollector;
import org.schabi.newpipe.extractor.search.SearchEngine;
public class SoundcloudSearchEngine extends SearchEngine {
public static final String CHARSET_UTF_8 = "UTF-8";
public SoundcloudSearchEngine(int serviceId) {
super(serviceId);
}
@Override
public InfoItemSearchCollector search(String query, int page, String languageCode, EnumSet<Filter> filter) throws IOException, ExtractionException {
InfoItemSearchCollector collector = getInfoItemSearchCollector();
Downloader downloader = NewPipe.getDownloader();
String url = "https://api-v2.soundcloud.com/search";
if (filter.contains(Filter.STREAM) && !filter.contains(Filter.CHANNEL)) {
url += "/tracks";
} else if (!filter.contains(Filter.STREAM) && filter.contains(Filter.CHANNEL)) {
url += "/users";
}
url += "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=10"
+ "&offset=" + Integer.toString(page * 10);
String searchJson = downloader.download(url);
JSONObject search = new JSONObject(searchJson);
JSONArray searchCollection = search.getJSONArray("collection");
if (searchCollection.length() == 0) {
throw new NothingFoundException("Nothing found");
}
for (int i = 0; i < searchCollection.length(); i++) {
JSONObject searchResult = searchCollection.getJSONObject(i);
String kind = searchResult.getString("kind");
if (kind.equals("user")) {
collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult));
} else if (kind.equals("track")) {
collector.commit(new SoundcloudStreamInfoItemExtractor(searchResult));
}
}
return collector;
}
}

73
services/soundcloud/SoundcloudService.java Normal file
View File

@ -0,0 +1,73 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import java.io.IOException;
public class SoundcloudService extends StreamingService {
public SoundcloudService(int id) {
super(id);
}
@Override
public ServiceInfo getServiceInfo() {
ServiceInfo serviceInfo = new ServiceInfo();
serviceInfo.name = "Soundcloud";
return serviceInfo;
}
@Override
public StreamExtractor getStreamExtractorInstance(String url)
throws ExtractionException, IOException {
UrlIdHandler urlIdHandler = SoundcloudStreamUrlIdHandler.getInstance();
if (urlIdHandler.acceptUrl(url)) {
return new SoundcloudStreamExtractor(urlIdHandler, url, getServiceId());
} else {
throw new IllegalArgumentException("supplied String is not a valid Soundcloud URL");
}
}
@Override
public SearchEngine getSearchEngineInstance() {
return new SoundcloudSearchEngine(getServiceId());
}
@Override
public UrlIdHandler getStreamUrlIdHandlerInstance() {
return SoundcloudStreamUrlIdHandler.getInstance();
}
@Override
public UrlIdHandler getChannelUrlIdHandlerInstance() {
return SoundcloudChannelUrlIdHandler.getInstance();
}
@Override
public UrlIdHandler getPlaylistUrlIdHandlerInstance() {
return SoundcloudPlaylistUrlIdHandler.getInstance();
}
@Override
public ChannelExtractor getChannelExtractorInstance(String url) throws ExtractionException, IOException {
return new SoundcloudChannelExtractor(getChannelUrlIdHandlerInstance(), url, getServiceId());
}
@Override
public PlaylistExtractor getPlaylistExtractorInstance(String url) throws ExtractionException, IOException {
return new SoundcloudPlaylistExtractor(getPlaylistUrlIdHandlerInstance(), url, getServiceId());
}
@Override
public SuggestionExtractor getSuggestionExtractorInstance() {
return new SoundcloudSuggestionExtractor(getServiceId());
}
}

230
services/soundcloud/SoundcloudStreamExtractor.java Normal file
View File

@ -0,0 +1,230 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import java.io.IOException;
import java.util.List;
import java.util.Vector;
import org.json.JSONArray;
import org.json.JSONObject;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
public class SoundcloudStreamExtractor extends StreamExtractor {
private String pageUrl;
private String trackId;
private JSONObject track;
public SoundcloudStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId) throws ExtractionException, IOException {
super(urlIdHandler, pageUrl, serviceId);
Downloader dl = NewPipe.getDownloader();
trackId = urlIdHandler.getId(pageUrl);
String apiUrl = "https://api-v2.soundcloud.com/tracks/" + trackId
+ "?client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl);
track = new JSONObject(response);
if (!track.getString("policy").equals("ALLOW") && !track.getString("policy").equals("MONETIZE")) {
throw new ContentNotAvailableException("Content not available: policy " + track.getString("policy"));
}
}
@Override
public String getId() {
return trackId;
}
@Override
public String getTitle() {
return track.getString("title");
}
@Override
public String getDescription() {
return track.getString("description");
}
@Override
public String getUploader() {
return track.getJSONObject("user").getString("username");
}
@Override
public int getLength() {
return track.getInt("duration") / 1000;
}
@Override
public long getViewCount() {
return track.getLong("playback_count");
}
@Override
public String getUploadDate() throws ParsingException {
return SoundcloudParsingHelper.toDateString(track.getString("created_at"));
}
@Override
public String getThumbnailUrl() {
return track.getString("artwork_url");
}
@Override
public String getUploaderThumbnailUrl() {
return track.getJSONObject("user").getString("avatar_url");
}
@Override
public String getDashMpdUrl() {
return null;
}
@Override
public List<AudioStream> getAudioStreams() throws ReCaptchaException, IOException, RegexException {
Vector<AudioStream> audioStreams = new Vector<>();
Downloader dl = NewPipe.getDownloader();
String apiUrl = "https://api.soundcloud.com/i1/tracks/" + trackId + "/streams"
+ "?client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl);
JSONObject responseObject = new JSONObject(response);
AudioStream audioStream = new AudioStream(responseObject.getString("http_mp3_128_url"), MediaFormat.MP3.id, 128);
audioStreams.add(audioStream);
return audioStreams;
}
@Override
public List<VideoStream> getVideoStreams() {
return null;
}
@Override
public List<VideoStream> getVideoOnlyStreams() {
return null;
}
@Override
public int getTimeStamp() throws ParsingException {
String timeStamp;
try {
timeStamp = Parser.matchGroup1("(#t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl);
} catch (Parser.RegexException e) {
// catch this instantly since an url does not necessarily have to have a time stamp
// -2 because well the testing system will then know its the regex that failed :/
// not good i know
return -2;
}
if (!timeStamp.isEmpty()) {
try {
String secondsString = "";
String minutesString = "";
String hoursString = "";
try {
secondsString = Parser.matchGroup1("(\\d{1,3})s", timeStamp);
minutesString = Parser.matchGroup1("(\\d{1,3})m", timeStamp);
hoursString = Parser.matchGroup1("(\\d{1,3})h", timeStamp);
} catch (Exception e) {
//it could be that time is given in another method
if (secondsString.isEmpty() //if nothing was got,
&& minutesString.isEmpty()//treat as unlabelled seconds
&& hoursString.isEmpty()) {
secondsString = Parser.matchGroup1("t=(\\d+)", timeStamp);
}
}
int seconds = secondsString.isEmpty() ? 0 : Integer.parseInt(secondsString);
int minutes = minutesString.isEmpty() ? 0 : Integer.parseInt(minutesString);
int hours = hoursString.isEmpty() ? 0 : Integer.parseInt(hoursString);
//don't trust BODMAS!
return seconds + (60 * minutes) + (3600 * hours);
//Log.d(TAG, "derived timestamp value:"+ret);
//the ordering varies internationally
} catch (ParsingException e) {
throw new ParsingException("Could not get timestamp.", e);
}
} else {
return 0;
}
}
@Override
public int getAgeLimit() {
return 0;
}
@Override
public String getAverageRating() {
return null;
}
@Override
public int getLikeCount() {
return track.getInt("likes_count");
}
@Override
public int getDislikeCount() {
return 0;
}
@Override
public StreamInfoItemExtractor getNextVideo() {
return null;
}
@Override
public StreamInfoItemCollector getRelatedVideos() throws ReCaptchaException, IOException, ParsingException {
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
Downloader dl = NewPipe.getDownloader();
String apiUrl = "https://api-v2.soundcloud.com/tracks/" + trackId + "/related"
+ "?client_id=" + SoundcloudParsingHelper.clientId();
String response = dl.download(apiUrl);
JSONObject responseObject = new JSONObject(response);
JSONArray responseCollection = responseObject.getJSONArray("collection");
for (int i = 0; i < responseCollection.length(); i++) {
JSONObject relatedVideo = responseCollection.getJSONObject(i);
collector.commit(new SoundcloudStreamInfoItemExtractor(relatedVideo));
}
return collector;
}
@Override
public String getChannelUrl() {
return track.getJSONObject("user").getString("permalink_url");
}
@Override
public StreamType getStreamType() {
return StreamType.AUDIO_STREAM;
}
@Override
public String getErrorMessage() {
return null;
}
}

60
services/soundcloud/SoundcloudStreamInfoItemExtractor.java Normal file
View File

@ -0,0 +1,60 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.json.JSONObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor {
private final JSONObject searchResult;
public SoundcloudStreamInfoItemExtractor(JSONObject searchResult) {
this.searchResult = searchResult;
}
@Override
public String getWebPageUrl() {
return searchResult.getString("permalink_url");
}
@Override
public String getTitle() {
return searchResult.getString("title");
}
@Override
public int getDuration() {
return searchResult.getInt("duration") / 1000;
}
@Override
public String getUploader() {
return searchResult.getJSONObject("user").getString("username");
}
@Override
public String getUploadDate() throws ParsingException {
return SoundcloudParsingHelper.toTimeAgoString(searchResult.getString("created_at"));
}
@Override
public long getViewCount() {
return searchResult.getLong("playback_count");
}
@Override
public String getThumbnailUrl() {
return searchResult.getString("artwork_url");
}
@Override
public StreamType getStreamType() {
return StreamType.AUDIO_STREAM;
}
@Override
public boolean isAd() {
return false;
}
}

77
services/soundcloud/SoundcloudStreamUrlIdHandler.java Normal file
View File

@ -0,0 +1,77 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Parser;
public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
private static final SoundcloudStreamUrlIdHandler instance = new SoundcloudStreamUrlIdHandler();
private SoundcloudStreamUrlIdHandler() {
}
public static SoundcloudStreamUrlIdHandler getInstance() {
return instance;
}
@Override
public String getUrl(String videoId) throws ParsingException {
try {
Downloader dl = NewPipe.getDownloader();
String response = dl.download("https://api-v2.soundcloud.com/tracks/" + videoId
+ "?client_id=" + SoundcloudParsingHelper.clientId());
JSONObject responseObject = new JSONObject(response);
return responseObject.getString("permalink_url");
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}
@Override
public String getId(String url) throws ParsingException {
try {
Downloader dl = NewPipe.getDownloader();
String response = dl.download(url);
Document doc = Jsoup.parse(response);
Element androidElement = doc.select("meta[property=al:android:url]").first();
String id = androidElement.attr("content").substring(20);
return id;
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}
@Override
public String cleanUrl(String complexUrl) throws ParsingException {
try {
Downloader dl = NewPipe.getDownloader();
String response = dl.download(complexUrl);
Document doc = Jsoup.parse(response);
Element ogElement = doc.select("meta[property=og:url]").first();
String url = ogElement.attr("content");
return url;
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}
}
@Override
public boolean acceptUrl(String videoUrl) {
String regex = "^https?://(www\\.)?soundcloud.com/[0-9a-z_-]+/(?!(tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?([#?].*)?$";
return Parser.isMatch(regex, videoUrl.toLowerCase());
}
}

46
services/soundcloud/SoundcloudSuggestionExtractor.java Normal file
View File

@ -0,0 +1,46 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
public static final String CHARSET_UTF_8 = "UTF-8";
public SoundcloudSuggestionExtractor(int serviceId) {
super(serviceId);
}
@Override
public List<String> suggestionList(String query, String contentCountry) throws RegexException, ReCaptchaException, IOException {
List<String> suggestions = new ArrayList<>();
Downloader dl = NewPipe.getDownloader();
String url = "https://api-v2.soundcloud.com/search/queries"
+ "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=10";
String response = dl.download(url);
JSONObject responseObject = new JSONObject(response);
JSONArray responseCollection = responseObject.getJSONArray("collection");
for (int i = 0; i < responseCollection.length(); i++) {
JSONObject suggestion = responseCollection.getJSONObject(i);
suggestions.add(suggestion.getString("query"));
}
return suggestions;
}
}

6
stream/StreamExtractor.java
View File

@ -23,7 +23,9 @@ package org.schabi.newpipe.extractor.stream;
import org.schabi.newpipe.extractor.Extractor;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import java.io.IOException;
import java.util.List;
/**
@ -46,7 +48,7 @@ public abstract class StreamExtractor extends Extractor {
public abstract String getUploadDate() throws ParsingException;
public abstract String getThumbnailUrl() throws ParsingException;
public abstract String getUploaderThumbnailUrl() throws ParsingException;
public abstract List<AudioStream> getAudioStreams() throws ParsingException;
public abstract List<AudioStream> getAudioStreams() throws ParsingException, ReCaptchaException, IOException;
public abstract List<VideoStream> getVideoStreams() throws ParsingException;
public abstract List<VideoStream> getVideoOnlyStreams() throws ParsingException;
public abstract String getDashMpdUrl() throws ParsingException;
@ -55,7 +57,7 @@ public abstract class StreamExtractor extends Extractor {
public abstract int getLikeCount() throws ParsingException;
public abstract int getDislikeCount() throws ParsingException;
public abstract StreamInfoItemExtractor getNextVideo() throws ParsingException;
public abstract StreamInfoItemCollector getRelatedVideos() throws ParsingException;
public abstract StreamInfoItemCollector getRelatedVideos() throws ParsingException, ReCaptchaException, IOException;
public abstract StreamType getStreamType() throws ParsingException;
/**

9
utils/Parser.java
View File

@ -63,6 +63,15 @@ public class Parser {
}
}
public static boolean isMatch(String pattern, String input) {
try {
matchGroup1(pattern, input);
return true;
} catch (RegexException e) {
return false;
}
}
public static Map<String, String> compatParseMap(final String input) throws UnsupportedEncodingException {
Map<String, String> map = new HashMap<>();
for (String arg : input.split("&")) {