2018-10-11 21:10:22 +02:00
|
|
|
package org.schabi.newpipe.extractor.services.peertube.extractors;
|
|
|
|
|
|
|
|
import java.io.IOException;
|
2018-12-26 09:59:23 +01:00
|
|
|
import java.io.UnsupportedEncodingException;
|
|
|
|
import java.net.URLEncoder;
|
2018-10-11 21:10:22 +02:00
|
|
|
import java.util.ArrayList;
|
|
|
|
import java.util.Collections;
|
|
|
|
import java.util.List;
|
|
|
|
|
|
|
|
import org.jsoup.helper.StringUtil;
|
|
|
|
import org.schabi.newpipe.extractor.MediaFormat;
|
|
|
|
import org.schabi.newpipe.extractor.ServiceList;
|
|
|
|
import org.schabi.newpipe.extractor.StreamingService;
|
2019-11-19 22:38:17 +01:00
|
|
|
import org.schabi.newpipe.extractor.downloader.Downloader;
|
|
|
|
import org.schabi.newpipe.extractor.downloader.Response;
|
2018-10-11 21:10:22 +02:00
|
|
|
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
|
|
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
|
|
|
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
|
|
|
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
|
2019-11-19 22:38:17 +01:00
|
|
|
import org.schabi.newpipe.extractor.localization.DateWrapper;
|
2018-10-11 21:10:22 +02:00
|
|
|
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
|
2018-12-26 09:59:23 +01:00
|
|
|
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
|
2018-10-11 21:10:22 +02:00
|
|
|
import org.schabi.newpipe.extractor.stream.AudioStream;
|
|
|
|
import org.schabi.newpipe.extractor.stream.Stream;
|
|
|
|
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
|
|
|
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
|
|
|
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
|
|
|
import org.schabi.newpipe.extractor.stream.StreamType;
|
2018-12-25 14:42:22 +01:00
|
|
|
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
|
2018-10-11 21:10:22 +02:00
|
|
|
import org.schabi.newpipe.extractor.stream.VideoStream;
|
|
|
|
import org.schabi.newpipe.extractor.utils.JsonUtils;
|
2019-11-21 00:30:09 +01:00
|
|
|
import org.schabi.newpipe.extractor.utils.Utils;
|
2018-10-11 21:10:22 +02:00
|
|
|
|
|
|
|
import com.grack.nanojson.JsonArray;
|
|
|
|
import com.grack.nanojson.JsonObject;
|
|
|
|
import com.grack.nanojson.JsonParser;
|
|
|
|
import com.grack.nanojson.JsonParserException;
|
|
|
|
|
|
|
|
public class PeertubeStreamExtractor extends StreamExtractor {
|
|
|
|
|
|
|
|
|
|
|
|
private JsonObject json;
|
2019-11-16 00:00:13 +01:00
|
|
|
private List<SubtitlesStream> subtitles = new ArrayList<>();
|
2018-10-11 21:10:22 +02:00
|
|
|
|
2019-11-19 22:38:17 +01:00
|
|
|
public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler) {
|
|
|
|
super(service, linkHandler);
|
2018-10-11 21:10:22 +02:00
|
|
|
}
|
2019-11-19 22:38:17 +01:00
|
|
|
|
2018-10-11 21:10:22 +02:00
|
|
|
@Override
|
2019-11-19 22:38:17 +01:00
|
|
|
public String getTextualUploadDate() throws ParsingException {
|
|
|
|
return JsonUtils.getString(json, "publishedAt");
|
2018-10-11 21:10:22 +02:00
|
|
|
}
|
|
|
|
|
2019-11-19 22:38:17 +01:00
|
|
|
@Override
|
|
|
|
public DateWrapper getUploadDate() throws ParsingException {
|
|
|
|
final String textualUploadDate = getTextualUploadDate();
|
|
|
|
|
|
|
|
if (textualUploadDate == null) {
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
|
|
|
|
return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate));
|
|
|
|
}
|
|
|
|
|
2018-10-11 21:10:22 +02:00
|
|
|
@Override
|
|
|
|
public String getThumbnailUrl() throws ParsingException {
|
|
|
|
return ServiceList.PeerTube.getBaseUrl() + JsonUtils.getString(json, "thumbnailPath");
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public String getDescription() throws ParsingException {
|
2018-10-11 22:29:13 +02:00
|
|
|
try {
|
|
|
|
return JsonUtils.getString(json, "description");
|
|
|
|
}catch(ParsingException e) {
|
|
|
|
return "No description";
|
|
|
|
}
|
2018-10-11 21:10:22 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public int getAgeLimit() throws ParsingException {
|
|
|
|
return NO_AGE_LIMIT;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public long getLength() throws ParsingException {
|
|
|
|
Number value = JsonUtils.getNumber(json, "duration");
|
|
|
|
return value.longValue();
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public long getTimeStamp() throws ParsingException {
|
|
|
|
//TODO fetch timestamp from url if present;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public long getViewCount() throws ParsingException {
|
|
|
|
Number value = JsonUtils.getNumber(json, "views");
|
|
|
|
return value.longValue();
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public long getLikeCount() throws ParsingException {
|
|
|
|
Number value = JsonUtils.getNumber(json, "likes");
|
|
|
|
return value.longValue();
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public long getDislikeCount() throws ParsingException {
|
|
|
|
Number value = JsonUtils.getNumber(json, "dislikes");
|
|
|
|
return value.longValue();
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public String getUploaderUrl() throws ParsingException {
|
|
|
|
String name = JsonUtils.getString(json, "account.name");
|
|
|
|
String host = JsonUtils.getString(json, "account.host");
|
2019-11-21 00:30:09 +01:00
|
|
|
String baseUrl = Utils.getBaseUrl(getUrl());
|
|
|
|
return getService().getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl();
|
2018-10-11 21:10:22 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public String getUploaderName() throws ParsingException {
|
|
|
|
return JsonUtils.getString(json, "account.displayName");
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public String getUploaderAvatarUrl() throws ParsingException {
|
2018-10-11 22:29:13 +02:00
|
|
|
String value;
|
|
|
|
try {
|
|
|
|
value = JsonUtils.getString(json, "account.avatar.path");
|
|
|
|
}catch(Exception e) {
|
|
|
|
value = "/client/assets/images/default-avatar.png";
|
|
|
|
}
|
|
|
|
return ServiceList.PeerTube.getBaseUrl() + value;
|
2018-10-11 21:10:22 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public String getDashMpdUrl() throws ParsingException {
|
|
|
|
return "";
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public String getHlsUrl() throws ParsingException {
|
|
|
|
return "";
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
|
|
|
|
assertPageFetched();
|
|
|
|
List<VideoStream> videoStreams = new ArrayList<>();
|
|
|
|
try {
|
|
|
|
JsonArray streams = json.getArray("files", new JsonArray());
|
|
|
|
for(Object s: streams) {
|
|
|
|
if(!(s instanceof JsonObject)) continue;
|
|
|
|
JsonObject stream = (JsonObject) s;
|
|
|
|
String url = JsonUtils.getString(stream, "fileUrl");
|
2018-12-30 10:30:57 +01:00
|
|
|
String torrentUrl = JsonUtils.getString(stream, "torrentUrl");
|
2018-10-11 21:10:22 +02:00
|
|
|
String resolution = JsonUtils.getString(stream, "resolution.label");
|
|
|
|
String extension = url.substring(url.lastIndexOf(".") + 1);
|
|
|
|
MediaFormat format = MediaFormat.getFromSuffix(extension);
|
2018-12-30 10:30:57 +01:00
|
|
|
VideoStream videoStream = new VideoStream(url, torrentUrl, format, resolution);
|
2018-10-11 21:10:22 +02:00
|
|
|
if (!Stream.containSimilarStream(videoStream, videoStreams)) {
|
|
|
|
videoStreams.add(videoStream);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} catch (Exception e) {
|
|
|
|
throw new ParsingException("Could not get video streams", e);
|
|
|
|
}
|
|
|
|
|
|
|
|
return videoStreams;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException {
|
|
|
|
// TODO Auto-generated method stub
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2018-12-25 14:42:22 +01:00
|
|
|
public List<SubtitlesStream> getSubtitlesDefault() throws IOException, ExtractionException {
|
2019-11-16 00:00:13 +01:00
|
|
|
return subtitles;
|
2018-10-11 21:10:22 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2018-12-25 14:42:22 +01:00
|
|
|
public List<SubtitlesStream> getSubtitles(final MediaFormat format) throws IOException, ExtractionException {
|
2019-11-16 00:00:13 +01:00
|
|
|
List<SubtitlesStream> filteredSubs = new ArrayList<>();
|
|
|
|
for(SubtitlesStream sub: subtitles) {
|
|
|
|
if(sub.getFormat() == format) {
|
|
|
|
filteredSubs.add(sub);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return filteredSubs;
|
2018-10-11 21:10:22 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public StreamType getStreamType() throws ParsingException {
|
|
|
|
return StreamType.VIDEO_STREAM;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2018-12-25 14:42:22 +01:00
|
|
|
public StreamInfoItem getNextStream() throws IOException, ExtractionException {
|
2018-10-11 21:10:22 +02:00
|
|
|
return null;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2018-12-25 14:42:22 +01:00
|
|
|
public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException {
|
2018-10-11 21:10:22 +02:00
|
|
|
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
2018-12-26 09:59:23 +01:00
|
|
|
List<String> tags = getTags();
|
|
|
|
String apiUrl = null;
|
|
|
|
if(!tags.isEmpty()) {
|
|
|
|
apiUrl = getRelatedStreamsUrl(tags);
|
|
|
|
|
|
|
|
}else {
|
|
|
|
apiUrl = getUploaderUrl() + "/videos?start=0&count=8";
|
|
|
|
}
|
|
|
|
if(!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl);
|
2018-10-11 21:10:22 +02:00
|
|
|
return collector;
|
|
|
|
}
|
2018-12-26 09:59:23 +01:00
|
|
|
|
|
|
|
private List<String> getTags(){
|
|
|
|
try {
|
|
|
|
return (List) JsonUtils.getArray(json, "tags");
|
|
|
|
} catch (Exception e) {
|
|
|
|
return Collections.emptyList();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private String getRelatedStreamsUrl(List<String> tags) throws UnsupportedEncodingException {
|
|
|
|
String url = ServiceList.PeerTube.getBaseUrl() + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT;
|
|
|
|
StringBuilder params = new StringBuilder();
|
|
|
|
params.append("start=0&count=8&sort=-createdAt");
|
|
|
|
for(String tag : tags) {
|
|
|
|
params.append("&tagsOneOf=");
|
|
|
|
params.append(URLEncoder.encode(tag, "UTF-8"));
|
|
|
|
}
|
|
|
|
return url + "?" + params.toString();
|
|
|
|
}
|
2018-10-11 21:10:22 +02:00
|
|
|
|
|
|
|
private void getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, IOException, ParsingException {
|
2019-11-19 22:38:17 +01:00
|
|
|
Response response = getDownloader().get(apiUrl);
|
2018-10-11 21:10:22 +02:00
|
|
|
JsonObject relatedVideosJson = null;
|
2019-11-19 22:38:17 +01:00
|
|
|
if(null != response && !StringUtil.isBlank(response.responseBody())) {
|
2018-10-11 21:10:22 +02:00
|
|
|
try {
|
2019-11-19 22:38:17 +01:00
|
|
|
relatedVideosJson = JsonParser.object().from(response.responseBody());
|
2018-10-11 21:10:22 +02:00
|
|
|
} catch (JsonParserException e) {
|
|
|
|
throw new ParsingException("Could not parse json data for related videos", e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if(relatedVideosJson != null) {
|
|
|
|
collectStreamsFrom(collector, relatedVideosJson);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json) throws ParsingException {
|
|
|
|
JsonArray contents;
|
|
|
|
try {
|
|
|
|
contents = (JsonArray) JsonUtils.getValue(json, "data");
|
|
|
|
}catch(Exception e) {
|
|
|
|
throw new ParsingException("unable to extract related videos", e);
|
|
|
|
}
|
|
|
|
|
2019-11-21 00:30:09 +01:00
|
|
|
String baseUrl = Utils.getBaseUrl(getUrl());
|
2018-10-11 21:10:22 +02:00
|
|
|
for(Object c: contents) {
|
|
|
|
if(c instanceof JsonObject) {
|
|
|
|
final JsonObject item = (JsonObject) c;
|
2019-11-21 00:30:09 +01:00
|
|
|
PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
|
2018-12-26 09:59:23 +01:00
|
|
|
//do not add the same stream in related streams
|
|
|
|
if(!extractor.getUrl().equals(getUrl())) collector.commit(extractor);
|
2018-10-11 21:10:22 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public String getErrorMessage() {
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
|
2019-11-19 22:38:17 +01:00
|
|
|
Response response = downloader.get(getUrl());
|
|
|
|
if(null != response && null != response.responseBody()) {
|
|
|
|
setInitialData(response.responseBody());
|
2018-10-11 21:10:22 +02:00
|
|
|
}else {
|
|
|
|
throw new ExtractionException("Unable to extract peertube channel data");
|
|
|
|
}
|
2019-11-16 00:00:13 +01:00
|
|
|
|
|
|
|
loadSubtitles();
|
2018-10-11 21:10:22 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
private void setInitialData(String responseBody) throws ExtractionException {
|
|
|
|
try {
|
|
|
|
json = JsonParser.object().from(responseBody);
|
|
|
|
} catch (JsonParserException e) {
|
|
|
|
throw new ExtractionException("Unable to extract peertube stream data", e);
|
|
|
|
}
|
|
|
|
if(null == json) throw new ExtractionException("Unable to extract peertube stream data");
|
2019-03-09 19:03:51 +01:00
|
|
|
PeertubeParsingHelper.validate(json);
|
2018-10-11 21:10:22 +02:00
|
|
|
}
|
2019-11-16 00:00:13 +01:00
|
|
|
|
|
|
|
private void loadSubtitles() {
|
|
|
|
if (subtitles.isEmpty()) {
|
|
|
|
try {
|
2019-11-19 22:38:17 +01:00
|
|
|
Response response = getDownloader().get(getUrl() + "/captions");
|
|
|
|
JsonObject captionsJson = JsonParser.object().from(response.responseBody());
|
2019-11-16 00:00:13 +01:00
|
|
|
JsonArray captions = JsonUtils.getArray(captionsJson, "data");
|
|
|
|
for(Object c: captions) {
|
|
|
|
if(c instanceof JsonObject) {
|
|
|
|
JsonObject caption = (JsonObject)c;
|
|
|
|
String url = ServiceList.PeerTube.getBaseUrl() + JsonUtils.getString(caption, "captionPath");
|
|
|
|
String languageCode = JsonUtils.getString(caption, "language.id");
|
|
|
|
String ext = url.substring(url.lastIndexOf(".") + 1);
|
|
|
|
MediaFormat fmt = MediaFormat.getFromSuffix(ext);
|
|
|
|
if(fmt != null && languageCode != null) subtitles.add(new SubtitlesStream(fmt, languageCode, url, false));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} catch (Exception e) {
|
|
|
|
// ignore all exceptions
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2018-10-11 21:10:22 +02:00
|
|
|
|
|
|
|
@Override
|
|
|
|
public String getName() throws ParsingException {
|
|
|
|
return JsonUtils.getString(json, "name");
|
|
|
|
}
|
|
|
|
|
2019-11-15 22:12:39 +01:00
|
|
|
@Override
|
|
|
|
public String getOriginalUrl() throws ParsingException {
|
|
|
|
return ServiceList.PeerTube.getBaseUrl() + "/videos/watch/" + getId();
|
|
|
|
}
|
|
|
|
|
2018-10-11 21:10:22 +02:00
|
|
|
}
|