lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #239 from B0pol/peertube 

[PeerTube] added metadata, fix descriptions, fix thumbnail, fix upload date, fix age limit, update tests.
This commit is contained in:
Tobias Groza 2020-02-08 22:31:02 +01:00 committed by GitHub
parent 62b81c3607 0e33249c85
commit 3add704f5d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 565 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -21,7 +21,7 @@ To test changes quickly you can build the library locally. Using the local Maven
2. It's _recommended_ that you change the `version` of this library (e.g. `LOCAL_SNAPSHOT`).
3. Run gradle's `ìnstall` task to deploy this library to your local repository (using the wrapper, present in the root of this project: `./gradlew install`)
4. Change the dependency version used in your project to match the one you chose in step 2 (`implementation 'com.github.TeamNewPipe:NewPipeExtractor:LOCAL_SNAPSHOT'`)
> Tip for Android Studio users: After you make changes and run the `install` task, use the menu option `File → "Sync with File System"` to refresh the library in your project.
## Supported sites

42
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java
View File

@ -17,6 +17,7 @@ import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
public class MediaCCCStreamExtractor extends StreamExtractor {
@ -47,8 +48,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getDescription() throws ParsingException {
return data.getString("description");
public Description getDescription() throws ParsingException {
return new Description(data.getString("description"), Description.PLAIN_TEXT);
}
@Override
@ -225,4 +226,41 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
public String getOriginalUrl() throws ParsingException {
return data.getString("frontend_link");
}
@Override
public String getHost() throws ParsingException {
return "";
}
@Override
public String getPrivacy() throws ParsingException {
return "";
}
@Override
public String getCategory() throws ParsingException {
return "";
}
@Override
public String getLicence() throws ParsingException {
return "";
}
@Override
public Locale getLanguageInfo() throws ParsingException {
return null;
}
@Nonnull
@Override
public List<String> getTags() throws ParsingException {
return new ArrayList<>();
}
@Nonnull
@Override
public String getSupportInfo() throws ParsingException {
return "";
}
}

11
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java
View File

@ -4,6 +4,7 @@ import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.TimeZone;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
@ -12,21 +13,23 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import com.grack.nanojson.JsonObject;
public class PeertubeParsingHelper {
private PeertubeParsingHelper() {
}
public static void validate(JsonObject json) throws ContentNotAvailableException {
String error = json.getString("error");
if(!StringUtil.isBlank(error)) {
if (!StringUtil.isBlank(error)) {
throw new ContentNotAvailableException(error);
}
}
public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
Date date;
try {
date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse(textualUploadDate);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
date = sdf.parse(textualUploadDate);
} catch (ParseException e) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java
View File

@ -67,7 +67,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
try {
Document doc = Jsoup.parse(htmlText);
return doc.body().text();
}catch(Exception e) {
} catch(Exception e) {
return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", "");
}
}
@ -83,7 +83,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
String value;
try {
value = JsonUtils.getString(item, "account.avatar.path");
}catch(Exception e) {
} catch(Exception e) {
value = "/client/assets/images/default-avatar.png";
}
return baseUrl + value;
@ -91,7 +91,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
@Override
public String getAuthorName() throws ParsingException {
return JsonUtils.getString(item, "account.displayName");
return JsonUtils.getString(item, "account.name") + "@" + JsonUtils.getString(item, "account.host");
}
@Override

163
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java
View File

@ -6,10 +6,11 @@ import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
@ -20,34 +21,28 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import javax.annotation.Nonnull;
public class PeertubeStreamExtractor extends StreamExtractor {
private final String baseUrl;
private JsonObject json;
private List<SubtitlesStream> subtitles = new ArrayList<>();
private final String baseUrl;
public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler) throws ParsingException {
super(service, linkHandler);
this.baseUrl = getBaseUrl();
}
@Override
public String getTextualUploadDate() throws ParsingException {
return JsonUtils.getString(json, "publishedAt");
@ -63,24 +58,42 @@ public class PeertubeStreamExtractor extends StreamExtractor {
return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate));
}
@Override
public String getThumbnailUrl() throws ParsingException {
return baseUrl + JsonUtils.getString(json, "thumbnailPath");
return baseUrl + JsonUtils.getString(json, "previewPath");
}
@Override
public String getDescription() throws ParsingException {
public Description getDescription() throws ParsingException {
String text;
try {
return JsonUtils.getString(json, "description");
}catch(ParsingException e) {
return "No description";
text = JsonUtils.getString(json, "description");
} catch (ParsingException e) {
return Description.emptyDescription;
}
if (text.length() == 250 && text.substring(247).equals("...")) {
//if description is shortened, get full description
Downloader dl = NewPipe.getDownloader();
try {
Response response = dl.get(getUrl() + "/description");
JsonObject jsonObject = JsonParser.object().from(response.responseBody());
text = JsonUtils.getString(jsonObject, "description");
} catch (ReCaptchaException | IOException | JsonParserException e) {
e.printStackTrace();
}
}
return new Description(text, Description.MARKDOWN);
}
@Override
public int getAgeLimit() throws ParsingException {
return NO_AGE_LIMIT;
boolean isNSFW = JsonUtils.getBoolean(json, "nsfw");
if (isNSFW) {
return 18;
} else {
return NO_AGE_LIMIT;
}
}
@Override
@ -130,7 +143,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
String value;
try {
value = JsonUtils.getString(json, "account.avatar.path");
}catch(Exception e) {
} catch (Exception e) {
value = "/client/assets/images/default-avatar.png";
}
return baseUrl + value;
@ -157,8 +170,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
List<VideoStream> videoStreams = new ArrayList<>();
try {
JsonArray streams = json.getArray("files", new JsonArray());
for(Object s: streams) {
if(!(s instanceof JsonObject)) continue;
for (Object s : streams) {
if (!(s instanceof JsonObject)) continue;
JsonObject stream = (JsonObject) s;
String url = JsonUtils.getString(stream, "fileUrl");
String torrentUrl = JsonUtils.getString(stream, "torrentUrl");
@ -192,8 +205,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
@Override
public List<SubtitlesStream> getSubtitles(final MediaFormat format) throws IOException, ExtractionException {
List<SubtitlesStream> filteredSubs = new ArrayList<>();
for(SubtitlesStream sub: subtitles) {
if(sub.getFormat() == format) {
for (SubtitlesStream sub : subtitles) {
if (sub.getFormat() == format) {
filteredSubs.add(sub);
}
}
@ -215,29 +228,40 @@ public class PeertubeStreamExtractor extends StreamExtractor {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
List<String> tags = getTags();
String apiUrl = null;
if(!tags.isEmpty()) {
if (!tags.isEmpty()) {
apiUrl = getRelatedStreamsUrl(tags);
}else {
} else {
apiUrl = getUploaderUrl() + "/videos?start=0&count=8";
}
if(!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl);
if (!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl);
return collector;
}
private List<String> getTags(){
@Override
public List<String> getTags() {
try {
return (List) JsonUtils.getArray(json, "tags");
} catch (Exception e) {
return Collections.emptyList();
}
}
@Nonnull
@Override
public String getSupportInfo() throws ParsingException {
try {
return JsonUtils.getString(json, "support");
} catch (ParsingException e) {
return "";
}
}
private String getRelatedStreamsUrl(List<String> tags) throws UnsupportedEncodingException {
String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT;
StringBuilder params = new StringBuilder();
params.append("start=0&count=8&sort=-createdAt");
for(String tag : tags) {
for (String tag : tags) {
params.append("&tagsOneOf=");
params.append(URLEncoder.encode(tag, "UTF-8"));
}
@ -247,38 +271,38 @@ public class PeertubeStreamExtractor extends StreamExtractor {
private void getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, IOException, ParsingException {
Response response = getDownloader().get(apiUrl);
JsonObject relatedVideosJson = null;
if(null != response && !StringUtil.isBlank(response.responseBody())) {
if (null != response && !StringUtil.isBlank(response.responseBody())) {
try {
relatedVideosJson = JsonParser.object().from(response.responseBody());
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json data for related videos", e);
}
}
if(relatedVideosJson != null) {
if (relatedVideosJson != null) {
collectStreamsFrom(collector, relatedVideosJson);
}
}
private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json) throws ParsingException {
JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
}catch(Exception e) {
} catch (Exception e) {
throw new ParsingException("unable to extract related videos", e);
}
for(Object c: contents) {
if(c instanceof JsonObject) {
for (Object c : contents) {
if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
//do not add the same stream in related streams
if(!extractor.getUrl().equals(getUrl())) collector.commit(extractor);
if (!extractor.getUrl().equals(getUrl())) collector.commit(extractor);
}
}
}
@Override
public String getErrorMessage() {
@ -288,12 +312,12 @@ public class PeertubeStreamExtractor extends StreamExtractor {
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
Response response = downloader.get(getUrl());
if(null != response && null != response.responseBody()) {
if (null != response && null != response.responseBody()) {
setInitialData(response.responseBody());
}else {
} else {
throw new ExtractionException("Unable to extract peertube channel data");
}
loadSubtitles();
}
@ -303,24 +327,25 @@ public class PeertubeStreamExtractor extends StreamExtractor {
} catch (JsonParserException e) {
throw new ExtractionException("Unable to extract peertube stream data", e);
}
if(null == json) throw new ExtractionException("Unable to extract peertube stream data");
if (null == json) throw new ExtractionException("Unable to extract peertube stream data");
PeertubeParsingHelper.validate(json);
}
private void loadSubtitles() {
if (subtitles.isEmpty()) {
try {
Response response = getDownloader().get(getUrl() + "/captions");
Response response = getDownloader().get(getUrl() + "/captions");
JsonObject captionsJson = JsonParser.object().from(response.responseBody());
JsonArray captions = JsonUtils.getArray(captionsJson, "data");
for(Object c: captions) {
if(c instanceof JsonObject) {
JsonObject caption = (JsonObject)c;
for (Object c : captions) {
if (c instanceof JsonObject) {
JsonObject caption = (JsonObject) c;
String url = baseUrl + JsonUtils.getString(caption, "captionPath");
String languageCode = JsonUtils.getString(caption, "language.id");
String ext = url.substring(url.lastIndexOf(".") + 1);
MediaFormat fmt = MediaFormat.getFromSuffix(ext);
if(fmt != null && languageCode != null) subtitles.add(new SubtitlesStream(fmt, languageCode, url, false));
if (fmt != null && languageCode != null)
subtitles.add(new SubtitlesStream(fmt, languageCode, url, false));
}
}
} catch (Exception e) {
@ -339,4 +364,32 @@ public class PeertubeStreamExtractor extends StreamExtractor {
return baseUrl + "/videos/watch/" + getId();
}
@Override
public String getHost() throws ParsingException {
return JsonUtils.getString(json, "account.host");
}
@Override
public String getPrivacy() throws ParsingException {
return JsonUtils.getString(json, "privacy.label");
}
@Override
public String getCategory() throws ParsingException {
return JsonUtils.getString(json, "category.label");
}
@Override
public String getLicence() throws ParsingException {
return JsonUtils.getString(json, "licence.label");
}
@Override
public Locale getLanguageInfo() throws ParsingException {
try {
return new Locale(JsonUtils.getString(json, "language.id"));
} catch (ParsingException e) {
return null;
}
}
}

43
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java
View File

@ -20,6 +20,7 @@ import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
public class SoundcloudStreamExtractor extends StreamExtractor {
private JsonObject track;
@ -73,10 +74,9 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
return artworkUrlBetterResolution;
}
@Nonnull
@Override
public String getDescription() {
return track.getString("description");
public Description getDescription() {
return new Description(track.getString("description"), Description.PLAIN_TEXT);
}
@Override
@ -254,4 +254,41 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
public String getErrorMessage() {
return null;
}
@Override
public String getHost() throws ParsingException {
return "";
}
@Override
public String getPrivacy() throws ParsingException {
return "";
}
@Override
public String getCategory() throws ParsingException {
return "";
}
@Override
public String getLicence() throws ParsingException {
return "";
}
@Override
public Locale getLanguageInfo() throws ParsingException {
return null;
}
@Nonnull
@Override
public List<String> getTags() throws ParsingException {
return new ArrayList<>();
}
@Nonnull
@Override
public String getSupportInfo() throws ParsingException {
return "";
}
}

43
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -180,15 +180,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getDescription() throws ParsingException {
public Description getDescription() throws ParsingException {
assertPageFetched();
try {
// first try to get html-formatted description
return parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html());
return new Description(parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html()), Description.HTML);
} catch (Exception e) {
try {
// fallback to raw non-html description
return playerResponse.getObject("videoDetails").getString("shortDescription");
return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT);
} catch (Exception ignored) {
throw new ParsingException("Could not get the description", e);
}
@ -1134,4 +1134,41 @@ public class YoutubeStreamExtractor extends StreamExtractor {
throw new ExtractionException(e);
}
}
@Override
public String getHost() throws ParsingException {
return "";
}
@Override
public String getPrivacy() throws ParsingException {
return "";
}
@Override
public String getCategory() throws ParsingException {
return "";
}
@Override
public String getLicence() throws ParsingException {
return "";
}
@Override
public Locale getLanguageInfo() throws ParsingException {
return null;
}
@Nonnull
@Override
public List<String> getTags() throws ParsingException {
return new ArrayList<>();
}
@Nonnull
@Override
public String getSupportInfo() throws ParsingException {
return "";
}
}

31
extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java Normal file
View File

@ -0,0 +1,31 @@
package org.schabi.newpipe.extractor.stream;
import java.io.Serializable;
public class Description implements Serializable {
public static final int HTML = 1;
public static final int MARKDOWN = 2;
public static final int PLAIN_TEXT = 3;
public static final Description emptyDescription = new Description("", PLAIN_TEXT);
private String content;
private int type;
public Description(String content, int type) {
this.type = type;
if (content == null) {
this.content = "";
} else {
this.content = content;
}
}
public String getContent() {
return content;
}
public int getType() {
return type;
}
}

78
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java
View File

@ -27,6 +27,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull;
@ -34,6 +35,7 @@ import javax.annotation.Nullable;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
/**
* Scrapes information from a video/audio streaming service (eg, YouTube).
@ -82,12 +84,12 @@ public abstract class StreamExtractor extends Extractor {
public abstract String getThumbnailUrl() throws ParsingException;
/**
* This is the stream description. On YouTube this is the video description. You can return simple HTML here.
* @return The description of the stream/video.
* This is the stream description.
* @return The description of the stream/video or Description.emptyDescription if the description is empty.
* @throws ParsingException
*/
@Nonnull
public abstract String getDescription() throws ParsingException;
public abstract Description getDescription() throws ParsingException;
/**
* Get the age limit.
@ -349,4 +351,74 @@ public abstract class StreamExtractor extends Extractor {
return 0;
}
}
/**
* The host of the stream (Eg. peertube.cpy.re).
* If the host is not available, or if the service doesn't use
* a federated system, but a centralised system,
* you can simply return an empty string.
* @return the host of the stream or an empty String.
* @throws ParsingException
*/
@Nonnull
public abstract String getHost() throws ParsingException;
/**
* The privacy of the stream (Eg. Public, Private, Unlisted…).
* If the privacy is not available you can simply return an empty string.
* @return the privacy of the stream or an empty String.
* @throws ParsingException
*/
@Nonnull
public abstract String getPrivacy() throws ParsingException;
/**
* The name of the category of the stream.
* If the category is not available you can simply return an empty string.
* @return the category of the stream or an empty String.
* @throws ParsingException
*/
@Nonnull
public abstract String getCategory() throws ParsingException;
/**
* The name of the licence of the stream.
* If the licence is not available you can simply return an empty string.
* @return the licence of the stream or an empty String.
* @throws ParsingException
*/
@Nonnull
public abstract String getLicence() throws ParsingException;
/**
* The locale language of the stream.
* If the language is not available you can simply return null.
* If the language is provided by a language code, you can return
* new Locale(language_code);
* @return the locale language of the stream or null.
* @throws ParsingException
*/
@Nullable
public abstract Locale getLanguageInfo() throws ParsingException;
/**
* The list of tags of the stream.
* If the tag list is not available you can simply return an empty list.
* @return the list of tags of the stream or an empty list.
* @throws ParsingException
*/
@Nonnull
public abstract List<String> getTags() throws ParsingException;
/**
* The support information of the stream.
* see: https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37
* (support button).
* If the support information are not available,
* you can simply return an empty String.
* @return the support information of the stream or an empty String.
* @throws ParsingException
*/
@Nonnull
public abstract String getSupportInfo() throws ParsingException;
}

107
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java
View File

@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
/*
* Created by Christian Schabesberger on 26.08.15.
@ -270,6 +271,43 @@ public class StreamInfo extends Info {
streamInfo.addError(e);
}
//additional info
try {
streamInfo.setHost(extractor.getHost());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setPrivacy(extractor.getPrivacy());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setCategory(extractor.getCategory());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setLicence(extractor.getLicence());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setLanguageInfo(extractor.getLanguageInfo());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setTags(extractor.getTags());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setSupportInfo(extractor.getSupportInfo());
} catch (Exception e) {
streamInfo.addError(e);
}
streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor));
return streamInfo;
@ -281,7 +319,7 @@ public class StreamInfo extends Info {
private DateWrapper uploadDate;
private long duration = -1;
private int ageLimit = -1;
private String description;
private Description description;
private long viewCount = -1;
private long likeCount = -1;
@ -308,6 +346,14 @@ public class StreamInfo extends Info {
private long startPosition = 0;
private List<SubtitlesStream> subtitles = new ArrayList<>();
private String host = "";
private String privacy = "";
private String category = "";
private String licence = "";
private String support = "";
private Locale language = null;
private List<String> tags = new ArrayList<>();
/**
* Get the stream type
*
@ -371,11 +417,11 @@ public class StreamInfo extends Info {
this.ageLimit = ageLimit;
}
public String getDescription() {
public Description getDescription() {
return description;
}
public void setDescription(String description) {
public void setDescription(Description description) {
this.description = description;
}
@ -533,4 +579,59 @@ public class StreamInfo extends Info {
this.subtitles = subtitles;
}
public String getHost() {
return this.host;
}
public void setHost(String str) {
this.host = str;
}
public String getPrivacy() {
return this.privacy;
}
public void setPrivacy(String str) {
this.privacy = str;
}
public String getCategory() {
return this.category;
}
public void setCategory(String cat) {
this.category = cat;
}
public String getLicence() {
return this.licence;
}
public void setLicence(String str) {
this.licence = str;
}
public Locale getLanguageInfo() {
return this.language;
}
public void setLanguageInfo(Locale lang) {
this.language = lang;
}
public List<String> getTags() {
return this.tags;
}
public void setTags(List<String> tags) {
this.tags = tags;
}
public void setSupportInfo(String support) {
this.support = support;
}
public String getSupportInfo() {
return this.support;
}
}

50
extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java
View File

@ -16,61 +16,71 @@ public class JsonUtils {
private JsonUtils() {
}
@Nonnull
public static Object getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
public static Object getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
List<String> keys = Arrays.asList(path.split("\\."));
object = getObject(object, keys.subList(0, keys.size() - 1));
if (null == object) throw new ParsingException("Unable to get " + path);
Object result = object.get(keys.get(keys.size() - 1));
if(null == result) throw new ParsingException("Unable to get " + path);
if (null == result) throw new ParsingException("Unable to get " + path);
return result;
}
@Nonnull
public static String getString(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
public static String getString(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
Object value = getValue(object, path);
if(value instanceof String) {
if (value instanceof String) {
return (String) value;
}else {
} else {
throw new ParsingException("Unable to get " + path);
}
}
@Nonnull
public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
public static Boolean getBoolean(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
Object value = getValue(object, path);
if(value instanceof Number) {
if (value instanceof Boolean) {
return (Boolean) value;
} else {
throw new ParsingException("Unable to get " + path);
}
}
@Nonnull
public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
Object value = getValue(object, path);
if (value instanceof Number) {
return (Number) value;
}else {
} else {
throw new ParsingException("Unable to get " + path);
}
}
@Nonnull
public static JsonObject getObject(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
public static JsonObject getObject(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
Object value = getValue(object, path);
if(value instanceof JsonObject) {
if (value instanceof JsonObject) {
return (JsonObject) value;
}else {
} else {
throw new ParsingException("Unable to get " + path);
}
}
@Nonnull
public static JsonArray getArray(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
public static JsonArray getArray(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException {
Object value = getValue(object, path);
if(value instanceof JsonArray) {
if (value instanceof JsonArray) {
return (JsonArray) value;
}else {
} else {
throw new ParsingException("Unable to get " + path);
}
}
@Nonnull
public static List<Object> getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException {
List<Object> result = new ArrayList<>();
for (int i = 0; i < array.size(); i++) {
JsonObject obj = array.getObject(i);

1
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java
View File

@ -185,5 +185,4 @@ public class Utils {
}
return uri.getProtocol() + "://" + uri.getAuthority();
}
}

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java
View File

@ -26,7 +26,7 @@ public class PeertubeCommentsExtractorTest {
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (PeertubeCommentsExtractor) PeerTube
.getCommentsExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
.getCommentsExtractor("https://framatube.org/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
}
@Test
@ -46,7 +46,7 @@ public class PeertubeCommentsExtractorTest {
@Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
boolean result = false;
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://peertube.mastodon.host/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
assertTrue("Comments".equals(commentsInfo.getName()));
result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");

77
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java
View File

@ -11,6 +11,8 @@ import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Locale;
import java.util.TimeZone;
import org.junit.BeforeClass;
import org.junit.Ignore;
@ -29,16 +31,28 @@ import org.schabi.newpipe.extractor.stream.StreamType;
*/
public class PeertubeStreamExtractorDefaultTest {
private static PeertubeStreamExtractor extractor;
private static final String expectedLargeDescription = "**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n*A decentralized video hosting network, based on free/libre software!*\r\n\r\n**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n*Directed by* Aryeom\r\n*Assistant* Jehan\r\n**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n\r\n**Sponsored by** [Framasoft](https://framasoft.org)\r\n\r\n**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n\r\n**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n\r\n**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/";
private static final String expectedSmallDescription = "https://www.kickstarter.com/projects/1587081065/nothing-to-hide-the-documentary";
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/afe5bf12-c58b-4efd-b56e-29c5a59e04bc");
PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "FramaTube"));
extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/9c9de5e8-0a1e-484a-b099-e80766180a6d");
extractor.fetchPage();
}
@Test
public void testGetUploadDate() throws ParsingException, ParseException {
final Calendar instance = Calendar.getInstance();
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
instance.setTime(sdf.parse("2018-10-01T10:52:46.396Z"));
assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
}
@Test
public void testGetInvalidTimeStamp() throws ParsingException {
assertTrue(extractor.getTimeStamp() + "",
@ -47,22 +61,37 @@ public class PeertubeStreamExtractorDefaultTest {
@Test
public void testGetTitle() throws ParsingException {
assertEquals(extractor.getName(), "Power Corrupts the Best");
assertEquals("What is PeerTube?", extractor.getName());
}
@Test
public void testGetDescription() throws ParsingException {
assertEquals(extractor.getDescription(), "A short reading from Bakunin, made for the group Audible Anarchist https://audibleanarchist.github.io/Webpage/");
public void testGetLargeDescription() throws ParsingException {
assertEquals(expectedLargeDescription, extractor.getDescription().getContent());
}
@Test
public void testGetEmptyDescription() throws Exception {
PeertubeStreamExtractor extractorEmpty = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/api/v1/videos/d5907aad-2252-4207-89ec-a4b687b9337d");
extractorEmpty.fetchPage();
assertEquals("", extractorEmpty.getDescription().getContent());
}
@Test
public void testGetSmallDescription() throws Exception {
PeerTube.setInstance(new PeertubeInstance("https://peertube.cpy.re", "PeerTube test server"));
PeertubeStreamExtractor extractorSmall = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.cpy.re/videos/watch/d2a5ec78-5f85-4090-8ec5-dc1102e022ea");
extractorSmall.fetchPage();
assertEquals(expectedSmallDescription, extractorSmall.getDescription().getContent());
}
@Test
public void testGetUploaderName() throws ParsingException {
assertEquals(extractor.getUploaderName(), "Rowsedower");
assertEquals("Framasoft", extractor.getUploaderName());
}
@Test
public void testGetLength() throws ParsingException {
assertEquals(extractor.getLength(), 269);
assertEquals(113, extractor.getLength());
}
@Test
@ -71,18 +100,10 @@ public class PeertubeStreamExtractorDefaultTest {
extractor.getViewCount() > 10);
}
@Test
public void testGetUploadDate() throws ParsingException, ParseException {
final Calendar instance = Calendar.getInstance();
instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-09-30T14:08:24.378Z"));
assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
}
@Test
public void testGetUploaderUrl() throws ParsingException {
assertIsSecureUrl(extractor.getUploaderUrl());
assertEquals("https://peertube.mastodon.host/api/v1/accounts/reddebrek@peertube.mastodon.host", extractor.getUploaderUrl());
assertEquals("https://framatube.org/api/v1/accounts/framasoft@framatube.org", extractor.getUploaderUrl());
}
@Test
@ -115,11 +136,31 @@ public class PeertubeStreamExtractorDefaultTest {
@Test
public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
assertTrue(extractor.getSubtitlesDefault().isEmpty());
assertFalse(extractor.getSubtitlesDefault().isEmpty());
}
@Test
public void testGetSubtitlesList() throws IOException, ExtractionException {
assertTrue(extractor.getSubtitlesDefault().isEmpty());
assertFalse(extractor.getSubtitlesDefault().isEmpty());
}
@Test
public void testGetAgeLimit() throws ExtractionException, IOException {
assertEquals(0, extractor.getAgeLimit());
PeertubeStreamExtractor ageLimit = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.co.uk/videos/watch/6762bb04-cad5-407b-81ee-c18eac4715a7");
ageLimit.fetchPage();
assertEquals(18, ageLimit.getAgeLimit());
}
@Test
public void testGetSupportInformation() throws ExtractionException, IOException {
PeertubeStreamExtractor supportInfoExtractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37");
supportInfoExtractor.fetchPage();
assertEquals("https://utip.io/chatsceptique", supportInfoExtractor.getSupportInfo());
}
@Test
public void testGetLanguageInformation() throws ParsingException {
assertEquals(new Locale("en"), extractor.getLanguageInfo());
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java
View File

@ -53,7 +53,7 @@ public class SoundcloudStreamExtractorDefaultTest {
@Test
public void testGetDescription() throws ParsingException {
assertEquals("The Perfect LUV Tape®️", extractor.getDescription());
assertEquals("The Perfect LUV Tape®️", extractor.getDescription().getContent());
}
@Test

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java
View File

@ -64,7 +64,7 @@ public class YoutubeStreamExtractorAgeRestrictedTest {
@Test
public void testGetDescription() throws ParsingException {
assertNotNull(extractor.getDescription());
assertFalse(extractor.getDescription().isEmpty());
assertFalse(extractor.getDescription().getContent().isEmpty());
}
@Test

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java
View File

@ -65,7 +65,7 @@ public class YoutubeStreamExtractorControversialTest {
@Test
public void testGetDescription() throws ParsingException {
assertNotNull(extractor.getDescription());
assertFalse(extractor.getDescription().isEmpty());
assertFalse(extractor.getDescription().getContent().isEmpty());
}
@Test

38
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java
View File

@ -83,13 +83,13 @@ public class YoutubeStreamExtractorDefaultTest {
@Test
public void testGetDescription() throws ParsingException {
assertNotNull(extractor.getDescription());
assertFalse(extractor.getDescription().isEmpty());
assertFalse(extractor.getDescription().getContent().isEmpty());
}
@Test
public void testGetFullLinksInDescription() throws ParsingException {
assertTrue(extractor.getDescription().contains("http://adele.com"));
assertFalse(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQi..."));
assertTrue(extractor.getDescription().getContent().contains("http://adele.com"));
assertFalse(extractor.getDescription().getContent().contains("http://smarturl.it/SubscribeAdele?IQi..."));
}
@Test
@ -215,18 +215,18 @@ public class YoutubeStreamExtractorDefaultTest {
@Test
public void testGetDescription() throws ParsingException {
assertNotNull(extractor.getDescription());
assertFalse(extractor.getDescription().isEmpty());
assertFalse(extractor.getDescription().getContent().isEmpty());
}
@Test
public void testGetFullLinksInDescription() throws ParsingException {
assertTrue(extractor.getDescription().contains("https://www.reddit.com/r/PewdiepieSubmissions/"));
assertTrue(extractor.getDescription().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng"));
assertTrue(extractor.getDescription().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/"));
assertTrue(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSubmissions/"));
assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng"));
assertTrue(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/"));
assertFalse(extractor.getDescription().contains("https://www.reddit.com/r/PewdiepieSub..."));
assertFalse(extractor.getDescription().contains("https://www.youtube.com/channel/UC3e8..."));
assertFalse(extractor.getDescription().contains("https://usa.clutchchairz.com/product/..."));
assertFalse(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSub..."));
assertFalse(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8..."));
assertFalse(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/..."));
}
}
@ -244,20 +244,20 @@ public class YoutubeStreamExtractorDefaultTest {
@Test
public void testGetDescription() throws ParsingException {
assertNotNull(extractor.getDescription());
assertFalse(extractor.getDescription().isEmpty());
assertFalse(extractor.getDescription().getContent().isEmpty());
}
@Test
public void testGetFullLinksInDescription() throws ParsingException {
assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertTrue(extractor.getDescription().contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertFalse(extractor.getDescription().contains("https://youtu.be/X7FLCHVXpsA?list=PL7..."));
assertFalse(extractor.getDescription().contains("https://youtu.be/Lqv6G0pDNnw?list=PL7..."));
assertFalse(extractor.getDescription().contains("https://youtu.be/XxaRBPyrnBU?list=PL7..."));
assertFalse(extractor.getDescription().contains("https://youtu.be/U-9tUEOFKNU?list=PL7..."));
assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/X7FLCHVXpsA?list=PL7..."));
assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/Lqv6G0pDNnw?list=PL7..."));
assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/XxaRBPyrnBU?list=PL7..."));
assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/U-9tUEOFKNU?list=PL7..."));
}
}

6
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java
View File

@ -44,13 +44,13 @@ public class YoutubeStreamExtractorLivestreamTest {
@Test
public void testGetDescription() throws ParsingException {
assertNotNull(extractor.getDescription());
assertFalse(extractor.getDescription().isEmpty());
assertFalse(extractor.getDescription().getContent().isEmpty());
}
@Test
public void testGetFullLinksInDescription() throws ParsingException {
assertTrue(extractor.getDescription().contains("https://www.instagram.com/nathalie.baraton/"));
assertFalse(extractor.getDescription().contains("https://www.instagram.com/nathalie.ba..."));
assertTrue(extractor.getDescription().getContent().contains("https://www.instagram.com/nathalie.baraton/"));
assertFalse(extractor.getDescription().getContent().contains("https://www.instagram.com/nathalie.ba..."));
}
@Test