added metadata, fix descriptions, fix thumbnail, update tests

thumbnail: quality before: https://peertube.cpy.re/static/thumbnails/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
quality after: https://peertube.cpy.re/static/previews/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg

description: we were getting about the first 260 characters, we now get full description (with fallback to first 260 chars if the get request for full description fails)
test: updated tests to match description, also changed some test: it was assertEquals(extracted, expected), but the proper way to do it is assertEquals(expected, extracted)

metadata: got host, privacy (public, private, unlisted), licence, language, tags
This commit is contained in:
bopol 2020-01-19 12:45:52 +01:00
parent 2ee558fbe7
commit a281519e77
10 changed files with 295 additions and 22 deletions

View File

@ -21,7 +21,7 @@ To test changes quickly you can build the library locally. Using the local Maven
2. It's _recommended_ that you change the `version` of this library (e.g. `LOCAL_SNAPSHOT`).
3. Run gradle's `ìnstall` task to deploy this library to your local repository (using the wrapper, present in the root of this project: `./gradlew install`)
4. Change the dependency version used in your project to match the one you chose in step 2 (`implementation 'com.github.TeamNewPipe:NewPipeExtractor:LOCAL_SNAPSHOT'`)
> Tip for Android Studio users: After you make changes and run the `install` task, use the menu option `File → "Sync with File System"` to refresh the library in your project.
## Supported sites

View File

@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import javax.annotation.Nonnull;
import java.io.IOException;
@ -225,4 +226,35 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
public String getOriginalUrl() throws ParsingException {
return data.getString("frontend_link");
}
@Override
public String getHost() throws ParsingException {
return "";
}
@Override
public String getPrivacy() throws ParsingException {
return "";
}
@Override
public String getCategory() throws ParsingException {
return "";
}
@Override
public String getLicence() throws ParsingException {
return "";
}
@Override
public String getStreamInfoLanguage() throws ParsingException {
return "";
}
@Nonnull
@Override
public List<String> getTags() throws ParsingException {
return new ArrayList<>();
}
}

View File

@ -9,7 +9,7 @@ import java.util.List;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
@ -29,7 +29,6 @@ import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
@ -66,16 +65,25 @@ public class PeertubeStreamExtractor extends StreamExtractor {
@Override
public String getThumbnailUrl() throws ParsingException {
return baseUrl + JsonUtils.getString(json, "thumbnailPath");
return baseUrl + JsonUtils.getString(json, "previewPath");
}
@Override
public String getDescription() throws ParsingException {
String description = "";
Downloader dl = NewPipe.getDownloader();
try {
return JsonUtils.getString(json, "description");
}catch(ParsingException e) {
return "No description";
Response response = dl.get(getUrl() + "/description");
JsonObject jsonObject = JsonParser.object().from(response.responseBody());
description = JsonUtils.getString(jsonObject, "description");
} catch (ReCaptchaException | IOException | JsonParserException e) {
e.printStackTrace();
}
if (description.equals("")) {
//if the request above failed
description = JsonUtils.getString(json, "description");
}
return description;
}
@Override
@ -224,8 +232,9 @@ public class PeertubeStreamExtractor extends StreamExtractor {
if(!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl);
return collector;
}
private List<String> getTags(){
@Override
public List<String> getTags(){
try {
return (List) JsonUtils.getArray(json, "tags");
} catch (Exception e) {
@ -339,4 +348,29 @@ public class PeertubeStreamExtractor extends StreamExtractor {
return baseUrl + "/videos/watch/" + getId();
}
//TODO: change privacy, category, licence by getting ID, therefore we will be able to translate it
@Override
public String getHost() throws ParsingException {
return JsonUtils.getString(json, "account.host");
}
@Override
public String getPrivacy() throws ParsingException {
return JsonUtils.getString(json, "privacy.label");
}
@Override
public String getCategory() throws ParsingException {
return JsonUtils.getString(json, "category.label");
}
@Override
public String getLicence() throws ParsingException {
return JsonUtils.getString(json, "licence.label");
}
@Override
public String getStreamInfoLanguage() throws ParsingException {
return JsonUtils.getString(json, "language.label");
}
}

View File

@ -254,4 +254,35 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
public String getErrorMessage() {
return null;
}
@Override
public String getHost() throws ParsingException {
return "";
}
@Override
public String getPrivacy() throws ParsingException {
return "";
}
@Override
public String getCategory() throws ParsingException {
return "";
}
@Override
public String getLicence() throws ParsingException {
return "";
}
@Override
public String getStreamInfoLanguage() throws ParsingException {
return "";
}
@Nonnull
@Override
public List<String> getTags() throws ParsingException {
return new ArrayList<>();
}
}

View File

@ -1113,4 +1113,35 @@ public class YoutubeStreamExtractor extends StreamExtractor {
throw new ExtractionException(e);
}
}
@Override
public String getHost() throws ParsingException {
return "";
}
@Override
public String getPrivacy() throws ParsingException {
return "";
}
@Override
public String getCategory() throws ParsingException {
return "";
}
@Override
public String getLicence() throws ParsingException {
return "";
}
@Override
public String getStreamInfoLanguage() throws ParsingException {
return "";
}
@Nonnull
@Override
public List<String> getTags() throws ParsingException {
return new ArrayList<>();
}
}

View File

@ -27,6 +27,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull;
@ -349,4 +350,60 @@ public abstract class StreamExtractor extends Extractor {
return 0;
}
}
/**
* The host of the stream (Eg. peertube.cpy.re).
* If the privacy is not available, or if the service doesn't use
* a federated system, but a centralised system,
* you can simply return an empty string.
* @return the host of the stream or an empty String.
* @throws ParsingException
*/
@Nonnull
public abstract String getHost() throws ParsingException;
/**
* The privacy of the stream (Eg. Public, Private, Unlisted).
* If the privacy is not available you can simply return an empty string.
* @return the privacy of the stream or an empty String.
* @throws ParsingException
*/
@Nonnull
public abstract String getPrivacy() throws ParsingException;
/**
* The name of the category of the stream.
* If the category is not available you can simply return an empty string.
* @return the category of the stream or an empty String.
* @throws ParsingException
*/
@Nonnull
public abstract String getCategory() throws ParsingException;
/**
* The name of the licence of the stream.
* If the licence is not available you can simply return an empty string.
* @return the licence of the stream or an empty String.
* @throws ParsingException
*/
@Nonnull
public abstract String getLicence() throws ParsingException;
/**
* The language of the stream.
* If the language is not available you can simply return an empty string.
* @return the licence of the stream or an empty String.
* @throws ParsingException
*/
@Nonnull
public abstract String getStreamInfoLanguage() throws ParsingException;
/**
* The list of tags of the stream.
* If the tag list is not available you can simply return an empty list.
* @return the list of tags of the stream or an empty list.
* @throws ParsingException
*/
@Nonnull
public abstract List<String> getTags() throws ParsingException;
}

View File

@ -270,6 +270,38 @@ public class StreamInfo extends Info {
streamInfo.addError(e);
}
//additional info
try {
streamInfo.setHost(extractor.getHost());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setPrivacy(extractor.getPrivacy());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setCategory(extractor.getCategory());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setLicence(extractor.getLicence());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setLanguage(extractor.getStreamInfoLanguage());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setTags(extractor.getTags());
} catch (Exception e) {
streamInfo.addError(e);
}
streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor));
return streamInfo;
@ -308,6 +340,13 @@ public class StreamInfo extends Info {
private long startPosition = 0;
private List<SubtitlesStream> subtitles = new ArrayList<>();
private String host = "";
private String privacy = "";
private String category = "";
private String licence = "";
private String language = "";
private List<String> tags = new ArrayList<>();
/**
* Get the stream type
*
@ -533,4 +572,53 @@ public class StreamInfo extends Info {
this.subtitles = subtitles;
}
public String getHost() {
return this.host;
}
public void setHost(String str) {
this.host = str;
}
public String getPrivacy() {
return this.privacy;
}
public void setPrivacy(String str) {
this.privacy = str;
}
public String getCategory() {
return this.category;
}
public void setCategory(String cat) {
this.category = cat;
}
public String getLicence() {
return this.licence;
}
public void setLicence(String str) {
this.licence = str;
}
public String getLanguage() {
return this.language;
}
public void setLanguage(String lang) {
this.language = lang;
}
public List<String> getTags() {
return this.tags;
}
public void setTags(List<String> tags) {
this.tags = tags;
}
}

View File

@ -185,5 +185,4 @@ public class Utils {
}
return uri.getProtocol() + "://" + uri.getAuthority();
}
}

View File

@ -26,7 +26,7 @@ public class PeertubeCommentsExtractorTest {
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (PeertubeCommentsExtractor) PeerTube
.getCommentsExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
.getCommentsExtractor("https://framatube.org/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
}
@Test
@ -46,7 +46,7 @@ public class PeertubeCommentsExtractorTest {
@Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
boolean result = false;
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://peertube.mastodon.host/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
assertTrue("Comments".equals(commentsInfo.getName()));
result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");

View File

@ -29,13 +29,14 @@ import org.schabi.newpipe.extractor.stream.StreamType;
*/
public class PeertubeStreamExtractorDefaultTest {
private static PeertubeStreamExtractor extractor;
private static final String expectedDescription = "**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n*A decentralized video hosting network, based on free/libre software!*\r\n\r\n**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n*Directed by* Aryeom\r\n*Assistant* Jehan\r\n**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n\r\n**Sponsored by** [Framasoft](https://framasoft.org)\r\n\r\n**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n\r\n**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n\r\n**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/";
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/afe5bf12-c58b-4efd-b56e-29c5a59e04bc");
PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "FramaTube"));
extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/9c9de5e8-0a1e-484a-b099-e80766180a6d");
extractor.fetchPage();
}
@ -47,22 +48,22 @@ public class PeertubeStreamExtractorDefaultTest {
@Test
public void testGetTitle() throws ParsingException {
assertEquals(extractor.getName(), "Power Corrupts the Best");
assertEquals("What is PeerTube?", extractor.getName());
}
@Test
public void testGetDescription() throws ParsingException {
assertEquals(extractor.getDescription(), "A short reading from Bakunin, made for the group Audible Anarchist https://audibleanarchist.github.io/Webpage/");
assertEquals(expectedDescription, extractor.getDescription());
}
@Test
public void testGetUploaderName() throws ParsingException {
assertEquals(extractor.getUploaderName(), "Rowsedower");
assertEquals("Framasoft", extractor.getUploaderName());
}
@Test
public void testGetLength() throws ParsingException {
assertEquals(extractor.getLength(), 269);
assertEquals(113, extractor.getLength());
}
@Test
@ -74,7 +75,7 @@ public class PeertubeStreamExtractorDefaultTest {
@Test
public void testGetUploadDate() throws ParsingException, ParseException {
final Calendar instance = Calendar.getInstance();
instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-09-30T14:08:24.378Z"));
instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-10-01T10:52:46.396Z"));
assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
}
@ -82,7 +83,7 @@ public class PeertubeStreamExtractorDefaultTest {
@Test
public void testGetUploaderUrl() throws ParsingException {
assertIsSecureUrl(extractor.getUploaderUrl());
assertEquals("https://peertube.mastodon.host/api/v1/accounts/reddebrek@peertube.mastodon.host", extractor.getUploaderUrl());
assertEquals("https://framatube.org/api/v1/accounts/framasoft@framatube.org", extractor.getUploaderUrl());
}
@Test
@ -115,11 +116,11 @@ public class PeertubeStreamExtractorDefaultTest {
@Test
public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
assertTrue(extractor.getSubtitlesDefault().isEmpty());
assertFalse(extractor.getSubtitlesDefault().isEmpty());
}
@Test
public void testGetSubtitlesList() throws IOException, ExtractionException {
assertTrue(extractor.getSubtitlesDefault().isEmpty());
assertFalse(extractor.getSubtitlesDefault().isEmpty());
}
}