lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #1061 from ChunkyProgrammer/develop/extract-yt-playlist-descrption 

Extract playlists description
This commit is contained in:
Tobi 2023-05-17 22:43:17 +02:00 committed by GitHub
parent a9ca5c49e4 c70bb83801
commit 3c036a9c03
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 166 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java
View File

@ -4,6 +4,7 @@ import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import javax.annotation.Nonnull;
@ -21,6 +22,9 @@ public abstract class PlaylistExtractor extends ListExtractor<StreamInfoItem> {
public abstract long getStreamCount() throws ParsingException;
@Nonnull
public abstract Description getDescription() throws ParsingException;
@Nonnull
public String getThumbnailUrl() throws ParsingException {
return "";

15
extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java
View File

@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
@ -102,6 +103,11 @@ public final class PlaylistInfo extends ListInfo<StreamInfoItem> {
} catch (final Exception e) {
info.addError(e);
}
try {
info.setDescription(extractor.getDescription());
} catch (final Exception e) {
info.addError(e);
}
try {
info.setThumbnailUrl(extractor.getThumbnailUrl());
} catch (final Exception e) {
@ -174,6 +180,7 @@ public final class PlaylistInfo extends ListInfo<StreamInfoItem> {
private String subChannelName;
private String subChannelAvatarUrl;
private long streamCount = 0;
private Description description;
private PlaylistType playlistType;
public String getThumbnailUrl() {
@ -248,6 +255,14 @@ public final class PlaylistInfo extends ListInfo<StreamInfoItem> {
this.streamCount = streamCount;
}
public Description getDescription() {
return description;
}
public void setDescription(final Description description) {
this.description = description;
}
public PlaylistType getPlaylistType() {
return playlistType;
}

10
extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItem.java
View File

@ -1,6 +1,7 @@
package org.schabi.newpipe.extractor.playlist;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.stream.Description;
import javax.annotation.Nullable;
@ -13,6 +14,7 @@ public class PlaylistInfoItem extends InfoItem {
* How many streams this playlist have
*/
private long streamCount = 0;
private Description description;
private PlaylistInfo.PlaylistType playlistType;
public PlaylistInfoItem(final int serviceId, final String url, final String name) {
@ -52,6 +54,14 @@ public class PlaylistInfoItem extends InfoItem {
this.streamCount = streamCount;
}
public Description getDescription() {
return description;
}
public void setDescription(final Description description) {
this.description = description;
}
public PlaylistInfo.PlaylistType getPlaylistType() {
return playlistType;
}

11
extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemExtractor.java
View File

@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.playlist;
import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.Description;
import javax.annotation.Nonnull;
@ -31,6 +32,16 @@ public interface PlaylistInfoItemExtractor extends InfoItemExtractor {
*/
long getStreamCount() throws ParsingException;
/**
* Get the description of the playlist if there is any.
* Otherwise, an {@link Description#EMPTY_DESCRIPTION EMPTY_DESCRIPTION} is returned.
* @return the playlist's description
*/
@Nonnull
default Description getDescription() throws ParsingException {
return Description.EMPTY_DESCRIPTION;
}
/**
* @return the type of this playlist, see {@link PlaylistInfo.PlaylistType} for a description
* of types. If not overridden always returns {@link PlaylistInfo.PlaylistType#NORMAL}.

5
extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemsCollector.java
View File

@ -41,6 +41,11 @@ public class PlaylistInfoItemsCollector
} catch (final Exception e) {
addError(e);
}
try {
resultItem.setDescription(extractor.getDescription());
} catch (final Exception e) {
addError(e);
}
try {
resultItem.setPlaylistType(extractor.getPlaylistType());
} catch (final Exception e) {

30
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java
View File

@ -11,6 +11,8 @@ import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -20,10 +22,12 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.BandcampPlaylistStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import java.io.IOException;
import java.util.Objects;
import javax.annotation.Nonnull;
@ -108,6 +112,32 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor {
return trackInfo.size();
}
@Nonnull
@Override
public Description getDescription() throws ParsingException {
final Element tInfo = document.getElementById("trackInfo");
if (tInfo == null) {
throw new ParsingException("Could not find trackInfo in document");
}
final Elements about = tInfo.getElementsByClass("tralbum-about");
final Elements credits = tInfo.getElementsByClass("tralbum-credits");
final Element license = document.getElementById("license");
if (about.isEmpty() && credits.isEmpty() && license == null) {
return Description.EMPTY_DESCRIPTION;
}
final StringBuilder sb = new StringBuilder();
if (!about.isEmpty()) {
sb.append(Objects.requireNonNull(about.first()).html());
}
if (!credits.isEmpty()) {
sb.append(Objects.requireNonNull(credits.first()).html());
}
if (license != null) {
sb.append(license.html());
}
return new Description(sb.toString(), Description.HTML);
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {

11
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java
View File

@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Utils;
@ -65,6 +66,16 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor {
return playlistInfo.getLong("videosLength");
}
@Nonnull
@Override
public Description getDescription() throws ParsingException {
final String description = playlistInfo.getString("description");
if (isNullOrEmpty(description)) {
return Description.EMPTY_DESCRIPTION;
}
return new Description(description, Description.PLAIN_TEXT);
}
@Nonnull
@Override
public String getSubChannelName() {

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistInfoItemExtractor.java
View File

@ -4,9 +4,12 @@ import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.Description;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
final JsonObject item;
@ -54,4 +57,14 @@ public class PeertubePlaylistInfoItemExtractor implements PlaylistInfoItemExtrac
public long getStreamCount() throws ParsingException {
return item.getInt("videosLength");
}
@Nonnull
@Override
public Description getDescription() throws ParsingException {
final String description = item.getString("description");
if (isNullOrEmpty(description)) {
return Description.EMPTY_DESCRIPTION;
}
return new Description(description, Description.PLAIN_TEXT);
}
}

11
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java
View File

@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
@ -118,6 +119,16 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
return playlist.getLong("track_count");
}
@Nonnull
@Override
public Description getDescription() throws ParsingException {
final String description = playlist.getString("description");
if (isNullOrEmpty(description)) {
return Description.EMPTY_DESCRIPTION;
}
return new Description(description, Description.PLAIN_TEXT);
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() {

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
View File

@ -31,6 +31,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
@ -169,6 +170,12 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
return ListExtractor.ITEM_COUNT_INFINITE;
}
@Nonnull
@Override
public Description getDescription() throws ParsingException {
return Description.EMPTY_DESCRIPTION;
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage()

12
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -26,6 +26,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Utils;
@ -294,6 +295,17 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
return ITEM_COUNT_UNKNOWN;
}
@Nonnull
@Override
public Description getDescription() throws ParsingException {
final String description = getTextFromObject(
getPlaylistInfo().getObject("description"),
true
);
return new Description(description, Description.HTML);
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {

12
extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampPlaylistExtractorTest.java
View File

@ -13,12 +13,14 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampPlaylistExtractor;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import java.io.IOException;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains;
import static org.schabi.newpipe.extractor.ServiceList.Bandcamp;
/**
@ -135,6 +137,16 @@ public class BandcampPlaylistExtractorTest {
assertEquals(5, extractor.getStreamCount());
}
@Test
public void testDescription() throws ParsingException {
final Description description = extractor.getDescription();
assertNotEquals(Description.EMPTY_DESCRIPTION, description);
assertContains("Artwork by Shona Radcliffe", description.getContent()); // about
assertContains("All tracks written, produced and recorded by Mac Benson",
description.getContent()); // credits
assertContains("all rights reserved", description.getContent()); // license
}
@Override
public void testUploaderVerified() throws Exception {
assertFalse(extractor.isUploaderVerified());

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java
View File

@ -60,6 +60,11 @@ public class PeertubePlaylistExtractorTest {
ExtractorAsserts.assertGreaterOrEqual(39, extractor.getStreamCount());
}
@Test
void testGetDescription() throws ParsingException {
ExtractorAsserts.assertContains("épisodes de Shocking", extractor.getDescription().getContent());
}
@Test
void testGetSubChannelUrl() {
assertEquals("https://skeptikon.fr/video-channels/metadechoc_channel", extractor.getSubChannelUrl());

20
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View File

@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertContains;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems;
@ -25,6 +26,7 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import java.io.IOException;
@ -163,6 +165,12 @@ public class YoutubePlaylistExtractorTest {
void getPlaylistType() throws ParsingException {
assertEquals(PlaylistInfo.PlaylistType.NORMAL, extractor.getPlaylistType());
}
@Test
public void testDescription() throws ParsingException {
final Description description = extractor.getDescription();
assertContains("pop songs list", description.getContent());
}
}
public static class HugePlaylist implements BasePlaylistExtractorTest {
@ -286,6 +294,12 @@ public class YoutubePlaylistExtractorTest {
void getPlaylistType() throws ParsingException {
assertEquals(PlaylistInfo.PlaylistType.NORMAL, extractor.getPlaylistType());
}
@Test
public void testDescription() throws ParsingException {
final Description description = extractor.getDescription();
assertContains("I Wanna Rock Super Gigantic Playlist", description.getContent());
}
}
public static class LearningPlaylist implements BasePlaylistExtractorTest {
@ -394,6 +408,12 @@ public class YoutubePlaylistExtractorTest {
void getPlaylistType() throws ParsingException {
assertEquals(PlaylistInfo.PlaylistType.NORMAL, extractor.getPlaylistType());
}
@Test
public void testDescription() throws ParsingException {
final Description description = extractor.getDescription();
assertContains("47 episodes", description.getContent());
}
}
public static class ContinuationsTests {