Merge pull request #479 from vkay94/stream-segments
Extract stream segments for YouTube
This commit is contained in:
commit
22a415156f
|
@ -16,6 +16,7 @@ import org.schabi.newpipe.extractor.stream.AudioStream;
|
|||
import org.schabi.newpipe.extractor.stream.Description;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.stream.StreamSegment;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
|
||||
import org.schabi.newpipe.extractor.stream.VideoStream;
|
||||
|
@ -294,4 +295,10 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
|
|||
public String getSupportInfo() {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public List<StreamSegment> getStreamSegments() {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -23,6 +23,7 @@ import org.schabi.newpipe.extractor.stream.Description;
|
|||
import org.schabi.newpipe.extractor.stream.Stream;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.stream.StreamSegment;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
|
||||
import org.schabi.newpipe.extractor.stream.VideoStream;
|
||||
|
@ -302,6 +303,12 @@ public class PeertubeStreamExtractor extends StreamExtractor {
|
|||
}
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public List<StreamSegment> getStreamSegments() {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
private String getRelatedStreamsUrl(final List<String> tags) throws UnsupportedEncodingException {
|
||||
final String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT;
|
||||
final StringBuilder params = new StringBuilder();
|
||||
|
|
|
@ -20,6 +20,7 @@ import org.schabi.newpipe.extractor.stream.AudioStream;
|
|||
import org.schabi.newpipe.extractor.stream.Description;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.stream.StreamSegment;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
|
||||
import org.schabi.newpipe.extractor.stream.VideoStream;
|
||||
|
@ -320,4 +321,10 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
|
|||
public String getSupportInfo() {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public List<StreamSegment> getStreamSegments() {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -35,6 +35,7 @@ import org.schabi.newpipe.extractor.stream.Stream;
|
|||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
||||
import org.schabi.newpipe.extractor.stream.StreamSegment;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
|
||||
import org.schabi.newpipe.extractor.stream.VideoStream;
|
||||
|
@ -1062,4 +1063,59 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
public String getSupportInfo() {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public List<StreamSegment> getStreamSegments() throws ParsingException {
|
||||
final ArrayList<StreamSegment> segments = new ArrayList<>();
|
||||
if (initialData.has("engagementPanels")) {
|
||||
final JsonArray panels = initialData.getArray("engagementPanels");
|
||||
JsonArray segmentsArray = null;
|
||||
|
||||
// Search for correct panel containing the data
|
||||
for (int i = 0; i < panels.size(); i++) {
|
||||
if (panels.getObject(i).getObject("engagementPanelSectionListRenderer")
|
||||
.getString("panelIdentifier").equals("engagement-panel-macro-markers")) {
|
||||
segmentsArray = panels.getObject(i).getObject("engagementPanelSectionListRenderer")
|
||||
.getObject("content").getObject("macroMarkersListRenderer").getArray("contents");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (segmentsArray != null) {
|
||||
final long duration = getLength();
|
||||
for (final Object object : segmentsArray) {
|
||||
final JsonObject segmentJson = ((JsonObject) object).getObject("macroMarkersListItemRenderer");
|
||||
|
||||
final int startTimeSeconds = segmentJson.getObject("onTap").getObject("watchEndpoint")
|
||||
.getInt("startTimeSeconds", -1);
|
||||
|
||||
if (startTimeSeconds == -1) {
|
||||
throw new ParsingException("Could not get stream segment start time.");
|
||||
}
|
||||
if (startTimeSeconds > duration) {
|
||||
break;
|
||||
}
|
||||
|
||||
final String title = getTextFromObject(segmentJson.getObject("title"));
|
||||
if (isNullOrEmpty(title)) {
|
||||
throw new ParsingException("Could not get stream segment title.");
|
||||
}
|
||||
|
||||
final StreamSegment segment = new StreamSegment(title, startTimeSeconds);
|
||||
segment.setUrl(getUrl() + "?t=" + startTimeSeconds);
|
||||
if (segmentJson.has("thumbnail")) {
|
||||
final JsonArray previewsArray = segmentJson.getObject("thumbnail").getArray("thumbnails");
|
||||
if (!previewsArray.isEmpty()) {
|
||||
// Assume that the thumbnail with the highest resolution is at the last position
|
||||
final String url = previewsArray.getObject(previewsArray.size() - 1).getString("url");
|
||||
segment.setPreviewUrl(fixThumbnailUrl(url));
|
||||
}
|
||||
}
|
||||
segments.add(segment);
|
||||
}
|
||||
}
|
||||
}
|
||||
return segments;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -476,4 +476,14 @@ public abstract class StreamExtractor extends Extractor {
|
|||
*/
|
||||
@Nonnull
|
||||
public abstract String getSupportInfo() throws ParsingException;
|
||||
|
||||
/**
|
||||
* The list of stream segments by timestamps for the stream.
|
||||
* If the segment list is not available you can simply return an empty list.
|
||||
*
|
||||
* @return The list of segments of the stream or an empty list.
|
||||
* @throws ParsingException
|
||||
*/
|
||||
@Nonnull
|
||||
public abstract List<StreamSegment> getStreamSegments() throws ParsingException;
|
||||
}
|
||||
|
|
|
@ -324,6 +324,11 @@ public class StreamInfo extends Info {
|
|||
} catch (Exception e) {
|
||||
streamInfo.addError(e);
|
||||
}
|
||||
try {
|
||||
streamInfo.setStreamSegments(extractor.getStreamSegments());
|
||||
} catch (Exception e) {
|
||||
streamInfo.addError(e);
|
||||
}
|
||||
|
||||
streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor));
|
||||
|
||||
|
@ -373,6 +378,7 @@ public class StreamInfo extends Info {
|
|||
private String support = "";
|
||||
private Locale language = null;
|
||||
private List<String> tags = new ArrayList<>();
|
||||
private List<StreamSegment> streamSegments = new ArrayList<>();
|
||||
|
||||
/**
|
||||
* Get the stream type
|
||||
|
@ -670,4 +676,12 @@ public class StreamInfo extends Info {
|
|||
public String getSupportInfo() {
|
||||
return this.support;
|
||||
}
|
||||
|
||||
public List<StreamSegment> getStreamSegments() {
|
||||
return streamSegments;
|
||||
}
|
||||
|
||||
public void setStreamSegments(List<StreamSegment> streamSegments) {
|
||||
this.streamSegments = streamSegments;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,69 @@
|
|||
package org.schabi.newpipe.extractor.stream;
|
||||
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.io.Serializable;
|
||||
|
||||
public class StreamSegment implements Serializable {
|
||||
/**
|
||||
* Title of this segment
|
||||
*/
|
||||
private String title;
|
||||
|
||||
/**
|
||||
* Timestamp of the starting point in seconds
|
||||
*/
|
||||
private int startTimeSeconds;
|
||||
|
||||
/**
|
||||
* Direct url to this segment. This can be null if the service doesn't provide such function.
|
||||
*/
|
||||
@Nullable
|
||||
public String url;
|
||||
|
||||
/**
|
||||
* Preview url for this segment. This can be null if the service doesn't provide such function
|
||||
* or there is no resource found.
|
||||
*/
|
||||
@Nullable
|
||||
private String previewUrl = null;
|
||||
|
||||
public StreamSegment(String title, int startTimeSeconds) {
|
||||
this.title = title;
|
||||
this.startTimeSeconds = startTimeSeconds;
|
||||
}
|
||||
|
||||
public String getTitle() {
|
||||
return title;
|
||||
}
|
||||
|
||||
public void setTitle(final String title) {
|
||||
this.title = title;
|
||||
}
|
||||
|
||||
public int getStartTimeSeconds() {
|
||||
return startTimeSeconds;
|
||||
}
|
||||
|
||||
public void setStartTimeSeconds(final int startTimeSeconds) {
|
||||
this.startTimeSeconds = startTimeSeconds;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
public String getUrl() {
|
||||
return url;
|
||||
}
|
||||
|
||||
public void setUrl(@Nullable final String url) {
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
public String getPreviewUrl() {
|
||||
return previewUrl;
|
||||
}
|
||||
|
||||
public void setPreviewUrl(@Nullable final String previewUrl) {
|
||||
this.previewUrl = previewUrl;
|
||||
}
|
||||
}
|
|
@ -66,6 +66,7 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
|
|||
public Locale expectedLanguageInfo() { return null; } // default: no language info available
|
||||
public List<String> expectedTags() { return Collections.emptyList(); } // default: no tags
|
||||
public String expectedSupportInfo() { return ""; } // default: no support info available
|
||||
public int expectedStreamSegmentsCount() { return -1; } // return 0 or greater to test (default is -1 to ignore)
|
||||
|
||||
@Test
|
||||
@Override
|
||||
|
@ -379,4 +380,11 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
|
|||
public void testSupportInfo() throws Exception {
|
||||
assertEquals(expectedSupportInfo(), extractor().getSupportInfo());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testStreamSegmentsCount() throws Exception {
|
||||
if (expectedStreamSegmentsCount() >= 0) {
|
||||
assertEquals(expectedStreamSegmentsCount(), extractor().getStreamSegments().size());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -57,6 +57,7 @@ public class MediaCCCStreamExtractorTest {
|
|||
@Override public boolean expectedHasSubtitles() { return false; }
|
||||
@Override public boolean expectedHasFrames() { return false; }
|
||||
@Override public List<String> expectedTags() { return Arrays.asList("gpn18", "105"); }
|
||||
@Override public int expectedStreamSegmentsCount() { return 0; }
|
||||
|
||||
@Override
|
||||
@Test
|
||||
|
|
|
@ -92,6 +92,7 @@ public class PeertubeStreamExtractorTest {
|
|||
@Override public String expectedLicence() { return "Attribution - Share Alike"; }
|
||||
@Override public Locale expectedLanguageInfo() { return Locale.forLanguageTag("en"); }
|
||||
@Override public List<String> expectedTags() { return Arrays.asList("framasoft", "peertube"); }
|
||||
@Override public int expectedStreamSegmentsCount() { return 0; }
|
||||
}
|
||||
|
||||
public static class AgeRestricted extends DefaultStreamExtractorTest {
|
||||
|
|
|
@ -55,6 +55,7 @@ public class SoundcloudStreamExtractorTest {
|
|||
@Override public boolean expectedHasVideoStreams() { return false; }
|
||||
@Override public boolean expectedHasSubtitles() { return false; }
|
||||
@Override public boolean expectedHasFrames() { return false; }
|
||||
@Override public int expectedStreamSegmentsCount() { return 0; }
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -9,6 +9,7 @@ import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
|
|||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.stream.StreamSegment;
|
||||
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
@ -16,7 +17,7 @@ import java.util.List;
|
|||
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
import static org.junit.Assert.fail;
|
||||
import static org.junit.Assert.*;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
|
||||
|
||||
/*
|
||||
|
@ -96,6 +97,7 @@ public class YoutubeStreamExtractorDefaultTest {
|
|||
@Nullable @Override public String expectedTextualUploadDate() { return "2019-08-24"; }
|
||||
@Override public long expectedLikeCountAtLeast() { return 5212900; }
|
||||
@Override public long expectedDislikeCountAtLeast() { return 30600; }
|
||||
@Override public int expectedStreamSegmentsCount() { return 0; }
|
||||
}
|
||||
|
||||
public static class DescriptionTestUnboxing extends DefaultStreamExtractorTest {
|
||||
|
@ -166,4 +168,94 @@ public class YoutubeStreamExtractorDefaultTest {
|
|||
@Override public long expectedLikeCountAtLeast() { return -1; }
|
||||
@Override public long expectedDislikeCountAtLeast() { return -1; }
|
||||
}
|
||||
|
||||
public static class StreamSegmentsTestOstCollection extends DefaultStreamExtractorTest {
|
||||
// StreamSegment example with single macro-makers panel
|
||||
private static final String ID = "2RYrHwnLHw0";
|
||||
private static final String URL = BASE_URL + ID;
|
||||
private static StreamExtractor extractor;
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(DownloaderTestImpl.getInstance());
|
||||
extractor = YouTube.getStreamExtractor(URL);
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
@Override public StreamExtractor extractor() { return extractor; }
|
||||
@Override public StreamingService expectedService() { return YouTube; }
|
||||
@Override public String expectedName() { return "1 Hour - Most Epic Anime Mix - Battle Anime OST"; }
|
||||
@Override public String expectedId() { return ID; }
|
||||
@Override public String expectedUrlContains() { return BASE_URL + ID; }
|
||||
@Override public String expectedOriginalUrlContains() { return URL; }
|
||||
|
||||
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
|
||||
@Override public String expectedUploaderName() { return "MathCaires"; }
|
||||
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UChFoHg6IT18SCqiwCp_KY7Q"; }
|
||||
@Override public List<String> expectedDescriptionContains() {
|
||||
return Arrays.asList("soundtracks", "9:49", "YouSeeBIGGIRLTT");
|
||||
}
|
||||
@Override public long expectedLength() { return 3889; }
|
||||
@Override public long expectedViewCountAtLeast() { return 2463261; }
|
||||
@Nullable @Override public String expectedUploadDate() { return "2019-06-26 00:00:00.000"; }
|
||||
@Nullable @Override public String expectedTextualUploadDate() { return "2019-06-26"; }
|
||||
@Override public long expectedLikeCountAtLeast() { return 32100; }
|
||||
@Override public long expectedDislikeCountAtLeast() { return 750; }
|
||||
@Override public boolean expectedHasSubtitles() { return false; }
|
||||
|
||||
@Override public int expectedStreamSegmentsCount() { return 17; }
|
||||
@Test
|
||||
public void testStreamSegment() throws Exception {
|
||||
final StreamSegment segment = extractor.getStreamSegments().get(3);
|
||||
assertEquals(589, segment.getStartTimeSeconds());
|
||||
assertEquals("Attack on Titan S2 - YouSeeBIGGIRLTT", segment.getTitle());
|
||||
assertEquals(BASE_URL + ID + "?t=589", segment.getUrl());
|
||||
assertNotNull(segment.getPreviewUrl());
|
||||
}
|
||||
}
|
||||
|
||||
public static class StreamSegmentsTestMaiLab extends DefaultStreamExtractorTest {
|
||||
// StreamSegment example with macro-makers panel and transcription panel
|
||||
private static final String ID = "ud9d5cMDP_0";
|
||||
private static final String URL = BASE_URL + ID;
|
||||
private static StreamExtractor extractor;
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(DownloaderTestImpl.getInstance());
|
||||
extractor = YouTube.getStreamExtractor(URL);
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
@Override public StreamExtractor extractor() { return extractor; }
|
||||
@Override public StreamingService expectedService() { return YouTube; }
|
||||
@Override public String expectedName() { return "Vitamin D wissenschaftlich gepr\u00fcft"; }
|
||||
@Override public String expectedId() { return ID; }
|
||||
@Override public String expectedUrlContains() { return BASE_URL + ID; }
|
||||
@Override public String expectedOriginalUrlContains() { return URL; }
|
||||
|
||||
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
|
||||
@Override public String expectedUploaderName() { return "maiLab"; }
|
||||
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCyHDQ5C6z1NDmJ4g6SerW8g"; }
|
||||
@Override public List<String> expectedDescriptionContains() {
|
||||
return Arrays.asList("Vitamin", "2:44", "Was ist Vitamin D?");
|
||||
}
|
||||
@Override public long expectedLength() { return 1010; }
|
||||
@Override public long expectedViewCountAtLeast() { return 815500; }
|
||||
@Nullable @Override public String expectedUploadDate() { return "2020-11-18 00:00:00.000"; }
|
||||
@Nullable @Override public String expectedTextualUploadDate() { return "2020-11-18"; }
|
||||
@Override public long expectedLikeCountAtLeast() { return 48500; }
|
||||
@Override public long expectedDislikeCountAtLeast() { return 20000; }
|
||||
@Override public boolean expectedHasSubtitles() { return true; }
|
||||
|
||||
@Override public int expectedStreamSegmentsCount() { return 7; }
|
||||
@Test
|
||||
public void testStreamSegment() throws Exception {
|
||||
final StreamSegment segment = extractor.getStreamSegments().get(1);
|
||||
assertEquals(164, segment.getStartTimeSeconds());
|
||||
assertEquals("Was ist Vitamin D?", segment.getTitle());
|
||||
assertEquals(BASE_URL + ID + "?t=164", segment.getUrl());
|
||||
assertNotNull(segment.getPreviewUrl());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue