lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #479 from vkay94/stream-segments 

Extract stream segments for YouTube
This commit is contained in:
Tobias Groza 2020-12-19 11:12:22 +01:00 committed by GitHub
parent e8cc302d5f 2ba27b39af
commit 22a415156f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 274 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java
View File

@ -16,6 +16,7 @@ import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
@ -294,4 +295,10 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
public String getSupportInfo() {
return "";
}
@Nonnull
@Override
public List<StreamSegment> getStreamSegments() {
return Collections.emptyList();
}
}

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java
View File

@ -23,6 +23,7 @@ import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
@ -302,6 +303,12 @@ public class PeertubeStreamExtractor extends StreamExtractor {
}
}
@Nonnull
@Override
public List<StreamSegment> getStreamSegments() {
return Collections.emptyList();
}
private String getRelatedStreamsUrl(final List<String> tags) throws UnsupportedEncodingException {
final String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT;
final StringBuilder params = new StringBuilder();

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java
View File

@ -20,6 +20,7 @@ import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
@ -320,4 +321,10 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
public String getSupportInfo() {
return "";
}
@Nonnull
@Override
public List<StreamSegment> getStreamSegments() {
return Collections.emptyList();
}
}

56
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -35,6 +35,7 @@ import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
@ -1062,4 +1063,59 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public String getSupportInfo() {
return "";
}
@Nonnull
@Override
public List<StreamSegment> getStreamSegments() throws ParsingException {
final ArrayList<StreamSegment> segments = new ArrayList<>();
if (initialData.has("engagementPanels")) {
final JsonArray panels = initialData.getArray("engagementPanels");
JsonArray segmentsArray = null;
// Search for correct panel containing the data
for (int i = 0; i < panels.size(); i++) {
if (panels.getObject(i).getObject("engagementPanelSectionListRenderer")
.getString("panelIdentifier").equals("engagement-panel-macro-markers")) {
segmentsArray = panels.getObject(i).getObject("engagementPanelSectionListRenderer")
.getObject("content").getObject("macroMarkersListRenderer").getArray("contents");
break;
}
}
if (segmentsArray != null) {
final long duration = getLength();
for (final Object object : segmentsArray) {
final JsonObject segmentJson = ((JsonObject) object).getObject("macroMarkersListItemRenderer");
final int startTimeSeconds = segmentJson.getObject("onTap").getObject("watchEndpoint")
.getInt("startTimeSeconds", -1);
if (startTimeSeconds == -1) {
throw new ParsingException("Could not get stream segment start time.");
}
if (startTimeSeconds > duration) {
break;
}
final String title = getTextFromObject(segmentJson.getObject("title"));
if (isNullOrEmpty(title)) {
throw new ParsingException("Could not get stream segment title.");
}
final StreamSegment segment = new StreamSegment(title, startTimeSeconds);
segment.setUrl(getUrl() + "?t=" + startTimeSeconds);
if (segmentJson.has("thumbnail")) {
final JsonArray previewsArray = segmentJson.getObject("thumbnail").getArray("thumbnails");
if (!previewsArray.isEmpty()) {
// Assume that the thumbnail with the highest resolution is at the last position
final String url = previewsArray.getObject(previewsArray.size() - 1).getString("url");
segment.setPreviewUrl(fixThumbnailUrl(url));
}
}
segments.add(segment);
}
}
}
return segments;
}
}

10
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java
View File

@ -476,4 +476,14 @@ public abstract class StreamExtractor extends Extractor {
*/
@Nonnull
public abstract String getSupportInfo() throws ParsingException;
/**
* The list of stream segments by timestamps for the stream.
* If the segment list is not available you can simply return an empty list.
*
* @return The list of segments of the stream or an empty list.
* @throws ParsingException
*/
@Nonnull
public abstract List<StreamSegment> getStreamSegments() throws ParsingException;
}

14
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java
View File

@ -324,6 +324,11 @@ public class StreamInfo extends Info {
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setStreamSegments(extractor.getStreamSegments());
} catch (Exception e) {
streamInfo.addError(e);
}
streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor));
@ -373,6 +378,7 @@ public class StreamInfo extends Info {
private String support = "";
private Locale language = null;
private List<String> tags = new ArrayList<>();
private List<StreamSegment> streamSegments = new ArrayList<>();
/**
* Get the stream type
@ -670,4 +676,12 @@ public class StreamInfo extends Info {
public String getSupportInfo() {
return this.support;
}
public List<StreamSegment> getStreamSegments() {
return streamSegments;
}
public void setStreamSegments(List<StreamSegment> streamSegments) {
this.streamSegments = streamSegments;
}
}

69
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamSegment.java Normal file
View File

@ -0,0 +1,69 @@
package org.schabi.newpipe.extractor.stream;
import javax.annotation.Nullable;
import java.io.Serializable;
public class StreamSegment implements Serializable {
/**
* Title of this segment
*/
private String title;
/**
* Timestamp of the starting point in seconds
*/
private int startTimeSeconds;
/**
* Direct url to this segment. This can be null if the service doesn't provide such function.
*/
@Nullable
public String url;
/**
* Preview url for this segment. This can be null if the service doesn't provide such function
* or there is no resource found.
*/
@Nullable
private String previewUrl = null;
public StreamSegment(String title, int startTimeSeconds) {
this.title = title;
this.startTimeSeconds = startTimeSeconds;
}
public String getTitle() {
return title;
}
public void setTitle(final String title) {
this.title = title;
}
public int getStartTimeSeconds() {
return startTimeSeconds;
}
public void setStartTimeSeconds(final int startTimeSeconds) {
this.startTimeSeconds = startTimeSeconds;
}
@Nullable
public String getUrl() {
return url;
}
public void setUrl(@Nullable final String url) {
this.url = url;
}
@Nullable
public String getPreviewUrl() {
return previewUrl;
}
public void setPreviewUrl(@Nullable final String previewUrl) {
this.previewUrl = previewUrl;
}
}

8
extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java
View File

@ -66,6 +66,7 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
public Locale expectedLanguageInfo() { return null; } // default: no language info available
public List<String> expectedTags() { return Collections.emptyList(); } // default: no tags
public String expectedSupportInfo() { return ""; } // default: no support info available
public int expectedStreamSegmentsCount() { return -1; } // return 0 or greater to test (default is -1 to ignore)
@Test
@Override
@ -379,4 +380,11 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
public void testSupportInfo() throws Exception {
assertEquals(expectedSupportInfo(), extractor().getSupportInfo());
}
@Test
public void testStreamSegmentsCount() throws Exception {
if (expectedStreamSegmentsCount() >= 0) {
assertEquals(expectedStreamSegmentsCount(), extractor().getStreamSegments().size());
}
}
}

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java
View File

@ -57,6 +57,7 @@ public class MediaCCCStreamExtractorTest {
@Override public boolean expectedHasSubtitles() { return false; }
@Override public boolean expectedHasFrames() { return false; }
@Override public List<String> expectedTags() { return Arrays.asList("gpn18", "105"); }
@Override public int expectedStreamSegmentsCount() { return 0; }
@Override
@Test

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java
View File

@ -92,6 +92,7 @@ public class PeertubeStreamExtractorTest {
@Override public String expectedLicence() { return "Attribution - Share Alike"; }
@Override public Locale expectedLanguageInfo() { return Locale.forLanguageTag("en"); }
@Override public List<String> expectedTags() { return Arrays.asList("framasoft", "peertube"); }
@Override public int expectedStreamSegmentsCount() { return 0; }
}
public static class AgeRestricted extends DefaultStreamExtractorTest {

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java
View File

@ -55,6 +55,7 @@ public class SoundcloudStreamExtractorTest {
@Override public boolean expectedHasVideoStreams() { return false; }
@Override public boolean expectedHasSubtitles() { return false; }
@Override public boolean expectedHasFrames() { return false; }
@Override public int expectedStreamSegmentsCount() { return 0; }
}
}

94
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java
View File

@ -9,6 +9,7 @@ import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import java.util.Arrays;
@ -16,7 +17,7 @@ import java.util.List;
import javax.annotation.Nullable;
import static org.junit.Assert.fail;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
/*
@ -96,6 +97,7 @@ public class YoutubeStreamExtractorDefaultTest {
@Nullable @Override public String expectedTextualUploadDate() { return "2019-08-24"; }
@Override public long expectedLikeCountAtLeast() { return 5212900; }
@Override public long expectedDislikeCountAtLeast() { return 30600; }
@Override public int expectedStreamSegmentsCount() { return 0; }
}
public static class DescriptionTestUnboxing extends DefaultStreamExtractorTest {
@ -166,4 +168,94 @@ public class YoutubeStreamExtractorDefaultTest {
@Override public long expectedLikeCountAtLeast() { return -1; }
@Override public long expectedDislikeCountAtLeast() { return -1; }
}
public static class StreamSegmentsTestOstCollection extends DefaultStreamExtractorTest {
// StreamSegment example with single macro-makers panel
private static final String ID = "2RYrHwnLHw0";
private static final String URL = BASE_URL + ID;
private static StreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "1 Hour - Most Epic Anime Mix - Battle Anime OST"; }
@Override public String expectedId() { return ID; }
@Override public String expectedUrlContains() { return BASE_URL + ID; }
@Override public String expectedOriginalUrlContains() { return URL; }
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
@Override public String expectedUploaderName() { return "MathCaires"; }
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UChFoHg6IT18SCqiwCp_KY7Q"; }
@Override public List<String> expectedDescriptionContains() {
return Arrays.asList("soundtracks", "9:49", "YouSeeBIGGIRLTT");
}
@Override public long expectedLength() { return 3889; }
@Override public long expectedViewCountAtLeast() { return 2463261; }
@Nullable @Override public String expectedUploadDate() { return "2019-06-26 00:00:00.000"; }
@Nullable @Override public String expectedTextualUploadDate() { return "2019-06-26"; }
@Override public long expectedLikeCountAtLeast() { return 32100; }
@Override public long expectedDislikeCountAtLeast() { return 750; }
@Override public boolean expectedHasSubtitles() { return false; }
@Override public int expectedStreamSegmentsCount() { return 17; }
@Test
public void testStreamSegment() throws Exception {
final StreamSegment segment = extractor.getStreamSegments().get(3);
assertEquals(589, segment.getStartTimeSeconds());
assertEquals("Attack on Titan S2 - YouSeeBIGGIRLTT", segment.getTitle());
assertEquals(BASE_URL + ID + "?t=589", segment.getUrl());
assertNotNull(segment.getPreviewUrl());
}
}
public static class StreamSegmentsTestMaiLab extends DefaultStreamExtractorTest {
// StreamSegment example with macro-makers panel and transcription panel
private static final String ID = "ud9d5cMDP_0";
private static final String URL = BASE_URL + ID;
private static StreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "Vitamin D wissenschaftlich gepr\u00fcft"; }
@Override public String expectedId() { return ID; }
@Override public String expectedUrlContains() { return BASE_URL + ID; }
@Override public String expectedOriginalUrlContains() { return URL; }
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
@Override public String expectedUploaderName() { return "maiLab"; }
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCyHDQ5C6z1NDmJ4g6SerW8g"; }
@Override public List<String> expectedDescriptionContains() {
return Arrays.asList("Vitamin", "2:44", "Was ist Vitamin D?");
}
@Override public long expectedLength() { return 1010; }
@Override public long expectedViewCountAtLeast() { return 815500; }
@Nullable @Override public String expectedUploadDate() { return "2020-11-18 00:00:00.000"; }
@Nullable @Override public String expectedTextualUploadDate() { return "2020-11-18"; }
@Override public long expectedLikeCountAtLeast() { return 48500; }
@Override public long expectedDislikeCountAtLeast() { return 20000; }
@Override public boolean expectedHasSubtitles() { return true; }
@Override public int expectedStreamSegmentsCount() { return 7; }
@Test
public void testStreamSegment() throws Exception {
final StreamSegment segment = extractor.getStreamSegments().get(1);
assertEquals(164, segment.getStartTimeSeconds());
assertEquals("Was ist Vitamin D?", segment.getTitle());
assertEquals(BASE_URL + ID + "?t=164", segment.getUrl());
assertNotNull(segment.getPreviewUrl());
}
}
}