lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Refactor collectors 

* Define a common iterface
  * Use generic types
  * Remove some duplicated code
  * Simplify InfoItemSearchCollector and remove unused methods
  * SearchResult: Make fields final
This commit is contained in:
Coffeemakr 2017-11-11 12:17:14 +01:00
parent bc44557bdb
commit ceb556384b
No known key found for this signature in database
GPG Key ID: 3F35676D8FF6E743
22 changed files with 223 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
src/main/java/org/schabi/newpipe/extractor/Collector.java Normal file
View File

@ -0,0 +1,46 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.util.List;
/**
* Collectors are used to simplify the collection of information
* from extractors
* @param <I> the item type
* @param <E> the extractor type
*/
public interface Collector<I, E> {
/**
* Try to add an extractor to the collection
* @param extractor the extractor to add
*/
void commit(E extractor);
/**
* Try to extract the item from an extractor without adding it to the collection
* @param extractor the extractor to use
* @return the item
* @throws ParsingException thrown if there is an error extracting the
* <b>required</b> fields of the item.
*/
I extract(E extractor) throws ParsingException;
/**
* Get all items
* @return the items
*/
List<I> getItemList();
/**
* Get all errors
* @return the errors
*/
List<Throwable> getErrors();
/**
* Reset all collected items and errors
*/
void reset();
}

6
src/main/java/org/schabi/newpipe/extractor/Extractor.java
View File

@ -32,6 +32,8 @@ public abstract class Extractor {
private String cleanUrl;
public Extractor(StreamingService service, String url) throws ExtractionException {
if(service == null) throw new NullPointerException("service is null");
if(url == null) throw new NullPointerException("url is null");
this.service = service;
this.originalUrl = url;
}
@ -53,6 +55,10 @@ public abstract class Extractor {
return originalUrl;
}
/**
* Get a clean url and as a fallback the original url.
* @return the clean url or the original url
*/
public String getCleanUrl() {
if (cleanUrl != null && !cleanUrl.isEmpty()) return cleanUrl;

2
src/main/java/org/schabi/newpipe/extractor/InfoItem.java
View File

@ -20,8 +20,6 @@ package org.schabi.newpipe.extractor;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import org.schabi.newpipe.extractor.stream.StreamType;
import java.io.Serializable;
public abstract class InfoItem implements Serializable {

65
src/main/java/org/schabi/newpipe/extractor/InfoItemCollector.java
View File

@ -1,8 +1,9 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/*
@ -25,42 +26,66 @@ import java.util.List;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public abstract class InfoItemCollector {
private List<InfoItem> itemList = new ArrayList<>();
private List<Throwable> errors = new ArrayList<>();
private int serviceId = -1;
public abstract class InfoItemCollector<I extends InfoItem, E> implements Collector<I,E> {
private final List<I> itemList = new ArrayList<>();
private final List<Throwable> errors = new ArrayList<>();
private final int serviceId;
/**
* Create a new collector
* @param serviceId the service id
*/
public InfoItemCollector(int serviceId) {
this.serviceId = serviceId;
}
public List<InfoItem> getItemList() {
return itemList;
@Override
public List<I> getItemList() {
return Collections.unmodifiableList(itemList);
}
@Override
public List<Throwable> getErrors() {
return errors;
return Collections.unmodifiableList(errors);
}
protected void addFromCollector(InfoItemCollector otherC) throws ExtractionException {
if (serviceId != otherC.serviceId) {
throw new ExtractionException("Service Id does not equal: "
+ NewPipe.getNameOfService(serviceId)
+ " and " + NewPipe.getNameOfService((otherC.serviceId)));
}
errors.addAll(otherC.errors);
itemList.addAll(otherC.itemList);
@Override
public void reset() {
itemList.clear();
errors.clear();
}
protected void addError(Exception e) {
errors.add(e);
/**
* Add an error
* @param error the error
*/
protected void addError(Exception error) {
errors.add(error);
}
protected void addItem(InfoItem item) {
/**
* Add an item
* @param item the item
*/
protected void addItem(I item) {
itemList.add(item);
}
protected int getServiceId() {
/**
* Get the service id
* @return the service id
*/
public int getServiceId() {
return serviceId;
}
@Override
public void commit(E extractor) {
try {
addItem(extract(extractor));
} catch (ParsingException e) {
addError(e);
}
}
}

12
src/main/java/org/schabi/newpipe/extractor/ListExtractor.java
View File

@ -87,6 +87,18 @@ public abstract class ListExtractor extends Extractor {
public boolean hasMoreStreams() {
return nextItemsUrl != null && !nextItemsUrl.isEmpty();
}
public List<InfoItem> getNextItemsList() {
return nextItemsList;
}
public String getNextItemsUrl() {
return nextItemsUrl;
}
public List<Throwable> getErrors() {
return errors;
}
}
}

6
src/main/java/org/schabi/newpipe/extractor/ListInfo.java
View File

@ -11,7 +11,7 @@ public abstract class ListInfo extends Info {
super(serviceId, id, url, name);
}
public List<InfoItem> getRelated_streams() {
public List<InfoItem> getRelatedStreams() {
return related_streams;
}
@ -19,7 +19,7 @@ public abstract class ListInfo extends Info {
this.related_streams = related_streams;
}
public boolean isHas_more_streams() {
public boolean hasMoreStreams() {
return has_more_streams;
}
@ -27,7 +27,7 @@ public abstract class ListInfo extends Info {
this.has_more_streams = has_more_streams;
}
public String getNext_streams_url() {
public String getNextStreamsUrl() {
return next_streams_url;
}

11
src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemCollector.java
View File

@ -23,11 +23,12 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class ChannelInfoItemCollector extends InfoItemCollector {
public class ChannelInfoItemCollector extends InfoItemCollector<ChannelInfoItem, ChannelInfoItemExtractor> {
public ChannelInfoItemCollector(int serviceId) {
super(serviceId);
}
@Override
public ChannelInfoItem extract(ChannelInfoItemExtractor extractor) throws ParsingException {
// important information
int serviceId = getServiceId();
@ -60,12 +61,4 @@ public class ChannelInfoItemCollector extends InfoItemCollector {
}
return resultItem;
}
public void commit(ChannelInfoItemExtractor extractor) throws ParsingException {
try {
addItem(extract(extractor));
} catch (Exception e) {
addError(e);
}
}
}

12
src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemCollector.java
View File

@ -3,11 +3,13 @@ package org.schabi.newpipe.extractor.playlist;
import org.schabi.newpipe.extractor.InfoItemCollector;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
public class PlaylistInfoItemCollector extends InfoItemCollector {
public class PlaylistInfoItemCollector extends InfoItemCollector<PlaylistInfoItem, PlaylistInfoItemExtractor> {
public PlaylistInfoItemCollector(int serviceId) {
super(serviceId);
}
@Override
public PlaylistInfoItem extract(PlaylistInfoItemExtractor extractor) throws ParsingException {
String name = extractor.getName();
@ -33,12 +35,4 @@ public class PlaylistInfoItemCollector extends InfoItemCollector {
}
return resultItem;
}
public void commit(PlaylistInfoItemExtractor extractor) throws ParsingException {
try {
addItem(extract(extractor));
} catch (Exception e) {
addError(e);
}
}
}

73
src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java
View File

@ -1,10 +1,10 @@
package org.schabi.newpipe.extractor.search;
import org.schabi.newpipe.extractor.InfoItemCollector;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemCollector;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
@ -30,13 +30,23 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class InfoItemSearchCollector extends InfoItemCollector {
/**
* Collector for search results
*
* This collector can handle the following extractor types:
* <ul>
* <li>{@link StreamInfoItemExtractor}</li>
* <li>{@link ChannelInfoItemExtractor}</li>
* <li>{@link PlaylistInfoItemExtractor}</li>
* </ul>
* Calling {@link #extract(InfoItemExtractor)} or {@link #commit(Object)} with any
* other extractor type will raise an exception.
*/
public class InfoItemSearchCollector extends InfoItemCollector<InfoItem, InfoItemExtractor> {
private String suggestion = "";
private StreamInfoItemCollector streamCollector;
private ChannelInfoItemCollector userCollector;
private PlaylistInfoItemCollector playlistCollector;
private SearchResult result = new SearchResult();
private final StreamInfoItemCollector streamCollector;
private final ChannelInfoItemCollector userCollector;
private final PlaylistInfoItemCollector playlistCollector;
InfoItemSearchCollector(int serviceId) {
super(serviceId);
@ -50,43 +60,20 @@ public class InfoItemSearchCollector extends InfoItemCollector {
}
public SearchResult getSearchResult() throws ExtractionException {
addFromCollector(userCollector);
addFromCollector(streamCollector);
addFromCollector(playlistCollector);
result.suggestion = suggestion;
result.errors = getErrors();
return result;
return new SearchResult(getServiceId(), suggestion, getItemList(), getErrors());
}
public void commit(StreamInfoItemExtractor extractor) {
try {
result.resultList.add(streamCollector.extract(extractor));
} catch (FoundAdException ae) {
System.err.println("Found ad");
} catch (Exception e) {
addError(e);
}
}
public void commit(ChannelInfoItemExtractor extractor) {
try {
result.resultList.add(userCollector.extract(extractor));
} catch (FoundAdException ae) {
System.err.println("Found ad");
} catch (Exception e) {
addError(e);
}
}
public void commit(PlaylistInfoItemExtractor extractor) {
try {
result.resultList.add(playlistCollector.extract(extractor));
} catch (FoundAdException ae) {
System.err.println("Found ad");
} catch (Exception e) {
addError(e);
@Override
public InfoItem extract(InfoItemExtractor extractor) throws ParsingException {
// Use the corresponding collector for each item extractor type
if(extractor instanceof StreamInfoItemExtractor) {
return streamCollector.extract((StreamInfoItemExtractor) extractor);
} else if(extractor instanceof ChannelInfoItemExtractor) {
return userCollector.extract((ChannelInfoItemExtractor) extractor);
} else if(extractor instanceof PlaylistInfoItemExtractor) {
return playlistCollector.extract((PlaylistInfoItemExtractor) extractor);
} else {
throw new IllegalArgumentException("Invalid extractor type: " + extractor);
}
}
}

33
src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java
View File

@ -5,6 +5,8 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
/*
@ -28,6 +30,18 @@ import java.util.List;
*/
public class SearchResult {
private final int serviceId;
public final String suggestion;
public final List<InfoItem> resultList;
public final List<Throwable> errors;
public SearchResult(int serviceId, String suggestion, List<InfoItem> results, List<Throwable> errors) {
this.serviceId = serviceId;
this.suggestion = suggestion;
this.resultList = Collections.unmodifiableList(new ArrayList<>(results));
this.errors = Collections.unmodifiableList(new ArrayList<>(errors));
}
public static SearchResult getSearchResult(SearchEngine engine, String query, int page, String languageCode, SearchEngine.Filter filter)
throws IOException, ExtractionException {
@ -47,7 +61,20 @@ public class SearchResult {
return result;
}
public String suggestion;
public List<InfoItem> resultList = new ArrayList<>();
public List<Throwable> errors = new ArrayList<>();
public String getSuggestion() {
return suggestion;
}
public List<InfoItem> getResults() {
return Collections.unmodifiableList(resultList);
}
public List<Throwable> getErrors() {
return errors;
}
public int getServiceId() {
return serviceId;
}
}

2
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java
View File

@ -214,7 +214,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
private void collectStreamsFrom(StreamInfoItemCollector collector, Element element) throws ParsingException {
collector.getItemList().clear();
collector.reset();
final String uploaderName = getName();
for (final Element li : element.children()) {

2
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java
View File

@ -195,7 +195,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
private void collectStreamsFrom(StreamInfoItemCollector collector, Element element) throws ParsingException {
collector.getItemList().clear();
collector.reset();
final UrlIdHandler streamUrlIdHandler = getService().getStreamUrlIdHandler();
for (final Element li : element.children()) {

12
src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java
View File

@ -25,18 +25,6 @@ import org.schabi.newpipe.extractor.MediaFormat;
public class AudioStream extends Stream {
public int average_bitrate = -1;
/**
* Create a new audio stream
* @param url the url
* @param format the id of the format
* @param averageBitrate the average bit rate
* @deprecated use {@link AudioStream#AudioStream(String, MediaFormat, int)} instead
*/
@Deprecated
public AudioStream(String url, int format, int averageBitrate) {
this(url, MediaFormat.getFormatById(format), averageBitrate);
}
/**
* Create a new audio stream
* @param url the url

11
src/main/java/org/schabi/newpipe/extractor/stream/Stream.java
View File

@ -8,6 +8,11 @@ import java.util.List;
public abstract class Stream implements Serializable {
private final MediaFormat mediaFormat;
public final String url;
/**
* @deprecated Use {@link #getFormat()} or {@link #getFormatId()}
*/
@Deprecated
public final int format;
public Stream(String url, MediaFormat format) {
@ -20,7 +25,7 @@ public abstract class Stream implements Serializable {
* Reveals whether two streams have the same stats (format and bitrate, for example)
*/
public boolean equalStats(Stream cmp) {
return cmp != null && format == cmp.format;
return cmp != null && getFormatId() == cmp.getFormatId();
}
/**
@ -48,4 +53,8 @@ public abstract class Stream implements Serializable {
public MediaFormat getFormat() {
return mediaFormat;
}
public int getFormatId() {
return mediaFormat.id;
}
}

8
src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemCollector.java
View File

@ -28,13 +28,14 @@ import java.util.Vector;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class StreamInfoItemCollector extends InfoItemCollector {
public class StreamInfoItemCollector extends InfoItemCollector<StreamInfoItem, StreamInfoItemExtractor> {
public StreamInfoItemCollector(int serviceId) {
super(serviceId);
}
public StreamInfoItem extract(StreamInfoItemExtractor extractor) throws Exception {
@Override
public StreamInfoItem extract(StreamInfoItemExtractor extractor) throws ParsingException {
if (extractor.isAd()) {
throw new FoundAdException("Found ad");
}
@ -82,7 +83,8 @@ public class StreamInfoItemCollector extends InfoItemCollector {
return resultItem;
}
public void commit(StreamInfoItemExtractor extractor) throws ParsingException {
@Override
public void commit(StreamInfoItemExtractor extractor) {
try {
addItem(extract(extractor));
} catch (FoundAdException ae) {

16
src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java
View File

@ -26,22 +26,6 @@ public class VideoStream extends Stream {
public String resolution;
public boolean isVideoOnly;
/**
* @deprecated use {@link VideoStream#VideoStream(String, MediaFormat, String)}
*/
@Deprecated
public VideoStream(String url, int format, String res) {
this(url, MediaFormat.getFormatById(format), res);
}
/**
* @deprecated use {@link VideoStream#VideoStream(String, MediaFormat, String, boolean)}
*/
@Deprecated
public VideoStream(String url, int format, String res, boolean isVideoOnly) {
this(url, MediaFormat.getFormatById(format), res, isVideoOnly);
}
public VideoStream(String url, MediaFormat format, String resolution) {
this(url, format, resolution, false);

7
src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java
View File

@ -13,6 +13,9 @@ import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.InfoItemCollector;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import java.util.List;
/**
* Test for {@link SoundcloudChartsUrlIdHandler}
@ -48,10 +51,10 @@ public class SoundcloudChartsExtractorTest {
@Test
public void testGetStreams() throws Exception {
InfoItemCollector collector = extractor.getStreams();
StreamInfoItemCollector collector = extractor.getStreams();
if(!collector.getErrors().isEmpty()) {
System.err.println("----------");
for(Throwable e : collector.getErrors()) {
for(Throwable e: collector.getErrors()) {
e.printStackTrace();
System.err.println("----------");
}

20
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java
View File

@ -1,6 +1,7 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
@ -36,12 +37,12 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
* Test for {@link SearchEngine}
*/
public class YoutubeSearchEngineAllTest {
private SearchResult result;
private static SearchResult result;
@Before
public void setUp() throws Exception {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance());
SearchEngine engine = YouTube.getService().getSearchEngine();
YoutubeSearchEngine engine = new YoutubeSearchEngine(1);
// Youtube will suggest "asdf" instead of "asdgff"
// keep in mind that the suggestions can change by country (the parameter "de")
@ -51,19 +52,22 @@ public class YoutubeSearchEngineAllTest {
@Test
public void testResultList() {
assertFalse(result.resultList.isEmpty());
System.out.println("Results: " + result.getResults());
assertFalse("Results are empty: " + result.resultList, result.resultList.isEmpty());
}
@Test
public void testResultErrors() {
if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace();
assertTrue(result.errors == null || result.errors.isEmpty());
for (Throwable error : result.getErrors()) {
error.printStackTrace();
}
assertTrue(result.getErrors().isEmpty());
}
@Ignore
@Test
public void testSuggestion() {
//todo write a real test
assertTrue(result.suggestion != null);
assertTrue(result.getSuggestion() != null);
}
}

4
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java
View File

@ -119,8 +119,8 @@ public class YoutubeStreamExtractorDefaultTest {
assertTrue(s.url,
s.url.contains(HTTPS));
assertTrue(s.resolution.length() > 0);
assertTrue(Integer.toString(s.format),
0 <= s.format && s.format <= 4);
assertTrue(Integer.toString(s.getFormatId()),
0 <= s.getFormatId() && s.getFormatId() <= 4);
}
}

8
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java
View File

@ -98,11 +98,11 @@ public class YoutubeStreamExtractorRestrictedTest {
@Test
public void testGetVideoStreams() throws IOException, ExtractionException {
for (VideoStream s : extractor.getVideoStreams()) {
assertTrue(s.url,
s.url.contains(HTTPS));
assertTrue(s.getUrl(),
s.getUrl().contains(HTTPS));
assertTrue(s.resolution.length() > 0);
assertTrue(Integer.toString(s.format),
0 <= s.format && s.format <= 4);
assertTrue(Integer.toString(s.getFormatId()),
0 <= s.getFormatId() && s.getFormatId() <= 4);
}
}

6
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTreindingKioskInfoTest.java
View File

@ -50,16 +50,16 @@ public class YoutubeTreindingKioskInfoTest {
@Test
public void getStreams() {
assertFalse(kioskInfo.related_streams.isEmpty());
assertFalse(kioskInfo.getRelatedStreams().isEmpty());
}
@Test
public void getId() {
assertEquals(kioskInfo.id, "Trending");
assertEquals(kioskInfo.getId(), "Trending");
}
@Test
public void getName() {
assertFalse(kioskInfo.name.isEmpty());
assertFalse(kioskInfo.getName().isEmpty());
}
}

5
src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java
View File

@ -26,6 +26,7 @@ import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.InfoItemCollector;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.assertEquals;
@ -67,7 +68,7 @@ public class YoutubeTrendingExtractorTest {
@Test
public void testGetStreams() throws Exception {
InfoItemCollector collector = extractor.getStreams();
StreamInfoItemCollector collector = extractor.getStreams();
if(!collector.getErrors().isEmpty()) {
System.err.println("----------");
for(Throwable e : collector.getErrors()) {
@ -95,7 +96,7 @@ public class YoutubeTrendingExtractorTest {
@Test
public void testGetNextStreams() throws Exception {
assertTrue("extractor has next streams", extractor.getNextStreams() == null
|| extractor.getNextStreams().nextItemsList.isEmpty());
|| extractor.getNextStreams().getNextItemsList().isEmpty());
}
@Test