lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

add conferences search

This commit is contained in:
Christian Schabesberger 2019-01-28 15:30:17 +01:00
parent 1503459292
commit 14e1ccdb86
6 changed files with 231 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java
View File

@ -8,22 +8,39 @@ import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferencesListLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Localization;
import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.CONFERENCES;
import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.EVENTS;
import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.ALL;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.List;
public class MediaCCCSearchExtractor extends SearchExtractor {
private JsonObject doc;
private MediaCCCConferenceKiosk conferenceKiosk;
public MediaCCCSearchExtractor(StreamingService service, SearchQueryHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
try {
conferenceKiosk = new MediaCCCConferenceKiosk(service,
new MediaCCCConferencesListLinkHandlerFactory().fromId("conferences"),
"conferences",
localization);
} catch (Exception e) {
e.printStackTrace();
}
}
@Override
@ -34,11 +51,22 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
InfoItemsCollector searchItems = getInfoItemSearchCollector();
JsonArray events = doc.getArray("events");
for(int i = 0; i < events.size(); i++) {
searchItems.commit(new MediaCCCStreamInfoItemExtractor(
events.getObject(i)));
InfoItemsSearchCollector searchItems = getInfoItemSearchCollector();
if(getLinkHandler().getContentFilters().contains(CONFERENCES)
|| getLinkHandler().getContentFilters().contains(ALL)) {
searchConferences(getSearchString(),
conferenceKiosk.getInitialPage().getItems(),
searchItems);
}
if(getLinkHandler().getContentFilters().contains(EVENTS)
|| getLinkHandler().getContentFilters().contains(ALL)) {
JsonArray events = doc.getArray("events");
for (int i = 0; i < events.size(); i++) {
searchItems.commit(new MediaCCCStreamInfoItemExtractor(
events.getObject(i)));
}
}
return new InfoItemsPage<>(searchItems, null);
}
@ -55,13 +83,60 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String site;
final String url = getUrl();
site = downloader.download(url, getLocalization());
try {
doc = JsonParser.object().from(site);
} catch (JsonParserException jpe) {
throw new ExtractionException("Could not parse json.", jpe);
if(getLinkHandler().getContentFilters().contains(EVENTS)
|| getLinkHandler().getContentFilters().contains(ALL)) {
final String site;
final String url = getUrl();
site = downloader.download(url, getLocalization());
try {
doc = JsonParser.object().from(site);
} catch (JsonParserException jpe) {
throw new ExtractionException("Could not parse json.", jpe);
}
}
if(getLinkHandler().getContentFilters().contains(CONFERENCES)
|| getLinkHandler().getContentFilters().contains(ALL))
conferenceKiosk.fetchPage();
}
private void searchConferences(String searchString,
List<ChannelInfoItem> channelItems,
InfoItemsSearchCollector collector) {
for(final ChannelInfoItem item : channelItems) {
if(item.getName().toUpperCase().contains(
searchString.toUpperCase())) {
collector.commit(new ChannelInfoItemExtractor() {
@Override
public String getDescription() throws ParsingException {
return item.getDescription();
}
@Override
public long getSubscriberCount() throws ParsingException {
return item.getSubscriberCount();
}
@Override
public long getStreamCount() throws ParsingException {
return item.getStreamCount();
}
@Override
public String getName() throws ParsingException {
return item.getName();
}
@Override
public String getUrl() throws ParsingException {
return item.getUrl();
}
@Override
public String getThumbnailUrl() throws ParsingException {
return item.getThumbnailUrl();
}
});
}
}
}
}

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java
View File

@ -9,9 +9,17 @@ import java.util.List;
public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String ALL = "all";
public static final String CONFERENCES = "conferences";
public static final String EVENTS = "events";
@Override
public String[] getAvailableContentFilter() {
return new String[0];
return new String[] {
ALL,
CONFERENCES,
EVENTS
};
}
@Override

60
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java Normal file
View File

@ -0,0 +1,60 @@
package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.Localization;
import java.util.Arrays;
import static junit.framework.TestCase.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
/**
* Test for {@link MediaCCCSearchExtractor}
*/
public class MediaCCCSearchExtractorAllTest {
private static SearchExtractor extractor;
private static ListExtractor.InfoItemsPage<InfoItem> itemsPage;
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory()
.fromQuery("c", Arrays.asList(new String[] {"all"}), "")
,new Localization("GB", "en"));
extractor.fetchPage();
itemsPage = extractor.getInitialPage();
}
@Test
public void testIfChannelInfoItemsAvailable() {
boolean isAvialable = false;
for(InfoItem item : itemsPage.getItems()) {
if(item instanceof ChannelInfoItem) {
isAvialable = true;
}
}
assertTrue("ChannelInfoItem not in all list", isAvialable);
}
@Test
public void testIfStreamInfoitemsAvailable() {
boolean isAvialable = false;
for(InfoItem item : itemsPage.getItems()) {
if(item instanceof StreamInfoItem) {
isAvialable = true;
}
}
assertTrue("ChannelInfoItem not in all list", isAvialable);
}
}

50
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java Normal file
View File

@ -0,0 +1,50 @@
package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.utils.Localization;
import java.util.Arrays;
import static junit.framework.TestCase.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
/**
* Test for {@link MediaCCCSearchExtractor}
*/
public class MediaCCCSearchExtractorConferencesTest {
private static SearchExtractor extractor;
private static ListExtractor.InfoItemsPage<InfoItem> itemsPage;
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory()
.fromQuery("c3", Arrays.asList(new String[] {"conferences"}), "")
,new Localization("GB", "en"));
extractor.fetchPage();
itemsPage = extractor.getInitialPage();
}
@Test
public void testReturnTypeChannel() {
for(InfoItem item : itemsPage.getItems()) {
assertTrue("Item is not of type channel", item instanceof ChannelInfoItem);
}
}
@Test
public void testItemCount() {
assertTrue("Count is to hight: " + itemsPage.getItems().size(), itemsPage.getItems().size() < 127);
assertTrue("Countis to low: " + itemsPage.getItems().size(), itemsPage.getItems().size() >= 29);
}
}

17
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorTest.java → extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java
View File

@ -8,8 +8,12 @@ import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.Localization;
import java.util.Arrays;
import static junit.framework.TestCase.assertEquals;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertFalse;
@ -18,14 +22,16 @@ import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
/**
* Test for {@link MediaCCCSearchExtractor}
*/
public class MediaCCCSearchExtractorTest {
public class MediaCCCSearchExtractorEventsTest {
private static SearchExtractor extractor;
private static ListExtractor.InfoItemsPage<InfoItem> itemsPage;
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
extractor = MediaCCC.getSearchExtractor("source");
extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory()
.fromQuery("linux", Arrays.asList(new String[] {"events"}), "")
,new Localization("GB", "en"));
extractor.fetchPage();
itemsPage = extractor.getInitialPage();
}
@ -58,4 +64,11 @@ public class MediaCCCSearchExtractorTest {
itemsPage.getItems().get(0).getThumbnailUrl().startsWith("https://static.media.ccc.de/media/")
&& itemsPage.getItems().get(0).getThumbnailUrl().endsWith(".jpg"));
}
@Test
public void testReturnTypeStream() throws Exception {
for(InfoItem item : itemsPage.getItems()) {
assertTrue("Item is not of type StreamInfoItem", item instanceof StreamInfoItem);
}
}
}

21
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java
View File

@ -8,10 +8,13 @@ import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.Localization;
import java.util.Arrays;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@ -44,26 +47,20 @@ public class SoundcloudSearchExtractorDefaultTest extends SoundcloudSearchExtrac
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
extractor = (SoundcloudSearchExtractor) SoundCloud.getSearchExtractor("lill uzi vert");
extractor = (SoundcloudSearchExtractor) SoundCloud.getSearchExtractor(
new SoundcloudSearchQueryHandlerFactory().fromQuery("lill uzi vert",
Arrays.asList(new String[]{"tracks"}), ""),
new Localization("GB", "en"));
extractor.fetchPage();
itemsPage = extractor.getInitialPage();
}
@Test
public void testGetSecondPageUrl() throws Exception {
assertEquals("https://api-v2.soundcloud.com/search?q=lill+uzi+vert&limit=10&offset=10",
assertEquals("https://api-v2.soundcloud.com/search/tracks?q=lill+uzi+vert&limit=10&offset=10",
removeClientId(extractor.getNextPageUrl()));
}
@Test
public void testResultList_FirstElement() {
InfoItem firstInfoItem = itemsPage.getItems().get(0);
// THe channel should be the first item
assertEquals("name", "Bad and Boujee (Feat. Lil Uzi Vert) [Prod. By Metro Boomin]", firstInfoItem.getName());
assertEquals("url","https://soundcloud.com/migosatl/bad-and-boujee-feat-lil-uzi-vert-prod-by-metro-boomin", firstInfoItem.getUrl());
}
@Test
public void testResultListCheckIfContainsStreamItems() {
boolean hasStreams = false;
@ -94,7 +91,7 @@ public class SoundcloudSearchExtractorDefaultTest extends SoundcloudSearchExtrac
}
assertFalse("First and second page are equal", equals);
assertEquals("https://api-v2.soundcloud.com/search?q=lill+uzi+vert&limit=10&offset=20",
assertEquals("https://api-v2.soundcloud.com/search/tracks?q=lill+uzi+vert&limit=10&offset=20",
removeClientId(secondPage.getNextPageUrl()));
}