lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'dev' into extractYTError

This commit is contained in:
Robin 2020-01-06 11:28:43 +01:00 committed by GitHub
parent 52b48a2e64 bdbfa26835
commit 03d61e589d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 77 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java
View File

@ -48,6 +48,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
InfoItemsSearchCollector searchItems = getInfoItemSearchCollector();
searchItems.reset();
if(getLinkHandler().getContentFilters().contains(CONFERENCES)
|| getLinkHandler().getContentFilters().contains(ALL)

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View File

@ -31,7 +31,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudParsingHelper {
private static final String HARDCODED_CLIENT_ID = "bkcJLoXNaiFlsLaKBQXOxO5FhW0NJVnu"; // Updated on 29/11/19
private static final String HARDCODED_CLIENT_ID = "r5ELVSy3RkcjX7ilaL7n2v1Z8irA9SL8"; // Updated on 31/12/19
private static String clientId;
private SoundcloudParsingHelper() {

1
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java
View File

@ -76,6 +76,7 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
private InfoItemsCollector<InfoItem, InfoItemExtractor> collectItems(JsonArray searchCollection) {
final InfoItemsSearchCollector collector = getInfoItemSearchCollector();
collector.reset();
for (Object result : searchCollection) {
if (!(result instanceof JsonObject)) continue;

1
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -106,6 +106,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
private InfoItemsSearchCollector collectItems(Document doc) throws NothingFoundException {
InfoItemsSearchCollector collector = getInfoItemSearchCollector();
collector.reset();
Element list = doc.select("ol[class=\"item-section\"]").first();
final TimeAgoParser timeAgoParser = getTimeAgoParser();

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -662,7 +662,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
return errorReason != null ? errorReason.toString() : null;
return errorReason != null ? errorReason.toString() : "";
}
/*//////////////////////////////////////////////////////////////////////////
@ -697,7 +697,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final String playerUrl;
// Check if the video is age restricted
if (!doc.select("meta[property=\"og:restrictions:age\"").isEmpty()) {
if (!doc.select("meta[property=\"og:restrictions:age\"]").isEmpty()) {
final EmbeddedInfo info = getEmbeddedInfo();
final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts);
final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody();

71
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchPagingTest.java Normal file
View File

@ -0,0 +1,71 @@
package org.schabi.newpipe.extractor.services.youtube.search;
import static java.util.Collections.singletonList;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
public class YoutubeSearchPagingTest {
private static ListExtractor.InfoItemsPage<InfoItem> page1;
private static ListExtractor.InfoItemsPage<InfoItem> page2;
private static Set<String> urlList1;
private static Set<String> urlList2;
private static int page1Size;
private static int page2Size;
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeSearchExtractor extractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("cirque du soleil",
singletonList(YoutubeSearchQueryHandlerFactory.VIDEOS), null);
extractor.fetchPage();
page1 = extractor.getInitialPage();
urlList1 = extractUrls(page1.getItems());
assertTrue("failed to load search result page one: too few items", 15 < page1.getItems().size());
page1Size = page1.getItems().size();
assertEquals("duplicated items in search result on page one", page1Size, urlList1.size());
assertTrue("search result has no second page", page1.hasNextPage());
assertNotNull("next page url is null", page1.getNextPageUrl());
page2 = extractor.getPage(page1.getNextPageUrl());
urlList2 = extractUrls(page2.getItems());
page2Size = page2.getItems().size();
}
private static Set<String> extractUrls(List<InfoItem> list) {
Set<String> result = new HashSet<>();
for (InfoItem item : list) {
result.add(item.getUrl());
}
return result;
}
@Test
public void secondPageUniqueVideos() {
assertEquals("Second search result page has duplicated items", page2Size, urlList2.size());
}
@Test
public void noRepeatingVideosInPages() {
Set<String> intersection = new HashSet<>(urlList2);
intersection.retainAll(urlList1);
assertEquals("Found the same item on first AND second search page", 0, intersection.size());
}
}