lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix: support richGridRenderer on channel page 

YouTube is currently A/B testing a new layout on their channel pages, which uses a RichGridRenderer.
This commit is contained in:
Theta-Dev 2022-10-12 15:29:36 +02:00
parent 6a858368c8
commit ed4559d4de
6 changed files with 685 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -332,16 +332,20 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
Page nextPage = null;
if (getVideoTab() != null) {
final JsonObject gridRenderer = getVideoTab().getObject("content")
final JsonObject tabContent = getVideoTab().getObject("content");
JsonArray items = tabContent
.getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("gridRenderer");
.getArray("contents").getObject(0).getObject("gridRenderer").getArray("items");
if (items.isEmpty()) {
items = tabContent.getObject("richGridRenderer").getArray("contents");
}
final List<String> channelIds = new ArrayList<>();
channelIds.add(getName());
channelIds.add(getUrl());
final JsonObject continuation = collectStreamsFrom(collector, gridRenderer
.getArray("items"), channelIds);
final JsonObject continuation = collectStreamsFrom(collector, items, channelIds);
nextPage = getNextPageFrom(continuation, channelIds);
}
@ -433,6 +437,21 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return uploaderUrl;
}
});
} else if (video.has("richItemRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(
video.getObject("richItemRenderer")
.getObject("content").getObject("videoRenderer"), timeAgoParser) {
@Override
public String getUploaderName() {
return uploaderName;
}
@Override
public String getUploaderUrl() {
return uploaderUrl;
}
});
} else if (video.has("continuationItemRenderer")) {
continuation = video.getObject("continuationItemRenderer");
}

15
extractor/src/test/java/org/schabi/newpipe/downloader/DownloaderFactory.java
View File

@ -37,7 +37,20 @@ public class DownloaderFactory {
* Preferably starting with {@link DownloaderFactory#RESOURCE_PATH}
*/
public static Downloader getDownloader(final String path) throws IOException {
final DownloaderType type = getDownloaderType();
return getDownloaderOfType(path, getDownloaderType());
}
/**
* <p>
* Returns a implementation of a {@link Downloader} with a given type.
* </p>
*
* @param path The path to the folder where mocks are saved/retrieved.
* Preferably starting with {@link DownloaderFactory#RESOURCE_PATH}
*
* @param type The downloader type
*/
public static Downloader getDownloaderOfType(final String path, final DownloaderType type) throws IOException {
switch (type) {
case REAL:
return DownloaderTestImpl.getInstance();

97
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java
View File

@ -16,6 +16,7 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderType;
import org.schabi.newpipe.extractor.ExtractorAsserts;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
@ -648,4 +649,100 @@ public class YoutubeChannelExtractorTest {
assertFalse(extractor.isVerified());
}
}
/**
* Test the extraction of channel data from a RichGridRenderer (currently A/B tested, as of 11.10.2022)
*/
public static class RichGrid implements BaseChannelExtractorTest {
private static YoutubeChannelExtractor extractor;
@BeforeAll
public static void setUp() throws Exception {
YoutubeTestsUtils.ensureStateless();
NewPipe.init(DownloaderFactory.getDownloaderOfType(RESOURCE_PATH + "richGrid", DownloaderType.MOCK));
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("https://www.youtube.com/channel/UC2DjFE7Xf11URZqWBigcVOQ");
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testServiceId() {
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
}
@Test
public void testName() throws Exception {
assertEquals("EEVblog", extractor.getName());
}
@Test
public void testId() throws Exception {
assertEquals("UC2DjFE7Xf11URZqWBigcVOQ", extractor.getId());
}
@Test
public void testUrl() throws ParsingException {
assertEquals("https://www.youtube.com/channel/UC2DjFE7Xf11URZqWBigcVOQ", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://www.youtube.com/channel/UC2DjFE7Xf11URZqWBigcVOQ", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Override
public void testMoreRelatedItems() throws Exception {
}
/*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testDescription() throws Exception {
ExtractorAsserts.assertContains("NO SCRIPT, NO FEAR, ALL OPINION", extractor.getDescription());
}
@Test
public void testAvatarUrl() throws Exception {
String avatarUrl = extractor.getAvatarUrl();
assertIsSecureUrl(avatarUrl);
ExtractorAsserts.assertContains("yt3", avatarUrl);
}
@Test
public void testBannerUrl() throws Exception {
String bannerUrl = extractor.getBannerUrl();
assertIsSecureUrl(bannerUrl);
ExtractorAsserts.assertContains("yt3", bannerUrl);
}
@Test
public void testFeedUrl() throws Exception {
assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UC2DjFE7Xf11URZqWBigcVOQ", extractor.getFeedUrl());
}
@Test
public void testSubscriberCount() throws Exception {
ExtractorAsserts.assertGreaterOrEqual(800_000, extractor.getSubscriberCount());
}
@Test
public void testVerified() throws Exception {
assertTrue(extractor.isVerified());
}
}
}

82
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/richGrid/generated_mock_0.json Normal file
View File

@ -0,0 +1,82 @@
{
"request": {
"httpMethod": "GET",
"url": "https://www.youtube.com/sw.js",
"headers": {
"Origin": [
"https://www.youtube.com"
],
"Referer": [
"https://www.youtube.com"
],
"Accept-Language": [
"en-GB, en;q\u003d0.9"
]
},
"localization": {
"languageCode": "en",
"countryCode": "GB"
}
},
"response": {
"responseCode": 200,
"responseMessage": "",
"responseHeaders": {
"access-control-allow-credentials": [
"true"
],
"access-control-allow-origin": [
"https://www.youtube.com"
],
"alt-svc": [
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
],
"cache-control": [
"private, max-age\u003d0"
],
"content-type": [
"text/javascript; charset\u003dutf-8"
],
"cross-origin-opener-policy-report-only": [
"same-origin; report-to\u003d\"youtube_main\""
],
"date": [
"Wed, 12 Oct 2022 12:56:05 GMT"
],
"expires": [
"Wed, 12 Oct 2022 12:56:05 GMT"
],
"p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
],
"permissions-policy": [
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
],
"report-to": [
"{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
],
"server": [
"ESF"
],
"set-cookie": [
"YSC\u003daZuttnu33k4; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dThu, 16-Jan-2020 12:56:05 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+225; expires\u003dFri, 11-Oct-2024 12:56:05 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
],
"strict-transport-security": [
"max-age\u003d31536000"
],
"x-content-type-options": [
"nosniff"
],
"x-frame-options": [
"SAMEORIGIN"
],
"x-xss-protection": [
"0"
]
},
"responseBody": "\n self.addEventListener(\u0027install\u0027, event \u003d\u003e {\n event.waitUntil(self.skipWaiting());\n });\n self.addEventListener(\u0027activate\u0027, event \u003d\u003e {\n event.waitUntil(\n self.clients.claim().then(() \u003d\u003e self.registration.unregister()));\n });\n ",
"latestUrl": "https://www.youtube.com/sw.js"
}
}

76
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/richGrid/generated_mock_1.json Normal file
View File

File diff suppressed because one or more lines are too long

393
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/richGrid/generated_mock_2.json Normal file
View File

File diff suppressed because one or more lines are too long