Merge 699b408639 into 6c3c2e25d7

2024-04-11 18:50:32 +02:00 · 2024-04-11 18:50:32 +02:00 · 8075b49e01
parent 6c3c2e25d7 699b408639
commit 8075b49e01
5 changed files with 370 additions and 0 deletions
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
@ -30,6 +30,7 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsE
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMusicSearchExtractor;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMusicTrendingExtractor;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
@ -39,6 +40,7 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingE
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeMusicTrendingLinkHandlerFactory;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
@ -158,6 +160,16 @@ public class YoutubeService extends StreamingService {

        // add kiosks here e.g.:
        try {
+            list.addKioskEntry(
+                    (streamingService, url, id) -> new YoutubeMusicTrendingExtractor(
+                            YoutubeService.this,
+                            new YoutubeMusicTrendingLinkHandlerFactory().fromUrl(url),
+                            id
+                    ),
+                    new YoutubeMusicTrendingLinkHandlerFactory(),
+                    "Trending Music"
+            );
+
            list.addKioskEntry(
                    (streamingService, url, id) -> new YoutubeTrendingExtractor(
                            YoutubeService.this,
@ -173,6 +185,7 @@ public class YoutubeService extends StreamingService {
        }

        return list;
+
    }

    @Override
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicTrendingExtractor.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicTrendingExtractor.java
@ -0,0 +1,158 @@
+/*
+ * Created by Christian Schabesberger on 12.08.17.
+ *
+ * Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
+ * YoutubeTrendingExtractor.java is part of NewPipe Extractor.
+ *
+ * NewPipe Extractor is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe Extractor is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.schabi.newpipe.extractor.services.youtube.extractors;
+
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextAtKey;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
+import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonWriter;
+
+import org.schabi.newpipe.extractor.Page;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.downloader.Downloader;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
+import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
+import org.schabi.newpipe.extractor.localization.TimeAgoParser;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+
+import java.io.IOException;
+
+import javax.annotation.Nonnull;
+
+public class YoutubeMusicTrendingExtractor extends KioskExtractor<StreamInfoItem> {
+    private JsonObject initialData;
+
+    public YoutubeMusicTrendingExtractor(final StreamingService service,
+                                         final ListLinkHandler linkHandler,
+                                         final String kioskId) {
+        super(service, linkHandler, kioskId);
+    }
+
+    @Override
+    public void onFetchPage(@Nonnull final Downloader downloader)
+            throws IOException, ExtractionException {
+        // @formatter:off
+        final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
+                getExtractorContentCountry())
+                .value("browseId", "MUtrending")
+                .done())
+                .getBytes(UTF_8);
+        // @formatter:on
+
+        initialData = getJsonPostResponse("browse", body, getExtractorLocalization());
+    }
+
+    @Override
+    public InfoItemsPage<StreamInfoItem> getPage(final Page page) {
+        return InfoItemsPage.emptyPage();
+    }
+
+    @Nonnull
+    @Override
+    public String getName() throws ParsingException {
+        final JsonObject header = initialData.getObject("header");
+        String name = null;
+        if (header.has("feedTabbedHeaderRenderer")) {
+            name = getTextAtKey(header.getObject("feedTabbedHeaderRenderer"), "title");
+        } else if (header.has("c4TabbedHeaderRenderer")) {
+            name = getTextAtKey(header.getObject("c4TabbedHeaderRenderer"), "title");
+        }
+
+        if (isNullOrEmpty(name)) {
+            throw new ParsingException("Could not get Trending name");
+        }
+        return name;
+    }
+
+    @Nonnull
+    @Override
+    public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
+        final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+        final TimeAgoParser timeAgoParser = getTimeAgoParser();
+        final JsonObject tabContent = getTrendingTabContent();
+
+        if (tabContent.has("richGridRenderer")) {
+            tabContent.getObject("richGridRenderer")
+                    .getArray("contents")
+                    .stream()
+                    .filter(JsonObject.class::isInstance)
+                    .map(JsonObject.class::cast)
+                    // Filter Trending shorts and Recently trending sections
+                    .filter(content -> content.has("richItemRenderer"))
+//                    .filter(shelfRenderer -> !shelfRenderer.has("music"))
+                    .map(content -> content.getObject("richItemRenderer")
+                            .getObject("content")
+                            .getObject("videoRenderer"))
+                    .forEachOrdered(videoRenderer -> collector.commit(
+                            new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser)));
+        } else if (tabContent.has("sectionListRenderer")) {
+            tabContent.getObject("sectionListRenderer")
+                    .getArray("contents")
+                    .stream()
+                    .filter(JsonObject.class::isInstance)
+                    .map(JsonObject.class::cast)
+                    .flatMap(content -> content.getObject("itemSectionRenderer")
+                            .getArray("contents")
+                            .stream())
+                    .filter(JsonObject.class::isInstance)
+                    .map(JsonObject.class::cast)
+                    .map(content -> content.getObject("shelfRenderer"))
+                    // Filter Trending shorts and Recently trending sections which have a title,
+                    // contrary to normal trends
+                    .filter(shelfRenderer -> !shelfRenderer.has("title"))
+//                    .filter(shelfRenderer -> !shelfRenderer.has("music"))
+                    .flatMap(shelfRenderer -> shelfRenderer.getObject("content")
+                            .getObject("expandedShelfContentsRenderer")
+                            .getArray("items")
+                            .stream())
+                    .filter(JsonObject.class::isInstance)
+                    .map(JsonObject.class::cast)
+                    .map(item -> item.getObject("videoRenderer"))
+                    .forEachOrdered(videoRenderer -> collector.commit(
+                            new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser)));
+        }
+
+        return new InfoItemsPage<>(collector, null);
+    }
+
+    private JsonObject getTrendingTabContent() throws ParsingException {
+        return initialData.getObject("contents")
+                .getObject("twoColumnBrowseResultsRenderer")
+                .getArray("tabs")
+                .stream()
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast)
+                .map(tab -> tab.getObject("tabRenderer"))
+                .filter(tabRenderer -> tabRenderer.getBoolean("selected"))
+                .filter(tabRenderer -> tabRenderer.has("content"))
+                // There should be at most one tab selected
+                .findFirst()
+                .orElseThrow(() -> new ParsingException("Could not get \"Now\" trending tab"))
+                .getObject("content");
+    }
+}
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeMusicTrendingLinkHandlerFactory.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeMusicTrendingLinkHandlerFactory.java
@ -0,0 +1,43 @@
+package org.schabi.newpipe.extractor.services.youtube.linkHandler;
+
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidioURL;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.List;
+import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
+import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
+import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.utils.Utils;
+
+public final class YoutubeMusicTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
+
+    public String getUrl(final String id,
+                         final List<String> contentFilters,
+                         final String sortFilter) {
+        return "https://www.youtube.com/feed/trending/music";
+    }
+
+    @Override
+    public String getId(final String url) {
+        return "Trending";
+    }
+
+    @Override
+    public boolean onAcceptUrl(final String url) {
+        final URL urlObj;
+        try {
+            urlObj = Utils.stringToURL(url);
+        } catch (final MalformedURLException e) {
+            return false;
+        }
+
+        final String urlPath = urlObj.getPath();
+        return Utils.isHTTP(urlObj) && (isYoutubeURL(urlObj) || isInvidioURL(urlObj))
+                && urlPath.equals("/feed/trending/music");
+    }
+}
--- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMusicTrendingKioskInfoTest.java
+++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMusicTrendingKioskInfoTest.java
@ -0,0 +1,69 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+/*
+ * Created by Christian Schabesberger on 12.08.17.
+ *
+ * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
+ * YoutubeTrendingKioskInfoTest.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.schabi.newpipe.downloader.DownloaderFactory;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.kiosk.KioskInfo;
+import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
+
+/**
+ * Test for {@link KioskInfo}
+ */
+public class YoutubeMusicTrendingKioskInfoTest {
+
+    private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "kiosk";
+
+    static KioskInfo kioskInfo;
+
+    @BeforeAll
+    public static void setUp()
+            throws Exception {
+        YoutubeTestsUtils.ensureStateless();
+        NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH));
+        LinkHandlerFactory LinkHandlerFactory = ((StreamingService) YouTube).getKioskList().getListLinkHandlerFactoryByType("Trending");
+
+        kioskInfo = KioskInfo.getInfo(YouTube, LinkHandlerFactory.fromId("Trending").getUrl());
+    }
+
+    @Test
+    public void getStreams() {
+        assertFalse(kioskInfo.getRelatedItems().isEmpty());
+    }
+
+    @Test
+    public void getId() {
+        assertTrue(kioskInfo.getId().equals("Trending Music")
+                || kioskInfo.getId().equals("Trends"));
+    }
+
+    @Test
+    public void getName() {
+        assertFalse(kioskInfo.getName().isEmpty());
+    }
+}
--- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMusicTrendingLinkHandlerFactoryTest.java
+++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMusicTrendingLinkHandlerFactoryTest.java
@ -0,0 +1,87 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+/*
+ * Created by Christian Schabesberger on 12.08.17.
+ *
+ * Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
+ * YoutubeTrendingLinkHandlerFactoryTest.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory;
+
+/**
+ * Test for {@link YoutubeTrendingLinkHandlerFactory}
+ */
+public class YoutubeMusicTrendingLinkHandlerFactoryTest {
+    private static LinkHandlerFactory LinkHandlerFactory;
+
+    @BeforeAll
+    public static void setUp() throws Exception {
+        LinkHandlerFactory = YouTube.getKioskList().getListLinkHandlerFactoryByType("Trending Music");
+        NewPipe.init(DownloaderTestImpl.getInstance());
+    }
+
+    @Test
+    public void getUrl()
+            throws Exception {
+        assertEquals(LinkHandlerFactory.fromId("").getUrl(), "https://www.youtube.com/feed/trending/music");
+    }
+
+    @Test
+    public void getId()
+            throws Exception {
+        assertEquals(LinkHandlerFactory.fromUrl("https://www.youtube.com/feed/trending/music").getId(), "Trending");
+    }
+
+    @Test
+    public void acceptUrl() throws ParsingException {
+        assertTrue(LinkHandlerFactory.acceptUrl("https://www.youtube.com/feed/trending/music"));
+        assertTrue(LinkHandlerFactory.acceptUrl("https://www.youtube.com/feed/trending/music/?adsf=fjaj#fhe"));
+        assertTrue(LinkHandlerFactory.acceptUrl("http://www.youtube.com/feed/trending/music/"));
+        assertTrue(LinkHandlerFactory.acceptUrl("www.youtube.com/feed/trending/music"));
+        assertTrue(LinkHandlerFactory.acceptUrl("youtube.com/feed/trending/music"));
+        assertTrue(LinkHandlerFactory.acceptUrl("youtube.com/feed/trending/music/?akdsakjf=dfije&kfj=dkjak"));
+        assertTrue(LinkHandlerFactory.acceptUrl("https://youtube.com/feed/trending/music"));
+        assertTrue(LinkHandlerFactory.acceptUrl("m.youtube.com/feed/trending/music"));
+
+        assertTrue(LinkHandlerFactory.acceptUrl("https://www.invidio.us/feed/trending/music"));
+        assertTrue(LinkHandlerFactory.acceptUrl("https://invidio.us/feed/trending/music"));
+        assertTrue(LinkHandlerFactory.acceptUrl("invidio.us/feed/trending/music"));
+
+        assertFalse(LinkHandlerFactory.acceptUrl("https://youtu.be/feed/trending/music"));
+        assertFalse(LinkHandlerFactory.acceptUrl("kdskjfiiejfia"));
+        assertFalse(LinkHandlerFactory.acceptUrl("https://www.youtube.com/bullshit/feed/trending/music"));
+        assertFalse(LinkHandlerFactory.acceptUrl("https://www.youtube.com/feed/trending/music/bullshit"));
+        assertFalse(LinkHandlerFactory.acceptUrl("https://www.youtube.com/feed/bullshit/trending/music"));
+        assertFalse(LinkHandlerFactory.acceptUrl("peter klaut aepferl youtube.com/feed/trending/music"));
+        assertFalse(LinkHandlerFactory.acceptUrl("youtube.com/feed/trending/music askjkf"));
+        assertFalse(LinkHandlerFactory.acceptUrl("askdjfi youtube.com/feed/trending/music askjkf"));
+        assertFalse(LinkHandlerFactory.acceptUrl("    youtube.com/feed/trending/music"));
+        assertFalse(LinkHandlerFactory.acceptUrl("https://www.youtube.com/feed/trending/music.html"));
+        assertFalse(LinkHandlerFactory.acceptUrl(""));
+    }
+}