package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYoutubeMusicClientVersion; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYoutubeMusicHeaders; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.utils.JsonUtils; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import java.util.Objects; import javax.annotation.Nonnull; import javax.annotation.Nullable; public class YoutubeMusicSearchExtractor extends SearchExtractor { private JsonObject initialData; public YoutubeMusicSearchExtractor(final StreamingService service, final SearchQueryHandler linkHandler) { super(service, linkHandler); } @Override public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { final String url = "https://music.youtube.com/youtubei/v1/search?" + DISABLE_PRETTY_PRINT_PARAMETER; final String params; switch (getLinkHandler().getContentFilters().get(0)) { case MUSIC_SONGS: params = "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D"; break; case MUSIC_VIDEOS: params = "Eg-KAQwIABABGAAgACgAMABqChAEEAUQAxAKEAk%3D"; break; case MUSIC_ALBUMS: params = "Eg-KAQwIABAAGAEgACgAMABqChAEEAUQAxAKEAk%3D"; break; case MUSIC_PLAYLISTS: params = "Eg-KAQwIABAAGAAgACgBMABqChAEEAUQAxAKEAk%3D"; break; case MUSIC_ARTISTS: params = "Eg-KAQwIABAAGAAgASgAMABqChAEEAUQAxAKEAk%3D"; break; default: params = null; break; } // @formatter:off final byte[] json = JsonWriter.string() .object() .object("context") .object("client") .value("clientName", "WEB_REMIX") .value("clientVersion", getYoutubeMusicClientVersion()) .value("hl", "en-GB") .value("gl", getExtractorContentCountry().getCountryCode()) .value("platform", "DESKTOP") .value("utcOffsetMinutes", 0) .end() .object("request") .array("internalExperimentFlags") .end() .value("useSsl", true) .end() .object("user") // TODO: provide a way to enable restricted mode with: // .value("enableSafetyMode", boolean) .value("lockedSafetyMode", false) .end() .end() .value("query", getSearchString()) .value("params", params) .end().done().getBytes(StandardCharsets.UTF_8); // @formatter:on final String responseBody = getValidJsonResponseBody( getDownloader().postWithContentTypeJson(url, getYoutubeMusicHeaders(), json)); try { initialData = JsonParser.object().from(responseBody); } catch (final JsonParserException e) { throw new ParsingException("Could not parse JSON", e); } } private List getItemSectionRendererContents() { return initialData .getObject("contents") .getObject("tabbedSearchResultsRenderer") .getArray("tabs") .getObject(0) .getObject("tabRenderer") .getObject("content") .getObject("sectionListRenderer") .getArray("contents") .stream() .filter(JsonObject.class::isInstance) .map(JsonObject.class::cast) .map(c -> c.getObject("itemSectionRenderer")) .filter(isr -> !isr.isEmpty()) .map(isr -> isr .getArray("contents") .getObject(0)) .collect(Collectors.toList()); } @Nonnull @Override public String getSearchSuggestion() throws ParsingException { for (final JsonObject obj : getItemSectionRendererContents()) { final JsonObject didYouMeanRenderer = obj .getObject("didYouMeanRenderer"); final JsonObject showingResultsForRenderer = obj .getObject("showingResultsForRenderer"); if (!didYouMeanRenderer.isEmpty()) { return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); } else if (!showingResultsForRenderer.isEmpty()) { return JsonUtils.getString(showingResultsForRenderer, "correctedQueryEndpoint.searchEndpoint.query"); } } return ""; } @Override public boolean isCorrectedSearch() throws ParsingException { return getItemSectionRendererContents() .stream() .anyMatch(obj -> obj.has("showingResultsForRenderer")); } @Nonnull @Override public List getMetaInfo() { return Collections.emptyList(); } @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); final JsonArray contents = JsonUtils.getArray(JsonUtils.getArray(initialData, "contents.tabbedSearchResultsRenderer.tabs").getObject(0), "tabRenderer.content.sectionListRenderer.contents"); Page nextPage = null; for (final Object content : contents) { if (((JsonObject) content).has("musicShelfRenderer")) { final JsonObject musicShelfRenderer = ((JsonObject) content) .getObject("musicShelfRenderer"); collectMusicStreamsFrom(collector, musicShelfRenderer.getArray("contents")); nextPage = getNextPageFrom(musicShelfRenderer.getArray("continuations")); } } return new InfoItemsPage<>(collector, nextPage); } @Override public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { if (page == null || isNullOrEmpty(page.getUrl())) { throw new IllegalArgumentException("Page doesn't contain an URL"); } final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); // @formatter:off final byte[] json = JsonWriter.string() .object() .object("context") .object("client") .value("clientName", "WEB_REMIX") .value("clientVersion", getYoutubeMusicClientVersion()) .value("hl", "en-GB") .value("gl", getExtractorContentCountry().getCountryCode()) .value("platform", "DESKTOP") .value("utcOffsetMinutes", 0) .end() .object("request") .array("internalExperimentFlags") .end() .value("useSsl", true) .end() .object("user") // TODO: provide a way to enable restricted mode with: // .value("enableSafetyMode", boolean) .value("lockedSafetyMode", false) .end() .end() .end().done().getBytes(StandardCharsets.UTF_8); // @formatter:on final String responseBody = getValidJsonResponseBody( getDownloader().postWithContentTypeJson( page.getUrl(), getYoutubeMusicHeaders(), json)); final JsonObject ajaxJson; try { ajaxJson = JsonParser.object().from(responseBody); } catch (final JsonParserException e) { throw new ParsingException("Could not parse JSON", e); } final JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents") .getObject("musicShelfContinuation"); collectMusicStreamsFrom(collector, musicShelfContinuation.getArray("contents")); final JsonArray continuations = musicShelfContinuation.getArray("continuations"); return new InfoItemsPage<>(collector, getNextPageFrom(continuations)); } private void collectMusicStreamsFrom(final MultiInfoItemsCollector collector, @Nonnull final JsonArray videos) { final String searchType = getLinkHandler().getContentFilters().get(0); videos.stream() .filter(JsonObject.class::isInstance) .map(JsonObject.class::cast) .map(item -> item.getObject("musicResponsiveListItemRenderer", null)) .filter(Objects::nonNull) .forEachOrdered(infoItem -> { final String displayPolicy = infoItem.getString( "musicItemRendererDisplayPolicy", ""); if (displayPolicy.equals("MUSIC_ITEM_RENDERER_DISPLAY_POLICY_GREY_OUT")) { // No info about URL available return; } final JsonArray descriptionElements = infoItem.getArray("flexColumns") .getObject(1) .getObject("musicResponsiveListItemFlexColumnRenderer") .getObject("text") .getArray("runs"); switch (searchType) { case MUSIC_SONGS: case MUSIC_VIDEOS: collector.commit(new YoutubeMusicSongOrVideoInfoItemExtractor( infoItem, descriptionElements, searchType)); break; case MUSIC_ARTISTS: collector.commit(new YoutubeMusicArtistInfoItemExtractor(infoItem)); break; case MUSIC_ALBUMS: case MUSIC_PLAYLISTS: collector.commit(new YoutubeMusicAlbumOrPlaylistInfoItemExtractor( infoItem, descriptionElements, searchType)); break; } }); } @Nullable private Page getNextPageFrom(final JsonArray continuations) { if (isNullOrEmpty(continuations)) { return null; } final JsonObject nextContinuationData = continuations.getObject(0) .getObject("nextContinuationData"); final String continuation = nextContinuationData.getString("continuation"); return new Page("https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation + "&continuation=" + continuation + "&" + DISABLE_PRETTY_PRINT_PARAMETER); } }