package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonWriter; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelTabExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import javax.annotation.Nonnull; import javax.annotation.Nullable; public class YoutubeChannelVideosTabExtractor extends ChannelTabExtractor { private final JsonObject tabRenderer; private final String channelName; private final String channelUrl; public YoutubeChannelVideosTabExtractor(final StreamingService service, final ListLinkHandler linkHandler, final JsonObject tabRenderer, final String channelName, final String channelUrl) { super(service, linkHandler); this.tabRenderer = tabRenderer; this.channelName = channelName; this.channelUrl = channelUrl; } @Override public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { // nothing to do, all data was already fetched and is stored in the link handler } @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); final JsonObject tabContent = tabRenderer.getObject("content"); JsonArray items = tabContent .getObject("sectionListRenderer") .getArray("contents").getObject(0).getObject("itemSectionRenderer") .getArray("contents").getObject(0).getObject("gridRenderer").getArray("items"); if (items.isEmpty()) { items = tabContent.getObject("richGridRenderer").getArray("contents"); } final List channelIds = new ArrayList<>(); channelIds.add(channelName); channelIds.add(channelUrl); final JsonObject continuation = collectStreamsFrom(collector, items, channelIds); return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelIds)); } @Override public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { if (page == null || isNullOrEmpty(page.getUrl())) { throw new IllegalArgumentException("Page doesn't contain an URL"); } final List channelIds = page.getIds(); final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); final JsonObject ajaxJson = getJsonPostResponse("browse", page.getBody(), getExtractorLocalization()); final JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions") .getObject(0) .getObject("appendContinuationItemsAction"); final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation .getArray("continuationItems"), channelIds); return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelIds)); } @Nullable private Page getNextPageFrom(final JsonObject continuations, final List channelIds) throws IOException, ExtractionException { if (isNullOrEmpty(continuations)) { return null; } final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint"); final String continuation = continuationEndpoint.getObject("continuationCommand") .getString("token"); final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(), getExtractorContentCountry()) .value("continuation", continuation) .done()) .getBytes(StandardCharsets.UTF_8); return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body); } /** * Collect streams from an array of items * * @param collector the collector where videos will be committed * @param videos the array to get videos from * @param channelIds the ids of the channel, which are its name and its URL * @return the continuation object */ private JsonObject collectStreamsFrom(@Nonnull final MultiInfoItemsCollector collector, @Nonnull final JsonArray videos, @Nonnull final List channelIds) { collector.reset(); final String uploaderName = channelIds.get(0); final String uploaderUrl = channelIds.get(1); final TimeAgoParser timeAgoParser = getTimeAgoParser(); JsonObject continuation = null; for (final Object object : videos) { final JsonObject video = (JsonObject) object; if (video.has("gridVideoRenderer")) { collector.commit(new YoutubeStreamInfoItemExtractor( video.getObject("gridVideoRenderer"), timeAgoParser) { @Override public String getUploaderName() { return uploaderName; } @Override public String getUploaderUrl() { return uploaderUrl; } }); } else if (video.has("richItemRenderer")) { collector.commit(new YoutubeStreamInfoItemExtractor( video.getObject("richItemRenderer") .getObject("content").getObject("videoRenderer"), timeAgoParser) { @Override public String getUploaderName() { return uploaderName; } @Override public String getUploaderUrl() { return uploaderUrl; } }); } else if (video.has("continuationItemRenderer")) { continuation = video.getObject("continuationItemRenderer"); } } return continuation; } }