2018-05-08 21:19:03 +02:00
|
|
|
package org.schabi.newpipe.extractor.services.youtube.extractors;
|
2017-03-01 18:47:52 +01:00
|
|
|
|
2017-11-22 18:39:38 +01:00
|
|
|
import com.grack.nanojson.JsonArray;
|
2017-08-16 04:40:03 +02:00
|
|
|
import com.grack.nanojson.JsonObject;
|
|
|
|
import com.grack.nanojson.JsonParser;
|
2020-02-22 20:19:41 +01:00
|
|
|
|
2017-03-01 18:47:52 +01:00
|
|
|
import org.jsoup.nodes.Document;
|
|
|
|
import org.jsoup.nodes.Element;
|
|
|
|
import org.mozilla.javascript.Context;
|
|
|
|
import org.mozilla.javascript.Function;
|
|
|
|
import org.mozilla.javascript.ScriptableObject;
|
2019-04-28 22:03:16 +02:00
|
|
|
import org.schabi.newpipe.extractor.MediaFormat;
|
|
|
|
import org.schabi.newpipe.extractor.NewPipe;
|
|
|
|
import org.schabi.newpipe.extractor.StreamingService;
|
|
|
|
import org.schabi.newpipe.extractor.downloader.Downloader;
|
|
|
|
import org.schabi.newpipe.extractor.downloader.Response;
|
2017-07-11 05:08:03 +02:00
|
|
|
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
|
2017-04-12 02:55:53 +02:00
|
|
|
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
|
|
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
|
|
|
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
2018-07-13 18:02:40 +02:00
|
|
|
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
|
2019-11-03 19:45:25 +01:00
|
|
|
import org.schabi.newpipe.extractor.localization.DateWrapper;
|
2019-12-16 08:35:43 +01:00
|
|
|
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
|
2018-05-08 21:19:03 +02:00
|
|
|
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
|
2019-10-29 06:00:29 +01:00
|
|
|
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
|
2020-02-22 20:19:41 +01:00
|
|
|
import org.schabi.newpipe.extractor.stream.AudioStream;
|
|
|
|
import org.schabi.newpipe.extractor.stream.Description;
|
|
|
|
import org.schabi.newpipe.extractor.stream.Frameset;
|
|
|
|
import org.schabi.newpipe.extractor.stream.Stream;
|
|
|
|
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
|
|
|
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
|
|
|
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
|
|
|
|
import org.schabi.newpipe.extractor.stream.StreamType;
|
|
|
|
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
|
|
|
|
import org.schabi.newpipe.extractor.stream.VideoStream;
|
2017-06-29 20:12:55 +02:00
|
|
|
import org.schabi.newpipe.extractor.utils.Parser;
|
2017-07-11 05:08:03 +02:00
|
|
|
import org.schabi.newpipe.extractor.utils.Utils;
|
2017-03-01 18:47:52 +01:00
|
|
|
|
|
|
|
import java.io.IOException;
|
2018-08-16 17:11:18 +02:00
|
|
|
import java.io.UnsupportedEncodingException;
|
2020-02-25 16:24:18 +01:00
|
|
|
import java.text.SimpleDateFormat;
|
|
|
|
import java.util.*;
|
2017-03-01 18:47:52 +01:00
|
|
|
|
2020-02-22 20:19:41 +01:00
|
|
|
import javax.annotation.Nonnull;
|
|
|
|
import javax.annotation.Nullable;
|
|
|
|
|
2017-06-29 20:12:55 +02:00
|
|
|
/*
|
2017-03-01 18:47:52 +01:00
|
|
|
* Created by Christian Schabesberger on 06.08.15.
|
|
|
|
*
|
2019-03-14 08:49:11 +01:00
|
|
|
* Copyright (C) Christian Schabesberger 2019 <chris.schabesberger@mailbox.org>
|
2017-03-01 18:47:52 +01:00
|
|
|
* YoutubeStreamExtractor.java is part of NewPipe.
|
|
|
|
*
|
|
|
|
* NewPipe is free software: you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License as published by
|
|
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
|
|
* (at your option) any later version.
|
|
|
|
*
|
|
|
|
* NewPipe is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
*/
|
|
|
|
|
|
|
|
public class YoutubeStreamExtractor extends StreamExtractor {
|
2017-07-11 05:08:03 +02:00
|
|
|
/*//////////////////////////////////////////////////////////////////////////
|
|
|
|
// Exceptions
|
|
|
|
//////////////////////////////////////////////////////////////////////////*/
|
2017-03-01 18:47:52 +01:00
|
|
|
|
|
|
|
public class DecryptException extends ParsingException {
|
|
|
|
DecryptException(String message, Throwable cause) {
|
|
|
|
super(message, cause);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-07-11 05:08:03 +02:00
|
|
|
/*//////////////////////////////////////////////////////////////////////////*/
|
2017-03-01 18:47:52 +01:00
|
|
|
|
2017-07-11 05:08:03 +02:00
|
|
|
private Document doc;
|
2017-11-25 01:10:04 +01:00
|
|
|
@Nullable
|
2017-08-16 04:40:03 +02:00
|
|
|
private JsonObject playerArgs;
|
2017-11-25 01:10:04 +01:00
|
|
|
@Nonnull
|
|
|
|
private final Map<String, String> videoInfoPage = new HashMap<>();
|
2019-09-11 19:05:41 +02:00
|
|
|
private JsonObject playerResponse;
|
2020-02-22 23:51:02 +01:00
|
|
|
private JsonObject initialData;
|
2018-02-02 08:24:22 +01:00
|
|
|
|
2018-02-01 22:27:14 +01:00
|
|
|
@Nonnull
|
2018-02-02 08:24:22 +01:00
|
|
|
private List<SubtitlesInfo> subtitlesInfos = new ArrayList<>();
|
2017-08-10 19:50:59 +02:00
|
|
|
|
|
|
|
private boolean isAgeRestricted;
|
2017-03-01 18:47:52 +01:00
|
|
|
|
2019-04-28 22:03:16 +02:00
|
|
|
public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) {
|
|
|
|
super(service, linkHandler);
|
2017-03-01 18:47:52 +01:00
|
|
|
}
|
|
|
|
|
2017-07-11 05:08:03 +02:00
|
|
|
/*//////////////////////////////////////////////////////////////////////////
|
|
|
|
// Impl
|
|
|
|
//////////////////////////////////////////////////////////////////////////*/
|
2017-03-01 18:47:52 +01:00
|
|
|
|
2017-11-25 01:10:04 +01:00
|
|
|
@Nonnull
|
2017-03-01 18:47:52 +01:00
|
|
|
@Override
|
2017-08-11 03:23:09 +02:00
|
|
|
public String getName() throws ParsingException {
|
2017-11-30 10:49:27 +01:00
|
|
|
assertPageFetched();
|
2020-02-25 09:50:22 +01:00
|
|
|
String title = null;
|
2020-01-06 20:45:57 +01:00
|
|
|
try {
|
2020-02-25 09:50:22 +01:00
|
|
|
title = getVideoPrimaryInfoRenderer().getObject("title").getArray("runs").getObject(0).getString("text");
|
|
|
|
} catch (Exception ignored) {}
|
|
|
|
if (title == null) {
|
2017-11-25 01:10:04 +01:00
|
|
|
try {
|
2020-02-25 10:05:53 +01:00
|
|
|
title = playerResponse.getObject("videoDetails").getString("title");
|
2020-02-25 09:50:22 +01:00
|
|
|
} catch (Exception ignored) {}
|
2017-11-25 01:10:04 +01:00
|
|
|
}
|
2020-02-25 09:50:22 +01:00
|
|
|
if (title != null) return title;
|
|
|
|
throw new ParsingException("Could not get name");
|
2017-03-01 18:47:52 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2019-04-28 22:03:16 +02:00
|
|
|
public String getTextualUploadDate() throws ParsingException {
|
|
|
|
if (getStreamType().equals(StreamType.LIVE_STREAM)) {
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
|
2020-02-25 16:24:18 +01:00
|
|
|
try {
|
|
|
|
//return playerResponse.getObject("microformat").getObject("playerMicroformatRenderer").getString("publishDate");
|
|
|
|
} catch (Exception ignored) {}
|
2020-02-25 09:50:22 +01:00
|
|
|
|
2017-03-01 18:47:52 +01:00
|
|
|
try {
|
2020-02-25 17:40:23 +01:00
|
|
|
// TODO this parses English formatted dates only, we need a better approach to parse the textual date
|
|
|
|
Date d = new SimpleDateFormat("dd MMM yyy").parse(getVideoPrimaryInfoRenderer()
|
|
|
|
.getObject("dateText").getString("simpleText"));
|
|
|
|
return new SimpleDateFormat("yyyy-MM-dd").format(d);
|
|
|
|
} catch (Exception ignored) {}
|
2020-02-25 16:24:18 +01:00
|
|
|
throw new ParsingException("Could not get upload date");
|
2017-03-01 18:47:52 +01:00
|
|
|
}
|
|
|
|
|
2019-04-28 22:03:16 +02:00
|
|
|
@Override
|
2019-11-03 19:45:25 +01:00
|
|
|
public DateWrapper getUploadDate() throws ParsingException {
|
2019-04-28 22:03:16 +02:00
|
|
|
final String textualUploadDate = getTextualUploadDate();
|
|
|
|
|
|
|
|
if (textualUploadDate == null) {
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
|
2019-12-16 08:35:43 +01:00
|
|
|
return new DateWrapper(YoutubeParsingHelper.parseDateFrom(textualUploadDate), true);
|
2019-04-28 22:03:16 +02:00
|
|
|
}
|
|
|
|
|
2017-11-25 01:10:04 +01:00
|
|
|
@Nonnull
|
2017-03-01 18:47:52 +01:00
|
|
|
@Override
|
2017-08-11 03:23:09 +02:00
|
|
|
public String getThumbnailUrl() throws ParsingException {
|
2017-11-30 10:49:27 +01:00
|
|
|
assertPageFetched();
|
2017-03-01 18:47:52 +01:00
|
|
|
try {
|
2020-01-06 20:45:57 +01:00
|
|
|
JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails");
|
|
|
|
// the last thumbnail is the one with the highest resolution
|
2020-01-20 22:52:36 +01:00
|
|
|
return thumbnails.getObject(thumbnails.size() - 1).getString("url");
|
2017-08-10 19:50:59 +02:00
|
|
|
|
2017-03-01 18:47:52 +01:00
|
|
|
} catch (Exception e) {
|
2020-02-25 09:50:22 +01:00
|
|
|
throw new ParsingException("Could not get thumbnail url");
|
2017-08-11 03:23:09 +02:00
|
|
|
}
|
2020-01-06 20:45:57 +01:00
|
|
|
|
2017-08-11 03:23:09 +02:00
|
|
|
}
|
|
|
|
|
2017-11-25 01:10:04 +01:00
|
|
|
@Nonnull
|
2017-08-11 03:23:09 +02:00
|
|
|
@Override
|
2020-02-06 23:35:46 +01:00
|
|
|
public Description getDescription() throws ParsingException {
|
2017-11-30 10:49:27 +01:00
|
|
|
assertPageFetched();
|
2020-02-25 16:24:18 +01:00
|
|
|
// raw non-html description
|
2017-08-11 03:23:09 +02:00
|
|
|
try {
|
2020-02-25 09:50:22 +01:00
|
|
|
return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT);
|
2020-02-25 16:24:18 +01:00
|
|
|
} catch (Exception ignored) { }
|
|
|
|
try {
|
|
|
|
JsonArray descriptions = getVideoSecondaryInfoRenderer().getObject("description").getArray("runs");
|
|
|
|
StringBuilder descriptionBuilder = new StringBuilder(descriptions.size());
|
|
|
|
for (Object textObjectHolder : descriptions) {
|
|
|
|
JsonObject textHolder = (JsonObject) textObjectHolder;
|
|
|
|
String text = textHolder.getString("text");
|
|
|
|
if (text != null) descriptionBuilder.append(text);
|
|
|
|
}
|
|
|
|
String description = descriptionBuilder.toString();
|
|
|
|
if (!description.isEmpty()) return new Description(description, Description.PLAIN_TEXT);
|
|
|
|
} catch (Exception ignored) { }
|
|
|
|
throw new ParsingException("Could not get description");
|
2017-08-11 03:23:09 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public int getAgeLimit() throws ParsingException {
|
2017-11-30 10:49:27 +01:00
|
|
|
assertPageFetched();
|
2017-08-11 03:23:09 +02:00
|
|
|
if (!isAgeRestricted) {
|
2017-11-25 02:03:30 +01:00
|
|
|
return NO_AGE_LIMIT;
|
2017-08-11 03:23:09 +02:00
|
|
|
}
|
|
|
|
try {
|
|
|
|
return Integer.valueOf(doc.select("meta[property=\"og:restrictions:age\"]")
|
|
|
|
.attr(CONTENT).replace("+", ""));
|
|
|
|
} catch (Exception e) {
|
|
|
|
throw new ParsingException("Could not get age restriction");
|
2017-03-01 18:47:52 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2017-08-10 19:50:59 +02:00
|
|
|
public long getLength() throws ParsingException {
|
2017-11-30 10:49:27 +01:00
|
|
|
assertPageFetched();
|
2019-07-30 20:53:23 +02:00
|
|
|
|
2017-08-10 19:50:59 +02:00
|
|
|
try {
|
2019-07-30 20:53:23 +02:00
|
|
|
String duration = playerResponse
|
|
|
|
.getObject("videoDetails")
|
|
|
|
.getString("lengthSeconds");
|
|
|
|
return Long.parseLong(duration);
|
2017-08-10 19:50:59 +02:00
|
|
|
} catch (Exception e) {
|
2020-01-06 20:45:57 +01:00
|
|
|
try {
|
|
|
|
String durationMs = playerResponse
|
|
|
|
.getObject("streamingData")
|
|
|
|
.getArray("formats")
|
|
|
|
.getObject(0)
|
|
|
|
.getString("approxDurationMs");
|
2020-01-20 22:52:48 +01:00
|
|
|
return Math.round(Long.parseLong(durationMs) / 1000f);
|
2020-01-06 20:45:57 +01:00
|
|
|
} catch (Exception ignored) {
|
|
|
|
throw new ParsingException("Could not get duration", e);
|
|
|
|
}
|
2017-03-01 18:47:52 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-08-11 03:23:09 +02:00
|
|
|
/**
|
|
|
|
* Attempts to parse (and return) the offset to start playing the video from.
|
|
|
|
*
|
|
|
|
* @return the offset (in seconds), or 0 if no timestamp is found.
|
|
|
|
*/
|
|
|
|
@Override
|
|
|
|
public long getTimeStamp() throws ParsingException {
|
2017-11-22 18:45:49 +01:00
|
|
|
return getTimestampSeconds("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)");
|
2017-08-11 03:23:09 +02:00
|
|
|
}
|
|
|
|
|
2017-03-01 18:47:52 +01:00
|
|
|
@Override
|
|
|
|
public long getViewCount() throws ParsingException {
|
2017-11-30 10:49:27 +01:00
|
|
|
assertPageFetched();
|
2020-02-25 09:50:22 +01:00
|
|
|
String views = null;
|
2017-03-01 18:47:52 +01:00
|
|
|
try {
|
2020-02-25 09:50:22 +01:00
|
|
|
views = getVideoPrimaryInfoRenderer().getObject("viewCount")
|
|
|
|
.getObject("videoViewCountRenderer").getObject("viewCount")
|
|
|
|
.getArray("runs").getObject(0).getString("text");
|
|
|
|
} catch (Exception ignored) {}
|
|
|
|
if (views == null) {
|
2020-01-06 20:45:57 +01:00
|
|
|
try {
|
2020-02-25 09:50:22 +01:00
|
|
|
views = getVideoPrimaryInfoRenderer().getObject("viewCount")
|
|
|
|
.getObject("videoViewCountRenderer").getObject("viewCount").getString("simpleText");
|
|
|
|
} catch (Exception ignored) {}
|
2020-02-24 16:04:01 +01:00
|
|
|
}
|
2020-02-25 09:50:22 +01:00
|
|
|
if (views == null) {
|
|
|
|
try {
|
|
|
|
views = playerResponse.getObject("videoDetails").getString("viewCount");
|
|
|
|
} catch (Exception ignored) {}
|
2020-02-24 16:04:01 +01:00
|
|
|
}
|
2020-02-25 10:08:52 +01:00
|
|
|
if (views != null) return Long.parseLong(Utils.removeNonDigitCharacters(views));
|
2020-02-25 09:50:22 +01:00
|
|
|
throw new ParsingException("Could not get view count");
|
2020-02-24 16:04:01 +01:00
|
|
|
}
|
|
|
|
|
2017-03-01 18:47:52 +01:00
|
|
|
@Override
|
2017-08-11 03:23:09 +02:00
|
|
|
public long getLikeCount() throws ParsingException {
|
2017-11-30 10:49:27 +01:00
|
|
|
assertPageFetched();
|
2017-08-11 03:23:09 +02:00
|
|
|
String likesString = "";
|
2017-03-01 18:47:52 +01:00
|
|
|
try {
|
2017-08-11 03:23:09 +02:00
|
|
|
try {
|
2020-02-24 16:04:01 +01:00
|
|
|
likesString = getVideoPrimaryInfoRenderer().getObject("sentimentBar")
|
|
|
|
.getObject("sentimentBarRenderer").getString("tooltip").split("/")[0];
|
2017-08-11 03:23:09 +02:00
|
|
|
} catch (NullPointerException e) {
|
2020-01-06 20:45:57 +01:00
|
|
|
//if this kicks in our button has no content and therefore ratings must be disabled
|
|
|
|
if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
|
|
|
|
throw new ParsingException("Ratings are enabled even though the like button is missing", e);
|
|
|
|
}
|
2017-08-11 03:23:09 +02:00
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
return Integer.parseInt(Utils.removeNonDigitCharacters(likesString));
|
|
|
|
} catch (NumberFormatException nfe) {
|
|
|
|
throw new ParsingException("Could not parse \"" + likesString + "\" as an Integer", nfe);
|
|
|
|
} catch (Exception e) {
|
|
|
|
throw new ParsingException("Could not get like count", e);
|
2017-03-01 18:47:52 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2017-08-11 03:23:09 +02:00
|
|
|
public long getDislikeCount() throws ParsingException {
|
2017-11-30 10:49:27 +01:00
|
|
|
assertPageFetched();
|
2017-08-11 03:23:09 +02:00
|
|
|
String dislikesString = "";
|
2017-03-01 18:47:52 +01:00
|
|
|
try {
|
2017-08-11 03:23:09 +02:00
|
|
|
try {
|
2020-02-24 16:04:01 +01:00
|
|
|
dislikesString = getVideoPrimaryInfoRenderer().getObject("sentimentBar")
|
|
|
|
.getObject("sentimentBarRenderer").getString("tooltip").split("/")[1];
|
2017-08-11 03:23:09 +02:00
|
|
|
} catch (NullPointerException e) {
|
2020-01-06 20:45:57 +01:00
|
|
|
//if this kicks in our button has no content and therefore ratings must be disabled
|
|
|
|
if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
|
|
|
|
throw new ParsingException("Ratings are enabled even though the dislike button is missing", e);
|
|
|
|
}
|
2017-08-11 03:23:09 +02:00
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString));
|
|
|
|
} catch (NumberFormatException nfe) {
|
|
|
|
throw new ParsingException("Could not parse \"" + dislikesString + "\" as an Integer", nfe);
|
|
|
|
} catch (Exception e) {
|
|
|
|
throw new ParsingException("Could not get dislike count", e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-11-25 01:10:04 +01:00
|
|
|
@Nonnull
|
2017-08-11 03:23:09 +02:00
|
|
|
@Override
|
|
|
|
public String getUploaderUrl() throws ParsingException {
|
2017-11-30 10:49:27 +01:00
|
|
|
assertPageFetched();
|
2020-02-25 10:05:53 +01:00
|
|
|
String uploaderId = null;
|
2017-08-11 03:23:09 +02:00
|
|
|
try {
|
2020-02-25 10:05:53 +01:00
|
|
|
uploaderId = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer")
|
|
|
|
.getObject("navigationEndpoint").getObject("browseEndpoint").getString("browseId");
|
|
|
|
} catch (Exception ignored) {}
|
|
|
|
if (uploaderId == null) {
|
2020-01-06 20:45:57 +01:00
|
|
|
try {
|
2020-02-25 10:05:53 +01:00
|
|
|
uploaderId = playerResponse.getObject("videoDetails").getString("channelId");
|
2020-01-06 20:45:57 +01:00
|
|
|
} catch (Exception ignored) {}
|
2017-11-25 01:10:04 +01:00
|
|
|
}
|
2020-02-25 10:05:53 +01:00
|
|
|
if (uploaderId != null) return "https://www.youtube.com/channel/" + uploaderId;
|
|
|
|
throw new ParsingException("Could not get uploader url");
|
2017-11-25 01:10:04 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Nonnull
|
2017-08-11 03:23:09 +02:00
|
|
|
@Override
|
|
|
|
public String getUploaderName() throws ParsingException {
|
2017-11-30 10:49:27 +01:00
|
|
|
assertPageFetched();
|
2020-02-25 10:05:53 +01:00
|
|
|
String uploaderName = null;
|
2020-01-06 20:45:57 +01:00
|
|
|
try {
|
2020-02-25 10:05:53 +01:00
|
|
|
uploaderName = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer")
|
|
|
|
.getObject("title").getArray("runs").getObject(0).getString("text");
|
|
|
|
} catch (Exception ignored) {}
|
|
|
|
if (uploaderName == null) {
|
2017-11-25 01:10:04 +01:00
|
|
|
try {
|
2020-02-25 10:05:53 +01:00
|
|
|
uploaderName = playerResponse.getObject("videoDetails").getString("author");
|
2020-01-06 20:45:57 +01:00
|
|
|
} catch (Exception ignored) {}
|
2017-08-10 19:50:59 +02:00
|
|
|
}
|
2020-02-25 10:05:53 +01:00
|
|
|
if (uploaderName != null) return uploaderName;
|
|
|
|
throw new ParsingException("Could not get uploader name");
|
2017-03-01 18:47:52 +01:00
|
|
|
}
|
|
|
|
|
2017-11-25 01:10:04 +01:00
|
|
|
@Nonnull
|
2017-03-01 18:47:52 +01:00
|
|
|
@Override
|
2017-08-08 23:36:11 +02:00
|
|
|
public String getUploaderAvatarUrl() throws ParsingException {
|
2017-11-30 10:49:27 +01:00
|
|
|
assertPageFetched();
|
2020-01-06 20:45:57 +01:00
|
|
|
|
2020-02-25 17:40:23 +01:00
|
|
|
String uploaderAvatarUrl = null;
|
2017-03-01 18:47:52 +01:00
|
|
|
try {
|
2020-02-22 23:51:02 +01:00
|
|
|
uploaderAvatarUrl = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("secondaryResults")
|
2020-02-17 11:41:11 +01:00
|
|
|
.getObject("secondaryResults").getArray("results").getObject(0).getObject("compactAutoplayRenderer")
|
|
|
|
.getArray("contents").getObject(0).getObject("compactVideoRenderer").getObject("channelThumbnail")
|
|
|
|
.getArray("thumbnails").getObject(0).getString("url");
|
2020-02-18 13:05:11 +01:00
|
|
|
if (uploaderAvatarUrl != null && !uploaderAvatarUrl.isEmpty()) {
|
|
|
|
return uploaderAvatarUrl;
|
|
|
|
}
|
|
|
|
} catch (Exception ignored) {}
|
2020-02-17 11:41:11 +01:00
|
|
|
|
2020-02-18 13:05:11 +01:00
|
|
|
try {
|
2020-02-25 17:40:23 +01:00
|
|
|
uploaderAvatarUrl = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer")
|
|
|
|
.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
|
|
|
|
} catch (Exception ignored) {}
|
2020-01-06 20:45:57 +01:00
|
|
|
|
|
|
|
if (uploaderAvatarUrl == null) {
|
|
|
|
throw new ParsingException("Could not get uploader avatar url");
|
2017-03-01 18:47:52 +01:00
|
|
|
}
|
2020-01-06 20:45:57 +01:00
|
|
|
return uploaderAvatarUrl;
|
2017-03-01 18:47:52 +01:00
|
|
|
}
|
|
|
|
|
2018-02-25 22:03:32 +01:00
|
|
|
@Nonnull
|
2017-03-01 18:47:52 +01:00
|
|
|
@Override
|
|
|
|
public String getDashMpdUrl() throws ParsingException {
|
2017-11-30 10:49:27 +01:00
|
|
|
assertPageFetched();
|
2017-03-01 18:47:52 +01:00
|
|
|
try {
|
2017-07-11 05:08:03 +02:00
|
|
|
String dashManifestUrl;
|
2017-11-25 01:10:04 +01:00
|
|
|
if (videoInfoPage.containsKey("dashmpd")) {
|
2017-03-01 18:47:52 +01:00
|
|
|
dashManifestUrl = videoInfoPage.get("dashmpd");
|
2017-11-25 01:10:04 +01:00
|
|
|
} else if (playerArgs != null && playerArgs.isString("dashmpd")) {
|
2017-08-16 04:40:03 +02:00
|
|
|
dashManifestUrl = playerArgs.getString("dashmpd", "");
|
2017-03-01 18:47:52 +01:00
|
|
|
} else {
|
|
|
|
return "";
|
|
|
|
}
|
2017-08-10 19:50:59 +02:00
|
|
|
|
2017-06-29 20:12:55 +02:00
|
|
|
if (!dashManifestUrl.contains("/signature/")) {
|
2017-03-01 18:47:52 +01:00
|
|
|
String encryptedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifestUrl);
|
|
|
|
String decryptedSig;
|
|
|
|
|
|
|
|
decryptedSig = decryptSignature(encryptedSig, decryptionCode);
|
|
|
|
dashManifestUrl = dashManifestUrl.replace("/s/" + encryptedSig, "/signature/" + decryptedSig);
|
|
|
|
}
|
2017-08-10 19:50:59 +02:00
|
|
|
|
2017-03-01 18:47:52 +01:00
|
|
|
return dashManifestUrl;
|
|
|
|
} catch (Exception e) {
|
2017-08-10 19:50:59 +02:00
|
|
|
throw new ParsingException("Could not get dash manifest url", e);
|
2017-03-01 18:47:52 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-02-25 22:03:32 +01:00
|
|
|
@Nonnull
|
|
|
|
@Override
|
|
|
|
public String getHlsUrl() throws ParsingException {
|
|
|
|
assertPageFetched();
|
2019-01-19 13:50:02 +01:00
|
|
|
|
2019-09-11 20:12:30 +02:00
|
|
|
try {
|
|
|
|
return playerResponse.getObject("streamingData").getString("hlsManifestUrl");
|
2018-02-25 22:03:32 +01:00
|
|
|
} catch (Exception e) {
|
2019-09-11 20:12:30 +02:00
|
|
|
if (playerArgs != null && playerArgs.isString("hlsvp")) {
|
|
|
|
return playerArgs.getString("hlsvp");
|
|
|
|
} else {
|
|
|
|
throw new ParsingException("Could not get hls manifest url", e);
|
|
|
|
}
|
2018-02-25 22:03:32 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-03-01 18:47:52 +01:00
|
|
|
@Override
|
2019-09-11 20:04:28 +02:00
|
|
|
public List<AudioStream> getAudioStreams() throws ExtractionException {
|
2017-11-30 10:49:27 +01:00
|
|
|
assertPageFetched();
|
2017-08-06 22:20:15 +02:00
|
|
|
List<AudioStream> audioStreams = new ArrayList<>();
|
2017-06-29 20:12:55 +02:00
|
|
|
try {
|
2019-09-11 20:04:28 +02:00
|
|
|
for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO).entrySet()) {
|
2017-08-10 19:50:59 +02:00
|
|
|
ItagItem itag = entry.getValue();
|
2017-03-01 18:47:52 +01:00
|
|
|
|
2017-11-11 01:21:43 +01:00
|
|
|
AudioStream audioStream = new AudioStream(entry.getKey(), itag.getMediaFormat(), itag.avgBitrate);
|
2017-08-10 19:50:59 +02:00
|
|
|
if (!Stream.containSimilarStream(audioStream, audioStreams)) {
|
|
|
|
audioStreams.add(audioStream);
|
2017-03-01 18:47:52 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
} catch (Exception e) {
|
2017-08-10 19:50:59 +02:00
|
|
|
throw new ParsingException("Could not get audio streams", e);
|
2017-03-01 18:47:52 +01:00
|
|
|
}
|
2017-08-10 19:50:59 +02:00
|
|
|
|
2017-03-01 18:47:52 +01:00
|
|
|
return audioStreams;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2019-09-11 20:04:28 +02:00
|
|
|
public List<VideoStream> getVideoStreams() throws ExtractionException {
|
2017-11-30 10:49:27 +01:00
|
|
|
assertPageFetched();
|
2017-08-06 22:20:15 +02:00
|
|
|
List<VideoStream> videoStreams = new ArrayList<>();
|
2017-06-29 20:12:55 +02:00
|
|
|
try {
|
2019-09-11 20:04:28 +02:00
|
|
|
for (Map.Entry<String, ItagItem> entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) {
|
2017-08-10 19:50:59 +02:00
|
|
|
ItagItem itag = entry.getValue();
|
|
|
|
|
2017-11-11 01:21:43 +01:00
|
|
|
VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString);
|
2017-08-10 19:50:59 +02:00
|
|
|
if (!Stream.containSimilarStream(videoStream, videoStreams)) {
|
|
|
|
videoStreams.add(videoStream);
|
2017-03-01 18:47:52 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
} catch (Exception e) {
|
2017-08-10 19:50:59 +02:00
|
|
|
throw new ParsingException("Could not get video streams", e);
|
2017-03-01 18:47:52 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
return videoStreams;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2018-08-21 17:23:56 +02:00
|
|
|
public List<VideoStream> getVideoOnlyStreams() throws ExtractionException {
|
2017-11-30 10:49:27 +01:00
|
|
|
assertPageFetched();
|
2017-08-06 22:20:15 +02:00
|
|
|
List<VideoStream> videoOnlyStreams = new ArrayList<>();
|
2017-04-12 02:55:53 +02:00
|
|
|
try {
|
2019-09-11 20:04:28 +02:00
|
|
|
for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) {
|
2017-08-10 19:50:59 +02:00
|
|
|
ItagItem itag = entry.getValue();
|
2017-04-12 02:55:53 +02:00
|
|
|
|
2017-11-11 01:21:43 +01:00
|
|
|
VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString, true);
|
2017-08-10 19:50:59 +02:00
|
|
|
if (!Stream.containSimilarStream(videoStream, videoOnlyStreams)) {
|
|
|
|
videoOnlyStreams.add(videoStream);
|
2017-04-12 02:55:53 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
} catch (Exception e) {
|
2017-08-10 19:50:59 +02:00
|
|
|
throw new ParsingException("Could not get video only streams", e);
|
2017-04-12 02:55:53 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
return videoOnlyStreams;
|
2017-03-01 18:47:52 +01:00
|
|
|
}
|
|
|
|
|
2017-11-22 18:39:38 +01:00
|
|
|
@Override
|
2018-02-01 22:27:14 +01:00
|
|
|
@Nonnull
|
2020-02-25 09:07:22 +01:00
|
|
|
public List<SubtitlesStream> getSubtitlesDefault() {
|
2018-09-24 21:04:22 +02:00
|
|
|
return getSubtitles(MediaFormat.TTML);
|
2017-11-23 16:33:03 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2018-02-01 22:27:14 +01:00
|
|
|
@Nonnull
|
2020-02-25 09:07:22 +01:00
|
|
|
public List<SubtitlesStream> getSubtitles(final MediaFormat format) {
|
2017-11-30 10:49:27 +01:00
|
|
|
assertPageFetched();
|
2018-09-24 21:04:22 +02:00
|
|
|
List<SubtitlesStream> subtitles = new ArrayList<>();
|
2018-02-02 08:24:22 +01:00
|
|
|
for (final SubtitlesInfo subtitlesInfo : subtitlesInfos) {
|
|
|
|
subtitles.add(subtitlesInfo.getSubtitle(format));
|
2017-11-23 11:47:05 +01:00
|
|
|
}
|
2018-02-01 22:27:14 +01:00
|
|
|
return subtitles;
|
2017-11-22 18:39:38 +01:00
|
|
|
}
|
|
|
|
|
2017-03-01 18:47:52 +01:00
|
|
|
@Override
|
2017-08-11 03:23:09 +02:00
|
|
|
public StreamType getStreamType() throws ParsingException {
|
2018-02-25 23:31:42 +01:00
|
|
|
assertPageFetched();
|
|
|
|
try {
|
2020-01-06 20:45:57 +01:00
|
|
|
if (!playerResponse.getObject("streamingData").has(FORMATS) ||
|
|
|
|
(playerArgs != null && playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))) {
|
2018-02-25 23:31:42 +01:00
|
|
|
return StreamType.LIVE_STREAM;
|
|
|
|
}
|
|
|
|
} catch (Exception e) {
|
2020-01-06 20:45:57 +01:00
|
|
|
throw new ParsingException("Could not get stream type", e);
|
2018-02-25 23:31:42 +01:00
|
|
|
}
|
2017-08-11 03:23:09 +02:00
|
|
|
return StreamType.VIDEO_STREAM;
|
2017-03-01 18:47:52 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2020-02-25 09:07:22 +01:00
|
|
|
public StreamInfoItem getNextStream() throws ExtractionException {
|
2017-11-30 10:49:27 +01:00
|
|
|
assertPageFetched();
|
2020-02-25 16:14:49 +01:00
|
|
|
if (isAgeRestricted) {
|
|
|
|
return null;
|
|
|
|
}
|
2017-03-01 18:47:52 +01:00
|
|
|
try {
|
2020-02-22 23:51:02 +01:00
|
|
|
final JsonObject videoInfo = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
|
2020-02-19 19:14:05 +01:00
|
|
|
.getObject("secondaryResults").getObject("secondaryResults").getArray("results")
|
|
|
|
.getObject(0).getObject("compactAutoplayRenderer").getArray("contents")
|
|
|
|
.getObject(0).getObject("compactVideoRenderer");
|
2019-04-28 22:03:16 +02:00
|
|
|
final TimeAgoParser timeAgoParser = getTimeAgoParser();
|
2020-02-19 19:14:05 +01:00
|
|
|
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
2017-08-10 19:50:59 +02:00
|
|
|
|
2020-02-22 20:19:41 +01:00
|
|
|
collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser));
|
2018-03-11 21:50:40 +01:00
|
|
|
return collector.getItems().get(0);
|
2017-06-29 20:12:55 +02:00
|
|
|
} catch (Exception e) {
|
2017-03-01 18:47:52 +01:00
|
|
|
throw new ParsingException("Could not get next video", e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2020-02-18 17:04:22 +01:00
|
|
|
public StreamInfoItemsCollector getRelatedStreams() throws ExtractionException {
|
2017-11-30 10:49:27 +01:00
|
|
|
assertPageFetched();
|
2020-02-25 16:14:49 +01:00
|
|
|
if (isAgeRestricted) {
|
|
|
|
return null;
|
|
|
|
}
|
2017-03-01 18:47:52 +01:00
|
|
|
try {
|
2018-02-24 22:20:50 +01:00
|
|
|
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
|
2020-02-22 23:51:02 +01:00
|
|
|
JsonArray results = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
|
2020-02-18 17:04:22 +01:00
|
|
|
.getObject("secondaryResults").getObject("secondaryResults").getArray("results");
|
|
|
|
|
2019-04-28 22:03:16 +02:00
|
|
|
final TimeAgoParser timeAgoParser = getTimeAgoParser();
|
|
|
|
|
2020-02-18 17:04:22 +01:00
|
|
|
for (Object ul : results) {
|
|
|
|
final JsonObject videoInfo = ((JsonObject) ul).getObject("compactVideoRenderer");
|
|
|
|
|
2020-02-22 20:19:41 +01:00
|
|
|
if (videoInfo != null) collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser));
|
2017-03-01 18:47:52 +01:00
|
|
|
}
|
|
|
|
return collector;
|
2017-06-29 20:12:55 +02:00
|
|
|
} catch (Exception e) {
|
2017-03-01 18:47:52 +01:00
|
|
|
throw new ParsingException("Could not get related videos", e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-06-29 20:12:55 +02:00
|
|
|
/**
|
2017-07-11 05:08:03 +02:00
|
|
|
* {@inheritDoc}
|
2017-06-29 20:12:55 +02:00
|
|
|
*/
|
2017-07-11 05:08:03 +02:00
|
|
|
@Override
|
|
|
|
public String getErrorMessage() {
|
|
|
|
StringBuilder errorReason;
|
2019-09-23 10:44:17 +02:00
|
|
|
Element errorElement = doc.select("h1[id=\"unavailable-message\"]").first();
|
2017-03-01 18:47:52 +01:00
|
|
|
|
2019-09-23 10:44:17 +02:00
|
|
|
if (errorElement == null) {
|
2017-07-11 05:08:03 +02:00
|
|
|
errorReason = null;
|
|
|
|
} else {
|
2019-09-23 10:44:17 +02:00
|
|
|
String errorMessage = errorElement.text();
|
|
|
|
if (errorMessage == null || errorMessage.isEmpty()) {
|
|
|
|
errorReason = null;
|
|
|
|
} else {
|
|
|
|
errorReason = new StringBuilder(errorMessage);
|
|
|
|
errorReason.append(" ");
|
|
|
|
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
|
|
|
|
}
|
2017-07-11 05:08:03 +02:00
|
|
|
}
|
2017-03-01 18:47:52 +01:00
|
|
|
|
2019-12-21 13:17:58 +01:00
|
|
|
return errorReason != null ? errorReason.toString() : "";
|
2017-07-11 05:08:03 +02:00
|
|
|
}
|
2017-03-01 18:47:52 +01:00
|
|
|
|
2017-07-11 05:08:03 +02:00
|
|
|
/*//////////////////////////////////////////////////////////////////////////
|
2017-08-10 19:50:59 +02:00
|
|
|
// Fetch page
|
2017-07-11 05:08:03 +02:00
|
|
|
//////////////////////////////////////////////////////////////////////////*/
|
2017-03-01 18:47:52 +01:00
|
|
|
|
2019-09-11 20:04:28 +02:00
|
|
|
private static final String FORMATS = "formats";
|
|
|
|
private static final String ADAPTIVE_FORMATS = "adaptiveFormats";
|
2017-07-11 05:08:03 +02:00
|
|
|
private static final String HTTPS = "https:";
|
|
|
|
private static final String CONTENT = "content";
|
|
|
|
private static final String DECRYPTION_FUNC_NAME = "decrypt";
|
|
|
|
|
2018-03-02 01:31:36 +01:00
|
|
|
private static final String VERIFIED_URL_PARAMS = "&has_verified=1&bpctr=9999999999";
|
|
|
|
|
2020-01-23 21:20:38 +01:00
|
|
|
private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX =
|
2019-03-14 08:49:11 +01:00
|
|
|
"([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;";
|
2020-01-24 01:59:39 +01:00
|
|
|
private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX_2 =
|
|
|
|
"\\b([\\w$]{2})\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;";
|
2018-09-08 07:25:07 +02:00
|
|
|
private final static String DECRYPTION_AKAMAIZED_STRING_REGEX =
|
2019-01-18 11:47:34 +01:00
|
|
|
"yt\\.akamaized\\.net/\\)\\s*\\|\\|\\s*.*?\\s*c\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\(";
|
2018-09-08 07:25:07 +02:00
|
|
|
private final static String DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX =
|
2019-01-18 11:47:34 +01:00
|
|
|
"\\bc\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\(";
|
2018-09-08 07:25:07 +02:00
|
|
|
|
2017-11-30 11:20:49 +01:00
|
|
|
private volatile String decryptionCode = "";
|
2017-07-11 05:08:03 +02:00
|
|
|
|
2017-11-30 11:20:49 +01:00
|
|
|
private String pageHtml = null;
|
2017-11-22 18:39:38 +01:00
|
|
|
|
2017-08-06 22:20:15 +02:00
|
|
|
@Override
|
2017-11-28 13:37:01 +01:00
|
|
|
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
|
2019-10-29 06:00:29 +01:00
|
|
|
final String verifiedUrl = getUrl() + VERIFIED_URL_PARAMS;
|
2019-04-28 22:03:16 +02:00
|
|
|
final Response response = downloader.get(verifiedUrl, getExtractorLocalization());
|
|
|
|
pageHtml = response.responseBody();
|
2019-10-29 06:00:29 +01:00
|
|
|
doc = YoutubeParsingHelper.parseAndCheckPage(verifiedUrl, response);
|
2017-07-11 05:08:03 +02:00
|
|
|
|
2017-12-18 23:05:58 +01:00
|
|
|
final String playerUrl;
|
2017-07-11 05:08:03 +02:00
|
|
|
// Check if the video is age restricted
|
2020-01-25 21:08:17 +01:00
|
|
|
if (!doc.select("meta[property=\"og:restrictions:age\"]").isEmpty()) {
|
2017-12-18 23:05:58 +01:00
|
|
|
final EmbeddedInfo info = getEmbeddedInfo();
|
|
|
|
final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts);
|
2019-04-28 22:03:16 +02:00
|
|
|
final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody();
|
2017-11-25 01:10:04 +01:00
|
|
|
videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
|
2017-12-18 23:05:58 +01:00
|
|
|
playerUrl = info.url;
|
2017-07-11 05:08:03 +02:00
|
|
|
isAgeRestricted = true;
|
|
|
|
} else {
|
2019-10-29 06:00:29 +01:00
|
|
|
final JsonObject ytPlayerConfig = getPlayerConfig();
|
2017-07-11 05:08:03 +02:00
|
|
|
playerArgs = getPlayerArgs(ytPlayerConfig);
|
|
|
|
playerUrl = getPlayerUrl(ytPlayerConfig);
|
|
|
|
isAgeRestricted = false;
|
|
|
|
}
|
2019-09-11 19:05:41 +02:00
|
|
|
playerResponse = getPlayerResponse();
|
2020-02-22 23:51:02 +01:00
|
|
|
initialData = YoutubeParsingHelper.getInitialData(pageHtml);
|
2017-07-11 05:08:03 +02:00
|
|
|
|
|
|
|
if (decryptionCode.isEmpty()) {
|
|
|
|
decryptionCode = loadDecryptionCode(playerUrl);
|
|
|
|
}
|
2018-02-01 22:27:14 +01:00
|
|
|
|
2018-02-02 08:24:22 +01:00
|
|
|
if (subtitlesInfos.isEmpty()) {
|
|
|
|
subtitlesInfos.addAll(getAvailableSubtitlesInfo());
|
2018-02-01 22:27:14 +01:00
|
|
|
}
|
2017-07-11 05:08:03 +02:00
|
|
|
}
|
|
|
|
|
2019-10-29 06:00:29 +01:00
|
|
|
private JsonObject getPlayerConfig() throws ParsingException {
|
2017-07-11 05:08:03 +02:00
|
|
|
try {
|
2019-10-29 06:00:29 +01:00
|
|
|
String ytPlayerConfigRaw = Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageHtml);
|
2017-08-16 04:40:03 +02:00
|
|
|
return JsonParser.object().from(ytPlayerConfigRaw);
|
2017-07-11 05:08:03 +02:00
|
|
|
} catch (Parser.RegexException e) {
|
|
|
|
String errorReason = getErrorMessage();
|
2020-02-25 09:07:22 +01:00
|
|
|
if (errorReason.isEmpty()) {
|
|
|
|
throw new ContentNotAvailableException("Content not available: player config empty", e);
|
2017-03-01 18:47:52 +01:00
|
|
|
}
|
2020-02-25 09:07:22 +01:00
|
|
|
throw new ContentNotAvailableException("Content not available", e);
|
2017-08-10 19:50:59 +02:00
|
|
|
} catch (Exception e) {
|
2017-07-11 05:08:03 +02:00
|
|
|
throw new ParsingException("Could not parse yt player config", e);
|
|
|
|
}
|
|
|
|
}
|
2017-03-01 18:47:52 +01:00
|
|
|
|
2017-08-16 04:40:03 +02:00
|
|
|
private JsonObject getPlayerArgs(JsonObject playerConfig) throws ParsingException {
|
|
|
|
JsonObject playerArgs;
|
2017-03-01 18:47:52 +01:00
|
|
|
|
2017-07-11 05:08:03 +02:00
|
|
|
//attempt to load the youtube js player JSON arguments
|
|
|
|
try {
|
2017-08-16 04:40:03 +02:00
|
|
|
playerArgs = playerConfig.getObject("args");
|
2017-08-10 19:50:59 +02:00
|
|
|
} catch (Exception e) {
|
2017-07-11 05:08:03 +02:00
|
|
|
throw new ParsingException("Could not parse yt player config", e);
|
|
|
|
}
|
2017-03-01 18:47:52 +01:00
|
|
|
|
2017-07-11 05:08:03 +02:00
|
|
|
return playerArgs;
|
|
|
|
}
|
|
|
|
|
2017-08-16 04:40:03 +02:00
|
|
|
private String getPlayerUrl(JsonObject playerConfig) throws ParsingException {
|
2017-07-11 05:08:03 +02:00
|
|
|
try {
|
|
|
|
// The Youtube service needs to be initialized by downloading the
|
|
|
|
// js-Youtube-player. This is done in order to get the algorithm
|
|
|
|
// for decrypting cryptic signatures inside certain stream urls.
|
|
|
|
String playerUrl;
|
|
|
|
|
2017-08-16 04:40:03 +02:00
|
|
|
JsonObject ytAssets = playerConfig.getObject("assets");
|
2017-07-11 05:08:03 +02:00
|
|
|
playerUrl = ytAssets.getString("js");
|
|
|
|
|
|
|
|
if (playerUrl.startsWith("//")) {
|
|
|
|
playerUrl = HTTPS + playerUrl;
|
2017-03-01 18:47:52 +01:00
|
|
|
}
|
2017-07-11 05:08:03 +02:00
|
|
|
return playerUrl;
|
2017-08-10 19:50:59 +02:00
|
|
|
} catch (Exception e) {
|
2017-08-16 04:40:03 +02:00
|
|
|
throw new ParsingException("Could not load decryption code for the Youtube service.", e);
|
2017-07-11 05:08:03 +02:00
|
|
|
}
|
2017-03-01 18:47:52 +01:00
|
|
|
}
|
|
|
|
|
2019-09-11 19:05:41 +02:00
|
|
|
private JsonObject getPlayerResponse() throws ParsingException {
|
|
|
|
try {
|
|
|
|
String playerResponseStr;
|
2020-01-24 01:27:54 +01:00
|
|
|
if (playerArgs != null) {
|
2019-09-11 19:05:41 +02:00
|
|
|
playerResponseStr = playerArgs.getString("player_response");
|
|
|
|
} else {
|
|
|
|
playerResponseStr = videoInfoPage.get("player_response");
|
|
|
|
}
|
|
|
|
return JsonParser.object().from(playerResponseStr);
|
|
|
|
} catch (Exception e) {
|
|
|
|
throw new ParsingException("Could not parse yt player response", e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-12-18 23:05:58 +01:00
|
|
|
@Nonnull
|
|
|
|
private EmbeddedInfo getEmbeddedInfo() throws ParsingException, ReCaptchaException {
|
2017-07-11 05:08:03 +02:00
|
|
|
try {
|
2017-12-18 23:05:58 +01:00
|
|
|
final Downloader downloader = NewPipe.getDownloader();
|
|
|
|
final String embedUrl = "https://www.youtube.com/embed/" + getId();
|
2019-04-28 22:03:16 +02:00
|
|
|
final String embedPageContent = downloader.get(embedUrl, getExtractorLocalization()).responseBody();
|
2017-12-18 23:05:58 +01:00
|
|
|
|
|
|
|
// Get player url
|
|
|
|
final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")";
|
|
|
|
String playerUrl = Parser.matchGroup1(assetsPattern, embedPageContent)
|
|
|
|
.replace("\\", "").replace("\"", "");
|
2017-07-11 05:08:03 +02:00
|
|
|
if (playerUrl.startsWith("//")) {
|
|
|
|
playerUrl = HTTPS + playerUrl;
|
|
|
|
}
|
2017-12-18 23:05:58 +01:00
|
|
|
|
2019-07-30 20:53:23 +02:00
|
|
|
try {
|
|
|
|
// Get embed sts
|
|
|
|
final String stsPattern = "\"sts\"\\s*:\\s*(\\d+)";
|
|
|
|
final String sts = Parser.matchGroup1(stsPattern, embedPageContent);
|
|
|
|
return new EmbeddedInfo(playerUrl, sts);
|
|
|
|
} catch (Exception i) {
|
2020-01-24 01:27:54 +01:00
|
|
|
// if it fails we simply reply with no sts as then it does not seem to be necessary
|
2019-07-30 20:53:23 +02:00
|
|
|
return new EmbeddedInfo(playerUrl, "");
|
|
|
|
}
|
2017-12-18 23:05:58 +01:00
|
|
|
|
2017-07-11 05:08:03 +02:00
|
|
|
} catch (IOException e) {
|
|
|
|
throw new ParsingException(
|
|
|
|
"Could load decryption code form restricted video for the Youtube service.", e);
|
|
|
|
}
|
|
|
|
}
|
2017-03-01 18:47:52 +01:00
|
|
|
|
|
|
|
private String loadDecryptionCode(String playerUrl) throws DecryptException {
|
|
|
|
try {
|
|
|
|
Downloader downloader = NewPipe.getDownloader();
|
2017-06-29 20:12:55 +02:00
|
|
|
if (!playerUrl.contains("https://youtube.com")) {
|
2017-03-01 18:47:52 +01:00
|
|
|
//sometimes the https://youtube.com part does not get send with
|
|
|
|
//than we have to add it by hand
|
|
|
|
playerUrl = "https://youtube.com" + playerUrl;
|
|
|
|
}
|
|
|
|
|
2019-04-28 22:03:16 +02:00
|
|
|
final String playerCode = downloader.get(playerUrl, getExtractorLocalization()).responseBody();
|
2019-03-14 09:07:19 +01:00
|
|
|
final String decryptionFunctionName = getDecryptionFuncName(playerCode);
|
2017-03-01 18:47:52 +01:00
|
|
|
|
2018-09-08 07:25:07 +02:00
|
|
|
final String functionPattern = "("
|
|
|
|
+ decryptionFunctionName.replace("$", "\\$")
|
2017-03-01 18:47:52 +01:00
|
|
|
+ "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})";
|
2018-09-08 07:25:07 +02:00
|
|
|
final String decryptionFunction = "var " + Parser.matchGroup1(functionPattern, playerCode) + ";";
|
2017-03-01 18:47:52 +01:00
|
|
|
|
2018-09-08 07:25:07 +02:00
|
|
|
final String helperObjectName =
|
|
|
|
Parser.matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", decryptionFunction);
|
|
|
|
final String helperPattern =
|
|
|
|
"(var " + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)";
|
|
|
|
final String helperObject =
|
|
|
|
Parser.matchGroup1(helperPattern, playerCode.replace("\n", ""));
|
2017-03-01 18:47:52 +01:00
|
|
|
|
2018-09-08 07:25:07 +02:00
|
|
|
final String callerFunction =
|
|
|
|
"function " + DECRYPTION_FUNC_NAME + "(a){return " + decryptionFunctionName + "(a);}";
|
2017-03-01 18:47:52 +01:00
|
|
|
|
2018-09-08 07:25:07 +02:00
|
|
|
return helperObject + decryptionFunction + callerFunction;
|
2017-06-29 20:12:55 +02:00
|
|
|
} catch (IOException ioe) {
|
2017-03-01 18:47:52 +01:00
|
|
|
throw new DecryptException("Could not load decrypt function", ioe);
|
2017-06-29 20:12:55 +02:00
|
|
|
} catch (Exception e) {
|
2017-03-01 18:47:52 +01:00
|
|
|
throw new DecryptException("Could not parse decrypt function ", e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-07-11 05:08:03 +02:00
|
|
|
private String decryptSignature(String encryptedSig, String decryptionCode) throws DecryptException {
|
2017-03-01 18:47:52 +01:00
|
|
|
Context context = Context.enter();
|
|
|
|
context.setOptimizationLevel(-1);
|
2017-11-23 11:47:05 +01:00
|
|
|
Object result;
|
2017-03-01 18:47:52 +01:00
|
|
|
try {
|
|
|
|
ScriptableObject scope = context.initStandardObjects();
|
|
|
|
context.evaluateString(scope, decryptionCode, "decryptionCode", 1, null);
|
|
|
|
Function decryptionFunc = (Function) scope.get("decrypt", scope);
|
|
|
|
result = decryptionFunc.call(context, scope, scope, new Object[]{encryptedSig});
|
|
|
|
} catch (Exception e) {
|
|
|
|
throw new DecryptException("could not get decrypt signature", e);
|
|
|
|
} finally {
|
|
|
|
Context.exit();
|
|
|
|
}
|
|
|
|
return result == null ? "" : result.toString();
|
|
|
|
}
|
|
|
|
|
2019-03-14 09:07:19 +01:00
|
|
|
private String getDecryptionFuncName(String playerCode) throws DecryptException {
|
2020-01-24 01:30:54 +01:00
|
|
|
String[] decryptionFuncNameRegexes = {
|
2020-01-24 01:59:39 +01:00
|
|
|
DECRYPTION_SIGNATURE_FUNCTION_REGEX_2,
|
2020-01-24 01:30:54 +01:00
|
|
|
DECRYPTION_SIGNATURE_FUNCTION_REGEX,
|
|
|
|
DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX,
|
|
|
|
DECRYPTION_AKAMAIZED_STRING_REGEX
|
|
|
|
};
|
|
|
|
Parser.RegexException exception = null;
|
|
|
|
for (String regex : decryptionFuncNameRegexes) {
|
2019-03-14 09:07:19 +01:00
|
|
|
try {
|
2020-01-24 01:30:54 +01:00
|
|
|
return Parser.matchGroup1(regex, playerCode);
|
|
|
|
} catch (Parser.RegexException re) {
|
|
|
|
if (exception == null)
|
|
|
|
exception = re;
|
2019-03-14 09:07:19 +01:00
|
|
|
}
|
|
|
|
}
|
2020-01-24 01:30:54 +01:00
|
|
|
throw new DecryptException("Could not find decrypt function with any of the given patterns.", exception);
|
2019-03-14 09:07:19 +01:00
|
|
|
}
|
|
|
|
|
2018-02-02 06:48:34 +01:00
|
|
|
@Nonnull
|
2020-02-25 09:07:22 +01:00
|
|
|
private List<SubtitlesInfo> getAvailableSubtitlesInfo() {
|
2018-02-02 06:48:34 +01:00
|
|
|
// If the video is age restricted getPlayerConfig will fail
|
2020-01-24 01:27:54 +01:00
|
|
|
if (isAgeRestricted) return Collections.emptyList();
|
2018-02-02 06:48:34 +01:00
|
|
|
|
|
|
|
final JsonObject captions;
|
2019-09-11 19:35:08 +02:00
|
|
|
if (!playerResponse.has("captions")) {
|
|
|
|
// Captions does not exist
|
|
|
|
return Collections.emptyList();
|
2018-02-02 06:48:34 +01:00
|
|
|
}
|
2019-09-11 19:35:08 +02:00
|
|
|
captions = playerResponse.getObject("captions");
|
2018-02-02 06:48:34 +01:00
|
|
|
|
2018-02-06 19:45:58 +01:00
|
|
|
final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer", new JsonObject());
|
|
|
|
final JsonArray captionsArray = renderer.getArray("captionTracks", new JsonArray());
|
2018-02-02 07:51:53 +01:00
|
|
|
// todo: use this to apply auto translation to different language from a source language
|
2020-02-25 09:07:22 +01:00
|
|
|
// final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages", new JsonArray());
|
2018-02-02 06:48:34 +01:00
|
|
|
|
2018-02-06 19:45:58 +01:00
|
|
|
// This check is necessary since there may be cases where subtitles metadata do not contain caption track info
|
|
|
|
// e.g. https://www.youtube.com/watch?v=-Vpwatutnko
|
2018-02-02 06:48:34 +01:00
|
|
|
final int captionsSize = captionsArray.size();
|
2020-01-24 01:27:54 +01:00
|
|
|
if (captionsSize == 0) return Collections.emptyList();
|
2018-02-02 06:48:34 +01:00
|
|
|
|
2018-02-02 08:24:22 +01:00
|
|
|
List<SubtitlesInfo> result = new ArrayList<>();
|
2018-02-02 06:48:34 +01:00
|
|
|
for (int i = 0; i < captionsSize; i++) {
|
2018-02-02 07:51:53 +01:00
|
|
|
final String languageCode = captionsArray.getObject(i).getString("languageCode");
|
|
|
|
final String baseUrl = captionsArray.getObject(i).getString("baseUrl");
|
2018-02-06 19:45:58 +01:00
|
|
|
final String vssId = captionsArray.getObject(i).getString("vssId");
|
2018-02-02 07:51:53 +01:00
|
|
|
|
2018-02-06 19:45:58 +01:00
|
|
|
if (languageCode != null && baseUrl != null && vssId != null) {
|
|
|
|
final boolean isAutoGenerated = vssId.startsWith("a.");
|
|
|
|
result.add(new SubtitlesInfo(baseUrl, languageCode, isAutoGenerated));
|
|
|
|
}
|
2018-02-01 22:27:14 +01:00
|
|
|
}
|
2018-02-02 06:48:34 +01:00
|
|
|
|
|
|
|
return result;
|
2018-02-01 22:27:14 +01:00
|
|
|
}
|
2017-12-18 23:05:58 +01:00
|
|
|
/*//////////////////////////////////////////////////////////////////////////
|
|
|
|
// Data Class
|
|
|
|
//////////////////////////////////////////////////////////////////////////*/
|
|
|
|
|
|
|
|
private class EmbeddedInfo {
|
|
|
|
final String url;
|
|
|
|
final String sts;
|
|
|
|
|
|
|
|
EmbeddedInfo(final String url, final String sts) {
|
|
|
|
this.url = url;
|
|
|
|
this.sts = sts;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-02-02 08:24:22 +01:00
|
|
|
private class SubtitlesInfo {
|
|
|
|
final String cleanUrl;
|
|
|
|
final String languageCode;
|
|
|
|
final boolean isGenerated;
|
|
|
|
|
|
|
|
public SubtitlesInfo(final String baseUrl, final String languageCode, final boolean isGenerated) {
|
|
|
|
this.cleanUrl = baseUrl
|
|
|
|
.replaceAll("&fmt=[^&]*", "") // Remove preexisting format if exists
|
|
|
|
.replaceAll("&tlang=[^&]*", ""); // Remove translation language
|
|
|
|
this.languageCode = languageCode;
|
|
|
|
this.isGenerated = isGenerated;
|
|
|
|
}
|
|
|
|
|
2018-09-24 21:04:22 +02:00
|
|
|
public SubtitlesStream getSubtitle(final MediaFormat format) {
|
|
|
|
return new SubtitlesStream(format, languageCode, cleanUrl + "&fmt=" + format.getSuffix(), isGenerated);
|
2018-02-02 08:24:22 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-08-10 19:50:59 +02:00
|
|
|
/*//////////////////////////////////////////////////////////////////////////
|
|
|
|
// Utils
|
|
|
|
//////////////////////////////////////////////////////////////////////////*/
|
|
|
|
|
2020-02-25 09:50:22 +01:00
|
|
|
private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException {
|
|
|
|
JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
|
|
|
|
.getObject("results").getObject("results").getArray("contents");
|
|
|
|
JsonObject videoPrimaryInfoRenderer = null;
|
|
|
|
|
|
|
|
for (Object content : contents) {
|
|
|
|
if (((JsonObject) content).getObject("videoPrimaryInfoRenderer") != null) {
|
|
|
|
videoPrimaryInfoRenderer = ((JsonObject) content).getObject("videoPrimaryInfoRenderer");
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (videoPrimaryInfoRenderer == null) {
|
|
|
|
throw new ParsingException("Could not find videoPrimaryInfoRenderer");
|
|
|
|
}
|
|
|
|
|
|
|
|
return videoPrimaryInfoRenderer;
|
|
|
|
}
|
|
|
|
|
2020-02-25 10:05:53 +01:00
|
|
|
private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException {
|
|
|
|
JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
|
|
|
|
.getObject("results").getObject("results").getArray("contents");
|
|
|
|
JsonObject videoSecondaryInfoRenderer = null;
|
|
|
|
|
|
|
|
for (Object content : contents) {
|
|
|
|
if (((JsonObject) content).getObject("videoSecondaryInfoRenderer") != null) {
|
|
|
|
videoSecondaryInfoRenderer = ((JsonObject) content).getObject("videoSecondaryInfoRenderer");
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (videoSecondaryInfoRenderer == null) {
|
|
|
|
throw new ParsingException("Could not find videoSecondaryInfoRenderer");
|
|
|
|
}
|
|
|
|
|
|
|
|
return videoSecondaryInfoRenderer;
|
|
|
|
}
|
|
|
|
|
2017-12-18 23:05:58 +01:00
|
|
|
@Nonnull
|
2018-02-01 22:27:14 +01:00
|
|
|
private static String getVideoInfoUrl(final String id, final String sts) {
|
2017-12-18 23:05:58 +01:00
|
|
|
return "https://www.youtube.com/get_video_info?" + "video_id=" + id +
|
|
|
|
"&eurl=https://youtube.googleapis.com/v/" + id +
|
|
|
|
"&sts=" + sts + "&ps=default&gl=US&hl=en";
|
|
|
|
}
|
|
|
|
|
2019-09-11 19:05:41 +02:00
|
|
|
private Map<String, ItagItem> getItags(String streamingDataKey, ItagItem.ItagType itagTypeWanted) throws ParsingException {
|
2017-08-10 19:50:59 +02:00
|
|
|
Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();
|
2019-09-12 15:08:17 +02:00
|
|
|
JsonObject streamingData = playerResponse.getObject("streamingData");
|
|
|
|
if (!streamingData.has(streamingDataKey)) {
|
|
|
|
return urlAndItags;
|
2017-08-10 19:50:59 +02:00
|
|
|
}
|
|
|
|
|
2019-09-12 15:08:17 +02:00
|
|
|
JsonArray formats = streamingData.getArray(streamingDataKey);
|
2019-09-11 19:05:41 +02:00
|
|
|
for (int i = 0; i != formats.size(); ++i) {
|
|
|
|
JsonObject formatData = formats.getObject(i);
|
|
|
|
int itag = formatData.getInt("itag");
|
2017-08-10 19:50:59 +02:00
|
|
|
|
2019-09-11 19:05:41 +02:00
|
|
|
if (ItagItem.isSupported(itag)) {
|
2019-09-11 19:56:16 +02:00
|
|
|
try {
|
2017-08-10 19:50:59 +02:00
|
|
|
ItagItem itagItem = ItagItem.getItag(itag);
|
|
|
|
if (itagItem.itagType == itagTypeWanted) {
|
2019-09-11 19:56:16 +02:00
|
|
|
String streamUrl;
|
|
|
|
if (formatData.has("url")) {
|
|
|
|
streamUrl = formatData.getString("url");
|
|
|
|
} else {
|
|
|
|
// this url has an encrypted signature
|
|
|
|
Map<String, String> cipher = Parser.compatParseMap(formatData.getString("cipher"));
|
|
|
|
streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + decryptSignature(cipher.get("s"), decryptionCode);
|
2017-08-10 19:50:59 +02:00
|
|
|
}
|
2019-09-11 19:56:16 +02:00
|
|
|
|
2017-08-10 19:50:59 +02:00
|
|
|
urlAndItags.put(streamUrl, itagItem);
|
|
|
|
}
|
2019-09-11 19:56:16 +02:00
|
|
|
} catch (UnsupportedEncodingException ignored) {
|
|
|
|
|
2017-08-10 19:50:59 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return urlAndItags;
|
|
|
|
}
|
|
|
|
|
2020-02-08 23:58:46 +01:00
|
|
|
@Nonnull
|
|
|
|
@Override
|
|
|
|
public List<Frameset> getFrames() throws ExtractionException {
|
|
|
|
try {
|
|
|
|
final String script = doc.select("#player-api").first().siblingElements().select("script").html();
|
|
|
|
int p = script.indexOf("ytplayer.config");
|
|
|
|
if (p == -1) {
|
|
|
|
return Collections.emptyList();
|
|
|
|
}
|
|
|
|
p = script.indexOf('{', p);
|
|
|
|
int e = script.indexOf("ytplayer.load", p);
|
|
|
|
if (e == -1) {
|
|
|
|
return Collections.emptyList();
|
|
|
|
}
|
|
|
|
JsonObject jo = JsonParser.object().from(script.substring(p, e - 1));
|
|
|
|
final String resp = jo.getObject("args").getString("player_response");
|
|
|
|
jo = JsonParser.object().from(resp);
|
|
|
|
final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|");
|
|
|
|
final String url = spec[0];
|
|
|
|
final ArrayList<Frameset> result = new ArrayList<>(spec.length - 1);
|
|
|
|
for (int i = 1; i < spec.length; ++i) {
|
|
|
|
final String[] parts = spec[i].split("#");
|
|
|
|
if (parts.length != 8) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
final int frameWidth = Integer.parseInt(parts[0]);
|
|
|
|
final int frameHeight = Integer.parseInt(parts[1]);
|
|
|
|
final int totalCount = Integer.parseInt(parts[2]);
|
|
|
|
final int framesPerPageX = Integer.parseInt(parts[3]);
|
|
|
|
final int framesPerPageY = Integer.parseInt(parts[4]);
|
|
|
|
final String baseUrl = url.replace("$L", String.valueOf(i - 1)).replace("$N", parts[6]) + "&sigh=" + parts[7];
|
|
|
|
final List<String> urls;
|
|
|
|
if (baseUrl.contains("$M")) {
|
|
|
|
final int totalPages = (int) Math.ceil(totalCount / (double) (framesPerPageX * framesPerPageY));
|
|
|
|
urls = new ArrayList<>(totalPages);
|
|
|
|
for (int j = 0; j < totalPages; j++) {
|
|
|
|
urls.add(baseUrl.replace("$M", String.valueOf(j)));
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
urls = Collections.singletonList(baseUrl);
|
|
|
|
}
|
|
|
|
result.add(new Frameset(
|
|
|
|
urls,
|
|
|
|
frameWidth,
|
|
|
|
frameHeight,
|
|
|
|
totalCount,
|
|
|
|
framesPerPageX,
|
|
|
|
framesPerPageY
|
|
|
|
));
|
|
|
|
}
|
|
|
|
result.trimToSize();
|
|
|
|
return result;
|
|
|
|
} catch (Exception e) {
|
|
|
|
throw new ExtractionException(e);
|
|
|
|
}
|
|
|
|
}
|
added metadata, fix descriptions, fix thumbnail, update tests
thumbnail: quality before: https://peertube.cpy.re/static/thumbnails/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
quality after: https://peertube.cpy.re/static/previews/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
description: we were getting about the first 260 characters, we now get full description (with fallback to first 260 chars if the get request for full description fails)
test: updated tests to match description, also changed some test: it was assertEquals(extracted, expected), but the proper way to do it is assertEquals(expected, extracted)
metadata: got host, privacy (public, private, unlisted), licence, language, tags
2020-01-19 12:45:52 +01:00
|
|
|
|
2020-02-25 09:07:22 +01:00
|
|
|
@Nonnull
|
added metadata, fix descriptions, fix thumbnail, update tests
thumbnail: quality before: https://peertube.cpy.re/static/thumbnails/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
quality after: https://peertube.cpy.re/static/previews/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
description: we were getting about the first 260 characters, we now get full description (with fallback to first 260 chars if the get request for full description fails)
test: updated tests to match description, also changed some test: it was assertEquals(extracted, expected), but the proper way to do it is assertEquals(expected, extracted)
metadata: got host, privacy (public, private, unlisted), licence, language, tags
2020-01-19 12:45:52 +01:00
|
|
|
@Override
|
2020-02-25 09:07:22 +01:00
|
|
|
public String getHost() {
|
added metadata, fix descriptions, fix thumbnail, update tests
thumbnail: quality before: https://peertube.cpy.re/static/thumbnails/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
quality after: https://peertube.cpy.re/static/previews/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
description: we were getting about the first 260 characters, we now get full description (with fallback to first 260 chars if the get request for full description fails)
test: updated tests to match description, also changed some test: it was assertEquals(extracted, expected), but the proper way to do it is assertEquals(expected, extracted)
metadata: got host, privacy (public, private, unlisted), licence, language, tags
2020-01-19 12:45:52 +01:00
|
|
|
return "";
|
|
|
|
}
|
|
|
|
|
2020-02-25 09:07:22 +01:00
|
|
|
@Nonnull
|
added metadata, fix descriptions, fix thumbnail, update tests
thumbnail: quality before: https://peertube.cpy.re/static/thumbnails/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
quality after: https://peertube.cpy.re/static/previews/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
description: we were getting about the first 260 characters, we now get full description (with fallback to first 260 chars if the get request for full description fails)
test: updated tests to match description, also changed some test: it was assertEquals(extracted, expected), but the proper way to do it is assertEquals(expected, extracted)
metadata: got host, privacy (public, private, unlisted), licence, language, tags
2020-01-19 12:45:52 +01:00
|
|
|
@Override
|
2020-02-25 09:07:22 +01:00
|
|
|
public String getPrivacy() {
|
added metadata, fix descriptions, fix thumbnail, update tests
thumbnail: quality before: https://peertube.cpy.re/static/thumbnails/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
quality after: https://peertube.cpy.re/static/previews/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
description: we were getting about the first 260 characters, we now get full description (with fallback to first 260 chars if the get request for full description fails)
test: updated tests to match description, also changed some test: it was assertEquals(extracted, expected), but the proper way to do it is assertEquals(expected, extracted)
metadata: got host, privacy (public, private, unlisted), licence, language, tags
2020-01-19 12:45:52 +01:00
|
|
|
return "";
|
|
|
|
}
|
|
|
|
|
2020-02-25 09:07:22 +01:00
|
|
|
@Nonnull
|
added metadata, fix descriptions, fix thumbnail, update tests
thumbnail: quality before: https://peertube.cpy.re/static/thumbnails/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
quality after: https://peertube.cpy.re/static/previews/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
description: we were getting about the first 260 characters, we now get full description (with fallback to first 260 chars if the get request for full description fails)
test: updated tests to match description, also changed some test: it was assertEquals(extracted, expected), but the proper way to do it is assertEquals(expected, extracted)
metadata: got host, privacy (public, private, unlisted), licence, language, tags
2020-01-19 12:45:52 +01:00
|
|
|
@Override
|
2020-02-25 09:07:22 +01:00
|
|
|
public String getCategory() {
|
added metadata, fix descriptions, fix thumbnail, update tests
thumbnail: quality before: https://peertube.cpy.re/static/thumbnails/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
quality after: https://peertube.cpy.re/static/previews/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
description: we were getting about the first 260 characters, we now get full description (with fallback to first 260 chars if the get request for full description fails)
test: updated tests to match description, also changed some test: it was assertEquals(extracted, expected), but the proper way to do it is assertEquals(expected, extracted)
metadata: got host, privacy (public, private, unlisted), licence, language, tags
2020-01-19 12:45:52 +01:00
|
|
|
return "";
|
|
|
|
}
|
|
|
|
|
2020-02-25 09:07:22 +01:00
|
|
|
@Nonnull
|
added metadata, fix descriptions, fix thumbnail, update tests
thumbnail: quality before: https://peertube.cpy.re/static/thumbnails/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
quality after: https://peertube.cpy.re/static/previews/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
description: we were getting about the first 260 characters, we now get full description (with fallback to first 260 chars if the get request for full description fails)
test: updated tests to match description, also changed some test: it was assertEquals(extracted, expected), but the proper way to do it is assertEquals(expected, extracted)
metadata: got host, privacy (public, private, unlisted), licence, language, tags
2020-01-19 12:45:52 +01:00
|
|
|
@Override
|
2020-02-25 09:07:22 +01:00
|
|
|
public String getLicence() {
|
added metadata, fix descriptions, fix thumbnail, update tests
thumbnail: quality before: https://peertube.cpy.re/static/thumbnails/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
quality after: https://peertube.cpy.re/static/previews/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
description: we were getting about the first 260 characters, we now get full description (with fallback to first 260 chars if the get request for full description fails)
test: updated tests to match description, also changed some test: it was assertEquals(extracted, expected), but the proper way to do it is assertEquals(expected, extracted)
metadata: got host, privacy (public, private, unlisted), licence, language, tags
2020-01-19 12:45:52 +01:00
|
|
|
return "";
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2020-02-25 09:07:22 +01:00
|
|
|
public Locale getLanguageInfo() {
|
2020-01-25 13:16:42 +01:00
|
|
|
return null;
|
added metadata, fix descriptions, fix thumbnail, update tests
thumbnail: quality before: https://peertube.cpy.re/static/thumbnails/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
quality after: https://peertube.cpy.re/static/previews/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
description: we were getting about the first 260 characters, we now get full description (with fallback to first 260 chars if the get request for full description fails)
test: updated tests to match description, also changed some test: it was assertEquals(extracted, expected), but the proper way to do it is assertEquals(expected, extracted)
metadata: got host, privacy (public, private, unlisted), licence, language, tags
2020-01-19 12:45:52 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
@Nonnull
|
|
|
|
@Override
|
2020-02-25 09:07:22 +01:00
|
|
|
public List<String> getTags() {
|
added metadata, fix descriptions, fix thumbnail, update tests
thumbnail: quality before: https://peertube.cpy.re/static/thumbnails/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
quality after: https://peertube.cpy.re/static/previews/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
description: we were getting about the first 260 characters, we now get full description (with fallback to first 260 chars if the get request for full description fails)
test: updated tests to match description, also changed some test: it was assertEquals(extracted, expected), but the proper way to do it is assertEquals(expected, extracted)
metadata: got host, privacy (public, private, unlisted), licence, language, tags
2020-01-19 12:45:52 +01:00
|
|
|
return new ArrayList<>();
|
|
|
|
}
|
2020-01-23 14:19:22 +01:00
|
|
|
|
|
|
|
@Nonnull
|
|
|
|
@Override
|
2020-02-25 09:07:22 +01:00
|
|
|
public String getSupportInfo() {
|
2020-01-23 14:19:22 +01:00
|
|
|
return "";
|
|
|
|
}
|
2017-03-01 18:47:52 +01:00
|
|
|
}
|