lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #986 from Isira-Seneviratne/Static_maps 

Use immutable Map factory methods.
This commit is contained in:
Stypox 2023-01-02 18:11:14 +01:00 committed by GitHub
parent 259de3cba6 219c5c5be5
commit 45636b0d00
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 101 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeTrendingLinkHandlerFactory.java
View File

@ -4,8 +4,6 @@ import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -14,27 +12,16 @@ public final class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFac
private static final PeertubeTrendingLinkHandlerFactory INSTANCE
= new PeertubeTrendingLinkHandlerFactory();
public static final Map<String, String> KIOSK_MAP;
public static final Map<String, String> REVERSE_KIOSK_MAP;
public static final String KIOSK_TRENDING = "Trending";
public static final String KIOSK_MOST_LIKED = "Most liked";
public static final String KIOSK_RECENT = "Recently added";
public static final String KIOSK_LOCAL = "Local";
static {
final Map<String, String> map = new HashMap<>();
map.put(KIOSK_TRENDING, "%s/api/v1/videos?sort=-trending");
map.put(KIOSK_MOST_LIKED, "%s/api/v1/videos?sort=-likes");
map.put(KIOSK_RECENT, "%s/api/v1/videos?sort=-publishedAt");
map.put(KIOSK_LOCAL, "%s/api/v1/videos?sort=-publishedAt&filter=local");
KIOSK_MAP = Collections.unmodifiableMap(map);
final Map<String, String> reverseMap = new HashMap<>();
for (final Map.Entry<String, String> entry : KIOSK_MAP.entrySet()) {
reverseMap.put(entry.getValue(), entry.getKey());
}
REVERSE_KIOSK_MAP = Collections.unmodifiableMap(reverseMap);
}
public static final Map<String, String> KIOSK_MAP = Map.of(
KIOSK_TRENDING, "%s/api/v1/videos?sort=-trending",
KIOSK_MOST_LIKED, "%s/api/v1/videos?sort=-likes",
KIOSK_RECENT, "%s/api/v1/videos?sort=-publishedAt",
KIOSK_LOCAL, "%s/api/v1/videos?sort=-publishedAt&filter=local");
public static PeertubeTrendingLinkHandlerFactory getInstance() {
return INSTANCE;
@ -66,10 +53,12 @@ public final class PeertubeTrendingLinkHandlerFactory extends ListLinkHandlerFac
return KIOSK_RECENT;
} else if (cleanUrl.contains("/videos/local")) {
return KIOSK_LOCAL;
} else if (REVERSE_KIOSK_MAP.containsKey(cleanUrl)) {
return REVERSE_KIOSK_MAP.get(cleanUrl);
} else {
throw new ParsingException("no id found for this url");
return KIOSK_MAP.entrySet().stream()
.filter(entry -> cleanUrl.equals(entry.getValue()))
.findFirst()
.map(Map.Entry::getKey)
.orElseThrow(() -> new ParsingException("no id found for this url"));
}
}

3
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View File

@ -64,8 +64,7 @@ public final class SoundcloudParsingHelper {
// The one containing the client id will likely be the last one
Collections.reverse(possibleScripts);
final Map<String, List<String>> headers = Collections.singletonMap("Range",
Collections.singletonList("bytes=0-50000"));
final var headers = Map.of("Range", List.of("bytes=0-50000"));
for (final Element element : possibleScripts) {
final String srcUrl = element.attr("src");

123
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -25,7 +25,6 @@ import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static java.util.Collections.singletonList;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonBuilder;
@ -61,7 +60,6 @@ import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
@ -92,6 +90,11 @@ public final class YoutubeParsingHelper {
public static final String YOUTUBEI_V1_GAPIS_URL =
"https://youtubei.googleapis.com/youtubei/v1/";
/**
* The base URL of YouTube Music.
*/
private static final String YOUTUBE_MUSIC_URL = "https://music.youtube.com";
/**
* A parameter to disable pretty-printed response of InnerTube requests, to reduce response
* sizes.
@ -554,9 +557,7 @@ public final class YoutubeParsingHelper {
.end().done().getBytes(StandardCharsets.UTF_8);
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", singletonList("1"));
headers.put("X-YouTube-Client-Version", singletonList(HARDCODED_CLIENT_VERSION));
final var headers = getClientHeaders("1", HARDCODED_CLIENT_VERSION);
// This endpoint is fetched by the YouTube website to get the items of its main menu and is
// pretty lightweight (around 30kB)
@ -578,9 +579,7 @@ public final class YoutubeParsingHelper {
return;
}
final String url = "https://www.youtube.com/sw.js";
final Map<String, List<String>> headers = new HashMap<>();
headers.put("Origin", singletonList("https://www.youtube.com"));
headers.put("Referer", singletonList("https://www.youtube.com"));
final var headers = getOriginReferrerHeaders("https://www.youtube.com");
final String response = getDownloader().get(url, headers).responseBody();
try {
clientVersion = getStringResultFromRegexArray(response,
@ -799,11 +798,9 @@ public final class YoutubeParsingHelper {
.end().done().getBytes(StandardCharsets.UTF_8);
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", singletonList(HARDCODED_YOUTUBE_MUSIC_KEY[1]));
headers.put("X-YouTube-Client-Version", singletonList(HARDCODED_YOUTUBE_MUSIC_KEY[2]));
headers.put("Origin", singletonList("https://music.youtube.com"));
headers.put("Referer", singletonList("https://music.youtube.com"));
final var headers = new HashMap<>(getOriginReferrerHeaders(YOUTUBE_MUSIC_URL));
headers.putAll(getClientHeaders(HARDCODED_YOUTUBE_MUSIC_KEY[1],
HARDCODED_YOUTUBE_MUSIC_KEY[2]));
final Response response = getDownloader().postWithContentTypeJson(url, headers, json);
// Ensure to have a valid response
@ -826,14 +823,12 @@ public final class YoutubeParsingHelper {
try {
final String url = "https://music.youtube.com/sw.js";
final Map<String, List<String>> headers = new HashMap<>();
headers.put("Origin", singletonList("https://music.youtube.com"));
headers.put("Referer", singletonList("https://music.youtube.com"));
final var headers = getOriginReferrerHeaders("https://music.youtube.com");
final String response = getDownloader().get(url, headers).responseBody();
musicClientVersion = getStringResultFromRegexArray(response,
INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1);
musicKey = getStringResultFromRegexArray(response, INNERTUBE_API_KEY_REGEXES, 1);
musicClientName = Parser.matchGroup1(INNERTUBE_CLIENT_NAME_REGEX, response);
musicClientVersion = getStringResultFromRegexArray(response,
INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1);
musicKey = getStringResultFromRegexArray(response, INNERTUBE_API_KEY_REGEXES, 1);
musicClientName = Parser.matchGroup1(INNERTUBE_CLIENT_NAME_REGEX, response);
} catch (final Exception e) {
final String url = "https://music.youtube.com/?ucbcb=1";
final String html = getDownloader().get(url, getCookieHeader()).responseBody();
@ -1176,8 +1171,7 @@ public final class YoutubeParsingHelper {
final byte[] body,
final Localization localization)
throws IOException, ExtractionException {
final Map<String, List<String>> headers = new HashMap<>();
addYouTubeHeaders(headers);
final var headers = getYouTubeHeaders();
return JsonUtils.toJsonObject(getValidJsonResponseBody(
getDownloader().postWithContentTypeJson(YOUTUBEI_V1_URL + endpoint + "?key="
@ -1209,9 +1203,8 @@ public final class YoutubeParsingHelper {
@Nonnull final String userAgent,
@Nonnull final String innerTubeApiKey,
@Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
final Map<String, List<String>> headers = new HashMap<>();
headers.put("User-Agent", singletonList(userAgent));
headers.put("X-Goog-Api-Format-Version", singletonList("2"));
final var headers = Map.of("User-Agent", List.of(userAgent),
"X-Goog-Api-Format-Version", List.of("2"));
final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?key=" + innerTubeApiKey
+ DISABLE_PRETTY_PRINT_PARAMETER;
@ -1423,40 +1416,60 @@ public final class YoutubeParsingHelper {
+ ")";
}
/**
* Returns a {@link Map} containing the required YouTube Music headers.
*/
@Nonnull
public static Map<String, List<String>> getYoutubeMusicHeaders() {
final Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKey[1]));
headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKey[2]));
headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
headers.put("Referer", Collections.singletonList("https://music.youtube.com"));
final var headers = new HashMap<>(getOriginReferrerHeaders(YOUTUBE_MUSIC_URL));
headers.putAll(getClientHeaders(youtubeMusicKey[1], youtubeMusicKey[2]));
return headers;
}
/**
* Add required headers and cookies to an existing headers Map.
* @see #addClientInfoHeaders(Map)
* @see #addCookieHeader(Map)
* Returns a {@link Map} containing the required YouTube headers, including the
* <code>CONSENT</code> cookie to prevent redirects to <code>consent.youtube.com</code>
*/
public static void addYouTubeHeaders(final Map<String, List<String>> headers)
throws IOException, ExtractionException {
addClientInfoHeaders(headers);
addCookieHeader(headers);
public static Map<String, List<String>> getYouTubeHeaders()
throws ExtractionException, IOException {
final var headers = getClientInfoHeaders();
headers.put("Cookie", List.of(generateConsentCookie()));
return headers;
}
/**
* Add the <code>X-YouTube-Client-Name</code>, <code>X-YouTube-Client-Version</code>,
* <code>Origin</code>, and <code>Referer</code> headers.
* @param headers The headers which should be completed
* Returns a {@link Map} containing the {@code X-YouTube-Client-Name},
* {@code X-YouTube-Client-Version}, {@code Origin}, and {@code Referer} headers.
*/
public static void addClientInfoHeaders(@Nonnull final Map<String, List<String>> headers)
throws IOException, ExtractionException {
headers.computeIfAbsent("Origin", k -> singletonList("https://www.youtube.com"));
headers.computeIfAbsent("Referer", k -> singletonList("https://www.youtube.com"));
headers.computeIfAbsent("X-YouTube-Client-Name", k -> singletonList("1"));
if (headers.get("X-YouTube-Client-Version") == null) {
headers.put("X-YouTube-Client-Version", singletonList(getClientVersion()));
}
public static Map<String, List<String>> getClientInfoHeaders()
throws ExtractionException, IOException {
final var headers = new HashMap<>(getOriginReferrerHeaders("https://www.youtube.com"));
headers.putAll(getClientHeaders("1", getClientVersion()));
return headers;
}
/**
* Returns an unmodifiable {@link Map} containing the {@code Origin} and {@code Referer}
* headers set to the given URL.
*
* @param url The URL to be set as the origin and referrer.
*/
private static Map<String, List<String>> getOriginReferrerHeaders(@Nonnull final String url) {
final var urlList = List.of(url);
return Map.of("Origin", urlList, "Referer", urlList);
}
/**
* Returns an unmodifiable {@link Map} containing the {@code X-YouTube-Client-Name} and
* {@code X-YouTube-Client-Version} headers.
*
* @param name The X-YouTube-Client-Name value.
* @param version X-YouTube-Client-Version value.
*/
private static Map<String, List<String>> getClientHeaders(@Nonnull final String name,
@Nonnull final String version) {
return Map.of("X-YouTube-Client-Name", List.of(name),
"X-YouTube-Client-Version", List.of(version));
}
/**
@ -1464,19 +1477,7 @@ public final class YoutubeParsingHelper {
* @return A singleton map containing the header.
*/
public static Map<String, List<String>> getCookieHeader() {
return Collections.singletonMap("Cookie", singletonList(generateConsentCookie()));
}
/**
* Add the <code>CONSENT</code> cookie to prevent redirect to <code>consent.youtube.com</code>
* @param headers the headers which should be completed
*/
public static void addCookieHeader(@Nonnull final Map<String, List<String>> headers) {
if (headers.get("Cookie") == null) {
headers.put("Cookie", Collections.singletonList(generateConsentCookie()));
} else {
headers.get("Cookie").add(generateConsentCookie());
}
return Map.of("Cookie", List.of(generateConsentCookie()));
}
@Nonnull

45
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java
View File

@ -1,5 +1,14 @@
package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientInfoHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isIosStreamingUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5SimplyEmbeddedPlayerStreamingUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebStreamingUrl;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -13,6 +22,14 @@ import org.w3c.dom.DOMException;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import java.io.IOException;
import java.io.StringWriter;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import javax.annotation.Nonnull;
import javax.xml.XMLConstants;
import javax.xml.parsers.DocumentBuilder;
@ -25,24 +42,6 @@ import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import java.io.IOException;
import java.io.StringWriter;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isIosStreamingUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5SimplyEmbeddedPlayerStreamingUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebStreamingUrl;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/**
* Utilities and constants for YouTube DASH manifest creators.
*
@ -583,9 +582,9 @@ public final class YoutubeDashManifestCreatorsUtils {
}
} else if (isAndroidStreamingUrl || isIosStreamingUrl) {
try {
final Map<String, List<String>> headers = Collections.singletonMap("User-Agent",
Collections.singletonList(isAndroidStreamingUrl
? getAndroidUserAgent(null) : getIosUserAgent(null)));
final var headers = Map.of("User-Agent",
List.of(isAndroidStreamingUrl ? getAndroidUserAgent(null)
: getIosUserAgent(null)));
final byte[] emptyBody = "".getBytes(StandardCharsets.UTF_8);
return downloader.post(baseStreamingUrl, headers, emptyBody);
} catch (final IOException | ExtractionException e) {
@ -705,9 +704,7 @@ public final class YoutubeDashManifestCreatorsUtils {
@Nonnull final String responseMimeTypeExpected)
throws CreationException {
try {
final Map<String, List<String>> headers = new HashMap<>();
headers.put("Origin", Collections.singletonList("https://www.youtube.com"));
headers.put("Referer", Collections.singletonList("https://www.youtube.com"));
final var headers = getClientInfoHeaders();
String responseMimeType = "";

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
View File

@ -2,11 +2,11 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addYouTubeHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYouTubeHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.getQueryValue;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -88,9 +88,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(StandardCharsets.UTF_8);
final Map<String, List<String>> headers = new HashMap<>();
// Cookie is required due to consent
addYouTubeHeaders(headers);
final var headers = getYouTubeHeaders();
final Response response = getDownloader().postWithContentTypeJson(
YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
@ -222,9 +221,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
}
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final Map<String, List<String>> headers = new HashMap<>();
// Cookie is required due to consent
addYouTubeHeaders(headers);
final var headers = getYouTubeHeaders();
final Response response = getDownloader().postWithContentTypeJson(page.getUrl(), headers,
page.getBody(), getExtractorLocalization());

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -14,6 +14,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -29,11 +30,12 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
public class YoutubePlaylistExtractor extends PlaylistExtractor {
// Names of some objects in JSON response frequently used in this class
private static final String PLAYLIST_VIDEO_RENDERER = "playlistVideoRenderer";