Use immutable sets in YoutubeParsingHelper.

This commit is contained in:
Isira Seneviratne 2022-08-09 09:14:34 +05:30
parent eb07d70a2c
commit 316d8573fa
7 changed files with 33 additions and 53 deletions

View File

@ -19,10 +19,10 @@ jobs:
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- name: set up JDK 8 - name: set up JDK 11
uses: actions/setup-java@v3 uses: actions/setup-java@v3
with: with:
java-version: '8' java-version: '11'
distribution: 'temurin' distribution: 'temurin'
- name: Cache Gradle dependencies - name: Cache Gradle dependencies

View File

@ -5,8 +5,8 @@ allprojects {
compileJava.options.encoding = 'UTF-8' compileJava.options.encoding = 'UTF-8'
compileTestJava.options.encoding = 'UTF-8' compileTestJava.options.encoding = 'UTF-8'
sourceCompatibility = 1.8 sourceCompatibility = JavaVersion.VERSION_11
targetCompatibility = 1.8 targetCompatibility = JavaVersion.VERSION_11
version 'v0.22.1' version 'v0.22.1'
group 'com.github.TeamNewPipe' group 'com.github.TeamNewPipe'

View File

@ -66,10 +66,12 @@ import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import java.util.Random; import java.util.Random;
import java.util.Set;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.stream.Stream; import java.util.stream.Stream;
@ -241,6 +243,18 @@ public final class YoutubeParsingHelper {
private static final Pattern C_ANDROID_PATTERN = Pattern.compile("&c=ANDROID"); private static final Pattern C_ANDROID_PATTERN = Pattern.compile("&c=ANDROID");
private static final Pattern C_IOS_PATTERN = Pattern.compile("&c=IOS"); private static final Pattern C_IOS_PATTERN = Pattern.compile("&c=IOS");
private static final Set<String> GOOGLE_URLS = Set.of("google.", "m.google.", "www.google.");
private static final Set<String> INVIDIOUS_URLS = Set.of("invidio.us", "dev.invidio.us",
"www.invidio.us", "redirect.invidious.io", "invidious.snopyta.org", "yewtu.be",
"tube.connect.cafe", "tubus.eduvid.org", "invidious.kavin.rocks", "invidious.site",
"invidious-us.kavin.rocks", "piped.kavin.rocks", "vid.mint.lgbt", "invidiou.site",
"invidious.fdn.fr", "invidious.048596.xyz", "invidious.zee.li", "vid.puffyan.us",
"ytprivate.com", "invidious.namazso.eu", "invidious.silkky.cloud", "ytb.trom.tf",
"invidious.exonip.de", "inv.riverside.rocks", "invidious.blamefran.net", "y.com.cm",
"invidious.moomoo.me", "yt.cyberhost.uk");
private static final Set<String> YOUTUBE_URLS = Set.of("youtube.com", "www.youtube.com",
"m.youtube.com", "music.youtube.com");
/** /**
* Determines how the consent cookie (that is required for YouTube) will be generated. * Determines how the consent cookie (that is required for YouTube) will be generated.
* *
@ -262,21 +276,14 @@ public final class YoutubeParsingHelper {
final String cachedUrl = extractCachedUrlIfNeeded(url); final String cachedUrl = extractCachedUrlIfNeeded(url);
try { try {
final URL u = new URL(cachedUrl); final URL u = new URL(cachedUrl);
final String host = u.getHost(); return GOOGLE_URLS.stream().anyMatch(item -> u.getHost().startsWith(item));
return host.startsWith("google.")
|| host.startsWith("m.google.")
|| host.startsWith("www.google.");
} catch (final MalformedURLException e) { } catch (final MalformedURLException e) {
return false; return false;
} }
} }
public static boolean isYoutubeURL(@Nonnull final URL url) { public static boolean isYoutubeURL(@Nonnull final URL url) {
final String host = url.getHost(); return YOUTUBE_URLS.contains(url.getHost().toLowerCase(Locale.ROOT));
return host.equalsIgnoreCase("youtube.com")
|| host.equalsIgnoreCase("www.youtube.com")
|| host.equalsIgnoreCase("m.youtube.com")
|| host.equalsIgnoreCase("music.youtube.com");
} }
public static boolean isYoutubeServiceURL(@Nonnull final URL url) { public static boolean isYoutubeServiceURL(@Nonnull final URL url) {
@ -290,36 +297,8 @@ public final class YoutubeParsingHelper {
return host.equalsIgnoreCase("hooktube.com"); return host.equalsIgnoreCase("hooktube.com");
} }
public static boolean isInvidioURL(@Nonnull final URL url) { public static boolean isInvidiousURL(@Nonnull final URL url) {
final String host = url.getHost(); return INVIDIOUS_URLS.contains(url.getHost().toLowerCase(Locale.ROOT));
return host.equalsIgnoreCase("invidio.us")
|| host.equalsIgnoreCase("dev.invidio.us")
|| host.equalsIgnoreCase("www.invidio.us")
|| host.equalsIgnoreCase("redirect.invidious.io")
|| host.equalsIgnoreCase("invidious.snopyta.org")
|| host.equalsIgnoreCase("yewtu.be")
|| host.equalsIgnoreCase("tube.connect.cafe")
|| host.equalsIgnoreCase("tubus.eduvid.org")
|| host.equalsIgnoreCase("invidious.kavin.rocks")
|| host.equalsIgnoreCase("invidious-us.kavin.rocks")
|| host.equalsIgnoreCase("piped.kavin.rocks")
|| host.equalsIgnoreCase("invidious.site")
|| host.equalsIgnoreCase("vid.mint.lgbt")
|| host.equalsIgnoreCase("invidiou.site")
|| host.equalsIgnoreCase("invidious.fdn.fr")
|| host.equalsIgnoreCase("invidious.048596.xyz")
|| host.equalsIgnoreCase("invidious.zee.li")
|| host.equalsIgnoreCase("vid.puffyan.us")
|| host.equalsIgnoreCase("ytprivate.com")
|| host.equalsIgnoreCase("invidious.namazso.eu")
|| host.equalsIgnoreCase("invidious.silkky.cloud")
|| host.equalsIgnoreCase("invidious.exonip.de")
|| host.equalsIgnoreCase("inv.riverside.rocks")
|| host.equalsIgnoreCase("invidious.blamefran.net")
|| host.equalsIgnoreCase("invidious.moomoo.me")
|| host.equalsIgnoreCase("ytb.trom.tf")
|| host.equalsIgnoreCase("yt.cyberhost.uk")
|| host.equalsIgnoreCase("y.com.cm");
} }
public static boolean isY2ubeURL(@Nonnull final URL url) { public static boolean isY2ubeURL(@Nonnull final URL url) {

View File

@ -20,7 +20,6 @@
package org.schabi.newpipe.extractor.services.youtube.linkHandler; package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import java.util.regex.Pattern;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
@ -29,6 +28,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.net.URL; import java.net.URL;
import java.util.List; import java.util.List;
import java.util.regex.Pattern;
import static org.schabi.newpipe.extractor.utils.Utils.isBlank; import static org.schabi.newpipe.extractor.utils.Utils.isBlank;
@ -90,7 +90,7 @@ public final class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFacto
String path = urlObj.getPath(); String path = urlObj.getPath();
if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj) if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj)
|| YoutubeParsingHelper.isInvidioURL(urlObj) || YoutubeParsingHelper.isInvidiousURL(urlObj)
|| YoutubeParsingHelper.isHooktubeURL(urlObj))) { || YoutubeParsingHelper.isHooktubeURL(urlObj))) {
throw new ParsingException("The URL given is not a YouTube URL"); throw new ParsingException("The URL given is not a YouTube URL");
} }

View File

@ -1,8 +1,5 @@
package org.schabi.newpipe.extractor.services.youtube.linkHandler; package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.List;
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
@ -11,6 +8,10 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.List;
public final class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { public final class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
private static final YoutubePlaylistLinkHandlerFactory INSTANCE = private static final YoutubePlaylistLinkHandlerFactory INSTANCE =
@ -35,7 +36,7 @@ public final class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFact
final URL urlObj = Utils.stringToURL(url); final URL urlObj = Utils.stringToURL(url);
if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj) if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj)
|| YoutubeParsingHelper.isInvidioURL(urlObj))) { || YoutubeParsingHelper.isInvidiousURL(urlObj))) {
throw new ParsingException("the url given is not a YouTube-URL"); throw new ParsingException("the url given is not a YouTube-URL");
} }

View File

@ -1,7 +1,7 @@
package org.schabi.newpipe.extractor.services.youtube.linkHandler; package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isHooktubeURL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isHooktubeURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidioURL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidiousURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isY2ubeURL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isY2ubeURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeServiceURL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeServiceURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL;
@ -122,7 +122,7 @@ public final class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
} }
if (!Utils.isHTTP(url) || !(isYoutubeURL(url) || isYoutubeServiceURL(url) if (!Utils.isHTTP(url) || !(isYoutubeURL(url) || isYoutubeServiceURL(url)
|| isHooktubeURL(url) || isInvidioURL(url) || isY2ubeURL(url))) { || isHooktubeURL(url) || isInvidiousURL(url) || isY2ubeURL(url))) {
if (host.equalsIgnoreCase("googleads.g.doubleclick.net")) { if (host.equalsIgnoreCase("googleads.g.doubleclick.net")) {
throw new FoundAdException("Error found ad: " + urlString); throw new FoundAdException("Error found ad: " + urlString);
} }

View File

@ -20,7 +20,7 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>. * along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/ */
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidioURL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidiousURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
@ -53,7 +53,7 @@ public class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
} }
final String urlPath = urlObj.getPath(); final String urlPath = urlObj.getPath();
return Utils.isHTTP(urlObj) && (isYoutubeURL(urlObj) || isInvidioURL(urlObj)) return Utils.isHTTP(urlObj) && (isYoutubeURL(urlObj) || isInvidiousURL(urlObj))
&& urlPath.equals("/feed/trending"); && urlPath.equals("/feed/trending");
} }
} }