diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index e21b17f3b..3d09d5094 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -16,6 +16,7 @@ import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; +import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nullable; import java.util.Collections; @@ -277,12 +278,13 @@ public abstract class StreamingService { * Figures out where the link is pointing to (a channel, a video, a playlist, etc.) * @param url the url on which it should be decided of which link type it is * @return the link type of url - * @throws ParsingException */ public final LinkType getLinkTypeByUrl(String url) throws ParsingException { - LinkHandlerFactory sH = getStreamLHFactory(); - LinkHandlerFactory cH = getChannelLHFactory(); - LinkHandlerFactory pH = getPlaylistLHFactory(); + url = Utils.followGoogleRedirectIfNeeded(url); + + final LinkHandlerFactory sH = getStreamLHFactory(); + final LinkHandlerFactory cH = getChannelLHFactory(); + final LinkHandlerFactory pH = getPlaylistLHFactory(); if (sH != null && sH.acceptUrl(url)) { return LinkType.STREAM; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java index 6bba7b4e5..7dcfe5f48 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java @@ -42,12 +42,30 @@ public abstract class LinkHandlerFactory { // Logic /////////////////////////////////// + /** + * Builds a {@link LinkHandler} from a url.
+ * Be sure to call {@link Utils#followGoogleRedirectIfNeeded(String)} on the url if overriding + * this function. + * @param url the url to extract path and id from + * @return a {@link LinkHandler} complete with information + */ public LinkHandler fromUrl(String url) throws ParsingException { if (url == null) throw new IllegalArgumentException("url can not be null"); + url = Utils.followGoogleRedirectIfNeeded(url); final String baseUrl = Utils.getBaseUrl(url); return fromUrl(url, baseUrl); } + /** + * Builds a {@link LinkHandler} from a url and a base url. The url is expected to be already + * polished from google search redirects (otherwise how could {@code baseUrl} have been + * extracted?).
+ * So do not call {@link Utils#followGoogleRedirectIfNeeded(String)} on the url if overriding + * this function, since that should be done in {@link #fromUrl(String)}. + * @param url the url without google search redirects to extract id from + * @param baseUrl the base url + * @return a {@link LinkHandler} complete with information + */ public LinkHandler fromUrl(String url, String baseUrl) throws ParsingException { if (url == null) throw new IllegalArgumentException("url can not be null"); if (!acceptUrl(url)) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java index 9ea478b02..cdbbab4f0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java @@ -32,7 +32,8 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory { @Override public ListLinkHandler fromUrl(String url) throws ParsingException { - String baseUrl = Utils.getBaseUrl(url); + url = Utils.followGoogleRedirectIfNeeded(url); + final String baseUrl = Utils.getBaseUrl(url); return fromUrl(url, baseUrl); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index c6bd508a2..e8823af86 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -196,6 +196,26 @@ public class Utils { } } + /** + * If the provided url is a Google search redirect, then the actual url is extracted from the + * {@code url=} query value and returned, otherwise the original url is returned. + * @param url the url which can possibly be a Google search redirect + * @return an url with no Google search redirects + */ + public static String followGoogleRedirectIfNeeded(final String url) { + // if the url is a redirect from a Google search, extract the actual url + try { + final URL decoded = Utils.stringToURL(url); + if (decoded.getHost().contains("google") && decoded.getPath().equals("/url")) { + return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), "UTF-8"); + } + } catch (final Exception ignored) { + } + + // url is not a google search redirect + return url; + } + public static boolean isNullOrEmpty(final String str) { return str == null || str.isEmpty(); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java index bdad6cb6b..5dbc43174 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java @@ -6,6 +6,7 @@ import java.util.HashSet; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.NewPipe.getServiceByUrl; +import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.ServiceList.YouTube; public class NewPipeTest { @@ -39,8 +40,10 @@ public class NewPipeTest { assertEquals(getServiceByUrl("https://www.youtube.com/watch?v=_r6CgaFNAGg"), YouTube); assertEquals(getServiceByUrl("https://www.youtube.com/channel/UCi2bIyFtz-JdI-ou8kaqsqg"), YouTube); assertEquals(getServiceByUrl("https://www.youtube.com/playlist?list=PLRqwX-V7Uu6ZiZxtDDRCi6uhfTH4FilpH"), YouTube); + assertEquals(getServiceByUrl("https://www.google.it/url?sa=t&rct=j&q=&esrc=s&cd=&cad=rja&uact=8&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DHu80uDzh8RY&source=video"), YouTube); - assertNotEquals(getServiceByUrl("https://soundcloud.com/pegboardnerds"), YouTube); + assertEquals(getServiceByUrl("https://soundcloud.com/pegboardnerds"), SoundCloud); + assertEquals(getServiceByUrl("https://www.google.com/url?sa=t&url=https%3A%2F%2Fsoundcloud.com%2Fciaoproduction&rct=j&q=&esrc=s&source=web&cd="), SoundCloud); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java index dcccc16da..e4a65505b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java @@ -30,4 +30,19 @@ public class UtilsTest { assertEquals("vnd.youtube", Utils.getBaseUrl("vnd.youtube://n8X9_MgEdCg")); assertEquals("https://music.youtube.com", Utils.getBaseUrl("https://music.youtube.com/watch?v=O0EDx9WAelc")); } + + @Test + public void testFollowGoogleRedirect() { + assertEquals("https://www.youtube.com/watch?v=Hu80uDzh8RY", + Utils.followGoogleRedirectIfNeeded("https://www.google.it/url?sa=t&rct=j&q=&esrc=s&cd=&cad=rja&uact=8&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DHu80uDzh8RY&source=video")); + assertEquals("https://www.youtube.com/watch?v=0b6cFWG45kA", + Utils.followGoogleRedirectIfNeeded("https://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=video&cd=&cad=rja&uact=8&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3D0b6cFWG45kA")); + assertEquals("https://soundcloud.com/ciaoproduction", + Utils.followGoogleRedirectIfNeeded("https://www.google.com/url?sa=t&url=https%3A%2F%2Fsoundcloud.com%2Fciaoproduction&rct=j&q=&esrc=s&source=web&cd=")); + + assertEquals("https://www.youtube.com/watch?v=Hu80uDzh8RY¶m=xyz", + Utils.followGoogleRedirectIfNeeded("https://www.youtube.com/watch?v=Hu80uDzh8RY¶m=xyz")); + assertEquals("https://www.youtube.com/watch?v=Hu80uDzh8RY&url=hello", + Utils.followGoogleRedirectIfNeeded("https://www.youtube.com/watch?v=Hu80uDzh8RY&url=hello")); + } }