From fe2583f3d72f646f3f961d5a84b3699f6258668c Mon Sep 17 00:00:00 2001 From: Coffeemakr Date: Thu, 4 Jan 2018 17:21:03 +0100 Subject: [PATCH 1/5] Generate and verify Https links --- .../soundcloud/SoundcloudExtractorHelper.java | 35 +++++++++++++++++++ .../SoundcloudPlaylistExtractor.java | 7 ++-- .../soundcloud/SoundcloudStreamExtractor.java | 6 ++-- .../newpipe/extractor/ExtractorAsserts.java | 17 +++++++-- .../SoundcloudChannelExtractorTest.java | 3 +- .../SoundcloudPlaylistExtractorTest.java | 8 +++-- .../SoundcloudStreamExtractorDefaultTest.java | 10 +++--- .../YoutubeStreamExtractorDefaultTest.java | 12 +++---- 8 files changed, 74 insertions(+), 24 deletions(-) create mode 100644 src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudExtractorHelper.java diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudExtractorHelper.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudExtractorHelper.java new file mode 100644 index 000000000..7ac81edd5 --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudExtractorHelper.java @@ -0,0 +1,35 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import com.grack.nanojson.JsonObject; + +import javax.annotation.Nonnull; + +public class SoundcloudExtractorHelper { + + private static final String HTTP = "http://"; + private static final String HTTPS = "https://"; + + + private static String replaceHttpWithHttps(final String url) { + if(!url.isEmpty() && url.startsWith(HTTP)) { + return HTTPS + url.substring(HTTP.length()); + } + return url; + } + + @Nonnull + static String getUploaderUrl(JsonObject object) { + String url = object.getObject("user").getString("permalink_url", ""); + return replaceHttpWithHttps(url); + } + + @Nonnull + static String getAvatarUrl(JsonObject object) { + String url = object.getObject("user", new JsonObject()).getString("avatar_url", ""); + return replaceHttpWithHttps(url); + } + + public static String getUploaderName(JsonObject object) { + return object.getObject("user").getString("username", ""); + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java index 35da76425..e0ea4a011 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java @@ -4,7 +4,6 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.Downloader; -import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -69,17 +68,17 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderUrl() { - return playlist.getObject("user").getString("permalink_url", ""); + return SoundcloudExtractorHelper.getUploaderUrl(playlist); } @Override public String getUploaderName() { - return playlist.getObject("user").getString("username", ""); + return SoundcloudExtractorHelper.getUploaderName(playlist); } @Override public String getUploaderAvatarUrl() { - return playlist.getObject("user", new JsonObject()).getString("avatar_url", ""); + return SoundcloudExtractorHelper.getAvatarUrl(playlist); } @Override diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index ca02b7a0c..c2dc61dc9 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -102,19 +102,19 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Nonnull @Override public String getUploaderUrl() { - return track.getObject("user").getString("permalink_url", ""); + return SoundcloudExtractorHelper.getUploaderUrl(track); } @Nonnull @Override public String getUploaderName() { - return track.getObject("user").getString("username", ""); + return SoundcloudExtractorHelper.getUploaderName(track); } @Nonnull @Override public String getUploaderAvatarUrl() { - return track.getObject("user", new JsonObject()).getString("avatar_url", ""); + return SoundcloudExtractorHelper.getAvatarUrl(track); } @Override diff --git a/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java b/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java index fe3fec14f..68c1053e1 100644 --- a/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java +++ b/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java @@ -1,9 +1,12 @@ package org.schabi.newpipe.extractor; +import javax.annotation.Nonnull; import java.net.MalformedURLException; import java.net.URL; import java.util.List; +import static org.junit.Assert.assertEquals; + public class ExtractorAsserts { public static void assertEmptyErrors(String message, List errors) { if(!errors.isEmpty()) { @@ -14,11 +17,21 @@ public class ExtractorAsserts { } } - public static void assertIsValidUrl(String url) { + @Nonnull + private static URL urlFromString(String url) { try { - new URL(url); + return new URL(url); } catch (MalformedURLException e) { throw new AssertionError("Invalid url: " + url, e); } } + + public static void assertIsValidUrl(String url) { + urlFromString(url); + } + + public static void assertIsSecureUrl(String urlToCheck) { + URL url = urlFromString(urlToCheck); + assertEquals("Protocol of URL is not secure", "https", url.getProtocol()); + } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java index e6db5da95..1272842a3 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; /** @@ -43,7 +44,7 @@ public class SoundcloudChannelExtractorTest { @Test public void testGetAvatarUrl() throws Exception { - assertTrue(extractor.getAvatarUrl().contains("https://")); + assertIsSecureUrl(extractor.getAvatarUrl()); } @Test diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java index a1b43816f..9813c4749 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; /** @@ -42,12 +43,13 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testGetThumbnailUrl() throws Exception { - assertTrue(extractor.getThumbnailUrl(), extractor.getThumbnailUrl().contains("https://")); + assertIsSecureUrl(extractor.getThumbnailUrl()); } @Test public void testGetUploaderUrl() throws Exception { - assertEquals(extractor.getUploaderUrl(), "http://soundcloud.com/liluzivert"); + assertIsSecureUrl(extractor.getUploaderUrl()); + assertEquals(extractor.getUploaderUrl(), "https://soundcloud.com/liluzivert"); } @Test @@ -57,7 +59,7 @@ public class SoundcloudPlaylistExtractorTest { @Test public void testGetUploaderAvatarUrl() throws Exception { - assertTrue(extractor.getUploaderAvatarUrl(), extractor.getUploaderAvatarUrl().contains("https://")); + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); } @Test diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java index d9dbc8fde..392535750 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java @@ -14,6 +14,7 @@ import org.schabi.newpipe.extractor.stream.SubtitlesFormat; import java.io.IOException; import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; /** @@ -69,22 +70,23 @@ public class SoundcloudStreamExtractorDefaultTest { @Test public void testGetUploadDate() throws ParsingException { - assertEquals(extractor.getUploadDate(), "2016-07-31"); + assertEquals("2016-07-31", extractor.getUploadDate()); } @Test public void testGetUploaderUrl() throws ParsingException { - assertEquals(extractor.getUploaderUrl(), "http://soundcloud.com/liluzivert"); + assertIsSecureUrl(extractor.getUploaderUrl()); + assertEquals("https://soundcloud.com/liluzivert", extractor.getUploaderUrl()); } @Test public void testGetThumbnailUrl() throws ParsingException { - assertTrue(extractor.getThumbnailUrl(), extractor.getThumbnailUrl().contains("https://")); + assertIsSecureUrl(extractor.getThumbnailUrl()); } @Test public void testGetUploaderAvatarUrl() throws ParsingException { - assertTrue(extractor.getUploaderAvatarUrl(), extractor.getUploaderAvatarUrl().contains("https://")); + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); } @Test diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java index 7e67cf085..7facda016 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube; import org.junit.BeforeClass; import org.junit.Test; import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.ExtractorAsserts; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -12,6 +13,7 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; /* @@ -38,7 +40,6 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube; * Test for {@link StreamExtractor} */ public class YoutubeStreamExtractorDefaultTest { - public static final String HTTPS = "https://"; private static YoutubeStreamExtractor extractor; @BeforeClass @@ -102,14 +103,12 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetThumbnailUrl() throws ParsingException { - assertTrue(extractor.getThumbnailUrl(), - extractor.getThumbnailUrl().contains(HTTPS)); + assertIsSecureUrl(extractor.getThumbnailUrl()); } @Test public void testGetUploaderAvatarUrl() throws ParsingException { - assertTrue(extractor.getUploaderAvatarUrl(), - extractor.getUploaderAvatarUrl().contains(HTTPS)); + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); } @Test @@ -120,8 +119,7 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetVideoStreams() throws IOException, ExtractionException { for (VideoStream s : extractor.getVideoStreams()) { - assertTrue(s.url, - s.url.contains(HTTPS)); + assertIsSecureUrl(s.url); assertTrue(s.resolution.length() > 0); assertTrue(Integer.toString(s.getFormatId()), 0 <= s.getFormatId() && s.getFormatId() <= 4); From 0ffd4d9743bdb7c69abd61ed8860ee9880f8bf3d Mon Sep 17 00:00:00 2001 From: Coffeemakr Date: Thu, 4 Jan 2018 18:01:25 +0100 Subject: [PATCH 2/5] Improve tests for search results and reuse code --- .../soundcloud/BaseSoundcloudSearchTest.java | 57 +++++++++++++++++++ .../SoundcloudSearchEngineAllTest.java | 17 +----- .../SoundcloudSearchEngineChannelTest.java | 15 +---- .../SoundcloudSearchEnginePlaylistTest.java | 15 +---- .../SoundcloudSearchEngineStreamTest.java | 17 +----- .../YoutubeStreamExtractorRestrictedTest.java | 7 +-- 6 files changed, 65 insertions(+), 63 deletions(-) create mode 100644 src/test/java/org/schabi/newpipe/extractor/services/soundcloud/BaseSoundcloudSearchTest.java diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/BaseSoundcloudSearchTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/BaseSoundcloudSearchTest.java new file mode 100644 index 000000000..a7b3d3a9d --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/BaseSoundcloudSearchTest.java @@ -0,0 +1,57 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import org.junit.Test; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.channel.ChannelInfoItem; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; +import org.schabi.newpipe.extractor.search.SearchResult; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamType; + +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; + +public abstract class BaseSoundcloudSearchTest { + + protected static SearchResult result; + + @Test + public void testResultList() { + assertFalse("Got empty result list", result.resultList.isEmpty()); + for(InfoItem infoItem: result.resultList) { + assertIsSecureUrl(infoItem.getUrl()); + assertIsSecureUrl(infoItem.getThumbnailUrl()); + assertFalse(infoItem.getName().isEmpty()); + assertFalse("Name is probably a URI: " + infoItem.getName(), + infoItem.getName().contains("://")); + if(infoItem instanceof StreamInfoItem) { + // test stream item + StreamInfoItem streamInfoItem = (StreamInfoItem) infoItem; + assertIsSecureUrl(streamInfoItem.getUploaderUrl()); + assertFalse(streamInfoItem.getUploadDate().isEmpty()); + assertFalse(streamInfoItem.getUploaderName().isEmpty()); + assertEquals(StreamType.AUDIO_STREAM, streamInfoItem.getStreamType()); + } else if(infoItem instanceof ChannelInfoItem) { + // Nothing special to check? + } else if(infoItem instanceof PlaylistInfoItem) { + // test playlist item + assertTrue(infoItem.getUrl().contains("/sets/")); + long streamCount = ((PlaylistInfoItem) infoItem).getStreamCount(); + assertTrue(streamCount > 0); + } else { + fail("Unknown infoItem type: " + infoItem); + } + } + } + + @Test + public void testResultErrors() { + assertNotNull(result.errors); + if (!result.errors.isEmpty()) { + for (Throwable error : result.errors) { + error.printStackTrace(); + } + } + assertTrue(result.errors.isEmpty()); + } +} diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineAllTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineAllTest.java index 064cc2612..0620b0d5b 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineAllTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineAllTest.java @@ -6,9 +6,7 @@ import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.search.SearchEngine; -import org.schabi.newpipe.extractor.search.SearchResult; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; @@ -16,8 +14,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; /** * Test for {@link SearchEngine} */ -public class SoundcloudSearchEngineAllTest { - private static SearchResult result; +public class SoundcloudSearchEngineAllTest extends BaseSoundcloudSearchTest { @BeforeClass public static void setUp() throws Exception { @@ -30,18 +27,6 @@ public class SoundcloudSearchEngineAllTest { .getSearchResult(); } - @Test - public void testResultList() { - assertFalse(result.resultList.isEmpty()); - } - - @Test - public void testResultErrors() { - assertNotNull(result.errors); - if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace(); - assertTrue(result.errors.isEmpty()); - } - @Ignore @Test public void testSuggestion() { diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineChannelTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineChannelTest.java index e2fe84572..488b2e960 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineChannelTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineChannelTest.java @@ -15,8 +15,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; /** * Test for {@link SearchEngine} */ -public class SoundcloudSearchEngineChannelTest { - private static SearchResult result; +public class SoundcloudSearchEngineChannelTest extends BaseSoundcloudSearchTest { @BeforeClass public static void setUp() throws Exception { @@ -29,11 +28,6 @@ public class SoundcloudSearchEngineChannelTest { .getSearchResult(); } - @Test - public void testResultList() { - assertFalse(result.resultList.isEmpty()); - } - @Test public void testResultsItemType() { for (InfoItem infoItem : result.resultList) { @@ -41,13 +35,6 @@ public class SoundcloudSearchEngineChannelTest { } } - @Test - public void testResultErrors() { - assertNotNull(result.errors); - if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace(); - assertTrue(result.errors.isEmpty()); - } - @Ignore @Test public void testSuggestion() { diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEnginePlaylistTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEnginePlaylistTest.java index 179646f1c..074cfbbf7 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEnginePlaylistTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEnginePlaylistTest.java @@ -36,8 +36,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; /** * Test for {@link SearchEngine} */ -public class SoundcloudSearchEnginePlaylistTest { - private static SearchResult result; +public class SoundcloudSearchEnginePlaylistTest extends BaseSoundcloudSearchTest { @BeforeClass public static void setUp() throws Exception { @@ -49,11 +48,6 @@ public class SoundcloudSearchEnginePlaylistTest { .getSearchResult(); } - @Test - public void testResultList() { - assertFalse(result.resultList.isEmpty()); - } - @Test public void testUserItemType() { for (InfoItem infoItem : result.resultList) { @@ -61,13 +55,6 @@ public class SoundcloudSearchEnginePlaylistTest { } } - @Test - public void testResultErrors() { - assertNotNull(result.errors); - if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace(); - assertTrue(result.errors.isEmpty()); - } - @Ignore @Test public void testSuggestion() { diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineStreamTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineStreamTest.java index 8cb70aa83..b4439afee 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineStreamTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngineStreamTest.java @@ -15,25 +15,19 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; /** * Test for {@link SearchEngine} */ -public class SoundcloudSearchEngineStreamTest { - private static SearchResult result; +public class SoundcloudSearchEngineStreamTest extends BaseSoundcloudSearchTest { @BeforeClass public static void setUp() throws Exception { NewPipe.init(Downloader.getInstance()); SearchEngine engine = SoundCloud.getService().getSearchEngine(); - // SoundCloud will suggest "lil uzi vert" instead of "lil uzi vert", + // SoundCloud will suggest "lil uzi vert" instead of "lill uzi vert", // keep in mind that the suggestions can NOT change by country (the parameter "de") result = engine.search("lill uzi vert", 0, "de", SearchEngine.Filter.STREAM) .getSearchResult(); } - @Test - public void testResultList() { - assertFalse(result.resultList.isEmpty()); - } - @Test public void testResultsItemType() { for (InfoItem infoItem : result.resultList) { @@ -41,13 +35,6 @@ public class SoundcloudSearchEngineStreamTest { } } - @Test - public void testResultErrors() { - assertNotNull(result.errors); - if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace(); - assertTrue(result.errors.isEmpty()); - } - @Ignore @Test public void testSuggestion() { diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java index bbcd73de1..622b1e868 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java @@ -16,6 +16,7 @@ import java.util.ArrayList; import java.util.List; import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; /** @@ -85,14 +86,12 @@ public class YoutubeStreamExtractorRestrictedTest { @Test public void testGetThumbnailUrl() throws ParsingException { - assertTrue(extractor.getThumbnailUrl(), - extractor.getThumbnailUrl().contains(HTTPS)); + assertIsSecureUrl(extractor.getThumbnailUrl()); } @Test public void testGetUploaderAvatarUrl() throws ParsingException { - assertTrue(extractor.getUploaderAvatarUrl(), - extractor.getUploaderAvatarUrl().contains(HTTPS)); + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); } // FIXME: 25.11.17 Are there no streams or are they not listed? From 06ea74cbb8dcc47f4270c81803f3350f19578b64 Mon Sep 17 00:00:00 2001 From: Coffeemakr Date: Thu, 4 Jan 2018 18:24:09 +0100 Subject: [PATCH 3/5] Add better tests for youtube search items --- .../youtube/BaseYoutubeSearchTest.java | 55 +++++++++++++++++++ .../youtube/YoutubeSearchEngineAllTest.java | 23 +------- .../YoutubeSearchEngineChannelTest.java | 18 +----- .../YoutubeSearchEnginePlaylistTest.java | 22 ++------ .../YoutubeSearchEngineStreamTest.java | 18 +----- 5 files changed, 64 insertions(+), 72 deletions(-) create mode 100644 src/test/java/org/schabi/newpipe/extractor/services/youtube/BaseYoutubeSearchTest.java diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/BaseYoutubeSearchTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/BaseYoutubeSearchTest.java new file mode 100644 index 000000000..41ef15170 --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/BaseYoutubeSearchTest.java @@ -0,0 +1,55 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.junit.Test; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.channel.ChannelInfoItem; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; +import org.schabi.newpipe.extractor.search.SearchResult; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; + +import static org.junit.Assert.*; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; + +public abstract class BaseYoutubeSearchTest { + + protected static SearchResult result; + + @Test + public void testResultList() { + assertFalse("Got empty result list", result.resultList.isEmpty()); + for(InfoItem infoItem: result.resultList) { + assertIsSecureUrl(infoItem.getUrl()); + assertIsSecureUrl(infoItem.getThumbnailUrl()); + assertFalse(infoItem.getName().isEmpty()); + assertFalse("Name is probably a URI: " + infoItem.getName(), + infoItem.getName().contains("://")); + if(infoItem instanceof StreamInfoItem) { + // test stream item + StreamInfoItem streamInfoItem = (StreamInfoItem) infoItem; + assertIsSecureUrl(streamInfoItem.getUploaderUrl()); + assertFalse(streamInfoItem.getUploadDate().isEmpty()); + assertFalse(streamInfoItem.getUploaderName().isEmpty()); + } else if(infoItem instanceof ChannelInfoItem) { + // Nothing special to check? + } else if(infoItem instanceof PlaylistInfoItem) { + // test playlist item + long streamCount = ((PlaylistInfoItem) infoItem).getStreamCount(); + assertTrue(streamCount > 0); + } else { + fail("Unknown infoItem type: " + infoItem); + } + } + } + + @Test + public void testResultErrors() { + assertNotNull(result.errors); + if (!result.errors.isEmpty()) { + for (Throwable error : result.errors) { + error.printStackTrace(); + } + } + assertTrue(result.errors.isEmpty()); + } +} diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java index 4f6701b3c..b24956b1b 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java @@ -40,8 +40,7 @@ import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsValidUrl; /** * Test for {@link SearchEngine} */ -public class YoutubeSearchEngineAllTest { - private static SearchResult result; +public class YoutubeSearchEngineAllTest extends BaseYoutubeSearchTest { @BeforeClass public static void setUpClass() throws Exception { @@ -53,29 +52,13 @@ public class YoutubeSearchEngineAllTest { } @Test - public void testResultList() { - final List results = result.getResults(); - assertFalse("Results are empty: " + results, results.isEmpty()); - - InfoItem firstInfoItem = results.get(0); + public void testResultList_FirstElement() { + InfoItem firstInfoItem = result.getResults().get(0); // THe channel should be the first item assertTrue(firstInfoItem instanceof ChannelInfoItem); assertEquals("name", "PewDiePie", firstInfoItem.name); assertEquals("url","https://www.youtube.com/user/PewDiePie", firstInfoItem.url); - - for(InfoItem item: results) { - assertIsValidUrl(item.url); - } - - } - - @Test - public void testResultErrors() { - for (Throwable error : result.getErrors()) { - error.printStackTrace(); - } - assertTrue(result.getErrors().isEmpty()); } @Ignore diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineChannelTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineChannelTest.java index b1eb2c68a..79f1b20a9 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineChannelTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineChannelTest.java @@ -37,8 +37,7 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube; /** * Test for {@link SearchEngine} */ -public class YoutubeSearchEngineChannelTest { - private static SearchResult result; +public class YoutubeSearchEngineChannelTest extends BaseYoutubeSearchTest { @BeforeClass public static void setUp() throws Exception { @@ -51,14 +50,6 @@ public class YoutubeSearchEngineChannelTest { .getSearchResult(); } - @Test - public void testResultList() { - assertFalse(result.resultList.isEmpty()); - for(InfoItem item: result.getResults()) { - assertIsValidUrl(item.url); - } - } - @Test public void testResultsItemType() { for (InfoItem infoItem : result.resultList) { @@ -66,13 +57,6 @@ public class YoutubeSearchEngineChannelTest { } } - @Test - public void testResultErrors() { - assertNotNull(result.errors); - if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace(); - assertTrue(result.errors.isEmpty()); - } - @Ignore @Test public void testSuggestion() { diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEnginePlaylistTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEnginePlaylistTest.java index b6e60404b..828829542 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEnginePlaylistTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEnginePlaylistTest.java @@ -6,6 +6,7 @@ import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchResult; @@ -37,8 +38,7 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube; /** * Test for {@link SearchEngine} */ -public class YoutubeSearchEnginePlaylistTest { - private static SearchResult result; +public class YoutubeSearchEnginePlaylistTest extends BaseYoutubeSearchTest { @BeforeClass public static void setUp() throws Exception { @@ -52,27 +52,13 @@ public class YoutubeSearchEnginePlaylistTest { } @Test - public void testResultList() { - assertFalse(result.resultList.isEmpty()); - for(InfoItem item: result.getResults()) { - assertIsValidUrl(item.url); - } - } - - @Test - public void testUserItemType() { + public void testInfoItemType() { for (InfoItem infoItem : result.resultList) { + assertTrue(infoItem instanceof PlaylistInfoItem); assertEquals(InfoItem.InfoType.PLAYLIST, infoItem.info_type); } } - @Test - public void testResultErrors() { - assertNotNull(result.errors); - if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace(); - assertTrue(result.errors.isEmpty()); - } - @Ignore @Test public void testSuggestion() { diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineStreamTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineStreamTest.java index 9ca946499..170419c51 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineStreamTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineStreamTest.java @@ -37,8 +37,7 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube; /** * Test for {@link SearchEngine} */ -public class YoutubeSearchEngineStreamTest { - private static SearchResult result; +public class YoutubeSearchEngineStreamTest extends BaseYoutubeSearchTest { @BeforeClass public static void setUp() throws Exception { @@ -51,14 +50,6 @@ public class YoutubeSearchEngineStreamTest { .getSearchResult(); } - @Test - public void testResultList() { - assertFalse(result.resultList.isEmpty()); - for(InfoItem item: result.getResults()) { - assertIsValidUrl(item.url); - } - } - @Test public void testResultsItemType() { for (InfoItem infoItem : result.resultList) { @@ -66,13 +57,6 @@ public class YoutubeSearchEngineStreamTest { } } - @Test - public void testResultErrors() { - assertNotNull(result.errors); - if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace(); - assertTrue(result.errors.isEmpty()); - } - @Ignore @Test public void testSuggestion() { From 7cfdaf39a433122c075b12c7593f54831da98d28 Mon Sep 17 00:00:00 2001 From: Coffeemakr Date: Thu, 4 Jan 2018 18:42:30 +0100 Subject: [PATCH 4/5] Fix relative url in uploader URL --- .../youtube/YoutubeStreamInfoItemExtractor.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java index 9b00d8a36..3e9f51365 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java @@ -94,13 +94,23 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { } } + private static String fixRelativeUrls(String url) { + if(url == null || url.isEmpty()) return url; + if(url.startsWith("/")) { + return "https://youtube.com" + url; + } else { + return url; + } + } + @Override public String getUploaderUrl() throws ParsingException { try { try { - return item.select("div[class=\"yt-lockup-byline\"]").first() + String url = item.select("div[class=\"yt-lockup-byline\"]").first() .select("a").first() .attr("href"); + return fixRelativeUrls(url); } catch (Exception e){} // try this if the first didn't work From 978a33fae2874e80b0c41238e05c6fb870f99885 Mon Sep 17 00:00:00 2001 From: Coffeemakr Date: Mon, 15 Jan 2018 12:33:09 +0100 Subject: [PATCH 5/5] Use 'abs:' prefix --- .../services/youtube/YoutubePlaylistExtractor.java | 2 +- .../youtube/YoutubeStreamInfoItemExtractor.java | 14 ++------------ .../services/youtube/YoutubeTrendingExtractor.java | 2 +- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java index c6e5c064d..112872378 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java @@ -265,7 +265,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderUrl() throws ParsingException { - return getUploaderLink().attr("href"); + return getUploaderLink().attr("abs:href"); } @Override diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java index 3e9f51365..7e792c1af 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java @@ -94,23 +94,13 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { } } - private static String fixRelativeUrls(String url) { - if(url == null || url.isEmpty()) return url; - if(url.startsWith("/")) { - return "https://youtube.com" + url; - } else { - return url; - } - } - @Override public String getUploaderUrl() throws ParsingException { try { try { - String url = item.select("div[class=\"yt-lockup-byline\"]").first() + return item.select("div[class=\"yt-lockup-byline\"]").first() .select("a").first() - .attr("href"); - return fixRelativeUrls(url); + .attr("abs:href"); } catch (Exception e){} // try this if the first didn't work diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java index 5c357405d..20177ef8f 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java @@ -110,7 +110,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor { @Override public String getUploaderUrl() throws ParsingException { try { - String link = getUploaderLink().attr("href"); + String link = getUploaderLink().attr("abs:href"); if (link.isEmpty()) { throw new IllegalArgumentException("is empty"); }