lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #822 from Stypox/more-refactors 

More refactors
This commit is contained in:
Stypox 2022-05-02 19:03:54 +02:00 committed by GitHub
parent 598ebb92ea c2b5370517
commit 2e1c5c119d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 105 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java
View File

@ -2,6 +2,8 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
@ -19,6 +21,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import java.io.IOException;
import java.util.Objects;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
@ -48,21 +52,18 @@ public class BandcampChannelExtractor extends ChannelExtractor {
*/
try {
final String html = getDownloader()
.get(channelInfo.getString("bandcamp_url")
.replace("http://", "https://"))
.get(replaceHttpWithHttps(channelInfo.getString("bandcamp_url")))
.responseBody();
return Jsoup.parse(html)
.getElementById("customHeader")
.getElementsByTag("img")
.first()
.attr("src");
return Stream.of(Jsoup.parse(html).getElementById("customHeader"))
.filter(Objects::nonNull)
.flatMap(element -> element.getElementsByTag("img").stream())
.map(element -> element.attr("src"))
.findFirst()
.orElse(""); // no banner available
} catch (final IOException | ReCaptchaException e) {
throw new ParsingException("Could not download artist web site", e);
} catch (final NullPointerException e) {
// No banner available
return "";
}
}

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelInfoItemExtractor.java
View File

@ -28,13 +28,7 @@ public class BandcampChannelInfoItemExtractor implements ChannelInfoItemExtracto
@Override
public String getThumbnailUrl() throws ParsingException {
final Element img = searchResult.getElementsByClass("art").first()
.getElementsByTag("img").first();
if (img != null) {
return img.attr("src");
} else {
return null;
}
return BandcampExtractorHelper.getThumbnailUrlFromSearchResult(searchResult);
}
@Override

18
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampCommentsInfoItemExtractor.java
View File

@ -4,6 +4,8 @@ import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.util.Objects;
public class BandcampCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
private final Element writing;
@ -16,7 +18,7 @@ public class BandcampCommentsInfoItemExtractor implements CommentsInfoItemExtrac
@Override
public String getName() throws ParsingException {
return writing.getElementsByClass("text").first().ownText();
return getCommentText();
}
@Override
@ -30,13 +32,21 @@ public class BandcampCommentsInfoItemExtractor implements CommentsInfoItemExtrac
}
@Override
public String getCommentText() {
return writing.getElementsByClass("text").first().ownText();
public String getCommentText() throws ParsingException {
return writing.getElementsByClass("text").stream()
.filter(Objects::nonNull)
.map(Element::ownText)
.findFirst()
.orElseThrow(() -> new ParsingException("Could not get comment text"));
}
@Override
public String getUploaderName() throws ParsingException {
return writing.getElementsByClass("name").first().text();
return writing.getElementsByClass("name").stream()
.filter(Objects::nonNull)
.map(Element::text)
.findFirst()
.orElseThrow(() -> new ParsingException("Could not get uploader name"));
}
@Override

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java
View File

@ -8,6 +8,7 @@ import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
@ -19,6 +20,8 @@ import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Locale;
import javax.annotation.Nullable;
public final class BandcampExtractorHelper {
public static final String BASE_URL = "https://bandcamp.com";
@ -134,4 +137,14 @@ public final class BandcampExtractorHelper {
throw new ParsingException("Could not parse date '" + textDate + "'", e);
}
}
@Nullable
public static String getThumbnailUrlFromSearchResult(final Element searchResult) {
return searchResult.getElementsByClass("art").stream()
.flatMap(element -> element.getElementsByTag("img").stream())
.map(element -> element.attr("src"))
.filter(string -> !string.isEmpty())
.findFirst()
.orElse(null);
}
}

25
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java
View File

@ -1,8 +1,15 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor.getAlbumInfoJson;
import static org.schabi.newpipe.extractor.utils.JsonUtils.getJsonData;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.Page;
@ -17,15 +24,9 @@ import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Objects;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl;
import static org.schabi.newpipe.extractor.utils.JsonUtils.getJsonData;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor.getAlbumInfoJson;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import javax.annotation.Nonnull;
public class BandcampPlaylistExtractor extends PlaylistExtractor {
@ -92,12 +93,10 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor {
@Override
public String getUploaderAvatarUrl() {
try {
return Objects.requireNonNull(document.getElementsByClass("band-photo").first())
.attr("src");
} catch (final NullPointerException e) {
return EMPTY_STRING;
}
return document.getElementsByClass("band-photo").stream()
.map(element -> element.attr("src"))
.findFirst()
.orElse(EMPTY_STRING);
}
@Override

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemExtractor.java
View File

@ -38,12 +38,6 @@ public class BandcampPlaylistInfoItemExtractor implements PlaylistInfoItemExtrac
@Override
public String getThumbnailUrl() {
final Element img = searchResult.getElementsByClass("art").first()
.getElementsByTag("img").first();
if (img != null) {
return img.attr("src");
} else {
return null;
}
return BandcampExtractorHelper.getThumbnailUrlFromSearchResult(searchResult);
}
}

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java
View File

@ -5,6 +5,7 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
@ -77,9 +78,11 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor {
@Nonnull
@Override
public String getUploaderName() {
return Jsoup.parse(showInfo.getString("image_caption"))
.getElementsByTag("a").first().text();
public String getUploaderName() throws ParsingException {
return Jsoup.parse(showInfo.getString("image_caption")).getElementsByTag("a").stream()
.map(Element::text)
.findFirst()
.orElseThrow(() -> new ParsingException("Could not get uploader name"));
}
@Nullable

38
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java
View File

@ -18,6 +18,7 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem.BandcampSearchStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
@ -34,7 +35,7 @@ public class BandcampSearchExtractor extends SearchExtractor {
@NonNull
@Override
public String getSearchSuggestion() {
return "";
return Utils.EMPTY_STRING;
}
@Override
@ -50,40 +51,30 @@ public class BandcampSearchExtractor extends SearchExtractor {
public InfoItemsPage<InfoItem> getPage(final Page page)
throws IOException, ExtractionException {
final String html = getDownloader().get(page.getUrl()).responseBody();
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
final Document d = Jsoup.parse(getDownloader().get(page.getUrl()).responseBody());
final Document d = Jsoup.parse(html);
final Elements searchResultsElements = d.getElementsByClass("searchresult");
for (final Element searchResult : searchResultsElements) {
final String type = searchResult.getElementsByClass("result-info").first()
.getElementsByClass("itemtype").first().text();
for (final Element searchResult : d.getElementsByClass("searchresult")) {
final String type = searchResult.getElementsByClass("result-info").stream()
.flatMap(element -> element.getElementsByClass("itemtype").stream())
.map(Element::text)
.findFirst()
.orElse(Utils.EMPTY_STRING);
switch (type) {
default:
continue;
case "FAN":
// don't display fan results
break;
case "ARTIST":
collector.commit(new BandcampChannelInfoItemExtractor(searchResult));
break;
case "ALBUM":
collector.commit(new BandcampPlaylistInfoItemExtractor(searchResult));
break;
case "TRACK":
collector.commit(new BandcampSearchStreamInfoItemExtractor(searchResult, null));
break;
default:
// don't display fan results ("FAN") or other things
break;
}
}
// Count pages
@ -92,7 +83,10 @@ public class BandcampSearchExtractor extends SearchExtractor {
return new InfoItemsPage<>(collector, null);
}
final Elements pages = pageLists.first().getElementsByTag("li");
final Elements pages = pageLists.stream()
.map(element -> element.getElementsByTag("li"))
.findFirst()
.orElseGet(Elements::new);
// Find current page
int currentPage = -1;

11
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java
View File

@ -119,7 +119,7 @@ public class BandcampStreamExtractor extends StreamExtractor {
@Override
public String getThumbnailUrl() throws ParsingException {
if (albumJson.isNull("art_id")) {
return "";
return Utils.EMPTY_STRING;
} else {
return getImageUrl(albumJson.getLong("art_id"), true);
}
@ -128,11 +128,10 @@ public class BandcampStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getUploaderAvatarUrl() {
try {
return document.getElementsByClass("band-photo").first().attr("src");
} catch (final NullPointerException e) {
return "";
}
return document.getElementsByClass("band-photo").stream()
.map(element -> element.attr("src"))
.findFirst()
.orElse(Utils.EMPTY_STRING);
}
@Nonnull

23
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java
View File

@ -4,7 +4,6 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_API_URL;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
@ -17,9 +16,9 @@ import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
public class BandcampSuggestionExtractor extends SuggestionExtractor {
@ -36,20 +35,12 @@ public class BandcampSuggestionExtractor extends SuggestionExtractor {
final JsonObject fuzzyResults = JsonParser.object().from(downloader
.get(AUTOCOMPLETE_URL + URLEncoder.encode(query, "UTF-8")).responseBody());
final JsonArray jsonArray = fuzzyResults.getObject("auto")
.getArray("results");
final List<String> suggestions = new ArrayList<>();
for (final Object fuzzyResult : jsonArray) {
final String res = ((JsonObject) fuzzyResult).getString("name");
if (!suggestions.contains(res)) {
suggestions.add(res);
}
}
return suggestions;
return fuzzyResults.getObject("auto").getArray("results").stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.map(jsonObject -> jsonObject.getString("name"))
.distinct()
.collect(Collectors.toList());
} catch (final JsonParserException e) {
return Collections.emptyList();
}

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/streaminfoitem/BandcampSearchStreamInfoItemExtractor.java
View File

@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors.streaminfoitem
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper;
import javax.annotation.Nullable;
@ -46,13 +47,7 @@ public class BandcampSearchStreamInfoItemExtractor extends BandcampStreamInfoIte
@Override
public String getThumbnailUrl() throws ParsingException {
final Element img = searchResult.getElementsByClass("art").first()
.getElementsByTag("img").first();
if (img != null) {
return img.attr("src");
} else {
return null;
}
return BandcampExtractorHelper.getThumbnailUrlFromSearchResult(searchResult);
}
@Override

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java
View File

@ -138,7 +138,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
}
@Override
public void onFetchPage(final Downloader downloader)
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
final Response response = downloader.get(
baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT + getId());

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java
View File

@ -23,12 +23,15 @@ import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelp
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import javax.annotation.Nonnull;
public class PeertubeCommentsExtractor extends CommentsExtractor {
public PeertubeCommentsExtractor(final StreamingService service,
final ListLinkHandler uiHandler) {
super(service, uiHandler);
}
@Nonnull
@Override
public InfoItemsPage<CommentsInfoItem> getInitialPage()
throws IOException, ExtractionException {
@ -84,6 +87,6 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
}
@Override
public void onFetchPage(final Downloader downloader) {
public void onFetchPage(@Nonnull final Downloader downloader) {
}
}

1
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java
View File

@ -63,6 +63,7 @@ public class PeertubeSearchExtractor extends SearchExtractor {
return Collections.emptyList();
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
return getPage(new Page(getUrl() + "&" + START_KEY + "=0&"

3
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java
View File

@ -388,7 +388,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
}
@Override
public void onFetchPage(final Downloader downloader) throws IOException, ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
final Response response = downloader.get(
baseUrl + PeertubeStreamLinkHandlerFactory.VIDEO_API_ENDPOINT + getId());
if (response != null) {

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java
View File

@ -33,11 +33,13 @@ public class PeertubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
super(streamingService, linkHandler, kioskId);
}
@Nonnull
@Override
public String getName() throws ParsingException {
return getId();
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
return getPage(new Page(getUrl() + "&" + START_KEY + "=0&"

3
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java
View File

@ -13,6 +13,8 @@ import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.VI
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import javax.annotation.Nonnull;
public class ItagItem {
/**
* List can be found here
@ -96,6 +98,7 @@ public class ItagItem {
return false;
}
@Nonnull
public static ItagItem getItag(final int itagId) throws ParsingException {
for (final ItagItem item : ITAG_LIST) {
if (itagId == item.id) {

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -975,7 +975,7 @@ public final class YoutubeParsingHelper {
if (html) {
text = text.replaceAll("\\n", "<br>");
text = text.replaceAll(" ", " &nbsp;");
text = text.replaceAll(" {2}", " &nbsp;");
}
return text;

1
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -134,6 +134,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
return !showingResultsForRenderer.isEmpty();
}
@Nonnull
@Override
public List<MetaInfo> getMetaInfo() throws ParsingException {
return YoutubeParsingHelper.getMetaInfo(

4
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java
View File

@ -29,8 +29,8 @@ public final class Utils {
@Deprecated
public static final String UTF_8 = "UTF-8";
public static final String EMPTY_STRING = "";
private static final Pattern M_PATTERN = Pattern.compile("(https?)?:\\/\\/m\\.");
private static final Pattern WWW_PATTERN = Pattern.compile("(https?)?:\\/\\/www\\.");
private static final Pattern M_PATTERN = Pattern.compile("(https?)?://m\\.");
private static final Pattern WWW_PATTERN = Pattern.compile("(https?)?://www\\.");
private Utils() {
// no instance