lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Implement time ago parser and improve localization handling 

- Handle special cases for languages where the number is not shown
- Rework the Downloader base implementation, allowing for more
advanced things to be done
- Separate the localization from the content country (just like
YouTube let's the user choose both).
This commit is contained in:
Mauricio Colli 2019-04-28 17:03:16 -03:00
parent 180836c180
commit 3638f0e0ea
No known key found for this signature in database
GPG Key ID: F200BFD6F29DDD85
274 changed files with 4770 additions and 3468 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
extractor/.attach_pid31246
View File

44
extractor/src/main/java/org/schabi/newpipe/extractor/DownloadRequest.java
View File

@ -1,44 +0,0 @@
package org.schabi.newpipe.extractor;
import java.util.Collections;
import java.util.List;
import java.util.Map;
public class DownloadRequest {
private final String requestBody;
private final Map<String, List<String>> requestHeaders;
public static final DownloadRequest emptyRequest = new DownloadRequest(null, null);
public DownloadRequest(String requestBody, Map<String, List<String>> headers) {
super();
this.requestBody = requestBody;
if(null != headers) {
this.requestHeaders = headers;
}else {
this.requestHeaders = Collections.emptyMap();
}
}
public String getRequestBody() {
return requestBody;
}
public Map<String, List<String>> getRequestHeaders() {
return requestHeaders;
}
public void setRequestCookies(List<String> cookies){
requestHeaders.put("Cookie", cookies);
}
public List<String> getRequestCookies(){
if(null == requestHeaders) return Collections.emptyList();
List<String> cookies = requestHeaders.get("Cookie");
if(null == cookies)
return Collections.emptyList();
else
return cookies;
}
}

42
extractor/src/main/java/org/schabi/newpipe/extractor/DownloadResponse.java
View File

@ -1,42 +0,0 @@
package org.schabi.newpipe.extractor;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
public class DownloadResponse {
private final int responseCode;
private final String responseBody;
private final Map<String, List<String>> responseHeaders;
public DownloadResponse(int responseCode, String responseBody, Map<String, List<String>> headers) {
super();
this.responseCode = responseCode;
this.responseBody = responseBody;
this.responseHeaders = headers;
}
public int getResponseCode() {
return responseCode;
}
public String getResponseBody() {
return responseBody;
}
public Map<String, List<String>> getResponseHeaders() {
return responseHeaders;
}
@Nonnull
public List<String> getResponseCookies(){
if(null == responseHeaders) return Collections.emptyList();
List<String> cookies = responseHeaders.get("Set-Cookie");
if(null == cookies)
return Collections.emptyList();
else
return cookies;
}
}

75
extractor/src/main/java/org/schabi/newpipe/extractor/Downloader.java
View File

@ -1,75 +0,0 @@
package org.schabi.newpipe.extractor;
import java.io.IOException;
import java.util.Map;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.utils.Localization;
/*
* Created by Christian Schabesberger on 28.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* Downloader.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public interface Downloader {
/**
* Download the text file at the supplied URL as in download(String), but set
* the HTTP header field "Accept-Language" to the supplied string.
*
* @param siteUrl the URL of the text file to return the contents of
* @param localization the language and country (usually a 2-character code for each)
* @return the contents of the specified text file
* @throws IOException
*/
String download(String siteUrl, Localization localization) throws IOException, ReCaptchaException;
/**
* Download the text file at the supplied URL as in download(String), but set
* the HTTP header field "Accept-Language" to the supplied string.
*
* @param siteUrl the URL of the text file to return the contents of
* @param customProperties set request header properties
* @return the contents of the specified text file
* @throws IOException
*/
String download(String siteUrl, Map<String, String> customProperties) throws IOException, ReCaptchaException;
/**
* Download (via HTTP) the text file located at the supplied URL, and return its
* contents. Primarily intended for downloading web pages.
*
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file
* @throws IOException
*/
String download(String siteUrl) throws IOException, ReCaptchaException;
DownloadResponse head(String siteUrl) throws IOException, ReCaptchaException;
DownloadResponse get(String siteUrl, Localization localization)
throws IOException, ReCaptchaException;
DownloadResponse get(String siteUrl, DownloadRequest request)
throws IOException, ReCaptchaException;
DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException;
DownloadResponse post(String siteUrl, DownloadRequest request)
throws IOException, ReCaptchaException;
}

40
extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java
View File

@ -1,9 +1,12 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@ -16,21 +19,20 @@ public abstract class Extractor{
* Useful for getting other things from a service (like the url handlers for cleaning/accepting/get id from urls).
*/
private final StreamingService service;
private final LinkHandler linkHandler;
private final Localization localization;
@Nullable
@Nullable private Localization forcedLocalization = null;
@Nullable private ContentCountry forcedContentCountry = null;
private boolean pageFetched = false;
private final Downloader downloader;
public Extractor(final StreamingService service, final LinkHandler linkHandler, final Localization localization) {
public Extractor(final StreamingService service, final LinkHandler linkHandler) {
if(service == null) throw new NullPointerException("service is null");
if(linkHandler == null) throw new NullPointerException("LinkHandler is null");
this.service = service;
this.linkHandler = linkHandler;
this.downloader = NewPipe.getDownloader();
this.localization = localization;
if(downloader == null) throw new NullPointerException("downloader is null");
}
@ -105,8 +107,30 @@ public abstract class Extractor{
return downloader;
}
/*//////////////////////////////////////////////////////////////////////////
// Localization
//////////////////////////////////////////////////////////////////////////*/
public void forceLocalization(Localization localization) {
this.forcedLocalization = localization;
}
public void forceContentCountry(ContentCountry contentCountry) {
this.forcedContentCountry = contentCountry;
}
@Nonnull
public Localization getLocalization() {
return localization;
public Localization getExtractorLocalization() {
return forcedLocalization == null ? getService().getLocalization() : forcedLocalization;
}
@Nonnull
public ContentCountry getExtractorContentCountry() {
return forcedContentCountry == null ? getService().getContentCountry() : forcedContentCountry;
}
@Nonnull
public TimeAgoParser getTimeAgoParser() {
return getService().getTimeAgoParser(getExtractorLocalization());
}
}

5
extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java
View File

@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.Localization;
import javax.annotation.Nonnull;
import java.io.IOException;
@ -14,8 +13,8 @@ import java.util.List;
*/
public abstract class ListExtractor<R extends InfoItem> extends Extractor {
public ListExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public ListExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
}
/**

62
extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java
View File

@ -20,24 +20,42 @@ package org.schabi.newpipe.extractor;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.List;
/**
* Provides access to streaming services supported by NewPipe.
*/
public class NewPipe {
private static Downloader downloader = null;
private static Localization localization = null;
private static Downloader downloader;
private static Localization preferredLocalization;
private static ContentCountry preferredContentCountry;
private NewPipe() {
}
public static void init(Downloader d) {
downloader = d;
preferredLocalization = Localization.DEFAULT;
preferredContentCountry = ContentCountry.DEFAULT;
}
public static void init(Downloader d, Localization l) {
downloader = d;
localization = l;
preferredLocalization = l;
preferredContentCountry = l.getCountryCode().isEmpty() ? ContentCountry.DEFAULT : new ContentCountry(l.getCountryCode());
}
public static void init(Downloader d, Localization l, ContentCountry c) {
downloader = d;
preferredLocalization = l;
preferredContentCountry = c;
}
public static Downloader getDownloader() {
@ -99,11 +117,41 @@ public class NewPipe {
}
}
public static void setLocalization(Localization localization) {
NewPipe.localization = localization;
/*//////////////////////////////////////////////////////////////////////////
// Localization
//////////////////////////////////////////////////////////////////////////*/
public static void setupLocalization(Localization preferredLocalization) {
setupLocalization(preferredLocalization, null);
}
public static void setupLocalization(Localization preferredLocalization, @Nullable ContentCountry preferredContentCountry) {
NewPipe.preferredLocalization = preferredLocalization;
if (preferredContentCountry != null) {
NewPipe.preferredContentCountry = preferredContentCountry;
} else {
NewPipe.preferredContentCountry = preferredLocalization.getCountryCode().isEmpty()
? ContentCountry.DEFAULT
: new ContentCountry(preferredLocalization.getCountryCode());
}
}
@Nonnull
public static Localization getPreferredLocalization() {
return localization;
return preferredLocalization == null ? Localization.DEFAULT : preferredLocalization;
}
public static void setPreferredLocalization(Localization preferredLocalization) {
NewPipe.preferredLocalization = preferredLocalization;
}
@Nonnull
public static ContentCountry getPreferredContentCountry() {
return preferredContentCountry == null ? ContentCountry.DEFAULT : preferredContentCountry;
}
public static void setPreferredContentCountry(ContentCountry preferredContentCountry) {
NewPipe.preferredContentCountry = preferredContentCountry;
}
}

219
extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java
View File

@ -1,25 +1,23 @@
package org.schabi.newpipe.extractor;
import java.util.Collections;
import java.util.List;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.*;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.TimeAgoParser;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.util.Collections;
import java.util.List;
/*
* Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
@ -113,9 +111,9 @@ public abstract class StreamingService {
return serviceId + ":" + serviceInfo.getName();
}
////////////////////////////////////////////
/*//////////////////////////////////////////////////////////////////////////
// Url Id handler
////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////*/
/**
* Must return a new instance of an implementation of LinkHandlerFactory for streams.
@ -144,25 +142,22 @@ public abstract class StreamingService {
public abstract SearchQueryHandlerFactory getSearchQHFactory();
public abstract ListLinkHandlerFactory getCommentsLHFactory();
////////////////////////////////////////////
// Extractor
////////////////////////////////////////////
/*//////////////////////////////////////////////////////////////////////////
// Extractors
//////////////////////////////////////////////////////////////////////////*/
/**
* Must create a new instance of a SearchExtractor implementation.
* @param queryHandler specifies the keyword lock for, and the filters which should be applied.
* @param localization specifies the language/country for the extractor.
* @return a new SearchExtractor instance
*/
public abstract SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler, Localization localization);
public abstract SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler);
/**
* Must create a new instance of a SuggestionExtractor implementation.
* @param localization specifies the language/country for the extractor.
* @return a new SuggestionExtractor instance
*/
public abstract SuggestionExtractor getSuggestionExtractor(Localization localization);
public abstract SuggestionExtractor getSuggestionExtractor();
/**
* Outdated or obsolete. null can be returned.
@ -180,113 +175,72 @@ public abstract class StreamingService {
/**
* Must create a new instance of a ChannelExtractor implementation.
* @param linkHandler is pointing to the channel which should be handled by this new instance.
* @param localization specifies the language used for the request.
* @return a new ChannelExtractor
* @throws ExtractionException
*/
public abstract ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler,
Localization localization) throws ExtractionException;
public abstract ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) throws ExtractionException;
/**
* Must crete a new instance of a PlaylistExtractor implementation.
* @param linkHandler is pointing to the playlist which should be handled by this new instance.
* @param localization specifies the language used for the request.
* @return a new PlaylistExtractor
* @throws ExtractionException
*/
public abstract PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler,
Localization localization) throws ExtractionException;
public abstract PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) throws ExtractionException;
/**
* Must create a new instance of a StreamExtractor implementation.
* @param linkHandler is pointing to the stream which should be handled by this new instance.
* @param localization specifies the language used for the request.
* @return a new StreamExtractor
* @throws ExtractionException
*/
public abstract StreamExtractor getStreamExtractor(LinkHandler linkHandler,
Localization localization) throws ExtractionException;
public abstract CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler,
Localization localization) throws ExtractionException;
////////////////////////////////////////////
// Extractor with default localization
////////////////////////////////////////////
public abstract StreamExtractor getStreamExtractor(LinkHandler linkHandler) throws ExtractionException;
public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) {
return getSearchExtractor(queryHandler, NewPipe.getPreferredLocalization());
}
public abstract CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler) throws ExtractionException;
public SuggestionExtractor getSuggestionExtractor() {
return getSuggestionExtractor(NewPipe.getPreferredLocalization());
}
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) throws ExtractionException {
return getChannelExtractor(linkHandler, NewPipe.getPreferredLocalization());
}
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) throws ExtractionException {
return getPlaylistExtractor(linkHandler, NewPipe.getPreferredLocalization());
}
public StreamExtractor getStreamExtractor(LinkHandler linkHandler) throws ExtractionException {
return getStreamExtractor(linkHandler, NewPipe.getPreferredLocalization());
}
public CommentsExtractor getCommentsExtractor(ListLinkHandler urlIdHandler) throws ExtractionException {
return getCommentsExtractor(urlIdHandler, NewPipe.getPreferredLocalization());
}
////////////////////////////////////////////
// Extractor without link handler
////////////////////////////////////////////
/*//////////////////////////////////////////////////////////////////////////
// Extractors without link handler
//////////////////////////////////////////////////////////////////////////*/
public SearchExtractor getSearchExtractor(String query,
List<String> contentFilter,
String sortFilter,
Localization localization) throws ExtractionException {
String sortFilter) throws ExtractionException {
return getSearchExtractor(getSearchQHFactory()
.fromQuery(query,
contentFilter,
sortFilter),
localization);
.fromQuery(query, contentFilter, sortFilter));
}
public ChannelExtractor getChannelExtractor(String id,
List<String> contentFilter,
String sortFilter,
Localization localization) throws ExtractionException {
return getChannelExtractor(getChannelLHFactory().fromQuery(id, contentFilter, sortFilter), localization);
String sortFilter) throws ExtractionException {
return getChannelExtractor(getChannelLHFactory()
.fromQuery(id, contentFilter, sortFilter));
}
public PlaylistExtractor getPlaylistExtractor(String id,
List<String> contentFilter,
String sortFilter,
Localization localization) throws ExtractionException {
String sortFilter) throws ExtractionException {
return getPlaylistExtractor(getPlaylistLHFactory()
.fromQuery(id,
contentFilter,
sortFilter),
localization);
.fromQuery(id, contentFilter, sortFilter));
}
////////////////////////////////////////////
// Short extractor without localization
////////////////////////////////////////////
/*//////////////////////////////////////////////////////////////////////////
// Short extractors overloads
//////////////////////////////////////////////////////////////////////////*/
public SearchExtractor getSearchExtractor(String query) throws ExtractionException {
return getSearchExtractor(getSearchQHFactory().fromQuery(query), NewPipe.getPreferredLocalization());
return getSearchExtractor(getSearchQHFactory().fromQuery(query));
}
public ChannelExtractor getChannelExtractor(String url) throws ExtractionException {
return getChannelExtractor(getChannelLHFactory().fromUrl(url), NewPipe.getPreferredLocalization());
return getChannelExtractor(getChannelLHFactory().fromUrl(url));
}
public PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException {
return getPlaylistExtractor(getPlaylistLHFactory().fromUrl(url), NewPipe.getPreferredLocalization());
return getPlaylistExtractor(getPlaylistLHFactory().fromUrl(url));
}
public StreamExtractor getStreamExtractor(String url) throws ExtractionException {
return getStreamExtractor(getStreamLHFactory().fromUrl(url), NewPipe.getPreferredLocalization());
return getStreamExtractor(getStreamLHFactory().fromUrl(url));
}
public CommentsExtractor getCommentsExtractor(String url) throws ExtractionException {
@ -294,12 +248,12 @@ public abstract class StreamingService {
if(null == llhf) {
return null;
}
return getCommentsExtractor(llhf.fromUrl(url), NewPipe.getPreferredLocalization());
return getCommentsExtractor(llhf.fromUrl(url));
}
public TimeAgoParser getTimeAgoParser() {
return new TimeAgoParser(TimeAgoParser.DEFAULT_AGO_PHRASES);
}
/*//////////////////////////////////////////////////////////////////////////
// Utils
//////////////////////////////////////////////////////////////////////////*/
/**
* Figures out where the link is pointing to (a channel, a video, a playlist, etc.)
@ -322,4 +276,95 @@ public abstract class StreamingService {
return LinkType.NONE;
}
}
/*//////////////////////////////////////////////////////////////////////////
// Localization
//////////////////////////////////////////////////////////////////////////*/
/**
* Returns a list of localizations that this service supports.
*/
public List<Localization> getSupportedLocalizations() {
return Collections.singletonList(Localization.DEFAULT);
}
/**
* Returns a list of countries that this service supports.<br>
*/
public List<ContentCountry> getSupportedCountries() {
return Collections.singletonList(ContentCountry.DEFAULT);
}
/**
* Returns the localization that should be used in this service. It will get which localization
* the user prefer (using {@link NewPipe#getPreferredLocalization()}), then it will:
* <ul>
* <li>Check if the exactly localization is supported by this service.</li>
* <li>If not, check if a less specific localization is available, using only the language code.</li>
* <li>Fallback to the {@link Localization#DEFAULT default} localization.</li>
* </ul>
*/
public Localization getLocalization() {
final Localization preferredLocalization = NewPipe.getPreferredLocalization();
// Check the localization's language and country
if (getSupportedLocalizations().contains(preferredLocalization)) {
return preferredLocalization;
}
// Fallback to the first supported language that matches the preferred language
for (Localization supportedLanguage : getSupportedLocalizations()) {
if (supportedLanguage.getLanguageCode().equals(preferredLocalization.getLanguageCode())) {
return supportedLanguage;
}
}
return Localization.DEFAULT;
}
/**
* Returns the country that should be used to fetch content in this service. It will get which country
* the user prefer (using {@link NewPipe#getPreferredContentCountry()}), then it will:
* <ul>
* <li>Check if the country is supported by this service.</li>
* <li>If not, fallback to the {@link ContentCountry#DEFAULT default} country.</li>
* </ul>
*/
public ContentCountry getContentCountry() {
final ContentCountry preferredContentCountry = NewPipe.getPreferredContentCountry();
if (getSupportedCountries().contains(preferredContentCountry)) {
return preferredContentCountry;
}
return ContentCountry.DEFAULT;
}
/**
* Get an instance of the time ago parser using the patterns related to the passed localization.<br>
* <br>
* Just like {@link #getLocalization()}, it will also try to fallback to a less specific localization if
* the exact one is not available/supported.
*
* @throws IllegalArgumentException if the localization is not supported (parsing patterns are not present).
*/
public TimeAgoParser getTimeAgoParser(Localization localization) {
final TimeAgoParser targetParser = TimeAgoPatternsManager.getTimeAgoParserFor(localization);
if (targetParser != null) {
return targetParser;
}
if (!localization.getCountryCode().isEmpty()) {
final Localization lessSpecificLocalization = new Localization(localization.getLanguageCode());
final TimeAgoParser lessSpecificParser = TimeAgoPatternsManager.getTimeAgoParserFor(lessSpecificLocalization);
if (lessSpecificParser != null) {
return lessSpecificParser;
}
}
throw new IllegalArgumentException("Localization is not supported (\"" + localization.toString() + "\")");
}
}

48
extractor/src/main/java/org/schabi/newpipe/extractor/SuggestionExtractor.java
View File

@ -1,48 +0,0 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.utils.Localization;
import java.io.IOException;
import java.util.List;
/*
* Created by Christian Schabesberger on 28.09.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* SuggestionExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public abstract class SuggestionExtractor {
private final int serviceId;
private final Localization localization;
public SuggestionExtractor(int serviceId, Localization localization) {
this.serviceId = serviceId;
this.localization = localization;
}
public abstract List<String> suggestionList(String query) throws IOException, ExtractionException;
public int getServiceId() {
return serviceId;
}
protected Localization getLocalization() {
return localization;
}
}

7
extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java
View File

@ -3,9 +3,8 @@ package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
/*
* Created by Christian Schabesberger on 25.07.16.
@ -29,8 +28,8 @@ import org.schabi.newpipe.extractor.utils.Localization;
public abstract class ChannelExtractor extends ListExtractor<StreamInfoItem> {
public ChannelExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public ChannelExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
}
public abstract String getAvatarUrl() throws ParsingException;

4
extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java
View File

@ -6,10 +6,10 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import org.schabi.newpipe.extractor.utils.Localization;
import java.io.IOException;

9
extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java
View File

@ -3,13 +3,12 @@ package org.schabi.newpipe.extractor.comments;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.Localization;
public abstract class CommentsExtractor extends ListExtractor<CommentsInfoItem> {
public CommentsExtractor(StreamingService service, ListLinkHandler uiHandler, Localization localization) {
super(service, uiHandler, localization);
// TODO Auto-generated constructor stub
}
public CommentsExtractor(StreamingService service, ListLinkHandler uiHandler) {
super(service, uiHandler);
// TODO Auto-generated constructor stub
}
}

121
extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java
View File

@ -2,75 +2,84 @@ package org.schabi.newpipe.extractor.comments;
import org.schabi.newpipe.extractor.InfoItem;
public class CommentsInfoItem extends InfoItem{
import java.util.Calendar;
private String commentId;
private String commentText;
private String authorName;
private String authorThumbnail;
private String authorEndpoint;
private String publishedTime;
private Integer likeCount;
public CommentsInfoItem(int serviceId, String url, String name) {
super(InfoType.COMMENT, serviceId, url, name);
// TODO Auto-generated constructor stub
}
public String getCommentText() {
return commentText;
}
public class CommentsInfoItem extends InfoItem {
public void setCommentText(String contentText) {
this.commentText = contentText;
}
private String commentId;
private String commentText;
private String authorName;
private String authorThumbnail;
private String authorEndpoint;
private String textualPublishedTime;
private Calendar publishedTime;
private int likeCount;
public String getAuthorName() {
return authorName;
}
public CommentsInfoItem(int serviceId, String url, String name) {
super(InfoType.COMMENT, serviceId, url, name);
}
public void setAuthorName(String authorName) {
this.authorName = authorName;
}
public String getCommentId() {
return commentId;
}
public String getAuthorThumbnail() {
return authorThumbnail;
}
public void setCommentId(String commentId) {
this.commentId = commentId;
}
public void setAuthorThumbnail(String authorThumbnail) {
this.authorThumbnail = authorThumbnail;
}
public String getCommentText() {
return commentText;
}
public String getAuthorEndpoint() {
return authorEndpoint;
}
public void setCommentText(String commentText) {
this.commentText = commentText;
}
public void setAuthorEndpoint(String authorEndpoint) {
this.authorEndpoint = authorEndpoint;
}
public String getAuthorName() {
return authorName;
}
public String getPublishedTime() {
return publishedTime;
}
public void setAuthorName(String authorName) {
this.authorName = authorName;
}
public void setPublishedTime(String publishedTime) {
this.publishedTime = publishedTime;
}
public String getAuthorThumbnail() {
return authorThumbnail;
}
public Integer getLikeCount() {
return likeCount;
}
public void setAuthorThumbnail(String authorThumbnail) {
this.authorThumbnail = authorThumbnail;
}
public void setLikeCount(Integer likeCount) {
this.likeCount = likeCount;
}
public String getAuthorEndpoint() {
return authorEndpoint;
}
public String getCommentId() {
return commentId;
}
public void setAuthorEndpoint(String authorEndpoint) {
this.authorEndpoint = authorEndpoint;
}
public void setCommentId(String commentId) {
this.commentId = commentId;
}
public String getTextualPublishedTime() {
return textualPublishedTime;
}
public void setTextualPublishedTime(String textualPublishedTime) {
this.textualPublishedTime = textualPublishedTime;
}
public Calendar getPublishedTime() {
return publishedTime;
}
public void setPublishedTime(Calendar publishedTime) {
this.publishedTime = publishedTime;
}
public int getLikeCount() {
return likeCount;
}
public void setLikeCount(int likeCount) {
this.likeCount = likeCount;
}
}

7
extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java
View File

@ -3,12 +3,15 @@ package org.schabi.newpipe.extractor.comments;
import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.util.Calendar;
public interface CommentsInfoItemExtractor extends InfoItemExtractor {
String getCommentId() throws ParsingException;
String getCommentText() throws ParsingException;
String getAuthorName() throws ParsingException;
String getAuthorThumbnail() throws ParsingException;
String getAuthorEndpoint() throws ParsingException;
String getPublishedTime() throws ParsingException;
Integer getLikeCount() throws ParsingException;
String getTextualPublishedTime() throws ParsingException;
Calendar getPublishedTime() throws ParsingException;
int getLikeCount() throws ParsingException;
}

11
extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemsCollector.java
View File

@ -1,12 +1,12 @@
package org.schabi.newpipe.extractor.comments;
import java.util.List;
import java.util.Vector;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.util.List;
import java.util.Vector;
public class CommentsInfoItemsCollector extends InfoItemsCollector<CommentsInfoItem, CommentsInfoItemExtractor> {
public CommentsInfoItemsCollector(int serviceId) {
@ -49,6 +49,11 @@ public class CommentsInfoItemsCollector extends InfoItemsCollector<CommentsInfoI
} catch (Exception e) {
addError(e);
}
try {
resultItem.setTextualPublishedTime(extractor.getTextualPublishedTime());
} catch (Exception e) {
addError(e);
}
try {
resultItem.setPublishedTime(extractor.getPublishedTime());
} catch (Exception e) {

144
extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java Normal file
View File

@ -0,0 +1,144 @@
package org.schabi.newpipe.extractor.downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.Localization;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.List;
import java.util.Map;
/**
* A base for downloader implementations that NewPipe will use
* to download needed resources during extraction.
*/
public abstract class Downloader {
/**
* Do a GET request to get the resource that the url is pointing to.<br>
* <br>
* This method calls {@link #get(String, Map, Localization)} with the default preferred localization. It should only be
* used when the resource that will be fetched won't be affected by the localization.
*
* @param url the URL that is pointing to the wanted resource
* @return the result of the GET request
*/
public Response get(String url) throws IOException, ReCaptchaException {
return get(url, null, NewPipe.getPreferredLocalization());
}
/**
* Do a GET request to get the resource that the url is pointing to.<br>
* <br>
* It will set the {@code Accept-Language} header to the language of the localization parameter.
*
* @param url the URL that is pointing to the wanted resource
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the GET request
*/
public Response get(String url, @Nullable Localization localization) throws IOException, ReCaptchaException {
return get(url, null, localization);
}
/**
* Do a GET request with the specified headers.
*
* @param url the URL that is pointing to the wanted resource
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @return the result of the GET request
*/
public Response get(String url, @Nullable Map<String, List<String>> headers) throws IOException, ReCaptchaException {
return get(url, headers, NewPipe.getPreferredLocalization());
}
/**
* Do a GET request with the specified headers.<br>
* <br>
* It will set the {@code Accept-Language} header to the language of the localization parameter.
*
* @param url the URL that is pointing to the wanted resource
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the GET request
*/
public Response get(String url, @Nullable Map<String, List<String>> headers, @Nullable Localization localization)
throws IOException, ReCaptchaException {
return execute(Request.newBuilder()
.get(url)
.headers(headers)
.localization(localization)
.build());
}
/**
* Do a HEAD request.
*
* @param url the URL that is pointing to the wanted resource
* @return the result of the HEAD request
*/
public Response head(String url) throws IOException, ReCaptchaException {
return head(url, null);
}
/**
* Do a HEAD request with the specified headers.
*
* @param url the URL that is pointing to the wanted resource
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @return the result of the HEAD request
*/
public Response head(String url, @Nullable Map<String, List<String>> headers)
throws IOException, ReCaptchaException {
return execute(Request.newBuilder()
.head(url)
.headers(headers)
.build());
}
/**
* Do a POST request with the specified headers, sending the data array.
*
* @param url the URL that is pointing to the wanted resource
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @return the result of the GET request
*/
public Response post(String url, @Nullable Map<String, List<String>> headers, @Nullable byte[] dataToSend)
throws IOException, ReCaptchaException {
return post(url, headers, dataToSend, NewPipe.getPreferredLocalization());
}
/**
* Do a POST request with the specified headers, sending the data array.
* <br>
* It will set the {@code Accept-Language} header to the language of the localization parameter.
*
* @param url the URL that is pointing to the wanted resource
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the GET request
*/
public Response post(String url, @Nullable Map<String, List<String>> headers, @Nullable byte[] dataToSend, @Nullable Localization localization)
throws IOException, ReCaptchaException {
return execute(Request.newBuilder()
.post(url, dataToSend)
.headers(headers)
.localization(localization)
.build());
}
/**
* Do a request using the specified {@link Request} object.
*
* @return the result of the request
*/
public abstract Response execute(@Nonnull Request request) throws IOException, ReCaptchaException;
}

244
extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Request.java Normal file
View File

@ -0,0 +1,244 @@
package org.schabi.newpipe.extractor.downloader;
import org.schabi.newpipe.extractor.localization.Localization;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.*;
/**
* An object that holds request information used when {@link Downloader#execute(Request) executing} a request.
*/
public class Request {
private final String httpMethod;
private final String url;
private final Map<String, List<String>> headers;
@Nullable private final byte[] dataToSend;
@Nullable private final Localization localization;
public Request(String httpMethod, String url, Map<String, List<String>> headers, @Nullable byte[] dataToSend,
@Nullable Localization localization, boolean automaticLocalizationHeader) {
if (httpMethod == null) throw new IllegalArgumentException("Request's httpMethod is null");
if (url == null) throw new IllegalArgumentException("Request's url is null");
this.httpMethod = httpMethod;
this.url = url;
this.dataToSend = dataToSend;
this.localization = localization;
Map<String, List<String>> headersToSet = null;
if (headers == null) headers = Collections.emptyMap();
if (automaticLocalizationHeader && localization != null) {
headersToSet = new LinkedHashMap<>(headersFromLocalization(localization));
headersToSet.putAll(headers);
}
this.headers = Collections.unmodifiableMap(headersToSet == null ? headers : headersToSet);
}
private Request(Builder builder) {
this(builder.httpMethod, builder.url, builder.headers, builder.dataToSend,
builder.localization, builder.automaticLocalizationHeader);
}
/**
* A http method (i.e. {@code GET, POST, HEAD}).
*/
public String httpMethod() {
return httpMethod;
}
/**
* The URL that is pointing to the wanted resource.
*/
public String url() {
return url;
}
/**
* A list of headers that will be used in the request.<br>
* Any default headers that the implementation may have, <b>should</b> be overridden by these.
*/
public Map<String, List<String>> headers() {
return headers;
}
/**
* An optional byte array that will be sent when doing the request, very commonly used in
* {@code POST} requests.<br>
* <br>
* The implementation should make note of some recommended headers
* (for example, {@code Content-Length} in a post request).
*/
@Nullable
public byte[] dataToSend() {
return dataToSend;
}
/**
* A localization object that should be used when executing a request.<br>
* <br>
* Usually the {@code Accept-Language} will be set to this value (a helper
* method to do this easily: {@link Request#headersFromLocalization(Localization)}).
*/
@Nullable
public Localization localization() {
return localization;
}
public static Builder newBuilder() {
return new Builder();
}
public static final class Builder {
private String httpMethod;
private String url;
private Map<String, List<String>> headers = new LinkedHashMap<>();
private byte[] dataToSend;
private Localization localization;
private boolean automaticLocalizationHeader = true;
public Builder() {
}
/**
* A http method (i.e. {@code GET, POST, HEAD}).
*/
public Builder httpMethod(String httpMethod) {
this.httpMethod = httpMethod;
return this;
}
/**
* The URL that is pointing to the wanted resource.
*/
public Builder url(String url) {
this.url = url;
return this;
}
/**
* A list of headers that will be used in the request.<br>
* Any default headers that the implementation may have, <b>should</b> be overridden by these.
*/
public Builder headers(@Nullable Map<String, List<String>> headers) {
if (headers == null) {
this.headers.clear();
return this;
}
this.headers.clear();
this.headers.putAll(headers);
return this;
}
/**
* An optional byte array that will be sent when doing the request, very commonly used in
* {@code POST} requests.<br>
* <br>
* The implementation should make note of some recommended headers
* (for example, {@code Content-Length} in a post request).
*/
public Builder dataToSend(byte[] dataToSend) {
this.dataToSend = dataToSend;
return this;
}
/**
* A localization object that should be used when executing a request.<br>
* <br>
* Usually the {@code Accept-Language} will be set to this value (a helper
* method to do this easily: {@link Request#headersFromLocalization(Localization)}).
*/
public Builder localization(Localization localization) {
this.localization = localization;
return this;
}
/**
* If localization headers should automatically be included in the request.
*/
public Builder automaticLocalizationHeader(boolean automaticLocalizationHeader) {
this.automaticLocalizationHeader = automaticLocalizationHeader;
return this;
}
public Request build() {
return new Request(this);
}
/*//////////////////////////////////////////////////////////////////////////
// Http Methods Utils
//////////////////////////////////////////////////////////////////////////*/
public Builder get(String url) {
this.httpMethod = "GET";
this.url = url;
return this;
}
public Builder head(String url) {
this.httpMethod = "HEAD";
this.url = url;
return this;
}
public Builder post(String url, @Nullable byte[] dataToSend) {
this.httpMethod = "POST";
this.url = url;
this.dataToSend = dataToSend;
return this;
}
/*//////////////////////////////////////////////////////////////////////////
// Additional Headers Utils
//////////////////////////////////////////////////////////////////////////*/
public Builder setHeaders(String headerName, List<String> headerValueList) {
this.headers.remove(headerName);
this.headers.put(headerName, headerValueList);
return this;
}
public Builder addHeaders(String headerName, List<String> headerValueList) {
@Nullable List<String> currentHeaderValueList = this.headers.get(headerName);
if (currentHeaderValueList == null) {
currentHeaderValueList = new ArrayList<>();
}
currentHeaderValueList.addAll(headerValueList);
this.headers.put(headerName, headerValueList);
return this;
}
public Builder setHeader(String headerName, String headerValue) {
return setHeaders(headerName, Collections.singletonList(headerValue));
}
public Builder addHeader(String headerName, String headerValue) {
return addHeaders(headerName, Collections.singletonList(headerValue));
}
}
/*//////////////////////////////////////////////////////////////////////////
// Utils
//////////////////////////////////////////////////////////////////////////*/
@SuppressWarnings("WeakerAccess")
@Nonnull
public static Map<String, List<String>> headersFromLocalization(@Nullable Localization localization) {
if (localization == null) return Collections.emptyMap();
final Map<String, List<String>> headers = new LinkedHashMap<>();
if (!localization.getCountryCode().isEmpty()) {
headers.put("Accept-Language", Collections.singletonList(localization.getLocalizationCode() +
", " + localization.getLanguageCode() + ";q=0.9"));
} else {
headers.put("Accept-Language", Collections.singletonList(localization.getLanguageCode()));
}
return headers;
}
}

66
extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java Normal file
View File

@ -0,0 +1,66 @@
package org.schabi.newpipe.extractor.downloader;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.Collections;
import java.util.List;
import java.util.Map;
/**
* A Data class used to hold the results from requests made by the Downloader implementation.
*/
public class Response {
private final int responseCode;
private final String responseMessage;
private final Map<String, List<String>> responseHeaders;
private final String responseBody;
public Response(int responseCode, String responseMessage, Map<String, List<String>> responseHeaders, @Nullable String responseBody) {
this.responseCode = responseCode;
this.responseMessage = responseMessage;
this.responseHeaders = responseHeaders != null ? responseHeaders : Collections.<String, List<String>>emptyMap();
this.responseBody = responseBody == null ? "" : responseBody;
}
public int responseCode() {
return responseCode;
}
public String responseMessage() {
return responseMessage;
}
public Map<String, List<String>> responseHeaders() {
return responseHeaders;
}
@Nonnull
public String responseBody() {
return responseBody;
}
/*//////////////////////////////////////////////////////////////////////////
// Utils
//////////////////////////////////////////////////////////////////////////*/
/**
* For easy access to some header value that (usually) don't repeat itself.
* <p>For getting all the values associated to the header, use {@link #responseHeaders()} (e.g. {@code Set-Cookie}).
*
* @param name the name of the header
* @return the first value assigned to this header
*/
@Nullable
public String getHeader(String name) {
for (Map.Entry<String, List<String>> headerEntry : responseHeaders.entrySet()) {
if (headerEntry.getKey().equalsIgnoreCase(name)) {
if (headerEntry.getValue().size() > 0) {
return headerEntry.getValue().get(0);
}
}
}
return null;
}
}

8
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java
View File

@ -24,9 +24,8 @@ import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import javax.annotation.Nonnull;
@ -35,9 +34,8 @@ public abstract class KioskExtractor<T extends InfoItem> extends ListExtractor<T
public KioskExtractor(StreamingService streamingService,
ListLinkHandler linkHandler,
String kioskId,
Localization localization) {
super(streamingService, linkHandler, localization);
String kioskId) {
super(streamingService, linkHandler);
this.id = kioskId;
}

13
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java
View File

@ -20,11 +20,14 @@ package org.schabi.newpipe.extractor.kiosk;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import java.io.IOException;
@ -37,7 +40,8 @@ public class KioskInfo extends ListInfo<StreamInfoItem> {
public static ListExtractor.InfoItemsPage<StreamInfoItem> getMoreItems(StreamingService service,
String url,
String pageUrl) throws IOException, ExtractionException {
String pageUrl)
throws IOException, ExtractionException {
KioskList kl = service.getKioskList();
KioskExtractor extractor = kl.getExtractorByUrl(url, pageUrl);
return extractor.getPage(pageUrl);
@ -47,8 +51,7 @@ public class KioskInfo extends ListInfo<StreamInfoItem> {
return getInfo(NewPipe.getServiceByUrl(url), url);
}
public static KioskInfo getInfo(StreamingService service,
String url) throws IOException, ExtractionException {
public static KioskInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
KioskList kl = service.getKioskList();
KioskExtractor extractor = kl.getExtractorByUrl(url, null);
extractor.fetchPage();

36
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java
View File

@ -2,28 +2,33 @@ package org.schabi.newpipe.extractor.kiosk;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
public class KioskList {
public interface KioskExtractorFactory {
KioskExtractor createNewKiosk(final StreamingService streamingService,
final String url,
final String kioskId,
final Localization localization)
final String kioskId)
throws ExtractionException, IOException;
}
private final int service_id;
private final StreamingService service;
private final HashMap<String, KioskEntry> kioskList = new HashMap<>();
private String defaultKiosk = null;
@Nullable private Localization forcedLocalization;
@Nullable private ContentCountry forcedContentCountry;
private class KioskEntry {
public KioskEntry(KioskExtractorFactory ef, ListLinkHandlerFactory h) {
extractorFactory = ef;
@ -33,8 +38,8 @@ public class KioskList {
final ListLinkHandlerFactory handlerFactory;
}
public KioskList(int service_id) {
this.service_id = service_id;
public KioskList(StreamingService service) {
this.service = service;
}
public void addKioskEntry(KioskExtractorFactory extractorFactory, ListLinkHandlerFactory handlerFactory, String id)
@ -89,8 +94,13 @@ public class KioskList {
if(ke == null) {
throw new ExtractionException("No kiosk found with the type: " + kioskId);
} else {
return ke.extractorFactory.createNewKiosk(NewPipe.getService(service_id),
ke.handlerFactory.fromId(kioskId).getUrl(), kioskId, localization);
final KioskExtractor kioskExtractor = ke.extractorFactory.createNewKiosk(service,
ke.handlerFactory.fromId(kioskId).getUrl(), kioskId);
if (forcedLocalization != null) kioskExtractor.forceLocalization(forcedLocalization);
if (forcedContentCountry != null) kioskExtractor.forceContentCountry(forcedContentCountry);
return kioskExtractor;
}
}
@ -117,4 +127,12 @@ public class KioskList {
public ListLinkHandlerFactory getListLinkHandlerFactoryByType(String type) {
return kioskList.get(type).handlerFactory;
}
public void forceLocalization(@Nullable Localization localization) {
this.forcedLocalization = localization;
}
public void forceContentCountry(@Nullable ContentCountry contentCountry) {
this.forcedContentCountry = contentCountry;
}
}

56
extractor/src/main/java/org/schabi/newpipe/extractor/localization/ContentCountry.java Normal file
View File

@ -0,0 +1,56 @@
package org.schabi.newpipe.extractor.localization;
import javax.annotation.Nonnull;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
* Represents a country that should be used when fetching content.
* <p>
* YouTube, for example, give different results in their feed depending on which country is selected.
* </p>
*/
public class ContentCountry implements Serializable {
public static final ContentCountry DEFAULT = new ContentCountry(Localization.DEFAULT.getCountryCode());
@Nonnull private final String countryCode;
public static List<ContentCountry> listFrom(String... countryCodeList) {
final List<ContentCountry> toReturn = new ArrayList<>();
for (String countryCode : countryCodeList) {
toReturn.add(new ContentCountry(countryCode));
}
return Collections.unmodifiableList(toReturn);
}
public ContentCountry(@Nonnull String countryCode) {
this.countryCode = countryCode;
}
@Nonnull
public String getCountryCode() {
return countryCode;
}
@Override
public String toString() {
return getCountryCode();
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof ContentCountry)) return false;
ContentCountry that = (ContentCountry) o;
return countryCode.equals(that.countryCode);
}
@Override
public int hashCode() {
return countryCode.hashCode();
}
}

99
extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java Normal file
View File

@ -0,0 +1,99 @@
package org.schabi.newpipe.extractor.localization;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.Serializable;
import java.util.*;
public class Localization implements Serializable {
public static final Localization DEFAULT = new Localization("en", "GB");
@Nonnull private final String languageCode;
@Nullable private final String countryCode;
/**
* @param localizationCodeList a list of localization code, formatted like {@link #getLocalizationCode()}
*/
public static List<Localization> listFrom(String... localizationCodeList) {
final List<Localization> toReturn = new ArrayList<>();
for (String localizationCode : localizationCodeList) {
toReturn.add(fromLocalizationCode(localizationCode));
}
return Collections.unmodifiableList(toReturn);
}
/**
* @param localizationCode a localization code, formatted like {@link #getLocalizationCode()}
*/
public static Localization fromLocalizationCode(String localizationCode) {
final int indexSeparator = localizationCode.indexOf("-");
final String languageCode, countryCode;
if (indexSeparator != -1) {
languageCode = localizationCode.substring(0, indexSeparator);
countryCode = localizationCode.substring(indexSeparator + 1);
} else {
languageCode = localizationCode;
countryCode = null;
}
return new Localization(languageCode, countryCode);
}
public Localization(@Nonnull String languageCode, @Nullable String countryCode) {
this.languageCode = languageCode;
this.countryCode = countryCode;
}
public Localization(@Nonnull String languageCode) {
this(languageCode, null);
}
public String getLanguageCode() {
return languageCode;
}
@Nonnull
public String getCountryCode() {
return countryCode == null ? "" : countryCode;
}
public Locale asLocale() {
return new Locale(getLanguageCode(), getCountryCode());
}
public static Localization fromLocale(@Nonnull Locale locale) {
return new Localization(locale.getLanguage(), locale.getCountry());
}
/**
* Return a formatted string in the form of: {@code language-Country}, or
* just {@code language} if country is {@code null}.
*/
public String getLocalizationCode() {
return languageCode + (countryCode == null ? "" : "-" + countryCode);
}
@Override
public String toString() {
return "Localization[" + getLocalizationCode() + "]";
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof Localization)) return false;
Localization that = (Localization) o;
if (!languageCode.equals(that.languageCode)) return false;
return countryCode != null ? countryCode.equals(that.countryCode) : that.countryCode == null;
}
@Override
public int hashCode() {
int result = languageCode.hashCode();
result = 31 * result + (countryCode != null ? countryCode.hashCode() : 0);
return result;
}
}

98
extractor/src/main/java/org/schabi/newpipe/extractor/stream/TimeAgoParser.java → extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java
View File

@ -1,32 +1,21 @@
package org.schabi.newpipe.extractor.stream;
/*
* Created by wojcik.online on 2018-01-25.
*/
package org.schabi.newpipe.extractor.localization;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.timeago.PatternsHolder;
import org.schabi.newpipe.extractor.timeago.TimeAgoUnit;
import org.schabi.newpipe.extractor.utils.Parser;
import java.util.Calendar;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumMap;
import java.util.Map;
import java.util.regex.Pattern;
/**
* A helper class that is meant to be used by services that need to parse upload dates in the
* format '2 days ago' or similar.
*/
public class TimeAgoParser {
/**
* A set of english phrases that are contained in the time units.
* (e.g. '7 minutes ago' contains 'min')
*/
public static Map<TimeAgoUnit, Collection<String>> DEFAULT_AGO_PHRASES =
new EnumMap<>(TimeAgoUnit.class);
private final Map<TimeAgoUnit, Collection<String>> agoPhrases;
private final PatternsHolder patternsHolder;
private final Calendar consistentNow;
/**
@ -34,22 +23,35 @@ public class TimeAgoParser {
* <p>
* Instantiate a new {@link TimeAgoParser} every time you extract a new batch of items.
* </p>
* @param agoPhrases A set of phrases how to recognize the time units in a given language.
* @param patternsHolder An object that holds the "time ago" patterns, special cases, and the language word separator.
*/
public TimeAgoParser(Map<TimeAgoUnit, Collection<String>> agoPhrases) {
this.agoPhrases = agoPhrases;
public TimeAgoParser(PatternsHolder patternsHolder) {
this.patternsHolder = patternsHolder;
consistentNow = Calendar.getInstance();
}
/**
* Parses a textual date in the format '2 days ago' into a Calendar representation.
* Beginning with days ago, marks the date as approximated by setting minutes, seconds
* Beginning with weeks ago, marks the date as approximated by setting minutes, seconds
* and milliseconds to 0.
*
* @param textualDate The original date as provided by the streaming service
* @return The parsed (approximated) time
* @throws ParsingException if the time unit could not be recognized
*/
public Calendar parse(String textualDate) throws ParsingException {
for (Map.Entry<TimeAgoUnit, Map<String, Integer>> caseUnitEntry : patternsHolder.specialCases().entrySet()) {
final TimeAgoUnit timeAgoUnit = caseUnitEntry.getKey();
for (Map.Entry<String, Integer> caseMapToAmountEntry : caseUnitEntry.getValue().entrySet()) {
final String caseText = caseMapToAmountEntry.getKey();
final Integer caseAmount = caseMapToAmountEntry.getValue();
if (textualDateMatches(textualDate, caseText)) {
return getCalendar(caseAmount, timeAgoUnit);
}
}
}
int timeAgoAmount;
try {
timeAgoAmount = parseTimeAgoAmount(textualDate);
@ -69,9 +71,11 @@ public class TimeAgoParser {
}
private TimeAgoUnit parseTimeAgoUnit(String textualDate) throws ParsingException {
for (TimeAgoUnit timeAgoUnit : agoPhrases.keySet()) {
for (String agoPhrase : agoPhrases.get(timeAgoUnit)) {
if (textualDate.toLowerCase().contains(agoPhrase.toLowerCase())){
for (Map.Entry<TimeAgoUnit, Collection<String>> entry : patternsHolder.asMap().entrySet()) {
final TimeAgoUnit timeAgoUnit = entry.getKey();
for (String agoPhrase : entry.getValue()) {
if (textualDateMatches(textualDate, agoPhrase)) {
return timeAgoUnit;
}
}
@ -80,6 +84,32 @@ public class TimeAgoParser {
throw new ParsingException("Unable to parse the date: " + textualDate);
}
private boolean textualDateMatches(String textualDate, String agoPhrase) {
if (textualDate.equals(agoPhrase)) {
return true;
}
if (patternsHolder.wordSeparator().isEmpty()) {
return textualDate.toLowerCase().contains(agoPhrase.toLowerCase());
} else {
final String escapedPhrase = Pattern.quote(agoPhrase.toLowerCase());
final String escapedSeparator;
if (patternsHolder.wordSeparator().equals(" ")) {
// From JDK8 → \h - Treat horizontal spaces as a normal one (non-breaking space, thin space, etc.)
escapedSeparator = "[ \\t\\xA0\\u1680\\u180e\\u2000-\\u200a\\u202f\\u205f\\u3000]";
} else {
escapedSeparator = Pattern.quote(patternsHolder.wordSeparator());
}
// (^|separator)pattern($|separator)
// Check if the pattern is surrounded by separators or start/end of the string.
final String pattern =
"(^|" + escapedSeparator + ")" + escapedPhrase + "($|" + escapedSeparator + ")";
return Parser.isMatch(pattern, textualDate.toLowerCase());
}
}
private Calendar getCalendar(int timeAgoAmount, TimeAgoUnit timeAgoUnit) {
Calendar calendarTime = getNow();
@ -135,24 +165,4 @@ public class TimeAgoParser {
calendarTime.set(Calendar.SECOND, 0);
calendarTime.set(Calendar.MILLISECOND, 0);
}
static {
DEFAULT_AGO_PHRASES.put(TimeAgoUnit.SECONDS, Collections.singleton("sec"));
DEFAULT_AGO_PHRASES.put(TimeAgoUnit.MINUTES, Collections.singleton("min"));
DEFAULT_AGO_PHRASES.put(TimeAgoUnit.HOURS, Collections.singleton("hour"));
DEFAULT_AGO_PHRASES.put(TimeAgoUnit.DAYS, Collections.singleton("day"));
DEFAULT_AGO_PHRASES.put(TimeAgoUnit.WEEKS, Collections.singleton("week"));
DEFAULT_AGO_PHRASES.put(TimeAgoUnit.MONTHS, Collections.singleton("month"));
DEFAULT_AGO_PHRASES.put(TimeAgoUnit.YEARS, Collections.singleton("year"));
}
public enum TimeAgoUnit {
SECONDS,
MINUTES,
HOURS,
DAYS,
WEEKS,
MONTHS,
YEARS,
}
}

25
extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoPatternsManager.java Normal file
View File

@ -0,0 +1,25 @@
package org.schabi.newpipe.extractor.localization;
import org.schabi.newpipe.extractor.timeago.PatternsHolder;
import org.schabi.newpipe.extractor.timeago.PatternsManager;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
public class TimeAgoPatternsManager {
@Nullable
private static PatternsHolder getPatternsFor(@Nonnull Localization localization) {
return PatternsManager.getPatterns(localization.getLanguageCode(), localization.getCountryCode());
}
@Nullable
public static TimeAgoParser getTimeAgoParserFor(@Nonnull Localization localization) {
final PatternsHolder holder = getPatternsFor(localization);
if (holder == null) {
return null;
}
return new TimeAgoParser(holder);
}
}

5
extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java
View File

@ -5,12 +5,11 @@ import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.Localization;
public abstract class PlaylistExtractor extends ListExtractor<StreamInfoItem> {
public PlaylistExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public PlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
}
public abstract String getThumbnailUrl() throws ParsingException;

1
extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java
View File

@ -9,7 +9,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import org.schabi.newpipe.extractor.utils.Localization;
import java.io.IOException;

10
extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java
View File

@ -6,7 +6,8 @@ import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.utils.Localization;
import javax.annotation.Nonnull;
public abstract class SearchExtractor extends ListExtractor<InfoItem> {
@ -18,10 +19,8 @@ public abstract class SearchExtractor extends ListExtractor<InfoItem> {
private final InfoItemsSearchCollector collector;
public SearchExtractor(StreamingService service,
SearchQueryHandler linkHandler,
Localization localization) {
super(service, linkHandler, localization);
public SearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
super(service, linkHandler);
collector = new InfoItemsSearchCollector(service.getServiceId());
}
@ -40,6 +39,7 @@ public abstract class SearchExtractor extends ListExtractor<InfoItem> {
return (SearchQueryHandler) super.getLinkHandler();
}
@Nonnull
@Override
public String getName() {
return getLinkHandler().getSearchString();

1
extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java
View File

@ -7,7 +7,6 @@ import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import org.schabi.newpipe.extractor.utils.Localization;
import java.io.IOException;

47
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java
View File

@ -1,24 +1,12 @@
package org.schabi.newpipe.extractor.services.media_ccc;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
import java.io.IOException;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.*;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor;
@ -31,7 +19,13 @@ import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearc
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.io.IOException;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
public class MediaCCCService extends StreamingService {
public MediaCCCService(int id) {
@ -39,8 +33,8 @@ public class MediaCCCService extends StreamingService {
}
@Override
public SearchExtractor getSearchExtractor(SearchQueryHandler query, Localization localization) {
return new MediaCCCSearchExtractor(this, query, localization);
public SearchExtractor getSearchExtractor(SearchQueryHandler query) {
return new MediaCCCSearchExtractor(this, query);
}
@Override
@ -64,28 +58,28 @@ public class MediaCCCService extends StreamingService {
}
@Override
public StreamExtractor getStreamExtractor(LinkHandler linkHandler, Localization localization) {
return new MediaCCCStreamExtractor(this, linkHandler, localization);
public StreamExtractor getStreamExtractor(LinkHandler linkHandler) {
return new MediaCCCStreamExtractor(this, linkHandler);
}
@Override
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler, Localization localization) {
return new MediaCCCConferenceExtractor(this, linkHandler, localization);
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) {
return new MediaCCCConferenceExtractor(this, linkHandler);
}
@Override
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler, Localization localization) {
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) {
return null;
}
@Override
public SuggestionExtractor getSuggestionExtractor(Localization localization) {
public SuggestionExtractor getSuggestionExtractor() {
return null;
}
@Override
public KioskList getKioskList() throws ExtractionException {
KioskList list = new KioskList(getServiceId());
KioskList list = new KioskList(this);
// add kiosks here e.g.:
try {
@ -93,10 +87,9 @@ public class MediaCCCService extends StreamingService {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService,
String url,
String kioskId,
Localization localization) throws ExtractionException, IOException {
String kioskId) throws ExtractionException, IOException {
return new MediaCCCConferenceKiosk(MediaCCCService.this,
new MediaCCCConferencesListLinkHandlerFactory().fromUrl(url), kioskId, localization);
new MediaCCCConferencesListLinkHandlerFactory().fromUrl(url), kioskId);
}
}, new MediaCCCConferencesListLinkHandlerFactory(), "conferences");
list.setDefaultKiosk("conferences");
@ -118,7 +111,7 @@ public class MediaCCCService extends StreamingService {
}
@Override
public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler, Localization localization)
public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler)
throws ExtractionException {
return null;
}

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java
View File

@ -4,16 +4,15 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Localization;
import javax.annotation.Nonnull;
import java.io.IOException;
@ -22,8 +21,8 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
private JsonObject conferenceData;
public MediaCCCConferenceExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public MediaCCCConferenceExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
@ -75,7 +74,7 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
try {
conferenceData = JsonParser.object().from(downloader.download(getUrl()));
conferenceData = JsonParser.object().from(downloader.get(getUrl()).responseBody());
} catch (JsonParserException jpe) {
throw new ExtractionException("Could not parse json returnd by url: " + getUrl());
}
@ -87,6 +86,7 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
return conferenceData.getString("title");
}
@Nonnull
@Override
public String getOriginalUrl() throws ParsingException {
return "https://media.ccc.de/c/" + conferenceData.getString("acronym");

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java
View File

@ -4,16 +4,15 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCConferenceInfoItemExtractor;
import org.schabi.newpipe.extractor.utils.Localization;
import javax.annotation.Nonnull;
import java.io.IOException;
@ -24,9 +23,8 @@ public class MediaCCCConferenceKiosk extends KioskExtractor<ChannelInfoItem> {
public MediaCCCConferenceKiosk(StreamingService streamingService,
ListLinkHandler linkHandler,
String kioskId,
Localization localization) {
super(streamingService, linkHandler, kioskId, localization);
String kioskId) {
super(streamingService, linkHandler, kioskId);
}
@Nonnull
@ -53,7 +51,7 @@ public class MediaCCCConferenceKiosk extends KioskExtractor<ChannelInfoItem> {
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
String site = downloader.download(getLinkHandler().getUrl());
String site = downloader.get(getLinkHandler().getUrl(), getExtractorLocalization()).responseBody();
try {
doc = JsonParser.object().from(site);
} catch (JsonParserException jpe) {

27
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java Normal file
View File

@ -0,0 +1,27 @@
package org.schabi.newpipe.extractor.services.media_ccc.extractors;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
public class MediaCCCParsingHelper {
private MediaCCCParsingHelper() {
}
public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
Date date;
try {
date = new SimpleDateFormat("yyyy-MM-dd").parse(textualUploadDate);
} catch (ParseException e) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
}
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
}
}

18
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java
View File

@ -4,11 +4,11 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
@ -16,26 +16,24 @@ import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferencesListLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Localization;
import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.CONFERENCES;
import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.EVENTS;
import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.ALL;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.List;
import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.*;
public class MediaCCCSearchExtractor extends SearchExtractor {
private JsonObject doc;
private MediaCCCConferenceKiosk conferenceKiosk;
public MediaCCCSearchExtractor(StreamingService service, SearchQueryHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public MediaCCCSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
super(service, linkHandler);
try {
conferenceKiosk = new MediaCCCConferenceKiosk(service,
new MediaCCCConferencesListLinkHandlerFactory().fromId("conferences"),
"conferences",
localization);
"conferences");
} catch (Exception e) {
e.printStackTrace();
}
@ -88,7 +86,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
|| getLinkHandler().getContentFilters().isEmpty()) {
final String site;
final String url = getUrl();
site = downloader.download(url, getLocalization());
site = downloader.get(url, getExtractorLocalization()).responseBody();
try {
doc = JsonParser.object().from(site);
} catch (JsonParserException jpe) {

25
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java
View File

@ -4,19 +4,21 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
public class MediaCCCStreamExtractor extends StreamExtractor {
@ -24,16 +26,22 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
private JsonObject data;
private JsonObject conferenceData;
public MediaCCCStreamExtractor(StreamingService service, LinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public MediaCCCStreamExtractor(StreamingService service, LinkHandler linkHandler) {
super(service, linkHandler);
}
@Nonnull
@Override
public String getUploadDate() throws ParsingException {
public String getTextualUploadDate() throws ParsingException {
return data.getString("release_date");
}
@Nonnull
@Override
public Calendar getUploadDate() throws ParsingException {
return MediaCCCParsingHelper.parseDateFrom(getTextualUploadDate());
}
@Nonnull
@Override
public String getThumbnailUrl() throws ParsingException {
@ -200,9 +208,9 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
try {
data = JsonParser.object().from(
downloader.download(getLinkHandler().getUrl()));
downloader.get(getLinkHandler().getUrl()).responseBody());
conferenceData = JsonParser.object()
.from(downloader.download(getUploaderUrl()));
.from(downloader.get(getUploaderUrl()).responseBody());
} catch (JsonParserException jpe) {
throw new ExtractionException("Could not parse json returned by url: " + getLinkHandler().getUrl(), jpe);
}
@ -215,6 +223,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
return data.getString("title");
}
@Nonnull
@Override
public String getOriginalUrl() throws ParsingException {
return data.getString("frontend_link");

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSuggestionExtractor.java
View File

@ -1,8 +1,8 @@
package org.schabi.newpipe.extractor.services.media_ccc.extractors;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.io.IOException;
import java.util.ArrayList;
@ -10,8 +10,8 @@ import java.util.List;
public class MediaCCCSuggestionExtractor extends SuggestionExtractor {
public MediaCCCSuggestionExtractor(int serviceId, Localization localization) {
super(serviceId, localization);
public MediaCCCSuggestionExtractor(StreamingService service) {
super(service);
}
@Override

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java
View File

@ -2,9 +2,12 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import java.util.Calendar;
public class MediaCCCStreamInfoItemExtractor implements StreamInfoItemExtractor {
JsonObject event;
@ -45,10 +48,15 @@ public class MediaCCCStreamInfoItemExtractor implements StreamInfoItemExtractor
}
@Override
public String getUploadDate() throws ParsingException {
public String getTextualUploadDate() throws ParsingException {
return event.getString("release_date");
}
@Override
public Calendar getUploadDate() throws ParsingException {
return MediaCCCParsingHelper.parseDateFrom(getTextualUploadDate());
}
@Override
public String getName() throws ParsingException {
return event.getString("title");

11
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java
View File

@ -4,15 +4,14 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Localization;
import javax.annotation.Nonnull;
import java.io.IOException;
@ -25,8 +24,8 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
private StreamInfoItemsCollector streamInfoItemsCollector = null;
private String nextPageUrl = null;
public SoundcloudChannelExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public SoundcloudChannelExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
@ -36,7 +35,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
String apiUrl = "https://api-v2.soundcloud.com/users/" + userId +
"?client_id=" + SoundcloudParsingHelper.clientId();
String response = downloader.download(apiUrl);
String response = downloader.get(apiUrl, getExtractorLocalization()).responseBody();
try {
user = JsonParser.object().from(response);
} catch (JsonParserException e) {

12
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java
View File

@ -1,18 +1,15 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Localization;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
private StreamInfoItemsCollector collector = null;
@ -20,9 +17,8 @@ public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
public SoundcloudChartsExtractor(StreamingService service,
ListLinkHandler linkHandler,
String kioskId,
Localization localization) {
super(service, linkHandler, kioskId, localization);
String kioskId) {
super(service, linkHandler, kioskId);
}
@Override

54
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View File

@ -8,10 +8,10 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
@ -24,10 +24,10 @@ import java.io.IOException;
import java.net.URLEncoder;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.*;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudParsingHelper {
@ -46,23 +46,23 @@ public class SoundcloudParsingHelper {
return clientId;
}
final DownloadResponse download = dl.get("https://soundcloud.com");
String response = download.getResponseBody();
final Response download = dl.get("https://soundcloud.com");
final String responseBody = download.responseBody();
final String clientIdPattern = ",client_id:\"(.*?)\"";
Document doc = Jsoup.parse(response);
Document doc = Jsoup.parse(responseBody);
final Elements possibleScripts = doc.select("script[src*=\"sndcdn.com/assets/\"][src$=\".js\"]");
// The one containing the client id will likely be the last one
Collections.reverse(possibleScripts);
final HashMap<String, String> headers = new HashMap<>();
headers.put("Range", "bytes=0-16384");
final HashMap<String, List<String>> headers = new HashMap<>();
headers.put("Range", singletonList("bytes=0-16384"));
for (Element element : possibleScripts) {
final String srcUrl = element.attr("src");
if (srcUrl != null && !srcUrl.isEmpty()) {
try {
return clientId = Parser.matchGroup1(clientIdPattern, dl.download(srcUrl, headers));
return clientId = Parser.matchGroup1(clientIdPattern, dl.get(srcUrl, headers).responseBody());
} catch (RegexException ignored) {
// Ignore it and proceed to try searching other script
}
@ -76,23 +76,24 @@ public class SoundcloudParsingHelper {
static boolean checkIfHardcodedClientIdIsValid(Downloader dl) throws IOException, ReCaptchaException {
final String apiUrl = "https://api.soundcloud.com/connect?client_id=" + HARDCODED_CLIENT_ID;
// Should return 200 to indicate that the client id is valid, a 401 is returned otherwise.
return dl.head(apiUrl).getResponseCode() == 200;
return dl.head(apiUrl).responseCode() == 200;
}
static Date parseDate(String time) throws ParsingException {
static Calendar parseDate(String textualUploadDate) throws ParsingException {
Date date;
try {
return new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").parse(time);
date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").parse(textualUploadDate);
} catch (ParseException e1) {
try {
return new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse(time);
date = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse(textualUploadDate);
} catch (ParseException e2) {
throw new ParsingException(e1.getMessage(), e2);
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"" + ", " + e1.getMessage(), e2);
}
}
}
static String toTextualDate(String time) throws ParsingException {
return new SimpleDateFormat("yyyy-MM-dd").format(parseDate(time));
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
}
/**
@ -106,7 +107,8 @@ public class SoundcloudParsingHelper {
+ "&client_id=" + clientId();
try {
return JsonParser.object().from(downloader.download(apiUrl));
final String response = downloader.get(apiUrl, SoundCloud.getLocalization()).responseBody();
return JsonParser.object().from(response);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
@ -119,8 +121,8 @@ public class SoundcloudParsingHelper {
*/
public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(apiUrl, "UTF-8"));
String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(apiUrl, "UTF-8"), SoundCloud.getLocalization()).responseBody();
return Jsoup.parse(response).select("link[rel=\"canonical\"]").first().attr("abs:href");
}
@ -132,8 +134,8 @@ public class SoundcloudParsingHelper {
*/
public static String resolveIdWithEmbedPlayer(String url) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(url, "UTF-8"));
String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(url, "UTF-8"), SoundCloud.getLocalization()).responseBody();
// handle playlists / sets different and get playlist id via uir field in JSON
if (url.contains("sets") && !url.endsWith("sets") && !url.endsWith("sets/"))
return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response);
@ -164,7 +166,7 @@ public class SoundcloudParsingHelper {
* @return the next streams url, empty if don't have
*/
public static String getUsersFromApi(ChannelInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().download(apiUrl);
String response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization()).responseBody();
JsonObject responseObject;
try {
responseObject = JsonParser.object().from(response);
@ -215,7 +217,7 @@ public class SoundcloudParsingHelper {
* @return the next streams url, empty if don't have
*/
public static String getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().download(apiUrl);
String response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization()).responseBody();
JsonObject responseObject;
try {
responseObject = JsonParser.object().from(response);

11
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java
View File

@ -3,15 +3,14 @@ package org.schabi.newpipe.extractor.services.soundcloud;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Localization;
import javax.annotation.Nonnull;
import java.io.IOException;
@ -24,8 +23,8 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
private StreamInfoItemsCollector streamInfoItemsCollector = null;
private String nextPageUrl = null;
public SoundcloudPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public SoundcloudPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
@ -36,7 +35,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
"?client_id=" + SoundcloudParsingHelper.clientId() +
"&representation=compact";
String response = downloader.download(apiUrl);
String response = downloader.get(apiUrl, getExtractorLocalization()).responseBody();
try {
playlist = JsonParser.object().from(response);
} catch (JsonParserException e) {

16
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java
View File

@ -5,12 +5,12 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull;
@ -25,10 +25,8 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
private JsonArray searchCollection;
public SoundcloudSearchExtractor(StreamingService service,
SearchQueryHandler linkHandler,
Localization localization) {
super(service, linkHandler, localization);
public SoundcloudSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
super(service, linkHandler);
}
@Override
@ -51,7 +49,8 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
public InfoItemsPage<InfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
final Downloader dl = getDownloader();
try {
searchCollection = JsonParser.object().from(dl.download(pageUrl)).getArray("collection");
final String response = dl.get(pageUrl, getExtractorLocalization()).responseBody();
searchCollection = JsonParser.object().from(response).getArray("collection");
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
@ -64,7 +63,8 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
final Downloader dl = getDownloader();
final String url = getUrl();
try {
searchCollection = JsonParser.object().from(dl.download(url)).getArray("collection");
final String response = dl.get(url, getExtractorLocalization()).responseBody();
searchCollection = JsonParser.object().from(response).getArray("collection");
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}

53
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java
View File

@ -1,26 +1,19 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.*;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.utils.Localization;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
public class SoundcloudService extends StreamingService {
@ -28,11 +21,6 @@ public class SoundcloudService extends StreamingService {
super(id, "SoundCloud", singletonList(AUDIO));
}
@Override
public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler, Localization localization) {
return new SoundcloudSearchExtractor(this, queryHandler, localization);
}
@Override
public SearchQueryHandlerFactory getSearchQHFactory() {
return new SoundcloudSearchQueryHandlerFactory();
@ -55,23 +43,28 @@ public class SoundcloudService extends StreamingService {
@Override
public StreamExtractor getStreamExtractor(LinkHandler LinkHandler, Localization localization) {
return new SoundcloudStreamExtractor(this, LinkHandler, localization);
public StreamExtractor getStreamExtractor(LinkHandler LinkHandler) {
return new SoundcloudStreamExtractor(this, LinkHandler);
}
@Override
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler, Localization localization) {
return new SoundcloudChannelExtractor(this, linkHandler, localization);
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) {
return new SoundcloudChannelExtractor(this, linkHandler);
}
@Override
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler, Localization localization) {
return new SoundcloudPlaylistExtractor(this, linkHandler, localization);
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) {
return new SoundcloudPlaylistExtractor(this, linkHandler);
}
@Override
public SuggestionExtractor getSuggestionExtractor(Localization localization) {
return new SoundcloudSuggestionExtractor(getServiceId(), localization);
public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) {
return new SoundcloudSearchExtractor(this, queryHandler);
}
@Override
public SoundcloudSuggestionExtractor getSuggestionExtractor() {
return new SoundcloudSuggestionExtractor(this);
}
@Override
@ -80,15 +73,14 @@ public class SoundcloudService extends StreamingService {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService,
String url,
String id,
Localization local)
String id)
throws ExtractionException {
return new SoundcloudChartsExtractor(SoundcloudService.this,
new SoundcloudChartsLinkHandlerFactory().fromUrl(url), id, local);
new SoundcloudChartsLinkHandlerFactory().fromUrl(url), id);
}
};
KioskList list = new KioskList(getServiceId());
KioskList list = new KioskList(this);
// add kiosks here e.g.:
final SoundcloudChartsLinkHandlerFactory h = new SoundcloudChartsLinkHandlerFactory();
@ -103,7 +95,6 @@ public class SoundcloudService extends StreamingService {
return list;
}
@Override
public SubscriptionExtractor getSubscriptionExtractor() {
return new SoundcloudSubscriptionExtractor(this);
@ -115,9 +106,9 @@ public class SoundcloudService extends StreamingService {
}
@Override
public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler, Localization localization)
public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler)
throws ExtractionException {
return null;
}
}

20
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java
View File

@ -4,26 +4,28 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.utils.Localization;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.List;
public class SoundcloudStreamExtractor extends StreamExtractor {
private JsonObject track;
public SoundcloudStreamExtractor(StreamingService service, LinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public SoundcloudStreamExtractor(StreamingService service, LinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
@ -50,8 +52,14 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getUploadDate() throws ParsingException {
return SoundcloudParsingHelper.toTextualDate(track.getString("created_at"));
public String getTextualUploadDate() {
return track.getString("created_at");
}
@Nonnull
@Override
public Calendar getUploadDate() throws ParsingException {
return SoundcloudParsingHelper.parseDate(getTextualUploadDate());
}
@Nonnull
@ -139,7 +147,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
String apiUrl = "https://api.soundcloud.com/i1/tracks/" + urlEncode(getId()) + "/streams"
+ "?client_id=" + urlEncode(SoundcloudParsingHelper.clientId());
String response = dl.download(apiUrl);
String response = dl.get(apiUrl, getExtractorLocalization()).responseBody();
JsonObject responseObject;
try {
responseObject = JsonParser.object().from(response);

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java
View File

@ -43,15 +43,13 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
}
@Override
public String getTextualUploadDate() throws ParsingException {
return SoundcloudParsingHelper.toTextualDate(getCreatedAt());
public String getTextualUploadDate() {
return itemObject.getString("created_at");
}
@Override
public Calendar getUploadDate() throws ParsingException {
Calendar uploadTime = Calendar.getInstance();
uploadTime.setTime(SoundcloudParsingHelper.parseDate(getCreatedAt()));
return uploadTime;
return SoundcloudParsingHelper.parseDate(getTextualUploadDate());
}
private String getCreatedAt() {

12
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java
View File

@ -4,12 +4,12 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.io.IOException;
import java.net.URLEncoder;
@ -20,8 +20,8 @@ public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
public static final String CHARSET_UTF_8 = "UTF-8";
public SoundcloudSuggestionExtractor(int serviceId, Localization localization) {
super(serviceId, localization);
public SoundcloudSuggestionExtractor(StreamingService service) {
super(service);
}
@Override
@ -35,7 +35,7 @@ public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=10";
String response = dl.download(url);
String response = dl.get(url, getExtractorLocalization()).responseBody();
try {
JsonArray collection = JsonParser.object().from(response).getArray("collection");
for (Object suggestion : collection) {

119
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View File

@ -1,43 +1,26 @@
package org.schabi.newpipe.extractor.services.youtube;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.*;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscriptionExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSuggestionExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.extractors.*;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.*;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.util.List;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.*;
/*
* Created by Christian Schabesberger on 23.08.15.
@ -65,11 +48,6 @@ public class YoutubeService extends StreamingService {
super(id, "YouTube", asList(AUDIO, VIDEO, LIVE, COMMENTS));
}
@Override
public SearchExtractor getSearchExtractor(SearchQueryHandler query, Localization localization) {
return new YoutubeSearchExtractor(this, query, localization);
}
@Override
public LinkHandlerFactory getStreamLHFactory() {
return YoutubeStreamLinkHandlerFactory.getInstance();
@ -91,28 +69,33 @@ public class YoutubeService extends StreamingService {
}
@Override
public StreamExtractor getStreamExtractor(LinkHandler linkHandler, Localization localization) {
return new YoutubeStreamExtractor(this, linkHandler, localization);
public StreamExtractor getStreamExtractor(LinkHandler linkHandler) {
return new YoutubeStreamExtractor(this, linkHandler);
}
@Override
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler, Localization localization) {
return new YoutubeChannelExtractor(this, linkHandler, localization);
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) {
return new YoutubeChannelExtractor(this, linkHandler);
}
@Override
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler, Localization localization) {
return new YoutubePlaylistExtractor(this, linkHandler, localization);
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) {
return new YoutubePlaylistExtractor(this, linkHandler);
}
@Override
public SuggestionExtractor getSuggestionExtractor(Localization localization) {
return new YoutubeSuggestionExtractor(getServiceId(), localization);
public SearchExtractor getSearchExtractor(SearchQueryHandler query) {
return new YoutubeSearchExtractor(this, query);
}
@Override
public SuggestionExtractor getSuggestionExtractor() {
return new YoutubeSuggestionExtractor(this);
}
@Override
public KioskList getKioskList() throws ExtractionException {
KioskList list = new KioskList(getServiceId());
KioskList list = new KioskList(this);
// add kiosks here e.g.:
try {
@ -120,11 +103,10 @@ public class YoutubeService extends StreamingService {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService,
String url,
String id,
Localization local)
String id)
throws ExtractionException {
return new YoutubeTrendingExtractor(YoutubeService.this,
new YoutubeTrendingLinkHandlerFactory().fromUrl(url), id, local);
new YoutubeTrendingLinkHandlerFactory().fromUrl(url), id);
}
}, new YoutubeTrendingLinkHandlerFactory(), "Trending");
list.setDefaultKiosk("Trending");
@ -146,9 +128,52 @@ public class YoutubeService extends StreamingService {
}
@Override
public CommentsExtractor getCommentsExtractor(ListLinkHandler urlIdHandler, Localization localization)
public CommentsExtractor getCommentsExtractor(ListLinkHandler urlIdHandler)
throws ExtractionException {
return new YoutubeCommentsExtractor(this, urlIdHandler, localization);
return new YoutubeCommentsExtractor(this, urlIdHandler);
}
/*//////////////////////////////////////////////////////////////////////////
// Localization
//////////////////////////////////////////////////////////////////////////*/
// https://www.youtube.com/picker_ajax?action_language_json=1
private static final List<Localization> SUPPORTED_LANGUAGES = Localization.listFrom(
"af", "am", "ar", "az", "be", "bg", "bn", "bs", "ca", "cs", "da", "de",
"el", "en", "en-GB", "es", "es-419", "es-US", "et", "eu", "fa", "fi", "fil", "fr",
"fr-CA", "gl", "gu", "hi", "hr", "hu", "hy", "id", "is", "it", "iw", "ja",
"ka", "kk", "km", "kn", "ko", "ky", "lo", "lt", "lv", "mk", "ml", "mn",
"mr", "ms", "my", "ne", "nl", "no", "pa", "pl", "pt", "pt-PT", "ro", "ru",
"si", "sk", "sl", "sq", "sr", "sr-Latn", "sv", "sw", "ta", "te", "th", "tr",
"uk", "ur", "uz", "vi", "zh-CN", "zh-HK", "zh-TW", "zu"
);
// https://www.youtube.com/picker_ajax?action_country_json=1
private static final List<ContentCountry> SUPPORTED_COUNTRIES = ContentCountry.listFrom(
"AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", "BA",
"BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV",
"BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU",
"CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES",
"ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM",
"GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", "ID", "IE",
"IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM",
"KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY",
"MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT",
"MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU",
"NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA",
"RE", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM",
"SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL",
"TM", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE",
"VG", "VI", "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW"
);
@Override
public List<Localization> getSupportedLocalizations() {
return SUPPORTED_LANGUAGES;
}
@Override
public List<ContentCountry> getSupportedCountries() {
return SUPPORTED_COUNTRIES;
}
}

24
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -1,32 +1,27 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.TimeAgoParser;
import org.schabi.newpipe.extractor.utils.DonationLinkHelper;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.ArrayList;
/*
* Created by Christian Schabesberger on 25.07.16.
@ -54,18 +49,16 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000&gl=US&hl=en";
private final TimeAgoParser timeAgoParser = getService().getTimeAgoParser();
private Document doc;
public YoutubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public YoutubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
String channelUrl = super.getUrl() + CHANNEL_URL_PARAMETERS;
final DownloadResponse response = downloader.get(channelUrl);
final Response response = downloader.get(channelUrl, getExtractorLocalization());
doc = YoutubeParsingHelper.parseAndCheckPage(channelUrl, response);
}
@ -191,7 +184,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
JsonObject ajaxJson;
try {
ajaxJson = JsonParser.object().from(NewPipe.getDownloader().download(pageUrl));
final String response = getDownloader().get(pageUrl, getExtractorLocalization()).responseBody();
ajaxJson = JsonParser.object().from(response);
} catch (JsonParserException pe) {
throw new ParsingException("Could not parse json data for next streams", pe);
}
@ -231,6 +225,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final String uploaderName = getName();
final String uploaderUrl = getUrl();
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (final Element li : element.children()) {
if (li.select("div[class=\"feed-item-dismissable\"]").first() != null) {
collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) {

61
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java
View File

@ -1,36 +1,31 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import javax.annotation.Nonnull;
import org.schabi.newpipe.extractor.DownloadRequest;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemsCollector;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Parser;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.*;
import java.util.regex.Pattern;
import static java.util.Collections.singletonList;
public class YoutubeCommentsExtractor extends CommentsExtractor {
@ -44,8 +39,8 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
private String title;
private InfoItemsPage<CommentsInfoItem> initPage;
public YoutubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler, Localization localization) {
super(service, uiHandler, localization);
public YoutubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) {
super(service, uiHandler);
}
@Override
@ -130,7 +125,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
for(Object c: comments) {
if(c instanceof JsonObject) {
CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor((JsonObject) c, getUrl());
CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor((JsonObject) c, getUrl(), getTimeAgoParser());
collector.commit(extractor);
}
}
@ -147,12 +142,11 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
}
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
Map<String, List<String>> requestHeaders = new HashMap<>();
requestHeaders.put("User-Agent", Arrays.asList(USER_AGENT));
DownloadRequest request = new DownloadRequest(null, requestHeaders);
DownloadResponse response = downloader.get(getUrl(), request);
String responseBody = response.getResponseBody();
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final Map<String, List<String>> requestHeaders = new HashMap<>();
requestHeaders.put("User-Agent", singletonList(USER_AGENT));
final Response response = downloader.get(getUrl(), requestHeaders, getExtractorLocalization());
String responseBody = response.responseBody();
ytClientVersion = findValue(responseBody, "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"", "\"");
ytClientName = Parser.matchGroup1(YT_CLIENT_NAME_PATTERN, responseBody);
String commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}");
@ -160,6 +154,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
initPage = getPage(getNextPageUrl(commentsToken));
}
@Nonnull
@Override
public String getName() throws ParsingException {
return title;
@ -168,13 +163,11 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
private String makeAjaxRequest(String siteUrl) throws IOException, ReCaptchaException {
Map<String, List<String>> requestHeaders = new HashMap<>();
requestHeaders.put("Accept", Arrays.asList("*/*"));
requestHeaders.put("User-Agent", Arrays.asList(USER_AGENT));
requestHeaders.put("X-YouTube-Client-Version", Arrays.asList(ytClientVersion));
requestHeaders.put("X-YouTube-Client-Name", Arrays.asList(ytClientName));
DownloadRequest request = new DownloadRequest(null, requestHeaders);
return NewPipe.getDownloader().get(siteUrl, request).getResponseBody();
requestHeaders.put("Accept", singletonList("*/*"));
requestHeaders.put("User-Agent", singletonList(USER_AGENT));
requestHeaders.put("X-YouTube-Client-Version", singletonList(ytClientVersion));
requestHeaders.put("X-YouTube-Client-Name", singletonList(ytClientName));
return getDownloader().get(siteUrl, requestHeaders, getExtractorLocalization()).responseBody();
}
private String getDataString(Map<String, String> params) throws UnsupportedEncodingException {

30
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java
View File

@ -1,21 +1,25 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import java.util.Calendar;
public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
private final JsonObject json;
private final String url;
private final TimeAgoParser timeAgoParser;
public YoutubeCommentsInfoItemExtractor(JsonObject json, String url) {
public YoutubeCommentsInfoItemExtractor(JsonObject json, String url, TimeAgoParser timeAgoParser) {
this.json = json;
this.url = url;
this.timeAgoParser = timeAgoParser;
}
@Override
@ -43,7 +47,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
}
@Override
public String getPublishedTime() throws ParsingException {
public String getTextualPublishedTime() throws ParsingException {
try {
return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "publishedTimeText"));
} catch (Exception e) {
@ -52,7 +56,17 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
}
@Override
public Integer getLikeCount() throws ParsingException {
public Calendar getPublishedTime() throws ParsingException {
String textualPublishedTime = getTextualPublishedTime();
if (timeAgoParser != null && textualPublishedTime != null && !textualPublishedTime.isEmpty()) {
return timeAgoParser.parse(textualPublishedTime);
} else {
return null;
}
}
@Override
public int getLikeCount() throws ParsingException {
try {
return JsonUtils.getNumber(json, "likeCount").intValue();
} catch (Exception e) {

20
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -6,20 +6,19 @@ import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.TimeAgoParser;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
@ -29,18 +28,16 @@ import java.io.IOException;
@SuppressWarnings("WeakerAccess")
public class YoutubePlaylistExtractor extends PlaylistExtractor {
private final TimeAgoParser timeAgoParser = getService().getTimeAgoParser();
private Document doc;
public YoutubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public YoutubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String url = getUrl();
final DownloadResponse response = downloader.get(url);
final Response response = downloader.get(url, getExtractorLocalization());
doc = YoutubeParsingHelper.parseAndCheckPage(url, response);
}
@ -144,7 +141,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
JsonObject pageJson;
try {
pageJson = JsonParser.object().from(getDownloader().download(pageUrl));
final String responseBody = getDownloader().get(pageUrl, getExtractorLocalization()).responseBody();
pageJson = JsonParser.object().from(responseBody);
} catch (JsonParserException pe) {
throw new ParsingException("Could not parse ajax json", pe);
}
@ -190,6 +188,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
final LinkHandlerFactory streamLinkHandlerFactory = getService().getStreamLHFactory();
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (final Element li : element.children()) {
if(isDeletedItem(li)) {
continue;

25
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -3,17 +3,16 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.Parser;
@ -47,22 +46,21 @@ public class YoutubeSearchExtractor extends SearchExtractor {
private Document doc;
public YoutubeSearchExtractor(StreamingService service,
SearchQueryHandler linkHandler,
Localization localization) {
super(service, linkHandler, localization);
public YoutubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String url = getUrl();
final DownloadResponse response = downloader.get(url, getLocalization());
final Response response = downloader.get(url, getExtractorLocalization());
doc = YoutubeParsingHelper.parseAndCheckPage(url, response);
}
@Nonnull
@Override
public String getUrl() throws ParsingException {
return super.getUrl() + "&gl="+ getLocalization().getCountry();
return super.getUrl() + "&gl=" + getExtractorContentCountry().getCountryCode();
}
@Override
@ -88,8 +86,8 @@ public class YoutubeSearchExtractor extends SearchExtractor {
@Override
public InfoItemsPage<InfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
String site = getDownloader().download(pageUrl);
doc = Jsoup.parse(site, pageUrl);
final String response = getDownloader().get(pageUrl, getExtractorLocalization()).responseBody();
doc = Jsoup.parse(response, pageUrl);
return new InfoItemsPage<>(collectItems(doc), getNextPageUrlFromCurrentUrl(pageUrl));
}
@ -110,6 +108,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
InfoItemsSearchCollector collector = getInfoItemSearchCollector();
Element list = doc.select("ol[class=\"item-section\"]").first();
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Element item : list.children()) {
/* First we need to determine which kind of item we are working with.
@ -130,7 +129,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
// video item type
} else if ((el = item.select("div[class*=\"yt-lockup-video\"]").first()) != null) {
collector.commit(new YoutubeStreamInfoItemExtractor(el, getService().getTimeAgoParser()));
collector.commit(new YoutubeStreamInfoItemExtractor(el, timeAgoParser));
} else if ((el = item.select("div[class*=\"yt-lockup-channel\"]").first()) != null) {
collector.commit(new YoutubeChannelInfoItemExtractor(el));
} else if ((el = item.select("div[class*=\"yt-lockup-playlist\"]").first()) != null &&

54
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -11,16 +11,22 @@ import org.jsoup.select.Elements;
import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Request;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
@ -75,8 +81,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
/*//////////////////////////////////////////////////////////////////////////*/
private final TimeAgoParser timeAgoParser = getService().getTimeAgoParser();
private Document doc;
@Nullable
private JsonObject playerArgs;
@ -89,8 +93,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private boolean isAgeRestricted;
public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) {
super(service, linkHandler);
}
/*//////////////////////////////////////////////////////////////////////////
@ -116,10 +120,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return name;
}
@Nonnull
@Override
public String getUploadDate() throws ParsingException {
assertPageFetched();
public String getTextualUploadDate() throws ParsingException {
if (getStreamType().equals(StreamType.LIVE_STREAM)) {
return null;
}
try {
return doc.select("meta[itemprop=datePublished]").attr(CONTENT);
} catch (Exception e) {//todo: add fallback method
@ -127,6 +133,17 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
@Override
public Calendar getUploadDate() throws ParsingException {
final String textualUploadDate = getTextualUploadDate();
if (textualUploadDate == null) {
return null;
}
return YoutubeParsingHelper.parseDateFrom(textualUploadDate);
}
@Nonnull
@Override
public String getThumbnailUrl() throws ParsingException {
@ -534,13 +551,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
assertPageFetched();
try {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final TimeAgoParser timeAgoParser = getTimeAgoParser();
Elements watch = doc.select("div[class=\"watch-sidebar-section\"]");
if (watch.size() < 1) {
return null;// prevent the snackbar notification "report error" on age-restricted videos
}
collector.commit(extractVideoPreviewInfo(watch.first().select("li").first()));
collector.commit(extractVideoPreviewInfo(watch.first().select("li").first(), timeAgoParser));
return collector.getItems().get(0);
} catch (Exception e) {
throw new ParsingException("Could not get next video", e);
@ -552,12 +570,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
assertPageFetched();
try {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final TimeAgoParser timeAgoParser = getTimeAgoParser();
Element ul = doc.select("ul[id=\"watch-related\"]").first();
if (ul != null) {
for (Element li : ul.children()) {
// first check if we have a playlist. If so leave them out
if (li.select("a[class*=\"content-link\"]").first() != null) {
collector.commit(extractVideoPreviewInfo(li));
collector.commit(extractVideoPreviewInfo(li, timeAgoParser));
}
}
}
@ -617,8 +637,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String verifiedUrl = getUrl() + VERIFIED_URL_PARAMS;
final DownloadResponse response = downloader.get(verifiedUrl);
pageHtml = response.getResponseBody();
final Response response = downloader.get(verifiedUrl, getExtractorLocalization());
pageHtml = response.responseBody();
doc = YoutubeParsingHelper.parseAndCheckPage(verifiedUrl, response);
final String playerUrl;
@ -626,7 +646,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (!doc.select("meta[property=\"og:restrictions:age\"").isEmpty()) {
final EmbeddedInfo info = getEmbeddedInfo();
final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts);
final String infoPageResponse = downloader.download(videoInfoUrl);
final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody();
videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
playerUrl = info.url;
isAgeRestricted = true;
@ -715,7 +735,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try {
final Downloader downloader = NewPipe.getDownloader();
final String embedUrl = "https://www.youtube.com/embed/" + getId();
final String embedPageContent = downloader.download(embedUrl);
final String embedPageContent = downloader.get(embedUrl, getExtractorLocalization()).responseBody();
// Get player url
final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")";
@ -750,7 +770,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
playerUrl = "https://youtube.com" + playerUrl;
}
final String playerCode = downloader.download(playerUrl);
final String playerCode = downloader.get(playerUrl, getExtractorLocalization()).responseBody();
final String decryptionFunctionName = getDecryptionFuncName(playerCode);
final String functionPattern = "("
@ -933,7 +953,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
* Provides information about links to other videos on the video page, such as related videos.
* This is encapsulated in a StreamInfoItem object, which is a subset of the fields in a full StreamInfo.
*/
private StreamInfoItemExtractor extractVideoPreviewInfo(final Element li) {
private StreamInfoItemExtractor extractVideoPreviewInfo(final Element li, final TimeAgoParser timeAgoParser) {
return new YoutubeStreamInfoItemExtractor(li, timeAgoParser) {
@Override

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java
View File

@ -6,7 +6,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.TimeAgoParser;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable;

14
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java
View File

@ -3,12 +3,12 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.io.IOException;
import java.net.URLEncoder;
@ -39,8 +39,8 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
public static final String CHARSET_UTF_8 = "UTF-8";
public YoutubeSuggestionExtractor(int serviceId, Localization localization) {
super(serviceId, localization);
public YoutubeSuggestionExtractor(StreamingService service) {
super(service);
}
@Override
@ -52,10 +52,10 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
+ "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml
+ "&jsonp=" + "JP"
+ "&ds=" + "yt"
+ "&hl=" + URLEncoder.encode(getLocalization().getCountry(), CHARSET_UTF_8)
+ "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), CHARSET_UTF_8)
+ "&q=" + URLEncoder.encode(query, CHARSET_UTF_8);
String response = dl.download(url);
String response = dl.get(url, getExtractorLocalization()).responseBody();
// trim JSONP part "JP(...)"
response = response.substring(3, response.length()-1);
try {

29
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View File

@ -20,48 +20,40 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.stream.TimeAgoParser;
import javax.annotation.Nonnull;
import java.io.IOException;
public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
private final TimeAgoParser timeAgoParser = getService().getTimeAgoParser();
private Document doc;
public YoutubeTrendingExtractor(StreamingService service,
ListLinkHandler linkHandler,
String kioskId,
Localization localization) {
super(service, linkHandler, kioskId, localization);
String kioskId) {
super(service, linkHandler, kioskId);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String contentCountry = getLocalization().getCountry();
String url = getUrl();
if(contentCountry != null && !contentCountry.isEmpty()) {
url += "?gl=" + contentCountry;
}
final String url = getUrl() +
"?gl=" + getExtractorContentCountry().getCountryCode();
final DownloadResponse response = downloader.get(url);
final Response response = downloader.get(url, getExtractorLocalization());
doc = YoutubeParsingHelper.parseAndCheckPage(url, response);
}
@ -93,6 +85,9 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Elements uls = doc.select("ul[class*=\"expanded-shelf-content-list\"]");
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for(Element ul : uls) {
for(final Element li : ul.children()) {
final Element el = li.select("div[class*=\"yt-lockup-dismissable\"]").first();

23
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java
View File

@ -3,11 +3,15 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import java.net.URL;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
/*
* Created by Christian Schabesberger on 02.03.16.
@ -39,8 +43,8 @@ public class YoutubeParsingHelper {
"input[name*=\"action_recaptcha_verify\"]"
};
public static Document parseAndCheckPage(final String url, final DownloadResponse response) throws ReCaptchaException {
final Document document = Jsoup.parse(response.getResponseBody(), url);
public static Document parseAndCheckPage(final String url, final Response response) throws ReCaptchaException {
final Document document = Jsoup.parse(response.responseBody(), url);
for (String detectionSelector : RECAPTCHA_DETECTION_SELECTORS) {
if (!document.select(detectionSelector).isEmpty()) {
@ -113,4 +117,17 @@ public class YoutubeParsingHelper {
+ Long.parseLong(minutes)) * 60)
+ Long.parseLong(seconds);
}
public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
Date date;
try {
date = new SimpleDateFormat("yyyy-MM-dd").parse(textualUploadDate);
} catch (ParseException e) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
}
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
}
}

36
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java
View File

@ -26,13 +26,13 @@ import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.List;
@ -43,18 +43,36 @@ public abstract class StreamExtractor extends Extractor {
public static final int NO_AGE_LIMIT = 0;
public StreamExtractor(StreamingService service, LinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public StreamExtractor(StreamingService service, LinkHandler linkHandler) {
super(service, linkHandler);
}
/**
* The day on which the stream got uploaded/created. The return information should be in the format
* dd.mm.yyyy, however it NewPipe will not crash if its not.
* @return The day on which the stream got uploaded.
* @throws ParsingException
* The original textual date provided by the service. Should be used as a fallback if
* {@link #getUploadDate()} isn't provided by the service, or it fails for some reason.
*
* <p>If the stream is a live stream, {@code null} should be returned.</p>
*
* @return The original textual date provided by the service, or {@code null}.
* @throws ParsingException if there is an error in the extraction
* @see #getUploadDate()
*/
@Nonnull
public abstract String getUploadDate() throws ParsingException;
@Nullable
public abstract String getTextualUploadDate() throws ParsingException;
/**
* A more general {@code Calendar} instance set to the date provided by the service.<br>
* Implementations usually will just parse the date returned from the {@link #getTextualUploadDate()}.
*
* <p>If the stream is a live stream, {@code null} should be returned.</p>
*
* @return The date this item was uploaded, or {@code null}.
* @throws ParsingException if there is an error in the extraction
* or the extracted date couldn't be parsed.
* @see #getTextualUploadDate()
*/
@Nullable
public abstract Calendar getUploadDate() throws ParsingException;
/**
* This will return the url to the thumbnail of the stream. Try to return the medium resolution here.

21
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java
View File

@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.stream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
import org.schabi.newpipe.extractor.Info;
@ -228,6 +229,11 @@ public class StreamInfo extends Info {
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setTextualUploadDate(extractor.getTextualUploadDate());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setUploadDate(extractor.getUploadDate());
} catch (Exception e) {
@ -271,7 +277,8 @@ public class StreamInfo extends Info {
private StreamType streamType;
private String thumbnailUrl = "";
private String uploadDate = "";
private String textualUploadDate;
private Calendar uploadDate;
private long duration = -1;
private int ageLimit = -1;
private String description;
@ -327,11 +334,19 @@ public class StreamInfo extends Info {
this.thumbnailUrl = thumbnailUrl;
}
public String getUploadDate() {
public String getTextualUploadDate() {
return textualUploadDate;
}
public void setTextualUploadDate(String textualUploadDate) {
this.textualUploadDate = textualUploadDate;
}
public Calendar getUploadDate() {
return uploadDate;
}
public void setUploadDate(String uploadDate) {
public void setUploadDate(Calendar uploadDate) {
this.uploadDate = uploadDate;
}

7
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java
View File

@ -22,6 +22,7 @@ package org.schabi.newpipe.extractor.stream;
import org.schabi.newpipe.extractor.InfoItem;
import javax.annotation.Nullable;
import java.util.Calendar;
/**
@ -83,18 +84,20 @@ public class StreamInfoItem extends InfoItem {
* @return The original textual upload date as returned by the streaming service.
* @see #getUploadDate()
*/
@Nullable
public String getTextualUploadDate() {
return textualUploadDate;
}
public void setTextualUploadDate(String upload_date) {
this.textualUploadDate = upload_date;
public void setTextualUploadDate(String uploadDate) {
this.textualUploadDate = uploadDate;
}
/**
* @return The (approximated) date and time this item was uploaded or {@code null}.
* @see #getTextualUploadDate()
*/
@Nullable
public Calendar getUploadDate() {
return uploadDate;
}

7
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java
View File

@ -66,11 +66,10 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor {
String getUploaderUrl() throws ParsingException;
/**
* Extract the textual upload date of this item.
* The original textual date provided by the service may be used if it is short;
* otherwise the format "yyyy-MM-dd" or an locale specific version is preferred.
* The original textual date provided by the service. Should be used as a fallback if
* {@link #getUploadDate()} isn't provided by the service, or it fails for some reason.
*
* @return The original textual upload date.
* @return The original textual date provided by the service.
* @throws ParsingException if there is an error in the extraction
* @see #getUploadDate()
*/

51
extractor/src/main/java/org/schabi/newpipe/extractor/suggestion/SuggestionExtractor.java Normal file
View File

@ -0,0 +1,51 @@
package org.schabi.newpipe.extractor.suggestion;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.List;
public abstract class SuggestionExtractor {
private final StreamingService service;
@Nullable private Localization forcedLocalization;
@Nullable private ContentCountry forcedContentCountry;
public SuggestionExtractor(StreamingService service) {
this.service = service;
}
public abstract List<String> suggestionList(String query) throws IOException, ExtractionException;
public int getServiceId() {
return service.getServiceId();
}
public StreamingService getService() {
return service;
}
// TODO: Create a more general Extractor class
public void forceLocalization(@Nullable Localization localization) {
this.forcedLocalization = localization;
}
public void forceContentCountry(@Nullable ContentCountry contentCountry) {
this.forcedContentCountry = contentCountry;
}
@Nonnull
public Localization getExtractorLocalization() {
return forcedLocalization == null ? getService().getLocalization() : forcedLocalization;
}
@Nonnull
public ContentCountry getExtractorContentCountry() {
return forcedContentCountry == null ? getService().getContentCountry() : forcedContentCountry;
}
}

4
extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java
View File

@ -1,6 +1,6 @@
package org.schabi.newpipe.extractor.utils;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -120,7 +120,7 @@ public class DashMpdParser {
String dashDoc;
Downloader downloader = NewPipe.getDownloader();
try {
dashDoc = downloader.download(streamInfo.getDashMpdUrl());
dashDoc = downloader.get(streamInfo.getDashMpdUrl()).responseBody();
} catch (IOException ioe) {
throw new DashMpdParsingException("Could not get dash mpd: " + streamInfo.getDashMpdUrl(), ioe);
}

19
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Localization.java
View File

@ -1,19 +0,0 @@
package org.schabi.newpipe.extractor.utils;
public class Localization {
private final String country;
private final String language;
public Localization(String country, String language) {
this.country = country;
this.language = language;
}
public String getCountry() {
return country;
}
public String getLanguage() {
return language;
}
}

258
extractor/src/test/java/org/schabi/newpipe/Downloader.java
View File

@ -1,258 +0,0 @@
package org.schabi.newpipe;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.net.ssl.HttpsURLConnection;
import org.schabi.newpipe.extractor.DownloadRequest;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.utils.Localization;
import static java.util.Collections.singletonList;
/*
* Created by Christian Schabesberger on 28.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* Downloader.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class Downloader implements org.schabi.newpipe.extractor.Downloader {
private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0";
private static final String DEFAULT_HTTP_ACCEPT_LANGUAGE = "en";
private static String mCookies = "";
private static Downloader instance = null;
private Downloader() {
}
public static Downloader getInstance() {
if (instance == null) {
synchronized (Downloader.class) {
if (instance == null) {
instance = new Downloader();
}
}
}
return instance;
}
public static synchronized void setCookies(String cookies) {
Downloader.mCookies = cookies;
}
public static synchronized String getCookies() {
return Downloader.mCookies;
}
/**
* Download the text file at the supplied URL as in download(String), but set
* the HTTP header field "Accept-Language" to the supplied string.
*
* @param siteUrl the URL of the text file to return the contents of
* @param localization the language and country (usually a 2-character code for both values)
* @return the contents of the specified text file
*/
public String download(String siteUrl, Localization localization) throws IOException, ReCaptchaException {
Map<String, String> requestProperties = new HashMap<>();
requestProperties.put("Accept-Language", localization.getLanguage());
return download(siteUrl, requestProperties);
}
/**
* Download the text file at the supplied URL as in download(String), but set
* the HTTP header field "Accept-Language" to the supplied string.
*
* @param siteUrl the URL of the text file to return the contents of
* @param customProperties set request header properties
* @return the contents of the specified text file
* @throws IOException
*/
public String download(String siteUrl, Map<String, String> customProperties)
throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
for (Map.Entry<String, String> pair : customProperties.entrySet()) {
con.setRequestProperty(pair.getKey(), pair.getValue());
}
return dl(con);
}
/**
* Common functionality between download(String url) and download(String url,
* String language)
*/
private static String dl(HttpsURLConnection con) throws IOException, ReCaptchaException {
StringBuilder response = new StringBuilder();
BufferedReader in = null;
try {
con.setRequestMethod("GET");
setDefaults(con);
in = new BufferedReader(new InputStreamReader(con.getInputStream()));
String inputLine;
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
} catch (UnknownHostException uhe) {// thrown when there's no internet
// connection
throw new IOException("unknown host or no network", uhe);
// Toast.makeText(getActivity(), uhe.getMessage(),
// Toast.LENGTH_LONG).show();
} catch (Exception e) {
/*
* HTTP 429 == Too Many Request Receive from Youtube.com = ReCaptcha challenge
* request See : https://github.com/rg3/youtube-dl/issues/5138
*/
if (con.getResponseCode() == 429) {
throw new ReCaptchaException("reCaptcha Challenge requested", con.getURL().toString());
}
throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e);
} finally {
if (in != null) {
in.close();
}
}
return response.toString();
}
private static void setDefaults(HttpsURLConnection con) {
con.setConnectTimeout(30 * 1000);// 30s
con.setReadTimeout(30 * 1000);// 30s
// set default user agent
if (null == con.getRequestProperty("User-Agent")) {
con.setRequestProperty("User-Agent", USER_AGENT);
}
// add default cookies
if (getCookies().length() > 0) {
con.addRequestProperty("Cookie", getCookies());
}
}
/**
* Download (via HTTP) the text file located at the supplied URL, and return its
* contents. Primarily intended for downloading web pages.
*
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file
*/
public String download(String siteUrl) throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
// HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
con.setRequestProperty("Accept-Language", DEFAULT_HTTP_ACCEPT_LANGUAGE);
return dl(con);
}
@Override
public DownloadResponse head(String siteUrl) throws IOException, ReCaptchaException {
final HttpsURLConnection con = (HttpsURLConnection) new URL(siteUrl).openConnection();
try {
con.setRequestMethod("HEAD");
setDefaults(con);
} catch (Exception e) {
/*
* HTTP 429 == Too Many Request Receive from Youtube.com = ReCaptcha challenge
* request See : https://github.com/rg3/youtube-dl/issues/5138
*/
if (con.getResponseCode() == 429) {
throw new ReCaptchaException("reCaptcha Challenge requested", con.getURL().toString());
}
throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e);
}
return new DownloadResponse(con.getResponseCode(), null, con.getHeaderFields());
}
@Override
public DownloadResponse get(String siteUrl, Localization localization) throws IOException, ReCaptchaException {
final Map<String, List<String>> requestHeaders = new HashMap<>();
requestHeaders.put("Accept-Language", singletonList(localization.getLanguage()));
return get(siteUrl, new DownloadRequest(null, requestHeaders));
}
@Override
public DownloadResponse get(String siteUrl, DownloadRequest request)
throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
for (Map.Entry<String, List<String>> pair : request.getRequestHeaders().entrySet()) {
for(String value: pair.getValue()) {
con.addRequestProperty(pair.getKey(), value);
}
}
String responseBody = dl(con);
return new DownloadResponse(con.getResponseCode(), responseBody, con.getHeaderFields());
}
@Override
public DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException {
return get(siteUrl, DownloadRequest.emptyRequest);
}
@Override
public DownloadResponse post(String siteUrl, DownloadRequest request)
throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
con.setRequestMethod("POST");
for (Map.Entry<String, List<String>> pair : request.getRequestHeaders().entrySet()) {
for(String value: pair.getValue()) {
con.addRequestProperty(pair.getKey(), value);
}
}
// set fields to default if not set already
setDefaults(con);
if(null != request.getRequestBody()) {
byte[] postDataBytes = request.getRequestBody().getBytes("UTF-8");
con.setRequestProperty("Content-Length", String.valueOf(postDataBytes.length));
con.setDoOutput(true);
con.getOutputStream().write(postDataBytes);
}
StringBuilder sb = new StringBuilder();
try (BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()))) {
String inputLine;
while ((inputLine = in.readLine()) != null) {
sb.append(inputLine);
}
}
return new DownloadResponse(con.getResponseCode(), sb.toString(), con.getHeaderFields());
}
}

120
extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java Normal file
View File

@ -0,0 +1,120 @@
package org.schabi.newpipe;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Request;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.Localization;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.net.ssl.HttpsURLConnection;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.List;
import java.util.Map;
public class DownloaderTestImpl extends Downloader {
private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0";
private static final String DEFAULT_HTTP_ACCEPT_LANGUAGE = "en";
private static DownloaderTestImpl instance = null;
private DownloaderTestImpl() {
}
public static DownloaderTestImpl getInstance() {
if (instance == null) {
synchronized (DownloaderTestImpl.class) {
if (instance == null) {
instance = new DownloaderTestImpl();
}
}
}
return instance;
}
private void setDefaultHeaders(URLConnection connection) {
connection.setRequestProperty("User-Agent", USER_AGENT);
connection.setRequestProperty("Accept-Language", DEFAULT_HTTP_ACCEPT_LANGUAGE);
}
@Override
public Response execute(@Nonnull Request request) throws IOException, ReCaptchaException {
final String httpMethod = request.httpMethod();
final String url = request.url();
final Map<String, List<String>> headers = request.headers();
@Nullable final byte[] dataToSend = request.dataToSend();
@Nullable final Localization localization = request.localization();
final HttpsURLConnection connection = (HttpsURLConnection) new URL(url).openConnection();
connection.setConnectTimeout(30 * 1000); // 30s
connection.setReadTimeout(30 * 1000); // 30s
connection.setRequestMethod(httpMethod);
setDefaultHeaders(connection);
for (Map.Entry<String, List<String>> pair : headers.entrySet()) {
final String headerName = pair.getKey();
final List<String> headerValueList = pair.getValue();
if (headerValueList.size() > 1) {
connection.setRequestProperty(headerName, null);
for (String headerValue : headerValueList) {
connection.addRequestProperty(headerName, headerValue);
}
} else if (headerValueList.size() == 1) {
connection.setRequestProperty(headerName, headerValueList.get(0));
}
}
@Nullable OutputStream outputStream = null;
@Nullable InputStreamReader input = null;
try {
if (dataToSend != null && dataToSend.length > 0) {
connection.setDoOutput(true);
connection.setRequestProperty("Content-Length", dataToSend.length + "");
outputStream = connection.getOutputStream();
outputStream.write(dataToSend);
}
final InputStream inputStream = connection.getInputStream();
final StringBuilder response = new StringBuilder();
// Not passing any charset for decoding here... something to keep in mind.
input = new InputStreamReader(inputStream);
int readCount;
char[] buffer = new char[32 * 1024];
while ((readCount = input.read(buffer)) != -1) {
response.append(buffer, 0, readCount);
}
final int responseCode = connection.getResponseCode();
final String responseMessage = connection.getResponseMessage();
final Map<String, List<String>> responseHeaders = connection.getHeaderFields();
return new Response(responseCode, responseMessage, responseHeaders, response.toString());
} catch (Exception e) {
/*
* HTTP 429 == Too Many Request
* Receive from Youtube.com = ReCaptcha challenge request
* See : https://github.com/rg3/youtube-dl/issues/5138
*/
if (connection.getResponseCode() == 429) {
throw new ReCaptchaException("reCaptcha Challenge requested", url);
}
throw new IOException(connection.getResponseCode() + " " + connection.getResponseMessage(), e);
} finally {
if (outputStream != null) outputStream.close();
if (input != null) input.close();
}
}
}

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java
View File

@ -2,11 +2,10 @@ package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor;
import org.schabi.newpipe.extractor.utils.Localization;
import static junit.framework.TestCase.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
@ -19,7 +18,7 @@ public class MediaCCCConferenceExtractorTest {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("en", "en_GB"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getChannelExtractor("https://api.media.ccc.de/public/conferences/froscon2017");
extractor.fetchPage();
}

9
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java
View File

@ -1,18 +1,17 @@
package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceKiosk;
import org.schabi.newpipe.extractor.utils.Localization;
import java.util.List;
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
/**
@ -24,7 +23,7 @@ public class MediaCCCConferenceListExtractorTest {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("en", "en_GB"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getKioskList().getDefaultKioskExtractor();
extractor.fetchPage();
}

6
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java
View File

@ -2,13 +2,11 @@ package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.utils.Localization;
import static junit.framework.TestCase.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
@ -22,7 +20,7 @@ public class MediaCCCOggTest {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/1317");
extractor.fetchPage();

8
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java
View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
@ -11,7 +11,6 @@ import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.Localization;
import java.util.Arrays;
@ -28,10 +27,9 @@ public class MediaCCCSearchExtractorAllTest {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory()
.fromQuery("c3", Arrays.asList(new String[0]), "")
,new Localization("GB", "en"));
.fromQuery("c3", Arrays.asList(new String[0]), ""));
extractor.fetchPage();
itemsPage = extractor.getInitialPage();
}

8
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java
View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
@ -10,7 +10,6 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.utils.Localization;
import java.util.Arrays;
@ -27,10 +26,9 @@ public class MediaCCCSearchExtractorConferencesTest {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory()
.fromQuery("c3", Arrays.asList(new String[] {"conferences"}), "")
,new Localization("GB", "en"));
.fromQuery("c3", Arrays.asList(new String[]{"conferences"}), ""));
extractor.fetchPage();
itemsPage = extractor.getInitialPage();
}

8
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java
View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
@ -10,7 +10,6 @@ import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.Localization;
import java.util.Arrays;
@ -28,10 +27,9 @@ public class MediaCCCSearchExtractorEventsTest {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory()
.fromQuery("linux", Arrays.asList(new String[] {"events"}), "")
,new Localization("GB", "en"));
.fromQuery("linux", Arrays.asList(new String[]{"events"}), ""));
extractor.fetchPage();
itemsPage = extractor.getInitialPage();
}

25
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java
View File

@ -1,13 +1,18 @@
package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseExtractorTest;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import static junit.framework.TestCase.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
@ -20,7 +25,7 @@ public class MediaCCCStreamExtractorTest implements BaseExtractorTest {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/8afc16c2-d76a-53f6-85e4-90494665835d");
extractor.fetchPage();
@ -80,4 +85,16 @@ public class MediaCCCStreamExtractorTest implements BaseExtractorTest {
public void testAudioStreams() throws Exception {
assertEquals(2, extractor.getAudioStreams().size());
}
@Test
public void testGetTextualUploadDate() throws ParsingException {
Assert.assertEquals("2018-05-11", extractor.getTextualUploadDate());
}
@Test
public void testGetUploadDate() throws ParsingException, ParseException {
final Calendar instance = Calendar.getInstance();
instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2018-05-11"));
Assert.assertEquals(instance, extractor.getUploadDate());
}
}

7
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java
View File

@ -2,12 +2,11 @@ package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.utils.Localization;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty;
@ -24,7 +23,7 @@ public class SoundcloudChannelExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (SoundcloudChannelExtractor) SoundCloud
.getChannelExtractor("http://soundcloud.com/liluzivert/sets");
extractor.fetchPage();
@ -108,7 +107,7 @@ public class SoundcloudChannelExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (SoundcloudChannelExtractor) SoundCloud
.getChannelExtractor("https://soundcloud.com/dubmatix");
extractor.fetchPage();

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java
View File

@ -3,12 +3,11 @@ package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.Localization;
import java.util.List;
@ -24,7 +23,7 @@ public class SoundcloudChartsExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = SoundCloud
.getKioskList()
.getExtractorById("Top 50", null);

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsLinkHandlerFactoryTest.java
View File

@ -2,10 +2,9 @@ package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Localization;
import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.assertEquals;
@ -20,7 +19,7 @@ public class SoundcloudChartsLinkHandlerFactoryTest {
@BeforeClass
public static void setUp() {
linkHandler = new SoundcloudChartsLinkHandlerFactory();
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test

7
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java
View File

@ -1,22 +1,21 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.*;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.utils.Localization;
import static org.junit.Assert.*;
public class SoundcloudParsingHelperTest {
@BeforeClass
public static void setUp() {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test
public void assertThatHardcodedClientIdIsValid() throws Exception {
assertTrue("Hardcoded client id is not valid anymore",
SoundcloudParsingHelper.checkIfHardcodedClientIdIsValid(Downloader.getInstance()));
SoundcloudParsingHelper.checkIfHardcodedClientIdIsValid(DownloaderTestImpl.getInstance()));
}
@Test

9
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java
View File

@ -4,14 +4,13 @@ import org.hamcrest.CoreMatchers;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.Localization;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
@ -27,7 +26,7 @@ public class SoundcloudPlaylistExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (SoundcloudPlaylistExtractor) SoundCloud
.getPlaylistExtractor("https://soundcloud.com/liluzivert/sets/the-perfect-luv-tape-r?test=123");
extractor.fetchPage();
@ -125,7 +124,7 @@ public class SoundcloudPlaylistExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (SoundcloudPlaylistExtractor) SoundCloud
.getPlaylistExtractor("https://soundcloud.com/micky96/sets/house");
extractor.fetchPage();
@ -217,7 +216,7 @@ public class SoundcloudPlaylistExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (SoundcloudPlaylistExtractor) SoundCloud
.getPlaylistExtractor("https://soundcloud.com/user350509423/sets/edm-xxx");
extractor.fetchPage();

20
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java
View File

@ -1,17 +1,20 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Localization;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
@ -25,7 +28,7 @@ public class SoundcloudStreamExtractorDefaultTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (SoundcloudStreamExtractor) SoundCloud.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon");
extractor.fetchPage();
}
@ -69,8 +72,15 @@ public class SoundcloudStreamExtractorDefaultTest {
}
@Test
public void testGetUploadDate() throws ParsingException {
assertEquals("2016-07-31", extractor.getUploadDate());
public void testGetTextualUploadDate() throws ParsingException {
Assert.assertEquals("2016/07/31 18:18:07 +0000", extractor.getTextualUploadDate());
}
@Test
public void testGetUploadDate() throws ParsingException, ParseException {
final Calendar instance = Calendar.getInstance();
instance.setTime(new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse("2016/07/31 18:18:07 +0000"));
Assert.assertEquals(instance, extractor.getUploadDate());
}
@Test

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactoryTest.java
View File

@ -2,10 +2,9 @@ package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Localization;
import java.util.ArrayList;
import java.util.List;
@ -21,7 +20,7 @@ public class SoundcloudStreamLinkHandlerFactoryTest {
@BeforeClass
public static void setUp() throws Exception {
linkHandler = SoundcloudStreamLinkHandlerFactory.getInstance();
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test(expected = IllegalArgumentException.class)

7
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java
View File

@ -2,14 +2,13 @@ package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
import org.schabi.newpipe.extractor.utils.Localization;
import java.io.IOException;
import java.util.Arrays;
@ -26,7 +25,7 @@ public class SoundcloudSubscriptionExtractorTest {
@BeforeClass
public static void setupClass() {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
subscriptionExtractor = new SoundcloudSubscriptionExtractor(ServiceList.SoundCloud);
urlHandler = ServiceList.SoundCloud.getChannelLHFactory();
}

7
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractorTest.java
View File

@ -2,11 +2,10 @@ package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.io.IOException;
@ -21,7 +20,7 @@ public class SoundcloudSuggestionExtractorTest {
@BeforeClass
public static void setUp() {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
suggestionExtractor = SoundCloud.getSuggestionExtractor();
}

10
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorChannelOnlyTest.java
View File

@ -2,14 +2,14 @@ package org.schabi.newpipe.extractor.services.soundcloud.search;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.utils.Localization;
import static java.util.Arrays.asList;
import static org.junit.Assert.*;
@ -19,9 +19,9 @@ public class SoundcloudSearchExtractorChannelOnlyTest extends SoundcloudSearchEx
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("DE", "de"));
NewPipe.init(DownloaderTestImpl.getInstance(), new Localization("de", "DE"));
extractor = (SoundcloudSearchExtractor) SoundCloud.getSearchExtractor("lill uzi vert",
asList(SoundcloudSearchQueryHandlerFactory.USERS), null, new Localization("DE", "de"));
asList(SoundcloudSearchQueryHandlerFactory.USERS), null);
extractor.fetchPage();
itemsPage = extractor.getInitialPage();
}
@ -29,7 +29,7 @@ public class SoundcloudSearchExtractorChannelOnlyTest extends SoundcloudSearchEx
@Test
public void testGetSecondPage() throws Exception {
SoundcloudSearchExtractor secondExtractor = (SoundcloudSearchExtractor) SoundCloud.getSearchExtractor("lill uzi vert",
asList(SoundcloudSearchQueryHandlerFactory.USERS), null, new Localization("DE", "de"));
asList(SoundcloudSearchQueryHandlerFactory.USERS), null);
ListExtractor.InfoItemsPage<InfoItem> secondPage = secondExtractor.getPage(itemsPage.getNextPageUrl());
assertTrue(Integer.toString(secondPage.getItems().size()),
secondPage.getItems().size() >= 3);

10
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java
View File

@ -2,22 +2,19 @@ package org.schabi.newpipe.extractor.services.soundcloud.search;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.Localization;
import java.util.Arrays;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
/*
* Created by Christian Schabesberger on 27.05.18
@ -46,11 +43,10 @@ public class SoundcloudSearchExtractorDefaultTest extends SoundcloudSearchExtrac
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (SoundcloudSearchExtractor) SoundCloud.getSearchExtractor(
new SoundcloudSearchQueryHandlerFactory().fromQuery("lill uzi vert",
Arrays.asList(new String[]{"tracks"}), ""),
new Localization("GB", "en"));
Arrays.asList(new String[]{"tracks"}), ""));
extractor.fetchPage();
itemsPage = extractor.getInitialPage();
}

9
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchQHTest.java
View File

@ -2,22 +2,19 @@ package org.schabi.newpipe.extractor.services.soundcloud.search;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.utils.Localization;
import static java.util.Arrays.asList;
import static org.junit.Assert.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory.PLAYLISTS;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory.TRACKS;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory.USERS;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory.*;
public class SoundcloudSearchQHTest {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
}
private static String removeClientId(String url) {

16
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java
View File

@ -2,14 +2,14 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
import org.schabi.newpipe.extractor.utils.Localization;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
@ -25,7 +25,7 @@ public class YoutubeChannelExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("http://www.youtube.com/user/Gronkh");
extractor.fetchPage();
@ -115,7 +115,7 @@ public class YoutubeChannelExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("https://www.youtube.com/user/Vsauce");
extractor.fetchPage();
@ -206,7 +206,7 @@ public class YoutubeChannelExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q");
extractor.fetchPage();
@ -308,7 +308,7 @@ public class YoutubeChannelExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("https://www.youtube.com/user/CaptainDisillusion/videos");
extractor.fetchPage();
@ -398,7 +398,7 @@ public class YoutubeChannelExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("https://www.youtube.com/user/EminemVEVO/");
extractor.fetchPage();
@ -491,7 +491,7 @@ public class YoutubeChannelExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w");
extractor.fetchPage();

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java
View File

@ -2,11 +2,10 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Localization;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@ -21,7 +20,7 @@ public class YoutubeChannelLinkHandlerFactoryTest {
@BeforeClass
public static void setUp() {
linkHandler = YoutubeChannelLinkHandlerFactory.getInstance();
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test

139
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java Normal file
View File

@ -0,0 +1,139 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import java.text.SimpleDateFormat;
import java.util.*;
import static org.junit.Assert.fail;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
/**
* A class that tests multiple channels and ranges of "time ago".
*/
@Ignore("Should be ran manually from time to time, as it's too time consuming.")
public class YoutubeChannelLocalizationTest {
private static final boolean DEBUG = true;
private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm");
@Test
public void testAllSupportedLocalizations() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
testLocalizationsFor("https://www.youtube.com/user/NBCNews");
testLocalizationsFor("https://www.youtube.com/channel/UCcmpeVbSSQlZRvHfdC-CRwg/videos");
testLocalizationsFor("https://www.youtube.com/channel/UC65afEgL62PGFWXY7n6CUbA");
testLocalizationsFor("https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg");
}
private void testLocalizationsFor(String channelUrl) throws Exception {
final List<Localization> supportedLocalizations = YouTube.getSupportedLocalizations();
// final List<Localization> supportedLocalizations = Arrays.asList(Localization.DEFAULT, new Localization("sr"));
final Map<Localization, List<StreamInfoItem>> results = new LinkedHashMap<>();
for (Localization currentLocalization : supportedLocalizations) {
if (DEBUG) System.out.println("Testing localization = " + currentLocalization);
ListExtractor.InfoItemsPage<StreamInfoItem> itemsPage;
try {
final ChannelExtractor extractor = YouTube.getChannelExtractor(channelUrl);
extractor.forceLocalization(currentLocalization);
extractor.fetchPage();
itemsPage = defaultTestRelatedItems(extractor, YouTube.getServiceId());
} catch (Throwable e) {
System.out.println("[!] " + currentLocalization + " → failed");
throw e;
}
final List<StreamInfoItem> items = itemsPage.getItems();
for (int i = 0; i < items.size(); i++) {
final StreamInfoItem item = items.get(i);
String debugMessage = "[" + String.format("%02d", i) + "] "
+ currentLocalization.getLocalizationCode() + " → " + item.getName()
+ "\n:::: " + item.getStreamType() + ", views = " + item.getViewCount();
final Calendar uploadDate = item.getUploadDate();
if (uploadDate != null) {
String dateAsText = dateFormat.format(uploadDate.getTime());
debugMessage += "\n:::: " + item.getTextualUploadDate() +
"\n:::: " + dateAsText;
}
if (DEBUG) System.out.println(debugMessage + "\n");
}
results.put(currentLocalization, itemsPage.getItems());
if (DEBUG) System.out.println("\n===============================\n");
}
// Check results
final List<StreamInfoItem> referenceList = results.get(Localization.DEFAULT);
boolean someFail = false;
for (Map.Entry<Localization, List<StreamInfoItem>> currentResultEntry : results.entrySet()) {
if (currentResultEntry.getKey().equals(Localization.DEFAULT)) {
continue;
}
final String currentLocalizationCode = currentResultEntry.getKey().getLocalizationCode();
final String referenceLocalizationCode = Localization.DEFAULT.getLocalizationCode();
if (DEBUG) {
System.out.println("Comparing " + referenceLocalizationCode + " with " +
currentLocalizationCode);
}
final List<StreamInfoItem> currentList = currentResultEntry.getValue();
if (referenceList.size() != currentList.size()) {
if (DEBUG) System.out.println("[!] " + currentLocalizationCode + " → Lists are not equal");
someFail = true;
continue;
}
for (int i = 0; i < referenceList.size() - 1; i++) {
final StreamInfoItem referenceItem = referenceList.get(i);
final StreamInfoItem currentItem = currentList.get(i);
final Calendar referenceUploadDate = referenceItem.getUploadDate();
final Calendar currentUploadDate = currentItem.getUploadDate();
final String referenceDateString = referenceUploadDate == null ? "null" :
dateFormat.format(referenceUploadDate.getTime());
final String currentDateString = currentUploadDate == null ? "null" :
dateFormat.format(currentUploadDate.getTime());
long difference = -1;
if (referenceUploadDate != null && currentUploadDate != null) {
difference = Math.abs(referenceUploadDate.getTimeInMillis() - currentUploadDate.getTimeInMillis());
}
final boolean areTimeEquals = difference < 5 * 60 * 1000L;
if (!areTimeEquals) {
System.out.println("" +
" [!] " + currentLocalizationCode + " → [" + i + "] dates are not equal\n" +
" " + referenceLocalizationCode + ": " +
referenceDateString + " → " + referenceItem.getTextualUploadDate() +
"\n " + currentLocalizationCode + ": " +
currentDateString + " → " + currentItem.getTextualUploadDate());
}
}
}
if (someFail) {
fail("Some localization failed");
} else {
if (DEBUG) System.out.print("All tests passed" +
"\n\n===============================\n\n");
}
}
}

27
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java
View File

@ -1,23 +1,23 @@
package org.schabi.newpipe.extractor.services.youtube;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import java.io.IOException;
import java.util.List;
import org.jsoup.helper.StringUtil;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.comments.CommentsInfo;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.services.DefaultTests;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
import org.schabi.newpipe.extractor.utils.Localization;
import java.io.IOException;
import java.util.List;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class YoutubeCommentsExtractorTest {
@ -25,7 +25,7 @@ public class YoutubeCommentsExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor("https://www.youtube.com/watch?v=D00Au7k3i6o");
}
@ -64,6 +64,8 @@ public class YoutubeCommentsExtractorTest {
@Test
public void testGetCommentsAllData() throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
DefaultTests.defaultTestListOfItems(YouTube.getServiceId(), comments.getItems(), comments.getErrors());
for(CommentsInfoItem c: comments.getItems()) {
assertFalse(StringUtil.isBlank(c.getAuthorEndpoint()));
assertFalse(StringUtil.isBlank(c.getAuthorName()));
@ -71,10 +73,11 @@ public class YoutubeCommentsExtractorTest {
assertFalse(StringUtil.isBlank(c.getCommentId()));
assertFalse(StringUtil.isBlank(c.getCommentText()));
assertFalse(StringUtil.isBlank(c.getName()));
assertFalse(StringUtil.isBlank(c.getPublishedTime()));
assertFalse(StringUtil.isBlank(c.getTextualPublishedTime()));
assertNotNull(c.getPublishedTime());
assertFalse(StringUtil.isBlank(c.getThumbnailUrl()));
assertFalse(StringUtil.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() == null);
assertFalse(c.getLikeCount() < 0);
}
}

7
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View File

@ -3,7 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
@ -12,7 +12,6 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.Localization;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@ -29,7 +28,7 @@ public class YoutubePlaylistExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubePlaylistExtractor) YouTube
.getPlaylistExtractor("http://www.youtube.com/watch?v=lp-EO5I60KA&list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj");
extractor.fetchPage();
@ -126,7 +125,7 @@ public class YoutubePlaylistExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubePlaylistExtractor) YouTube
.getPlaylistExtractor("https://www.youtube.com/watch?v=8SbUC-UaAxE&list=PLWwAypAcFRgKAIIFqBr9oy-ZYZnixa_Fj");
extractor.fetchPage();

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java
View File

@ -2,11 +2,10 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Localization;
import static org.junit.Assert.*;
@ -18,8 +17,8 @@ public class YoutubePlaylistLinkHandlerFactoryTest {
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
linkHandler = YoutubePlaylistLinkHandlerFactory.getInstance();
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
}
@Test(expected = IllegalArgumentException.class)

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeServiceTest.java
View File

@ -22,11 +22,10 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.utils.Localization;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@ -41,7 +40,7 @@ public class YoutubeServiceTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
service = YouTube;
kioskList = service.getKioskList();
}

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java
View File

@ -2,12 +2,11 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Localization;
import java.util.ArrayList;
import java.util.List;
@ -24,7 +23,7 @@ public class YoutubeStreamLinkHandlerFactoryTest {
@BeforeClass
public static void setUp() {
linkHandler = YoutubeStreamLinkHandlerFactory.getInstance();
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test(expected = IllegalArgumentException.class)

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java
View File

@ -2,14 +2,13 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscriptionExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
import org.schabi.newpipe.extractor.utils.Localization;
import java.io.ByteArrayInputStream;
import java.io.File;
@ -28,7 +27,7 @@ public class YoutubeSubscriptionExtractorTest {
@BeforeClass
public static void setupClass() {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
subscriptionExtractor = new YoutubeSubscriptionExtractor(ServiceList.YouTube);
urlHandler = ServiceList.YouTube.getChannelLHFactory();
}

8
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractorTest.java
View File

@ -22,11 +22,11 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.io.IOException;
@ -41,7 +41,7 @@ public class YoutubeSuggestionExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("DE", "de"));
NewPipe.init(DownloaderTestImpl.getInstance(), new Localization("de", "DE"));
suggestionExtractor = YouTube.getSuggestionExtractor();
}

7
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java
View File

@ -22,13 +22,13 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Utils;
import static junit.framework.TestCase.assertFalse;
@ -46,10 +46,11 @@ public class YoutubeTrendingExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeTrendingExtractor) YouTube
.getKioskList()
.getExtractorById("Trending", null);
extractor.forceContentCountry(new ContentCountry("de"));
extractor.fetchPage();
}

7
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java
View File

@ -22,12 +22,11 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.kiosk.KioskInfo;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@ -42,7 +41,7 @@ public class YoutubeTrendingKioskInfoTest {
@BeforeClass
public static void setUp()
throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
StreamingService service = YouTube;
LinkHandlerFactory LinkHandlerFactory = service.getKioskList().getListLinkHandlerFactoryByType("Trending");

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingLinkHandlerFactoryTest.java
View File

@ -22,12 +22,11 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Localization;
import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.assertEquals;
@ -43,7 +42,7 @@ public class YoutubeTrendingLinkHandlerFactoryTest {
@BeforeClass
public static void setUp() throws Exception {
LinkHandlerFactory = YouTube.getKioskList().getListLinkHandlerFactoryByType("Trending");
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test

7
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java
View File

@ -2,12 +2,11 @@ package org.schabi.newpipe.extractor.services.youtube.search;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.utils.Localization;
import static java.util.Collections.singletonList;
import static junit.framework.TestCase.assertTrue;
@ -20,9 +19,9 @@ public class YoutubeSearchCountTest {
public static class YoutubeChannelViewCountTest extends YoutubeSearchExtractorBaseTest {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("pewdiepie",
singletonList(YoutubeSearchQueryHandlerFactory.CHANNELS), null, new Localization("GB", "en"));
singletonList(YoutubeSearchQueryHandlerFactory.CHANNELS), null);
extractor.fetchPage();
itemsPage = extractor.getInitialPage();
}

9
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java
View File

@ -4,14 +4,13 @@ import org.hamcrest.CoreMatchers;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.utils.Localization;
import static java.util.Arrays.asList;
import static org.junit.Assert.*;
@ -21,9 +20,9 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("pewdiepie",
asList(YoutubeSearchQueryHandlerFactory.CHANNELS), null, new Localization("GB", "en"));
asList(YoutubeSearchQueryHandlerFactory.CHANNELS), null);
extractor.fetchPage();
itemsPage = extractor.getInitialPage();
}
@ -31,7 +30,7 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto
@Test
public void testGetSecondPage() throws Exception {
YoutubeSearchExtractor secondExtractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("pewdiepie",
asList(YoutubeSearchQueryHandlerFactory.CHANNELS), null, new Localization("GB", "en"));
asList(YoutubeSearchQueryHandlerFactory.CHANNELS), null);
ListExtractor.InfoItemsPage<InfoItem> secondPage = secondExtractor.getPage(itemsPage.getNextPageUrl());
assertTrue(Integer.toString(secondPage.getItems().size()),
secondPage.getItems().size() > 10);

Some files were not shown because too many files have changed in this diff Show More