lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

merged upstream/dev

This commit is contained in:
yausername 2019-11-20 03:08:17 +05:30
parent 318f600527 0c6e2c8f23
commit f3a59a6cdc
301 changed files with 5358 additions and 3579 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
README.md
View File

@ -13,6 +13,17 @@ If you're using Gradle, you could add NewPipe Extractor as a dependency with the
1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`.
2. Add `compile 'com.github.TeamNewPipe:NewPipeExtractor:v0.11.0'`the `dependencies` in your `build.gradle`. Replace `v0.11.0` with the latest release.
### Testing changes
To test changes quickly you can build the library locally. Using the local Maven repository is a good approach, here's a gist of how to use it:
1. Add `mavenLocal()` in your project `repositories` list (usually as the first entry to give priority above the others).
2. It's _recommended_ that you change the `version` of this library (e.g. `LOCAL_SNAPSHOT`).
3. Run gradle's `ìnstall` task to deploy this library to your local repository (using the wrapper, present in the root of this project: `./gradlew install`)
4. Change the dependency version used in your project to match the one you chose in step 2 (`implementation 'com.github.TeamNewPipe:NewPipeExtractor:LOCAL_SNAPSHOT'`)
> Tip for Android Studio users: After you make changes and run the `install` task, use the menu option `File → "Sync with File System"` to refresh the library in your project.
## Supported sites
The following sites are currently supported:

8
build.gradle
View File

@ -1,15 +1,23 @@
allprojects {
apply plugin: 'java-library'
apply plugin: 'maven'
sourceCompatibility = 1.7
targetCompatibility = 1.7
version 'v0.13.0'
group 'com.github.TeamNewPipe'
repositories {
jcenter()
}
}
dependencies {
implementation project(':extractor')
implementation project(':timeago-parser')
}
subprojects {
task sourcesJar(type: Jar, dependsOn: classes) {
classifier = 'sources'

0
extractor/.attach_pid31246
View File

44
extractor/src/main/java/org/schabi/newpipe/extractor/DownloadRequest.java
View File

@ -1,44 +0,0 @@
package org.schabi.newpipe.extractor;
import java.util.Collections;
import java.util.List;
import java.util.Map;
public class DownloadRequest {
private final String requestBody;
private final Map<String, List<String>> requestHeaders;
public static final DownloadRequest emptyRequest = new DownloadRequest(null, null);
public DownloadRequest(String requestBody, Map<String, List<String>> headers) {
super();
this.requestBody = requestBody;
if(null != headers) {
this.requestHeaders = headers;
}else {
this.requestHeaders = Collections.emptyMap();
}
}
public String getRequestBody() {
return requestBody;
}
public Map<String, List<String>> getRequestHeaders() {
return requestHeaders;
}
public void setRequestCookies(List<String> cookies){
requestHeaders.put("Cookie", cookies);
}
public List<String> getRequestCookies(){
if(null == requestHeaders) return Collections.emptyList();
List<String> cookies = requestHeaders.get("Cookie");
if(null == cookies)
return Collections.emptyList();
else
return cookies;
}
}

42
extractor/src/main/java/org/schabi/newpipe/extractor/DownloadResponse.java
View File

@ -1,42 +0,0 @@
package org.schabi.newpipe.extractor;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
public class DownloadResponse {
private final int responseCode;
private final String responseBody;
private final Map<String, List<String>> responseHeaders;
public DownloadResponse(int responseCode, String responseBody, Map<String, List<String>> headers) {
super();
this.responseCode = responseCode;
this.responseBody = responseBody;
this.responseHeaders = headers;
}
public int getResponseCode() {
return responseCode;
}
public String getResponseBody() {
return responseBody;
}
public Map<String, List<String>> getResponseHeaders() {
return responseHeaders;
}
@Nonnull
public List<String> getResponseCookies(){
if(null == responseHeaders) return Collections.emptyList();
List<String> cookies = responseHeaders.get("Set-Cookie");
if(null == cookies)
return Collections.emptyList();
else
return cookies;
}
}

75
extractor/src/main/java/org/schabi/newpipe/extractor/Downloader.java
View File

@ -1,75 +0,0 @@
package org.schabi.newpipe.extractor;
import java.io.IOException;
import java.util.Map;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.utils.Localization;
/*
* Created by Christian Schabesberger on 28.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* Downloader.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public interface Downloader {
/**
* Download the text file at the supplied URL as in download(String), but set
* the HTTP header field "Accept-Language" to the supplied string.
*
* @param siteUrl the URL of the text file to return the contents of
* @param localization the language and country (usually a 2-character code for each)
* @return the contents of the specified text file
* @throws IOException
*/
String download(String siteUrl, Localization localization) throws IOException, ReCaptchaException;
/**
* Download the text file at the supplied URL as in download(String), but set
* the HTTP header field "Accept-Language" to the supplied string.
*
* @param siteUrl the URL of the text file to return the contents of
* @param customProperties set request header properties
* @return the contents of the specified text file
* @throws IOException
*/
String download(String siteUrl, Map<String, String> customProperties) throws IOException, ReCaptchaException;
/**
* Download (via HTTP) the text file located at the supplied URL, and return its
* contents. Primarily intended for downloading web pages.
*
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file
* @throws IOException
*/
String download(String siteUrl) throws IOException, ReCaptchaException;
DownloadResponse head(String siteUrl) throws IOException, ReCaptchaException;
DownloadResponse get(String siteUrl, Localization localization)
throws IOException, ReCaptchaException;
DownloadResponse get(String siteUrl, DownloadRequest request)
throws IOException, ReCaptchaException;
DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException;
DownloadResponse post(String siteUrl, DownloadRequest request)
throws IOException, ReCaptchaException;
}

40
extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java
View File

@ -1,9 +1,12 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@ -16,21 +19,20 @@ public abstract class Extractor{
* Useful for getting other things from a service (like the url handlers for cleaning/accepting/get id from urls).
*/
private final StreamingService service;
private final LinkHandler linkHandler;
private final Localization localization;
@Nullable
@Nullable private Localization forcedLocalization = null;
@Nullable private ContentCountry forcedContentCountry = null;
private boolean pageFetched = false;
private final Downloader downloader;
public Extractor(final StreamingService service, final LinkHandler linkHandler, final Localization localization) {
public Extractor(final StreamingService service, final LinkHandler linkHandler) {
if(service == null) throw new NullPointerException("service is null");
if(linkHandler == null) throw new NullPointerException("LinkHandler is null");
this.service = service;
this.linkHandler = linkHandler;
this.downloader = NewPipe.getDownloader();
this.localization = localization;
if(downloader == null) throw new NullPointerException("downloader is null");
}
@ -105,8 +107,30 @@ public abstract class Extractor{
return downloader;
}
/*//////////////////////////////////////////////////////////////////////////
// Localization
//////////////////////////////////////////////////////////////////////////*/
public void forceLocalization(Localization localization) {
this.forcedLocalization = localization;
}
public void forceContentCountry(ContentCountry contentCountry) {
this.forcedContentCountry = contentCountry;
}
@Nonnull
public Localization getLocalization() {
return localization;
public Localization getExtractorLocalization() {
return forcedLocalization == null ? getService().getLocalization() : forcedLocalization;
}
@Nonnull
public ContentCountry getExtractorContentCountry() {
return forcedContentCountry == null ? getService().getContentCountry() : forcedContentCountry;
}
@Nonnull
public TimeAgoParser getTimeAgoParser() {
return getService().getTimeAgoParser(getExtractorLocalization());
}
}

5
extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java
View File

@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.Localization;
import javax.annotation.Nonnull;
import java.io.IOException;
@ -14,8 +13,8 @@ import java.util.List;
*/
public abstract class ListExtractor<R extends InfoItem> extends Extractor {
public ListExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public ListExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
}
/**

62
extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java
View File

@ -20,24 +20,42 @@ package org.schabi.newpipe.extractor;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.List;
/**
* Provides access to streaming services supported by NewPipe.
*/
public class NewPipe {
private static Downloader downloader = null;
private static Localization localization = null;
private static Downloader downloader;
private static Localization preferredLocalization;
private static ContentCountry preferredContentCountry;
private NewPipe() {
}
public static void init(Downloader d) {
downloader = d;
preferredLocalization = Localization.DEFAULT;
preferredContentCountry = ContentCountry.DEFAULT;
}
public static void init(Downloader d, Localization l) {
downloader = d;
localization = l;
preferredLocalization = l;
preferredContentCountry = l.getCountryCode().isEmpty() ? ContentCountry.DEFAULT : new ContentCountry(l.getCountryCode());
}
public static void init(Downloader d, Localization l, ContentCountry c) {
downloader = d;
preferredLocalization = l;
preferredContentCountry = c;
}
public static Downloader getDownloader() {
@ -99,11 +117,41 @@ public class NewPipe {
}
}
public static void setLocalization(Localization localization) {
NewPipe.localization = localization;
/*//////////////////////////////////////////////////////////////////////////
// Localization
//////////////////////////////////////////////////////////////////////////*/
public static void setupLocalization(Localization preferredLocalization) {
setupLocalization(preferredLocalization, null);
}
public static void setupLocalization(Localization preferredLocalization, @Nullable ContentCountry preferredContentCountry) {
NewPipe.preferredLocalization = preferredLocalization;
if (preferredContentCountry != null) {
NewPipe.preferredContentCountry = preferredContentCountry;
} else {
NewPipe.preferredContentCountry = preferredLocalization.getCountryCode().isEmpty()
? ContentCountry.DEFAULT
: new ContentCountry(preferredLocalization.getCountryCode());
}
}
@Nonnull
public static Localization getPreferredLocalization() {
return localization;
return preferredLocalization == null ? Localization.DEFAULT : preferredLocalization;
}
public static void setPreferredLocalization(Localization preferredLocalization) {
NewPipe.preferredLocalization = preferredLocalization;
}
@Nonnull
public static ContentCountry getPreferredContentCountry() {
return preferredContentCountry == null ? ContentCountry.DEFAULT : preferredContentCountry;
}
public static void setPreferredContentCountry(ContentCountry preferredContentCountry) {
NewPipe.preferredContentCountry = preferredContentCountry;
}
}

209
extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java
View File

@ -14,11 +14,15 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
/*
* Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
@ -116,10 +120,12 @@ public abstract class StreamingService {
public String toString() {
return serviceId + ":" + serviceInfo.getName();
}
public abstract String getBaseUrl();
////////////////////////////////////////////
/*//////////////////////////////////////////////////////////////////////////
// Url Id handler
////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////*/
/**
* Must return a new instance of an implementation of LinkHandlerFactory for streams.
@ -148,25 +154,22 @@ public abstract class StreamingService {
public abstract SearchQueryHandlerFactory getSearchQHFactory();
public abstract ListLinkHandlerFactory getCommentsLHFactory();
////////////////////////////////////////////
// Extractor
////////////////////////////////////////////
/*//////////////////////////////////////////////////////////////////////////
// Extractors
//////////////////////////////////////////////////////////////////////////*/
/**
* Must create a new instance of a SearchExtractor implementation.
* @param queryHandler specifies the keyword lock for, and the filters which should be applied.
* @param localization specifies the language/country for the extractor.
* @return a new SearchExtractor instance
*/
public abstract SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler, Localization localization);
public abstract SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler);
/**
* Must create a new instance of a SuggestionExtractor implementation.
* @param localization specifies the language/country for the extractor.
* @return a new SuggestionExtractor instance
*/
public abstract SuggestionExtractor getSuggestionExtractor(Localization localization);
public abstract SuggestionExtractor getSuggestionExtractor();
/**
* Outdated or obsolete. null can be returned.
@ -184,124 +187,85 @@ public abstract class StreamingService {
/**
* Must create a new instance of a ChannelExtractor implementation.
* @param linkHandler is pointing to the channel which should be handled by this new instance.
* @param localization specifies the language used for the request.
* @return a new ChannelExtractor
* @throws ExtractionException
*/
public abstract ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler,
Localization localization) throws ExtractionException;
public abstract ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) throws ExtractionException;
/**
* Must crete a new instance of a PlaylistExtractor implementation.
* @param linkHandler is pointing to the playlist which should be handled by this new instance.
* @param localization specifies the language used for the request.
* @return a new PlaylistExtractor
* @throws ExtractionException
*/
public abstract PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler,
Localization localization) throws ExtractionException;
public abstract PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) throws ExtractionException;
/**
* Must create a new instance of a StreamExtractor implementation.
* @param linkHandler is pointing to the stream which should be handled by this new instance.
* @param localization specifies the language used for the request.
* @return a new StreamExtractor
* @throws ExtractionException
*/
public abstract StreamExtractor getStreamExtractor(LinkHandler linkHandler,
Localization localization) throws ExtractionException;
public abstract CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler,
Localization localization) throws ExtractionException;
////////////////////////////////////////////
// Extractor with default localization
////////////////////////////////////////////
public abstract StreamExtractor getStreamExtractor(LinkHandler linkHandler) throws ExtractionException;
public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) {
return getSearchExtractor(queryHandler, NewPipe.getPreferredLocalization());
}
public abstract CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler) throws ExtractionException;
public SuggestionExtractor getSuggestionExtractor() {
return getSuggestionExtractor(NewPipe.getPreferredLocalization());
}
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) throws ExtractionException {
return getChannelExtractor(linkHandler, NewPipe.getPreferredLocalization());
}
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) throws ExtractionException {
return getPlaylistExtractor(linkHandler, NewPipe.getPreferredLocalization());
}
public StreamExtractor getStreamExtractor(LinkHandler linkHandler) throws ExtractionException {
return getStreamExtractor(linkHandler, NewPipe.getPreferredLocalization());
}
public CommentsExtractor getCommentsExtractor(ListLinkHandler urlIdHandler) throws ExtractionException {
return getCommentsExtractor(urlIdHandler, NewPipe.getPreferredLocalization());
}
////////////////////////////////////////////
// Extractor without link handler
////////////////////////////////////////////
/*//////////////////////////////////////////////////////////////////////////
// Extractors without link handler
//////////////////////////////////////////////////////////////////////////*/
public SearchExtractor getSearchExtractor(String query,
List<String> contentFilter,
String sortFilter,
Localization localization) throws ExtractionException {
String sortFilter) throws ExtractionException {
return getSearchExtractor(getSearchQHFactory()
.fromQuery(query,
contentFilter,
sortFilter),
localization);
.fromQuery(query, contentFilter, sortFilter));
}
public ChannelExtractor getChannelExtractor(String id,
List<String> contentFilter,
String sortFilter,
Localization localization) throws ExtractionException {
return getChannelExtractor(getChannelLHFactory().fromQuery(id, contentFilter, sortFilter), localization);
String sortFilter) throws ExtractionException {
return getChannelExtractor(getChannelLHFactory()
.fromQuery(id, contentFilter, sortFilter));
}
public PlaylistExtractor getPlaylistExtractor(String id,
List<String> contentFilter,
String sortFilter,
Localization localization) throws ExtractionException {
String sortFilter) throws ExtractionException {
return getPlaylistExtractor(getPlaylistLHFactory()
.fromQuery(id,
contentFilter,
sortFilter),
localization);
.fromQuery(id, contentFilter, sortFilter));
}
////////////////////////////////////////////
// Short extractor without localization
////////////////////////////////////////////
/*//////////////////////////////////////////////////////////////////////////
// Short extractors overloads
//////////////////////////////////////////////////////////////////////////*/
public SearchExtractor getSearchExtractor(String query) throws ExtractionException {
return getSearchExtractor(getSearchQHFactory().fromQuery(query), NewPipe.getPreferredLocalization());
return getSearchExtractor(getSearchQHFactory().fromQuery(query));
}
public ChannelExtractor getChannelExtractor(String url) throws ExtractionException {
return getChannelExtractor(getChannelLHFactory().fromUrl(url), NewPipe.getPreferredLocalization());
return getChannelExtractor(getChannelLHFactory().fromUrl(url));
}
public PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException {
return getPlaylistExtractor(getPlaylistLHFactory().fromUrl(url), NewPipe.getPreferredLocalization());
return getPlaylistExtractor(getPlaylistLHFactory().fromUrl(url));
}
public StreamExtractor getStreamExtractor(String url) throws ExtractionException {
return getStreamExtractor(getStreamLHFactory().fromUrl(url), NewPipe.getPreferredLocalization());
return getStreamExtractor(getStreamLHFactory().fromUrl(url));
}
public CommentsExtractor getCommentsExtractor(String url) throws ExtractionException {
ListLinkHandlerFactory llhf = getCommentsLHFactory();
if(null == llhf) {
return null;
}
return getCommentsExtractor(llhf.fromUrl(url), NewPipe.getPreferredLocalization());
return getCommentsExtractor(llhf.fromUrl(url));
}
public abstract String getBaseUrl();
/*//////////////////////////////////////////////////////////////////////////
// Utils
//////////////////////////////////////////////////////////////////////////*/
/**
* Figures out where the link is pointing to (a channel, a video, a playlist, etc.)
@ -324,4 +288,95 @@ public abstract class StreamingService {
return LinkType.NONE;
}
}
/*//////////////////////////////////////////////////////////////////////////
// Localization
//////////////////////////////////////////////////////////////////////////*/
/**
* Returns a list of localizations that this service supports.
*/
public List<Localization> getSupportedLocalizations() {
return Collections.singletonList(Localization.DEFAULT);
}
/**
* Returns a list of countries that this service supports.<br>
*/
public List<ContentCountry> getSupportedCountries() {
return Collections.singletonList(ContentCountry.DEFAULT);
}
/**
* Returns the localization that should be used in this service. It will get which localization
* the user prefer (using {@link NewPipe#getPreferredLocalization()}), then it will:
* <ul>
* <li>Check if the exactly localization is supported by this service.</li>
* <li>If not, check if a less specific localization is available, using only the language code.</li>
* <li>Fallback to the {@link Localization#DEFAULT default} localization.</li>
* </ul>
*/
public Localization getLocalization() {
final Localization preferredLocalization = NewPipe.getPreferredLocalization();
// Check the localization's language and country
if (getSupportedLocalizations().contains(preferredLocalization)) {
return preferredLocalization;
}
// Fallback to the first supported language that matches the preferred language
for (Localization supportedLanguage : getSupportedLocalizations()) {
if (supportedLanguage.getLanguageCode().equals(preferredLocalization.getLanguageCode())) {
return supportedLanguage;
}
}
return Localization.DEFAULT;
}
/**
* Returns the country that should be used to fetch content in this service. It will get which country
* the user prefer (using {@link NewPipe#getPreferredContentCountry()}), then it will:
* <ul>
* <li>Check if the country is supported by this service.</li>
* <li>If not, fallback to the {@link ContentCountry#DEFAULT default} country.</li>
* </ul>
*/
public ContentCountry getContentCountry() {
final ContentCountry preferredContentCountry = NewPipe.getPreferredContentCountry();
if (getSupportedCountries().contains(preferredContentCountry)) {
return preferredContentCountry;
}
return ContentCountry.DEFAULT;
}
/**
* Get an instance of the time ago parser using the patterns related to the passed localization.<br>
* <br>
* Just like {@link #getLocalization()}, it will also try to fallback to a less specific localization if
* the exact one is not available/supported.
*
* @throws IllegalArgumentException if the localization is not supported (parsing patterns are not present).
*/
public TimeAgoParser getTimeAgoParser(Localization localization) {
final TimeAgoParser targetParser = TimeAgoPatternsManager.getTimeAgoParserFor(localization);
if (targetParser != null) {
return targetParser;
}
if (!localization.getCountryCode().isEmpty()) {
final Localization lessSpecificLocalization = new Localization(localization.getLanguageCode());
final TimeAgoParser lessSpecificParser = TimeAgoPatternsManager.getTimeAgoParserFor(lessSpecificLocalization);
if (lessSpecificParser != null) {
return lessSpecificParser;
}
}
throw new IllegalArgumentException("Localization is not supported (\"" + localization.toString() + "\")");
}
}

48
extractor/src/main/java/org/schabi/newpipe/extractor/SuggestionExtractor.java
View File

@ -1,48 +0,0 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.utils.Localization;
import java.io.IOException;
import java.util.List;
/*
* Created by Christian Schabesberger on 28.09.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* SuggestionExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public abstract class SuggestionExtractor {
private final int serviceId;
private final Localization localization;
public SuggestionExtractor(int serviceId, Localization localization) {
this.serviceId = serviceId;
this.localization = localization;
}
public abstract List<String> suggestionList(String query) throws IOException, ExtractionException;
public int getServiceId() {
return serviceId;
}
protected Localization getLocalization() {
return localization;
}
}

7
extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java
View File

@ -3,9 +3,8 @@ package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
/*
* Created by Christian Schabesberger on 25.07.16.
@ -29,8 +28,8 @@ import org.schabi.newpipe.extractor.utils.Localization;
public abstract class ChannelExtractor extends ListExtractor<StreamInfoItem> {
public ChannelExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public ChannelExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
}
public abstract String getAvatarUrl() throws ParsingException;

4
extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java
View File

@ -6,10 +6,10 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import org.schabi.newpipe.extractor.utils.Localization;
import java.io.IOException;

9
extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java
View File

@ -3,13 +3,12 @@ package org.schabi.newpipe.extractor.comments;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.Localization;
public abstract class CommentsExtractor extends ListExtractor<CommentsInfoItem> {
public CommentsExtractor(StreamingService service, ListLinkHandler uiHandler, Localization localization) {
super(service, uiHandler, localization);
// TODO Auto-generated constructor stub
}
public CommentsExtractor(StreamingService service, ListLinkHandler uiHandler) {
super(service, uiHandler);
// TODO Auto-generated constructor stub
}
}

123
extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java
View File

@ -1,76 +1,87 @@
package org.schabi.newpipe.extractor.comments;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.localization.DateWrapper;
public class CommentsInfoItem extends InfoItem{
import javax.annotation.Nullable;
private String commentId;
private String commentText;
private String authorName;
private String authorThumbnail;
private String authorEndpoint;
private String publishedTime;
private Integer likeCount;
public CommentsInfoItem(int serviceId, String url, String name) {
super(InfoType.COMMENT, serviceId, url, name);
// TODO Auto-generated constructor stub
}
public String getCommentText() {
return commentText;
}
public class CommentsInfoItem extends InfoItem {
public void setCommentText(String contentText) {
this.commentText = contentText;
}
private String commentId;
private String commentText;
private String authorName;
private String authorThumbnail;
private String authorEndpoint;
private String textualPublishedTime;
@Nullable private DateWrapper publishedTime;
private int likeCount;
public String getAuthorName() {
return authorName;
}
public CommentsInfoItem(int serviceId, String url, String name) {
super(InfoType.COMMENT, serviceId, url, name);
}
public void setAuthorName(String authorName) {
this.authorName = authorName;
}
public String getCommentId() {
return commentId;
}
public String getAuthorThumbnail() {
return authorThumbnail;
}
public void setCommentId(String commentId) {
this.commentId = commentId;
}
public void setAuthorThumbnail(String authorThumbnail) {
this.authorThumbnail = authorThumbnail;
}
public String getCommentText() {
return commentText;
}
public String getAuthorEndpoint() {
return authorEndpoint;
}
public void setCommentText(String commentText) {
this.commentText = commentText;
}
public void setAuthorEndpoint(String authorEndpoint) {
this.authorEndpoint = authorEndpoint;
}
public String getAuthorName() {
return authorName;
}
public String getPublishedTime() {
return publishedTime;
}
public void setAuthorName(String authorName) {
this.authorName = authorName;
}
public void setPublishedTime(String publishedTime) {
this.publishedTime = publishedTime;
}
public String getAuthorThumbnail() {
return authorThumbnail;
}
public Integer getLikeCount() {
return likeCount;
}
public void setAuthorThumbnail(String authorThumbnail) {
this.authorThumbnail = authorThumbnail;
}
public void setLikeCount(Integer likeCount) {
this.likeCount = likeCount;
}
public String getAuthorEndpoint() {
return authorEndpoint;
}
public String getCommentId() {
return commentId;
}
public void setAuthorEndpoint(String authorEndpoint) {
this.authorEndpoint = authorEndpoint;
}
public void setCommentId(String commentId) {
this.commentId = commentId;
}
public String getTextualPublishedTime() {
return textualPublishedTime;
}
public void setTextualPublishedTime(String textualPublishedTime) {
this.textualPublishedTime = textualPublishedTime;
}
@Nullable
public DateWrapper getPublishedTime() {
return publishedTime;
}
public void setPublishedTime(@Nullable DateWrapper publishedTime) {
this.publishedTime = publishedTime;
}
public int getLikeCount() {
return likeCount;
}
public void setLikeCount(int likeCount) {
this.likeCount = likeCount;
}
}

9
extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java
View File

@ -2,6 +2,9 @@ package org.schabi.newpipe.extractor.comments;
import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import javax.annotation.Nullable;
public interface CommentsInfoItemExtractor extends InfoItemExtractor {
String getCommentId() throws ParsingException;
@ -9,6 +12,8 @@ public interface CommentsInfoItemExtractor extends InfoItemExtractor {
String getAuthorName() throws ParsingException;
String getAuthorThumbnail() throws ParsingException;
String getAuthorEndpoint() throws ParsingException;
String getPublishedTime() throws ParsingException;
Integer getLikeCount() throws ParsingException;
String getTextualPublishedTime() throws ParsingException;
@Nullable
DateWrapper getPublishedTime() throws ParsingException;
int getLikeCount() throws ParsingException;
}

11
extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemsCollector.java
View File

@ -1,12 +1,12 @@
package org.schabi.newpipe.extractor.comments;
import java.util.List;
import java.util.Vector;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.util.List;
import java.util.Vector;
public class CommentsInfoItemsCollector extends InfoItemsCollector<CommentsInfoItem, CommentsInfoItemExtractor> {
public CommentsInfoItemsCollector(int serviceId) {
@ -49,6 +49,11 @@ public class CommentsInfoItemsCollector extends InfoItemsCollector<CommentsInfoI
} catch (Exception e) {
addError(e);
}
try {
resultItem.setTextualPublishedTime(extractor.getTextualPublishedTime());
} catch (Exception e) {
addError(e);
}
try {
resultItem.setPublishedTime(extractor.getPublishedTime());
} catch (Exception e) {

144
extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java Normal file
View File

@ -0,0 +1,144 @@
package org.schabi.newpipe.extractor.downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.Localization;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.List;
import java.util.Map;
/**
* A base for downloader implementations that NewPipe will use
* to download needed resources during extraction.
*/
public abstract class Downloader {
/**
* Do a GET request to get the resource that the url is pointing to.<br>
* <br>
* This method calls {@link #get(String, Map, Localization)} with the default preferred localization. It should only be
* used when the resource that will be fetched won't be affected by the localization.
*
* @param url the URL that is pointing to the wanted resource
* @return the result of the GET request
*/
public Response get(String url) throws IOException, ReCaptchaException {
return get(url, null, NewPipe.getPreferredLocalization());
}
/**
* Do a GET request to get the resource that the url is pointing to.<br>
* <br>
* It will set the {@code Accept-Language} header to the language of the localization parameter.
*
* @param url the URL that is pointing to the wanted resource
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the GET request
*/
public Response get(String url, @Nullable Localization localization) throws IOException, ReCaptchaException {
return get(url, null, localization);
}
/**
* Do a GET request with the specified headers.
*
* @param url the URL that is pointing to the wanted resource
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @return the result of the GET request
*/
public Response get(String url, @Nullable Map<String, List<String>> headers) throws IOException, ReCaptchaException {
return get(url, headers, NewPipe.getPreferredLocalization());
}
/**
* Do a GET request with the specified headers.<br>
* <br>
* It will set the {@code Accept-Language} header to the language of the localization parameter.
*
* @param url the URL that is pointing to the wanted resource
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the GET request
*/
public Response get(String url, @Nullable Map<String, List<String>> headers, @Nullable Localization localization)
throws IOException, ReCaptchaException {
return execute(Request.newBuilder()
.get(url)
.headers(headers)
.localization(localization)
.build());
}
/**
* Do a HEAD request.
*
* @param url the URL that is pointing to the wanted resource
* @return the result of the HEAD request
*/
public Response head(String url) throws IOException, ReCaptchaException {
return head(url, null);
}
/**
* Do a HEAD request with the specified headers.
*
* @param url the URL that is pointing to the wanted resource
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @return the result of the HEAD request
*/
public Response head(String url, @Nullable Map<String, List<String>> headers)
throws IOException, ReCaptchaException {
return execute(Request.newBuilder()
.head(url)
.headers(headers)
.build());
}
/**
* Do a POST request with the specified headers, sending the data array.
*
* @param url the URL that is pointing to the wanted resource
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @return the result of the GET request
*/
public Response post(String url, @Nullable Map<String, List<String>> headers, @Nullable byte[] dataToSend)
throws IOException, ReCaptchaException {
return post(url, headers, dataToSend, NewPipe.getPreferredLocalization());
}
/**
* Do a POST request with the specified headers, sending the data array.
* <br>
* It will set the {@code Accept-Language} header to the language of the localization parameter.
*
* @param url the URL that is pointing to the wanted resource
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the GET request
*/
public Response post(String url, @Nullable Map<String, List<String>> headers, @Nullable byte[] dataToSend, @Nullable Localization localization)
throws IOException, ReCaptchaException {
return execute(Request.newBuilder()
.post(url, dataToSend)
.headers(headers)
.localization(localization)
.build());
}
/**
* Do a request using the specified {@link Request} object.
*
* @return the result of the request
*/
public abstract Response execute(@Nonnull Request request) throws IOException, ReCaptchaException;
}

244
extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Request.java Normal file
View File

@ -0,0 +1,244 @@
package org.schabi.newpipe.extractor.downloader;
import org.schabi.newpipe.extractor.localization.Localization;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.*;
/**
* An object that holds request information used when {@link Downloader#execute(Request) executing} a request.
*/
public class Request {
private final String httpMethod;
private final String url;
private final Map<String, List<String>> headers;
@Nullable private final byte[] dataToSend;
@Nullable private final Localization localization;
public Request(String httpMethod, String url, Map<String, List<String>> headers, @Nullable byte[] dataToSend,
@Nullable Localization localization, boolean automaticLocalizationHeader) {
if (httpMethod == null) throw new IllegalArgumentException("Request's httpMethod is null");
if (url == null) throw new IllegalArgumentException("Request's url is null");
this.httpMethod = httpMethod;
this.url = url;
this.dataToSend = dataToSend;
this.localization = localization;
Map<String, List<String>> headersToSet = null;
if (headers == null) headers = Collections.emptyMap();
if (automaticLocalizationHeader && localization != null) {
headersToSet = new LinkedHashMap<>(headersFromLocalization(localization));
headersToSet.putAll(headers);
}
this.headers = Collections.unmodifiableMap(headersToSet == null ? headers : headersToSet);
}
private Request(Builder builder) {
this(builder.httpMethod, builder.url, builder.headers, builder.dataToSend,
builder.localization, builder.automaticLocalizationHeader);
}
/**
* A http method (i.e. {@code GET, POST, HEAD}).
*/
public String httpMethod() {
return httpMethod;
}
/**
* The URL that is pointing to the wanted resource.
*/
public String url() {
return url;
}
/**
* A list of headers that will be used in the request.<br>
* Any default headers that the implementation may have, <b>should</b> be overridden by these.
*/
public Map<String, List<String>> headers() {
return headers;
}
/**
* An optional byte array that will be sent when doing the request, very commonly used in
* {@code POST} requests.<br>
* <br>
* The implementation should make note of some recommended headers
* (for example, {@code Content-Length} in a post request).
*/
@Nullable
public byte[] dataToSend() {
return dataToSend;
}
/**
* A localization object that should be used when executing a request.<br>
* <br>
* Usually the {@code Accept-Language} will be set to this value (a helper
* method to do this easily: {@link Request#headersFromLocalization(Localization)}).
*/
@Nullable
public Localization localization() {
return localization;
}
public static Builder newBuilder() {
return new Builder();
}
public static final class Builder {
private String httpMethod;
private String url;
private Map<String, List<String>> headers = new LinkedHashMap<>();
private byte[] dataToSend;
private Localization localization;
private boolean automaticLocalizationHeader = true;
public Builder() {
}
/**
* A http method (i.e. {@code GET, POST, HEAD}).
*/
public Builder httpMethod(String httpMethod) {
this.httpMethod = httpMethod;
return this;
}
/**
* The URL that is pointing to the wanted resource.
*/
public Builder url(String url) {
this.url = url;
return this;
}
/**
* A list of headers that will be used in the request.<br>
* Any default headers that the implementation may have, <b>should</b> be overridden by these.
*/
public Builder headers(@Nullable Map<String, List<String>> headers) {
if (headers == null) {
this.headers.clear();
return this;
}
this.headers.clear();
this.headers.putAll(headers);
return this;
}
/**
* An optional byte array that will be sent when doing the request, very commonly used in
* {@code POST} requests.<br>
* <br>
* The implementation should make note of some recommended headers
* (for example, {@code Content-Length} in a post request).
*/
public Builder dataToSend(byte[] dataToSend) {
this.dataToSend = dataToSend;
return this;
}
/**
* A localization object that should be used when executing a request.<br>
* <br>
* Usually the {@code Accept-Language} will be set to this value (a helper
* method to do this easily: {@link Request#headersFromLocalization(Localization)}).
*/
public Builder localization(Localization localization) {
this.localization = localization;
return this;
}
/**
* If localization headers should automatically be included in the request.
*/
public Builder automaticLocalizationHeader(boolean automaticLocalizationHeader) {
this.automaticLocalizationHeader = automaticLocalizationHeader;
return this;
}
public Request build() {
return new Request(this);
}
/*//////////////////////////////////////////////////////////////////////////
// Http Methods Utils
//////////////////////////////////////////////////////////////////////////*/
public Builder get(String url) {
this.httpMethod = "GET";
this.url = url;
return this;
}
public Builder head(String url) {
this.httpMethod = "HEAD";
this.url = url;
return this;
}
public Builder post(String url, @Nullable byte[] dataToSend) {
this.httpMethod = "POST";
this.url = url;
this.dataToSend = dataToSend;
return this;
}
/*//////////////////////////////////////////////////////////////////////////
// Additional Headers Utils
//////////////////////////////////////////////////////////////////////////*/
public Builder setHeaders(String headerName, List<String> headerValueList) {
this.headers.remove(headerName);
this.headers.put(headerName, headerValueList);
return this;
}
public Builder addHeaders(String headerName, List<String> headerValueList) {
@Nullable List<String> currentHeaderValueList = this.headers.get(headerName);
if (currentHeaderValueList == null) {
currentHeaderValueList = new ArrayList<>();
}
currentHeaderValueList.addAll(headerValueList);
this.headers.put(headerName, headerValueList);
return this;
}
public Builder setHeader(String headerName, String headerValue) {
return setHeaders(headerName, Collections.singletonList(headerValue));
}
public Builder addHeader(String headerName, String headerValue) {
return addHeaders(headerName, Collections.singletonList(headerValue));
}
}
/*//////////////////////////////////////////////////////////////////////////
// Utils
//////////////////////////////////////////////////////////////////////////*/
@SuppressWarnings("WeakerAccess")
@Nonnull
public static Map<String, List<String>> headersFromLocalization(@Nullable Localization localization) {
if (localization == null) return Collections.emptyMap();
final Map<String, List<String>> headers = new LinkedHashMap<>();
if (!localization.getCountryCode().isEmpty()) {
headers.put("Accept-Language", Collections.singletonList(localization.getLocalizationCode() +
", " + localization.getLanguageCode() + ";q=0.9"));
} else {
headers.put("Accept-Language", Collections.singletonList(localization.getLanguageCode()));
}
return headers;
}
}

66
extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java Normal file
View File

@ -0,0 +1,66 @@
package org.schabi.newpipe.extractor.downloader;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.Collections;
import java.util.List;
import java.util.Map;
/**
* A Data class used to hold the results from requests made by the Downloader implementation.
*/
public class Response {
private final int responseCode;
private final String responseMessage;
private final Map<String, List<String>> responseHeaders;
private final String responseBody;
public Response(int responseCode, String responseMessage, Map<String, List<String>> responseHeaders, @Nullable String responseBody) {
this.responseCode = responseCode;
this.responseMessage = responseMessage;
this.responseHeaders = responseHeaders != null ? responseHeaders : Collections.<String, List<String>>emptyMap();
this.responseBody = responseBody == null ? "" : responseBody;
}
public int responseCode() {
return responseCode;
}
public String responseMessage() {
return responseMessage;
}
public Map<String, List<String>> responseHeaders() {
return responseHeaders;
}
@Nonnull
public String responseBody() {
return responseBody;
}
/*//////////////////////////////////////////////////////////////////////////
// Utils
//////////////////////////////////////////////////////////////////////////*/
/**
* For easy access to some header value that (usually) don't repeat itself.
* <p>For getting all the values associated to the header, use {@link #responseHeaders()} (e.g. {@code Set-Cookie}).
*
* @param name the name of the header
* @return the first value assigned to this header
*/
@Nullable
public String getHeader(String name) {
for (Map.Entry<String, List<String>> headerEntry : responseHeaders.entrySet()) {
if (headerEntry.getKey().equalsIgnoreCase(name)) {
if (headerEntry.getValue().size() > 0) {
return headerEntry.getValue().get(0);
}
}
}
return null;
}
}

8
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java
View File

@ -24,9 +24,8 @@ import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import javax.annotation.Nonnull;
@ -35,9 +34,8 @@ public abstract class KioskExtractor<T extends InfoItem> extends ListExtractor<T
public KioskExtractor(StreamingService streamingService,
ListLinkHandler linkHandler,
String kioskId,
Localization localization) {
super(streamingService, linkHandler, localization);
String kioskId) {
super(streamingService, linkHandler);
this.id = kioskId;
}

13
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java
View File

@ -20,11 +20,14 @@ package org.schabi.newpipe.extractor.kiosk;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import java.io.IOException;
@ -37,7 +40,8 @@ public class KioskInfo extends ListInfo<StreamInfoItem> {
public static ListExtractor.InfoItemsPage<StreamInfoItem> getMoreItems(StreamingService service,
String url,
String pageUrl) throws IOException, ExtractionException {
String pageUrl)
throws IOException, ExtractionException {
KioskList kl = service.getKioskList();
KioskExtractor extractor = kl.getExtractorByUrl(url, pageUrl);
return extractor.getPage(pageUrl);
@ -47,8 +51,7 @@ public class KioskInfo extends ListInfo<StreamInfoItem> {
return getInfo(NewPipe.getServiceByUrl(url), url);
}
public static KioskInfo getInfo(StreamingService service,
String url) throws IOException, ExtractionException {
public static KioskInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
KioskList kl = service.getKioskList();
KioskExtractor extractor = kl.getExtractorByUrl(url, null);
extractor.fetchPage();

36
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java
View File

@ -2,28 +2,33 @@ package org.schabi.newpipe.extractor.kiosk;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
public class KioskList {
public interface KioskExtractorFactory {
KioskExtractor createNewKiosk(final StreamingService streamingService,
final String url,
final String kioskId,
final Localization localization)
final String kioskId)
throws ExtractionException, IOException;
}
private final int service_id;
private final StreamingService service;
private final HashMap<String, KioskEntry> kioskList = new HashMap<>();
private String defaultKiosk = null;
@Nullable private Localization forcedLocalization;
@Nullable private ContentCountry forcedContentCountry;
private class KioskEntry {
public KioskEntry(KioskExtractorFactory ef, ListLinkHandlerFactory h) {
extractorFactory = ef;
@ -33,8 +38,8 @@ public class KioskList {
final ListLinkHandlerFactory handlerFactory;
}
public KioskList(int service_id) {
this.service_id = service_id;
public KioskList(StreamingService service) {
this.service = service;
}
public void addKioskEntry(KioskExtractorFactory extractorFactory, ListLinkHandlerFactory handlerFactory, String id)
@ -89,8 +94,13 @@ public class KioskList {
if(ke == null) {
throw new ExtractionException("No kiosk found with the type: " + kioskId);
} else {
return ke.extractorFactory.createNewKiosk(NewPipe.getService(service_id),
ke.handlerFactory.fromId(kioskId).getUrl(), kioskId, localization);
final KioskExtractor kioskExtractor = ke.extractorFactory.createNewKiosk(service,
ke.handlerFactory.fromId(kioskId).getUrl(), kioskId);
if (forcedLocalization != null) kioskExtractor.forceLocalization(forcedLocalization);
if (forcedContentCountry != null) kioskExtractor.forceContentCountry(forcedContentCountry);
return kioskExtractor;
}
}
@ -117,4 +127,12 @@ public class KioskList {
public ListLinkHandlerFactory getListLinkHandlerFactoryByType(String type) {
return kioskList.get(type).handlerFactory;
}
public void forceLocalization(@Nullable Localization localization) {
this.forcedLocalization = localization;
}
public void forceContentCountry(@Nullable ContentCountry contentCountry) {
this.forcedContentCountry = contentCountry;
}
}

56
extractor/src/main/java/org/schabi/newpipe/extractor/localization/ContentCountry.java Normal file
View File

@ -0,0 +1,56 @@
package org.schabi.newpipe.extractor.localization;
import javax.annotation.Nonnull;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
* Represents a country that should be used when fetching content.
* <p>
* YouTube, for example, give different results in their feed depending on which country is selected.
* </p>
*/
public class ContentCountry implements Serializable {
public static final ContentCountry DEFAULT = new ContentCountry(Localization.DEFAULT.getCountryCode());
@Nonnull private final String countryCode;
public static List<ContentCountry> listFrom(String... countryCodeList) {
final List<ContentCountry> toReturn = new ArrayList<>();
for (String countryCode : countryCodeList) {
toReturn.add(new ContentCountry(countryCode));
}
return Collections.unmodifiableList(toReturn);
}
public ContentCountry(@Nonnull String countryCode) {
this.countryCode = countryCode;
}
@Nonnull
public String getCountryCode() {
return countryCode;
}
@Override
public String toString() {
return getCountryCode();
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof ContentCountry)) return false;
ContentCountry that = (ContentCountry) o;
return countryCode.equals(that.countryCode);
}
@Override
public int hashCode() {
return countryCode.hashCode();
}
}

39
extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java Normal file
View File

@ -0,0 +1,39 @@
package org.schabi.newpipe.extractor.localization;
import edu.umd.cs.findbugs.annotations.NonNull;
import java.io.Serializable;
import java.util.Calendar;
/**
* A wrapper class that provides a field to describe if the date is precise or just an approximation.
*/
public class DateWrapper implements Serializable {
@NonNull private final Calendar date;
private final boolean isApproximation;
public DateWrapper(@NonNull Calendar date) {
this(date, false);
}
public DateWrapper(@NonNull Calendar date, boolean isApproximation) {
this.date = date;
this.isApproximation = isApproximation;
}
/**
* @return the wrapped date.
*/
@NonNull
public Calendar date() {
return date;
}
/**
* @return if the date is considered is precise or just an approximation (e.g. service only returns an approximation
* like 2 weeks ago instead of a precise date).
*/
public boolean isApproximation() {
return isApproximation;
}
}

99
extractor/src/main/java/org/schabi/newpipe/extractor/localization/Localization.java Normal file
View File

@ -0,0 +1,99 @@
package org.schabi.newpipe.extractor.localization;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.Serializable;
import java.util.*;
public class Localization implements Serializable {
public static final Localization DEFAULT = new Localization("en", "GB");
@Nonnull private final String languageCode;
@Nullable private final String countryCode;
/**
* @param localizationCodeList a list of localization code, formatted like {@link #getLocalizationCode()}
*/
public static List<Localization> listFrom(String... localizationCodeList) {
final List<Localization> toReturn = new ArrayList<>();
for (String localizationCode : localizationCodeList) {
toReturn.add(fromLocalizationCode(localizationCode));
}
return Collections.unmodifiableList(toReturn);
}
/**
* @param localizationCode a localization code, formatted like {@link #getLocalizationCode()}
*/
public static Localization fromLocalizationCode(String localizationCode) {
final int indexSeparator = localizationCode.indexOf("-");
final String languageCode, countryCode;
if (indexSeparator != -1) {
languageCode = localizationCode.substring(0, indexSeparator);
countryCode = localizationCode.substring(indexSeparator + 1);
} else {
languageCode = localizationCode;
countryCode = null;
}
return new Localization(languageCode, countryCode);
}
public Localization(@Nonnull String languageCode, @Nullable String countryCode) {
this.languageCode = languageCode;
this.countryCode = countryCode;
}
public Localization(@Nonnull String languageCode) {
this(languageCode, null);
}
public String getLanguageCode() {
return languageCode;
}
@Nonnull
public String getCountryCode() {
return countryCode == null ? "" : countryCode;
}
public Locale asLocale() {
return new Locale(getLanguageCode(), getCountryCode());
}
public static Localization fromLocale(@Nonnull Locale locale) {
return new Localization(locale.getLanguage(), locale.getCountry());
}
/**
* Return a formatted string in the form of: {@code language-Country}, or
* just {@code language} if country is {@code null}.
*/
public String getLocalizationCode() {
return languageCode + (countryCode == null ? "" : "-" + countryCode);
}
@Override
public String toString() {
return "Localization[" + getLocalizationCode() + "]";
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof Localization)) return false;
Localization that = (Localization) o;
if (!languageCode.equals(that.languageCode)) return false;
return countryCode != null ? countryCode.equals(that.countryCode) : that.countryCode == null;
}
@Override
public int hashCode() {
int result = languageCode.hashCode();
result = 31 * result + (countryCode != null ? countryCode.hashCode() : 0);
return result;
}
}

174
extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java Normal file
View File

@ -0,0 +1,174 @@
package org.schabi.newpipe.extractor.localization;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.timeago.PatternsHolder;
import org.schabi.newpipe.extractor.timeago.TimeAgoUnit;
import org.schabi.newpipe.extractor.utils.Parser;
import java.util.Calendar;
import java.util.Collection;
import java.util.Map;
import java.util.regex.Pattern;
/**
* A helper class that is meant to be used by services that need to parse upload dates in the
* format '2 days ago' or similar.
*/
public class TimeAgoParser {
private final PatternsHolder patternsHolder;
private final Calendar consistentNow;
/**
* Creates a helper to parse upload dates in the format '2 days ago'.
* <p>
* Instantiate a new {@link TimeAgoParser} every time you extract a new batch of items.
* </p>
* @param patternsHolder An object that holds the "time ago" patterns, special cases, and the language word separator.
*/
public TimeAgoParser(PatternsHolder patternsHolder) {
this.patternsHolder = patternsHolder;
consistentNow = Calendar.getInstance();
}
/**
* Parses a textual date in the format '2 days ago' into a Calendar representation which is then wrapped in a
* {@link DateWrapper} object.
* <p>
* Beginning with days ago, the date is considered as an approximation.
*
* @param textualDate The original date as provided by the streaming service
* @return The parsed time (can be approximated)
* @throws ParsingException if the time unit could not be recognized
*/
public DateWrapper parse(String textualDate) throws ParsingException {
for (Map.Entry<TimeAgoUnit, Map<String, Integer>> caseUnitEntry : patternsHolder.specialCases().entrySet()) {
final TimeAgoUnit timeAgoUnit = caseUnitEntry.getKey();
for (Map.Entry<String, Integer> caseMapToAmountEntry : caseUnitEntry.getValue().entrySet()) {
final String caseText = caseMapToAmountEntry.getKey();
final Integer caseAmount = caseMapToAmountEntry.getValue();
if (textualDateMatches(textualDate, caseText)) {
return getResultFor(caseAmount, timeAgoUnit);
}
}
}
int timeAgoAmount;
try {
timeAgoAmount = parseTimeAgoAmount(textualDate);
} catch (NumberFormatException e) {
// If there is no valid number in the textual date,
// assume it is 1 (as in 'a second ago').
timeAgoAmount = 1;
}
final TimeAgoUnit timeAgoUnit = parseTimeAgoUnit(textualDate);
return getResultFor(timeAgoAmount, timeAgoUnit);
}
private int parseTimeAgoAmount(String textualDate) throws NumberFormatException {
String timeValueStr = textualDate.replaceAll("\\D+", "");
return Integer.parseInt(timeValueStr);
}
private TimeAgoUnit parseTimeAgoUnit(String textualDate) throws ParsingException {
for (Map.Entry<TimeAgoUnit, Collection<String>> entry : patternsHolder.asMap().entrySet()) {
final TimeAgoUnit timeAgoUnit = entry.getKey();
for (String agoPhrase : entry.getValue()) {
if (textualDateMatches(textualDate, agoPhrase)) {
return timeAgoUnit;
}
}
}
throw new ParsingException("Unable to parse the date: " + textualDate);
}
private boolean textualDateMatches(String textualDate, String agoPhrase) {
if (textualDate.equals(agoPhrase)) {
return true;
}
if (patternsHolder.wordSeparator().isEmpty()) {
return textualDate.toLowerCase().contains(agoPhrase.toLowerCase());
} else {
final String escapedPhrase = Pattern.quote(agoPhrase.toLowerCase());
final String escapedSeparator;
if (patternsHolder.wordSeparator().equals(" ")) {
// From JDK8 → \h - Treat horizontal spaces as a normal one (non-breaking space, thin space, etc.)
escapedSeparator = "[ \\t\\xA0\\u1680\\u180e\\u2000-\\u200a\\u202f\\u205f\\u3000]";
} else {
escapedSeparator = Pattern.quote(patternsHolder.wordSeparator());
}
// (^|separator)pattern($|separator)
// Check if the pattern is surrounded by separators or start/end of the string.
final String pattern =
"(^|" + escapedSeparator + ")" + escapedPhrase + "($|" + escapedSeparator + ")";
return Parser.isMatch(pattern, textualDate.toLowerCase());
}
}
private DateWrapper getResultFor(int timeAgoAmount, TimeAgoUnit timeAgoUnit) {
final Calendar calendarTime = getNow();
boolean isApproximation = false;
switch (timeAgoUnit) {
case SECONDS:
calendarTime.add(Calendar.SECOND, -timeAgoAmount);
break;
case MINUTES:
calendarTime.add(Calendar.MINUTE, -timeAgoAmount);
break;
case HOURS:
calendarTime.add(Calendar.HOUR_OF_DAY, -timeAgoAmount);
break;
case DAYS:
calendarTime.add(Calendar.DAY_OF_MONTH, -timeAgoAmount);
isApproximation = true;
break;
case WEEKS:
calendarTime.add(Calendar.WEEK_OF_YEAR, -timeAgoAmount);
isApproximation = true;
break;
case MONTHS:
calendarTime.add(Calendar.MONTH, -timeAgoAmount);
isApproximation = true;
break;
case YEARS:
calendarTime.add(Calendar.YEAR, -timeAgoAmount);
// Prevent `PrettyTime` from showing '12 months ago'.
calendarTime.add(Calendar.DAY_OF_MONTH, -1);
isApproximation = true;
break;
}
if (isApproximation) {
markApproximatedTime(calendarTime);
}
return new DateWrapper(calendarTime, isApproximation);
}
private Calendar getNow() {
return (Calendar) consistentNow.clone();
}
/**
* Marks the time as approximated by setting minutes, seconds and milliseconds to 0.
* @param calendarTime Time to be marked as approximated
*/
private void markApproximatedTime(Calendar calendarTime) {
calendarTime.set(Calendar.MINUTE, 0);
calendarTime.set(Calendar.SECOND, 0);
calendarTime.set(Calendar.MILLISECOND, 0);
}
}

25
extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoPatternsManager.java Normal file
View File

@ -0,0 +1,25 @@
package org.schabi.newpipe.extractor.localization;
import org.schabi.newpipe.extractor.timeago.PatternsHolder;
import org.schabi.newpipe.extractor.timeago.PatternsManager;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
public class TimeAgoPatternsManager {
@Nullable
private static PatternsHolder getPatternsFor(@Nonnull Localization localization) {
return PatternsManager.getPatterns(localization.getLanguageCode(), localization.getCountryCode());
}
@Nullable
public static TimeAgoParser getTimeAgoParserFor(@Nonnull Localization localization) {
final PatternsHolder holder = getPatternsFor(localization);
if (holder == null) {
return null;
}
return new TimeAgoParser(holder);
}
}

5
extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java
View File

@ -5,12 +5,11 @@ import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.Localization;
public abstract class PlaylistExtractor extends ListExtractor<StreamInfoItem> {
public PlaylistExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public PlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
}
public abstract String getThumbnailUrl() throws ParsingException;

1
extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java
View File

@ -9,7 +9,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import org.schabi.newpipe.extractor.utils.Localization;
import java.io.IOException;

10
extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java
View File

@ -6,7 +6,8 @@ import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.utils.Localization;
import javax.annotation.Nonnull;
public abstract class SearchExtractor extends ListExtractor<InfoItem> {
@ -18,10 +19,8 @@ public abstract class SearchExtractor extends ListExtractor<InfoItem> {
private final InfoItemsSearchCollector collector;
public SearchExtractor(StreamingService service,
SearchQueryHandler linkHandler,
Localization localization) {
super(service, linkHandler, localization);
public SearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
super(service, linkHandler);
collector = new InfoItemsSearchCollector(service.getServiceId());
}
@ -40,6 +39,7 @@ public abstract class SearchExtractor extends ListExtractor<InfoItem> {
return (SearchQueryHandler) super.getLinkHandler();
}
@Nonnull
@Override
public String getName() {
return getLinkHandler().getSearchString();

1
extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java
View File

@ -7,7 +7,6 @@ import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import org.schabi.newpipe.extractor.utils.Localization;
import java.io.IOException;

47
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java
View File

@ -1,24 +1,12 @@
package org.schabi.newpipe.extractor.services.media_ccc;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
import java.io.IOException;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.*;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor;
@ -31,7 +19,13 @@ import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearc
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.io.IOException;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
public class MediaCCCService extends StreamingService {
public MediaCCCService(int id) {
@ -39,8 +33,8 @@ public class MediaCCCService extends StreamingService {
}
@Override
public SearchExtractor getSearchExtractor(SearchQueryHandler query, Localization localization) {
return new MediaCCCSearchExtractor(this, query, localization);
public SearchExtractor getSearchExtractor(SearchQueryHandler query) {
return new MediaCCCSearchExtractor(this, query);
}
@Override
@ -64,28 +58,28 @@ public class MediaCCCService extends StreamingService {
}
@Override
public StreamExtractor getStreamExtractor(LinkHandler linkHandler, Localization localization) {
return new MediaCCCStreamExtractor(this, linkHandler, localization);
public StreamExtractor getStreamExtractor(LinkHandler linkHandler) {
return new MediaCCCStreamExtractor(this, linkHandler);
}
@Override
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler, Localization localization) {
return new MediaCCCConferenceExtractor(this, linkHandler, localization);
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) {
return new MediaCCCConferenceExtractor(this, linkHandler);
}
@Override
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler, Localization localization) {
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) {
return null;
}
@Override
public SuggestionExtractor getSuggestionExtractor(Localization localization) {
public SuggestionExtractor getSuggestionExtractor() {
return null;
}
@Override
public KioskList getKioskList() throws ExtractionException {
KioskList list = new KioskList(getServiceId());
KioskList list = new KioskList(this);
// add kiosks here e.g.:
try {
@ -93,10 +87,9 @@ public class MediaCCCService extends StreamingService {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService,
String url,
String kioskId,
Localization localization) throws ExtractionException, IOException {
String kioskId) throws ExtractionException, IOException {
return new MediaCCCConferenceKiosk(MediaCCCService.this,
new MediaCCCConferencesListLinkHandlerFactory().fromUrl(url), kioskId, localization);
new MediaCCCConferencesListLinkHandlerFactory().fromUrl(url), kioskId);
}
}, new MediaCCCConferencesListLinkHandlerFactory(), "conferences");
list.setDefaultKiosk("conferences");
@ -118,7 +111,7 @@ public class MediaCCCService extends StreamingService {
}
@Override
public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler, Localization localization)
public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler)
throws ExtractionException {
return null;
}

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java
View File

@ -4,16 +4,15 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Localization;
import javax.annotation.Nonnull;
import java.io.IOException;
@ -22,8 +21,8 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
private JsonObject conferenceData;
public MediaCCCConferenceExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public MediaCCCConferenceExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
@ -75,7 +74,7 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
try {
conferenceData = JsonParser.object().from(downloader.download(getUrl()));
conferenceData = JsonParser.object().from(downloader.get(getUrl()).responseBody());
} catch (JsonParserException jpe) {
throw new ExtractionException("Could not parse json returnd by url: " + getUrl());
}
@ -87,6 +86,7 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
return conferenceData.getString("title");
}
@Nonnull
@Override
public String getOriginalUrl() throws ParsingException {
return "https://media.ccc.de/c/" + conferenceData.getString("acronym");

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java
View File

@ -4,16 +4,15 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCConferenceInfoItemExtractor;
import org.schabi.newpipe.extractor.utils.Localization;
import javax.annotation.Nonnull;
import java.io.IOException;
@ -24,9 +23,8 @@ public class MediaCCCConferenceKiosk extends KioskExtractor<ChannelInfoItem> {
public MediaCCCConferenceKiosk(StreamingService streamingService,
ListLinkHandler linkHandler,
String kioskId,
Localization localization) {
super(streamingService, linkHandler, kioskId, localization);
String kioskId) {
super(streamingService, linkHandler, kioskId);
}
@Nonnull
@ -53,7 +51,7 @@ public class MediaCCCConferenceKiosk extends KioskExtractor<ChannelInfoItem> {
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
String site = downloader.download(getLinkHandler().getUrl());
String site = downloader.get(getLinkHandler().getUrl(), getExtractorLocalization()).responseBody();
try {
doc = JsonParser.object().from(site);
} catch (JsonParserException jpe) {

27
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java Normal file
View File

@ -0,0 +1,27 @@
package org.schabi.newpipe.extractor.services.media_ccc.extractors;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
public class MediaCCCParsingHelper {
private MediaCCCParsingHelper() {
}
public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
Date date;
try {
date = new SimpleDateFormat("yyyy-MM-dd").parse(textualUploadDate);
} catch (ParseException e) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
}
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
}
}

18
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java
View File

@ -4,11 +4,11 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
@ -16,26 +16,24 @@ import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferencesListLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Localization;
import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.CONFERENCES;
import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.EVENTS;
import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.ALL;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.List;
import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.*;
public class MediaCCCSearchExtractor extends SearchExtractor {
private JsonObject doc;
private MediaCCCConferenceKiosk conferenceKiosk;
public MediaCCCSearchExtractor(StreamingService service, SearchQueryHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public MediaCCCSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
super(service, linkHandler);
try {
conferenceKiosk = new MediaCCCConferenceKiosk(service,
new MediaCCCConferencesListLinkHandlerFactory().fromId("conferences"),
"conferences",
localization);
"conferences");
} catch (Exception e) {
e.printStackTrace();
}
@ -88,7 +86,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
|| getLinkHandler().getContentFilters().isEmpty()) {
final String site;
final String url = getUrl();
site = downloader.download(url, getLocalization());
site = downloader.get(url, getExtractorLocalization()).responseBody();
try {
doc = JsonParser.object().from(site);
} catch (JsonParserException jpe) {

22
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java
View File

@ -4,15 +4,14 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull;
import java.io.IOException;
@ -24,16 +23,22 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
private JsonObject data;
private JsonObject conferenceData;
public MediaCCCStreamExtractor(StreamingService service, LinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public MediaCCCStreamExtractor(StreamingService service, LinkHandler linkHandler) {
super(service, linkHandler);
}
@Nonnull
@Override
public String getUploadDate() throws ParsingException {
public String getTextualUploadDate() throws ParsingException {
return data.getString("release_date");
}
@Nonnull
@Override
public DateWrapper getUploadDate() throws ParsingException {
return new DateWrapper(MediaCCCParsingHelper.parseDateFrom(getTextualUploadDate()));
}
@Nonnull
@Override
public String getThumbnailUrl() throws ParsingException {
@ -200,9 +205,9 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
try {
data = JsonParser.object().from(
downloader.download(getLinkHandler().getUrl()));
downloader.get(getLinkHandler().getUrl()).responseBody());
conferenceData = JsonParser.object()
.from(downloader.download(getUploaderUrl()));
.from(downloader.get(getUploaderUrl()).responseBody());
} catch (JsonParserException jpe) {
throw new ExtractionException("Could not parse json returned by url: " + getLinkHandler().getUrl(), jpe);
}
@ -215,6 +220,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
return data.getString("title");
}
@Nonnull
@Override
public String getOriginalUrl() throws ParsingException {
return data.getString("frontend_link");

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSuggestionExtractor.java
View File

@ -1,8 +1,8 @@
package org.schabi.newpipe.extractor.services.media_ccc.extractors;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.io.IOException;
import java.util.ArrayList;
@ -10,8 +10,8 @@ import java.util.List;
public class MediaCCCSuggestionExtractor extends SuggestionExtractor {
public MediaCCCSuggestionExtractor(int serviceId, Localization localization) {
super(serviceId, localization);
public MediaCCCSuggestionExtractor(StreamingService service) {
super(service);
}
@Override

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java
View File

@ -2,9 +2,13 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import javax.annotation.Nullable;
public class MediaCCCStreamInfoItemExtractor implements StreamInfoItemExtractor {
JsonObject event;
@ -44,11 +48,18 @@ public class MediaCCCStreamInfoItemExtractor implements StreamInfoItemExtractor
return event.getString("conference_url");
}
@Nullable
@Override
public String getUploadDate() throws ParsingException {
public String getTextualUploadDate() throws ParsingException {
return event.getString("release_date");
}
@Nullable
@Override
public DateWrapper getUploadDate() throws ParsingException {
return new DateWrapper(MediaCCCParsingHelper.parseDateFrom(getTextualUploadDate()));
}
@Override
public String getName() throws ParsingException {
return event.getString("title");

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java
View File

@ -3,9 +3,9 @@ package org.schabi.newpipe.extractor.services.peertube;
import java.io.IOException;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.utils.JsonUtils;
@ -37,7 +37,7 @@ public class PeertubeInstance {
private String validateInstance(String url) throws IOException {
Downloader downloader = NewPipe.getDownloader();
DownloadResponse response = null;
Response response = null;
try {
response = downloader.get(url + "/api/v1/config");
@ -45,11 +45,11 @@ public class PeertubeInstance {
throw new IOException("unable to configure instance " + url, e);
}
if(null == response || StringUtil.isBlank(response.getResponseBody())) {
if(null == response || StringUtil.isBlank(response.responseBody())) {
throw new IOException("unable to configure instance " + url);
}
return response.getResponseBody();
return response.responseBody();
}
private void setInstanceMetaData(String responseBody) {

25
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java
View File

@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.peertube;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import org.jsoup.helper.StringUtil;
@ -15,20 +16,24 @@ public class PeertubeParsingHelper {
private PeertubeParsingHelper() {
}
public static String toDateString(String time) throws ParsingException {
try {
Date date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse(time);
SimpleDateFormat newDateFormat = new SimpleDateFormat("yyyy-MM-dd");
return newDateFormat.format(date);
} catch (ParseException e) {
throw new ParsingException(e.getMessage(), e);
}
}
public static void validate(JsonObject json) throws ContentNotAvailableException {
String error = json.getString("error");
if(!StringUtil.isBlank(error)) {
throw new ContentNotAvailableException(error);
}
}
public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
Date date;
try {
date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse(textualUploadDate);
} catch (ParseException e) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
}
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
}
}

32
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java
View File

@ -10,7 +10,6 @@ import java.util.List;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -37,7 +36,7 @@ import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStream
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
public class PeertubeService extends StreamingService {
@ -83,13 +82,13 @@ public class PeertubeService extends StreamingService {
}
@Override
public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler, Localization localization) {
return new PeertubeSearchExtractor(this, queryHandler, localization);
public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) {
return new PeertubeSearchExtractor(this, queryHandler);
}
@Override
public SuggestionExtractor getSuggestionExtractor(Localization localization) {
return new PeertubeSuggestionExtractor(this.getServiceId(), localization);
public SuggestionExtractor getSuggestionExtractor() {
return new PeertubeSuggestionExtractor(this);
}
@Override
@ -99,28 +98,28 @@ public class PeertubeService extends StreamingService {
}
@Override
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler, Localization localization)
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler)
throws ExtractionException {
return new PeertubeChannelExtractor(this, linkHandler, localization);
return new PeertubeChannelExtractor(this, linkHandler);
}
@Override
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler, Localization localization)
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler)
throws ExtractionException {
// TODO Auto-generated method stub
return null;
}
@Override
public StreamExtractor getStreamExtractor(LinkHandler linkHandler, Localization localization)
public StreamExtractor getStreamExtractor(LinkHandler linkHandler)
throws ExtractionException {
return new PeertubeStreamExtractor(this, linkHandler, localization);
return new PeertubeStreamExtractor(this, linkHandler);
}
@Override
public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler, Localization localization)
public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler)
throws ExtractionException {
return new PeertubeCommentsExtractor(this, linkHandler, localization);
return new PeertubeCommentsExtractor(this, linkHandler);
}
@Override
@ -152,15 +151,14 @@ public class PeertubeService extends StreamingService {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService,
String url,
String id,
Localization local)
String id)
throws ExtractionException {
return new PeertubeTrendingExtractor(PeertubeService.this,
new PeertubeTrendingLinkHandlerFactory().fromId(id), id, local);
new PeertubeTrendingLinkHandlerFactory().fromId(id), id);
}
};
KioskList list = new KioskList(getServiceId());
KioskList list = new KioskList(this);
// add kiosks here e.g.:
final PeertubeTrendingLinkHandlerFactory h = new PeertubeTrendingLinkHandlerFactory();

21
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java
View File

@ -3,11 +3,11 @@ package org.schabi.newpipe.extractor.services.peertube.extractors;
import java.io.IOException;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@ -15,7 +15,6 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
@ -36,8 +35,8 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
private JsonObject json;
public PeertubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public PeertubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
@ -108,11 +107,11 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
@Override
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
DownloadResponse response = getDownloader().get(pageUrl);
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if(null != response && !StringUtil.isBlank(response.getResponseBody())) {
if(null != response && !StringUtil.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.getResponseBody());
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
throw new ParsingException("Could not parse json data for kiosk info", e);
}
@ -155,9 +154,9 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
DownloadResponse response = downloader.get(getUrl());
if(null != response && null != response.getResponseBody()) {
setInitialData(response.getResponseBody());
Response response = downloader.get(getUrl());
if(null != response && null != response.responseBody()) {
setInitialData(response.responseBody());
}else {
throw new ExtractionException("Unable to extract peertube channel data");
}

15
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java
View File

@ -3,17 +3,16 @@ package org.schabi.newpipe.extractor.services.peertube.extractors;
import java.io.IOException;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemsCollector;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
@ -31,8 +30,8 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
private InfoItemsPage<CommentsInfoItem> initPage;
private long total;
public PeertubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler, Localization localization) {
super(service, uiHandler, localization);
public PeertubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) {
super(service, uiHandler);
}
@Override
@ -72,11 +71,11 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
@Override
public InfoItemsPage<CommentsInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
DownloadResponse response = getDownloader().get(pageUrl);
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if(null != response && !StringUtil.isBlank(response.getResponseBody())) {
if(null != response && !StringUtil.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.getResponseBody());
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
throw new ParsingException("Could not parse json data for comments info", e);
}

34
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java
View File

@ -1,15 +1,12 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Locale;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.JsonUtils;
@ -48,13 +45,18 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
}
@Override
public String getPublishedTime() throws ParsingException {
String date = JsonUtils.getString(item, "createdAt");
return getFormattedDate(date);
public String getTextualPublishedTime() throws ParsingException {
return JsonUtils.getString(item, "createdAt");
}
@Override
public Integer getLikeCount() throws ParsingException {
public DateWrapper getPublishedTime() throws ParsingException {
String textualUploadDate = getTextualPublishedTime();
return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate));
}
@Override
public int getLikeCount() throws ParsingException {
return 0;
}
@ -98,14 +100,4 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
return PeertubeChannelLinkHandlerFactory.getInstance().fromId(name + "@" + host).getUrl();
}
private String getFormattedDate(String date) {
DateFormat sourceDf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
DateFormat targetDf = new SimpleDateFormat("dd-MM-yyyy hh:mm a", Locale.ENGLISH);
try {
return targetDf.format(sourceDf.parse(date));
} catch (ParseException e) {
return date;
}
}
}
}

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java
View File

@ -2,19 +2,18 @@ package org.schabi.newpipe.extractor.services.peertube.extractors;
import java.io.IOException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.Localization;
public class PeertubePlaylistExtractor extends PlaylistExtractor{
public PeertubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public PeertubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
// TODO Auto-generated constructor stub
}

16
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java
View File

@ -3,19 +3,18 @@ package org.schabi.newpipe.extractor.services.peertube.extractors;
import java.io.IOException;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
@ -33,9 +32,8 @@ public class PeertubeSearchExtractor extends SearchExtractor {
private InfoItemsPage<InfoItem> initPage;
private long total;
public PeertubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler,
Localization localization) {
super(service, linkHandler, localization);
public PeertubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
super(service, linkHandler);
}
@Override
@ -80,11 +78,11 @@ public class PeertubeSearchExtractor extends SearchExtractor {
@Override
public InfoItemsPage<InfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
DownloadResponse response = getDownloader().get(pageUrl);
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if(null != response && !StringUtil.isBlank(response.getResponseBody())) {
if(null != response && !StringUtil.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.getResponseBody());
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
throw new ParsingException("Could not parse json data for search info", e);
}

44
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java
View File

@ -8,15 +8,16 @@ import java.util.Collections;
import java.util.List;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
@ -29,7 +30,6 @@ import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Localization;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
@ -42,16 +42,26 @@ public class PeertubeStreamExtractor extends StreamExtractor {
private JsonObject json;
private List<SubtitlesStream> subtitles = new ArrayList<>();
public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
public String getTextualUploadDate() throws ParsingException {
return JsonUtils.getString(json, "publishedAt");
}
@Override
public String getUploadDate() throws ParsingException {
String date = JsonUtils.getString(json, "publishedAt");
return PeertubeParsingHelper.toDateString(date);
}
public DateWrapper getUploadDate() throws ParsingException {
final String textualUploadDate = getTextualUploadDate();
if (textualUploadDate == null) {
return null;
}
return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate));
}
@Override
public String getThumbnailUrl() throws ParsingException {
return ServiceList.PeerTube.getBaseUrl() + JsonUtils.getString(json, "thumbnailPath");
@ -233,11 +243,11 @@ public class PeertubeStreamExtractor extends StreamExtractor {
}
private void getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, IOException, ParsingException {
DownloadResponse response = getDownloader().get(apiUrl);
Response response = getDownloader().get(apiUrl);
JsonObject relatedVideosJson = null;
if(null != response && !StringUtil.isBlank(response.getResponseBody())) {
if(null != response && !StringUtil.isBlank(response.responseBody())) {
try {
relatedVideosJson = JsonParser.object().from(response.getResponseBody());
relatedVideosJson = JsonParser.object().from(response.responseBody());
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json data for related videos", e);
}
@ -275,9 +285,9 @@ public class PeertubeStreamExtractor extends StreamExtractor {
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
DownloadResponse response = downloader.get(getUrl());
if(null != response && null != response.getResponseBody()) {
setInitialData(response.getResponseBody());
Response response = downloader.get(getUrl());
if(null != response && null != response.responseBody()) {
setInitialData(response.responseBody());
}else {
throw new ExtractionException("Unable to extract peertube channel data");
}
@ -298,8 +308,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
private void loadSubtitles() {
if (subtitles.isEmpty()) {
try {
DownloadResponse response = getDownloader().get(getUrl() + "/captions");
JsonObject captionsJson = JsonParser.object().from(response.getResponseBody());
Response response = getDownloader().get(getUrl() + "/captions");
JsonObject captionsJson = JsonParser.object().from(response.responseBody());
JsonArray captions = JsonUtils.getArray(captionsJson, "data");
for(Object c: captions) {
if(c instanceof JsonObject) {

20
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java
View File

@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.peertube.extractors;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory;
@ -60,11 +61,21 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor
}
@Override
public String getUploadDate() throws ParsingException {
String date = JsonUtils.getString(item, "publishedAt");
return PeertubeParsingHelper.toDateString(date);
public String getTextualUploadDate() throws ParsingException {
return JsonUtils.getString(item, "publishedAt");
}
@Override
public DateWrapper getUploadDate() throws ParsingException {
final String textualUploadDate = getTextualUploadDate();
if (textualUploadDate == null) {
return null;
}
return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate));
}
@Override
public StreamType getStreamType() throws ParsingException {
return StreamType.VIDEO_STREAM;
@ -75,4 +86,5 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor
Number value = JsonUtils.getNumber(item, "duration");
return value.longValue();
}
}

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java
View File

@ -4,14 +4,14 @@ import java.io.IOException;
import java.util.Collections;
import java.util.List;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
public class PeertubeSuggestionExtractor extends SuggestionExtractor{
public PeertubeSuggestionExtractor(int serviceId, Localization localization) {
super(serviceId, localization);
public PeertubeSuggestionExtractor(StreamingService service) {
super(service);
}
@Override

21
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java
View File

@ -3,22 +3,16 @@ package org.schabi.newpipe.extractor.services.peertube.extractors;
import java.io.IOException;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
@ -36,9 +30,8 @@ public class PeertubeTrendingExtractor extends KioskExtractor {
private InfoItemsPage<StreamInfoItem> initPage;
private long total;
public PeertubeTrendingExtractor(StreamingService streamingService, ListLinkHandler linkHandler, String kioskId,
Localization localization) {
super(streamingService, linkHandler, kioskId, localization);
public PeertubeTrendingExtractor(StreamingService streamingService, ListLinkHandler linkHandler, String kioskId) {
super(streamingService, linkHandler, kioskId);
}
@Override
@ -78,11 +71,11 @@ public class PeertubeTrendingExtractor extends KioskExtractor {
@Override
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
DownloadResponse response = getDownloader().get(pageUrl);
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if(null != response && !StringUtil.isBlank(response.getResponseBody())) {
if(null != response && !StringUtil.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.getResponseBody());
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
throw new ParsingException("Could not parse json data for kiosk info", e);
}

11
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java
View File

@ -4,15 +4,14 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Localization;
import javax.annotation.Nonnull;
import java.io.IOException;
@ -25,8 +24,8 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
private StreamInfoItemsCollector streamInfoItemsCollector = null;
private String nextPageUrl = null;
public SoundcloudChannelExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public SoundcloudChannelExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
@ -36,7 +35,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
String apiUrl = "https://api-v2.soundcloud.com/users/" + userId +
"?client_id=" + SoundcloudParsingHelper.clientId();
String response = downloader.download(apiUrl);
String response = downloader.get(apiUrl, getExtractorLocalization()).responseBody();
try {
user = JsonParser.object().from(response);
} catch (JsonParserException e) {

12
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java
View File

@ -1,18 +1,15 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Localization;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
private StreamInfoItemsCollector collector = null;
@ -20,9 +17,8 @@ public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
public SoundcloudChartsExtractor(StreamingService service,
ListLinkHandler linkHandler,
String kioskId,
Localization localization) {
super(service, linkHandler, kioskId, localization);
String kioskId) {
super(service, linkHandler, kioskId);
}
@Override

61
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View File

@ -8,10 +8,10 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
@ -24,10 +24,10 @@ import java.io.IOException;
import java.net.URLEncoder;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.*;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudParsingHelper {
@ -46,23 +46,23 @@ public class SoundcloudParsingHelper {
return clientId;
}
final DownloadResponse download = dl.get("https://soundcloud.com");
String response = download.getResponseBody();
final Response download = dl.get("https://soundcloud.com");
final String responseBody = download.responseBody();
final String clientIdPattern = ",client_id:\"(.*?)\"";
Document doc = Jsoup.parse(response);
Document doc = Jsoup.parse(responseBody);
final Elements possibleScripts = doc.select("script[src*=\"sndcdn.com/assets/\"][src$=\".js\"]");
// The one containing the client id will likely be the last one
Collections.reverse(possibleScripts);
final HashMap<String, String> headers = new HashMap<>();
headers.put("Range", "bytes=0-16384");
final HashMap<String, List<String>> headers = new HashMap<>();
headers.put("Range", singletonList("bytes=0-16384"));
for (Element element : possibleScripts) {
final String srcUrl = element.attr("src");
if (srcUrl != null && !srcUrl.isEmpty()) {
try {
return clientId = Parser.matchGroup1(clientIdPattern, dl.download(srcUrl, headers));
return clientId = Parser.matchGroup1(clientIdPattern, dl.get(srcUrl, headers).responseBody());
} catch (RegexException ignored) {
// Ignore it and proceed to try searching other script
}
@ -76,24 +76,24 @@ public class SoundcloudParsingHelper {
static boolean checkIfHardcodedClientIdIsValid(Downloader dl) throws IOException, ReCaptchaException {
final String apiUrl = "https://api.soundcloud.com/connect?client_id=" + HARDCODED_CLIENT_ID;
// Should return 200 to indicate that the client id is valid, a 401 is returned otherwise.
return dl.head(apiUrl).getResponseCode() == 200;
return dl.head(apiUrl).responseCode() == 200;
}
public static String toDateString(String time) throws ParsingException {
static Calendar parseDate(String textualUploadDate) throws ParsingException {
Date date;
try {
Date date;
// Have two date formats, one for the 'api.soundc...' and the other 'api-v2.soundc...'.
date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").parse(textualUploadDate);
} catch (ParseException e1) {
try {
date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").parse(time);
} catch (Exception e) {
date = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse(time);
date = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse(textualUploadDate);
} catch (ParseException e2) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"" + ", " + e1.getMessage(), e2);
}
SimpleDateFormat newDateFormat = new SimpleDateFormat("yyyy-MM-dd");
return newDateFormat.format(date);
} catch (ParseException e) {
throw new ParsingException(e.getMessage(), e);
}
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
}
/**
@ -107,7 +107,8 @@ public class SoundcloudParsingHelper {
+ "&client_id=" + clientId();
try {
return JsonParser.object().from(downloader.download(apiUrl));
final String response = downloader.get(apiUrl, SoundCloud.getLocalization()).responseBody();
return JsonParser.object().from(response);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
@ -120,8 +121,8 @@ public class SoundcloudParsingHelper {
*/
public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(apiUrl, "UTF-8"));
String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(apiUrl, "UTF-8"), SoundCloud.getLocalization()).responseBody();
return Jsoup.parse(response).select("link[rel=\"canonical\"]").first().attr("abs:href");
}
@ -133,8 +134,8 @@ public class SoundcloudParsingHelper {
*/
public static String resolveIdWithEmbedPlayer(String url) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(url, "UTF-8"));
String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(url, "UTF-8"), SoundCloud.getLocalization()).responseBody();
// handle playlists / sets different and get playlist id via uir field in JSON
if (url.contains("sets") && !url.endsWith("sets") && !url.endsWith("sets/"))
return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response);
@ -165,7 +166,7 @@ public class SoundcloudParsingHelper {
* @return the next streams url, empty if don't have
*/
public static String getUsersFromApi(ChannelInfoItemsCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().download(apiUrl);
String response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization()).responseBody();
JsonObject responseObject;
try {
responseObject = JsonParser.object().from(response);
@ -216,7 +217,7 @@ public class SoundcloudParsingHelper {
* @return the next streams url, empty if don't have
*/
public static String getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().download(apiUrl);
String response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization()).responseBody();
JsonObject responseObject;
try {
responseObject = JsonParser.object().from(response);

11
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java
View File

@ -3,15 +3,14 @@ package org.schabi.newpipe.extractor.services.soundcloud;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Localization;
import javax.annotation.Nonnull;
import java.io.IOException;
@ -24,8 +23,8 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
private StreamInfoItemsCollector streamInfoItemsCollector = null;
private String nextPageUrl = null;
public SoundcloudPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public SoundcloudPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
@ -36,7 +35,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
"?client_id=" + SoundcloudParsingHelper.clientId() +
"&representation=compact";
String response = downloader.download(apiUrl);
String response = downloader.get(apiUrl, getExtractorLocalization()).responseBody();
try {
playlist = JsonParser.object().from(response);
} catch (JsonParserException e) {

16
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java
View File

@ -5,12 +5,12 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull;
@ -25,10 +25,8 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
private JsonArray searchCollection;
public SoundcloudSearchExtractor(StreamingService service,
SearchQueryHandler linkHandler,
Localization localization) {
super(service, linkHandler, localization);
public SoundcloudSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
super(service, linkHandler);
}
@Override
@ -51,7 +49,8 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
public InfoItemsPage<InfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
final Downloader dl = getDownloader();
try {
searchCollection = JsonParser.object().from(dl.download(pageUrl)).getArray("collection");
final String response = dl.get(pageUrl, getExtractorLocalization()).responseBody();
searchCollection = JsonParser.object().from(response).getArray("collection");
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
@ -64,7 +63,8 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
final Downloader dl = getDownloader();
final String url = getUrl();
try {
searchCollection = JsonParser.object().from(dl.download(url)).getArray("collection");
final String response = dl.get(url, getExtractorLocalization()).responseBody();
searchCollection = JsonParser.object().from(response).getArray("collection");
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}

44
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java
View File

@ -4,7 +4,6 @@ import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -20,17 +19,16 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.utils.Localization;
public class SoundcloudService extends StreamingService {
public SoundcloudService(int id) {
super(id, "SoundCloud", singletonList(AUDIO));
}
@Override
public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler, Localization localization) {
return new SoundcloudSearchExtractor(this, queryHandler, localization);
public String getBaseUrl() {
return "https://soundcloud.com";
}
@Override
@ -55,23 +53,28 @@ public class SoundcloudService extends StreamingService {
@Override
public StreamExtractor getStreamExtractor(LinkHandler LinkHandler, Localization localization) {
return new SoundcloudStreamExtractor(this, LinkHandler, localization);
public StreamExtractor getStreamExtractor(LinkHandler LinkHandler) {
return new SoundcloudStreamExtractor(this, LinkHandler);
}
@Override
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler, Localization localization) {
return new SoundcloudChannelExtractor(this, linkHandler, localization);
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) {
return new SoundcloudChannelExtractor(this, linkHandler);
}
@Override
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler, Localization localization) {
return new SoundcloudPlaylistExtractor(this, linkHandler, localization);
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) {
return new SoundcloudPlaylistExtractor(this, linkHandler);
}
@Override
public SuggestionExtractor getSuggestionExtractor(Localization localization) {
return new SoundcloudSuggestionExtractor(getServiceId(), localization);
public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) {
return new SoundcloudSearchExtractor(this, queryHandler);
}
@Override
public SoundcloudSuggestionExtractor getSuggestionExtractor() {
return new SoundcloudSuggestionExtractor(this);
}
@Override
@ -80,15 +83,14 @@ public class SoundcloudService extends StreamingService {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService,
String url,
String id,
Localization local)
String id)
throws ExtractionException {
return new SoundcloudChartsExtractor(SoundcloudService.this,
new SoundcloudChartsLinkHandlerFactory().fromUrl(url), id, local);
new SoundcloudChartsLinkHandlerFactory().fromUrl(url), id);
}
};
KioskList list = new KioskList(getServiceId());
KioskList list = new KioskList(this);
// add kiosks here e.g.:
final SoundcloudChartsLinkHandlerFactory h = new SoundcloudChartsLinkHandlerFactory();
@ -103,7 +105,6 @@ public class SoundcloudService extends StreamingService {
return list;
}
@Override
public SubscriptionExtractor getSubscriptionExtractor() {
return new SoundcloudSubscriptionExtractor(this);
@ -115,14 +116,9 @@ public class SoundcloudService extends StreamingService {
}
@Override
public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler, Localization localization)
public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler)
throws ExtractionException {
return null;
}
@Override
public String getBaseUrl() {
return "https://soundcloud.com";
}
}

19
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java
View File

@ -4,12 +4,13 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.utils.Localization;
import javax.annotation.Nonnull;
import java.io.IOException;
@ -22,8 +23,8 @@ import java.util.List;
public class SoundcloudStreamExtractor extends StreamExtractor {
private JsonObject track;
public SoundcloudStreamExtractor(StreamingService service, LinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public SoundcloudStreamExtractor(StreamingService service, LinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
@ -50,8 +51,14 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getUploadDate() throws ParsingException {
return SoundcloudParsingHelper.toDateString(track.getString("created_at"));
public String getTextualUploadDate() {
return track.getString("created_at");
}
@Nonnull
@Override
public DateWrapper getUploadDate() throws ParsingException {
return new DateWrapper(SoundcloudParsingHelper.parseDate(getTextualUploadDate()));
}
@Nonnull
@ -139,7 +146,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
String apiUrl = "https://api.soundcloud.com/i1/tracks/" + urlEncode(getId()) + "/streams"
+ "?client_id=" + urlEncode(SoundcloudParsingHelper.clientId());
String response = dl.download(apiUrl);
String response = dl.get(apiUrl, getExtractorLocalization()).responseBody();
JsonObject responseObject;
try {
responseObject = JsonParser.object().from(response);

14
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java
View File

@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.soundcloud;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
@ -41,8 +42,17 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
}
@Override
public String getUploadDate() throws ParsingException {
return SoundcloudParsingHelper.toDateString(itemObject.getString("created_at"));
public String getTextualUploadDate() {
return itemObject.getString("created_at");
}
@Override
public DateWrapper getUploadDate() throws ParsingException {
return new DateWrapper(SoundcloudParsingHelper.parseDate(getTextualUploadDate()));
}
private String getCreatedAt() {
return itemObject.getString("created_at");
}
@Override

12
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java
View File

@ -4,12 +4,12 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.io.IOException;
import java.net.URLEncoder;
@ -20,8 +20,8 @@ public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
public static final String CHARSET_UTF_8 = "UTF-8";
public SoundcloudSuggestionExtractor(int serviceId, Localization localization) {
super(serviceId, localization);
public SoundcloudSuggestionExtractor(StreamingService service) {
super(service);
}
@Override
@ -35,7 +35,7 @@ public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=10";
String response = dl.download(url);
String response = dl.get(url, getExtractorLocalization()).responseBody();
try {
JsonArray collection = JsonParser.object().from(response).getArray("collection");
for (Object suggestion : collection) {

87
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View File

@ -6,8 +6,9 @@ import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCap
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
import java.util.List;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -19,6 +20,8 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
@ -37,7 +40,7 @@ import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLi
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
/*
* Created by Christian Schabesberger on 23.08.15.
@ -66,10 +69,10 @@ public class YoutubeService extends StreamingService {
}
@Override
public SearchExtractor getSearchExtractor(SearchQueryHandler query, Localization localization) {
return new YoutubeSearchExtractor(this, query, localization);
public String getBaseUrl() {
return "https://youtube.com";
}
@Override
public LinkHandlerFactory getStreamLHFactory() {
return YoutubeStreamLinkHandlerFactory.getInstance();
@ -91,28 +94,33 @@ public class YoutubeService extends StreamingService {
}
@Override
public StreamExtractor getStreamExtractor(LinkHandler linkHandler, Localization localization) {
return new YoutubeStreamExtractor(this, linkHandler, localization);
public StreamExtractor getStreamExtractor(LinkHandler linkHandler) {
return new YoutubeStreamExtractor(this, linkHandler);
}
@Override
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler, Localization localization) {
return new YoutubeChannelExtractor(this, linkHandler, localization);
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) {
return new YoutubeChannelExtractor(this, linkHandler);
}
@Override
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler, Localization localization) {
return new YoutubePlaylistExtractor(this, linkHandler, localization);
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) {
return new YoutubePlaylistExtractor(this, linkHandler);
}
@Override
public SuggestionExtractor getSuggestionExtractor(Localization localization) {
return new YoutubeSuggestionExtractor(getServiceId(), localization);
public SearchExtractor getSearchExtractor(SearchQueryHandler query) {
return new YoutubeSearchExtractor(this, query);
}
@Override
public SuggestionExtractor getSuggestionExtractor() {
return new YoutubeSuggestionExtractor(this);
}
@Override
public KioskList getKioskList() throws ExtractionException {
KioskList list = new KioskList(getServiceId());
KioskList list = new KioskList(this);
// add kiosks here e.g.:
try {
@ -120,11 +128,10 @@ public class YoutubeService extends StreamingService {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService,
String url,
String id,
Localization local)
String id)
throws ExtractionException {
return new YoutubeTrendingExtractor(YoutubeService.this,
new YoutubeTrendingLinkHandlerFactory().fromUrl(url), id, local);
new YoutubeTrendingLinkHandlerFactory().fromUrl(url), id);
}
}, new YoutubeTrendingLinkHandlerFactory(), "Trending");
list.setDefaultKiosk("Trending");
@ -146,14 +153,52 @@ public class YoutubeService extends StreamingService {
}
@Override
public CommentsExtractor getCommentsExtractor(ListLinkHandler urlIdHandler, Localization localization)
public CommentsExtractor getCommentsExtractor(ListLinkHandler urlIdHandler)
throws ExtractionException {
return new YoutubeCommentsExtractor(this, urlIdHandler, localization);
return new YoutubeCommentsExtractor(this, urlIdHandler);
}
/*//////////////////////////////////////////////////////////////////////////
// Localization
//////////////////////////////////////////////////////////////////////////*/
// https://www.youtube.com/picker_ajax?action_language_json=1
private static final List<Localization> SUPPORTED_LANGUAGES = Localization.listFrom(
"en-GB"
/*"af", "am", "ar", "az", "be", "bg", "bn", "bs", "ca", "cs", "da", "de",
"el", "en", "en-GB", "es", "es-419", "es-US", "et", "eu", "fa", "fi", "fil", "fr",
"fr-CA", "gl", "gu", "hi", "hr", "hu", "hy", "id", "is", "it", "iw", "ja",
"ka", "kk", "km", "kn", "ko", "ky", "lo", "lt", "lv", "mk", "ml", "mn",
"mr", "ms", "my", "ne", "nl", "no", "pa", "pl", "pt", "pt-PT", "ro", "ru",
"si", "sk", "sl", "sq", "sr", "sr-Latn", "sv", "sw", "ta", "te", "th", "tr",
"uk", "ur", "uz", "vi", "zh-CN", "zh-HK", "zh-TW", "zu"*/
);
// https://www.youtube.com/picker_ajax?action_country_json=1
private static final List<ContentCountry> SUPPORTED_COUNTRIES = ContentCountry.listFrom(
"AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", "BA",
"BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV",
"BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU",
"CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES",
"ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM",
"GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", "ID", "IE",
"IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM",
"KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY",
"MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT",
"MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU",
"NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA",
"RE", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM",
"SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL",
"TM", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE",
"VG", "VI", "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW"
);
@Override
public String getBaseUrl() {
return "https://youtube.com";
public List<Localization> getSupportedLocalizations() {
return SUPPORTED_LANGUAGES;
}
public List<ContentCountry> getSupportedCountries() {
return SUPPORTED_COUNTRIES;
}
}

25
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -1,31 +1,27 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.DonationLinkHelper;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.ArrayList;
/*
* Created by Christian Schabesberger on 25.07.16.
@ -51,18 +47,18 @@ import java.util.ArrayList;
public class YoutubeChannelExtractor extends ChannelExtractor {
/*package-private*/ static final String CHANNEL_URL_BASE = "https://www.youtube.com/channel/";
private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000&gl=US&hl=en";
private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000";
private Document doc;
public YoutubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public YoutubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
String channelUrl = super.getUrl() + CHANNEL_URL_PARAMETERS;
final DownloadResponse response = downloader.get(channelUrl);
final Response response = downloader.get(channelUrl, getExtractorLocalization());
doc = YoutubeParsingHelper.parseAndCheckPage(channelUrl, response);
}
@ -188,7 +184,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
JsonObject ajaxJson;
try {
ajaxJson = JsonParser.object().from(NewPipe.getDownloader().download(pageUrl));
final String response = getDownloader().get(pageUrl, getExtractorLocalization()).responseBody();
ajaxJson = JsonParser.object().from(response);
} catch (JsonParserException pe) {
throw new ParsingException("Could not parse json data for next streams", pe);
}
@ -228,9 +225,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final String uploaderName = getName();
final String uploaderUrl = getUrl();
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (final Element li : element.children()) {
if (li.select("div[class=\"feed-item-dismissable\"]").first() != null) {
collector.commit(new YoutubeStreamInfoItemExtractor(li) {
collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) {
@Override
public String getUrl() throws ParsingException {
try {

61
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java
View File

@ -1,36 +1,31 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import javax.annotation.Nonnull;
import org.schabi.newpipe.extractor.DownloadRequest;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemsCollector;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Parser;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.*;
import java.util.regex.Pattern;
import static java.util.Collections.singletonList;
public class YoutubeCommentsExtractor extends CommentsExtractor {
@ -44,8 +39,8 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
private String title;
private InfoItemsPage<CommentsInfoItem> initPage;
public YoutubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler, Localization localization) {
super(service, uiHandler, localization);
public YoutubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) {
super(service, uiHandler);
}
@Override
@ -130,7 +125,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
for(Object c: comments) {
if(c instanceof JsonObject) {
CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor((JsonObject) c, getUrl());
CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor((JsonObject) c, getUrl(), getTimeAgoParser());
collector.commit(extractor);
}
}
@ -147,12 +142,11 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
}
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
Map<String, List<String>> requestHeaders = new HashMap<>();
requestHeaders.put("User-Agent", Arrays.asList(USER_AGENT));
DownloadRequest request = new DownloadRequest(null, requestHeaders);
DownloadResponse response = downloader.get(getUrl(), request);
String responseBody = response.getResponseBody();
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final Map<String, List<String>> requestHeaders = new HashMap<>();
requestHeaders.put("User-Agent", singletonList(USER_AGENT));
final Response response = downloader.get(getUrl(), requestHeaders, getExtractorLocalization());
String responseBody = response.responseBody();
ytClientVersion = findValue(responseBody, "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"", "\"");
ytClientName = Parser.matchGroup1(YT_CLIENT_NAME_PATTERN, responseBody);
String commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}");
@ -160,6 +154,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
initPage = getPage(getNextPageUrl(commentsToken));
}
@Nonnull
@Override
public String getName() throws ParsingException {
return title;
@ -168,13 +163,11 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
private String makeAjaxRequest(String siteUrl) throws IOException, ReCaptchaException {
Map<String, List<String>> requestHeaders = new HashMap<>();
requestHeaders.put("Accept", Arrays.asList("*/*"));
requestHeaders.put("User-Agent", Arrays.asList(USER_AGENT));
requestHeaders.put("X-YouTube-Client-Version", Arrays.asList(ytClientVersion));
requestHeaders.put("X-YouTube-Client-Name", Arrays.asList(ytClientName));
DownloadRequest request = new DownloadRequest(null, requestHeaders);
return NewPipe.getDownloader().get(siteUrl, request).getResponseBody();
requestHeaders.put("Accept", singletonList("*/*"));
requestHeaders.put("User-Agent", singletonList(USER_AGENT));
requestHeaders.put("X-YouTube-Client-Version", singletonList(ytClientVersion));
requestHeaders.put("X-YouTube-Client-Name", singletonList(ytClientName));
return getDownloader().get(siteUrl, requestHeaders, getExtractorLocalization()).responseBody();
}
private String getDataString(Map<String, String> params) throws UnsupportedEncodingException {

32
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java
View File

@ -1,21 +1,26 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable;
public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
private final JsonObject json;
private final String url;
private final TimeAgoParser timeAgoParser;
public YoutubeCommentsInfoItemExtractor(JsonObject json, String url) {
public YoutubeCommentsInfoItemExtractor(JsonObject json, String url, TimeAgoParser timeAgoParser) {
this.json = json;
this.url = url;
this.timeAgoParser = timeAgoParser;
}
@Override
@ -43,7 +48,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
}
@Override
public String getPublishedTime() throws ParsingException {
public String getTextualPublishedTime() throws ParsingException {
try {
return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "publishedTimeText"));
} catch (Exception e) {
@ -51,8 +56,19 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
}
}
@Nullable
@Override
public Integer getLikeCount() throws ParsingException {
public DateWrapper getPublishedTime() throws ParsingException {
String textualPublishedTime = getTextualPublishedTime();
if (timeAgoParser != null && textualPublishedTime != null && !textualPublishedTime.isEmpty()) {
return timeAgoParser.parse(textualPublishedTime);
} else {
return null;
}
}
@Override
public int getLikeCount() throws ParsingException {
try {
return JsonUtils.getNumber(json, "likeCount").intValue();
} catch (Exception e) {

21
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -6,19 +6,19 @@ import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
@ -30,14 +30,14 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
private Document doc;
public YoutubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public YoutubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String url = getUrl();
final DownloadResponse response = downloader.get(url);
final Response response = downloader.get(url, getExtractorLocalization());
doc = YoutubeParsingHelper.parseAndCheckPage(url, response);
}
@ -141,7 +141,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
JsonObject pageJson;
try {
pageJson = JsonParser.object().from(getDownloader().download(pageUrl));
final String responseBody = getDownloader().get(pageUrl, getExtractorLocalization()).responseBody();
pageJson = JsonParser.object().from(responseBody);
} catch (JsonParserException pe) {
throw new ParsingException("Could not parse ajax json", pe);
}
@ -187,12 +188,14 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
final LinkHandlerFactory streamLinkHandlerFactory = getService().getStreamLHFactory();
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (final Element li : element.children()) {
if(isDeletedItem(li)) {
continue;
}
collector.commit(new YoutubeStreamInfoItemExtractor(li) {
collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) {
public Element uploaderLink;
@Override
@ -258,7 +261,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
@Override
public String getUploadDate() throws ParsingException {
public String getTextualUploadDate() throws ParsingException {
return "";
}

26
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -3,16 +3,16 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.Parser;
@ -46,22 +46,21 @@ public class YoutubeSearchExtractor extends SearchExtractor {
private Document doc;
public YoutubeSearchExtractor(StreamingService service,
SearchQueryHandler linkHandler,
Localization localization) {
super(service, linkHandler, localization);
public YoutubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String url = getUrl();
final DownloadResponse response = downloader.get(url, getLocalization());
final Response response = downloader.get(url, getExtractorLocalization());
doc = YoutubeParsingHelper.parseAndCheckPage(url, response);
}
@Nonnull
@Override
public String getUrl() throws ParsingException {
return super.getUrl() + "&gl="+ getLocalization().getCountry();
return super.getUrl() + "&gl=" + getExtractorContentCountry().getCountryCode();
}
@Override
@ -87,8 +86,8 @@ public class YoutubeSearchExtractor extends SearchExtractor {
@Override
public InfoItemsPage<InfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
String site = getDownloader().download(pageUrl);
doc = Jsoup.parse(site, pageUrl);
final String response = getDownloader().get(pageUrl, getExtractorLocalization()).responseBody();
doc = Jsoup.parse(response, pageUrl);
return new InfoItemsPage<>(collectItems(doc), getNextPageUrlFromCurrentUrl(pageUrl));
}
@ -109,6 +108,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
InfoItemsSearchCollector collector = getInfoItemSearchCollector();
Element list = doc.select("ol[class=\"item-section\"]").first();
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Element item : list.children()) {
/* First we need to determine which kind of item we are working with.
@ -129,7 +129,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
// video item type
} else if ((el = item.select("div[class*=\"yt-lockup-video\"]").first()) != null) {
collector.commit(new YoutubeStreamInfoItemExtractor(el));
collector.commit(new YoutubeStreamInfoItemExtractor(el, timeAgoParser));
} else if ((el = item.select("div[class*=\"yt-lockup-channel\"]").first()) != null) {
collector.commit(new YoutubeChannelInfoItemExtractor(el));
} else if ((el = item.select("div[class*=\"yt-lockup-playlist\"]").first()) != null &&

123
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -11,16 +11,23 @@ import org.jsoup.select.Elements;
import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Request;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
@ -87,8 +94,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private boolean isAgeRestricted;
public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) {
super(service, linkHandler);
}
/*//////////////////////////////////////////////////////////////////////////
@ -114,10 +121,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return name;
}
@Nonnull
@Override
public String getUploadDate() throws ParsingException {
assertPageFetched();
public String getTextualUploadDate() throws ParsingException {
if (getStreamType().equals(StreamType.LIVE_STREAM)) {
return null;
}
try {
return doc.select("meta[itemprop=datePublished]").attr(CONTENT);
} catch (Exception e) {//todo: add fallback method
@ -125,6 +134,17 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
@Override
public DateWrapper getUploadDate() throws ParsingException {
final String textualUploadDate = getTextualUploadDate();
if (textualUploadDate == null) {
return null;
}
return new DateWrapper(YoutubeParsingHelper.parseDateFrom(textualUploadDate));
}
@Nonnull
@Override
public String getThumbnailUrl() throws ParsingException {
@ -285,12 +305,65 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public long getViewCount() throws ParsingException {
assertPageFetched();
try {
if (getStreamType().equals(StreamType.LIVE_STREAM)) {
return getLiveStreamWatchingCount();
}
return Long.parseLong(doc.select("meta[itemprop=interactionCount]").attr(CONTENT));
} catch (Exception e) {//todo: find fallback method
throw new ParsingException("Could not get number of views", e);
}
}
private long getLiveStreamWatchingCount() throws ExtractionException, IOException, JsonParserException {
// https://www.youtube.com/youtubei/v1/updated_metadata?alt=json&key=
String innerTubeKey = null, clientVersion = null;
if (playerArgs != null && !playerArgs.isEmpty()) {
innerTubeKey = playerArgs.getString("innertube_api_key");
clientVersion = playerArgs.getString("innertube_context_client_version");
} else if (!videoInfoPage.isEmpty()) {
innerTubeKey = videoInfoPage.get("innertube_api_key");
clientVersion = videoInfoPage.get("innertube_context_client_version");
}
if (innerTubeKey == null || innerTubeKey.isEmpty()) {
throw new ExtractionException("Couldn't get innerTube key");
}
if (clientVersion == null || clientVersion.isEmpty()) {
throw new ExtractionException("Couldn't get innerTube client version");
}
final String metadataUrl = "https://www.youtube.com/youtubei/v1/updated_metadata?alt=json&key=" + innerTubeKey;
final byte[] dataBody = ("{\"context\":{\"client\":{\"clientName\":1,\"clientVersion\":\"" + clientVersion + "\"}}" +
",\"videoId\":\"" + getId() + "\"}").getBytes("UTF-8");
final Response response = getDownloader().execute(Request.newBuilder()
.post(metadataUrl, dataBody)
.addHeader("Content-Type", "application/json")
.build());
final JsonObject jsonObject = JsonParser.object().from(response.responseBody());
for (Object actionEntry : jsonObject.getArray("actions")) {
if (!(actionEntry instanceof JsonObject)) continue;
final JsonObject entry = (JsonObject) actionEntry;
final JsonObject updateViewershipAction = entry.getObject("updateViewershipAction", null);
if (updateViewershipAction == null) continue;
final JsonArray viewCountRuns = JsonUtils.getArray(updateViewershipAction, "viewership.videoViewCountRenderer.viewCount.runs");
if (viewCountRuns.isEmpty()) continue;
final JsonObject textObject = viewCountRuns.getObject(0);
if (!textObject.has("text")) {
throw new ExtractionException("Response don't have \"text\" element");
}
return Long.parseLong(Utils.removeNonDigitCharacters(textObject.getString("text")));
}
throw new ExtractionException("Could not find correct results in response");
}
@Override
public long getLikeCount() throws ParsingException {
assertPageFetched();
@ -532,13 +605,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
assertPageFetched();
try {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final TimeAgoParser timeAgoParser = getTimeAgoParser();
Elements watch = doc.select("div[class=\"watch-sidebar-section\"]");
if (watch.size() < 1) {
return null;// prevent the snackbar notification "report error" on age-restricted videos
}
collector.commit(extractVideoPreviewInfo(watch.first().select("li").first()));
collector.commit(extractVideoPreviewInfo(watch.first().select("li").first(), timeAgoParser));
return collector.getItems().get(0);
} catch (Exception e) {
throw new ParsingException("Could not get next video", e);
@ -550,12 +624,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
assertPageFetched();
try {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final TimeAgoParser timeAgoParser = getTimeAgoParser();
Element ul = doc.select("ul[id=\"watch-related\"]").first();
if (ul != null) {
for (Element li : ul.children()) {
// first check if we have a playlist. If so leave them out
if (li.select("a[class*=\"content-link\"]").first() != null) {
collector.commit(extractVideoPreviewInfo(li));
collector.commit(extractVideoPreviewInfo(li, timeAgoParser));
}
}
}
@ -615,8 +691,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String verifiedUrl = getUrl() + VERIFIED_URL_PARAMS;
final DownloadResponse response = downloader.get(verifiedUrl);
pageHtml = response.getResponseBody();
final Response response = downloader.get(verifiedUrl, getExtractorLocalization());
pageHtml = response.responseBody();
doc = YoutubeParsingHelper.parseAndCheckPage(verifiedUrl, response);
final String playerUrl;
@ -624,7 +700,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (!doc.select("meta[property=\"og:restrictions:age\"").isEmpty()) {
final EmbeddedInfo info = getEmbeddedInfo();
final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts);
final String infoPageResponse = downloader.download(videoInfoUrl);
final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody();
videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
playerUrl = info.url;
isAgeRestricted = true;
@ -713,7 +789,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try {
final Downloader downloader = NewPipe.getDownloader();
final String embedUrl = "https://www.youtube.com/embed/" + getId();
final String embedPageContent = downloader.download(embedUrl);
final String embedPageContent = downloader.get(embedUrl, getExtractorLocalization()).responseBody();
// Get player url
final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")";
@ -748,7 +824,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
playerUrl = "https://youtube.com" + playerUrl;
}
final String playerCode = downloader.download(playerUrl);
final String playerCode = downloader.get(playerUrl, getExtractorLocalization()).responseBody();
final String decryptionFunctionName = getDecryptionFuncName(playerCode);
final String functionPattern = "("
@ -931,8 +1007,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
* Provides information about links to other videos on the video page, such as related videos.
* This is encapsulated in a StreamInfoItem object, which is a subset of the fields in a full StreamInfo.
*/
private StreamInfoItemExtractor extractVideoPreviewInfo(final Element li) {
return new YoutubeStreamInfoItemExtractor(li) {
private StreamInfoItemExtractor extractVideoPreviewInfo(final Element li, final TimeAgoParser timeAgoParser) {
return new YoutubeStreamInfoItemExtractor(li, timeAgoParser) {
@Override
public String getUrl() throws ParsingException {
@ -959,23 +1035,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
@Override
public String getUploadDate() throws ParsingException {
public String getTextualUploadDate() throws ParsingException {
return "";
}
@Override
public long getViewCount() throws ParsingException {
try {
if (getStreamType() == StreamType.LIVE_STREAM) return -1;
return Long.parseLong(Utils.removeNonDigitCharacters(
li.select("span.view-count").first().text()));
} catch (Exception e) {
//related videos sometimes have no view count
return 0;
}
}
@Override
public String getThumbnailUrl() throws ParsingException {
Element img = li.select("img").first();

127
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java
View File

@ -1,12 +1,20 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
/*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* YoutubeStreamInfoItemExtractor.java is part of NewPipe.
@ -28,9 +36,18 @@ import org.schabi.newpipe.extractor.utils.Utils;
public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
private final Element item;
private final TimeAgoParser timeAgoParser;
public YoutubeStreamInfoItemExtractor(Element item) {
private String cachedUploadDate;
/**
* Creates an extractor of StreamInfoItems from a YouTube page.
* @param item The page element
* @param timeAgoParser A parser of the textual dates or {@code null}.
*/
public YoutubeStreamInfoItemExtractor(Element item, @Nullable TimeAgoParser timeAgoParser) {
this.item = item;
this.timeAgoParser = timeAgoParser;
}
@Override
@ -125,42 +142,94 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
}
}
@Nullable
@Override
public String getUploadDate() throws ParsingException {
public String getTextualUploadDate() throws ParsingException {
if (getStreamType().equals(StreamType.LIVE_STREAM)) {
return null;
}
if (cachedUploadDate != null) {
return cachedUploadDate;
}
try {
if (isVideoReminder()) {
final Calendar calendar = getDateFromReminder();
if (calendar != null) {
return cachedUploadDate = new SimpleDateFormat("yyyy-MM-dd HH:mm")
.format(calendar.getTime());
}
}
Element meta = item.select("div[class=\"yt-lockup-meta\"]").first();
if (meta == null) return "";
Element li = meta.select("li").first();
if(li == null) return "";
final Elements li = meta.select("li");
if (li.isEmpty()) return "";
return meta.select("li").first().text();
return cachedUploadDate = li.first().text();
} catch (Exception e) {
throw new ParsingException("Could not get upload date", e);
}
}
@Nullable
@Override
public DateWrapper getUploadDate() throws ParsingException {
if (getStreamType().equals(StreamType.LIVE_STREAM)) {
return null;
}
if (isVideoReminder()) {
return new DateWrapper(getDateFromReminder());
}
String textualUploadDate = getTextualUploadDate();
if (timeAgoParser != null && textualUploadDate != null && !textualUploadDate.isEmpty()) {
return timeAgoParser.parse(textualUploadDate);
} else {
return null;
}
}
@Override
public long getViewCount() throws ParsingException {
String input;
try {
// TODO: Return the actual live stream's watcher count
// -1 for no view count
if (getStreamType() == StreamType.LIVE_STREAM) return -1;
Element meta = item.select("div[class=\"yt-lockup-meta\"]").first();
if (meta == null) return -1;
final Element spanViewCount = item.select("span.view-count").first();
if (spanViewCount != null) {
input = spanViewCount.text();
// This case can happen if google releases a special video
if(meta.select("li").size() < 2) return -1;
} else if (getStreamType().equals(StreamType.LIVE_STREAM)) {
Element meta = item.select("ul.yt-lockup-meta-info").first();
if (meta == null) return 0;
input = meta.select("li").get(1).text();
final Elements li = meta.select("li");
if (li.isEmpty()) return 0;
} catch (IndexOutOfBoundsException e) {
throw new ParsingException("Could not parse yt-lockup-meta although available: " + getUrl(), e);
input = li.first().text();
} else {
try {
Element meta = item.select("div.yt-lockup-meta").first();
if (meta == null) return -1;
// This case can happen if google releases a special video
if (meta.select("li").size() < 2) return -1;
input = meta.select("li").get(1).text();
} catch (IndexOutOfBoundsException e) {
throw new ParsingException("Could not parse yt-lockup-meta although available: " + getUrl(), e);
}
}
if (input == null) {
throw new ParsingException("Input is null");
}
try {
return Long.parseLong(Utils.removeNonDigitCharacters(input));
} catch (NumberFormatException e) {
// if this happens the video probably has no views
@ -191,6 +260,32 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
}
}
private boolean isVideoReminder() {
return !item.select("span.yt-uix-livereminder").isEmpty();
}
private Calendar getDateFromReminder() throws ParsingException {
final Element timeFuture = item.select("span.yt-badge.localized-date").first();
if (timeFuture == null) {
throw new ParsingException("Span timeFuture is null");
}
final String timestamp = timeFuture.attr("data-timestamp");
if (!timestamp.isEmpty()) {
try {
final Calendar calendar = Calendar.getInstance();
calendar.setTime(new Date(Long.parseLong(timestamp) * 1000L));
return calendar;
} catch (Exception e) {
throw new ParsingException("Could not parse = \"" + timestamp + "\"");
}
}
throw new ParsingException("Could not parse date from reminder element: \"" + timeFuture + "\"");
}
/**
* Generic method that checks if the element contains any clues that it's a livestream item
*/

14
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java
View File

@ -3,12 +3,12 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.io.IOException;
import java.net.URLEncoder;
@ -39,8 +39,8 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
public static final String CHARSET_UTF_8 = "UTF-8";
public YoutubeSuggestionExtractor(int serviceId, Localization localization) {
super(serviceId, localization);
public YoutubeSuggestionExtractor(StreamingService service) {
super(service);
}
@Override
@ -52,10 +52,10 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
+ "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml
+ "&jsonp=" + "JP"
+ "&ds=" + "yt"
+ "&hl=" + URLEncoder.encode(getLocalization().getCountry(), CHARSET_UTF_8)
+ "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), CHARSET_UTF_8)
+ "&q=" + URLEncoder.encode(query, CHARSET_UTF_8);
String response = dl.download(url);
String response = dl.get(url, getExtractorLocalization()).responseBody();
// trim JSONP part "JP(...)"
response = response.substring(3, response.length()-1);
try {

28
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View File

@ -20,21 +20,20 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Localization;
import javax.annotation.Nonnull;
import java.io.IOException;
@ -45,20 +44,16 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
public YoutubeTrendingExtractor(StreamingService service,
ListLinkHandler linkHandler,
String kioskId,
Localization localization) {
super(service, linkHandler, kioskId, localization);
String kioskId) {
super(service, linkHandler, kioskId);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String contentCountry = getLocalization().getCountry();
String url = getUrl();
if(contentCountry != null && !contentCountry.isEmpty()) {
url += "?gl=" + contentCountry;
}
final String url = getUrl() +
"?gl=" + getExtractorContentCountry().getCountryCode();
final DownloadResponse response = downloader.get(url);
final Response response = downloader.get(url, getExtractorLocalization());
doc = YoutubeParsingHelper.parseAndCheckPage(url, response);
}
@ -90,10 +85,13 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Elements uls = doc.select("ul[class*=\"expanded-shelf-content-list\"]");
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for(Element ul : uls) {
for(final Element li : ul.children()) {
final Element el = li.select("div[class*=\"yt-lockup-dismissable\"]").first();
collector.commit(new YoutubeStreamInfoItemExtractor(li) {
collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) {
@Override
public String getUrl() throws ParsingException {
try {

23
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java
View File

@ -3,11 +3,15 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import java.net.URL;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
/*
* Created by Christian Schabesberger on 02.03.16.
@ -39,8 +43,8 @@ public class YoutubeParsingHelper {
"input[name*=\"action_recaptcha_verify\"]"
};
public static Document parseAndCheckPage(final String url, final DownloadResponse response) throws ReCaptchaException {
final Document document = Jsoup.parse(response.getResponseBody(), url);
public static Document parseAndCheckPage(final String url, final Response response) throws ReCaptchaException {
final Document document = Jsoup.parse(response.responseBody(), url);
for (String detectionSelector : RECAPTCHA_DETECTION_SELECTORS) {
if (!document.select(detectionSelector).isEmpty()) {
@ -113,4 +117,17 @@ public class YoutubeParsingHelper {
+ Long.parseLong(minutes)) * 60)
+ Long.parseLong(seconds);
}
public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
Date date;
try {
date = new SimpleDateFormat("yyyy-MM-dd").parse(textualUploadDate);
} catch (ParseException e) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
}
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
}
}

37
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java
View File

@ -26,13 +26,12 @@ import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@ -43,18 +42,36 @@ public abstract class StreamExtractor extends Extractor {
public static final int NO_AGE_LIMIT = 0;
public StreamExtractor(StreamingService service, LinkHandler linkHandler, Localization localization) {
super(service, linkHandler, localization);
public StreamExtractor(StreamingService service, LinkHandler linkHandler) {
super(service, linkHandler);
}
/**
* The day on which the stream got uploaded/created. The return information should be in the format
* dd.mm.yyyy, however it NewPipe will not crash if its not.
* @return The day on which the stream got uploaded.
* @throws ParsingException
* The original textual date provided by the service. Should be used as a fallback if
* {@link #getUploadDate()} isn't provided by the service, or it fails for some reason.
*
* <p>If the stream is a live stream, {@code null} should be returned.</p>
*
* @return The original textual date provided by the service, or {@code null}.
* @throws ParsingException if there is an error in the extraction
* @see #getUploadDate()
*/
@Nonnull
public abstract String getUploadDate() throws ParsingException;
@Nullable
public abstract String getTextualUploadDate() throws ParsingException;
/**
* A more general {@code Calendar} instance set to the date provided by the service.<br>
* Implementations usually will just parse the date returned from the {@link #getTextualUploadDate()}.
*
* <p>If the stream is a live stream, {@code null} should be returned.</p>
*
* @return The date this item was uploaded, or {@code null}.
* @throws ParsingException if there is an error in the extraction
* or the extracted date couldn't be parsed.
* @see #getTextualUploadDate()
*/
@Nullable
public abstract DateWrapper getUploadDate() throws ParsingException;
/**
* This will return the url to the thumbnail of the stream. Try to return the medium resolution here.

29
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java
View File

@ -1,18 +1,19 @@
package org.schabi.newpipe.extractor.stream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.schabi.newpipe.extractor.Info;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.utils.DashMpdParser;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/*
* Created by Christian Schabesberger on 26.08.15.
*
@ -228,6 +229,11 @@ public class StreamInfo extends Info {
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setTextualUploadDate(extractor.getTextualUploadDate());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setUploadDate(extractor.getUploadDate());
} catch (Exception e) {
@ -271,7 +277,8 @@ public class StreamInfo extends Info {
private StreamType streamType;
private String thumbnailUrl = "";
private String uploadDate = "";
private String textualUploadDate;
private DateWrapper uploadDate;
private long duration = -1;
private int ageLimit = -1;
private String description;
@ -327,11 +334,19 @@ public class StreamInfo extends Info {
this.thumbnailUrl = thumbnailUrl;
}
public String getUploadDate() {
public String getTextualUploadDate() {
return textualUploadDate;
}
public void setTextualUploadDate(String textualUploadDate) {
this.textualUploadDate = textualUploadDate;
}
public DateWrapper getUploadDate() {
return uploadDate;
}
public void setUploadDate(String uploadDate) {
public void setUploadDate(DateWrapper uploadDate) {
this.uploadDate = uploadDate;
}

34
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java
View File

@ -21,6 +21,9 @@ package org.schabi.newpipe.extractor.stream;
*/
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import javax.annotation.Nullable;
/**
* Info object for previews of unopened videos, eg search results, related videos
@ -29,7 +32,8 @@ public class StreamInfoItem extends InfoItem {
private final StreamType streamType;
private String uploaderName;
private String uploadDate;
private String textualUploadDate;
@Nullable private DateWrapper uploadDate;
private long viewCount = -1;
private long duration = -1;
@ -52,14 +56,6 @@ public class StreamInfoItem extends InfoItem {
this.uploaderName = uploader_name;
}
public String getUploadDate() {
return uploadDate;
}
public void setUploadDate(String upload_date) {
this.uploadDate = upload_date;
}
public long getViewCount() {
return viewCount;
}
@ -84,12 +80,30 @@ public class StreamInfoItem extends InfoItem {
this.uploaderUrl = uploaderUrl;
}
@Nullable
public String getTextualUploadDate() {
return textualUploadDate;
}
public void setTextualUploadDate(String uploadDate) {
this.textualUploadDate = uploadDate;
}
@Nullable
public DateWrapper getUploadDate() {
return uploadDate;
}
public void setUploadDate(@Nullable DateWrapper uploadDate) {
this.uploadDate = uploadDate;
}
@Override
public String toString() {
return "StreamInfoItem{" +
"streamType=" + streamType +
", uploaderName='" + uploaderName + '\'' +
", uploadDate='" + uploadDate + '\'' +
", textualUploadDate='" + textualUploadDate + '\'' +
", viewCount=" + viewCount +
", duration=" + duration +
", uploaderUrl='" + uploaderUrl + '\'' +

31
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java
View File

@ -2,6 +2,9 @@ package org.schabi.newpipe.extractor.stream;
import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import javax.annotation.Nullable;
/*
* Created by Christian Schabesberger on 28.02.16.
@ -64,10 +67,30 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor {
String getUploaderUrl() throws ParsingException;
/**
* Extract the uploader name
* @return the uploader name
* @throws ParsingException thrown if there is an error in the extraction
* The original textual date provided by the service. Should be used as a fallback if
* {@link #getUploadDate()} isn't provided by the service, or it fails for some reason.
*
* @return The original textual date provided by the service or {@code null} if not provided.
* @throws ParsingException if there is an error in the extraction
* @see #getUploadDate()
*/
String getUploadDate() throws ParsingException;
@Nullable
String getTextualUploadDate() throws ParsingException;
/**
* Extracts the upload date and time of this item and parses it.
* <p>
* If the service doesn't provide an exact time, an approximation can be returned.
* <br>
* If the service doesn't provide any date at all, then {@code null} should be returned.
* </p>
*
* @return The date and time (can be approximated) this item was uploaded or {@code null}.
* @throws ParsingException if there is an error in the extraction
* or the extracted date couldn't be parsed.
* @see #getTextualUploadDate()
*/
@Nullable
DateWrapper getUploadDate() throws ParsingException;
}

7
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemsCollector.java
View File

@ -61,10 +61,15 @@ public class StreamInfoItemsCollector extends InfoItemsCollector<StreamInfoItem,
addError(e);
}
try {
resultItem.setUploadDate(extractor.getUploadDate());
resultItem.setTextualUploadDate(extractor.getTextualUploadDate());
} catch (Exception e) {
addError(e);
}
try {
resultItem.setUploadDate(extractor.getUploadDate());
} catch (ParsingException e) {
addError(e);
}
try {
resultItem.setViewCount(extractor.getViewCount());
} catch (Exception e) {

51
extractor/src/main/java/org/schabi/newpipe/extractor/suggestion/SuggestionExtractor.java Normal file
View File

@ -0,0 +1,51 @@
package org.schabi.newpipe.extractor.suggestion;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.List;
public abstract class SuggestionExtractor {
private final StreamingService service;
@Nullable private Localization forcedLocalization;
@Nullable private ContentCountry forcedContentCountry;
public SuggestionExtractor(StreamingService service) {
this.service = service;
}
public abstract List<String> suggestionList(String query) throws IOException, ExtractionException;
public int getServiceId() {
return service.getServiceId();
}
public StreamingService getService() {
return service;
}
// TODO: Create a more general Extractor class
public void forceLocalization(@Nullable Localization localization) {
this.forcedLocalization = localization;
}
public void forceContentCountry(@Nullable ContentCountry contentCountry) {
this.forcedContentCountry = contentCountry;
}
@Nonnull
public Localization getExtractorLocalization() {
return forcedLocalization == null ? getService().getLocalization() : forcedLocalization;
}
@Nonnull
public ContentCountry getExtractorContentCountry() {
return forcedContentCountry == null ? getService().getContentCountry() : forcedContentCountry;
}
}

4
extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java
View File

@ -1,6 +1,6 @@
package org.schabi.newpipe.extractor.utils;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -120,7 +120,7 @@ public class DashMpdParser {
String dashDoc;
Downloader downloader = NewPipe.getDownloader();
try {
dashDoc = downloader.download(streamInfo.getDashMpdUrl());
dashDoc = downloader.get(streamInfo.getDashMpdUrl()).responseBody();
} catch (IOException ioe) {
throw new DashMpdParsingException("Could not get dash mpd: " + streamInfo.getDashMpdUrl(), ioe);
}

19
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Localization.java
View File

@ -1,19 +0,0 @@
package org.schabi.newpipe.extractor.utils;
public class Localization {
private final String country;
private final String language;
public Localization(String country, String language) {
this.country = country;
this.language = language;
}
public String getCountry() {
return country;
}
public String getLanguage() {
return language;
}
}

256
extractor/src/test/java/org/schabi/newpipe/Downloader.java
View File

@ -1,256 +0,0 @@
package org.schabi.newpipe;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.net.ssl.HttpsURLConnection;
import org.schabi.newpipe.extractor.DownloadRequest;
import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.utils.Localization;
import static java.util.Collections.singletonList;
/*
* Created by Christian Schabesberger on 28.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* Downloader.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class Downloader implements org.schabi.newpipe.extractor.Downloader {
private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0";
private static String mCookies = "";
private static Downloader instance = null;
private Downloader() {
}
public static Downloader getInstance() {
if (instance == null) {
synchronized (Downloader.class) {
if (instance == null) {
instance = new Downloader();
}
}
}
return instance;
}
public static synchronized void setCookies(String cookies) {
Downloader.mCookies = cookies;
}
public static synchronized String getCookies() {
return Downloader.mCookies;
}
/**
* Download the text file at the supplied URL as in download(String), but set
* the HTTP header field "Accept-Language" to the supplied string.
*
* @param siteUrl the URL of the text file to return the contents of
* @param localization the language and country (usually a 2-character code for both values)
* @return the contents of the specified text file
*/
public String download(String siteUrl, Localization localization) throws IOException, ReCaptchaException {
Map<String, String> requestProperties = new HashMap<>();
requestProperties.put("Accept-Language", localization.getLanguage());
return download(siteUrl, requestProperties);
}
/**
* Download the text file at the supplied URL as in download(String), but set
* the HTTP header field "Accept-Language" to the supplied string.
*
* @param siteUrl the URL of the text file to return the contents of
* @param customProperties set request header properties
* @return the contents of the specified text file
* @throws IOException
*/
public String download(String siteUrl, Map<String, String> customProperties)
throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
for (Map.Entry<String, String> pair : customProperties.entrySet()) {
con.setRequestProperty(pair.getKey(), pair.getValue());
}
return dl(con);
}
/**
* Common functionality between download(String url) and download(String url,
* String language)
*/
private static String dl(HttpsURLConnection con) throws IOException, ReCaptchaException {
StringBuilder response = new StringBuilder();
BufferedReader in = null;
try {
con.setRequestMethod("GET");
setDefaults(con);
in = new BufferedReader(new InputStreamReader(con.getInputStream()));
String inputLine;
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
} catch (UnknownHostException uhe) {// thrown when there's no internet
// connection
throw new IOException("unknown host or no network", uhe);
// Toast.makeText(getActivity(), uhe.getMessage(),
// Toast.LENGTH_LONG).show();
} catch (Exception e) {
/*
* HTTP 429 == Too Many Request Receive from Youtube.com = ReCaptcha challenge
* request See : https://github.com/rg3/youtube-dl/issues/5138
*/
if (con.getResponseCode() == 429) {
throw new ReCaptchaException("reCaptcha Challenge requested", con.getURL().toString());
}
throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e);
} finally {
if (in != null) {
in.close();
}
}
return response.toString();
}
private static void setDefaults(HttpsURLConnection con) {
con.setConnectTimeout(30 * 1000);// 30s
con.setReadTimeout(30 * 1000);// 30s
// set default user agent
if (null == con.getRequestProperty("User-Agent")) {
con.setRequestProperty("User-Agent", USER_AGENT);
}
// add default cookies
if (getCookies().length() > 0) {
con.addRequestProperty("Cookie", getCookies());
}
}
/**
* Download (via HTTP) the text file located at the supplied URL, and return its
* contents. Primarily intended for downloading web pages.
*
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file
*/
public String download(String siteUrl) throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
// HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
return dl(con);
}
@Override
public DownloadResponse head(String siteUrl) throws IOException, ReCaptchaException {
final HttpsURLConnection con = (HttpsURLConnection) new URL(siteUrl).openConnection();
try {
con.setRequestMethod("HEAD");
setDefaults(con);
} catch (Exception e) {
/*
* HTTP 429 == Too Many Request Receive from Youtube.com = ReCaptcha challenge
* request See : https://github.com/rg3/youtube-dl/issues/5138
*/
if (con.getResponseCode() == 429) {
throw new ReCaptchaException("reCaptcha Challenge requested", con.getURL().toString());
}
throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e);
}
return new DownloadResponse(con.getResponseCode(), null, con.getHeaderFields());
}
@Override
public DownloadResponse get(String siteUrl, Localization localization) throws IOException, ReCaptchaException {
final Map<String, List<String>> requestHeaders = new HashMap<>();
requestHeaders.put("Accept-Language", singletonList(localization.getLanguage()));
return get(siteUrl, new DownloadRequest(null, requestHeaders));
}
@Override
public DownloadResponse get(String siteUrl, DownloadRequest request)
throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
for (Map.Entry<String, List<String>> pair : request.getRequestHeaders().entrySet()) {
for(String value: pair.getValue()) {
con.addRequestProperty(pair.getKey(), value);
}
}
String responseBody = dl(con);
return new DownloadResponse(con.getResponseCode(), responseBody, con.getHeaderFields());
}
@Override
public DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException {
return get(siteUrl, DownloadRequest.emptyRequest);
}
@Override
public DownloadResponse post(String siteUrl, DownloadRequest request)
throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
con.setRequestMethod("POST");
for (Map.Entry<String, List<String>> pair : request.getRequestHeaders().entrySet()) {
for(String value: pair.getValue()) {
con.addRequestProperty(pair.getKey(), value);
}
}
// set fields to default if not set already
setDefaults(con);
if(null != request.getRequestBody()) {
byte[] postDataBytes = request.getRequestBody().getBytes("UTF-8");
con.setRequestProperty("Content-Length", String.valueOf(postDataBytes.length));
con.setDoOutput(true);
con.getOutputStream().write(postDataBytes);
}
StringBuilder sb = new StringBuilder();
try (BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()))) {
String inputLine;
while ((inputLine = in.readLine()) != null) {
sb.append(inputLine);
}
}
return new DownloadResponse(con.getResponseCode(), sb.toString(), con.getHeaderFields());
}
}

120
extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java Normal file
View File

@ -0,0 +1,120 @@
package org.schabi.newpipe;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Request;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.Localization;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.net.ssl.HttpsURLConnection;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.List;
import java.util.Map;
public class DownloaderTestImpl extends Downloader {
private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0";
private static final String DEFAULT_HTTP_ACCEPT_LANGUAGE = "en";
private static DownloaderTestImpl instance = null;
private DownloaderTestImpl() {
}
public static DownloaderTestImpl getInstance() {
if (instance == null) {
synchronized (DownloaderTestImpl.class) {
if (instance == null) {
instance = new DownloaderTestImpl();
}
}
}
return instance;
}
private void setDefaultHeaders(URLConnection connection) {
connection.setRequestProperty("User-Agent", USER_AGENT);
connection.setRequestProperty("Accept-Language", DEFAULT_HTTP_ACCEPT_LANGUAGE);
}
@Override
public Response execute(@Nonnull Request request) throws IOException, ReCaptchaException {
final String httpMethod = request.httpMethod();
final String url = request.url();
final Map<String, List<String>> headers = request.headers();
@Nullable final byte[] dataToSend = request.dataToSend();
@Nullable final Localization localization = request.localization();
final HttpsURLConnection connection = (HttpsURLConnection) new URL(url).openConnection();
connection.setConnectTimeout(30 * 1000); // 30s
connection.setReadTimeout(30 * 1000); // 30s
connection.setRequestMethod(httpMethod);
setDefaultHeaders(connection);
for (Map.Entry<String, List<String>> pair : headers.entrySet()) {
final String headerName = pair.getKey();
final List<String> headerValueList = pair.getValue();
if (headerValueList.size() > 1) {
connection.setRequestProperty(headerName, null);
for (String headerValue : headerValueList) {
connection.addRequestProperty(headerName, headerValue);
}
} else if (headerValueList.size() == 1) {
connection.setRequestProperty(headerName, headerValueList.get(0));
}
}
@Nullable OutputStream outputStream = null;
@Nullable InputStreamReader input = null;
try {
if (dataToSend != null && dataToSend.length > 0) {
connection.setDoOutput(true);
connection.setRequestProperty("Content-Length", dataToSend.length + "");
outputStream = connection.getOutputStream();
outputStream.write(dataToSend);
}
final InputStream inputStream = connection.getInputStream();
final StringBuilder response = new StringBuilder();
// Not passing any charset for decoding here... something to keep in mind.
input = new InputStreamReader(inputStream);
int readCount;
char[] buffer = new char[32 * 1024];
while ((readCount = input.read(buffer)) != -1) {
response.append(buffer, 0, readCount);
}
final int responseCode = connection.getResponseCode();
final String responseMessage = connection.getResponseMessage();
final Map<String, List<String>> responseHeaders = connection.getHeaderFields();
return new Response(responseCode, responseMessage, responseHeaders, response.toString());
} catch (Exception e) {
/*
* HTTP 429 == Too Many Request
* Receive from Youtube.com = ReCaptcha challenge request
* See : https://github.com/rg3/youtube-dl/issues/5138
*/
if (connection.getResponseCode() == 429) {
throw new ReCaptchaException("reCaptcha Challenge requested", url);
}
throw new IOException(connection.getResponseCode() + " " + connection.getResponseMessage(), e);
} finally {
if (outputStream != null) outputStream.close();
if (input != null) input.close();
}
}
}

10
extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java
View File

@ -2,8 +2,10 @@ package org.schabi.newpipe.extractor.services;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import java.util.Calendar;
import java.util.List;
import static org.junit.Assert.*;
@ -27,6 +29,14 @@ public final class DefaultTests {
StreamInfoItem streamInfoItem = (StreamInfoItem) item;
assertNotEmpty("Uploader name not set: " + item, streamInfoItem.getUploaderName());
assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl());
final String textualUploadDate = streamInfoItem.getTextualUploadDate();
if (textualUploadDate != null && !textualUploadDate.isEmpty()) {
final DateWrapper uploadDate = streamInfoItem.getUploadDate();
assertNotNull("No parsed upload date", uploadDate);
assertTrue("Upload date not in the past", uploadDate.date().before(Calendar.getInstance()));
}
}
}
}

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java
View File

@ -2,11 +2,10 @@ package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor;
import org.schabi.newpipe.extractor.utils.Localization;
import static junit.framework.TestCase.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
@ -19,7 +18,7 @@ public class MediaCCCConferenceExtractorTest {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("en", "en_GB"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getChannelExtractor("https://api.media.ccc.de/public/conferences/froscon2017");
extractor.fetchPage();
}

9
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java
View File

@ -1,18 +1,17 @@
package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceKiosk;
import org.schabi.newpipe.extractor.utils.Localization;
import java.util.List;
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
/**
@ -24,7 +23,7 @@ public class MediaCCCConferenceListExtractorTest {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("en", "en_GB"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getKioskList().getDefaultKioskExtractor();
extractor.fetchPage();
}

6
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java
View File

@ -2,13 +2,11 @@ package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.utils.Localization;
import static junit.framework.TestCase.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
@ -22,7 +20,7 @@ public class MediaCCCOggTest {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/1317");
extractor.fetchPage();

8
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java
View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
@ -11,7 +11,6 @@ import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.Localization;
import java.util.Arrays;
@ -28,10 +27,9 @@ public class MediaCCCSearchExtractorAllTest {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory()
.fromQuery("c3", Arrays.asList(new String[0]), "")
,new Localization("GB", "en"));
.fromQuery("c3", Arrays.asList(new String[0]), ""));
extractor.fetchPage();
itemsPage = extractor.getInitialPage();
}

8
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java
View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
@ -10,7 +10,6 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.utils.Localization;
import java.util.Arrays;
@ -27,10 +26,9 @@ public class MediaCCCSearchExtractorConferencesTest {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory()
.fromQuery("c3", Arrays.asList(new String[] {"conferences"}), "")
,new Localization("GB", "en"));
.fromQuery("c3", Arrays.asList(new String[]{"conferences"}), ""));
extractor.fetchPage();
itemsPage = extractor.getInitialPage();
}

8
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java
View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
@ -10,7 +10,6 @@ import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.Localization;
import java.util.Arrays;
@ -28,10 +27,9 @@ public class MediaCCCSearchExtractorEventsTest {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory()
.fromQuery("linux", Arrays.asList(new String[] {"events"}), "")
,new Localization("GB", "en"));
.fromQuery("linux", Arrays.asList(new String[]{"events"}), ""));
extractor.fetchPage();
itemsPage = extractor.getInitialPage();
}

26
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java
View File

@ -1,14 +1,20 @@
package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseExtractorTest;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor;
import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import static java.util.Objects.requireNonNull;
import static junit.framework.TestCase.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
@ -20,7 +26,7 @@ public class MediaCCCStreamExtractorTest implements BaseExtractorTest {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/8afc16c2-d76a-53f6-85e4-90494665835d");
extractor.fetchPage();
@ -80,4 +86,16 @@ public class MediaCCCStreamExtractorTest implements BaseExtractorTest {
public void testAudioStreams() throws Exception {
assertEquals(2, extractor.getAudioStreams().size());
}
@Test
public void testGetTextualUploadDate() throws ParsingException {
Assert.assertEquals("2018-05-11", extractor.getTextualUploadDate());
}
@Test
public void testGetUploadDate() throws ParsingException, ParseException {
final Calendar instance = Calendar.getInstance();
instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2018-05-11"));
assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
}
}

7
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java
View File

@ -13,13 +13,12 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRela
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelExtractor;
import org.schabi.newpipe.extractor.utils.Localization;
/**
* Test for {@link PeertubeChannelExtractor}
@ -30,7 +29,7 @@ public class PeertubeChannelExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host");
extractor = (PeertubeChannelExtractor) PeerTube
@ -117,7 +116,7 @@ public class PeertubeChannelExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host");
extractor = (PeertubeChannelExtractor) PeerTube

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java
View File

@ -5,11 +5,10 @@ import static org.junit.Assert.assertTrue;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Localization;
/**
* Test for {@link PeertubeChannelLinkHandlerFactory}
@ -21,7 +20,7 @@ public class PeertubeChannelLinkHandlerFactoryTest {
@BeforeClass
public static void setUp() {
linkHandler = PeertubeChannelLinkHandlerFactory.getInstance();
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test

9
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java
View File

@ -10,14 +10,13 @@ import java.util.List;
import org.jsoup.helper.StringUtil;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.comments.CommentsInfo;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor;
import org.schabi.newpipe.extractor.utils.Localization;
public class PeertubeCommentsExtractorTest {
@ -25,7 +24,7 @@ public class PeertubeCommentsExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (PeertubeCommentsExtractor) PeerTube
.getCommentsExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
}
@ -71,10 +70,10 @@ public class PeertubeCommentsExtractorTest {
assertFalse(StringUtil.isBlank(c.getCommentId()));
assertFalse(StringUtil.isBlank(c.getCommentText()));
assertFalse(StringUtil.isBlank(c.getName()));
assertFalse(StringUtil.isBlank(c.getPublishedTime()));
assertFalse(StringUtil.isBlank(c.getTextualPublishedTime()));
assertFalse(StringUtil.isBlank(c.getThumbnailUrl()));
assertFalse(StringUtil.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() == null);
assertFalse(c.getLikeCount() == -1);
}
}

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsLinkHandlerFactoryTest.java
View File

@ -5,11 +5,10 @@ import static org.junit.Assert.assertTrue;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Localization;
/**
* Test for {@link PeertubeCommentsLinkHandlerFactory}
@ -21,7 +20,7 @@ public class PeertubeCommentsLinkHandlerFactoryTest {
@BeforeClass
public static void setUp() {
linkHandler = PeertubeCommentsLinkHandlerFactory.getInstance();
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java
View File

@ -5,11 +5,10 @@ import static org.junit.Assert.assertTrue;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Localization;
/**
* Test for {@link PeertubePlaylistLinkHandlerFactory}
@ -21,7 +20,7 @@ public class PeertubePlaylistLinkHandlerFactoryTest {
@BeforeClass
public static void setUp() {
linkHandler = PeertubePlaylistLinkHandlerFactory.getInstance();
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test

16
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java
View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.peertube;
import static java.util.Objects.requireNonNull;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@ -7,11 +8,14 @@ import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -19,7 +23,6 @@ import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamE
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Localization;
/**
* Test for {@link StreamExtractor}
@ -29,7 +32,7 @@ public class PeertubeStreamExtractorDefaultTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host");
extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/afe5bf12-c58b-4efd-b56e-29c5a59e04bc");
@ -69,8 +72,11 @@ public class PeertubeStreamExtractorDefaultTest {
}
@Test
public void testGetUploadDate() throws ParsingException {
assertEquals("2018-09-30", extractor.getUploadDate());
public void testGetUploadDate() throws ParsingException, ParseException {
final Calendar instance = Calendar.getInstance();
instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-09-30T14:08:24.378Z"));
assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
}
@Test

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java
View File

@ -5,11 +5,10 @@ import static org.junit.Assert.assertTrue;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Localization;
/**
* Test for {@link PeertubeStreamLinkHandlerFactory}
@ -20,7 +19,7 @@ public class PeertubeStreamLinkHandlerFactoryTest {
@BeforeClass
public static void setUp() throws Exception {
linkHandler = PeertubeStreamLinkHandlerFactory.getInstance();
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java
View File

@ -10,13 +10,12 @@ import java.util.List;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.Localization;
/**
* Test for {@link PeertubeTrendingExtractor}
@ -27,7 +26,7 @@ public class PeertubeTrendingExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host");
extractor = PeerTube

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingLinkHandlerFactoryTest.java
View File

@ -6,12 +6,11 @@ import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeTrendingLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Localization;
/**
* Test for {@link PeertubeTrendingLinkHandlerFactory}
@ -24,7 +23,7 @@ public class PeertubeTrendingLinkHandlerFactoryTest {
// setting instance might break test when running in parallel
PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host");
LinkHandlerFactory = new PeertubeTrendingLinkHandlerFactory();
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorDefaultTest.java
View File

@ -7,13 +7,12 @@ import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSearchExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.utils.Localization;
/**
* Test for {@link PeertubeSearchExtractor}
@ -22,7 +21,7 @@ public class PeertubeSearchExtractorDefaultTest extends PeertubeSearchExtractorB
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host");
extractor = (PeertubeSearchExtractor) PeerTube.getSearchExtractor("kde");

Some files were not shown because too many files have changed in this diff Show More