lasers
/
NewPipe
mirror of https://github.com/TeamNewPipe/NewPipe
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

remove extractor

This commit is contained in:
Christian Schabesberger 2017-03-01 19:06:27 +01:00
parent ecabc65e94
commit cffd049c8a
41 changed files with 0 additions and 4055 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
app/src/main/java/org/schabi/newpipe/extractor/AbstractStreamInfo.java
View File

@ -1,41 +0,0 @@
package org.schabi.newpipe.extractor;
/**
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* AbstractStreamInfo.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/**Common properties between StreamInfo and StreamInfoItem.*/
public abstract class AbstractStreamInfo {
public enum StreamType {
NONE, // placeholder to check if stream type was checked or not
VIDEO_STREAM,
AUDIO_STREAM,
LIVE_STREAM,
AUDIO_LIVE_STREAM,
FILE
}
public StreamType stream_type;
public int service_id = -1;
public String id = "";
public String title = "";
public String uploader = "";
public String thumbnail_url = "";
public String webpage_url = "";
public String upload_date = "";
public long view_count = -1;
}

93
app/src/main/java/org/schabi/newpipe/extractor/DashMpdParser.java
View File

@ -1,93 +0,0 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream_info.AudioStream;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.Vector;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
/**
* Created by Christian Schabesberger on 02.02.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* DashMpdParser.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class DashMpdParser {
private DashMpdParser() {
}
static class DashMpdParsingException extends ParsingException {
DashMpdParsingException(String message, Exception e) {
super(message, e);
}
}
public static List<AudioStream> getAudioStreams(String dashManifestUrl)
throws DashMpdParsingException, ReCaptchaException {
String dashDoc;
Downloader downloader = NewPipe.getDownloader();
try {
dashDoc = downloader.download(dashManifestUrl);
} catch(IOException ioe) {
throw new DashMpdParsingException("Could not get dash mpd: " + dashManifestUrl, ioe);
} catch (ReCaptchaException e) {
throw new ReCaptchaException("reCaptcha Challenge needed");
}
Vector<AudioStream> audioStreams = new Vector<>();
try {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
DocumentBuilder builder = factory.newDocumentBuilder();
InputStream stream = new ByteArrayInputStream(dashDoc.getBytes());
Document doc = builder.parse(stream);
NodeList adaptationSetList = doc.getElementsByTagName("AdaptationSet");
for(int i = 0; i < adaptationSetList.getLength(); i++) {
Element adaptationSet = (Element) adaptationSetList.item(i);
String memeType = adaptationSet.getAttribute("mimeType");
if(memeType.contains("audio")) {
Element representation = (Element) adaptationSet.getElementsByTagName("Representation").item(0);
String url = representation.getElementsByTagName("BaseURL").item(0).getTextContent();
int bandwidth = Integer.parseInt(representation.getAttribute("bandwidth"));
int samplingRate = Integer.parseInt(representation.getAttribute("audioSamplingRate"));
int format = -1;
if(memeType.equals(MediaFormat.WEBMA.mimeType)) {
format = MediaFormat.WEBMA.id;
} else if(memeType.equals(MediaFormat.M4A.mimeType)) {
format = MediaFormat.M4A.id;
}
audioStreams.add(new AudioStream(url, format, bandwidth, samplingRate));
}
}
}
catch(Exception e) {
throw new DashMpdParsingException("Could not parse Dash mpd", e);
}
return audioStreams;
}
}

52
app/src/main/java/org/schabi/newpipe/extractor/Downloader.java
View File

@ -1,52 +0,0 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import java.io.IOException;
import java.util.Map;
/**
* Created by Christian Schabesberger on 28.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* Downloader.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public interface Downloader {
/**Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string.
* @param siteUrl the URL of the text file to return the contents of
* @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file
* @throws IOException*/
String download(String siteUrl, String language) throws IOException, ReCaptchaException;
/**Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string.
* @param siteUrl the URL of the text file to return the contents of
* @param customProperties set request header properties
* @return the contents of the specified text file
* @throws IOException*/
String download(String siteUrl, Map<String, String> customProperties) throws IOException, ReCaptchaException;
/**Download (via HTTP) the text file located at the supplied URL, and return its contents.
* Primarily intended for downloading web pages.
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file
* @throws IOException*/
String download(String siteUrl) throws IOException, ReCaptchaException;
}

33
app/src/main/java/org/schabi/newpipe/extractor/InfoItem.java
View File

@ -1,33 +0,0 @@
package org.schabi.newpipe.extractor;
/**
* Created by the-scrabi on 11.02.17.
*
* Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
* InfoItem.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public interface InfoItem {
public enum InfoType {
STREAM,
PLAYLIST,
CHANNEL
}
InfoType infoType();
String getTitle();
String getLink();
}

61
app/src/main/java/org/schabi/newpipe/extractor/InfoItemCollector.java
View File

@ -1,61 +0,0 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.util.List;
import java.util.Vector;
/**
* Created by Christian Schabesberger on 12.02.17.
*
* Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
* InfoItemCollector.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class InfoItemCollector {
private List<InfoItem> itemList = new Vector<>();
private List<Throwable> errors = new Vector<>();
private int serviceId = -1;
public InfoItemCollector(int serviceId) {
this.serviceId = serviceId;
}
public List<InfoItem> getItemList() {
return itemList;
}
public List<Throwable> getErrors() {
return errors;
}
protected void addFromCollector(InfoItemCollector otherC) throws ExtractionException {
if(serviceId != otherC.serviceId) {
throw new ExtractionException("Service Id does not equal: "
+ NewPipe.getNameOfService(serviceId)
+ " and " + NewPipe.getNameOfService(otherC.serviceId));
}
errors.addAll(otherC.errors);
itemList.addAll(otherC.itemList);
}
protected void addError(Exception e) {
errors.add(e);
}
protected void addItem(InfoItem item) {
itemList.add(item);
}
protected int getServiceId() {
return serviceId;
}
}

83
app/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java
View File

@ -1,83 +0,0 @@
package org.schabi.newpipe.extractor;
/**
* Created by Adam Howard on 08/11/15.
*
* Copyright (c) Christian Schabesberger <chris.schabesberger@mailbox.org>
* and Adam Howard <achdisposable1@gmail.com> 2015
*
* MediaFormat.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/**Static data about various media formats support by Newpipe, eg mime type, extension*/
public enum MediaFormat {
//video and audio combined formats
// id name suffix mime type
MPEG_4 (0x0, "MPEG-4", "mp4", "video/mp4"),
v3GPP (0x1, "3GPP", "3gp", "video/3gpp"),
WEBM (0x2, "WebM", "webm", "video/webm"),
// audio formats
M4A (0x3, "m4a", "m4a", "audio/mp4"),
WEBMA (0x4, "WebM", "webm", "audio/webm");
public final int id;
@SuppressWarnings("WeakerAccess")
public final String name;
@SuppressWarnings("WeakerAccess")
public final String suffix;
public final String mimeType;
MediaFormat(int id, String name, String suffix, String mimeType) {
this.id = id;
this.name = name;
this.suffix = suffix;
this.mimeType = mimeType;
}
/**Return the friendly name of the media format with the supplied id
* @param ident the id of the media format. Currently an arbitrary, NewPipe-specific number.
* @return the friendly name of the MediaFormat associated with this ids,
* or an empty String if none match it.*/
public static String getNameById(int ident) {
for (MediaFormat vf : MediaFormat.values()) {
if(vf.id == ident) return vf.name;
}
return "";
}
/**Return the file extension of the media format with the supplied id
* @param ident the id of the media format. Currently an arbitrary, NewPipe-specific number.
* @return the file extension of the MediaFormat associated with this ids,
* or an empty String if none match it.*/
public static String getSuffixById(int ident) {
for (MediaFormat vf : MediaFormat.values()) {
if(vf.id == ident) return vf.suffix;
}
return "";
}
/**Return the MIME type of the media format with the supplied id
* @param ident the id of the media format. Currently an arbitrary, NewPipe-specific number.
* @return the MIME type of the MediaFormat associated with this ids,
* or an empty String if none match it.*/
public static String getMimeById(int ident) {
for (MediaFormat vf : MediaFormat.values()) {
if(vf.id == ident) return vf.mimeType;
}
return "";
}
}

88
app/src/main/java/org/schabi/newpipe/extractor/NewPipe.java
View File

@ -1,88 +0,0 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
/**
* Created by Christian Schabesberger on 23.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* NewPipe.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/**Provides access to the video streaming services supported by NewPipe.
* Currently only Youtube until the API becomes more stable.*/
@SuppressWarnings("ALL")
public class NewPipe {
private NewPipe() {
}
private static final String TAG = NewPipe.class.toString();
private static Downloader downloader = null;
public static StreamingService[] getServices() {
return ServiceList.serviceList;
}
public static StreamingService getService(int serviceId)throws ExtractionException {
for(StreamingService s : ServiceList.serviceList) {
if(s.getServiceId() == serviceId) {
return s;
}
}
return null;
}
public static StreamingService getService(String serviceName) throws ExtractionException {
return ServiceList.serviceList[getIdOfService(serviceName)];
}
public static String getNameOfService(int id) {
try {
return getService(id).getServiceInfo().name;
} catch (Exception e) {
System.err.println("Service id not known");
e.printStackTrace();
return "";
}
}
public static int getIdOfService(String serviceName) {
for(int i = 0; i < ServiceList.serviceList.length; i++) {
if(ServiceList.serviceList[i].getServiceInfo().name.equals(serviceName)) {
return i;
}
}
return -1;
}
public static void init(Downloader d) {
downloader = d;
}
public static Downloader getDownloader() {
return downloader;
}
public static StreamingService getServiceByUrl(String url) {
for(StreamingService s : ServiceList.serviceList) {
if(s.getLinkTypeByUrl(url) != StreamingService.LinkType.NONE) {
return s;
}
}
return null;
}
}

77
app/src/main/java/org/schabi/newpipe/extractor/Parser.java
View File

@ -1,77 +0,0 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Created by Christian Schabesberger on 02.02.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* Parser.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/** avoid using regex !!! */
public class Parser {
private Parser() {
}
public static class RegexException extends ParsingException {
public RegexException(String message) {
super(message);
}
}
public static String matchGroup1(String pattern, String input) throws RegexException {
return matchGroup(pattern, input, 1);
}
public static String matchGroup(String pattern, String input, int group) throws RegexException {
Pattern pat = Pattern.compile(pattern);
Matcher mat = pat.matcher(input);
boolean foundMatch = mat.find();
if (foundMatch) {
return mat.group(group);
}
else {
//Log.e(TAG, "failed to find pattern \""+pattern+"\" inside of \""+input+"\"");
if(input.length() > 1024) {
throw new RegexException("failed to find pattern \""+pattern);
} else {
throw new RegexException("failed to find pattern \"" + pattern + " inside of " + input + "\"");
}
}
}
public static Map<String, String> compatParseMap(final String input) throws UnsupportedEncodingException {
Map<String, String> map = new HashMap<>();
for(String arg : input.split("&")) {
String[] splitArg = arg.split("=");
if(splitArg.length > 1) {
map.put(splitArg[0], URLDecoder.decode(splitArg[1], "UTF-8"));
} else {
map.put(splitArg[0], "");
}
}
return map;
}
}

13
app/src/main/java/org/schabi/newpipe/extractor/ServiceList.java
View File

@ -1,13 +0,0 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
/**
* Created by the-scrabi on 18.02.17.
*/
class ServiceList {
public static final StreamingService[] serviceList = {
new YoutubeService(0)
};
}

78
app/src/main/java/org/schabi/newpipe/extractor/StreamingService.java
View File

@ -1,78 +0,0 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.stream_info.StreamExtractor;
import java.io.IOException;
/**
* Created by Christian Schabesberger on 23.08.15.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* StreamingService.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public abstract class StreamingService {
public class ServiceInfo {
public String name = "";
}
public enum LinkType {
NONE,
STREAM,
CHANNEL,
PLAYLIST
}
private int serviceId;
public StreamingService(int id) {
serviceId = id;
}
public abstract ServiceInfo getServiceInfo();
public abstract StreamExtractor getExtractorInstance(String url)
throws IOException, ExtractionException;
public abstract SearchEngine getSearchEngineInstance();
public abstract UrlIdHandler getStreamUrlIdHandlerInstance();
public abstract UrlIdHandler getChannelUrlIdHandlerInstance();
public abstract ChannelExtractor getChannelExtractorInstance(String url, int page)
throws ExtractionException, IOException;
public abstract SuggestionExtractor getSuggestionExtractorInstance();
public final int getServiceId() {
return serviceId;
}
/**
* figure out where the link is pointing to (a channel, video, playlist, etc.)
*/
public final LinkType getLinkTypeByUrl(String url) {
UrlIdHandler sH = getStreamUrlIdHandlerInstance();
UrlIdHandler cH = getChannelUrlIdHandlerInstance();
if(sH.acceptUrl(url)) {
return LinkType.STREAM;
} else if(cH.acceptUrl(url)) {
return LinkType.CHANNEL;
} else {
return LinkType.NONE;
}
}
}

43
app/src/main/java/org/schabi/newpipe/extractor/SuggestionExtractor.java
View File

@ -1,43 +0,0 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.io.IOException;
import java.util.List;
/**
* Created by Christian Schabesberger on 28.09.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* SuggestionExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public abstract class SuggestionExtractor {
private int serviceId;
public SuggestionExtractor(int serviceId) {
this.serviceId = serviceId;
}
public abstract List<String> suggestionList(
String query,String contentCountry)
throws ExtractionException, IOException;
public int getServiceId() {
return serviceId;
}
}

35
app/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java
View File

@ -1,35 +0,0 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
/**
* Created by Christian Schabesberger on 26.07.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* UrlIdHandler.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public interface UrlIdHandler {
String getUrl(String videoId);
String getId(String siteUrl) throws ParsingException;
String cleanUrl(String siteUrl) throws ParsingException;
/**When a VIEW_ACTION is caught this function will test if the url delivered within the calling
Intent was meant to be watched with this Service.
Return false if this service shall not allow to be called through ACTIONs.*/
boolean acceptUrl(String videoUrl);
}

62
app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java
View File

@ -1,62 +0,0 @@
package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream_info.StreamInfoItemCollector;
import java.io.IOException;
/**
* Created by Christian Schabesberger on 25.07.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* ChannelExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public abstract class ChannelExtractor {
private int serviceId;
private String url;
private UrlIdHandler urlIdHandler;
private StreamInfoItemCollector previewInfoCollector;
private int page = -1;
public ChannelExtractor(UrlIdHandler urlIdHandler, String url, int page, int serviceId)
throws ExtractionException, IOException {
this.url = url;
this.page = page;
this.serviceId = serviceId;
this.urlIdHandler = urlIdHandler;
previewInfoCollector = new StreamInfoItemCollector(urlIdHandler, serviceId);
}
public String getUrl() { return url; }
public UrlIdHandler getUrlIdHandler() { return urlIdHandler; }
public StreamInfoItemCollector getStreamPreviewInfoCollector() {
return previewInfoCollector;
}
public abstract String getChannelName() throws ParsingException;
public abstract String getAvatarUrl() throws ParsingException;
public abstract String getBannerUrl() throws ParsingException;
public abstract String getFeedUrl() throws ParsingException;
public abstract StreamInfoItemCollector getStreams() throws ParsingException;
public abstract long getSubscriberCount() throws ParsingException;
public abstract boolean hasNextPage() throws ParsingException;
public int getServiceId() {
return serviceId;
}
}

86
app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java
View File

@ -1,86 +0,0 @@
package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream_info.StreamInfoItem;
import org.schabi.newpipe.extractor.stream_info.StreamInfoItemCollector;
import java.util.List;
import java.util.Vector;
/**
* Created by Christian Schabesberger on 31.07.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* ChannelInfo.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class ChannelInfo {
public void addException(Exception e) {
errors.add(e);
}
public static ChannelInfo getInfo(ChannelExtractor extractor)
throws ParsingException {
ChannelInfo info = new ChannelInfo();
// importand data
info.service_id = extractor.getServiceId();
info.channel_name = extractor.getChannelName();
info.hasNextPage = extractor.hasNextPage();
try {
info.avatar_url = extractor.getAvatarUrl();
} catch (Exception e) {
info.errors.add(e);
}
try {
info.banner_url = extractor.getBannerUrl();
} catch (Exception e) {
info.errors.add(e);
}
try {
info.feed_url = extractor.getFeedUrl();
} catch(Exception e) {
info.errors.add(e);
}
try {
StreamInfoItemCollector c = extractor.getStreams();
info.related_streams = c.getItemList();
info.errors.addAll(c.getErrors());
} catch(Exception e) {
info.errors.add(e);
}
try {
info.subscriberCount = extractor.getSubscriberCount();
} catch (Exception e) {
info.errors.add(e);
}
return info;
}
public int service_id = -1;
public String channel_name = "";
public String avatar_url = "";
public String banner_url = "";
public String feed_url = "";
public List<InfoItem> related_streams = null;
public long subscriberCount = -1;
public boolean hasNextPage = false;
public List<Throwable> errors = new Vector<>();
}

44
app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItem.java
View File

@ -1,44 +0,0 @@
package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.InfoItem;
/**
* Created by Christian Schabesberger on 11.02.17.
*
* Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
* ChannelInfoItem.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class ChannelInfoItem implements InfoItem {
public int serviceId = -1;
public String channelName = "";
public String thumbnailUrl = "";
public String webPageUrl = "";
public String description = "";
public long subscriberCount = -1;
public int videoAmount = -1;
public InfoType infoType() {
return InfoType.CHANNEL;
}
public String getTitle() {
return channelName;
}
public String getLink() {
return webPageUrl;
}
}

74
app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemCollector.java
View File

@ -1,74 +0,0 @@
package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.InfoItemCollector;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream_info.StreamInfoItem;
import org.schabi.newpipe.extractor.stream_info.StreamInfoItemExtractor;
/**
* Created by Christian Schabesberger on 12.02.17.
*
* Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
* ChannelInfoItemCollector.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class ChannelInfoItemCollector extends InfoItemCollector {
public ChannelInfoItemCollector(int serviceId) {
super(serviceId);
}
public ChannelInfoItem extract(ChannelInfoItemExtractor extractor) throws ParsingException {
ChannelInfoItem resultItem = new ChannelInfoItem();
// importand information
resultItem.channelName = extractor.getChannelName();
resultItem.serviceId = getServiceId();
resultItem.webPageUrl = extractor.getWebPageUrl();
// optional information
try {
resultItem.subscriberCount = extractor.getSubscriberCount();
} catch (Exception e) {
addError(e);
}
try {
resultItem.videoAmount = extractor.getVideoAmount();
} catch (Exception e) {
addError(e);
}
try {
resultItem.thumbnailUrl = extractor.getThumbnailUrl();
} catch (Exception e) {
addError(e);
}
try {
resultItem.description = extractor.getDescription();
} catch (Exception e) {
addError(e);
}
return resultItem;
}
public void commit(ChannelInfoItemExtractor extractor) throws ParsingException {
try {
addItem(extract(extractor));
} catch (Exception e) {
addError(e);
}
}
}

32
app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemExtractor.java
View File

@ -1,32 +0,0 @@
package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
/**
* Created by Christian Schabesberger on 12.02.17.
*
* Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
* ChannelInfoItemExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public interface ChannelInfoItemExtractor {
String getThumbnailUrl() throws ParsingException;
String getChannelName() throws ParsingException;
String getWebPageUrl() throws ParsingException;
String getDescription() throws ParsingException;
long getSubscriberCount() throws ParsingException;
int getVideoAmount() throws ParsingException;
}

33
app/src/main/java/org/schabi/newpipe/extractor/exceptions/ExtractionException.java
View File

@ -1,33 +0,0 @@
package org.schabi.newpipe.extractor.exceptions;
/**
* Created by Christian Schabesberger on 30.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* ExtractionException.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class ExtractionException extends Exception {
public ExtractionException(String message) {
super(message);
}
public ExtractionException(Throwable cause) {
super(cause);
}
public ExtractionException(String message, Throwable cause) {
super(message, cause);
}
}

30
app/src/main/java/org/schabi/newpipe/extractor/exceptions/FoundAdException.java
View File

@ -1,30 +0,0 @@
package org.schabi.newpipe.extractor.exceptions;
/**
* Created by Christian Schabesberger on 12.09.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* FoundAdException.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class FoundAdException extends ParsingException {
public FoundAdException(String message) {
super(message);
}
public FoundAdException(String message, Throwable cause) {
super(message, cause);
}
}

31
app/src/main/java/org/schabi/newpipe/extractor/exceptions/ParsingException.java
View File

@ -1,31 +0,0 @@
package org.schabi.newpipe.extractor.exceptions;
/**
* Created by Christian Schabesberger on 31.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* ParsingException.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class ParsingException extends ExtractionException {
public ParsingException(String message) {
super(message);
}
public ParsingException(String message, Throwable cause) {
super(message, cause);
}
}

27
app/src/main/java/org/schabi/newpipe/extractor/exceptions/ReCaptchaException.java
View File

@ -1,27 +0,0 @@
package org.schabi.newpipe.extractor.exceptions;
/**
* Created by beneth <bmauduit@beneth.fr> on 07.12.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* ReCaptchaException.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class ReCaptchaException extends ExtractionException {
public ReCaptchaException(String message) {
super(message);
}
}

79
app/src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java
View File

@ -1,79 +0,0 @@
package org.schabi.newpipe.extractor.search;
import org.schabi.newpipe.extractor.InfoItemCollector;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream_info.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.stream_info.StreamInfoItemExtractor;
/**
* Created by Christian Schabesberger on 12.02.17.
*
* Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
* InfoItemSearchCollector.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class InfoItemSearchCollector extends InfoItemCollector {
private String suggestion = "";
private StreamInfoItemCollector streamCollector;
private ChannelInfoItemCollector channelCollector;
SearchResult result = new SearchResult();
InfoItemSearchCollector(UrlIdHandler handler, int serviceId) {
super(serviceId);
streamCollector = new StreamInfoItemCollector(handler, serviceId);
channelCollector = new ChannelInfoItemCollector(serviceId);
}
public void setSuggestion(String suggestion) {
this.suggestion = suggestion;
}
public SearchResult getSearchResult() throws ExtractionException {
addFromCollector(channelCollector);
addFromCollector(streamCollector);
result.suggestion = suggestion;
result.errors = getErrors();
return result;
}
public void commit(StreamInfoItemExtractor extractor) {
try {
result.resultList.add(streamCollector.extract(extractor));
} catch(FoundAdException ae) {
System.err.println("Found add");
} catch (Exception e) {
addError(e);
}
}
public void commit(ChannelInfoItemExtractor extractor) {
try {
result.resultList.add(channelCollector.extract(extractor));
} catch(FoundAdException ae) {
System.err.println("Found add");
} catch (Exception e) {
addError(e);
}
}
}

53
app/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java
View File

@ -1,53 +0,0 @@
package org.schabi.newpipe.extractor.search;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.stream_info.StreamInfoItemCollector;
import java.io.IOException;
import java.util.EnumSet;
/**
* Created by Christian Schabesberger on 10.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* SearchEngine.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public abstract class SearchEngine {
public enum Filter {
STREAM, CHANNEL, PLAY_LIST
}
public static class NothingFoundException extends ExtractionException {
public NothingFoundException(String message) {
super(message);
}
}
private InfoItemSearchCollector collector;
public SearchEngine(UrlIdHandler urlIdHandler, int serviceId) {
collector = new InfoItemSearchCollector(urlIdHandler, serviceId);
}
protected InfoItemSearchCollector getInfoItemSearchCollector() {
return collector;
}
//Result search(String query, int page);
public abstract InfoItemSearchCollector search(
String query, int page, String contentCountry, EnumSet<Filter> filter)
throws ExtractionException, IOException;
}

56
app/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java
View File

@ -1,56 +0,0 @@
package org.schabi.newpipe.extractor.search;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.stream_info.StreamInfoItem;
import java.io.IOException;
import java.util.EnumSet;
import java.util.List;
import java.util.Vector;
/**
* Created by Christian Schabesberger on 29.02.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* SearchResult.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class SearchResult {
public static SearchResult getSearchResult(SearchEngine engine, String query,
int page, String languageCode, EnumSet<SearchEngine.Filter> filter)
throws ExtractionException, IOException {
SearchResult result = engine
.search(query, page, languageCode, filter)
.getSearchResult();
if(result.resultList.isEmpty()) {
if(result.suggestion.isEmpty()) {
if(result.errors.isEmpty()) {
throw new ExtractionException("Empty result despite no error");
}
} else {
// This is used as a fallback. Do not relay on it !!!
throw new SearchEngine.NothingFoundException(result.suggestion);
}
}
return result;
}
public String suggestion = "";
public List<InfoItem> resultList = new Vector<>();
public List<Throwable> errors = new Vector<>();
}

361
app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java
View File

@ -1,361 +0,0 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.json.JSONException;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.AbstractStreamInfo;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Parser;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream_info.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.stream_info.StreamInfoItemExtractor;
import java.io.IOException;
/**
* Created by Christian Schabesberger on 25.07.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* YoutubeChannelExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeChannelExtractor extends ChannelExtractor {
private static final String TAG = YoutubeChannelExtractor.class.toString();
// private CSSOMParser cssParser = new CSSOMParser(new SACParserCSS3());
private Document doc = null;
private boolean isAjaxPage = false;
private static String userUrl = "";
private static String channelName = "";
private static String avatarUrl = "";
private static String bannerUrl = "";
private static String feedUrl = "";
private static long subscriberCount = -1;
// the fist page is html all other pages are ajax. Every new page can be requested by sending
// this request url.
private static String nextPageUrl = "";
public YoutubeChannelExtractor(UrlIdHandler urlIdHandler, String url, int page, int serviceId)
throws ExtractionException, IOException {
super(urlIdHandler, url, page, serviceId);
Downloader downloader = NewPipe.getDownloader();
url = urlIdHandler.cleanUrl(url) ; //+ "/video?veiw=0&flow=list&sort=dd";
if(page == 0) {
if (isUserUrl(url)) {
userUrl = url;
} else {
// we first need to get the user url. Otherwise we can't find videos
String channelPageContent = downloader.download(url);
Document channelDoc = Jsoup.parse(channelPageContent, url);
userUrl = getUserUrl(channelDoc);
}
userUrl = userUrl + "/videos?veiw=0&flow=list&sort=dd&live_view=10000";
String pageContent = downloader.download(userUrl);
doc = Jsoup.parse(pageContent, userUrl);
nextPageUrl = getNextPageUrl(doc);
isAjaxPage = false;
} else {
String ajaxDataRaw = downloader.download(nextPageUrl);
JSONObject ajaxData;
try {
ajaxData = new JSONObject(ajaxDataRaw);
String htmlDataRaw = ajaxData.getString("content_html");
doc = Jsoup.parse(htmlDataRaw, nextPageUrl);
String nextPageHtmlDataRaw = ajaxData.getString("load_more_widget_html");
if(!nextPageHtmlDataRaw.isEmpty()) {
Document nextPageData = Jsoup.parse(nextPageHtmlDataRaw, nextPageUrl);
nextPageUrl = getNextPageUrl(nextPageData);
} else {
nextPageUrl = "";
}
} catch (JSONException e) {
throw new ParsingException("Could not parse json data for next page", e);
}
isAjaxPage = true;
}
}
@Override
public String getChannelName() throws ParsingException {
try {
if(!isAjaxPage) {
channelName = doc.select("span[class=\"qualified-channel-title-text\"]").first()
.select("a").first().text();
}
return channelName;
} catch(Exception e) {
throw new ParsingException("Could not get channel name");
}
}
@Override
public String getAvatarUrl() throws ParsingException {
try {
if(!isAjaxPage) {
avatarUrl = doc.select("img[class=\"channel-header-profile-image\"]")
.first().attr("abs:src");
}
return avatarUrl;
} catch(Exception e) {
throw new ParsingException("Could not get avatar", e);
}
}
@Override
public String getBannerUrl() throws ParsingException {
try {
if(!isAjaxPage) {
Element el = doc.select("div[id=\"gh-banner\"]").first().select("style").first();
String cssContent = el.html();
String url = "https:" + Parser.matchGroup1("url\\(([^)]+)\\)", cssContent);
if (url.contains("s.ytimg.com") || url.contains("default_banner")) {
bannerUrl = null;
} else {
bannerUrl = url;
}
}
return bannerUrl;
} catch(Exception e) {
throw new ParsingException("Could not get Banner", e);
}
}
@Override
public StreamInfoItemCollector getStreams() throws ParsingException {
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
Element ul;
if(isAjaxPage) {
ul = doc.select("body").first();
} else {
ul = doc.select("ul[id=\"browse-items-primary\"]").first();
}
for(final Element li : ul.children()) {
if (li.select("div[class=\"feed-item-dismissable\"]").first() != null) {
collector.commit(new StreamInfoItemExtractor() {
@Override
public AbstractStreamInfo.StreamType getStreamType() throws ParsingException {
return AbstractStreamInfo.StreamType.VIDEO_STREAM;
}
@Override
public boolean isAd() throws ParsingException {
if(!li.select("span[class*=\"icon-not-available\"]").isEmpty()) {
return true;
} else {
return false;
}
}
@Override
public String getWebPageUrl() throws ParsingException {
try {
Element el = li.select("div[class=\"feed-item-dismissable\"]").first();
Element dl = el.select("h3").first().select("a").first();
return dl.attr("abs:href");
} catch (Exception e) {
throw new ParsingException("Could not get web page url for the video", e);
}
}
@Override
public String getTitle() throws ParsingException {
try {
Element el = li.select("div[class=\"feed-item-dismissable\"]").first();
Element dl = el.select("h3").first().select("a").first();
return dl.text();
} catch (Exception e) {
throw new ParsingException("Could not get title", e);
}
}
@Override
public int getDuration() throws ParsingException {
try {
return YoutubeParsingHelper.parseDurationString(
li.select("span[class=\"video-time\"]").first().text());
} catch(Exception e) {
if(isLiveStream(li)) {
// -1 for no duration
return -1;
} else {
throw new ParsingException("Could not get Duration: " + getTitle(), e);
}
}
}
@Override
public String getUploader() throws ParsingException {
return getChannelName();
}
@Override
public String getUploadDate() throws ParsingException {
try {
Element meta = li.select("div[class=\"yt-lockup-meta\"]").first();
Element li = meta.select("li").first();
if (li == null && meta != null) {
//this means we have a youtube red video
return "";
}else {
return li.text();
}
} catch(Exception e) {
throw new ParsingException("Could not get uplaod date", e);
}
}
@Override
public long getViewCount() throws ParsingException {
String output;
String input;
try {
input = li.select("div[class=\"yt-lockup-meta\"]").first()
.select("li").get(1)
.text();
} catch (IndexOutOfBoundsException e) {
return -1;
}
output = Parser.matchGroup1("([0-9,\\. ]*)", input)
.replace(" ", "")
.replace(".", "")
.replace(",", "");
try {
return Long.parseLong(output);
} catch (NumberFormatException e) {
// if this happens the video probably has no views
if(!input.isEmpty()) {
return 0;
} else {
throw new ParsingException("Could not handle input: " + input, e);
}
}
}
@Override
public String getThumbnailUrl() throws ParsingException {
try {
String url;
Element te = li.select("span[class=\"yt-thumb-clip\"]").first()
.select("img").first();
url = te.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we've caught such an item.
if (url.contains(".gif")) {
url = te.attr("abs:data-thumb");
}
return url;
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
}
private boolean isLiveStream(Element item) {
Element bla = item.select("span[class*=\"yt-badge-live\"]").first();
if(bla == null) {
// sometimes livestreams dont have badges but sill are live streams
// if video time is not available we most likly have an offline livestream
if(item.select("span[class*=\"video-time\"]").first() == null) {
return true;
}
}
return bla != null;
}
});
}
}
return collector;
}
@Override
public long getSubscriberCount() throws ParsingException {
Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]")
.first();
if(el != null) {
subscriberCount = Long.parseLong(el.text().replaceAll("\\D+",""));
} else if(el == null && subscriberCount == -1) {
throw new ParsingException("Could not get subscriber count");
}
return subscriberCount;
}
@Override
public String getFeedUrl() throws ParsingException {
try {
if(userUrl.contains("channel")) {
//channels don't have feeds in youtube, only user can provide such
return "";
}
if(!isAjaxPage) {
feedUrl = doc.select("link[title=\"RSS\"]").first().attr("abs:href");
}
return feedUrl;
} catch(Exception e) {
throw new ParsingException("Could not get feed url", e);
}
}
@Override
public boolean hasNextPage() throws ParsingException {
return !nextPageUrl.isEmpty();
}
private String getUserUrl(Document d) throws ParsingException {
return d.select("span[class=\"qualified-channel-title-text\"]").first()
.select("a").first().attr("abs:href");
}
private boolean isUserUrl(String url) throws ParsingException {
return url.contains("/user/");
}
private String getNextPageUrl(Document d) throws ParsingException {
try {
Element button = d.select("button[class*=\"yt-uix-load-more\"]").first();
if(button != null) {
return button.attr("abs:data-uix-load-more-href");
} else {
// sometimes channels are simply so small, they don't have a second/next4q page
return "";
}
} catch(Exception e) {
throw new ParsingException("could not load next page url", e);
}
}
}

83
app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java
View File

@ -1,83 +0,0 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.schabi.newpipe.extractor.Parser;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.jsoup.nodes.Element;
/**
* Created by Christian Schabesberger on 12.02.17.
*
* Copyright (C) Christian Schabesberger 2017 <chris.schabesberger@mailbox.org>
* YoutubeChannelInfoItemExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor {
private Element el;
public YoutubeChannelInfoItemExtractor(Element el) {
this.el = el;
}
public String getThumbnailUrl() throws ParsingException {
Element img = el.select("span[class*=\"yt-thumb-simple\"]").first()
.select("img").first();
String url = img.attr("abs:src");
if(url.contains("gif")) {
url = img.attr("abs:data-thumb");
}
return url;
}
public String getChannelName() throws ParsingException {
return el.select("a[class*=\"yt-uix-tile-link\"]").first()
.text();
}
public String getWebPageUrl() throws ParsingException {
return el.select("a[class*=\"yt-uix-tile-link\"]").first()
.attr("abs:href");
}
public long getSubscriberCount() throws ParsingException {
Element subsEl = el.select("span[class*=\"yt-subscriber-count\"]").first();
if(subsEl == null) {
return 0;
} else {
return Integer.parseInt(subsEl.text().replaceAll("\\D+",""));
}
}
public int getVideoAmount() throws ParsingException {
Element metaEl = el.select("ul[class*=\"yt-lockup-meta-info\"]").first();
if(metaEl == null) {
return 0;
} else {
return Integer.parseInt(metaEl.text().replaceAll("\\D+",""));
}
}
public String getDescription() throws ParsingException {
Element desEl = el.select("div[class*=\"yt-lockup-description\"]").first();
if(desEl == null) {
return "";
} else {
return desEl.text();
}
}
}

47
app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandler.java
View File

@ -1,47 +0,0 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.schabi.newpipe.extractor.Parser;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
/**
* Created by Christian Schabesberger on 25.07.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* YoutubeChannelUrlIdHandler.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeChannelUrlIdHandler implements UrlIdHandler {
public String getUrl(String channelId) {
return "https://www.youtube.com/" + channelId;
}
public String getId(String siteUrl) throws ParsingException {
return Parser.matchGroup1("/(user/[A-Za-z0-9_-]*|channel/[A-Za-z0-9_-]*)", siteUrl);
}
public String cleanUrl(String siteUrl) throws ParsingException {
return getUrl(getId(siteUrl));
}
public boolean acceptUrl(String videoUrl) {
return (videoUrl.contains("youtube") ||
videoUrl.contains("youtu.be")) &&
( videoUrl.contains("/user/") ||
videoUrl.contains("/channel/"));
}
}

66
app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -1,66 +0,0 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
/**
* Created by Christian Schabesberger on 02.03.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* YoutubeParsingHelper.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeParsingHelper {
private YoutubeParsingHelper() {
}
public static int parseDurationString(String input)
throws ParsingException, NumberFormatException {
String[] splitInput = input.split(":");
String days = "0";
String hours = "0";
String minutes = "0";
String seconds;
switch(splitInput.length) {
case 4:
days = splitInput[0];
hours = splitInput[1];
minutes = splitInput[2];
seconds = splitInput[3];
break;
case 3:
hours = splitInput[0];
minutes = splitInput[1];
seconds = splitInput[2];
break;
case 2:
minutes = splitInput[0];
seconds = splitInput[1];
break;
case 1:
seconds = splitInput[0];
break;
default:
throw new ParsingException("Error duration string with unknown format: " + input);
}
return ((((Integer.parseInt(days) * 24)
+ Integer.parseInt(hours) * 60)
+ Integer.parseInt(minutes)) * 60)
+ Integer.parseInt(seconds);
}
}

119
app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java
View File

@ -1,119 +0,0 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.search.InfoItemSearchCollector;
import org.schabi.newpipe.extractor.search.SearchEngine;
import java.net.URLEncoder;
import java.io.IOException;
import java.util.EnumSet;
/**
* Created by Christian Schabesberger on 09.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* YoutubeSearchEngine.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeSearchEngine extends SearchEngine {
private static final String TAG = YoutubeSearchEngine.class.toString();
public static final String CHARSET_UTF_8 = "UTF-8";
public YoutubeSearchEngine(UrlIdHandler urlIdHandler, int serviceId) {
super(urlIdHandler, serviceId);
}
@Override
public InfoItemSearchCollector search(String query,
int page,
String languageCode,
EnumSet<Filter> filter)
throws IOException, ExtractionException {
InfoItemSearchCollector collector = getInfoItemSearchCollector();
Downloader downloader = NewPipe.getDownloader();
String url = "https://www.youtube.com/results"
+ "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
+ "&page=" + Integer.toString(page + 1);
if(filter.contains(Filter.STREAM) && !filter.contains(Filter.CHANNEL)) {
url += "&sp=EgIQAQ%253D%253D";
} else if(!filter.contains(Filter.STREAM) && filter.contains(Filter.CHANNEL)) {
url += "&sp=EgIQAg%253D%253D";
}
String site;
//String url = builder.build().toString();
//if we've been passed a valid language code, append it to the URL
if(!languageCode.isEmpty()) {
//assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode);
site = downloader.download(url, languageCode);
}
else {
site = downloader.download(url);
}
Document doc = Jsoup.parse(site, url);
Element list = doc.select("ol[class=\"item-section\"]").first();
for (Element item : list.children()) {
/* First we need to determine which kind of item we are working with.
Youtube depicts five different kinds of items on its search result page. These are
regular videos, playlists, channels, two types of video suggestions, and a "no video
found" item. Since we only want videos, we need to filter out all the others.
An example for this can be seen here:
https://www.youtube.com/results?search_query=asdf&page=1
We already applied a filter to the url, so we don't need to care about channels and
playlists now.
*/
Element el;
// both types of spell correction item
if ((el = item.select("div[class*=\"spell-correction\"]").first()) != null) {
collector.setSuggestion(el.select("a").first().text());
if(list.children().size() == 1) {
throw new NothingFoundException("Did you mean: " + el.select("a").first().text());
}
// search message item
} else if ((el = item.select("div[class*=\"search-message\"]").first()) != null) {
throw new NothingFoundException(el.text());
// video item type
} else if ((el = item.select("div[class*=\"yt-lockup-video\"]").first()) != null) {
collector.commit(new YoutubeStreamInfoItemExtractor(el));
} else if((el = item.select("div[class*=\"yt-lockup-channel\"]").first()) != null) {
collector.commit(new YoutubeChannelInfoItemExtractor(el));
} else {
// noinspection ConstantConditions
// simply ignore not known items
// throw new ExtractionException("unexpected element found: \"" + item + "\"");
}
}
return collector;
}
}

82
app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View File

@ -1,82 +0,0 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.stream_info.StreamExtractor;
import java.io.IOException;
/**
* Created by Christian Schabesberger on 23.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* YoutubeService.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeService extends StreamingService {
public YoutubeService(int id) {
super(id);
}
@Override
public ServiceInfo getServiceInfo() {
ServiceInfo serviceInfo = new ServiceInfo();
serviceInfo.name = "Youtube";
return serviceInfo;
}
@Override
public StreamExtractor getExtractorInstance(String url)
throws ExtractionException, IOException {
UrlIdHandler urlIdHandler = YoutubeStreamUrlIdHandler.getInstance();
if(urlIdHandler.acceptUrl(url)) {
return new YoutubeStreamExtractor(urlIdHandler, url, getServiceId());
}
else {
throw new IllegalArgumentException("supplied String is not a valid Youtube URL");
}
}
@Override
public SearchEngine getSearchEngineInstance() {
return new YoutubeSearchEngine(getStreamUrlIdHandlerInstance(), getServiceId());
}
@Override
public UrlIdHandler getStreamUrlIdHandlerInstance() {
return YoutubeStreamUrlIdHandler.getInstance();
}
@Override
public UrlIdHandler getChannelUrlIdHandlerInstance() {
return new YoutubeChannelUrlIdHandler();
}
@Override
public ChannelExtractor getChannelExtractorInstance(String url, int page)
throws ExtractionException, IOException {
return new YoutubeChannelExtractor(getChannelUrlIdHandlerInstance(), url, page, getServiceId());
}
@Override
public SuggestionExtractor getSuggestionExtractorInstance() {
return new YoutubeSuggestionExtractor(getServiceId());
}
}

878
app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java
View File

@ -1,878 +0,0 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.json.JSONException;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.extractor.AbstractStreamInfo;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream_info.AudioStream;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Parser;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.stream_info.StreamExtractor;
import org.schabi.newpipe.extractor.stream_info.StreamInfo;
import org.schabi.newpipe.extractor.stream_info.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.stream_info.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream_info.VideoStream;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Vector;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Created by Christian Schabesberger on 06.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* YoutubeStreamExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeStreamExtractor extends StreamExtractor {
public static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map";
public static final String HTTPS = "https:";
public static final String CONTENT = "content";
public static final String REGEX_INT = "[^\\d]";
// exceptions
public class DecryptException extends ParsingException {
DecryptException(String message, Throwable cause) {
super(message, cause);
}
}
// special content not available exceptions
public class GemaException extends ContentNotAvailableException {
GemaException(String message) {
super(message);
}
}
public class LiveStreamException extends ContentNotAvailableException {
LiveStreamException(String message) {
super(message);
}
}
// ----------------
// Sometimes if the html page of youtube is already downloaded, youtube web page will internally
// download the /get_video_info page. Since a certain date dashmpd url is only available over
// this /get_video_info page, so we always need to download this one to.
// %%video_id%% will be replaced by the actual video id
// $$el_type$$ will be replaced by the actual el_type (se the declarations below)
private static final String GET_VIDEO_INFO_URL =
"https://www.youtube.com/get_video_info?video_id=%%video_id%%$$el_type$$&ps=default&eurl=&gl=US&hl=en";
// eltype is necessary for the url above
private static final String EL_INFO = "el=info";
public enum ItagType {
AUDIO,
VIDEO,
VIDEO_ONLY
}
private static class ItagItem {
public ItagItem(int id, ItagType type, MediaFormat format, String res, int fps) {
this.id = id;
this.itagType = type;
this.mediaFormatId = format.id;
this.resolutionString = res;
this.fps = fps;
}
public ItagItem(int id, ItagType type, MediaFormat format, int samplingRate, int bandWidth) {
this.id = id;
this.itagType = type;
this.mediaFormatId = format.id;
this.samplingRate = samplingRate;
this.bandWidth = bandWidth;
}
public int id;
public ItagType itagType;
public int mediaFormatId;
public String resolutionString;
public int fps = -1;
public int samplingRate = -1;
public int bandWidth = -1;
}
private static final ItagItem[] itagList = {
// video streams
// id, ItagType, MediaFormat, Resolution, fps
new ItagItem(17, ItagType.VIDEO, MediaFormat.v3GPP, "144p", 12),
new ItagItem(18, ItagType.VIDEO, MediaFormat.MPEG_4, "360p", 24),
new ItagItem(22, ItagType.VIDEO, MediaFormat.MPEG_4, "720p", 24),
new ItagItem(36, ItagType.VIDEO, MediaFormat.v3GPP, "240p", 24),
new ItagItem(37, ItagType.VIDEO, MediaFormat.MPEG_4, "1080p", 24),
new ItagItem(38, ItagType.VIDEO, MediaFormat.MPEG_4, "1080p", 24),
new ItagItem(43, ItagType.VIDEO, MediaFormat.WEBM, "360p", 24),
new ItagItem(44, ItagType.VIDEO, MediaFormat.WEBM, "480p", 24),
new ItagItem(45, ItagType.VIDEO, MediaFormat.WEBM, "720p", 24),
new ItagItem(46, ItagType.VIDEO, MediaFormat.WEBM, "1080p", 24),
// audio streams
// id, ItagType, MediaFormat, samplingR, bandwidth
new ItagItem(249, ItagType.AUDIO, MediaFormat.WEBMA, 0, 0), // bandwith/samplingR 0 because not known
new ItagItem(250, ItagType.AUDIO, MediaFormat.WEBMA, 0, 0),
new ItagItem(171, ItagType.AUDIO, MediaFormat.WEBMA, 0, 0),
new ItagItem(140, ItagType.AUDIO, MediaFormat.M4A, 0, 0),
new ItagItem(251, ItagType.AUDIO, MediaFormat.WEBMA, 0, 0),
// video only streams
new ItagItem(160, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "144p", 24),
new ItagItem(133, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "240p", 24),
new ItagItem(134, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "360p", 24),
new ItagItem(135, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "480p", 24),
new ItagItem(136, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "720p", 24),
new ItagItem(137, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "1080p", 24),
};
/**These lists only contain itag formats that are supported by the common Android Video player.
However if you are looking for a list showing all itag formats, look at
https://github.com/rg3/youtube-dl/issues/1687 */
public static boolean itagIsSupported(int itag) {
for(ItagItem item : itagList) {
if(itag == item.id) {
return true;
}
}
return false;
}
public static ItagItem getItagItem(int itag) throws ParsingException {
for(ItagItem item : itagList) {
if(itag == item.id) {
return item;
}
}
throw new ParsingException("itag=" + Integer.toString(itag) + " not supported");
}
private static final String TAG = YoutubeStreamExtractor.class.toString();
private final Document doc;
private JSONObject playerArgs;
private boolean isAgeRestricted;
private Map<String, String> videoInfoPage;
// static values
private static final String DECRYPTION_FUNC_NAME="decrypt";
// cached values
private static volatile String decryptionCode = "";
UrlIdHandler urlidhandler = YoutubeStreamUrlIdHandler.getInstance();
String pageUrl = "";
public YoutubeStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId)
throws ExtractionException, IOException {
super(urlIdHandler, pageUrl, serviceId);
//most common videoInfo fields are now set in our superclass, for all services
this.pageUrl = pageUrl;
Downloader downloader = NewPipe.getDownloader();
String pageContent = downloader.download(urlidhandler.cleanUrl(pageUrl));
doc = Jsoup.parse(pageContent, pageUrl);
JSONObject ytPlayerConfig;
String playerUrl;
// Check if the video is age restricted
if (pageContent.contains("<meta property=\"og:restrictions:age")) {
String videoInfoUrl = GET_VIDEO_INFO_URL.replace("%%video_id%%",
urlidhandler.getId(pageUrl)).replace("$$el_type$$", "&" + EL_INFO);
String videoInfoPageString = downloader.download(videoInfoUrl);
videoInfoPage = Parser.compatParseMap(videoInfoPageString);
playerUrl = getPlayerUrlFromRestrictedVideo(pageUrl);
isAgeRestricted = true;
} else {
ytPlayerConfig = getPlayerConfig(pageContent);
playerArgs = getPlayerArgs(ytPlayerConfig);
playerUrl = getPlayerUrl(ytPlayerConfig);
isAgeRestricted = false;
}
if(decryptionCode.isEmpty()) {
decryptionCode = loadDecryptionCode(playerUrl);
}
}
private JSONObject getPlayerConfig(String pageContent) throws ParsingException {
try {
String ytPlayerConfigRaw =
Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
return new JSONObject(ytPlayerConfigRaw);
} catch (Parser.RegexException e) {
String errorReason = findErrorReason(doc);
switch(errorReason) {
case "GEMA":
throw new GemaException(errorReason);
case "":
throw new ContentNotAvailableException("Content not available: player config empty", e);
default:
throw new ContentNotAvailableException("Content not available", e);
}
} catch (JSONException e) {
throw new ParsingException("Could not parse yt player config", e);
}
}
private JSONObject getPlayerArgs(JSONObject playerConfig) throws ParsingException {
JSONObject playerArgs;
//attempt to load the youtube js player JSON arguments
boolean isLiveStream = false; //used to determine if this is a livestream or not
try {
playerArgs = playerConfig.getJSONObject("args");
// check if we have a live stream. We need to filter it, since its not yet supported.
if((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))
|| (playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
isLiveStream = true;
}
} catch (JSONException e) {
throw new ParsingException("Could not parse yt player config", e);
}
if (isLiveStream) {
throw new LiveStreamException("This is a Life stream. Can't use those right now.");
}
return playerArgs;
}
private String getPlayerUrl(JSONObject playerConfig) throws ParsingException {
try {
// The Youtube service needs to be initialized by downloading the
// js-Youtube-player. This is done in order to get the algorithm
// for decrypting cryptic signatures inside certain stream urls.
String playerUrl = "";
JSONObject ytAssets = playerConfig.getJSONObject("assets");
playerUrl = ytAssets.getString("js");
if (playerUrl.startsWith("//")) {
playerUrl = HTTPS + playerUrl;
}
return playerUrl;
} catch (JSONException e) {
throw new ParsingException(
"Could not load decryption code for the Youtube service.", e);
}
}
private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException, ReCaptchaException {
try {
Downloader downloader = NewPipe.getDownloader();
String playerUrl = "";
String videoId = urlidhandler.getId(pageUrl);
String embedUrl = "https://www.youtube.com/embed/" + videoId;
String embedPageContent = downloader.download(embedUrl);
//todo: find out if this can be reapaced by Parser.matchGroup1()
Pattern assetsPattern = Pattern.compile("\"assets\":.+?\"js\":\\s*(\"[^\"]+\")");
Matcher patternMatcher = assetsPattern.matcher(embedPageContent);
while (patternMatcher.find()) {
playerUrl = patternMatcher.group(1);
}
playerUrl = playerUrl.replace("\\", "").replace("\"", "");
if (playerUrl.startsWith("//")) {
playerUrl = HTTPS + playerUrl;
}
return playerUrl;
} catch (IOException e) {
throw new ParsingException(
"Could load decryption code form restricted video for the Youtube service.", e);
} catch (ReCaptchaException e) {
throw new ReCaptchaException("reCaptcha Challenge requested");
}
}
@Override
public String getTitle() throws ParsingException {
try {
if (playerArgs == null) {
return videoInfoPage.get("title");
}
//json player args method
return playerArgs.getString("title");
} catch(JSONException je) {//html <meta> method
je.printStackTrace();
System.err.println("failed to load title from JSON args; trying to extract it from HTML");
try { // fall through to fall-back
return doc.select("meta[name=title]").attr(CONTENT);
} catch (Exception e) {
throw new ParsingException("failed permanently to load title.", e);
}
}
}
@Override
public String getDescription() throws ParsingException {
try {
return doc.select("p[id=\"eow-description\"]").first().html();
} catch (Exception e) {//todo: add fallback method <-- there is no ... as long as i know
throw new ParsingException("failed to load description.", e);
}
}
@Override
public String getUploader() throws ParsingException {
try {
if (playerArgs == null) {
return videoInfoPage.get("author");
}
//json player args method
return playerArgs.getString("author");
} catch(JSONException je) {
je.printStackTrace();
System.err.println(
"failed to load uploader name from JSON args; trying to extract it from HTML");
} try {//fall through to fallback HTML method
return doc.select("div.yt-user-info").first().text();
} catch (Exception e) {
throw new ParsingException("failed permanently to load uploader name.", e);
}
}
@Override
public int getLength() throws ParsingException {
try {
if (playerArgs == null) {
return Integer.valueOf(videoInfoPage.get("length_seconds"));
}
return playerArgs.getInt("length_seconds");
} catch (JSONException e) {//todo: find fallback method
throw new ParsingException("failed to load video duration from JSON args", e);
}
}
@Override
public long getViewCount() throws ParsingException {
try {
String viewCountString = doc.select("meta[itemprop=interactionCount]").attr(CONTENT);
return Long.parseLong(viewCountString);
} catch (Exception e) {//todo: find fallback method
throw new ParsingException("failed to get number of views", e);
}
}
@Override
public String getUploadDate() throws ParsingException {
try {
return doc.select("meta[itemprop=datePublished]").attr(CONTENT);
} catch (Exception e) {//todo: add fallback method
throw new ParsingException("failed to get upload date.", e);
}
}
@Override
public String getThumbnailUrl() throws ParsingException {
//first attempt getting a small image version
//in the html extracting part we try to get a thumbnail with a higher resolution
// Try to get high resolution thumbnail if it fails use low res from the player instead
try {
return doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href");
} catch(Exception e) {
System.err.println("Could not find high res Thumbnail. Using low res instead");
}
try { //fall through to fallback
return playerArgs.getString("thumbnail_url");
} catch (JSONException je) {
throw new ParsingException(
"failed to extract thumbnail URL from JSON args; trying to extract it from HTML", je);
} catch (NullPointerException ne) {
// Get from the video info page instead
return videoInfoPage.get("thumbnail_url");
}
}
@Override
public String getUploaderThumbnailUrl() throws ParsingException {
try {
return doc.select("a[class*=\"yt-user-photo\"]").first()
.select("img").first()
.attr("abs:data-thumb");
} catch (Exception e) {//todo: add fallback method
throw new ParsingException("failed to get uploader thumbnail URL.", e);
}
}
@Override
public String getDashMpdUrl() throws ParsingException {
try {
String dashManifestUrl = "";
if(videoInfoPage != null && videoInfoPage.containsKey("dashmpd")) {
dashManifestUrl = videoInfoPage.get("dashmpd");
} else if (playerArgs.has("dashmpd")) {
dashManifestUrl = playerArgs.getString("dashmpd");
} else {
return "";
}
if(!dashManifestUrl.contains("/signature/")) {
String encryptedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifestUrl);
String decryptedSig;
decryptedSig = decryptSignature(encryptedSig, decryptionCode);
dashManifestUrl = dashManifestUrl.replace("/s/" + encryptedSig, "/signature/" + decryptedSig);
}
return dashManifestUrl;
} catch (Exception e) {
throw new ParsingException(
"Could not get \"dashmpd\" maybe VideoInfoPage is broken.", e);
}
}
@Override
public List<AudioStream> getAudioStreams() throws ParsingException {
Vector<AudioStream> audioStreams = new Vector<>();
try{
String encodedUrlMap;
// playerArgs could be null if the video is age restricted
if (playerArgs == null) {
if(videoInfoPage.containsKey("adaptive_fmts")) {
encodedUrlMap = videoInfoPage.get("adaptive_fmts");
} else {
return null;
}
} else {
if(playerArgs.has("adaptive_fmts")) {
encodedUrlMap = playerArgs.getString("adaptive_fmts");
} else {
return null;
}
}
for(String url_data_str : encodedUrlMap.split(",")) {
// This loop iterates through multiple streams, therefor tags
// is related to one and the same stream at a time.
Map<String, String> tags = Parser.compatParseMap(
org.jsoup.parser.Parser.unescapeEntities(url_data_str, true));
int itag = Integer.parseInt(tags.get("itag"));
if (itagIsSupported(itag)) {
ItagItem itagItem = getItagItem(itag);
if (itagItem.itagType == ItagType.AUDIO) {
String streamUrl = tags.get("url");
// if video has a signature: decrypt it and add it to the url
if (tags.get("s") != null) {
streamUrl = streamUrl + "&signature="
+ decryptSignature(tags.get("s"), decryptionCode);
}
audioStreams.add(new AudioStream(streamUrl,
itagItem.mediaFormatId,
itagItem.bandWidth,
itagItem.samplingRate));
}
}
}
} catch (Exception e) {
throw new ParsingException("Could not get audiostreams", e);
}
return audioStreams;
}
@Override
public List<VideoStream> getVideoStreams() throws ParsingException {
Vector<VideoStream> videoStreams = new Vector<>();
try{
String encodedUrlMap;
// playerArgs could be null if the video is age restricted
if (playerArgs == null) {
encodedUrlMap = videoInfoPage.get(URL_ENCODED_FMT_STREAM_MAP);
} else {
encodedUrlMap = playerArgs.getString(URL_ENCODED_FMT_STREAM_MAP);
}
for(String url_data_str : encodedUrlMap.split(",")) {
try {
// This loop iterates through multiple streams, therefor tags
// is related to one and the same stream at a time.
Map<String, String> tags = Parser.compatParseMap(
org.jsoup.parser.Parser.unescapeEntities(url_data_str, true));
int itag = Integer.parseInt(tags.get("itag"));
if (itagIsSupported(itag)) {
ItagItem itagItem = getItagItem(itag);
if(itagItem.itagType == ItagType.VIDEO) {
String streamUrl = tags.get("url");
// if video has a signature: decrypt it and add it to the url
if (tags.get("s") != null) {
streamUrl = streamUrl + "&signature="
+ decryptSignature(tags.get("s"), decryptionCode);
}
videoStreams.add(new VideoStream(
streamUrl,
itagItem.mediaFormatId,
itagItem.resolutionString));
}
}
} catch (Exception e) {
//todo: dont log throw an error
System.err.println("Could not get Video stream.");
e.printStackTrace();
}
}
} catch (Exception e) {
throw new ParsingException("Failed to get video streams", e);
}
if(videoStreams.isEmpty()) {
throw new ParsingException("Failed to get any video stream");
}
return videoStreams;
}
@Override
public List<VideoStream> getVideoOnlyStreams() throws ParsingException {
return null;
}
/**Attempts to parse (and return) the offset to start playing the video from.
* @return the offset (in seconds), or 0 if no timestamp is found.*/
@Override
public int getTimeStamp() throws ParsingException {
String timeStamp;
try {
timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl);
} catch (Parser.RegexException e) {
// catch this instantly since an url does not necessarily have to have a time stamp
// -2 because well the testing system will then know its the regex that failed :/
// not good i know
return -2;
}
if(!timeStamp.isEmpty()) {
try {
String secondsString = "";
String minutesString = "";
String hoursString = "";
try {
secondsString = Parser.matchGroup1("(\\d{1,3})s", timeStamp);
minutesString = Parser.matchGroup1("(\\d{1,3})m", timeStamp);
hoursString = Parser.matchGroup1("(\\d{1,3})h", timeStamp);
} catch (Exception e) {
//it could be that time is given in another method
if (secondsString.isEmpty() //if nothing was got,
&& minutesString.isEmpty()//treat as unlabelled seconds
&& hoursString.isEmpty()) {
secondsString = Parser.matchGroup1("t=(\\d+)", timeStamp);
}
}
int seconds = secondsString.isEmpty() ? 0 : Integer.parseInt(secondsString);
int minutes = minutesString.isEmpty() ? 0 : Integer.parseInt(minutesString);
int hours = hoursString.isEmpty() ? 0 : Integer.parseInt(hoursString);
//don't trust BODMAS!
return seconds + (60 * minutes) + (3600 * hours);
//Log.d(TAG, "derived timestamp value:"+ret);
//the ordering varies internationally
} catch (ParsingException e) {
throw new ParsingException("Could not get timestamp.", e);
}
} else {
return 0;
}
}
@Override
public int getAgeLimit() throws ParsingException {
if (!isAgeRestricted) {
return 0;
}
try {
return Integer.valueOf(doc.head()
.getElementsByAttributeValue("property", "og:restrictions:age")
.attr(CONTENT).replace("+", ""));
} catch (Exception e) {
throw new ParsingException("Could not get age restriction");
}
}
@Override
public String getAverageRating() throws ParsingException {
try {
if (playerArgs == null) {
return videoInfoPage.get("avg_rating");
}
return playerArgs.getString("avg_rating");
} catch (JSONException e) {
throw new ParsingException("Could not get Average rating", e);
}
}
@Override
public int getLikeCount() throws ParsingException {
String likesString = "";
try {
Element button = doc.select("button.like-button-renderer-like-button").first();
try {
likesString = button.select("span.yt-uix-button-content").first().text();
} catch (NullPointerException e) {
//if this ckicks in our button has no content and thefore likes/dislikes are disabled
return -1;
}
return Integer.parseInt(likesString.replaceAll(REGEX_INT, ""));
} catch (NumberFormatException nfe) {
throw new ParsingException(
"failed to parse likesString \"" + likesString + "\" as integers", nfe);
} catch (Exception e) {
throw new ParsingException("Could not get like count", e);
}
}
@Override
public int getDislikeCount() throws ParsingException {
String dislikesString = "";
try {
Element button = doc.select("button.like-button-renderer-dislike-button").first();
try {
dislikesString = button.select("span.yt-uix-button-content").first().text();
} catch (NullPointerException e) {
//if this kicks in our button has no content and therefore likes/dislikes are disabled
return -1;
}
return Integer.parseInt(dislikesString.replaceAll(REGEX_INT, ""));
} catch(NumberFormatException nfe) {
throw new ParsingException(
"failed to parse dislikesString \"" + dislikesString + "\" as integers", nfe);
} catch(Exception e) {
throw new ParsingException("Could not get dislike count", e);
}
}
@Override
public StreamInfoItemExtractor getNextVideo() throws ParsingException {
try {
return extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first()
.select("li").first());
} catch(Exception e) {
throw new ParsingException("Could not get next video", e);
}
}
@Override
public StreamInfoItemCollector getRelatedVideos() throws ParsingException {
try {
StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
Element ul = doc.select("ul[id=\"watch-related\"]").first();
if(ul != null) {
for (Element li : ul.children()) {
// first check if we have a playlist. If so leave them out
if (li.select("a[class*=\"content-link\"]").first() != null) {
collector.commit(extractVideoPreviewInfo(li));
}
}
}
return collector;
} catch(Exception e) {
throw new ParsingException("Could not get related videos", e);
}
}
@Override
public String getPageUrl() {
return pageUrl;
}
@Override
public String getChannelUrl() throws ParsingException {
try {
return doc.select("div[class=\"yt-user-info\"]").first().children()
.select("a").first().attr("abs:href");
} catch(Exception e) {
throw new ParsingException("Could not get channel link", e);
}
}
@Override
public StreamInfo.StreamType getStreamType() throws ParsingException {
//todo: if implementing livestream support this value should be generated dynamically
return StreamInfo.StreamType.VIDEO_STREAM;
}
/**Provides information about links to other videos on the video page, such as related videos.
* This is encapsulated in a StreamInfoItem object,
* which is a subset of the fields in a full StreamInfo.*/
private StreamInfoItemExtractor extractVideoPreviewInfo(final Element li) {
return new StreamInfoItemExtractor() {
@Override
public AbstractStreamInfo.StreamType getStreamType() throws ParsingException {
return AbstractStreamInfo.StreamType.VIDEO_STREAM;
}
@Override
public boolean isAd() throws ParsingException {
if(!li.select("span[class*=\"icon-not-available\"]").isEmpty()) {
return true;
} else {
return false;
}
}
@Override
public String getWebPageUrl() throws ParsingException {
return li.select("a.content-link").first().attr("abs:href");
}
@Override
public String getTitle() throws ParsingException {
//todo: check NullPointerException causing
return li.select("span.title").first().text();
//this page causes the NullPointerException, after finding it by searching for "tjvg":
//https://www.youtube.com/watch?v=Uqg0aEhLFAg
}
@Override
public int getDuration() throws ParsingException {
return YoutubeParsingHelper.parseDurationString(
li.select("span.video-time").first().text());
}
@Override
public String getUploader() throws ParsingException {
return li.select("span.g-hovercard").first().text();
}
@Override
public String getUploadDate() throws ParsingException {
return null;
}
@Override
public long getViewCount() throws ParsingException {
//this line is unused
//String views = li.select("span.view-count").first().text();
//Log.i(TAG, "title:"+info.title);
//Log.i(TAG, "view count:"+views);
try {
return Long.parseLong(li.select("span.view-count")
.first().text().replaceAll(REGEX_INT, ""));
} catch (Exception e) {
//related videos sometimes have no view count
return 0;
}
}
@Override
public String getThumbnailUrl() throws ParsingException {
Element img = li.select("img").first();
String thumbnailUrl = img.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we caught such an item.
if (thumbnailUrl.contains(".gif")) {
thumbnailUrl = img.attr("data-thumb");
}
if (thumbnailUrl.startsWith("//")) {
thumbnailUrl = HTTPS + thumbnailUrl;
}
return thumbnailUrl;
}
};
}
private String loadDecryptionCode(String playerUrl) throws DecryptException {
String decryptionFuncName;
String decryptionFunc;
String helperObjectName;
String helperObject;
String callerFunc = "function " + DECRYPTION_FUNC_NAME + "(a){return %%(a);}";
String decryptionCode;
try {
Downloader downloader = NewPipe.getDownloader();
if(!playerUrl.contains("https://youtube.com")) {
//sometimes the https://youtube.com part does not get send with
//than we have to add it by hand
playerUrl = "https://youtube.com" + playerUrl;
}
String playerCode = downloader.download(playerUrl);
decryptionFuncName =
Parser.matchGroup("([\"\\'])signature\\1\\s*,\\s*([a-zA-Z0-9$]+)\\(", playerCode, 2);
String functionPattern = "("
+ decryptionFuncName.replace("$", "\\$")
+ "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})";
decryptionFunc = "var " + Parser.matchGroup1(functionPattern, playerCode) + ";";
helperObjectName = Parser
.matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", decryptionFunc);
String helperPattern = "(var "
+ helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)";
helperObject = Parser.matchGroup1(helperPattern, playerCode);
callerFunc = callerFunc.replace("%%", decryptionFuncName);
decryptionCode = helperObject + decryptionFunc + callerFunc;
} catch(IOException ioe) {
throw new DecryptException("Could not load decrypt function", ioe);
} catch(Exception e) {
throw new DecryptException("Could not parse decrypt function ", e);
}
return decryptionCode;
}
private String decryptSignature(String encryptedSig, String decryptionCode)
throws DecryptException{
Context context = Context.enter();
context.setOptimizationLevel(-1);
Object result = null;
try {
ScriptableObject scope = context.initStandardObjects();
context.evaluateString(scope, decryptionCode, "decryptionCode", 1, null);
Function decryptionFunc = (Function) scope.get("decrypt", scope);
result = decryptionFunc.call(context, scope, scope, new Object[]{encryptedSig});
} catch (Exception e) {
throw new DecryptException("could not get decrypt signature", e);
} finally {
Context.exit();
}
return result == null ? "" : result.toString();
}
private String findErrorReason(Document doc) {
String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text();
if(errorMessage.contains("GEMA")) {
// Gema sometimes blocks youtube music content in germany:
// https://www.gema.de/en/
// Detailed description:
// https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
return "GEMA";
}
return "";
}
}

186
app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java
View File

@ -1,186 +0,0 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.AbstractStreamInfo;
import org.schabi.newpipe.extractor.Parser;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream_info.StreamInfoItemExtractor;
/**
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* YoutubeStreamInfoItemExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
private final Element item;
public YoutubeStreamInfoItemExtractor(Element item) throws FoundAdException {
this.item = item;
}
@Override
public String getWebPageUrl() throws ParsingException {
try {
Element el = item.select("div[class*=\"yt-lockup-video\"").first();
Element dl = el.select("h3").first().select("a").first();
return dl.attr("abs:href");
} catch (Exception e) {
throw new ParsingException("Could not get web page url for the video", e);
}
}
@Override
public String getTitle() throws ParsingException {
try {
Element el = item.select("div[class*=\"yt-lockup-video\"").first();
Element dl = el.select("h3").first().select("a").first();
return dl.text();
} catch (Exception e) {
throw new ParsingException("Could not get title", e);
}
}
@Override
public int getDuration() throws ParsingException {
try {
return YoutubeParsingHelper.parseDurationString(
item.select("span[class=\"video-time\"]").first().text());
} catch(Exception e) {
if(isLiveStream(item)) {
// -1 for no duration
return -1;
} else {
throw new ParsingException("Could not get Duration: " + getTitle(), e);
}
}
}
@Override
public String getUploader() throws ParsingException {
try {
return item.select("div[class=\"yt-lockup-byline\"]").first()
.select("a").first()
.text();
} catch (Exception e) {
throw new ParsingException("Could not get uploader", e);
}
}
@Override
public String getUploadDate() throws ParsingException {
try {
Element div = item.select("div[class=\"yt-lockup-meta\"]").first();
if(div == null) {
return null;
} else {
return div.select("li").first().text();
}
} catch(Exception e) {
throw new ParsingException("Could not get uplaod date", e);
}
}
@Override
public long getViewCount() throws ParsingException {
String output;
String input;
try {
Element div = item.select("div[class=\"yt-lockup-meta\"]").first();
if(div == null) {
return -1;
} else {
input = div.select("li").get(1)
.text();
}
} catch (IndexOutOfBoundsException e) {
if(isLiveStream(item)) {
// -1 for no view count
return -1;
} else {
throw new ParsingException(
"Could not parse yt-lockup-meta although available: " + getTitle(), e);
}
}
output = Parser.matchGroup1("([0-9,\\. ]*)", input)
.replace(" ", "")
.replace(".", "")
.replace(",", "");
try {
return Long.parseLong(output);
} catch (NumberFormatException e) {
// if this happens the video probably has no views
if(!input.isEmpty()) {
return 0;
} else {
throw new ParsingException("Could not handle input: " + input, e);
}
}
}
@Override
public String getThumbnailUrl() throws ParsingException {
try {
String url;
Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first()
.select("img").first();
url = te.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we've caught such an item.
if (url.contains(".gif")) {
url = te.attr("abs:data-thumb");
}
return url;
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
}
@Override
public AbstractStreamInfo.StreamType getStreamType() {
if(isLiveStream(item)) {
return AbstractStreamInfo.StreamType.LIVE_STREAM;
} else {
return AbstractStreamInfo.StreamType.VIDEO_STREAM;
}
}
@Override
public boolean isAd() throws ParsingException {
if(!item.select("span[class*=\"icon-not-available\"]").isEmpty()) {
return true;
} else {
return false;
}
}
private boolean isLiveStream(Element item) {
Element bla = item.select("span[class*=\"yt-badge-live\"]").first();
if(bla == null) {
// sometimes livestreams dont have badges but sill are live streams
// if video time is not available we most likly have an offline livestream
if(item.select("span[class*=\"video-time\"]").first() == null) {
return true;
}
}
return bla != null;
}
}

163
app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java
View File

@ -1,163 +0,0 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Parser;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLDecoder;
/**
* Created by Christian Schabesberger on 02.02.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* YoutubeStreamUrlIdHandler.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeStreamUrlIdHandler implements UrlIdHandler {
private static final YoutubeStreamUrlIdHandler instance = new YoutubeStreamUrlIdHandler();
private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{11})";
private YoutubeStreamUrlIdHandler() {}
public static YoutubeStreamUrlIdHandler getInstance() {
return instance;
}
@Override
public String getUrl(String videoId) {
return "https://www.youtube.com/watch?v=" + videoId;
}
@Override
public String getId(String url) throws ParsingException, IllegalArgumentException {
if(url.isEmpty()) {
throw new IllegalArgumentException("The url parameter should not be empty");
}
String id;
String lowercaseUrl = url.toLowerCase();
if(lowercaseUrl.contains("youtube")) {
if (url.contains("attribution_link")) {
try {
String escapedQuery = Parser.matchGroup1("u=(.[^&|$]*)", url);
String query = URLDecoder.decode(escapedQuery, "UTF-8");
id = Parser.matchGroup1("v=" + ID_PATTERN, query);
} catch (UnsupportedEncodingException uee) {
throw new ParsingException("Could not parse attribution_link", uee);
}
} else if(lowercaseUrl.contains("youtube.com/shared?ci=")) {
return getRealIdFromSharedLink(url);
} else if (url.contains("vnd.youtube")) {
id = Parser.matchGroup1(ID_PATTERN, url);
} else if (url.contains("embed")) {
id = Parser.matchGroup1("embed/" + ID_PATTERN, url);
} else if(url.contains("googleads")) {
throw new FoundAdException("Error found add: " + url);
} else {
id = Parser.matchGroup1("[?&]v=" + ID_PATTERN, url);
}
}
else if(lowercaseUrl.contains("youtu.be")) {
if(url.contains("v=")) {
id = Parser.matchGroup1("v=" + ID_PATTERN, url);
} else {
id = Parser.matchGroup1("[Yy][Oo][Uu][Tt][Uu]\\.[Bb][Ee]/" + ID_PATTERN, url);
}
}
else {
throw new ParsingException("Error no suitable url: " + url);
}
if(!id.isEmpty()){
return id;
} else {
throw new ParsingException("Error could not parse url: " + url);
}
}
/**
* Get the real url from a shared uri.
*
* Shared URI's look like this:
* <pre>
* * https://www.youtube.com/shared?ci=PJICrTByb3E
* * vnd.youtube://www.youtube.com/shared?ci=PJICrTByb3E&feature=twitter-deep-link
* </pre>
* @param url The shared url
* @return the id of the stream
* @throws ParsingException
*/
private String getRealIdFromSharedLink(String url) throws ParsingException {
URI uri;
try {
uri = new URI(url);
} catch (URISyntaxException e) {
throw new ParsingException("Invalid shared link", e);
}
String sharedId = getSharedId(uri);
Downloader downloader = NewPipe.getDownloader();
String content;
try {
content = downloader.download("https://www.youtube.com/shared?ci=" + sharedId);
} catch (IOException | ReCaptchaException e) {
throw new ParsingException("Unable to resolve shared link", e);
}
// is this bad? is this fragile?:
String realId = Parser.matchGroup1("rel=\"shortlink\" href=\"https://youtu.be/" + ID_PATTERN, content);
if(sharedId.equals(realId)) {
throw new ParsingException("Got same id for as shared id: " + sharedId);
}
return realId;
}
private String getSharedId(URI uri) throws ParsingException {
if (!"/shared".equals(uri.getPath())) {
throw new ParsingException("Not a shared link: " + uri.toString() + " (path != " + uri.getPath() + ")");
}
return Parser.matchGroup1("ci=" + ID_PATTERN, uri.getQuery());
}
public String cleanUrl(String complexUrl) throws ParsingException {
return getUrl(getId(complexUrl));
}
@Override
public boolean acceptUrl(String videoUrl) {
String lowercaseUrl = videoUrl.toLowerCase();
if(lowercaseUrl.contains("youtube") ||
lowercaseUrl.contains("youtu.be")) {
// bad programming I know
try {
getId(videoUrl);
return true;
} catch (Exception e) {
return false;
}
} else {
return false;
}
}
}

99
app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java
View File

@ -1,99 +0,0 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
/**
* Created by Christian Schabesberger on 28.09.16.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* YoutubeSuggestionExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeSuggestionExtractor extends SuggestionExtractor {
public static final String CHARSET_UTF_8 = "UTF-8";
public YoutubeSuggestionExtractor(int serviceId) {
super(serviceId);
}
@Override
public List<String> suggestionList(
String query, String contentCountry)
throws ExtractionException, IOException {
List<String> suggestions = new ArrayList<>();
Downloader dl = NewPipe.getDownloader();
String url = "https://suggestqueries.google.com/complete/search"
+ "?client=" + ""
+ "&output=" + "toolbar"
+ "&ds=" + "yt"
+ "&hl=" + URLEncoder.encode(contentCountry, CHARSET_UTF_8)
+ "&q=" + URLEncoder.encode(query, CHARSET_UTF_8);
String response = dl.download(url);
//TODO: Parse xml data using Jsoup not done
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder dBuilder;
org.w3c.dom.Document doc = null;
try {
dBuilder = dbFactory.newDocumentBuilder();
doc = dBuilder.parse(new InputSource(
new ByteArrayInputStream(response.getBytes(CHARSET_UTF_8))));
doc.getDocumentElement().normalize();
} catch (ParserConfigurationException | SAXException | IOException e) {
throw new ParsingException("Could not parse document.");
}
try {
NodeList nList = doc.getElementsByTagName("CompleteSuggestion");
for (int temp = 0; temp < nList.getLength(); temp++) {
NodeList nList1 = doc.getElementsByTagName("suggestion");
Node nNode1 = nList1.item(temp);
if (nNode1.getNodeType() == Node.ELEMENT_NODE) {
org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1;
suggestions.add(eElement.getAttribute("data"));
}
}
return suggestions;
} catch(Exception e) {
throw new ParsingException("Could not get suggestions form document.", e);
}
}
}

46
app/src/main/java/org/schabi/newpipe/extractor/stream_info/AudioStream.java
View File

@ -1,46 +0,0 @@
package org.schabi.newpipe.extractor.stream_info;
/**
* Created by Christian Schabesberger on 04.03.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* AudioStream.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class AudioStream {
public String url = "";
public int format = -1;
public int bandwidth = -1;
public int sampling_rate = -1;
public AudioStream(String url, int format, int bandwidth, int samplingRate) {
this.url = url; this.format = format;
this.bandwidth = bandwidth; this.sampling_rate = samplingRate;
}
// reveals whether two streams are the same, but have different urls
public boolean equalStats(AudioStream cmp) {
return format == cmp.format
&& bandwidth == cmp.bandwidth
&& sampling_rate == cmp.sampling_rate;
}
// reveals whether two streams are equal
public boolean equals(AudioStream cmp) {
return cmp != null && equalStats(cmp)
&& url == cmp.url;
}
}

104
app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamExtractor.java
View File

@ -1,104 +0,0 @@
package org.schabi.newpipe.extractor.stream_info;
/**
* Created by Christian Schabesberger on 10.08.15.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* StreamExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.util.List;
/**Scrapes information from a video streaming service (eg, YouTube).*/
@SuppressWarnings("ALL")
public abstract class StreamExtractor {
private int serviceId;
private String url;
private UrlIdHandler urlIdHandler;
private StreamInfoItemCollector previewInfoCollector;
public class ExtractorInitException extends ExtractionException {
public ExtractorInitException(String message) {
super(message);
}
public ExtractorInitException(Throwable cause) {
super(cause);
}
public ExtractorInitException(String message, Throwable cause) {
super(message, cause);
}
}
public class ContentNotAvailableException extends ParsingException {
public ContentNotAvailableException(String message) {
super(message);
}
public ContentNotAvailableException(String message, Throwable cause) {
super(message, cause);
}
}
public StreamExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) {
this.serviceId = serviceId;
this.urlIdHandler = urlIdHandler;
previewInfoCollector = new StreamInfoItemCollector(urlIdHandler, serviceId);
}
protected StreamInfoItemCollector getStreamPreviewInfoCollector() {
return previewInfoCollector;
}
public String getUrl() {
return url;
}
public UrlIdHandler getUrlIdHandler() {
return urlIdHandler;
}
public abstract int getTimeStamp() throws ParsingException;
public abstract String getTitle() throws ParsingException;
public abstract String getDescription() throws ParsingException;
public abstract String getUploader() throws ParsingException;
public abstract String getChannelUrl() throws ParsingException;
public abstract int getLength() throws ParsingException;
public abstract long getViewCount() throws ParsingException;
public abstract String getUploadDate() throws ParsingException;
public abstract String getThumbnailUrl() throws ParsingException;
public abstract String getUploaderThumbnailUrl() throws ParsingException;
public abstract List<AudioStream> getAudioStreams() throws ParsingException;
public abstract List<VideoStream> getVideoStreams() throws ParsingException;
public abstract List<VideoStream> getVideoOnlyStreams() throws ParsingException;
public abstract String getDashMpdUrl() throws ParsingException;
public abstract int getAgeLimit() throws ParsingException;
public abstract String getAverageRating() throws ParsingException;
public abstract int getLikeCount() throws ParsingException;
public abstract int getDislikeCount() throws ParsingException;
public abstract StreamInfoItemExtractor getNextVideo() throws ParsingException;
public abstract StreamInfoItemCollector getRelatedVideos() throws ParsingException;
public abstract String getPageUrl();
public abstract StreamInfo.StreamType getStreamType() throws ParsingException;
public int getServiceId() {
return serviceId;
}
}

294
app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamInfo.java
View File

@ -1,294 +0,0 @@
package org.schabi.newpipe.extractor.stream_info;
import org.schabi.newpipe.extractor.AbstractStreamInfo;
import org.schabi.newpipe.extractor.DashMpdParser;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.io.IOException;
import java.util.List;
import java.util.Vector;
/**
* Created by Christian Schabesberger on 26.08.15.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* StreamInfo.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/**Info object for opened videos, ie the video ready to play.*/
@SuppressWarnings("ALL")
public class StreamInfo extends AbstractStreamInfo {
public static class StreamExctractException extends ExtractionException {
StreamExctractException(String message) {
super(message);
}
}
public StreamInfo() {}
/**Creates a new StreamInfo object from an existing AbstractVideoInfo.
* All the shared properties are copied to the new StreamInfo.*/
@SuppressWarnings("WeakerAccess")
public StreamInfo(AbstractStreamInfo avi) {
this.id = avi.id;
this.title = avi.title;
this.uploader = avi.uploader;
this.thumbnail_url = avi.thumbnail_url;
this.webpage_url = avi.webpage_url;
this.upload_date = avi.upload_date;
this.upload_date = avi.upload_date;
this.view_count = avi.view_count;
//todo: better than this
if(avi instanceof StreamInfoItem) {
//shitty String to convert code
/*
String dur = ((StreamInfoItem)avi).duration;
int minutes = Integer.parseInt(dur.substring(0, dur.indexOf(":")));
int seconds = Integer.parseInt(dur.substring(dur.indexOf(":")+1, dur.length()));
*/
this.duration = ((StreamInfoItem)avi).duration;
}
}
public void addException(Exception e) {
errors.add(e);
}
/**Fills out the video info fields which are common to all services.
* Probably needs to be overridden by subclasses*/
public static StreamInfo getVideoInfo(StreamExtractor extractor)
throws ExtractionException, IOException {
StreamInfo streamInfo = new StreamInfo();
streamInfo = extractImportantData(streamInfo, extractor);
streamInfo = extractStreams(streamInfo, extractor);
streamInfo = extractOptionalData(streamInfo, extractor);
return streamInfo;
}
private static StreamInfo extractImportantData(
StreamInfo streamInfo, StreamExtractor extractor)
throws ExtractionException, IOException {
/* ---- important data, withoug the video can't be displayed goes here: ---- */
// if one of these is not available an exception is meant to be thrown directly into the frontend.
UrlIdHandler uiconv = extractor.getUrlIdHandler();
streamInfo.service_id = extractor.getServiceId();
streamInfo.webpage_url = extractor.getPageUrl();
streamInfo.stream_type = extractor.getStreamType();
streamInfo.id = uiconv.getId(extractor.getPageUrl());
streamInfo.title = extractor.getTitle();
streamInfo.age_limit = extractor.getAgeLimit();
if((streamInfo.stream_type == StreamType.NONE)
|| (streamInfo.webpage_url == null || streamInfo.webpage_url.isEmpty())
|| (streamInfo.id == null || streamInfo.id.isEmpty())
|| (streamInfo.title == null /* streamInfo.title can be empty of course */)
|| (streamInfo.age_limit == -1)) {
throw new ExtractionException("Some importand stream information was not given.");
}
return streamInfo;
}
private static StreamInfo extractStreams(
StreamInfo streamInfo, StreamExtractor extractor)
throws ExtractionException, IOException {
/* ---- stream extraction goes here ---- */
// At least one type of stream has to be available,
// otherwise an exception will be thrown directly into the frontend.
try {
streamInfo.dashMpdUrl = extractor.getDashMpdUrl();
} catch(Exception e) {
streamInfo.addException(new ExtractionException("Couldn't get Dash manifest", e));
}
/* Load and extract audio */
try {
streamInfo.audio_streams = extractor.getAudioStreams();
} catch(Exception e) {
streamInfo.addException(new ExtractionException("Couldn't get audio streams", e));
}
// also try to get streams from the dashMpd
if(streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) {
if(streamInfo.audio_streams == null) {
streamInfo.audio_streams = new Vector<>();
}
//todo: make this quick and dirty solution a real fallback
// same as the quick and dirty above
try {
streamInfo.audio_streams.addAll(
DashMpdParser.getAudioStreams(streamInfo.dashMpdUrl));
} catch(Exception e) {
streamInfo.addException(
new ExtractionException("Couldn't get audio streams from dash mpd", e));
}
}
/* Extract video stream url*/
try {
streamInfo.video_streams = extractor.getVideoStreams();
} catch (Exception e) {
streamInfo.addException(
new ExtractionException("Couldn't get video streams", e));
}
/* Extract video only stream url*/
try {
streamInfo.video_only_streams = extractor.getVideoOnlyStreams();
} catch(Exception e) {
streamInfo.addException(
new ExtractionException("Couldn't get video only streams", e));
}
// either dash_mpd audio_only or video has to be available, otherwise we didn't get a stream,
// and therefore failed. (Since video_only_streams are just optional they don't caunt).
if((streamInfo.video_streams == null || streamInfo.video_streams.isEmpty())
&& (streamInfo.audio_streams == null || streamInfo.audio_streams.isEmpty())
&& (streamInfo.dashMpdUrl == null || streamInfo.dashMpdUrl.isEmpty())) {
throw new StreamExctractException(
"Could not get any stream. See error variable to get further details.");
}
return streamInfo;
}
private static StreamInfo extractOptionalData(
StreamInfo streamInfo, StreamExtractor extractor) {
/* ---- optional data goes here: ---- */
// If one of these fails, the frontend needs to handle that they are not available.
// Exceptions are therefore not thrown into the frontend, but stored into the error List,
// so the frontend can afterwards check where errors happened.
try {
streamInfo.thumbnail_url = extractor.getThumbnailUrl();
} catch(Exception e) {
streamInfo.addException(e);
}
try {
streamInfo.duration = extractor.getLength();
} catch(Exception e) {
streamInfo.addException(e);
}
try {
streamInfo.uploader = extractor.getUploader();
} catch(Exception e) {
streamInfo.addException(e);
}
try {
streamInfo.channel_url = extractor.getChannelUrl();
} catch(Exception e) {
streamInfo.addException(e);
}
try {
streamInfo.description = extractor.getDescription();
} catch(Exception e) {
streamInfo.addException(e);
}
try {
streamInfo.view_count = extractor.getViewCount();
} catch(Exception e) {
streamInfo.addException(e);
}
try {
streamInfo.upload_date = extractor.getUploadDate();
} catch(Exception e) {
streamInfo.addException(e);
}
try {
streamInfo.uploader_thumbnail_url = extractor.getUploaderThumbnailUrl();
} catch(Exception e) {
streamInfo.addException(e);
}
try {
streamInfo.start_position = extractor.getTimeStamp();
} catch(Exception e) {
streamInfo.addException(e);
}
try {
streamInfo.average_rating = extractor.getAverageRating();
} catch(Exception e) {
streamInfo.addException(e);
}
try {
streamInfo.like_count = extractor.getLikeCount();
} catch(Exception e) {
streamInfo.addException(e);
}
try {
streamInfo.dislike_count = extractor.getDislikeCount();
} catch(Exception e) {
streamInfo.addException(e);
}
try {
// get next video
if(streamInfo.next_video != null)
{
StreamInfoItemCollector c = new StreamInfoItemCollector(
extractor.getUrlIdHandler(), extractor.getServiceId());
StreamInfoItemExtractor nextVideo = extractor.getNextVideo();
c.commit(nextVideo);
if(c.getItemList().size() != 0) {
streamInfo.next_video = (StreamInfoItem) c.getItemList().get(0);
}
streamInfo.errors.addAll(c.getErrors());
}
}
catch(Exception e) {
streamInfo.addException(e);
}
try {
// get related videos
StreamInfoItemCollector c = extractor.getRelatedVideos();
streamInfo.related_streams = c.getItemList();
streamInfo.errors.addAll(c.getErrors());
} catch(Exception e) {
streamInfo.addException(e);
}
return streamInfo;
}
public String uploader_thumbnail_url = "";
public String channel_url = "";
public String description = "";
public List<VideoStream> video_streams = null;
public List<AudioStream> audio_streams = null;
public List<VideoStream> video_only_streams = null;
// video streams provided by the dash mpd do not need to be provided as VideoStream.
// Later on this will also aplly to audio streams. Since dash mpd is standarized,
// crawling such a file is not service dependent. Therefore getting audio only streams by yust
// providing the dash mpd fille will be possible in the future.
public String dashMpdUrl = "";
public int duration = -1;
public int age_limit = -1;
public int like_count = -1;
public int dislike_count = -1;
public String average_rating = "";
public StreamInfoItem next_video = null;
public List<InfoItem> related_streams = null;
//in seconds. some metadata is not passed using a StreamInfo object!
public int start_position = 0;
public List<Throwable> errors = new Vector<>();
}

41
app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamInfoItem.java
View File

@ -1,41 +0,0 @@
package org.schabi.newpipe.extractor.stream_info;
/**
* Created by Christian Schabesberger on 26.08.15.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* StreamInfoItem.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import org.schabi.newpipe.extractor.AbstractStreamInfo;
import org.schabi.newpipe.extractor.InfoItem;
/**Info object for previews of unopened videos, eg search results, related videos*/
public class StreamInfoItem extends AbstractStreamInfo implements InfoItem {
public int duration;
public InfoType infoType() {
return InfoType.STREAM;
}
public String getTitle() {
return title;
}
public String getLink() {
return webpage_url;
}
}

102
app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamInfoItemCollector.java
View File

@ -1,102 +0,0 @@
package org.schabi.newpipe.extractor.stream_info;
import org.schabi.newpipe.extractor.InfoItemCollector;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.util.List;
import java.util.Vector;
/**
* Created by Christian Schabesberger on 28.02.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* StreamInfoItemCollector.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class StreamInfoItemCollector extends InfoItemCollector {
private UrlIdHandler urlIdHandler;
public StreamInfoItemCollector(UrlIdHandler handler, int serviceId) {
super(serviceId);
urlIdHandler = handler;
}
private UrlIdHandler getUrlIdHandler() {
return urlIdHandler;
}
public StreamInfoItem extract(StreamInfoItemExtractor extractor) throws Exception {
if(extractor.isAd()) {
throw new FoundAdException("Found ad");
}
StreamInfoItem resultItem = new StreamInfoItem();
// importand information
resultItem.service_id = getServiceId();
resultItem.webpage_url = extractor.getWebPageUrl();
if (getUrlIdHandler() == null) {
throw new ParsingException("Error: UrlIdHandler not set");
} else if (!resultItem.webpage_url.isEmpty()) {
resultItem.id = NewPipe.getService(getServiceId())
.getStreamUrlIdHandlerInstance()
.getId(resultItem.webpage_url);
}
resultItem.title = extractor.getTitle();
resultItem.stream_type = extractor.getStreamType();
// optional information
try {
resultItem.duration = extractor.getDuration();
} catch (Exception e) {
addError(e);
}
try {
resultItem.uploader = extractor.getUploader();
} catch (Exception e) {
addError(e);
}
try {
resultItem.upload_date = extractor.getUploadDate();
} catch (Exception e) {
addError(e);
}
try {
resultItem.view_count = extractor.getViewCount();
} catch (Exception e) {
addError(e);
}
try {
resultItem.thumbnail_url = extractor.getThumbnailUrl();
} catch (Exception e) {
addError(e);
}
return resultItem;
}
public void commit(StreamInfoItemExtractor extractor) throws ParsingException {
try {
addItem(extract(extractor));
} catch(FoundAdException ae) {
//System.out.println("AD_WARNING: " + ae.getMessage());
} catch (Exception e) {
addError(e);
}
}
}

36
app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamInfoItemExtractor.java
View File

@ -1,36 +0,0 @@
package org.schabi.newpipe.extractor.stream_info;
import org.schabi.newpipe.extractor.AbstractStreamInfo;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
/**
* Created by Christian Schabesberger on 28.02.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* StreamInfoItemExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public interface StreamInfoItemExtractor {
AbstractStreamInfo.StreamType getStreamType() throws ParsingException;
String getWebPageUrl() throws ParsingException;
String getTitle() throws ParsingException;
int getDuration() throws ParsingException;
String getUploader() throws ParsingException;
String getUploadDate() throws ParsingException;
long getViewCount() throws ParsingException;
String getThumbnailUrl() throws ParsingException;
boolean isAd() throws ParsingException;
}

44
app/src/main/java/org/schabi/newpipe/extractor/stream_info/VideoStream.java
View File

@ -1,44 +0,0 @@
package org.schabi.newpipe.extractor.stream_info;
/**
* Created by Christian Schabesberger on 04.03.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* VideoStream.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class VideoStream {
//url of the stream
public String url = "";
public int format = -1;
public String resolution = "";
public VideoStream(String url, int format, String res) {
this.url = url; this.format = format; resolution = res;
}
// reveals wether two streams are the same, but have diferent urls
public boolean equalStats(VideoStream cmp) {
return format == cmp.format
&& resolution == cmp.resolution;
}
// revelas wether two streams are equal
public boolean equals(VideoStream cmp) {
return cmp != null && equalStats(cmp)
&& url == cmp.url;
}
}