lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Create InfoItemExtractor

This commit is contained in:
Mauricio Colli 2017-08-11 15:21:49 -03:00
parent b719e59fae
commit 82824cdd72
28 changed files with 90 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/org/schabi/newpipe/extractor/Info.java
View File

@ -16,4 +16,9 @@ public abstract class Info implements Serializable {
public String name;
public List<Throwable> errors = new ArrayList<>();
@Override
public String toString() {
return getClass().getSimpleName() + "[url=\"" + url + "\", name=\"" + name + "\"]";
}
}

6
src/main/java/org/schabi/newpipe/extractor/InfoItem.java
View File

@ -38,4 +38,10 @@ public abstract class InfoItem implements Serializable {
public int service_id = -1;
public String url;
public String name;
public String thumbnail_url;
@Override
public String toString() {
return getClass().getSimpleName() + "[url=\"" + url + "\", name=\"" + name + "\"]";
}
}

9
src/main/java/org/schabi/newpipe/extractor/InfoItemExtractor.java Normal file
View File

@ -0,0 +1,9 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
public interface InfoItemExtractor {
String getName() throws ParsingException;
String getUrl() throws ParsingException;
String getThumbnailUrl() throws ParsingException;
}

2
src/main/java/org/schabi/newpipe/extractor/StreamingService.java
View File

@ -1,10 +1,10 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import java.io.IOException;

2
src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java
View File

@ -108,5 +108,5 @@ public class ChannelInfo extends ListInfo {
public String banner_url;
public String feed_url;
public long subscriber_count = -1;
public String description = "";
public String description;
}

1
src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItem.java
View File

@ -24,7 +24,6 @@ import org.schabi.newpipe.extractor.InfoItem;
public class ChannelInfoItem extends InfoItem {
public String thumbnail_url;
public String description;
public long subscriber_count = -1;
public long stream_count = -1;

4
src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemCollector.java
View File

@ -32,8 +32,8 @@ public class ChannelInfoItemCollector extends InfoItemCollector {
ChannelInfoItem resultItem = new ChannelInfoItem();
// important information
resultItem.service_id = getServiceId();
resultItem.name = extractor.getChannelName();
resultItem.url = extractor.getWebPageUrl();
resultItem.name = extractor.getName();
resultItem.url = extractor.getUrl();
// optional information
try {

7
src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemExtractor.java
View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
/*
@ -22,11 +23,9 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public interface ChannelInfoItemExtractor {
String getThumbnailUrl() throws ParsingException;
String getChannelName() throws ParsingException;
String getWebPageUrl() throws ParsingException;
public interface ChannelInfoItemExtractor extends InfoItemExtractor {
String getDescription() throws ParsingException;
long getSubscriberCount() throws ParsingException;
long getStreamCount() throws ParsingException;
}

1
src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItem.java
View File

@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.InfoItem;
public class PlaylistInfoItem extends InfoItem {
public String uploader_name;
public String thumbnail_url;
/**
* How many streams this playlist have
*/

4
src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemCollector.java
View File

@ -11,9 +11,9 @@ public class PlaylistInfoItemCollector extends InfoItemCollector {
public PlaylistInfoItem extract(PlaylistInfoItemExtractor extractor) throws ParsingException {
final PlaylistInfoItem resultItem = new PlaylistInfoItem();
resultItem.name = extractor.getPlaylistName();
resultItem.name = extractor.getName();
resultItem.service_id = getServiceId();
resultItem.url = extractor.getWebPageUrl();
resultItem.url = extractor.getUrl();
try {
resultItem.uploader_name = extractor.getUploaderName();

6
src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemExtractor.java
View File

@ -1,11 +1,9 @@
package org.schabi.newpipe.extractor.playlist;
import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
public interface PlaylistInfoItemExtractor {
String getThumbnailUrl() throws ParsingException;
public interface PlaylistInfoItemExtractor extends InfoItemExtractor {
String getUploaderName() throws ParsingException;
String getPlaylistName() throws ParsingException;
String getWebPageUrl() throws ParsingException;
long getStreamCount() throws ParsingException;
}

2
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java
View File

@ -72,7 +72,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Override
public long getSubscriberCount() {
return user.optLong("followers_count", 0L);
return user.optLong("followers_count");
}
@Override

24
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java
View File

@ -10,29 +10,29 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
this.searchResult = searchResult;
}
@Override
public String getName() {
return searchResult.getString("username");
}
@Override
public String getUrl() {
return searchResult.getString("permalink_url");
}
@Override
public String getThumbnailUrl() {
return searchResult.optString("avatar_url");
}
@Override
public String getChannelName() {
return searchResult.getString("username");
}
@Override
public String getWebPageUrl() {
return searchResult.getString("permalink_url");
}
@Override
public long getSubscriberCount() {
return searchResult.optLong("followers_count", 0L);
return searchResult.optLong("followers_count");
}
@Override
public long getStreamCount() {
return searchResult.getLong("track_count");
return searchResult.optLong("track_count");
}
@Override

8
src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java
View File

@ -14,18 +14,18 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
}
@Override
public String getWebPageUrl() {
public String getUrl() {
return searchResult.getString("permalink_url");
}
@Override
public String getTitle() {
public String getName() {
return searchResult.getString("title");
}
@Override
public int getDuration() {
return searchResult.getInt("duration") / 1000;
public long getDuration() {
return searchResult.getLong("duration") / 1000L;
}
@Override

10
src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java
View File

@ -3,14 +3,8 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import static org.schabi.newpipe.extractor.MediaFormat.M4A;
import static org.schabi.newpipe.extractor.MediaFormat.MPEG_4;
import static org.schabi.newpipe.extractor.MediaFormat.WEBM;
import static org.schabi.newpipe.extractor.MediaFormat.WEBMA;
import static org.schabi.newpipe.extractor.MediaFormat.v3GPP;
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.AUDIO;
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.VIDEO;
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.VIDEO_ONLY;
import static org.schabi.newpipe.extractor.MediaFormat.*;
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.*;
public class ItagItem {
/**

6
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java
View File

@ -196,7 +196,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
if (li.select("div[class=\"feed-item-dismissable\"]").first() != null) {
collector.commit(new YoutubeStreamInfoItemExtractor(li) {
@Override
public String getWebPageUrl() throws ParsingException {
public String getUrl() throws ParsingException {
try {
Element el = li.select("div[class=\"feed-item-dismissable\"]").first();
Element dl = el.select("h3").first().select("a").first();
@ -207,7 +207,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
@Override
public String getTitle() throws ParsingException {
public String getName() throws ParsingException {
try {
Element el = li.select("div[class=\"feed-item-dismissable\"]").first();
Element dl = el.select("h3").first().select("a").first();
@ -219,7 +219,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getUploaderName() throws ParsingException {
return getName();
return YoutubeChannelExtractor.this.getName();
}
@Override

4
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java
View File

@ -46,13 +46,13 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
}
@Override
public String getChannelName() throws ParsingException {
public String getName() throws ParsingException {
return el.select("a[class*=\"yt-uix-tile-link\"]").first()
.text();
}
@Override
public String getWebPageUrl() throws ParsingException {
public String getUrl() throws ParsingException {
return el.select("a[class*=\"yt-uix-tile-link\"]").first()
.attr("abs:href");
}

10
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -28,7 +28,7 @@ public class YoutubeParsingHelper {
private YoutubeParsingHelper() {
}
public static int parseDurationString(String input)
public static long parseDurationString(String input)
throws ParsingException, NumberFormatException {
String[] splitInput = input.split(":");
String days = "0";
@ -58,9 +58,9 @@ public class YoutubeParsingHelper {
default:
throw new ParsingException("Error duration string with unknown format: " + input);
}
return ((((Integer.parseInt(days) * 24)
+ Integer.parseInt(hours) * 60)
+ Integer.parseInt(minutes)) * 60)
+ Integer.parseInt(seconds);
return ((((Long.parseLong(days) * 24)
+ Long.parseLong(hours) * 60)
+ Long.parseLong(minutes)) * 60)
+ Long.parseLong(seconds);
}
}

10
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java
View File

@ -205,7 +205,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
@Override
public String getWebPageUrl() throws ParsingException {
public String getUrl() throws ParsingException {
try {
return streamUrlIdHandler.getUrl(li.attr("data-video-id"));
} catch (Exception e) {
@ -214,7 +214,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
@Override
public String getTitle() throws ParsingException {
public String getName() throws ParsingException {
try {
return li.attr("data-title");
} catch (Exception e) {
@ -223,7 +223,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
@Override
public int getDuration() throws ParsingException {
public long getDuration() throws ParsingException {
try {
if (getStreamType() == StreamType.LIVE_STREAM) return -1;
@ -236,7 +236,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
return YoutubeParsingHelper.parseDurationString(first.text());
} catch (Exception e) {
throw new ParsingException("Could not get Duration: " + getTitle(), e);
throw new ParsingException("Could not get duration" + getUrl(), e);
}
}
@ -258,7 +258,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getThumbnailUrl() throws ParsingException {
try {
return "https://i.ytimg.com/vi/" + streamUrlIdHandler.getId(getWebPageUrl()) + "/hqdefault.jpg";
return "https://i.ytimg.com/vi/" + streamUrlIdHandler.getId(getUrl()) + "/hqdefault.jpg";
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}

2
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View File

@ -3,11 +3,11 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.SuggestionExtractor;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import java.io.IOException;

4
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java
View File

@ -725,12 +725,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return new YoutubeStreamInfoItemExtractor(li) {
@Override
public String getWebPageUrl() throws ParsingException {
public String getUrl() throws ParsingException {
return li.select("a.content-link").first().attr("abs:href");
}
@Override
public String getTitle() throws ParsingException {
public String getName() throws ParsingException {
//todo: check NullPointerException causing
return li.select("span.title").first().text();
//this page causes the NullPointerException, after finding it by searching for "tjvg":

10
src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java
View File

@ -48,7 +48,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
}
@Override
public String getWebPageUrl() throws ParsingException {
public String getUrl() throws ParsingException {
try {
Element el = item.select("div[class*=\"yt-lockup-video\"").first();
Element dl = el.select("h3").first().select("a").first();
@ -59,7 +59,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
}
@Override
public String getTitle() throws ParsingException {
public String getName() throws ParsingException {
try {
Element el = item.select("div[class*=\"yt-lockup-video\"").first();
Element dl = el.select("h3").first().select("a").first();
@ -70,13 +70,13 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
}
@Override
public int getDuration() throws ParsingException {
public long getDuration() throws ParsingException {
try {
if (getStreamType() == StreamType.LIVE_STREAM) return -1;
return YoutubeParsingHelper.parseDurationString(item.select("span[class*=\"video-time\"]").first().text());
} catch (Exception e) {
throw new ParsingException("Could not get Duration: " + getTitle(), e);
throw new ParsingException("Could not get Duration: " + getUrl(), e);
}
}
@ -116,7 +116,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
input = meta.select("li").get(1).text();
} catch (IndexOutOfBoundsException e) {
throw new ParsingException("Could not parse yt-lockup-meta although available: " + getTitle(), e);
throw new ParsingException("Could not parse yt-lockup-meta although available: " + getUrl(), e);
}
try {

8
src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java
View File

@ -1,10 +1,6 @@
package org.schabi.newpipe.extractor.stream;
import org.schabi.newpipe.extractor.Info;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.utils.DashMpdParser;
@ -261,6 +257,7 @@ public class StreamInfo extends Info {
public String upload_date;
public long duration = -1;
public int age_limit = -1;
public String description;
public long view_count = -1;
public long like_count = -1;
@ -283,5 +280,4 @@ public class StreamInfo extends Info {
public List<InfoItem> related_streams;
//in seconds. some metadata is not passed using a StreamInfo object!
public long start_position = 0;
public String description = "";
}

3
src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItem.java
View File

@ -29,10 +29,9 @@ public class StreamInfoItem extends InfoItem {
public StreamType stream_type;
public String uploader_name;
public String thumbnail_url;
public String upload_date;
public long view_count = -1;
public int duration = -1;
public long duration = -1;
public StreamInfoItem() {
super(InfoType.STREAM);

5
src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemCollector.java
View File

@ -38,9 +38,8 @@ public class StreamInfoItemCollector extends InfoItemCollector {
StreamInfoItem resultItem = new StreamInfoItem();
// important information
resultItem.service_id = getServiceId();
resultItem.url = extractor.getWebPageUrl();
resultItem.name = extractor.getTitle();
resultItem.url = extractor.getUrl();
resultItem.name = extractor.getName();
resultItem.stream_type = extractor.getStreamType();
// optional information

15
src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemExtractor.java
View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.stream;
import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
/*
@ -22,14 +23,14 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public interface StreamInfoItemExtractor {
public interface StreamInfoItemExtractor extends InfoItemExtractor {
StreamType getStreamType() throws ParsingException;
String getWebPageUrl() throws ParsingException;
String getTitle() throws ParsingException;
int getDuration() throws ParsingException;
boolean isAd() throws ParsingException;
long getDuration() throws ParsingException;
long getViewCount() throws ParsingException;
String getUploaderName() throws ParsingException;
String getUploadDate() throws ParsingException;
long getViewCount() throws ParsingException;
String getThumbnailUrl() throws ParsingException;
boolean isAd() throws ParsingException;
}

5
src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java
View File

@ -14,13 +14,12 @@ import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
/*
* Created by Christian Schabesberger on 02.02.16.
*

4
src/test/java/org/schabi/newpipe/Downloader.java
View File

@ -2,6 +2,7 @@ package org.schabi.newpipe;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import javax.net.ssl.HttpsURLConnection;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
@ -11,8 +12,6 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import javax.net.ssl.HttpsURLConnection;
/*
* Created by Christian Schabesberger on 28.01.16.
@ -106,6 +105,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
BufferedReader in = null;
try {
con.setConnectTimeout(30 * 1000);// 30s
con.setReadTimeout(30 * 1000);// 30s
con.setRequestMethod("GET");
con.setRequestProperty("User-Agent", USER_AGENT);