lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

add parse linkspan

This commit is contained in:
Christian Schabesberger 2018-05-15 22:17:57 +02:00
parent 0501a2f543
commit e4a2dc8018
4 changed files with 71 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java
View File

@ -20,6 +20,7 @@ package org.schabi.newpipe.extractor.stream;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import org.nibor.autolink.LinkSpan;
import org.schabi.newpipe.extractor.Extractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.Subtitles;
@ -143,6 +144,9 @@ public abstract class StreamExtractor extends Extractor {
public abstract String[] getDonationLinks() throws ExtractionException;
public abstract String[] getAffiliateLinks() throws ExtractionException;
public LinkSpan[] getLinksFromDescription() throws ExtractionException {
return Parser.getLinkSpanFromString(getDescription());
}
/**
* Analyses the webpage's document and extracts any error message there might be.

43
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java
View File

@ -1,10 +1,14 @@
package org.schabi.newpipe.extractor.stream;
import org.nibor.autolink.LinkSpan;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.DashMpdParser;
import org.schabi.newpipe.extractor.utils.DonationLinkHelper;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.util.ArrayList;
@ -62,6 +66,7 @@ public class StreamInfo extends Info {
streamInfo = extractImportantData(extractor);
streamInfo = extractStreams(streamInfo, extractor);
streamInfo = extractOptionalData(streamInfo, extractor);
streamInfo = extractLinks(streamInfo, extractor);
} catch (ExtractionException e) {
// Currently YouTube does not distinguish between age restricted videos and videos blocked
// by country. This means that during the initialisation of the extractor, the extractor
@ -171,6 +176,25 @@ public class StreamInfo extends Info {
return streamInfo;
}
private static StreamInfo extractLinks(StreamInfo streamInfo, StreamExtractor extractor) throws ParsingException {
try {
streamInfo.setLinksInDescription(extractor.getLinksFromDescription());
} catch (Exception e) {
e.printStackTrace();
}
try {
streamInfo.setAffiliateLinks(extractor.getAffiliateLinks());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setDonationLinks(extractor.getDonationLinks());
} catch (Exception e) {
streamInfo.addError(e);
}
}
private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtractor extractor) {
/* ---- optional data goes here: ---- */
// If one of these fails, the frontend needs to handle that they are not available.
@ -242,16 +266,6 @@ public class StreamInfo extends Info {
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setAffiliateLinks(extractor.getAffiliateLinks());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setDonationLinks(extractor.getDonationLinks());
} catch (Exception e) {
streamInfo.addError(e);
}
streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor));
return streamInfo;
@ -284,6 +298,7 @@ public class StreamInfo extends Info {
private long startPosition = 0;
private List<Subtitles> subtitles;
private LinkSpan[] linksInDescription;
private String[] donationLinks;
private String[] affiliateLinks;
@ -495,4 +510,12 @@ public class StreamInfo extends Info {
public void setAffiliateLinks(String[] affiliateLinks) {
this.affiliateLinks = affiliateLinks;
}
public LinkSpan[] getLinksInDescription() {
return linksInDescription;
}
public void setLinksInDescription(LinkSpan[] linksInDescription) {
this.linksInDescription = linksInDescription;
}
}

44
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java
View File

@ -88,20 +88,38 @@ public class Parser {
public static String[] getLinksFromString(final String txt) throws ParsingException {
try {
ArrayList<String> links = new ArrayList<>();
LinkExtractor linkExtractor = LinkExtractor.builder()
.linkTypes(EnumSet.of(LinkType.URL, LinkType.WWW))
.build();
Iterable<LinkSpan> linkss = linkExtractor.extractLinks(txt);
for(LinkSpan ls : linkss) {
links.add(txt.substring(ls.getBeginIndex(), ls.getEndIndex()));
}
String[] linksarray = new String[links.size()];
linksarray = links.toArray(linksarray);
return linksarray;
return getLinksFromLinkSpan(getLinkSpanFromString(txt), txt);
} catch (Exception e) {
throw new ParsingException("Could not get links from string", e);
}
}
}
public static LinkSpan[] getLinkSpanFromString(final String txt) throws ParsingException {
try {
ArrayList<LinkSpan> linksSpans = new ArrayList<>();
LinkExtractor linkExtractor = LinkExtractor.builder()
.linkTypes(EnumSet.of(LinkType.URL, LinkType.WWW))
.build();
for (LinkSpan ls : linkExtractor.extractLinks(txt)) {
linksSpans.add(ls);
}
LinkSpan[] linksarray = new LinkSpan[linksSpans.size()];
return linksSpans.toArray(linksarray);
} catch (Exception e) {
throw new ParsingException("Could not get links from string", e);
}
}
public static String[] getLinksFromLinkSpan(final LinkSpan[] links, final String txt) throws ParsingException {
try {
ArrayList<String> linksout = new ArrayList<>();
for (LinkSpan ls : links) {
linksout.add(txt.substring(ls.getBeginIndex(), ls.getEndIndex()));
}
String[] linksarray = new String[linksout.size()];
return linksout.toArray(linksarray);
} catch (Exception e) {
throw new ParsingException("Could not get LinkSpans from string", e);
}
}
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java
View File

@ -6,6 +6,9 @@ import java.util.List;
public class Utils {
private static final String HTTP = "http://";
private static final String HTTPS = "https://";
private Utils() {
//no instance
}
@ -46,9 +49,6 @@ public class Utils {
}
}
private static final String HTTP = "http://";
private static final String HTTPS = "https://";
public static String replaceHttpWithHttps(final String url) {
if (url == null) return null;