lasers
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Bandcmap: Use nanojson fork instead of org.json

This commit is contained in:
Fynn Godau 2020-03-19 11:18:29 +01:00
parent 9c239371f4
commit b100b9873f
11 changed files with 142 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
extractor/build.gradle
View File

@ -1,5 +1,3 @@
dependencies {
implementation project(':timeago-parser')
@ -8,7 +6,6 @@ dependencies {
implementation 'org.mozilla:rhino:1.7.7.1'
implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0'
implementation 'org.nibor.autolink:autolink:0.8.0'
implementation 'org.json:json:20190722'
testImplementation 'junit:junit:4.12'
}
}

31
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampChannelExtractor.java
View File

@ -2,11 +2,11 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
@ -23,7 +23,7 @@ import java.io.IOException;
public class BandcampChannelExtractor extends ChannelExtractor {
private JSONObject channelInfo;
private JsonObject channelInfo;
public BandcampChannelExtractor(StreamingService service, ListLinkHandler linkHandler) throws ParsingException {
super(service, linkHandler);
@ -36,17 +36,17 @@ public class BandcampChannelExtractor extends ChannelExtractor {
* <a href=https://notabug.org/fynngodau/bandcampDirect/wiki/rewindBandcamp+%E2%80%93+Fetching+artist+details>
* I once took a moment to note down how it works.</a>
*/
public static JSONObject getArtistDetails(String id) throws ParsingException {
public static JsonObject getArtistDetails(String id) throws ParsingException {
try {
return
new JSONObject(
JsonParser.object().from(
NewPipe.getDownloader().post(
"https://bandcamp.com/api/mobile/22/band_details",
null,
("{\"band_id\":\"" + id + "\"}").getBytes()
).responseBody()
);
} catch (IOException | ReCaptchaException e) {
} catch (IOException | ReCaptchaException | JsonParserException e) {
throw new ParsingException("Could not download band details", e);
}
}
@ -63,12 +63,9 @@ public class BandcampChannelExtractor extends ChannelExtractor {
@Override
public String getAvatarUrl() {
try {
return getImageUrl(channelInfo.getLong("bio_image_id"), false);
} catch (JSONException e) {
// In this case, the id is null and no image is available
return "";
}
if (channelInfo.getLong("bio_image_id") == 0) return "";
return getImageUrl(channelInfo.getLong("bio_image_id"), false);
}
/**
@ -120,11 +117,11 @@ public class BandcampChannelExtractor extends ChannelExtractor {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
JSONArray discography = channelInfo.getJSONArray("discography");
JsonArray discography = channelInfo.getArray("discography");
for (int i = 0; i < discography.length(); i++) {
for (int i = 0; i < discography.size(); i++) {
// I define discograph as an item that can appear in a discography
JSONObject discograph = discography.getJSONObject(i);
JsonObject discograph = discography.getObject(i);
if (!discograph.getString("item_type").equals("track")) continue;

15
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java
View File

@ -2,8 +2,9 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import org.json.JSONException;
import org.json.JSONObject;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
@ -24,7 +25,7 @@ public class BandcampExtractorHelper {
* @param variable Name of the variable
* @return The JsonObject stored in the variable with this name
*/
public static JSONObject getJSONFromJavaScriptVariables(String html, String variable) throws JSONException, ArrayIndexOutOfBoundsException, ParsingException {
public static JsonObject getJSONFromJavaScriptVariables(String html, String variable) throws JsonParserException, ArrayIndexOutOfBoundsException, ParsingException {
String[] part = html.split("var " + variable + " = ");
@ -44,7 +45,7 @@ public class BandcampExtractorHelper {
case '}':
level--;
if (level == 0) {
return new JSONObject(firstHalfGone.substring(0, position + 1)
return JsonParser.object().from(firstHalfGone.substring(0, position + 1)
.replaceAll(" {4}//.+", "") // Remove "for the curious" in JSON
.replaceAll("// xxx: note - don't internationalize this variable", "") // Remove this comment
);
@ -62,14 +63,14 @@ public class BandcampExtractorHelper {
public static String getStreamUrlFromIds(long bandId, long itemId, String itemType) throws ParsingException {
try {
String html = NewPipe.getDownloader().get(
String jsonString = NewPipe.getDownloader().get(
"https://bandcamp.com/api/mobile/22/tralbum_details?band_id=" + bandId
+ "&tralbum_id=" + itemId + "&tralbum_type=" + itemType.substring(0, 1))
.responseBody();
return new JSONObject(html).getString("bandcamp_url").replace("http://", "https://");
return JsonParser.object().from(jsonString).getString("bandcamp_url").replace("http://", "https://");
} catch (JSONException | ReCaptchaException | IOException e) {
} catch (JsonParserException | ReCaptchaException | IOException e) {
throw new ParsingException("Ids could not be translated to URL", e);
}

57
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java
View File

@ -2,8 +2,10 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import org.json.JSONArray;
import org.json.JSONObject;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Collector;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.InfoItemsCollector;
@ -49,34 +51,41 @@ public class BandcampFeaturedExtractor extends KioskExtractor<InfoItem> {
InfoItemsCollector c = new PlaylistInfoItemsCollector(getServiceId());
JSONObject json = new JSONObject(
getDownloader().post(
FEATURED_API_URL, null, "{\"platform\":\"\",\"version\":0}".getBytes()
).responseBody()
);
try {
JSONArray featuredStories = json.getJSONObject("feed_content")
.getJSONObject("stories")
.getJSONArray("featured");
for (int i = 0; i < featuredStories.length(); i++) {
JSONObject featuredStory = featuredStories.getJSONObject(i);
JsonObject json = JsonParser.object().from(
getDownloader().post(
FEATURED_API_URL, null, "{\"platform\":\"\",\"version\":0}".getBytes()
).responseBody()
);
if (featuredStory.isNull("album_title")) {
// Is not an album, ignore
continue;
JsonArray featuredStories = json.getObject("feed_content")
.getObject("stories")
.getArray("featured");
for (int i = 0; i < featuredStories.size(); i++) {
JsonObject featuredStory = featuredStories.getObject(i);
if (featuredStory.isNull("album_title")) {
// Is not an album, ignore
continue;
}
c.commit(new BandcampPlaylistInfoItemExtractor(
featuredStory.getString("album_title"),
featuredStory.getString("band_name"),
featuredStory.getString("item_url"),
featuredStory.has("art_id") ? getImageUrl(featuredStory.getLong("art_id"), true) : "",
featuredStory.getInt("num_streamable_tracks")
));
}
c.commit(new BandcampPlaylistInfoItemExtractor(
featuredStory.getString("album_title"),
featuredStory.getString("band_name"),
featuredStory.getString("item_url"),
featuredStory.has("art_id") ? getImageUrl(featuredStory.getLong("art_id"), true) : "",
featuredStory.getInt("num_streamable_tracks")
));
return new InfoItemsPage<InfoItem>(c, null);
} catch (JsonParserException e) {
e.printStackTrace();
throw new ParsingException("JSON error", e);
}
return new InfoItemsPage<InfoItem>(c, null);
}
@Override

24
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java
View File

@ -1,8 +1,8 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.StreamingService;
@ -32,8 +32,8 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor {
private static final int MAXIMUM_INDIVIDUAL_COVER_ARTS = 10;
private Document document;
private JSONObject albumJson;
private JSONArray trackInfo;
private JsonObject albumJson;
private JsonArray trackInfo;
private String name;
public BandcampPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
@ -45,11 +45,11 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor {
String html = downloader.get(getLinkHandler().getUrl()).responseBody();
document = Jsoup.parse(html);
albumJson = getAlbumInfoJson(html);
trackInfo = albumJson.getJSONArray("trackinfo");
trackInfo = albumJson.getArray("trackinfo");
try {
name = getJSONFromJavaScriptVariables(html, "EmbedData").getString("album_title");
} catch (JSONException e) {
} catch (JsonParserException e) {
throw new ParsingException("Faulty JSON; page likely does not contain album data", e);
} catch (ArrayIndexOutOfBoundsException e) {
throw new ParsingException("JSON does not exist", e);
@ -57,7 +57,7 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor {
if (trackInfo.length() <= 0) {
if (trackInfo.size() <= 0) {
// Albums without trackInfo need to be purchased before they can be played
throw new ContentNotAvailableException("Album needs to be purchased");
}
@ -97,7 +97,7 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor {
@Override
public long getStreamCount() {
return trackInfo.length();
return trackInfo.size();
}
@Nonnull
@ -106,10 +106,10 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
for (int i = 0; i < trackInfo.length(); i++) {
JSONObject track = trackInfo.getJSONObject(i);
for (int i = 0; i < trackInfo.size(); i++) {
JsonObject track = trackInfo.getObject(i);
if (trackInfo.length() < MAXIMUM_INDIVIDUAL_COVER_ARTS) {
if (trackInfo.size() < MAXIMUM_INDIVIDUAL_COVER_ARTS) {
// Load cover art of every track individually
collector.commit(new BandcampStreamInfoItemExtractor(
track.getString("title"),

38
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java
View File

@ -2,8 +2,10 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import org.json.JSONArray;
import org.json.JSONObject;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.StreamingService;
@ -12,7 +14,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
@ -43,20 +45,26 @@ public class BandcampRadioExtractor extends KioskExtractor<InfoItem> {
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
InfoItemsCollector c = new StreamInfoItemsCollector(getServiceId());
JSONObject json = new JSONObject(
getDownloader().get(
RADIO_API_URL
).responseBody()
);
try {
JSONArray radioShows = json.getJSONArray("results");
for (int i = 0; i < radioShows.length(); i++) {
JSONObject radioShow = radioShows.getJSONObject(i);
c.commit(
new BandcampRadioInfoItemExtractor(radioShow)
JsonObject json = JsonParser.object().from(
getDownloader().get(
RADIO_API_URL
).responseBody()
);
JsonArray radioShows = json.getArray("results");
for (int i = 0; i < radioShows.size(); i++) {
JsonObject radioShow = radioShows.getObject(i);
c.commit(
new BandcampRadioInfoItemExtractor(radioShow)
);
}
} catch (JsonParserException e) {
e.printStackTrace();
}
return new InfoItemsPage<InfoItem>(c, null);

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java
View File

@ -2,7 +2,7 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import org.json.JSONObject;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
@ -14,9 +14,9 @@ import static org.schabi.newpipe.extractor.services.bandcamp.extractors.Bandcamp
public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor {
private JSONObject show;
private JsonObject show;
public BandcampRadioInfoItemExtractor(JSONObject radioShow) {
public BandcampRadioInfoItemExtractor(JsonObject radioShow) {
show = radioShow;
}

14
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java
View File

@ -1,6 +1,8 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import org.json.JSONObject;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
@ -23,7 +25,7 @@ import static org.schabi.newpipe.extractor.services.bandcamp.extractors.Bandcamp
public class BandcampRadioStreamExtractor extends BandcampStreamExtractor {
private JSONObject showInfo;
private JsonObject showInfo;
private LinkHandler linkHandler;
public BandcampRadioStreamExtractor(StreamingService service, LinkHandler linkHandler) {
@ -31,12 +33,12 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor {
this.linkHandler = linkHandler;
}
static JSONObject query(int id) throws ParsingException {
static JsonObject query(int id) throws ParsingException {
try {
return new JSONObject(
return JsonParser.object().from(
NewPipe.getDownloader().get("https://bandcamp.com/api/bcweekly/1/get?id=" + id).responseBody()
);
} catch (IOException | ReCaptchaException e) {
} catch (IOException | ReCaptchaException | JsonParserException e) {
throw new ParsingException("could not get show data", e);
}
}
@ -104,7 +106,7 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor {
@Override
public List<AudioStream> getAudioStreams() {
ArrayList<AudioStream> list = new ArrayList<>();
JSONObject streams = showInfo.getJSONObject("audio_stream");
JsonObject streams = showInfo.getObject("audio_stream");
if (streams.has("opus-lo")) {
list.add(new AudioStream(

38
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java
View File

@ -2,8 +2,8 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import org.json.JSONException;
import org.json.JSONObject;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
@ -28,8 +28,8 @@ import static org.schabi.newpipe.extractor.services.bandcamp.extractors.Bandcamp
public class BandcampStreamExtractor extends StreamExtractor {
private JSONObject albumJson;
private JSONObject current;
private JsonObject albumJson;
private JsonObject current;
private Document document;
public BandcampStreamExtractor(StreamingService service, LinkHandler linkHandler) {
@ -42,9 +42,9 @@ public class BandcampStreamExtractor extends StreamExtractor {
String html = downloader.get(getLinkHandler().getUrl()).responseBody();
document = Jsoup.parse(html);
albumJson = getAlbumInfoJson(html);
current = albumJson.getJSONObject("current");
current = albumJson.getObject("current");
if (albumJson.getJSONArray("trackinfo").length() > 1) {
if (albumJson.getArray("trackinfo").size() > 1) {
// In this case, we are actually viewing an album page!
throw new ExtractionException("Page is actually an album, not a track");
}
@ -57,10 +57,10 @@ public class BandcampStreamExtractor extends StreamExtractor {
* @return Album metadata JSON
* @throws ParsingException In case of a faulty website
*/
public static JSONObject getAlbumInfoJson(String html) throws ParsingException {
public static JsonObject getAlbumInfoJson(String html) throws ParsingException {
try {
return BandcampExtractorHelper.getJSONFromJavaScriptVariables(html, "TralbumData");
} catch (JSONException e) {
} catch (JsonParserException e) {
throw new ParsingException("Faulty JSON; page likely does not contain album data", e);
} catch (ArrayIndexOutOfBoundsException e) {
throw new ParsingException("JSON does not exist", e);
@ -127,26 +127,14 @@ public class BandcampStreamExtractor extends StreamExtractor {
public Description getDescription() {
String s = BandcampExtractorHelper.smartConcatenate(
new String[]{
getStringOrNull(current, "about"),
getStringOrNull(current, "lyrics"),
getStringOrNull(current, "credits")
current.getString("about"),
current.getString("lyrics"),
current.getString("credits")
}, "\n\n"
);
return new Description(s, Description.PLAIN_TEXT);
}
/**
* Avoid exceptions like "<code>JSONObject["about"] not a string.</code>" and instead just return null.
* This is for the case that the actual JSON has something like <code>"about": null</code>.
*/
private String getStringOrNull(JSONObject jsonObject, String value) {
try {
return jsonObject.getString(value);
} catch (JSONException e) {
return null;
}
}
@Override
public int getAgeLimit() throws ParsingException {
return 0;
@ -194,8 +182,8 @@ public class BandcampStreamExtractor extends StreamExtractor {
List<AudioStream> audioStreams = new ArrayList<>();
audioStreams.add(new AudioStream(
albumJson.getJSONArray("trackinfo").getJSONObject(0)
.getJSONObject("file").getString("mp3-128"),
albumJson.getArray("trackinfo").getObject(0)
.getObject("file").getString("mp3-128"),
MediaFormat.MP3, 128
));
return audioStreams;

36
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java
View File

@ -2,8 +2,10 @@
package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import org.json.JSONArray;
import org.json.JSONObject;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -28,24 +30,30 @@ public class BandcampSuggestionExtractor extends SuggestionExtractor {
public List<String> suggestionList(String query) throws IOException, ExtractionException {
Downloader downloader = NewPipe.getDownloader();
JSONObject fuzzyResults = new JSONObject(
downloader.get(AUTOCOMPLETE_URL + URLEncoder.encode(query, CHARSET_UTF_8)).responseBody()
);
try {
JsonObject fuzzyResults = JsonParser.object().from(
downloader.get(AUTOCOMPLETE_URL + URLEncoder.encode(query, CHARSET_UTF_8)).responseBody()
);
JSONArray jsonArray = fuzzyResults.getJSONObject("auto")
.getJSONArray("results");
JsonArray jsonArray = fuzzyResults.getObject("auto")
.getArray("results");
ArrayList<String> suggestions = new ArrayList<>();
ArrayList<String> suggestions = new ArrayList<>();
for (int i = 0; i < jsonArray.length(); i++) {
JSONObject fuzzyResult = jsonArray.getJSONObject(i);
String res = fuzzyResult.getString("name");
for (int i = 0; i < jsonArray.size(); i++) {
JsonObject fuzzyResult = jsonArray.getObject(i);
String res = fuzzyResult.getString("name");
if (!suggestions.contains(res)) suggestions.add(res);
if (!suggestions.contains(res)) suggestions.add(res);
}
return suggestions;
} catch (JsonParserException e) {
e.printStackTrace();
return new ArrayList<>();
}
return suggestions;
}
}

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java
View File

@ -2,8 +2,8 @@
package org.schabi.newpipe.extractor.services.bandcamp.linkHandler;
import org.json.JSONException;
import org.json.JSONObject;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
@ -26,11 +26,11 @@ public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory {
String response = NewPipe.getDownloader().get(url).responseBody();
// This variable contains band data!
JSONObject bandData = BandcampExtractorHelper.getJSONFromJavaScriptVariables(response, "BandData");
JsonObject bandData = BandcampExtractorHelper.getJSONFromJavaScriptVariables(response, "BandData");
return String.valueOf(bandData.getLong("id"));
} catch (IOException | ReCaptchaException | ArrayIndexOutOfBoundsException e) {
} catch (IOException | ReCaptchaException | ArrayIndexOutOfBoundsException | JsonParserException e) {
throw new ParsingException("Download failed", e);
}
}
@ -44,7 +44,7 @@ public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory {
return BandcampChannelExtractor.getArtistDetails(id)
.getString("bandcamp_url")
.replace("http://", "https://");
} catch (JSONException e) {
} catch (NullPointerException e) {
throw new ParsingException("JSON does not contain URL (invalid id?) or is otherwise invalid", e);
}