lasers
/
NewPipe
mirror of https://github.com/TeamNewPipe/NewPipe
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Space reserving tweaks for huge video resolutions 

* improve space reserving, allows write better 4K/8K video data
* do not use cache dirs in the muxers, Android can force close NewPipe if the device is running out of storage. Is a aggressive cache cleaning >:/
* (for devs) webm & mkv are the same thing
* calculate the final file size inside of the mission, instead getting from the UI
* simplify ps algorithms constructors
* [missing old commit message] simplify the loading of pending downloads
This commit is contained in:
kapodamy 2019-04-25 00:34:29 -03:00
parent 34b2b96158
commit 7b948f83c3
11 changed files with 608 additions and 550 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java
View File

@ -767,7 +767,8 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck
psArgs = null; psArgs = null;
long videoSize = wrappedVideoStreams.getSizeInBytes((VideoStream) selectedStream); long videoSize = wrappedVideoStreams.getSizeInBytes((VideoStream) selectedStream);
// set nearLength, only, if both sizes are fetched or known. this probably does not work on slow networks // set nearLength, only, if both sizes are fetched or known. This probably
// does not work on slow networks but is later updated in the downloader
if (secondaryStream.getSizeInBytes() > 0 && videoSize > 0) { if (secondaryStream.getSizeInBytes() > 0 && videoSize > 0) {
nearLength = secondaryStream.getSizeInBytes() + videoSize; nearLength = secondaryStream.getSizeInBytes() + videoSize;
} }
@ -793,7 +794,7 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck
if (secondaryStreamUrl == null) { if (secondaryStreamUrl == null) {
urls = new String[]{selectedStream.getUrl()}; urls = new String[]{selectedStream.getUrl()};
} else { } else {
urls = new String[]{secondaryStreamUrl, selectedStream.getUrl()}; urls = new String[]{selectedStream.getUrl(), secondaryStreamUrl};
} }
DownloadManagerService.startMission(context, urls, storage, kind, threads, currentInfo.getUrl(), psName, psArgs, nearLength); DownloadManagerService.startMission(context, urls, storage, kind, threads, currentInfo.getUrl(), psName, psArgs, nearLength);

349
app/src/main/java/us/shandian/giga/get/DownloadInitializer.java
View File

@ -1,158 +1,191 @@
package us.shandian.giga.get; package us.shandian.giga.get;
import android.support.annotation.NonNull; import android.support.annotation.NonNull;
import android.util.Log; import android.util.Log;
import org.schabi.newpipe.streams.io.SharpStream; import org.schabi.newpipe.streams.io.SharpStream;
import java.io.IOException; import java.io.IOException;
import java.io.InterruptedIOException; import java.io.InterruptedIOException;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.nio.channels.ClosedByInterruptException; import java.nio.channels.ClosedByInterruptException;
import us.shandian.giga.util.Utility; import us.shandian.giga.util.Utility;
import static org.schabi.newpipe.BuildConfig.DEBUG; import static org.schabi.newpipe.BuildConfig.DEBUG;
public class DownloadInitializer extends Thread { public class DownloadInitializer extends Thread {
private final static String TAG = "DownloadInitializer"; private final static String TAG = "DownloadInitializer";
final static int mId = 0; final static int mId = 0;
private final static int RESERVE_SPACE_DEFAULT = 5 * 1024 * 1024;// 5 MiB
private DownloadMission mMission; private final static int RESERVE_SPACE_MAXIMUM = 150 * 1024 * 1024;// 150 MiB
private HttpURLConnection mConn;
private DownloadMission mMission;
DownloadInitializer(@NonNull DownloadMission mission) { private HttpURLConnection mConn;
mMission = mission;
mConn = null; DownloadInitializer(@NonNull DownloadMission mission) {
} mMission = mission;
mConn = null;
@Override }
public void run() {
if (mMission.current > 0) mMission.resetState(false, true, DownloadMission.ERROR_NOTHING); @Override
public void run() {
int retryCount = 0; if (mMission.current > 0) mMission.resetState(false, true, DownloadMission.ERROR_NOTHING);
while (true) {
try { int retryCount = 0;
mMission.currentThreadCount = mMission.threadCount; while (true) {
try {
mConn = mMission.openConnection(mId, -1, -1); mMission.currentThreadCount = mMission.threadCount;
mMission.establishConnection(mId, mConn);
if (mMission.blocks < 0 && mMission.current == 0) {
if (!mMission.running || Thread.interrupted()) return; // calculate the whole size of the mission
long finalLength = 0;
mMission.length = Utility.getContentLength(mConn); long lowestSize = Long.MAX_VALUE;
for (int i = 0; i < mMission.urls.length && mMission.running; i++) {
if (mMission.length == 0) { mConn = mMission.openConnection(mMission.urls[i], mId, -1, -1);
mMission.notifyError(DownloadMission.ERROR_HTTP_NO_CONTENT, null); mMission.establishConnection(mId, mConn);
return;
} if (Thread.interrupted()) return;
long length = Utility.getContentLength(mConn);
// check for dynamic generated content
if (mMission.length == -1 && mConn.getResponseCode() == 200) { if (i == 0) mMission.length = length;
mMission.blocks = 0; if (length > 0) finalLength += length;
mMission.length = 0; if (length < lowestSize) lowestSize = length;
mMission.fallback = true; }
mMission.unknownLength = true;
mMission.currentThreadCount = 1; mMission.nearLength = finalLength;
if (DEBUG) { // reserve space at the start of the file
Log.d(TAG, "falling back (unknown length)"); if (mMission.psAlgorithm != null && mMission.psAlgorithm.reserveSpace) {
} if (lowestSize < 1) {
} else { // the length is unknown use the default size
// Open again mMission.offsets[0] = RESERVE_SPACE_DEFAULT;
mConn = mMission.openConnection(mId, mMission.length - 10, mMission.length); } else {
mMission.establishConnection(mId, mConn); // use the smallest resource size to download, otherwise, use the maximum
mMission.offsets[0] = lowestSize < RESERVE_SPACE_MAXIMUM ? lowestSize : RESERVE_SPACE_MAXIMUM;
if (!mMission.running || Thread.interrupted()) return; }
}
synchronized (mMission.blockState) { } else {
if (mConn.getResponseCode() == 206) { // ask for the current resource length
if (mMission.currentThreadCount > 1) { mConn = mMission.openConnection(mId, -1, -1);
mMission.blocks = mMission.length / DownloadMission.BLOCK_SIZE; mMission.establishConnection(mId, mConn);
if (mMission.currentThreadCount > mMission.blocks) { if (!mMission.running || Thread.interrupted()) return;
mMission.currentThreadCount = (int) mMission.blocks;
} mMission.length = Utility.getContentLength(mConn);
if (mMission.currentThreadCount <= 0) { }
mMission.currentThreadCount = 1;
} if (mMission.length == 0 || mConn.getResponseCode() == 204) {
if (mMission.blocks * DownloadMission.BLOCK_SIZE < mMission.length) { mMission.notifyError(DownloadMission.ERROR_HTTP_NO_CONTENT, null);
mMission.blocks++; return;
} }
} else {
// if one thread is solicited don't calculate blocks, is useless // check for dynamic generated content
mMission.blocks = 1; if (mMission.length == -1 && mConn.getResponseCode() == 200) {
mMission.fallback = true; mMission.blocks = 0;
mMission.unknownLength = false; mMission.length = 0;
} mMission.fallback = true;
mMission.unknownLength = true;
if (DEBUG) { mMission.currentThreadCount = 1;
Log.d(TAG, "http response code = " + mConn.getResponseCode());
} if (DEBUG) {
} else { Log.d(TAG, "falling back (unknown length)");
// Fallback to single thread }
mMission.blocks = 0; } else {
mMission.fallback = true; // Open again
mMission.unknownLength = false; mConn = mMission.openConnection(mId, mMission.length - 10, mMission.length);
mMission.currentThreadCount = 1; mMission.establishConnection(mId, mConn);
if (DEBUG) { if (!mMission.running || Thread.interrupted()) return;
Log.d(TAG, "falling back due http response code = " + mConn.getResponseCode());
} synchronized (mMission.blockState) {
} if (mConn.getResponseCode() == 206) {
if (mMission.currentThreadCount > 1) {
for (long i = 0; i < mMission.currentThreadCount; i++) { mMission.blocks = mMission.length / DownloadMission.BLOCK_SIZE;
mMission.threadBlockPositions.add(i);
mMission.threadBytePositions.add(0L); if (mMission.currentThreadCount > mMission.blocks) {
} mMission.currentThreadCount = (int) mMission.blocks;
} }
if (mMission.currentThreadCount <= 0) {
if (!mMission.running || Thread.interrupted()) return; mMission.currentThreadCount = 1;
} }
if (mMission.blocks * DownloadMission.BLOCK_SIZE < mMission.length) {
SharpStream fs = mMission.storage.getStream(); mMission.blocks++;
fs.setLength(mMission.offsets[mMission.current] + mMission.length); }
fs.seek(mMission.offsets[mMission.current]); } else {
fs.close(); // if one thread is solicited don't calculate blocks, is useless
mMission.blocks = 1;
if (!mMission.running || Thread.interrupted()) return; mMission.fallback = true;
mMission.unknownLength = false;
mMission.running = false; }
break;
} catch (InterruptedIOException | ClosedByInterruptException e) { if (DEBUG) {
return; Log.d(TAG, "http response code = " + mConn.getResponseCode());
} catch (Exception e) { }
if (!mMission.running) return; } else {
// Fallback to single thread
if (e instanceof IOException && e.getMessage().contains("Permission denied")) { mMission.blocks = 0;
mMission.notifyError(DownloadMission.ERROR_PERMISSION_DENIED, e); mMission.fallback = true;
return; mMission.unknownLength = false;
} mMission.currentThreadCount = 1;
if (retryCount++ > mMission.maxRetry) { if (DEBUG) {
Log.e(TAG, "initializer failed", e); Log.d(TAG, "falling back due http response code = " + mConn.getResponseCode());
mMission.notifyError(e); }
return; }
}
for (long i = 0; i < mMission.currentThreadCount; i++) {
Log.e(TAG, "initializer failed, retrying", e); mMission.threadBlockPositions.add(i);
} mMission.threadBytePositions.add(0L);
} }
}
mMission.start();
} if (!mMission.running || Thread.interrupted()) return;
}
@Override
public void interrupt() { SharpStream fs = mMission.storage.getStream();
super.interrupt(); fs.setLength(mMission.offsets[mMission.current] + mMission.length);
fs.seek(mMission.offsets[mMission.current]);
if (mConn != null) { fs.close();
try {
mConn.disconnect(); if (!mMission.running || Thread.interrupted()) return;
} catch (Exception e) {
// nothing to do mMission.running = false;
} break;
} } catch (InterruptedIOException | ClosedByInterruptException e) {
} return;
} } catch (Exception e) {
if (!mMission.running) return;
if (e instanceof IOException && e.getMessage().contains("Permission denied")) {
mMission.notifyError(DownloadMission.ERROR_PERMISSION_DENIED, e);
return;
}
if (retryCount++ > mMission.maxRetry) {
Log.e(TAG, "initializer failed", e);
mMission.notifyError(e);
return;
}
Log.e(TAG, "initializer failed, retrying", e);
}
}
mMission.start();
}
@Override
public void interrupt() {
super.interrupt();
if (mConn != null) {
try {
mConn.disconnect();
} catch (Exception e) {
// nothing to do
}
}
}
}

30
app/src/main/java/us/shandian/giga/get/DownloadMission.java
View File

@ -147,14 +147,10 @@ public class DownloadMission extends Mission {
this.enqueued = true; this.enqueued = true;
this.maxRetry = 3; this.maxRetry = 3;
this.storage = storage; this.storage = storage;
this.psAlgorithm = psInstance;
if (psInstance != null) { if (DEBUG && psInstance == null && urls.length > 1) {
this.psAlgorithm = psInstance; Log.w(TAG, "mission created with multiple urls ¿missing post-processing algorithm?");
this.offsets[0] = psInstance.recommendedReserve;
} else {
if (DEBUG && urls.length > 1) {
Log.w(TAG, "mission created with multiple urls ¿missing post-processing algorithm?");
}
} }
} }
@ -233,10 +229,14 @@ public class DownloadMission extends Mission {
* @throws IOException if an I/O exception occurs. * @throws IOException if an I/O exception occurs.
*/ */
HttpURLConnection openConnection(int threadId, long rangeStart, long rangeEnd) throws IOException { HttpURLConnection openConnection(int threadId, long rangeStart, long rangeEnd) throws IOException {
URL url = new URL(urls[current]); return openConnection(urls[current], threadId, rangeStart, rangeEnd);
HttpURLConnection conn = (HttpURLConnection) url.openConnection(); }
HttpURLConnection openConnection(String url, int threadId, long rangeStart, long rangeEnd) throws IOException {
HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
conn.setInstanceFollowRedirects(true); conn.setInstanceFollowRedirects(true);
conn.setRequestProperty("User-Agent", Downloader.USER_AGENT); conn.setRequestProperty("User-Agent", Downloader.USER_AGENT);
conn.setRequestProperty("Accept", "*/*");
// BUG workaround: switching between networks can freeze the download forever // BUG workaround: switching between networks can freeze the download forever
conn.setConnectTimeout(30000); conn.setConnectTimeout(30000);
@ -536,8 +536,11 @@ public class DownloadMission extends Mission {
@Override @Override
public boolean delete() { public boolean delete() {
deleted = true; deleted = true;
if (psAlgorithm != null) psAlgorithm.cleanupTemporalDir();
boolean res = deleteThisFromFile(); boolean res = deleteThisFromFile();
if (!super.delete()) res = false;
if (!super.delete()) return false;
return res; return res;
} }
@ -626,6 +629,11 @@ public class DownloadMission extends Mission {
return blocks >= 0; // DownloadMissionInitializer was executed return blocks >= 0; // DownloadMissionInitializer was executed
} }
/**
* Gets the approximated final length of the file
*
* @return the length in bytes
*/
public long getLength() { public long getLength() {
long calculated; long calculated;
if (psState == 1 || psState == 3) { if (psState == 1 || psState == 3) {
@ -681,6 +689,8 @@ public class DownloadMission extends Mission {
private boolean doPostprocessing() { private boolean doPostprocessing() {
if (psAlgorithm == null || psState == 2) return true; if (psAlgorithm == null || psState == 2) return true;
errObject = null;
notifyPostProcessing(1); notifyPostProcessing(1);
notifyProgress(0); notifyProgress(0);

4
app/src/main/java/us/shandian/giga/postprocessing/M4aNoDash.java
View File

@ -6,10 +6,10 @@ import org.schabi.newpipe.streams.io.SharpStream;
import java.io.IOException; import java.io.IOException;
public class M4aNoDash extends Postprocessing { class M4aNoDash extends Postprocessing {
M4aNoDash() { M4aNoDash() {
super(0, true); super(false, true, ALGORITHM_M4A_NO_DASH);
} }
@Override @Override

2
app/src/main/java/us/shandian/giga/postprocessing/Mp4FromDashMuxer.java
View File

@ -11,7 +11,7 @@ import java.io.IOException;
class Mp4FromDashMuxer extends Postprocessing { class Mp4FromDashMuxer extends Postprocessing {
Mp4FromDashMuxer() { Mp4FromDashMuxer() {
super(3 * 1024 * 1024/* 3 MiB */, true); super(true, true, ALGORITHM_MP4_FROM_DASH_MUXER);
} }
@Override @Override

503
app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java
View File

@ -1,247 +1,256 @@
package us.shandian.giga.postprocessing; package us.shandian.giga.postprocessing;
import android.os.Message; import android.os.Message;
import android.support.annotation.NonNull; import android.support.annotation.NonNull;
import android.util.Log; import android.util.Log;
import org.schabi.newpipe.streams.io.SharpStream; import org.schabi.newpipe.streams.io.SharpStream;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.Serializable; import java.io.Serializable;
import us.shandian.giga.get.DownloadMission; import us.shandian.giga.get.DownloadMission;
import us.shandian.giga.io.ChunkFileInputStream; import us.shandian.giga.io.ChunkFileInputStream;
import us.shandian.giga.io.CircularFileWriter; import us.shandian.giga.io.CircularFileWriter;
import us.shandian.giga.io.CircularFileWriter.OffsetChecker; import us.shandian.giga.io.CircularFileWriter.OffsetChecker;
import us.shandian.giga.service.DownloadManagerService; import us.shandian.giga.service.DownloadManagerService;
import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING; import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING;
import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING_HOLD; import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING_HOLD;
import static us.shandian.giga.get.DownloadMission.ERROR_UNKNOWN_EXCEPTION; import static us.shandian.giga.get.DownloadMission.ERROR_UNKNOWN_EXCEPTION;
public abstract class Postprocessing implements Serializable { public abstract class Postprocessing implements Serializable {
static transient final byte OK_RESULT = ERROR_NOTHING; static transient final byte OK_RESULT = ERROR_NOTHING;
public transient static final String ALGORITHM_TTML_CONVERTER = "ttml"; public transient static final String ALGORITHM_TTML_CONVERTER = "ttml";
public transient static final String ALGORITHM_WEBM_MUXER = "webm"; public transient static final String ALGORITHM_WEBM_MUXER = "webm";
public transient static final String ALGORITHM_MP4_FROM_DASH_MUXER = "mp4D-mp4"; public transient static final String ALGORITHM_MP4_FROM_DASH_MUXER = "mp4D-mp4";
public transient static final String ALGORITHM_M4A_NO_DASH = "mp4D-m4a"; public transient static final String ALGORITHM_M4A_NO_DASH = "mp4D-m4a";
public static Postprocessing getAlgorithm(@NonNull String algorithmName, String[] args, @NonNull File cacheDir) { public static Postprocessing getAlgorithm(@NonNull String algorithmName, String[] args) {
Postprocessing instance; Postprocessing instance;
switch (algorithmName) { switch (algorithmName) {
case ALGORITHM_TTML_CONVERTER: case ALGORITHM_TTML_CONVERTER:
instance = new TtmlConverter(); instance = new TtmlConverter();
break; break;
case ALGORITHM_WEBM_MUXER: case ALGORITHM_WEBM_MUXER:
instance = new WebMMuxer(); instance = new WebMMuxer();
break; break;
case ALGORITHM_MP4_FROM_DASH_MUXER: case ALGORITHM_MP4_FROM_DASH_MUXER:
instance = new Mp4FromDashMuxer(); instance = new Mp4FromDashMuxer();
break; break;
case ALGORITHM_M4A_NO_DASH: case ALGORITHM_M4A_NO_DASH:
instance = new M4aNoDash(); instance = new M4aNoDash();
break; break;
/*case "example-algorithm": /*case "example-algorithm":
instance = new ExampleAlgorithm();*/ instance = new ExampleAlgorithm();*/
default: default:
throw new RuntimeException("Unimplemented post-processing algorithm: " + algorithmName); throw new UnsupportedOperationException("Unimplemented post-processing algorithm: " + algorithmName);
} }
instance.args = args; instance.args = args;
instance.name = algorithmName;// for debug only, maybe remove this field in the future return instance;
instance.cacheDir = cacheDir; }
return instance; /**
} * Get a boolean value that indicate if the given algorithm work on the same
* file
/** */
* Get a boolean value that indicate if the given algorithm work on the same public final boolean worksOnSameFile;
* file
*/ /**
public boolean worksOnSameFile; * Indicates whether the selected algorithm needs space reserved at the beginning of the file
*/
/** public final boolean reserveSpace;
* Get the recommended space to reserve for the given algorithm. The amount
* is in bytes /**
*/ * Gets the given algorithm short name
public int recommendedReserve; */
private final String name;
/**
* the download to post-process
*/ private String[] args;
protected transient DownloadMission mission;
protected transient DownloadMission mission;
public transient File cacheDir;
private File tempFile;
private String[] args;
Postprocessing(boolean reserveSpace, boolean worksOnSameFile, String algorithmName) {
private String name; this.reserveSpace = reserveSpace;
this.worksOnSameFile = worksOnSameFile;
Postprocessing(int recommendedReserve, boolean worksOnSameFile) { this.name = algorithmName;// for debugging only
this.recommendedReserve = recommendedReserve; }
this.worksOnSameFile = worksOnSameFile;
} public void setTemporalDir(@NonNull File directory) {
long rnd = (int) (Math.random() * 100000f);
public void run(DownloadMission target) throws IOException { tempFile = new File(directory, rnd + "_" + System.nanoTime() + ".tmp");
this.mission = target; }
File temp = null; public void cleanupTemporalDir() {
CircularFileWriter out = null; if (tempFile != null && tempFile.exists()) {
int result; //noinspection ResultOfMethodCallIgnored
long finalLength = -1; tempFile.delete();
}
mission.done = 0; }
mission.length = mission.storage.length();
if (worksOnSameFile) { public void run(DownloadMission target) throws IOException {
ChunkFileInputStream[] sources = new ChunkFileInputStream[mission.urls.length]; this.mission = target;
try {
int i = 0; CircularFileWriter out = null;
for (; i < sources.length - 1; i++) { int result;
sources[i] = new ChunkFileInputStream(mission.storage.getStream(), mission.offsets[i], mission.offsets[i + 1]); long finalLength = -1;
}
sources[i] = new ChunkFileInputStream(mission.storage.getStream(), mission.offsets[i]); mission.done = 0;
mission.length = mission.storage.length();
if (test(sources)) {
for (SharpStream source : sources) source.rewind(); if (worksOnSameFile) {
ChunkFileInputStream[] sources = new ChunkFileInputStream[mission.urls.length];
OffsetChecker checker = () -> { try {
for (ChunkFileInputStream source : sources) { int i = 0;
/* for (; i < sources.length - 1; i++) {
* WARNING: never use rewind() in any chunk after any writing (especially on first chunks) sources[i] = new ChunkFileInputStream(mission.storage.getStream(), mission.offsets[i], mission.offsets[i + 1]);
* or the CircularFileWriter can lead to unexpected results }
*/ sources[i] = new ChunkFileInputStream(mission.storage.getStream(), mission.offsets[i]);
if (source.isClosed() || source.available() < 1) {
continue;// the selected source is not used anymore if (test(sources)) {
} for (SharpStream source : sources) source.rewind();
return source.getFilePointer() - 1; OffsetChecker checker = () -> {
} for (ChunkFileInputStream source : sources) {
/*
return -1; * WARNING: never use rewind() in any chunk after any writing (especially on first chunks)
}; * or the CircularFileWriter can lead to unexpected results
*/
temp = new File(cacheDir, mission.storage.getName() + ".tmp"); if (source.isClosed() || source.available() < 1) {
continue;// the selected source is not used anymore
out = new CircularFileWriter(mission.storage.getStream(), temp, checker); }
out.onProgress = this::progressReport;
return source.getFilePointer() - 1;
out.onWriteError = (err) -> { }
mission.psState = 3;
mission.notifyError(ERROR_POSTPROCESSING_HOLD, err); return -1;
};
try {
synchronized (this) { out = new CircularFileWriter(mission.storage.getStream(), tempFile, checker);
while (mission.psState == 3) out.onProgress = this::progressReport;
wait();
} out.onWriteError = (err) -> {
} catch (InterruptedException e) { mission.psState = 3;
// nothing to do mission.notifyError(ERROR_POSTPROCESSING_HOLD, err);
Log.e(this.getClass().getSimpleName(), "got InterruptedException");
} try {
synchronized (this) {
return mission.errCode == ERROR_NOTHING; while (mission.psState == 3)
}; wait();
}
result = process(out, sources); } catch (InterruptedException e) {
// nothing to do
if (result == OK_RESULT) Log.e(this.getClass().getSimpleName(), "got InterruptedException");
finalLength = out.finalizeFile(); }
} else {
result = OK_RESULT; return mission.errCode == ERROR_NOTHING;
} };
} finally {
for (SharpStream source : sources) { result = process(out, sources);
if (source != null && !source.isClosed()) {
source.close(); if (result == OK_RESULT)
} finalLength = out.finalizeFile();
} } else {
if (out != null) { result = OK_RESULT;
out.close(); }
} } finally {
if (temp != null) { for (SharpStream source : sources) {
//noinspection ResultOfMethodCallIgnored if (source != null && !source.isClosed()) {
temp.delete(); source.close();
} }
} }
} else { if (out != null) {
result = test() ? process(null) : OK_RESULT; out.close();
} }
if (tempFile != null) {
if (result == OK_RESULT) { //noinspection ResultOfMethodCallIgnored
if (finalLength != -1) { tempFile.delete();
mission.done = finalLength; tempFile = null;
mission.length = finalLength; }
} }
} else { } else {
mission.errCode = ERROR_UNKNOWN_EXCEPTION; result = test() ? process(null) : OK_RESULT;
mission.errObject = new RuntimeException("post-processing algorithm returned " + result); }
}
if (result == OK_RESULT) {
if (result != OK_RESULT && worksOnSameFile) mission.storage.delete(); if (finalLength != -1) {
mission.done = finalLength;
this.mission = null; mission.length = finalLength;
} }
} else {
/** mission.errCode = ERROR_UNKNOWN_EXCEPTION;
* Test if the post-processing algorithm can be skipped mission.errObject = new RuntimeException("post-processing algorithm returned " + result);
* }
* @param sources files to be processed
* @return {@code true} if the post-processing is required, otherwise, {@code false} if (result != OK_RESULT && worksOnSameFile) mission.storage.delete();
* @throws IOException if an I/O error occurs.
*/ this.mission = null;
boolean test(SharpStream... sources) throws IOException { }
return true;
} /**
* Test if the post-processing algorithm can be skipped
/** *
* Abstract method to execute the post-processing algorithm * @param sources files to be processed
* * @return {@code true} if the post-processing is required, otherwise, {@code false}
* @param out output stream * @throws IOException if an I/O error occurs.
* @param sources files to be processed */
* @return a error code, 0 means the operation was successful boolean test(SharpStream... sources) throws IOException {
* @throws IOException if an I/O error occurs. return true;
*/ }
abstract int process(SharpStream out, SharpStream... sources) throws IOException;
/**
String getArgumentAt(int index, String defaultValue) { * Abstract method to execute the post-processing algorithm
if (args == null || index >= args.length) { *
return defaultValue; * @param out output stream
} * @param sources files to be processed
* @return a error code, 0 means the operation was successful
return args[index]; * @throws IOException if an I/O error occurs.
} */
abstract int process(SharpStream out, SharpStream... sources) throws IOException;
private void progressReport(long done) {
mission.done = done; String getArgumentAt(int index, String defaultValue) {
if (mission.length < mission.done) mission.length = mission.done; if (args == null || index >= args.length) {
return defaultValue;
Message m = new Message(); }
m.what = DownloadManagerService.MESSAGE_PROGRESS;
m.obj = mission; return args[index];
}
mission.mHandler.sendMessage(m);
} private void progressReport(long done) {
mission.done = done;
@NonNull if (mission.length < mission.done) mission.length = mission.done;
@Override
public String toString() { Message m = new Message();
StringBuilder str = new StringBuilder(); m.what = DownloadManagerService.MESSAGE_PROGRESS;
m.obj = mission;
str.append("name=").append(name).append('[');
mission.mHandler.sendMessage(m);
if (args != null) { }
for (String arg : args) {
str.append(", "); @NonNull
str.append(arg); @Override
} public String toString() {
str.delete(0, 1); StringBuilder str = new StringBuilder();
}
str.append("name=").append(name).append('[');
return str.append(']').toString();
} if (args != null) {
} for (String arg : args) {
str.append(", ");
str.append(arg);
}
str.delete(0, 1);
}
return str.append(']').toString();
}
}

144
app/src/main/java/us/shandian/giga/postprocessing/TtmlConverter.java
View File

@ -1,72 +1,72 @@
package us.shandian.giga.postprocessing; package us.shandian.giga.postprocessing;
import android.util.Log; import android.util.Log;
import org.schabi.newpipe.streams.SubtitleConverter; import org.schabi.newpipe.streams.SubtitleConverter;
import org.schabi.newpipe.streams.io.SharpStream; import org.schabi.newpipe.streams.io.SharpStream;
import org.xml.sax.SAXException; import org.xml.sax.SAXException;
import java.io.IOException; import java.io.IOException;
import java.text.ParseException; import java.text.ParseException;
import javax.xml.parsers.ParserConfigurationException; import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathExpressionException;
/** /**
* @author kapodamy * @author kapodamy
*/ */
class TtmlConverter extends Postprocessing { class TtmlConverter extends Postprocessing {
private static final String TAG = "TtmlConverter"; private static final String TAG = "TtmlConverter";
TtmlConverter() { TtmlConverter() {
// due how XmlPullParser works, the xml is fully loaded on the ram // due how XmlPullParser works, the xml is fully loaded on the ram
super(0, true); super(false, true, ALGORITHM_TTML_CONVERTER);
} }
@Override @Override
int process(SharpStream out, SharpStream... sources) throws IOException { int process(SharpStream out, SharpStream... sources) throws IOException {
// check if the subtitle is already in srt and copy, this should never happen // check if the subtitle is already in srt and copy, this should never happen
String format = getArgumentAt(0, null); String format = getArgumentAt(0, null);
if (format == null || format.equals("ttml")) { if (format == null || format.equals("ttml")) {
SubtitleConverter ttmlDumper = new SubtitleConverter(); SubtitleConverter ttmlDumper = new SubtitleConverter();
try { try {
ttmlDumper.dumpTTML( ttmlDumper.dumpTTML(
sources[0], sources[0],
out, out,
getArgumentAt(1, "true").equals("true"), getArgumentAt(1, "true").equals("true"),
getArgumentAt(2, "true").equals("true") getArgumentAt(2, "true").equals("true")
); );
} catch (Exception err) { } catch (Exception err) {
Log.e(TAG, "subtitle parse failed", err); Log.e(TAG, "subtitle parse failed", err);
if (err instanceof IOException) { if (err instanceof IOException) {
return 1; return 1;
} else if (err instanceof ParseException) { } else if (err instanceof ParseException) {
return 2; return 2;
} else if (err instanceof SAXException) { } else if (err instanceof SAXException) {
return 3; return 3;
} else if (err instanceof ParserConfigurationException) { } else if (err instanceof ParserConfigurationException) {
return 4; return 4;
} else if (err instanceof XPathExpressionException) { } else if (err instanceof XPathExpressionException) {
return 7; return 7;
} }
return 8; return 8;
} }
return OK_RESULT; return OK_RESULT;
} else if (format.equals("srt")) { } else if (format.equals("srt")) {
byte[] buffer = new byte[8 * 1024]; byte[] buffer = new byte[8 * 1024];
int read; int read;
while ((read = sources[0].read(buffer)) > 0) { while ((read = sources[0].read(buffer)) > 0) {
out.write(buffer, 0, read); out.write(buffer, 0, read);
} }
return OK_RESULT; return OK_RESULT;
} }
throw new UnsupportedOperationException("Can't convert this subtitle, unimplemented format: " + format); throw new UnsupportedOperationException("Can't convert this subtitle, unimplemented format: " + format);
} }
} }

88
app/src/main/java/us/shandian/giga/postprocessing/WebMMuxer.java
View File

@ -1,44 +1,44 @@
package us.shandian.giga.postprocessing; package us.shandian.giga.postprocessing;
import org.schabi.newpipe.streams.WebMReader.TrackKind; import org.schabi.newpipe.streams.WebMReader.TrackKind;
import org.schabi.newpipe.streams.WebMReader.WebMTrack; import org.schabi.newpipe.streams.WebMReader.WebMTrack;
import org.schabi.newpipe.streams.WebMWriter; import org.schabi.newpipe.streams.WebMWriter;
import org.schabi.newpipe.streams.io.SharpStream; import org.schabi.newpipe.streams.io.SharpStream;
import java.io.IOException; import java.io.IOException;
/** /**
* @author kapodamy * @author kapodamy
*/ */
class WebMMuxer extends Postprocessing { class WebMMuxer extends Postprocessing {
WebMMuxer() { WebMMuxer() {
super(5 * 1024 * 1024/* 5 MiB */, true); super(true, true, ALGORITHM_WEBM_MUXER);
} }
@Override @Override
int process(SharpStream out, SharpStream... sources) throws IOException { int process(SharpStream out, SharpStream... sources) throws IOException {
WebMWriter muxer = new WebMWriter(sources); WebMWriter muxer = new WebMWriter(sources);
muxer.parseSources(); muxer.parseSources();
// youtube uses a webm with a fake video track that acts as a "cover image" // youtube uses a webm with a fake video track that acts as a "cover image"
int[] indexes = new int[sources.length]; int[] indexes = new int[sources.length];
for (int i = 0; i < sources.length; i++) { for (int i = 0; i < sources.length; i++) {
WebMTrack[] tracks = muxer.getTracksFromSource(i); WebMTrack[] tracks = muxer.getTracksFromSource(i);
for (int j = 0; j < tracks.length; j++) { for (int j = 0; j < tracks.length; j++) {
if (tracks[j].kind == TrackKind.Audio) { if (tracks[j].kind == TrackKind.Audio) {
indexes[i] = j; indexes[i] = j;
i = sources.length; i = sources.length;
break; break;
} }
} }
} }
muxer.selectTracks(indexes); muxer.selectTracks(indexes);
muxer.build(out); muxer.build(out);
return OK_RESULT; return OK_RESULT;
} }
} }

25
app/src/main/java/us/shandian/giga/service/DownloadManager.java
View File

@ -154,7 +154,9 @@ public class DownloadManager {
if (mis.psAlgorithm.worksOnSameFile) { if (mis.psAlgorithm.worksOnSameFile) {
// Incomplete post-processing results in a corrupted download file // Incomplete post-processing results in a corrupted download file
// because the selected algorithm works on the same file to save space. // because the selected algorithm works on the same file to save space.
if (exists && !mis.storage.delete()) // the file will be deleted if the storage API
// is Java IO (avoid showing the "Save as..." dialog)
if (exists && mis.storage.isDirect() && !mis.storage.delete())
Log.w(TAG, "Unable to delete incomplete download file: " + sub.getPath()); Log.w(TAG, "Unable to delete incomplete download file: " + sub.getPath());
exists = true; exists = true;
@ -162,7 +164,6 @@ public class DownloadManager {
mis.psState = 0; mis.psState = 0;
mis.errCode = DownloadMission.ERROR_POSTPROCESSING_STOPPED; mis.errCode = DownloadMission.ERROR_POSTPROCESSING_STOPPED;
mis.errObject = null;
} else if (!exists) { } else if (!exists) {
tryRecover(mis); tryRecover(mis);
@ -171,8 +172,10 @@ public class DownloadManager {
mis.resetState(true, true, DownloadMission.ERROR_PROGRESS_LOST); mis.resetState(true, true, DownloadMission.ERROR_PROGRESS_LOST);
} }
if (mis.psAlgorithm != null) if (mis.psAlgorithm != null) {
mis.psAlgorithm.cacheDir = pickAvailableCacheDir(ctx); mis.psAlgorithm.cleanupTemporalDir();
mis.psAlgorithm.setTemporalDir(pickAvailableTemporalDir(ctx));
}
mis.recovered = exists; mis.recovered = exists;
mis.metadata = sub; mis.metadata = sub;
@ -532,14 +535,14 @@ public class DownloadManager {
} }
private static boolean isDirectoryAvailable(File directory) { private static boolean isDirectoryAvailable(File directory) {
return directory != null && directory.canWrite(); return directory != null && directory.canWrite() && directory.exists();
} }
static File pickAvailableCacheDir(@NonNull Context ctx) { static File pickAvailableTemporalDir(@NonNull Context ctx) {
if (isDirectoryAvailable(ctx.getExternalCacheDir())) if (isDirectoryAvailable(ctx.getExternalFilesDir(null)))
return ctx.getExternalCacheDir(); return ctx.getExternalFilesDir(null);
else if (isDirectoryAvailable(ctx.getCacheDir())) else if (isDirectoryAvailable(ctx.getFilesDir()))
return ctx.getCacheDir(); return ctx.getFilesDir();
// this never should happen // this never should happen
return ctx.getDir("tmp", Context.MODE_PRIVATE); return ctx.getDir("tmp", Context.MODE_PRIVATE);
@ -550,7 +553,7 @@ public class DownloadManager {
if (tag.equals(TAG_AUDIO)) return mMainStorageAudio; if (tag.equals(TAG_AUDIO)) return mMainStorageAudio;
if (tag.equals(TAG_VIDEO)) return mMainStorageVideo; if (tag.equals(TAG_VIDEO)) return mMainStorageVideo;
Log.w(TAG, "Unknown download category, not [audio video]: " + String.valueOf(tag)); Log.w(TAG, "Unknown download category, not [audio video]: " + tag);
return null;// this never should happen return null;// this never should happen
} }

5
app/src/main/java/us/shandian/giga/service/DownloadManagerService.java
View File

@ -450,13 +450,16 @@ public class DownloadManagerService extends Service {
if (psName == null) if (psName == null)
ps = null; ps = null;
else else
ps = Postprocessing.getAlgorithm(psName, psArgs, DownloadManager.pickAvailableCacheDir(this)); ps = Postprocessing.getAlgorithm(psName, psArgs);
final DownloadMission mission = new DownloadMission(urls, storage, kind, ps); final DownloadMission mission = new DownloadMission(urls, storage, kind, ps);
mission.threadCount = threads; mission.threadCount = threads;
mission.source = source; mission.source = source;
mission.nearLength = nearLength; mission.nearLength = nearLength;
if (ps != null)
ps.setTemporalDir(DownloadManager.pickAvailableTemporalDir(this));
handleConnectivityState(true);// first check the actual network status handleConnectivityState(true);// first check the actual network status
mManager.startMission(mission); mManager.startMission(mission);

3
app/src/main/java/us/shandian/giga/util/Utility.java
View File

@ -267,8 +267,7 @@ public class Utility {
} }
try { try {
long length = Long.parseLong(connection.getHeaderField("Content-Length")); return Long.parseLong(connection.getHeaderField("Content-Length"));
if (length >= 0) return length;
} catch (Exception err) { } catch (Exception err) {
// nothing to do // nothing to do
} }