From 7b948f83c37879de114ac0cbac83ba9b7af80973 Mon Sep 17 00:00:00 2001 From: kapodamy Date: Thu, 25 Apr 2019 00:34:29 -0300 Subject: [PATCH] Space reserving tweaks for huge video resolutions * improve space reserving, allows write better 4K/8K video data * do not use cache dirs in the muxers, Android can force close NewPipe if the device is running out of storage. Is a aggressive cache cleaning >:/ * (for devs) webm & mkv are the same thing * calculate the final file size inside of the mission, instead getting from the UI * simplify ps algorithms constructors * [missing old commit message] simplify the loading of pending downloads --- .../newpipe/download/DownloadDialog.java | 5 +- .../giga/get/DownloadInitializer.java | 349 ++++++------ .../us/shandian/giga/get/DownloadMission.java | 30 +- .../giga/postprocessing/M4aNoDash.java | 4 +- .../giga/postprocessing/Mp4FromDashMuxer.java | 2 +- .../giga/postprocessing/Postprocessing.java | 503 +++++++++--------- .../giga/postprocessing/TtmlConverter.java | 144 ++--- .../giga/postprocessing/WebMMuxer.java | 88 +-- .../giga/service/DownloadManager.java | 25 +- .../giga/service/DownloadManagerService.java | 5 +- .../java/us/shandian/giga/util/Utility.java | 3 +- 11 files changed, 608 insertions(+), 550 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java index f27e7467e..8fef9a995 100644 --- a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java +++ b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java @@ -767,7 +767,8 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck psArgs = null; long videoSize = wrappedVideoStreams.getSizeInBytes((VideoStream) selectedStream); - // set nearLength, only, if both sizes are fetched or known. this probably does not work on slow networks + // set nearLength, only, if both sizes are fetched or known. This probably + // does not work on slow networks but is later updated in the downloader if (secondaryStream.getSizeInBytes() > 0 && videoSize > 0) { nearLength = secondaryStream.getSizeInBytes() + videoSize; } @@ -793,7 +794,7 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck if (secondaryStreamUrl == null) { urls = new String[]{selectedStream.getUrl()}; } else { - urls = new String[]{secondaryStreamUrl, selectedStream.getUrl()}; + urls = new String[]{selectedStream.getUrl(), secondaryStreamUrl}; } DownloadManagerService.startMission(context, urls, storage, kind, threads, currentInfo.getUrl(), psName, psArgs, nearLength); diff --git a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java index 590c3704c..5239c5bb7 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java @@ -1,158 +1,191 @@ -package us.shandian.giga.get; - -import android.support.annotation.NonNull; -import android.util.Log; - -import org.schabi.newpipe.streams.io.SharpStream; - -import java.io.IOException; -import java.io.InterruptedIOException; -import java.net.HttpURLConnection; -import java.nio.channels.ClosedByInterruptException; - -import us.shandian.giga.util.Utility; - -import static org.schabi.newpipe.BuildConfig.DEBUG; - -public class DownloadInitializer extends Thread { - private final static String TAG = "DownloadInitializer"; - final static int mId = 0; - - private DownloadMission mMission; - private HttpURLConnection mConn; - - DownloadInitializer(@NonNull DownloadMission mission) { - mMission = mission; - mConn = null; - } - - @Override - public void run() { - if (mMission.current > 0) mMission.resetState(false, true, DownloadMission.ERROR_NOTHING); - - int retryCount = 0; - while (true) { - try { - mMission.currentThreadCount = mMission.threadCount; - - mConn = mMission.openConnection(mId, -1, -1); - mMission.establishConnection(mId, mConn); - - if (!mMission.running || Thread.interrupted()) return; - - mMission.length = Utility.getContentLength(mConn); - - - if (mMission.length == 0) { - mMission.notifyError(DownloadMission.ERROR_HTTP_NO_CONTENT, null); - return; - } - - // check for dynamic generated content - if (mMission.length == -1 && mConn.getResponseCode() == 200) { - mMission.blocks = 0; - mMission.length = 0; - mMission.fallback = true; - mMission.unknownLength = true; - mMission.currentThreadCount = 1; - - if (DEBUG) { - Log.d(TAG, "falling back (unknown length)"); - } - } else { - // Open again - mConn = mMission.openConnection(mId, mMission.length - 10, mMission.length); - mMission.establishConnection(mId, mConn); - - if (!mMission.running || Thread.interrupted()) return; - - synchronized (mMission.blockState) { - if (mConn.getResponseCode() == 206) { - if (mMission.currentThreadCount > 1) { - mMission.blocks = mMission.length / DownloadMission.BLOCK_SIZE; - - if (mMission.currentThreadCount > mMission.blocks) { - mMission.currentThreadCount = (int) mMission.blocks; - } - if (mMission.currentThreadCount <= 0) { - mMission.currentThreadCount = 1; - } - if (mMission.blocks * DownloadMission.BLOCK_SIZE < mMission.length) { - mMission.blocks++; - } - } else { - // if one thread is solicited don't calculate blocks, is useless - mMission.blocks = 1; - mMission.fallback = true; - mMission.unknownLength = false; - } - - if (DEBUG) { - Log.d(TAG, "http response code = " + mConn.getResponseCode()); - } - } else { - // Fallback to single thread - mMission.blocks = 0; - mMission.fallback = true; - mMission.unknownLength = false; - mMission.currentThreadCount = 1; - - if (DEBUG) { - Log.d(TAG, "falling back due http response code = " + mConn.getResponseCode()); - } - } - - for (long i = 0; i < mMission.currentThreadCount; i++) { - mMission.threadBlockPositions.add(i); - mMission.threadBytePositions.add(0L); - } - } - - if (!mMission.running || Thread.interrupted()) return; - } - - SharpStream fs = mMission.storage.getStream(); - fs.setLength(mMission.offsets[mMission.current] + mMission.length); - fs.seek(mMission.offsets[mMission.current]); - fs.close(); - - if (!mMission.running || Thread.interrupted()) return; - - mMission.running = false; - break; - } catch (InterruptedIOException | ClosedByInterruptException e) { - return; - } catch (Exception e) { - if (!mMission.running) return; - - if (e instanceof IOException && e.getMessage().contains("Permission denied")) { - mMission.notifyError(DownloadMission.ERROR_PERMISSION_DENIED, e); - return; - } - - if (retryCount++ > mMission.maxRetry) { - Log.e(TAG, "initializer failed", e); - mMission.notifyError(e); - return; - } - - Log.e(TAG, "initializer failed, retrying", e); - } - } - - mMission.start(); - } - - @Override - public void interrupt() { - super.interrupt(); - - if (mConn != null) { - try { - mConn.disconnect(); - } catch (Exception e) { - // nothing to do - } - } - } -} +package us.shandian.giga.get; + +import android.support.annotation.NonNull; +import android.util.Log; + +import org.schabi.newpipe.streams.io.SharpStream; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.net.HttpURLConnection; +import java.nio.channels.ClosedByInterruptException; + +import us.shandian.giga.util.Utility; + +import static org.schabi.newpipe.BuildConfig.DEBUG; + +public class DownloadInitializer extends Thread { + private final static String TAG = "DownloadInitializer"; + final static int mId = 0; + private final static int RESERVE_SPACE_DEFAULT = 5 * 1024 * 1024;// 5 MiB + private final static int RESERVE_SPACE_MAXIMUM = 150 * 1024 * 1024;// 150 MiB + + private DownloadMission mMission; + private HttpURLConnection mConn; + + DownloadInitializer(@NonNull DownloadMission mission) { + mMission = mission; + mConn = null; + } + + @Override + public void run() { + if (mMission.current > 0) mMission.resetState(false, true, DownloadMission.ERROR_NOTHING); + + int retryCount = 0; + while (true) { + try { + mMission.currentThreadCount = mMission.threadCount; + + if (mMission.blocks < 0 && mMission.current == 0) { + // calculate the whole size of the mission + long finalLength = 0; + long lowestSize = Long.MAX_VALUE; + + for (int i = 0; i < mMission.urls.length && mMission.running; i++) { + mConn = mMission.openConnection(mMission.urls[i], mId, -1, -1); + mMission.establishConnection(mId, mConn); + + if (Thread.interrupted()) return; + long length = Utility.getContentLength(mConn); + + if (i == 0) mMission.length = length; + if (length > 0) finalLength += length; + if (length < lowestSize) lowestSize = length; + } + + mMission.nearLength = finalLength; + + // reserve space at the start of the file + if (mMission.psAlgorithm != null && mMission.psAlgorithm.reserveSpace) { + if (lowestSize < 1) { + // the length is unknown use the default size + mMission.offsets[0] = RESERVE_SPACE_DEFAULT; + } else { + // use the smallest resource size to download, otherwise, use the maximum + mMission.offsets[0] = lowestSize < RESERVE_SPACE_MAXIMUM ? lowestSize : RESERVE_SPACE_MAXIMUM; + } + } + } else { + // ask for the current resource length + mConn = mMission.openConnection(mId, -1, -1); + mMission.establishConnection(mId, mConn); + + if (!mMission.running || Thread.interrupted()) return; + + mMission.length = Utility.getContentLength(mConn); + } + + if (mMission.length == 0 || mConn.getResponseCode() == 204) { + mMission.notifyError(DownloadMission.ERROR_HTTP_NO_CONTENT, null); + return; + } + + // check for dynamic generated content + if (mMission.length == -1 && mConn.getResponseCode() == 200) { + mMission.blocks = 0; + mMission.length = 0; + mMission.fallback = true; + mMission.unknownLength = true; + mMission.currentThreadCount = 1; + + if (DEBUG) { + Log.d(TAG, "falling back (unknown length)"); + } + } else { + // Open again + mConn = mMission.openConnection(mId, mMission.length - 10, mMission.length); + mMission.establishConnection(mId, mConn); + + if (!mMission.running || Thread.interrupted()) return; + + synchronized (mMission.blockState) { + if (mConn.getResponseCode() == 206) { + if (mMission.currentThreadCount > 1) { + mMission.blocks = mMission.length / DownloadMission.BLOCK_SIZE; + + if (mMission.currentThreadCount > mMission.blocks) { + mMission.currentThreadCount = (int) mMission.blocks; + } + if (mMission.currentThreadCount <= 0) { + mMission.currentThreadCount = 1; + } + if (mMission.blocks * DownloadMission.BLOCK_SIZE < mMission.length) { + mMission.blocks++; + } + } else { + // if one thread is solicited don't calculate blocks, is useless + mMission.blocks = 1; + mMission.fallback = true; + mMission.unknownLength = false; + } + + if (DEBUG) { + Log.d(TAG, "http response code = " + mConn.getResponseCode()); + } + } else { + // Fallback to single thread + mMission.blocks = 0; + mMission.fallback = true; + mMission.unknownLength = false; + mMission.currentThreadCount = 1; + + if (DEBUG) { + Log.d(TAG, "falling back due http response code = " + mConn.getResponseCode()); + } + } + + for (long i = 0; i < mMission.currentThreadCount; i++) { + mMission.threadBlockPositions.add(i); + mMission.threadBytePositions.add(0L); + } + } + + if (!mMission.running || Thread.interrupted()) return; + } + + SharpStream fs = mMission.storage.getStream(); + fs.setLength(mMission.offsets[mMission.current] + mMission.length); + fs.seek(mMission.offsets[mMission.current]); + fs.close(); + + if (!mMission.running || Thread.interrupted()) return; + + mMission.running = false; + break; + } catch (InterruptedIOException | ClosedByInterruptException e) { + return; + } catch (Exception e) { + if (!mMission.running) return; + + if (e instanceof IOException && e.getMessage().contains("Permission denied")) { + mMission.notifyError(DownloadMission.ERROR_PERMISSION_DENIED, e); + return; + } + + if (retryCount++ > mMission.maxRetry) { + Log.e(TAG, "initializer failed", e); + mMission.notifyError(e); + return; + } + + Log.e(TAG, "initializer failed, retrying", e); + } + } + + mMission.start(); + } + + @Override + public void interrupt() { + super.interrupt(); + + if (mConn != null) { + try { + mConn.disconnect(); + } catch (Exception e) { + // nothing to do + } + } + } +} diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMission.java b/app/src/main/java/us/shandian/giga/get/DownloadMission.java index 9bc46e3af..b3e32a43c 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMission.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMission.java @@ -147,14 +147,10 @@ public class DownloadMission extends Mission { this.enqueued = true; this.maxRetry = 3; this.storage = storage; + this.psAlgorithm = psInstance; - if (psInstance != null) { - this.psAlgorithm = psInstance; - this.offsets[0] = psInstance.recommendedReserve; - } else { - if (DEBUG && urls.length > 1) { - Log.w(TAG, "mission created with multiple urls ¿missing post-processing algorithm?"); - } + if (DEBUG && psInstance == null && urls.length > 1) { + Log.w(TAG, "mission created with multiple urls ¿missing post-processing algorithm?"); } } @@ -233,10 +229,14 @@ public class DownloadMission extends Mission { * @throws IOException if an I/O exception occurs. */ HttpURLConnection openConnection(int threadId, long rangeStart, long rangeEnd) throws IOException { - URL url = new URL(urls[current]); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + return openConnection(urls[current], threadId, rangeStart, rangeEnd); + } + + HttpURLConnection openConnection(String url, int threadId, long rangeStart, long rangeEnd) throws IOException { + HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); conn.setInstanceFollowRedirects(true); conn.setRequestProperty("User-Agent", Downloader.USER_AGENT); + conn.setRequestProperty("Accept", "*/*"); // BUG workaround: switching between networks can freeze the download forever conn.setConnectTimeout(30000); @@ -536,8 +536,11 @@ public class DownloadMission extends Mission { @Override public boolean delete() { deleted = true; + if (psAlgorithm != null) psAlgorithm.cleanupTemporalDir(); + boolean res = deleteThisFromFile(); - if (!super.delete()) res = false; + + if (!super.delete()) return false; return res; } @@ -626,6 +629,11 @@ public class DownloadMission extends Mission { return blocks >= 0; // DownloadMissionInitializer was executed } + /** + * Gets the approximated final length of the file + * + * @return the length in bytes + */ public long getLength() { long calculated; if (psState == 1 || psState == 3) { @@ -681,6 +689,8 @@ public class DownloadMission extends Mission { private boolean doPostprocessing() { if (psAlgorithm == null || psState == 2) return true; + errObject = null; + notifyPostProcessing(1); notifyProgress(0); diff --git a/app/src/main/java/us/shandian/giga/postprocessing/M4aNoDash.java b/app/src/main/java/us/shandian/giga/postprocessing/M4aNoDash.java index ee7e4cba1..aa5170908 100644 --- a/app/src/main/java/us/shandian/giga/postprocessing/M4aNoDash.java +++ b/app/src/main/java/us/shandian/giga/postprocessing/M4aNoDash.java @@ -6,10 +6,10 @@ import org.schabi.newpipe.streams.io.SharpStream; import java.io.IOException; -public class M4aNoDash extends Postprocessing { +class M4aNoDash extends Postprocessing { M4aNoDash() { - super(0, true); + super(false, true, ALGORITHM_M4A_NO_DASH); } @Override diff --git a/app/src/main/java/us/shandian/giga/postprocessing/Mp4FromDashMuxer.java b/app/src/main/java/us/shandian/giga/postprocessing/Mp4FromDashMuxer.java index f12c1c2d2..74cb43116 100644 --- a/app/src/main/java/us/shandian/giga/postprocessing/Mp4FromDashMuxer.java +++ b/app/src/main/java/us/shandian/giga/postprocessing/Mp4FromDashMuxer.java @@ -11,7 +11,7 @@ import java.io.IOException; class Mp4FromDashMuxer extends Postprocessing { Mp4FromDashMuxer() { - super(3 * 1024 * 1024/* 3 MiB */, true); + super(true, true, ALGORITHM_MP4_FROM_DASH_MUXER); } @Override diff --git a/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java b/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java index 3d10628e7..15c4f575d 100644 --- a/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java +++ b/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java @@ -1,247 +1,256 @@ -package us.shandian.giga.postprocessing; - -import android.os.Message; -import android.support.annotation.NonNull; -import android.util.Log; - -import org.schabi.newpipe.streams.io.SharpStream; - -import java.io.File; -import java.io.IOException; -import java.io.Serializable; - -import us.shandian.giga.get.DownloadMission; -import us.shandian.giga.io.ChunkFileInputStream; -import us.shandian.giga.io.CircularFileWriter; -import us.shandian.giga.io.CircularFileWriter.OffsetChecker; -import us.shandian.giga.service.DownloadManagerService; - -import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING; -import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING_HOLD; -import static us.shandian.giga.get.DownloadMission.ERROR_UNKNOWN_EXCEPTION; - -public abstract class Postprocessing implements Serializable { - - static transient final byte OK_RESULT = ERROR_NOTHING; - - public transient static final String ALGORITHM_TTML_CONVERTER = "ttml"; - public transient static final String ALGORITHM_WEBM_MUXER = "webm"; - public transient static final String ALGORITHM_MP4_FROM_DASH_MUXER = "mp4D-mp4"; - public transient static final String ALGORITHM_M4A_NO_DASH = "mp4D-m4a"; - - public static Postprocessing getAlgorithm(@NonNull String algorithmName, String[] args, @NonNull File cacheDir) { - Postprocessing instance; - - switch (algorithmName) { - case ALGORITHM_TTML_CONVERTER: - instance = new TtmlConverter(); - break; - case ALGORITHM_WEBM_MUXER: - instance = new WebMMuxer(); - break; - case ALGORITHM_MP4_FROM_DASH_MUXER: - instance = new Mp4FromDashMuxer(); - break; - case ALGORITHM_M4A_NO_DASH: - instance = new M4aNoDash(); - break; - /*case "example-algorithm": - instance = new ExampleAlgorithm();*/ - default: - throw new RuntimeException("Unimplemented post-processing algorithm: " + algorithmName); - } - - instance.args = args; - instance.name = algorithmName;// for debug only, maybe remove this field in the future - instance.cacheDir = cacheDir; - - return instance; - } - - /** - * Get a boolean value that indicate if the given algorithm work on the same - * file - */ - public boolean worksOnSameFile; - - /** - * Get the recommended space to reserve for the given algorithm. The amount - * is in bytes - */ - public int recommendedReserve; - - /** - * the download to post-process - */ - protected transient DownloadMission mission; - - public transient File cacheDir; - - private String[] args; - - private String name; - - Postprocessing(int recommendedReserve, boolean worksOnSameFile) { - this.recommendedReserve = recommendedReserve; - this.worksOnSameFile = worksOnSameFile; - } - - public void run(DownloadMission target) throws IOException { - this.mission = target; - - File temp = null; - CircularFileWriter out = null; - int result; - long finalLength = -1; - - mission.done = 0; - mission.length = mission.storage.length(); - - if (worksOnSameFile) { - ChunkFileInputStream[] sources = new ChunkFileInputStream[mission.urls.length]; - try { - int i = 0; - for (; i < sources.length - 1; i++) { - sources[i] = new ChunkFileInputStream(mission.storage.getStream(), mission.offsets[i], mission.offsets[i + 1]); - } - sources[i] = new ChunkFileInputStream(mission.storage.getStream(), mission.offsets[i]); - - if (test(sources)) { - for (SharpStream source : sources) source.rewind(); - - OffsetChecker checker = () -> { - for (ChunkFileInputStream source : sources) { - /* - * WARNING: never use rewind() in any chunk after any writing (especially on first chunks) - * or the CircularFileWriter can lead to unexpected results - */ - if (source.isClosed() || source.available() < 1) { - continue;// the selected source is not used anymore - } - - return source.getFilePointer() - 1; - } - - return -1; - }; - - temp = new File(cacheDir, mission.storage.getName() + ".tmp"); - - out = new CircularFileWriter(mission.storage.getStream(), temp, checker); - out.onProgress = this::progressReport; - - out.onWriteError = (err) -> { - mission.psState = 3; - mission.notifyError(ERROR_POSTPROCESSING_HOLD, err); - - try { - synchronized (this) { - while (mission.psState == 3) - wait(); - } - } catch (InterruptedException e) { - // nothing to do - Log.e(this.getClass().getSimpleName(), "got InterruptedException"); - } - - return mission.errCode == ERROR_NOTHING; - }; - - result = process(out, sources); - - if (result == OK_RESULT) - finalLength = out.finalizeFile(); - } else { - result = OK_RESULT; - } - } finally { - for (SharpStream source : sources) { - if (source != null && !source.isClosed()) { - source.close(); - } - } - if (out != null) { - out.close(); - } - if (temp != null) { - //noinspection ResultOfMethodCallIgnored - temp.delete(); - } - } - } else { - result = test() ? process(null) : OK_RESULT; - } - - if (result == OK_RESULT) { - if (finalLength != -1) { - mission.done = finalLength; - mission.length = finalLength; - } - } else { - mission.errCode = ERROR_UNKNOWN_EXCEPTION; - mission.errObject = new RuntimeException("post-processing algorithm returned " + result); - } - - if (result != OK_RESULT && worksOnSameFile) mission.storage.delete(); - - this.mission = null; - } - - /** - * Test if the post-processing algorithm can be skipped - * - * @param sources files to be processed - * @return {@code true} if the post-processing is required, otherwise, {@code false} - * @throws IOException if an I/O error occurs. - */ - boolean test(SharpStream... sources) throws IOException { - return true; - } - - /** - * Abstract method to execute the post-processing algorithm - * - * @param out output stream - * @param sources files to be processed - * @return a error code, 0 means the operation was successful - * @throws IOException if an I/O error occurs. - */ - abstract int process(SharpStream out, SharpStream... sources) throws IOException; - - String getArgumentAt(int index, String defaultValue) { - if (args == null || index >= args.length) { - return defaultValue; - } - - return args[index]; - } - - private void progressReport(long done) { - mission.done = done; - if (mission.length < mission.done) mission.length = mission.done; - - Message m = new Message(); - m.what = DownloadManagerService.MESSAGE_PROGRESS; - m.obj = mission; - - mission.mHandler.sendMessage(m); - } - - @NonNull - @Override - public String toString() { - StringBuilder str = new StringBuilder(); - - str.append("name=").append(name).append('['); - - if (args != null) { - for (String arg : args) { - str.append(", "); - str.append(arg); - } - str.delete(0, 1); - } - - return str.append(']').toString(); - } -} +package us.shandian.giga.postprocessing; + +import android.os.Message; +import android.support.annotation.NonNull; +import android.util.Log; + +import org.schabi.newpipe.streams.io.SharpStream; + +import java.io.File; +import java.io.IOException; +import java.io.Serializable; + +import us.shandian.giga.get.DownloadMission; +import us.shandian.giga.io.ChunkFileInputStream; +import us.shandian.giga.io.CircularFileWriter; +import us.shandian.giga.io.CircularFileWriter.OffsetChecker; +import us.shandian.giga.service.DownloadManagerService; + +import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING; +import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING_HOLD; +import static us.shandian.giga.get.DownloadMission.ERROR_UNKNOWN_EXCEPTION; + +public abstract class Postprocessing implements Serializable { + + static transient final byte OK_RESULT = ERROR_NOTHING; + + public transient static final String ALGORITHM_TTML_CONVERTER = "ttml"; + public transient static final String ALGORITHM_WEBM_MUXER = "webm"; + public transient static final String ALGORITHM_MP4_FROM_DASH_MUXER = "mp4D-mp4"; + public transient static final String ALGORITHM_M4A_NO_DASH = "mp4D-m4a"; + + public static Postprocessing getAlgorithm(@NonNull String algorithmName, String[] args) { + Postprocessing instance; + + switch (algorithmName) { + case ALGORITHM_TTML_CONVERTER: + instance = new TtmlConverter(); + break; + case ALGORITHM_WEBM_MUXER: + instance = new WebMMuxer(); + break; + case ALGORITHM_MP4_FROM_DASH_MUXER: + instance = new Mp4FromDashMuxer(); + break; + case ALGORITHM_M4A_NO_DASH: + instance = new M4aNoDash(); + break; + /*case "example-algorithm": + instance = new ExampleAlgorithm();*/ + default: + throw new UnsupportedOperationException("Unimplemented post-processing algorithm: " + algorithmName); + } + + instance.args = args; + return instance; + } + + /** + * Get a boolean value that indicate if the given algorithm work on the same + * file + */ + public final boolean worksOnSameFile; + + /** + * Indicates whether the selected algorithm needs space reserved at the beginning of the file + */ + public final boolean reserveSpace; + + /** + * Gets the given algorithm short name + */ + private final String name; + + + private String[] args; + + protected transient DownloadMission mission; + + private File tempFile; + + Postprocessing(boolean reserveSpace, boolean worksOnSameFile, String algorithmName) { + this.reserveSpace = reserveSpace; + this.worksOnSameFile = worksOnSameFile; + this.name = algorithmName;// for debugging only + } + + public void setTemporalDir(@NonNull File directory) { + long rnd = (int) (Math.random() * 100000f); + tempFile = new File(directory, rnd + "_" + System.nanoTime() + ".tmp"); + } + + public void cleanupTemporalDir() { + if (tempFile != null && tempFile.exists()) { + //noinspection ResultOfMethodCallIgnored + tempFile.delete(); + } + } + + + public void run(DownloadMission target) throws IOException { + this.mission = target; + + CircularFileWriter out = null; + int result; + long finalLength = -1; + + mission.done = 0; + mission.length = mission.storage.length(); + + if (worksOnSameFile) { + ChunkFileInputStream[] sources = new ChunkFileInputStream[mission.urls.length]; + try { + int i = 0; + for (; i < sources.length - 1; i++) { + sources[i] = new ChunkFileInputStream(mission.storage.getStream(), mission.offsets[i], mission.offsets[i + 1]); + } + sources[i] = new ChunkFileInputStream(mission.storage.getStream(), mission.offsets[i]); + + if (test(sources)) { + for (SharpStream source : sources) source.rewind(); + + OffsetChecker checker = () -> { + for (ChunkFileInputStream source : sources) { + /* + * WARNING: never use rewind() in any chunk after any writing (especially on first chunks) + * or the CircularFileWriter can lead to unexpected results + */ + if (source.isClosed() || source.available() < 1) { + continue;// the selected source is not used anymore + } + + return source.getFilePointer() - 1; + } + + return -1; + }; + + out = new CircularFileWriter(mission.storage.getStream(), tempFile, checker); + out.onProgress = this::progressReport; + + out.onWriteError = (err) -> { + mission.psState = 3; + mission.notifyError(ERROR_POSTPROCESSING_HOLD, err); + + try { + synchronized (this) { + while (mission.psState == 3) + wait(); + } + } catch (InterruptedException e) { + // nothing to do + Log.e(this.getClass().getSimpleName(), "got InterruptedException"); + } + + return mission.errCode == ERROR_NOTHING; + }; + + result = process(out, sources); + + if (result == OK_RESULT) + finalLength = out.finalizeFile(); + } else { + result = OK_RESULT; + } + } finally { + for (SharpStream source : sources) { + if (source != null && !source.isClosed()) { + source.close(); + } + } + if (out != null) { + out.close(); + } + if (tempFile != null) { + //noinspection ResultOfMethodCallIgnored + tempFile.delete(); + tempFile = null; + } + } + } else { + result = test() ? process(null) : OK_RESULT; + } + + if (result == OK_RESULT) { + if (finalLength != -1) { + mission.done = finalLength; + mission.length = finalLength; + } + } else { + mission.errCode = ERROR_UNKNOWN_EXCEPTION; + mission.errObject = new RuntimeException("post-processing algorithm returned " + result); + } + + if (result != OK_RESULT && worksOnSameFile) mission.storage.delete(); + + this.mission = null; + } + + /** + * Test if the post-processing algorithm can be skipped + * + * @param sources files to be processed + * @return {@code true} if the post-processing is required, otherwise, {@code false} + * @throws IOException if an I/O error occurs. + */ + boolean test(SharpStream... sources) throws IOException { + return true; + } + + /** + * Abstract method to execute the post-processing algorithm + * + * @param out output stream + * @param sources files to be processed + * @return a error code, 0 means the operation was successful + * @throws IOException if an I/O error occurs. + */ + abstract int process(SharpStream out, SharpStream... sources) throws IOException; + + String getArgumentAt(int index, String defaultValue) { + if (args == null || index >= args.length) { + return defaultValue; + } + + return args[index]; + } + + private void progressReport(long done) { + mission.done = done; + if (mission.length < mission.done) mission.length = mission.done; + + Message m = new Message(); + m.what = DownloadManagerService.MESSAGE_PROGRESS; + m.obj = mission; + + mission.mHandler.sendMessage(m); + } + + @NonNull + @Override + public String toString() { + StringBuilder str = new StringBuilder(); + + str.append("name=").append(name).append('['); + + if (args != null) { + for (String arg : args) { + str.append(", "); + str.append(arg); + } + str.delete(0, 1); + } + + return str.append(']').toString(); + } +} diff --git a/app/src/main/java/us/shandian/giga/postprocessing/TtmlConverter.java b/app/src/main/java/us/shandian/giga/postprocessing/TtmlConverter.java index bba0b299a..5a5b687f7 100644 --- a/app/src/main/java/us/shandian/giga/postprocessing/TtmlConverter.java +++ b/app/src/main/java/us/shandian/giga/postprocessing/TtmlConverter.java @@ -1,72 +1,72 @@ -package us.shandian.giga.postprocessing; - -import android.util.Log; - -import org.schabi.newpipe.streams.SubtitleConverter; -import org.schabi.newpipe.streams.io.SharpStream; -import org.xml.sax.SAXException; - -import java.io.IOException; -import java.text.ParseException; - -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.xpath.XPathExpressionException; - -/** - * @author kapodamy - */ -class TtmlConverter extends Postprocessing { - private static final String TAG = "TtmlConverter"; - - TtmlConverter() { - // due how XmlPullParser works, the xml is fully loaded on the ram - super(0, true); - } - - @Override - int process(SharpStream out, SharpStream... sources) throws IOException { - // check if the subtitle is already in srt and copy, this should never happen - String format = getArgumentAt(0, null); - - if (format == null || format.equals("ttml")) { - SubtitleConverter ttmlDumper = new SubtitleConverter(); - - try { - ttmlDumper.dumpTTML( - sources[0], - out, - getArgumentAt(1, "true").equals("true"), - getArgumentAt(2, "true").equals("true") - ); - } catch (Exception err) { - Log.e(TAG, "subtitle parse failed", err); - - if (err instanceof IOException) { - return 1; - } else if (err instanceof ParseException) { - return 2; - } else if (err instanceof SAXException) { - return 3; - } else if (err instanceof ParserConfigurationException) { - return 4; - } else if (err instanceof XPathExpressionException) { - return 7; - } - - return 8; - } - - return OK_RESULT; - } else if (format.equals("srt")) { - byte[] buffer = new byte[8 * 1024]; - int read; - while ((read = sources[0].read(buffer)) > 0) { - out.write(buffer, 0, read); - } - return OK_RESULT; - } - - throw new UnsupportedOperationException("Can't convert this subtitle, unimplemented format: " + format); - } - -} +package us.shandian.giga.postprocessing; + +import android.util.Log; + +import org.schabi.newpipe.streams.SubtitleConverter; +import org.schabi.newpipe.streams.io.SharpStream; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.text.ParseException; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPathExpressionException; + +/** + * @author kapodamy + */ +class TtmlConverter extends Postprocessing { + private static final String TAG = "TtmlConverter"; + + TtmlConverter() { + // due how XmlPullParser works, the xml is fully loaded on the ram + super(false, true, ALGORITHM_TTML_CONVERTER); + } + + @Override + int process(SharpStream out, SharpStream... sources) throws IOException { + // check if the subtitle is already in srt and copy, this should never happen + String format = getArgumentAt(0, null); + + if (format == null || format.equals("ttml")) { + SubtitleConverter ttmlDumper = new SubtitleConverter(); + + try { + ttmlDumper.dumpTTML( + sources[0], + out, + getArgumentAt(1, "true").equals("true"), + getArgumentAt(2, "true").equals("true") + ); + } catch (Exception err) { + Log.e(TAG, "subtitle parse failed", err); + + if (err instanceof IOException) { + return 1; + } else if (err instanceof ParseException) { + return 2; + } else if (err instanceof SAXException) { + return 3; + } else if (err instanceof ParserConfigurationException) { + return 4; + } else if (err instanceof XPathExpressionException) { + return 7; + } + + return 8; + } + + return OK_RESULT; + } else if (format.equals("srt")) { + byte[] buffer = new byte[8 * 1024]; + int read; + while ((read = sources[0].read(buffer)) > 0) { + out.write(buffer, 0, read); + } + return OK_RESULT; + } + + throw new UnsupportedOperationException("Can't convert this subtitle, unimplemented format: " + format); + } + +} diff --git a/app/src/main/java/us/shandian/giga/postprocessing/WebMMuxer.java b/app/src/main/java/us/shandian/giga/postprocessing/WebMMuxer.java index 618c1ec5a..ea1676482 100644 --- a/app/src/main/java/us/shandian/giga/postprocessing/WebMMuxer.java +++ b/app/src/main/java/us/shandian/giga/postprocessing/WebMMuxer.java @@ -1,44 +1,44 @@ -package us.shandian.giga.postprocessing; - -import org.schabi.newpipe.streams.WebMReader.TrackKind; -import org.schabi.newpipe.streams.WebMReader.WebMTrack; -import org.schabi.newpipe.streams.WebMWriter; -import org.schabi.newpipe.streams.io.SharpStream; - -import java.io.IOException; - -/** - * @author kapodamy - */ -class WebMMuxer extends Postprocessing { - - WebMMuxer() { - super(5 * 1024 * 1024/* 5 MiB */, true); - } - - @Override - int process(SharpStream out, SharpStream... sources) throws IOException { - WebMWriter muxer = new WebMWriter(sources); - muxer.parseSources(); - - // youtube uses a webm with a fake video track that acts as a "cover image" - int[] indexes = new int[sources.length]; - - for (int i = 0; i < sources.length; i++) { - WebMTrack[] tracks = muxer.getTracksFromSource(i); - for (int j = 0; j < tracks.length; j++) { - if (tracks[j].kind == TrackKind.Audio) { - indexes[i] = j; - i = sources.length; - break; - } - } - } - - muxer.selectTracks(indexes); - muxer.build(out); - - return OK_RESULT; - } - -} +package us.shandian.giga.postprocessing; + +import org.schabi.newpipe.streams.WebMReader.TrackKind; +import org.schabi.newpipe.streams.WebMReader.WebMTrack; +import org.schabi.newpipe.streams.WebMWriter; +import org.schabi.newpipe.streams.io.SharpStream; + +import java.io.IOException; + +/** + * @author kapodamy + */ +class WebMMuxer extends Postprocessing { + + WebMMuxer() { + super(true, true, ALGORITHM_WEBM_MUXER); + } + + @Override + int process(SharpStream out, SharpStream... sources) throws IOException { + WebMWriter muxer = new WebMWriter(sources); + muxer.parseSources(); + + // youtube uses a webm with a fake video track that acts as a "cover image" + int[] indexes = new int[sources.length]; + + for (int i = 0; i < sources.length; i++) { + WebMTrack[] tracks = muxer.getTracksFromSource(i); + for (int j = 0; j < tracks.length; j++) { + if (tracks[j].kind == TrackKind.Audio) { + indexes[i] = j; + i = sources.length; + break; + } + } + } + + muxer.selectTracks(indexes); + muxer.build(out); + + return OK_RESULT; + } + +} diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManager.java b/app/src/main/java/us/shandian/giga/service/DownloadManager.java index 479c4b92f..c2bba7396 100644 --- a/app/src/main/java/us/shandian/giga/service/DownloadManager.java +++ b/app/src/main/java/us/shandian/giga/service/DownloadManager.java @@ -154,7 +154,9 @@ public class DownloadManager { if (mis.psAlgorithm.worksOnSameFile) { // Incomplete post-processing results in a corrupted download file // because the selected algorithm works on the same file to save space. - if (exists && !mis.storage.delete()) + // the file will be deleted if the storage API + // is Java IO (avoid showing the "Save as..." dialog) + if (exists && mis.storage.isDirect() && !mis.storage.delete()) Log.w(TAG, "Unable to delete incomplete download file: " + sub.getPath()); exists = true; @@ -162,7 +164,6 @@ public class DownloadManager { mis.psState = 0; mis.errCode = DownloadMission.ERROR_POSTPROCESSING_STOPPED; - mis.errObject = null; } else if (!exists) { tryRecover(mis); @@ -171,8 +172,10 @@ public class DownloadManager { mis.resetState(true, true, DownloadMission.ERROR_PROGRESS_LOST); } - if (mis.psAlgorithm != null) - mis.psAlgorithm.cacheDir = pickAvailableCacheDir(ctx); + if (mis.psAlgorithm != null) { + mis.psAlgorithm.cleanupTemporalDir(); + mis.psAlgorithm.setTemporalDir(pickAvailableTemporalDir(ctx)); + } mis.recovered = exists; mis.metadata = sub; @@ -532,14 +535,14 @@ public class DownloadManager { } private static boolean isDirectoryAvailable(File directory) { - return directory != null && directory.canWrite(); + return directory != null && directory.canWrite() && directory.exists(); } - static File pickAvailableCacheDir(@NonNull Context ctx) { - if (isDirectoryAvailable(ctx.getExternalCacheDir())) - return ctx.getExternalCacheDir(); - else if (isDirectoryAvailable(ctx.getCacheDir())) - return ctx.getCacheDir(); + static File pickAvailableTemporalDir(@NonNull Context ctx) { + if (isDirectoryAvailable(ctx.getExternalFilesDir(null))) + return ctx.getExternalFilesDir(null); + else if (isDirectoryAvailable(ctx.getFilesDir())) + return ctx.getFilesDir(); // this never should happen return ctx.getDir("tmp", Context.MODE_PRIVATE); @@ -550,7 +553,7 @@ public class DownloadManager { if (tag.equals(TAG_AUDIO)) return mMainStorageAudio; if (tag.equals(TAG_VIDEO)) return mMainStorageVideo; - Log.w(TAG, "Unknown download category, not [audio video]: " + String.valueOf(tag)); + Log.w(TAG, "Unknown download category, not [audio video]: " + tag); return null;// this never should happen } diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java b/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java index f25147507..aab0257db 100755 --- a/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java +++ b/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java @@ -450,13 +450,16 @@ public class DownloadManagerService extends Service { if (psName == null) ps = null; else - ps = Postprocessing.getAlgorithm(psName, psArgs, DownloadManager.pickAvailableCacheDir(this)); + ps = Postprocessing.getAlgorithm(psName, psArgs); final DownloadMission mission = new DownloadMission(urls, storage, kind, ps); mission.threadCount = threads; mission.source = source; mission.nearLength = nearLength; + if (ps != null) + ps.setTemporalDir(DownloadManager.pickAvailableTemporalDir(this)); + handleConnectivityState(true);// first check the actual network status mManager.startMission(mission); diff --git a/app/src/main/java/us/shandian/giga/util/Utility.java b/app/src/main/java/us/shandian/giga/util/Utility.java index 793cbea18..dc6a67b4b 100644 --- a/app/src/main/java/us/shandian/giga/util/Utility.java +++ b/app/src/main/java/us/shandian/giga/util/Utility.java @@ -267,8 +267,7 @@ public class Utility { } try { - long length = Long.parseLong(connection.getHeaderField("Content-Length")); - if (length >= 0) return length; + return Long.parseLong(connection.getHeaderField("Content-Length")); } catch (Exception err) { // nothing to do }