Space reserving tweaks for huge video resolutions

* improve space reserving, allows write better 4K/8K video data
* do not use cache dirs in the muxers, Android can force close NewPipe if the device is running out of storage. Is a aggressive cache cleaning >:/
* (for devs) webm & mkv are the same thing
* calculate the final file size inside of the mission, instead getting from the UI
* simplify ps algorithms constructors
* [missing old commit message] simplify the loading of pending downloads

app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java

@@ -767,7 +767,8 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck
                     psArgs = null;
                     long videoSize = wrappedVideoStreams.getSizeInBytes((VideoStream) selectedStream);
 
-                    // set nearLength, only, if both sizes are fetched or known. this probably does not work on slow networks
+                    // set nearLength, only, if both sizes are fetched or known. This probably
+                    // does not work on slow networks but is later updated in the downloader
                     if (secondaryStream.getSizeInBytes() > 0 && videoSize > 0) {
                         nearLength = secondaryStream.getSizeInBytes() + videoSize;
                     }
@@ -793,7 +794,7 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck
         if (secondaryStreamUrl == null) {
             urls = new String[]{selectedStream.getUrl()};
         } else {
-            urls = new String[]{secondaryStreamUrl, selectedStream.getUrl()};
+            urls = new String[]{selectedStream.getUrl(), secondaryStreamUrl};
         }
 
         DownloadManagerService.startMission(context, urls, storage, kind, threads, currentInfo.getUrl(), psName, psArgs, nearLength);

app/src/main/java/us/shandian/giga/get/DownloadInitializer.java

@@ -17,6 +17,8 @@ import static org.schabi.newpipe.BuildConfig.DEBUG;
 public class DownloadInitializer extends Thread {
     private final static String TAG = "DownloadInitializer";
     final static int mId = 0;
+    private final static int RESERVE_SPACE_DEFAULT = 5 * 1024 * 1024;// 5 MiB
+    private final static int RESERVE_SPACE_MAXIMUM = 150 * 1024 * 1024;// 150 MiB
 
     private DownloadMission mMission;
     private HttpURLConnection mConn;
@@ -35,15 +37,46 @@ public class DownloadInitializer extends Thread {
             try {
                 mMission.currentThreadCount = mMission.threadCount;
 
+                if (mMission.blocks < 0 && mMission.current == 0) {
+                    // calculate the whole size of the mission
+                    long finalLength = 0;
+                    long lowestSize = Long.MAX_VALUE;
+
+                    for (int i = 0; i < mMission.urls.length && mMission.running; i++) {
+                        mConn = mMission.openConnection(mMission.urls[i], mId, -1, -1);
+                        mMission.establishConnection(mId, mConn);
+
+                        if (Thread.interrupted()) return;
+                        long length = Utility.getContentLength(mConn);
+
+                        if (i == 0) mMission.length = length;
+                        if (length > 0) finalLength += length;
+                        if (length < lowestSize) lowestSize = length;
+                    }
+
+                    mMission.nearLength = finalLength;
+
+                    // reserve space at the start of the file
+                    if (mMission.psAlgorithm != null && mMission.psAlgorithm.reserveSpace) {
+                        if (lowestSize < 1) {
+                            // the length is unknown use the default size
+                            mMission.offsets[0] = RESERVE_SPACE_DEFAULT;
+                        } else {
+                            // use the smallest resource size to download, otherwise, use the maximum
+                            mMission.offsets[0] = lowestSize < RESERVE_SPACE_MAXIMUM ? lowestSize : RESERVE_SPACE_MAXIMUM;
+                        }
+                    }
+                } else {
+                    // ask for the current resource length
                     mConn = mMission.openConnection(mId, -1, -1);
                     mMission.establishConnection(mId, mConn);
 
                     if (!mMission.running || Thread.interrupted()) return;
 
                     mMission.length = Utility.getContentLength(mConn);
+                }
 
-
+                if (mMission.length == 0 || mConn.getResponseCode() == 204) {
-                if (mMission.length == 0) {
                     mMission.notifyError(DownloadMission.ERROR_HTTP_NO_CONTENT, null);
                     return;
                 }

app/src/main/java/us/shandian/giga/get/DownloadMission.java

@@ -147,16 +147,12 @@ public class DownloadMission extends Mission {
         this.enqueued = true;
         this.maxRetry = 3;
         this.storage = storage;
-
-        if (psInstance != null) {
         this.psAlgorithm = psInstance;
-            this.offsets[0] = psInstance.recommendedReserve;
+
-        } else {
+        if (DEBUG && psInstance == null && urls.length > 1) {
-            if (DEBUG && urls.length > 1) {
             Log.w(TAG, "mission created with multiple urls ¿missing post-processing algorithm?");
         }
     }
-    }
 
     private void checkBlock(long block) {
         if (block < 0 || block >= blocks) {
@@ -233,10 +229,14 @@ public class DownloadMission extends Mission {
      * @throws IOException if an I/O exception occurs.
      */
     HttpURLConnection openConnection(int threadId, long rangeStart, long rangeEnd) throws IOException {
-        URL url = new URL(urls[current]);
+        return openConnection(urls[current], threadId, rangeStart, rangeEnd);
-        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+    }
+
+    HttpURLConnection openConnection(String url, int threadId, long rangeStart, long rangeEnd) throws IOException {
+        HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
         conn.setInstanceFollowRedirects(true);
         conn.setRequestProperty("User-Agent", Downloader.USER_AGENT);
+        conn.setRequestProperty("Accept", "*/*");
 
         // BUG workaround: switching between networks can freeze the download forever
         conn.setConnectTimeout(30000);
@@ -536,8 +536,11 @@ public class DownloadMission extends Mission {
     @Override
     public boolean delete() {
         deleted = true;
+        if (psAlgorithm != null) psAlgorithm.cleanupTemporalDir();
+
         boolean res = deleteThisFromFile();
-        if (!super.delete()) res = false;
+
+        if (!super.delete()) return false;
         return res;
     }
 
@@ -626,6 +629,11 @@ public class DownloadMission extends Mission {
         return blocks >= 0; // DownloadMissionInitializer was executed
     }
 
+    /**
+     * Gets the approximated final length of the file
+     *
+     * @return the length in bytes
+     */
     public long getLength() {
         long calculated;
         if (psState == 1 || psState == 3) {
@@ -681,6 +689,8 @@ public class DownloadMission extends Mission {
     private boolean doPostprocessing() {
         if (psAlgorithm == null || psState == 2) return true;
 
+        errObject = null;
+
         notifyPostProcessing(1);
         notifyProgress(0);
 

app/src/main/java/us/shandian/giga/postprocessing/M4aNoDash.java

@@ -6,10 +6,10 @@ import org.schabi.newpipe.streams.io.SharpStream;
 
 import java.io.IOException;
 
-public class M4aNoDash extends Postprocessing {
+class M4aNoDash extends Postprocessing {
 
     M4aNoDash() {
-        super(0, true);
+        super(false, true, ALGORITHM_M4A_NO_DASH);
     }
 
     @Override

app/src/main/java/us/shandian/giga/postprocessing/Mp4FromDashMuxer.java

@@ -11,7 +11,7 @@ import java.io.IOException;
 class Mp4FromDashMuxer extends Postprocessing {
 
     Mp4FromDashMuxer() {
-        super(3 * 1024 * 1024/* 3 MiB */, true);
+        super(true, true, ALGORITHM_MP4_FROM_DASH_MUXER);
     }
 
     @Override

app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java

@@ -29,7 +29,7 @@ public abstract class Postprocessing implements Serializable {
     public transient static final String ALGORITHM_MP4_FROM_DASH_MUXER = "mp4D-mp4";
     public transient static final String ALGORITHM_M4A_NO_DASH = "mp4D-m4a";
 
-    public static Postprocessing getAlgorithm(@NonNull String algorithmName, String[] args, @NonNull File cacheDir) {
+    public static Postprocessing getAlgorithm(@NonNull String algorithmName, String[] args) {
         Postprocessing instance;
 
         switch (algorithmName) {
@@ -48,13 +48,10 @@ public abstract class Postprocessing implements Serializable {
             /*case "example-algorithm":
                 instance = new ExampleAlgorithm();*/
             default:
-                throw new RuntimeException("Unimplemented post-processing algorithm: " + algorithmName);
+                throw new UnsupportedOperationException("Unimplemented post-processing algorithm: " + algorithmName);
         }
 
         instance.args = args;
-        instance.name = algorithmName;// for debug only, maybe remove this field in the future
-        instance.cacheDir = cacheDir;
-
         return instance;
     }
 
@@ -62,34 +59,47 @@ public abstract class Postprocessing implements Serializable {
      * Get a boolean value that indicate if the given algorithm work on the same
      * file
      */
-    public boolean worksOnSameFile;
+    public final boolean worksOnSameFile;
 
     /**
-     * Get the recommended space to reserve for the given algorithm. The amount
+     * Indicates whether the selected algorithm needs space reserved at the beginning of the file
-     * is in bytes
      */
-    public int recommendedReserve;
+    public final boolean reserveSpace;
 
     /**
-     * the download to post-process
+     * Gets the given algorithm short name
      */
-    protected transient DownloadMission mission;
+    private final String name;
 
-    public transient File cacheDir;
 
     private String[] args;
 
-    private String name;
+    protected transient DownloadMission mission;
 
-    Postprocessing(int recommendedReserve, boolean worksOnSameFile) {
+    private File tempFile;
-        this.recommendedReserve = recommendedReserve;
+
+    Postprocessing(boolean reserveSpace, boolean worksOnSameFile, String algorithmName) {
+        this.reserveSpace = reserveSpace;
         this.worksOnSameFile = worksOnSameFile;
+        this.name = algorithmName;// for debugging only
     }
 
+    public void setTemporalDir(@NonNull File directory) {
+        long rnd = (int) (Math.random() * 100000f);
+        tempFile = new File(directory, rnd + "_" + System.nanoTime() + ".tmp");
+    }
+
+    public void cleanupTemporalDir() {
+        if (tempFile != null && tempFile.exists()) {
+            //noinspection ResultOfMethodCallIgnored
+            tempFile.delete();
+        }
+    }
+
+
     public void run(DownloadMission target) throws IOException {
         this.mission = target;
 
-        File temp = null;
         CircularFileWriter out = null;
         int result;
         long finalLength = -1;
@@ -125,9 +135,7 @@ public abstract class Postprocessing implements Serializable {
                         return -1;
                     };
 
-                    temp = new File(cacheDir, mission.storage.getName() + ".tmp");
+                    out = new CircularFileWriter(mission.storage.getStream(), tempFile, checker);
-
-                    out = new CircularFileWriter(mission.storage.getStream(), temp, checker);
                     out.onProgress = this::progressReport;
 
                     out.onWriteError = (err) -> {
@@ -163,9 +171,10 @@ public abstract class Postprocessing implements Serializable {
                 if (out != null) {
                     out.close();
                 }
-                if (temp != null) {
+                if (tempFile != null) {
                     //noinspection ResultOfMethodCallIgnored
-                    temp.delete();
+                    tempFile.delete();
+                    tempFile = null;
                 }
             }
         } else {

app/src/main/java/us/shandian/giga/postprocessing/TtmlConverter.java

@@ -20,7 +20,7 @@ class TtmlConverter extends Postprocessing {
 
     TtmlConverter() {
         // due how XmlPullParser works, the xml is fully loaded on the ram
-        super(0, true);
+        super(false, true, ALGORITHM_TTML_CONVERTER);
     }
 
     @Override

app/src/main/java/us/shandian/giga/postprocessing/WebMMuxer.java

@@ -13,7 +13,7 @@ import java.io.IOException;
 class WebMMuxer extends Postprocessing {
 
     WebMMuxer() {
-        super(5 * 1024 * 1024/* 5 MiB */, true);
+        super(true, true, ALGORITHM_WEBM_MUXER);
     }
 
     @Override

app/src/main/java/us/shandian/giga/service/DownloadManager.java

@@ -154,7 +154,9 @@ public class DownloadManager {
                 if (mis.psAlgorithm.worksOnSameFile) {
                     // Incomplete post-processing results in a corrupted download file
                     // because the selected algorithm works on the same file to save space.
-                    if (exists && !mis.storage.delete())
+                    // the file will be deleted if the storage API
+                    // is Java IO (avoid showing the "Save as..." dialog)
+                    if (exists && mis.storage.isDirect() && !mis.storage.delete())
                         Log.w(TAG, "Unable to delete incomplete download file: " + sub.getPath());
 
                     exists = true;
@@ -162,7 +164,6 @@ public class DownloadManager {
 
                 mis.psState = 0;
                 mis.errCode = DownloadMission.ERROR_POSTPROCESSING_STOPPED;
-                mis.errObject = null;
             } else if (!exists) {
                 tryRecover(mis);
 
@@ -171,8 +172,10 @@ public class DownloadManager {
                     mis.resetState(true, true, DownloadMission.ERROR_PROGRESS_LOST);
             }
 
-            if (mis.psAlgorithm != null)
+            if (mis.psAlgorithm != null) {
-                mis.psAlgorithm.cacheDir = pickAvailableCacheDir(ctx);
+                mis.psAlgorithm.cleanupTemporalDir();
+                mis.psAlgorithm.setTemporalDir(pickAvailableTemporalDir(ctx));
+            }
 
             mis.recovered = exists;
             mis.metadata = sub;
@@ -532,14 +535,14 @@ public class DownloadManager {
     }
 
     private static boolean isDirectoryAvailable(File directory) {
-        return directory != null && directory.canWrite();
+        return directory != null && directory.canWrite() && directory.exists();
     }
 
-    static File pickAvailableCacheDir(@NonNull Context ctx) {
+    static File pickAvailableTemporalDir(@NonNull Context ctx) {
-        if (isDirectoryAvailable(ctx.getExternalCacheDir()))
+        if (isDirectoryAvailable(ctx.getExternalFilesDir(null)))
-            return ctx.getExternalCacheDir();
+            return ctx.getExternalFilesDir(null);
-        else if (isDirectoryAvailable(ctx.getCacheDir()))
+        else if (isDirectoryAvailable(ctx.getFilesDir()))
-            return ctx.getCacheDir();
+            return ctx.getFilesDir();
 
         // this never should happen
         return ctx.getDir("tmp", Context.MODE_PRIVATE);
@@ -550,7 +553,7 @@ public class DownloadManager {
         if (tag.equals(TAG_AUDIO)) return mMainStorageAudio;
         if (tag.equals(TAG_VIDEO)) return mMainStorageVideo;
 
-        Log.w(TAG, "Unknown download category, not [audio video]: " + String.valueOf(tag));
+        Log.w(TAG, "Unknown download category, not [audio video]: " + tag);
 
         return null;// this never should happen
     }

app/src/main/java/us/shandian/giga/service/DownloadManagerService.java

@@ -450,13 +450,16 @@ public class DownloadManagerService extends Service {
         if (psName == null)
             ps = null;
         else
-            ps = Postprocessing.getAlgorithm(psName, psArgs, DownloadManager.pickAvailableCacheDir(this));
+            ps = Postprocessing.getAlgorithm(psName, psArgs);
 
         final DownloadMission mission = new DownloadMission(urls, storage, kind, ps);
         mission.threadCount = threads;
         mission.source = source;
         mission.nearLength = nearLength;
 
+        if (ps != null)
+            ps.setTemporalDir(DownloadManager.pickAvailableTemporalDir(this));
+
         handleConnectivityState(true);// first check the actual network status
 
         mManager.startMission(mission);

app/src/main/java/us/shandian/giga/util/Utility.java

@@ -267,8 +267,7 @@ public class Utility {
         }
 
         try {
-            long length = Long.parseLong(connection.getHeaderField("Content-Length"));
+            return Long.parseLong(connection.getHeaderField("Content-Length"));
-            if (length >= 0) return length;
         } catch (Exception err) {
             // nothing to do
         }