cloud_sync: WebDAV, S3 and Google Drive downloads go straight to disk

A download from these servers was gathered whole in memory, up to the
256 MB body limit, then written to the local file in one piece; the
WebDAV driver carried a TODO asking for exactly this.  They now use
task_push_http_download_file_with_headers: the body goes to the file
in pieces as it arrives, and the callback opens the finished file to
hand on.  The HTTP error text these drivers log still reaches them.

A body with no Content-Length or chunked framing is still refused, and
its file is removed with it.  A 404 (or S3's 400) still means the
server does not have the file; the download removed the file already.

network/cloud_sync/google_drive.c

@@ -23,6 +23,7 @@
 #include "../cloud_sync_driver.h"
 #include "../../retroarch.h"
 #include "../../tasks/tasks_internal.h"
+#include "../../tasks/task_file_transfer.h"
 #include "../../verbosity.h"
 
 /* OAuth client credentials (XOR-obfuscated to avoid automated secret scanners;
@@ -1315,6 +1316,7 @@ static void gdrive_read_download_cb(retro_task_t *task, void *task_data,
    {
       RARCH_WARN(GDPFX "%s: response body has no Content-Length or "
             "chunked framing; treating as failure.\n", cb_st->path);
+      filestream_delete(cb_st->file);
       success = false;
    }
 
@@ -1324,21 +1326,12 @@ static void gdrive_read_download_cb(retro_task_t *task, void *task_data,
     * server" (gdrive_read_search_cb()). */
    if (success)
    {
-      file = filestream_open(cb_st->file,
-            RETRO_VFS_FILE_ACCESS_READ_WRITE,
-            RETRO_VFS_FILE_ACCESS_HINT_SEQUENTIAL_BULK);
-      if (   file
-          && data->data && data->len
-          && filestream_write(file, data->data, data->len) != (int64_t)data->len)
+      /* The body went to the file as it arrived */
+      if (!(file = filestream_open(cb_st->file,
+                  RETRO_VFS_FILE_ACCESS_READ,
+                  RETRO_VFS_FILE_ACCESS_HINT_SEQUENTIAL_BULK)))
       {
-         filestream_close(file);
-         file = NULL;
-      }
-      if (file)
-         filestream_seek(file, 0, SEEK_SET);
-      else
-      {
-         RARCH_WARN(GDPFX "Could not write \"%s\".\n", cb_st->file);
+         RARCH_WARN(GDPFX "Could not open \"%s\".\n", cb_st->file);
          success = false;
       }
    }
@@ -1383,8 +1376,9 @@ static void gdrive_read_search_cb(retro_task_t *task, void *task_data,
 
    RARCH_DBG(GDPFX "GET %s\n", cb_st->path);
    headers = gdrive_get_headers(NULL);
-   if (!task_push_http_transfer_with_headers(url, true, NULL,
-         headers, gdrive_read_download_cb, cb_st))
+   /* Downloaded straight to the file, as it arrives */
+   if (!task_push_http_download_file_with_headers(url, cb_st->file, true,
+         NULL, headers, gdrive_read_download_cb, cb_st))
    {
       RARCH_WARN(GDPFX "Could not download %s\n", cb_st->path);
       gdrive_cb_state_finish(cb_st, false);

network/cloud_sync/s3.c

@@ -24,6 +24,7 @@
 #include "../cloud_sync_driver.h"
 #include "../../retroarch.h"
 #include "../../tasks/tasks_internal.h"
+#include "../../tasks/task_file_transfer.h"
 #include "../../verbosity.h"
 #include <compat/strl.h>
 
@@ -859,22 +860,16 @@ static void s3_read_cb(retro_task_t *task, void *task_data, void *user_data, con
    else if (!success)
       s3_log_http_failure(s3_cb_st->path, data, err);
 
+   /* The body went to the file as it arrived */
    if (success && data && data->status >= 200 && data->status < 300)
    {
-      file = filestream_open(s3_cb_st->file,
-            RETRO_VFS_FILE_ACCESS_READ_WRITE,
-            RETRO_VFS_FILE_ACCESS_HINT_SEQUENTIAL_BULK);
-      if (!file)
+      if (!(file = filestream_open(s3_cb_st->file,
+                  RETRO_VFS_FILE_ACCESS_READ,
+                  RETRO_VFS_FILE_ACCESS_HINT_SEQUENTIAL_BULK)))
       {
          RARCH_WARN(S3_PFX "Failed to open local file for read '%s'\n", s3_cb_st->file);
          success = false;
       }
-      else
-      {
-         if (data->data && data->len > 0)
-            filestream_write(file, data->data, data->len);
-         filestream_seek(file, 0, SEEK_SET);
-      }
    }
 
    if (s3_cb_st->cb)
@@ -926,10 +921,11 @@ static void s3_delete_cb(retro_task_t *task, void *task_data, void *user_data, c
    free(s3_cb_st);
 }
 
+/* GET straight into @cb_state's file, as the body arrives */
 static bool s3_http_get_async(const char *url, const char *headers, s3_cb_state_t *cb_state)
 {
-   return task_push_http_transfer_with_headers(url, true, NULL,
-         headers, s3_read_cb, cb_state) != NULL;
+   return task_push_http_download_file_with_headers(url, cb_state->file,
+         true, NULL, headers, s3_read_cb, cb_state) != NULL;
 }
 
 /* PUT @cb_state's body, streamed from its file */

network/cloud_sync/webdav.c

@@ -27,6 +27,7 @@
 #include "../cloud_sync_driver.h"
 #include "../../retroarch.h"
 #include "../../tasks/tasks_internal.h"
+#include "../../tasks/task_file_transfer.h"
 #include "../../verbosity.h"
 
 typedef struct
@@ -901,7 +902,9 @@ static void webdav_read_cb(retro_task_t *task, void *task_data, void *user_data,
 
       RARCH_DBG("[webdav] GET %s\n", url_encoded);
       auth_header = webdav_get_auth_header("GET", url_encoded);
-      task_push_http_transfer_with_headers(url_encoded, true, NULL, auth_header, webdav_read_cb, webdav_cb_st);
+      task_push_http_download_file_with_headers(url_encoded,
+            webdav_cb_st->file, true, NULL, auth_header,
+            webdav_read_cb, webdav_cb_st);
       free(auth_header);
       return;
    }
@@ -912,6 +915,7 @@ static void webdav_read_cb(retro_task_t *task, void *task_data, void *user_data,
    {
       RARCH_WARN("[webdav] %s: response body has no Content-Length or "
             "chunked framing; treating as failure.\n", webdav_cb_st->path);
+      filestream_delete(webdav_cb_st->file);
       found   = false;
       success = false;
    }
@@ -922,23 +926,12 @@ static void webdav_read_cb(retro_task_t *task, void *task_data, void *user_data,
     * server does not have it (404). */
    if (found)
    {
-      /* TODO/FIXME: it would be better if writing
-       * to the file happened during the network reads */
-      file = filestream_open(webdav_cb_st->file,
-                             RETRO_VFS_FILE_ACCESS_READ_WRITE,
-                             RETRO_VFS_FILE_ACCESS_HINT_SEQUENTIAL_BULK);
-      if (   file
-          && data->data && data->len
-          && filestream_write(file, data->data, data->len) != (int64_t)data->len)
+      /* The body went to the file as it arrived */
+      if (!(file = filestream_open(webdav_cb_st->file,
+                  RETRO_VFS_FILE_ACCESS_READ,
+                  RETRO_VFS_FILE_ACCESS_HINT_SEQUENTIAL_BULK)))
       {
-         filestream_close(file);
-         file = NULL;
-      }
-      if (file)
-         filestream_seek(file, 0, SEEK_SET);
-      else
-      {
-         RARCH_WARN("[webdav] Could not write \"%s\".\n", webdav_cb_st->file);
+         RARCH_WARN("[webdav] Could not open \"%s\".\n", webdav_cb_st->file);
          success = false;
       }
    }
@@ -969,8 +962,10 @@ static bool webdav_read(const char *path, const char *file,
 
    RARCH_DBG("[webdav] GET %s\n", url_encoded);
    auth_header = webdav_get_auth_header("GET", url_encoded);
-   t = task_push_http_transfer_with_headers(url_encoded, true, NULL,
-         auth_header, webdav_read_cb, webdav_cb_st);
+   /* Downloaded straight to the file, as it arrives */
+   t = task_push_http_download_file_with_headers(url_encoded,
+         webdav_cb_st->file, true, NULL, auth_header,
+         webdav_read_cb, webdav_cb_st);
    free(auth_header);
    return (t != NULL);
 }