Require yes/no from user if wasm downloading > 10 MB of data (#156)

* Separate read buffer class instead of in response class

* Store response in stream

* Show prompt for large downloads

Author: ianthomas23

CMakeLists.txt

@@ -103,6 +103,8 @@ set(GIT2CPP_SRC
     ${GIT2CPP_SOURCE_DIR}/wasm/constants.hpp
     ${GIT2CPP_SOURCE_DIR}/wasm/libgit2_internals.cpp
     ${GIT2CPP_SOURCE_DIR}/wasm/libgit2_internals.hpp
+    ${GIT2CPP_SOURCE_DIR}/wasm/read_buffer.cpp
+    ${GIT2CPP_SOURCE_DIR}/wasm/read_buffer.hpp
     ${GIT2CPP_SOURCE_DIR}/wasm/response.cpp
     ${GIT2CPP_SOURCE_DIR}/wasm/response.hpp
     ${GIT2CPP_SOURCE_DIR}/wasm/scope.cpp

src/wasm/read_buffer.cpp

@@ -0,0 +1,12 @@
+#ifdef EMSCRIPTEN
+
+#    include "read_buffer.hpp"
+
+read_buffer_t::read_buffer_t(char* buffer, size_t buffer_size, size_t* bytes_read)
+    : m_buffer(buffer)
+    , m_buffer_size(buffer_size)
+    , m_bytes_read(bytes_read)
+{
+}
+
+#endif  // EMSCRIPTEN

src/wasm/read_buffer.hpp

@@ -0,0 +1,18 @@
+#pragma once
+
+#ifdef EMSCRIPTEN
+
+#    include <stddef.h>
+
+// Buffer used when reading remote data is created by libgit2 and passed to transport layer to use.
+// We only fill the buffer and otherwise return it unmodified.
+struct read_buffer_t
+{
+    read_buffer_t(char* buffer, size_t buffer_size, size_t* bytes_read);
+
+    char* m_buffer;
+    size_t m_buffer_size;
+    size_t* m_bytes_read;
+};
+
+#endif  // EMSCRIPTEN

src/wasm/response.cpp

@@ -5,13 +5,11 @@
 #    include "../utils/common.hpp"
 #    include "libgit2_internals.hpp"
 
-wasm_http_response::wasm_http_response(char* buffer, size_t buffer_size, size_t* bytes_read)
-    : m_buffer(buffer)
-    , m_buffer_size(buffer_size)
-    , m_bytes_read(bytes_read)
-    , m_status(0)
+wasm_http_response::wasm_http_response()
+    : m_status(0)
+    , m_read_count(0)
+    , m_total_bytes(0)
 {
-    *m_bytes_read = 0;
 }
 
 void wasm_http_response::add_header(const std::string& key, const std::string& value)
@@ -21,9 +19,10 @@ void wasm_http_response::add_header(const std::string& key, const std::string& v
 
 void wasm_http_response::clear()
 {
-    *m_bytes_read = 0;
     m_status = 0;
     m_status_text.clear();
+    m_read_count = 0;
+    m_total_bytes = 0;
     m_response_headers.clear();
 }
 

src/wasm/response.hpp

@@ -13,7 +13,7 @@ class wasm_http_response
 {
 public:
 
-    wasm_http_response(char* buffer, size_t buffer_size, size_t* bytes_read);
+    wasm_http_response();
 
     void add_header(const std::string& key, const std::string& value);
 
@@ -29,11 +29,10 @@ class wasm_http_response
 
     void set_git_error(std::string_view url) const;
 
-    char* m_buffer;  // Not owned.
-    size_t m_buffer_size;
-    size_t* m_bytes_read;  // Not owned.
-    int32_t m_status;      // Specific type corresponding to i32 in emscripten setValue call.
+    int32_t m_status;  // Specific type corresponding to i32 in emscripten setValue call.
     std::string m_status_text;
+    unsigned int m_read_count;
+    int64_t m_total_bytes;  // Specific type corresponding to i64 in emscripten setValue call.
 
 private:
 

src/wasm/stream.cpp

@@ -9,7 +9,9 @@
 
 #    include "../utils/common.hpp"
 #    include "constants.hpp"
+#    include "read_buffer.hpp"
 #    include "response.hpp"
+#    include "utils.hpp"
 
 // Buffer size used in transport_smart, hardcoded in libgit2.
 #    define EMFORGE_BUFSIZE 65536
@@ -157,8 +159,10 @@ EM_JS(
     (int request_index,
      char* buffer,
      size_t buffer_size,
+     bool first_read,  // Set subsequent arguments only if this is true.
      int32_t* status,
      const char** status_text,
+     int64_t* total_bytes,
      const char** response_headers),
     {
         try
@@ -173,22 +177,28 @@ EM_JS(
                 request.content = null;
             }
 
-            let bytes_read = 0;
+            let byte_length = 0;  // Total number of bytes that will be returned.
+            let bytes_read = 0;   // Number of bytes returned by this call.
             if (xhr.response && xhr.response.byteLength)
             {
-                bytes_read = xhr.response.byteLength - request.result_buffer_pointer;
+                byte_length = xhr.response.byteLength;
+                bytes_read = byte_length - request.result_buffer_pointer;
                 if (bytes_read > buffer_size)
                 {
                     bytes_read = buffer_size;
                 }
             }
 
-            // Caller must delete the returned status_text and response_headers.
-            // clang-format off
-            setValue(status, xhr.status, 'i32*');
-            setValue(status_text, stringToNewUTF8(xhr.statusText ?? ""), 'i8**');
-            setValue(response_headers, stringToNewUTF8(xhr.getAllResponseHeaders() ?? ""), 'i8**');
-            // clang-format on
+            if (first_read)
+            {
+                // Caller must delete the returned status_text and response_headers.
+                // clang-format off
+                setValue(status, xhr.status, 'i32*');
+                setValue(status_text, stringToNewUTF8(xhr.statusText ?? ""), 'i8**');
+                setValue(total_bytes, byte_length, 'i64*');
+                setValue(response_headers, stringToNewUTF8(xhr.getAllResponseHeaders() ?? ""), 'i8**');
+                // clang-format on
+            }
 
             if (bytes_read > 0)
             {
@@ -308,7 +318,7 @@ static void delete_request(wasm_http_stream* stream)
     }
 }
 
-static int read(wasm_http_stream* stream, wasm_http_response& response, bool is_read_response)
+static int read(wasm_http_stream* stream, read_buffer_t& read_buffer, bool is_read_response)
 {
     if (is_read_response)
     {
@@ -342,18 +352,23 @@ static int read(wasm_http_stream* stream, wasm_http_response& response, bool is_
         }
     }
 
+    auto& response = stream->m_response;
+    bool first_read = response.m_read_count == 0;
     const char* status_text = nullptr;
     const char* response_headers = nullptr;
 
     // Actual read.
     size_t bytes_read = js_read(
         stream->m_request_index,
-        response.m_buffer,
-        response.m_buffer_size,
+        read_buffer.m_buffer,
+        read_buffer.m_buffer_size,
+        first_read,
         &response.m_status,
         &status_text,
+        &response.m_total_bytes,
         &response_headers
     );
+    stream->m_response.m_read_count++;
     if (bytes_read == static_cast<size_t>(-1))
     {
         convert_js_to_git_error(stream);
@@ -363,43 +378,52 @@ static int read(wasm_http_stream* stream, wasm_http_response& response, bool is_
         return -1;
     }
 
-    response.m_status_text = status_text;
-    delete status_text;  // Delete const char* allocated in JavaScript.
-
-    // Split single string with response headers separated by \r\n into individual headers.
-    auto lines = split_input_at_newlines(response_headers);
-    for (const auto& line : lines)
+    if (first_read)
     {
-        auto pos = line.find(":");
-        if (pos == std::string::npos)
+        response.m_status_text = status_text;
+        delete status_text;  // Delete const char* allocated in JavaScript.
+
+        // Split single string with response headers separated by \r\n into individual headers.
+        auto lines = split_input_at_newlines(response_headers);
+        for (const auto& line : lines)
         {
-            // Skip invalid lines.  Should this be an error condition?
-            continue;
+            auto pos = line.find(":");
+            if (pos == std::string::npos)
+            {
+                // Skip invalid lines.  Should this be an error condition?
+                continue;
+            }
+            response.add_header(line.substr(0, pos), line.substr(pos + 1));
         }
-        response.add_header(line.substr(0, pos), line.substr(pos + 1));
-    }
-    delete response_headers;  // Delete const char* allocated in JavaScript.
+        delete response_headers;  // Delete const char* allocated in JavaScript.
 
-    // If successful, check expected response content-type is correct.
-    if (response.m_status == GIT_HTTP_STATUS_OK)
-    {
-        auto expected_response_type = stream->m_service.m_response_type;
-        if (!expected_response_type.empty()
-            && !response.has_header_matches("content-type", expected_response_type))
+        // If successful, check expected response content-type is correct.
+        if (response.m_status == GIT_HTTP_STATUS_OK)
         {
-            // Not sure this should be checked at all, as CORS proxy may be doing something
-            // with it.
-            git_error_set(
-                GIT_ERROR_HTTP,
-                "expected response content-type header '%s' to request %s",
-                expected_response_type.c_str(),
-                stream->m_unconverted_url.c_str()
-            );
+            auto expected_response_type = stream->m_service.m_response_type;
+            if (!expected_response_type.empty()
+                && !response.has_header_matches("content-type", expected_response_type))
+            {
+                // Not sure this should be checked at all, as CORS proxy may be doing something
+                // with it.
+                git_error_set(
+                    GIT_ERROR_HTTP,
+                    "expected response content-type header '%s' to request %s",
+                    expected_response_type.c_str(),
+                    stream->m_unconverted_url.c_str()
+                );
+                return -1;
+            }
+        }
+
+        if (!maybe_prompt_to_download(response.m_total_bytes))
+        {
+            git_error_set(GIT_ERROR_NONE, "Download aborted");
             return -1;
         }
     }
 
-    *response.m_bytes_read = bytes_read;
+    *read_buffer.m_bytes_read = bytes_read;
     return 0;
 }
 
@@ -427,7 +451,7 @@ static int write(wasm_http_stream* stream, const char* buffer, size_t buffer_siz
 
 // C credential functions.
 
-static int create_credential(wasm_http_stream* stream, const wasm_http_response& response)
+static int create_credential(wasm_http_stream* stream)
 {
     wasm_http_subtransport* subtransport = stream->m_subtransport;
 
@@ -440,7 +464,7 @@ static int create_credential(wasm_http_stream* stream, const wasm_http_response&
     subtransport->m_authorization_header = "";
 
     // Check that response headers show support for 'www-authenticate: Basic'.
-    if (!response.has_header_starts_with("www-authenticate", "Basic"))
+    if (!stream->m_response.has_header_starts_with("www-authenticate", "Basic"))
     {
         git_error_set(
             GIT_ERROR_HTTP,
@@ -541,36 +565,36 @@ void wasm_http_stream_free(git_smart_subtransport_stream* s)
 int wasm_http_stream_read(git_smart_subtransport_stream* s, char* buffer, size_t buffer_size, size_t* bytes_read)
 {
     wasm_http_stream* stream = reinterpret_cast<wasm_http_stream*>(s);
-    wasm_http_response response(buffer, buffer_size, bytes_read);
+    read_buffer_t read_buffer(buffer, buffer_size, bytes_read);
 
     bool send = true;
     while (send)
     {
-        if (read(stream, response, false) == static_cast<size_t>(-1))
+        if (read(stream, read_buffer, false) == static_cast<size_t>(-1))
         {
             return -1;  // git error already set.
         }
         send = false;
 
-        auto final_url_header = response.get_header("x-final-url");
+        auto final_url_header = stream->m_response.get_header("x-final-url");
         if (final_url_header.has_value() && stream->ensure_final_url(final_url_header.value())
-            && response.m_status != GIT_HTTP_STATUS_OK)
+            && stream->m_response.m_status != GIT_HTTP_STATUS_OK)
         {
             // Resend only if status not OK, if OK next request will use updated URL.
             send = true;
         }
 
-        if (response.has_header("strict-transport-security") && stream->ensure_https()
-            && response.m_status != GIT_HTTP_STATUS_OK)
+        if (stream->m_response.has_header("strict-transport-security") && stream->ensure_https()
+            && stream->m_response.m_status != GIT_HTTP_STATUS_OK)
         {
             // Resend only if status not OK, if OK next request will use https not http.
             send = true;
         }
 
-        if (response.m_status == GIT_HTTP_STATUS_UNAUTHORIZED)
+        if (stream->m_response.m_status == GIT_HTTP_STATUS_UNAUTHORIZED)
         {
             // Request and create new credentials.
-            if (create_credential(stream, response) < 0)
+            if (create_credential(stream) < 0)
             {
                 return -1;  // git error already set.
             }
@@ -580,13 +604,13 @@ int wasm_http_stream_read(git_smart_subtransport_stream* s, char* buffer, size_t
         if (send)
         {
             delete_request(stream);
-            response.clear();
+            stream->m_response.clear();
         }
     }
 
-    if (response.m_status != GIT_HTTP_STATUS_OK)
+    if (stream->m_response.m_status != GIT_HTTP_STATUS_OK)
     {
-        response.set_git_error(stream->m_unconverted_url);
+        stream->m_response.set_git_error(stream->m_unconverted_url);
         return -1;
     }
 
@@ -597,15 +621,15 @@ int wasm_http_stream_read_response(git_smart_subtransport_stream* s, char* buffe
 {
     wasm_http_stream* stream = reinterpret_cast<wasm_http_stream*>(s);
 
-    wasm_http_response response(buffer, buffer_size, bytes_read);
-    int error = read(stream, response, true);
+    read_buffer_t read_buffer(buffer, buffer_size, bytes_read);
+    int error = read(stream, read_buffer, true);
 
     // May need similar handling of response status and headers as occurs in read() above, but so
     // far this has not been necessary.
 
-    if (error == 0 && response.m_status != GIT_HTTP_STATUS_OK)
+    if (error == 0 && stream->m_response.m_status != GIT_HTTP_STATUS_OK)
     {
-        response.set_git_error(stream->m_unconverted_url);
+        stream->m_response.set_git_error(stream->m_unconverted_url);
         error = -1;
     }
 

src/wasm/stream.hpp

@@ -5,6 +5,7 @@
 #    include <git2/sys/transport.h>
 
 #    include "libgit2_internals.hpp"
+#    include "response.hpp"
 #    include "subtransport.hpp"
 
 // A stream represents a single http/https request.
@@ -26,6 +27,7 @@ struct wasm_http_stream
     http_service m_service;
     std::string m_unconverted_url;
     int m_request_index;
+    wasm_http_response m_response;
 };
 
 void wasm_http_stream_free(git_smart_subtransport_stream* s);