docs and style fixes, remove traces of manifest idea

src/lib/libfetchfs.js

@@ -8,8 +8,11 @@ addToLibrary({
   $FETCHFS__deps: ['$stringToUTF8OnStack', 'wasmfs_create_fetch_backend'],
   $FETCHFS: {
     createBackend(opts) {
-      return _wasmfs_create_fetch_backend(stringToUTF8OnStack(opts.base_url), opts.chunkSize | 0);
-    }
+      opts.base_url ??= "";
+      return withStackSave(() => {
+        return _wasmfs_create_fetch_backend(stringToUTF8OnStack(opts.base_url), opts.chunkSize | 0);
+      });
+    },
   },
 });
 

src/lib/libwasmfs_fetch.js

@@ -31,25 +31,26 @@ addToLibrary({
         try {
           var u = new URL(fileUrl, self.location.origin);
           url = u.toString();
-        } catch (e) {
+        } catch (_e) {
+          throw {status: 404};
         }
       }
       var chunkSize = __wasmfs_fetch_get_chunk_size(file);
-      offset = offset || 0;
+      offset ??= 0;
       len = len || chunkSize;
       var firstChunk = (offset / chunkSize) | 0;
       var lastChunk = ((offset+len) / chunkSize) | 0;
       if (!(file in wasmFS$JSMemoryRanges)) {
         var fileInfo = await fetch(url,{method:"HEAD", headers:{"Range": "bytes=0-"}});
-        if(fileInfo.ok &&
+        if (fileInfo.ok &&
            fileInfo.headers.has("Content-Length") &&
            fileInfo.headers.get("Accept-Ranges") == "bytes" &&
            (parseInt(fileInfo.headers.get("Content-Length")) > chunkSize*2)) {
           wasmFS$JSMemoryRanges[file] = {size:parseInt(fileInfo.headers.get("Content-Length")), chunks:[], chunkSize:chunkSize};
         } else {
           // may as well/forced to download the whole file
           var wholeFileReq = await fetch(url);
-          if(!wholeFileReq.ok) {
+          if (!wholeFileReq.ok) {
             throw wholeFileReq;
           }
           var wholeFileData = new Uint8Array(await wholeFileReq.arrayBuffer());
@@ -60,11 +61,11 @@ addToLibrary({
       }
       var allPresent = true;
       var i;
-      if(lastChunk * chunkSize < offset+len) {
+      if (lastChunk * chunkSize < offset+len) {
         lastChunk += 1;
       }
-      for(i = firstChunk; i < lastChunk; i++) {
-        if(!wasmFS$JSMemoryRanges[file].chunks[i]) {
+      for (i = firstChunk; i < lastChunk; i++) {
+        if (!wasmFS$JSMemoryRanges[file].chunks[i]) {
           allPresent = false;
           break;
         }
@@ -78,14 +79,13 @@ addToLibrary({
       var start = firstChunk*chunkSize;
       var end = lastChunk*chunkSize;
       var response = await fetch(url, {headers:{"Range": `bytes=${start}-${end-1}`}});
-      if (response.ok) {
-        var bytes = new Uint8Array(await response['arrayBuffer']());
-        for (i = firstChunk; i < lastChunk; i++) {
-          wasmFS$JSMemoryRanges[file].chunks[i] = bytes.slice(i*chunkSize-start,(i+1)*chunkSize-start);
-        }
-      } else {
+      if (!response.ok) {
         throw response;
       }
+      var bytes = new Uint8Array(await response['arrayBuffer']());
+      for (i = firstChunk; i < lastChunk; i++) {
+        wasmFS$JSMemoryRanges[file].chunks[i] = bytes.slice(i*chunkSize-start,(i+1)*chunkSize-start);
+      }
       return Promise.resolve();
     }
 
@@ -110,24 +110,21 @@ addToLibrary({
       read: async (file, buffer, length, offset) => {
         try {
           await getFileRange(file, offset || 0, length);
-        } catch (response) {
-          return response.status === 404 ? -{{{ cDefs.ENOENT }}} : -{{{ cDefs.EBADF }}};
+        } catch (failedResponse) {
+          return failedResponse.status === 404 ? -{{{ cDefs.ENOENT }}} : -{{{ cDefs.EBADF }}};
         }
         var fileInfo = wasmFS$JSMemoryRanges[file];
         var fileData = fileInfo.chunks;
         var chunkSize = fileInfo.chunkSize;
         var firstChunk = (offset / chunkSize) | 0;
         var lastChunk = ((offset+length) / chunkSize) | 0;
-        if(offset + length > lastChunk * chunkSize) {
+        if (offset + length > lastChunk * chunkSize) {
           lastChunk += 1;
         }
         var readLength = 0;
         for (var i = firstChunk; i < lastChunk; i++) {
           var chunk = fileData[i];
           var start = Math.max(i*chunkSize, offset);
-          if(!chunk) {
-            throw [fileData.length, firstChunk, lastChunk, i];
-          }
           var chunkStart = i*chunkSize;
           var end = Math.min(chunkStart+chunkSize, offset+length);
           HEAPU8.set(chunk.subarray(start-chunkStart, end-chunkStart), buffer+(start-offset));
@@ -138,7 +135,7 @@ addToLibrary({
       getSize: async (file) => {
         try {
           await getFileRange(file, 0, 0);
-        } catch (response) {
+        } catch (failedResponse) {
           return 0;
         }
         return wasmFS$JSMemoryRanges[file].size;

system/include/emscripten/wasmfs.h

@@ -48,6 +48,18 @@ typedef backend_t (*backend_constructor_t)(void*);
 
 backend_t wasmfs_create_memory_backend(void);
 
+// Fetch backend
+//
+// Creates a new fetchfs backend.  FetchFS will backstop filesystem
+// reads to HTTP fetch requests, which will download just specific
+// ranges of the requested files.  FetchFS works best when your web
+// server supports HTTP range requests, and it's important that those
+// files are not stored encrypted or compressed at rest.  FetchFS by
+// default will dispatch HTTP requests to URLs beginning with base_url
+// and ending with whatever the file's path is relative to where the
+// fetchfs directory is mounted.
+//
+//
 // Note: this cannot be called on the browser main thread because it might
 // deadlock while waiting for its dedicated worker thread to be spawned.
 //
@@ -57,7 +69,9 @@ backend_t wasmfs_create_memory_backend(void);
 //
 // TODO: Add an async version of this function that will work on the main
 // thread.
-backend_t wasmfs_create_fetch_backend(const char* base_url __attribute__((nonnull)), uint32_t);
+//
+backend_t wasmfs_create_fetch_backend(const char* base_url __attribute__((nonnull)),
+                                      uint32_t chunkSize);
 
 backend_t wasmfs_create_node_backend(const char* root __attribute__((nonnull)));
 

system/lib/wasmfs/backends/fetch_backend.cpp

@@ -21,16 +21,14 @@ class FetchBackend : public wasmfs::ProxiedAsyncJSBackend {
   FetchBackend(const std::string& baseUrl,
                uint32_t chunkSize,
                std::function<void(backend_t)> setupOnThread)
-    : ProxiedAsyncJSBackend(setupOnThread), baseUrl(baseUrl), chunkSize(chunkSize)
-      // TODO manifest
-  {}
+    : ProxiedAsyncJSBackend(setupOnThread), baseUrl(baseUrl), chunkSize(chunkSize) {}
   std::shared_ptr<DataFile> createFile(mode_t mode) override;
   std::shared_ptr<Directory> createDirectory(mode_t mode) override;
   const std::string getFileURL(const std::string& filePath);
   uint32_t getChunkSize();
 };
 
-  
+
 class FetchFile : public ProxiedAsyncJSImplFile {
   std::string filePath;
   std::string fileUrl;
@@ -81,6 +79,10 @@ class FetchDirectory : public MemoryDirectory {
   std::string getChildPath(const std::string& name) const {
     return dirPath + '/' + name;
   }
+
+  std::shared_ptr<File> getChild(const std::string& name) override {
+    return MemoryDirectory::getChild(name);
+  }
 };
 
 std::shared_ptr<DataFile> FetchBackend::createFile(mode_t mode) {
@@ -92,41 +94,38 @@ std::shared_ptr<Directory> FetchBackend::createDirectory(mode_t mode) {
 }
 
 const std::string FetchBackend::getFileURL(const std::string& filePath) {
-  // TODO use manifest
   if(filePath == "") {
     return baseUrl;
   }
   return baseUrl + "/" + filePath;
 }
+
 uint32_t FetchBackend::getChunkSize() {
   return chunkSize;
 }
 
 extern "C" {
-  backend_t wasmfs_create_fetch_backend(const char* base_url, uint32_t chunkSize /* TODO manifest */) {
+  backend_t wasmfs_create_fetch_backend(const char* base_url, uint32_t chunkSize) {
   // ProxyWorker cannot safely be synchronously spawned from the main browser
   // thread. See comment in thread_utils.h for more details.
   assert(!emscripten_is_main_browser_thread() &&
          "Cannot safely create fetch backend on main browser thread");
   return wasmFS.addBackend(std::make_unique<FetchBackend>(
     base_url ? base_url : "",
-    chunkSize != 0 ? chunkSize : DEFAULT_CHUNK_SIZE,
-    /* TODO manifest */
+    chunkSize ? chunkSize : DEFAULT_CHUNK_SIZE,
     [](backend_t backend) { _wasmfs_create_fetch_backend_js(backend); }));
   }
 
-const char* EMSCRIPTEN_KEEPALIVE _wasmfs_fetch_get_file_path(void* ptr) {
-  auto* file = reinterpret_cast<wasmfs::FetchFile*>(ptr);
-  return file ? file->getPath().data() : nullptr;
-}
-const char* EMSCRIPTEN_KEEPALIVE _wasmfs_fetch_get_file_url(void* ptr) {
+const char* _wasmfs_fetch_get_file_url(void* ptr) {
   auto* file = reinterpret_cast<wasmfs::FetchFile*>(ptr);
   return file ? file->getURL().data() : nullptr;
 }
-uint32_t EMSCRIPTEN_KEEPALIVE _wasmfs_fetch_get_chunk_size(void* ptr) {
+
+uint32_t _wasmfs_fetch_get_chunk_size(void* ptr) {
   auto* file = reinterpret_cast<wasmfs::FetchFile*>(ptr);
   return file ? file->getChunkSize() : DEFAULT_CHUNK_SIZE;
 }
+
 }
 
 } // namespace wasmfs

system/lib/wasmfs/backends/fetch_backend.h

@@ -6,6 +6,6 @@
 #include "wasmfs.h"
 
 extern "C" {
-  // See library_wasmfs_fetch.js
-  void _wasmfs_create_fetch_backend_js(wasmfs::backend_t);
+// See library_wasmfs_fetch.js
+void _wasmfs_create_fetch_backend_js(wasmfs::backend_t);
 }

test/common.py

@@ -2025,7 +2025,7 @@ def send_head(self):
         f = open(path, 'rb')
         fs = os.fstat(f.fileno())
       except IOError:
-        self.send_error(404, "File not found: " + path)
+        self.send_error(404, f'File not found {path}')
         return None
       if self.path.endswith('.js'):
         self.send_response(200)
@@ -2034,16 +2034,16 @@ def send_head(self):
         self.send_header('Content-length', fs[6])
         self.end_headers()
         return f
-      elif self.headers.get("Range"):
+      elif self.headers.get('Range'):
         self.send_response(206)
         ctype = self.guess_type(path)
         self.send_header('Content-Type', ctype)
-        pieces = self.headers.get("Range").split("=")[1].split("-")
+        pieces = self.headers.get('Range').split('=')[1].split('-')
         start = int(pieces[0]) if pieces[0] != '' else 0
         end = int(pieces[1]) if pieces[1] != '' else fs[6] - 1
         end = min(fs[6] - 1, end)
         length = end - start + 1
-        self.send_header('Content-Range', "bytes " + str(start) + "-" + str(end) + "/" + str(fs[6]))
+        self.send_header('Content-Range', f'bytes {start}-{end}/{fs[6]}')
         self.send_header('Content-Length', str(length))
         self.end_headers()
         return f
@@ -2147,19 +2147,19 @@ def do_GET(self):
         # Use SimpleHTTPServer default file serving operation for GET.
         if DEBUG:
           print('[simple HTTP serving:', unquote_plus(self.path), ']')
-        if self.headers.get("Range"):
+        if self.headers.get('Range'):
           self.send_response(206)
           path = self.translate_path(self.path)
           data = read_binary(path)
           ctype = self.guess_type(path)
           self.send_header('Content-type', ctype)
-          pieces = self.headers.get("range").split("=")[1].split("-")
+          pieces = self.headers.get('Range').split('=')[1].split('-')
           start = int(pieces[0]) if pieces[0] != '' else 0
           end = int(pieces[1]) if pieces[1] != '' else len(data) - 1
           end = min(len(data) - 1, end)
           length = end - start + 1
           self.send_header('Content-Length', str(length))
-          self.send_header('Content-Range', "bytes " + str(start) + "-" + str(end) + "/" + str(len(data)))
+          self.send_header('Content-Range', f'bytes {start}-{end}/{len(data)}')
           self.end_headers()
           self.wfile.write(data[start:end + 1])
         else:

test/wasmfs/wasmfs_fetch.c

@@ -6,6 +6,7 @@
  */
 
 #include <assert.h>
+#include <fcntl.h>
 #include <dirent.h>
 #include <emscripten/emscripten.h>
 #include <emscripten/wasmfs.h>