From 0770dc8bcdd1f50bc98177fdab62357d74c51fb0 Mon Sep 17 00:00:00 2001 From: "Joseph C. Osborn" Date: Wed, 29 Jan 2025 09:07:54 -0800 Subject: [PATCH] docs and style fixes, remove traces of manifest idea --- src/lib/libfetchfs.js | 7 ++-- src/lib/libwasmfs_fetch.js | 37 +++++++++----------- system/include/emscripten/wasmfs.h | 16 ++++++++- system/lib/wasmfs/backends/fetch_backend.cpp | 27 +++++++------- system/lib/wasmfs/backends/fetch_backend.h | 4 +-- test/common.py | 14 ++++---- test/wasmfs/wasmfs_fetch.c | 1 + 7 files changed, 60 insertions(+), 46 deletions(-) diff --git a/src/lib/libfetchfs.js b/src/lib/libfetchfs.js index 3a13511a205c4..30b5d287f428e 100644 --- a/src/lib/libfetchfs.js +++ b/src/lib/libfetchfs.js @@ -8,8 +8,11 @@ addToLibrary({ $FETCHFS__deps: ['$stringToUTF8OnStack', 'wasmfs_create_fetch_backend'], $FETCHFS: { createBackend(opts) { - return _wasmfs_create_fetch_backend(stringToUTF8OnStack(opts.base_url), opts.chunkSize | 0); - } + opts.base_url ??= ""; + return withStackSave(() => { + return _wasmfs_create_fetch_backend(stringToUTF8OnStack(opts.base_url), opts.chunkSize | 0); + }); + }, }, }); diff --git a/src/lib/libwasmfs_fetch.js b/src/lib/libwasmfs_fetch.js index d4cafcd671e0f..9e74e0a1d0739 100644 --- a/src/lib/libwasmfs_fetch.js +++ b/src/lib/libwasmfs_fetch.js @@ -31,17 +31,18 @@ addToLibrary({ try { var u = new URL(fileUrl, self.location.origin); url = u.toString(); - } catch (e) { + } catch (_e) { + throw {status: 404}; } } var chunkSize = __wasmfs_fetch_get_chunk_size(file); - offset = offset || 0; + offset ??= 0; len = len || chunkSize; var firstChunk = (offset / chunkSize) | 0; var lastChunk = ((offset+len) / chunkSize) | 0; if (!(file in wasmFS$JSMemoryRanges)) { var fileInfo = await fetch(url,{method:"HEAD", headers:{"Range": "bytes=0-"}}); - if(fileInfo.ok && + if (fileInfo.ok && fileInfo.headers.has("Content-Length") && fileInfo.headers.get("Accept-Ranges") == "bytes" && (parseInt(fileInfo.headers.get("Content-Length")) > chunkSize*2)) { @@ -49,7 +50,7 @@ addToLibrary({ } else { // may as well/forced to download the whole file var wholeFileReq = await fetch(url); - if(!wholeFileReq.ok) { + if (!wholeFileReq.ok) { throw wholeFileReq; } var wholeFileData = new Uint8Array(await wholeFileReq.arrayBuffer()); @@ -60,11 +61,11 @@ addToLibrary({ } var allPresent = true; var i; - if(lastChunk * chunkSize < offset+len) { + if (lastChunk * chunkSize < offset+len) { lastChunk += 1; } - for(i = firstChunk; i < lastChunk; i++) { - if(!wasmFS$JSMemoryRanges[file].chunks[i]) { + for (i = firstChunk; i < lastChunk; i++) { + if (!wasmFS$JSMemoryRanges[file].chunks[i]) { allPresent = false; break; } @@ -78,14 +79,13 @@ addToLibrary({ var start = firstChunk*chunkSize; var end = lastChunk*chunkSize; var response = await fetch(url, {headers:{"Range": `bytes=${start}-${end-1}`}}); - if (response.ok) { - var bytes = new Uint8Array(await response['arrayBuffer']()); - for (i = firstChunk; i < lastChunk; i++) { - wasmFS$JSMemoryRanges[file].chunks[i] = bytes.slice(i*chunkSize-start,(i+1)*chunkSize-start); - } - } else { + if (!response.ok) { throw response; } + var bytes = new Uint8Array(await response['arrayBuffer']()); + for (i = firstChunk; i < lastChunk; i++) { + wasmFS$JSMemoryRanges[file].chunks[i] = bytes.slice(i*chunkSize-start,(i+1)*chunkSize-start); + } return Promise.resolve(); } @@ -110,24 +110,21 @@ addToLibrary({ read: async (file, buffer, length, offset) => { try { await getFileRange(file, offset || 0, length); - } catch (response) { - return response.status === 404 ? -{{{ cDefs.ENOENT }}} : -{{{ cDefs.EBADF }}}; + } catch (failedResponse) { + return failedResponse.status === 404 ? -{{{ cDefs.ENOENT }}} : -{{{ cDefs.EBADF }}}; } var fileInfo = wasmFS$JSMemoryRanges[file]; var fileData = fileInfo.chunks; var chunkSize = fileInfo.chunkSize; var firstChunk = (offset / chunkSize) | 0; var lastChunk = ((offset+length) / chunkSize) | 0; - if(offset + length > lastChunk * chunkSize) { + if (offset + length > lastChunk * chunkSize) { lastChunk += 1; } var readLength = 0; for (var i = firstChunk; i < lastChunk; i++) { var chunk = fileData[i]; var start = Math.max(i*chunkSize, offset); - if(!chunk) { - throw [fileData.length, firstChunk, lastChunk, i]; - } var chunkStart = i*chunkSize; var end = Math.min(chunkStart+chunkSize, offset+length); HEAPU8.set(chunk.subarray(start-chunkStart, end-chunkStart), buffer+(start-offset)); @@ -138,7 +135,7 @@ addToLibrary({ getSize: async (file) => { try { await getFileRange(file, 0, 0); - } catch (response) { + } catch (failedResponse) { return 0; } return wasmFS$JSMemoryRanges[file].size; diff --git a/system/include/emscripten/wasmfs.h b/system/include/emscripten/wasmfs.h index 34d79761336c3..34ff18f7b39a8 100644 --- a/system/include/emscripten/wasmfs.h +++ b/system/include/emscripten/wasmfs.h @@ -48,6 +48,18 @@ typedef backend_t (*backend_constructor_t)(void*); backend_t wasmfs_create_memory_backend(void); +// Fetch backend +// +// Creates a new fetchfs backend. FetchFS will backstop filesystem +// reads to HTTP fetch requests, which will download just specific +// ranges of the requested files. FetchFS works best when your web +// server supports HTTP range requests, and it's important that those +// files are not stored encrypted or compressed at rest. FetchFS by +// default will dispatch HTTP requests to URLs beginning with base_url +// and ending with whatever the file's path is relative to where the +// fetchfs directory is mounted. +// +// // Note: this cannot be called on the browser main thread because it might // deadlock while waiting for its dedicated worker thread to be spawned. // @@ -57,7 +69,9 @@ backend_t wasmfs_create_memory_backend(void); // // TODO: Add an async version of this function that will work on the main // thread. -backend_t wasmfs_create_fetch_backend(const char* base_url __attribute__((nonnull)), uint32_t); +// +backend_t wasmfs_create_fetch_backend(const char* base_url __attribute__((nonnull)), + uint32_t chunkSize); backend_t wasmfs_create_node_backend(const char* root __attribute__((nonnull))); diff --git a/system/lib/wasmfs/backends/fetch_backend.cpp b/system/lib/wasmfs/backends/fetch_backend.cpp index 08200e43e2e11..36c76fd7c6f7c 100644 --- a/system/lib/wasmfs/backends/fetch_backend.cpp +++ b/system/lib/wasmfs/backends/fetch_backend.cpp @@ -21,16 +21,14 @@ class FetchBackend : public wasmfs::ProxiedAsyncJSBackend { FetchBackend(const std::string& baseUrl, uint32_t chunkSize, std::function setupOnThread) - : ProxiedAsyncJSBackend(setupOnThread), baseUrl(baseUrl), chunkSize(chunkSize) - // TODO manifest - {} + : ProxiedAsyncJSBackend(setupOnThread), baseUrl(baseUrl), chunkSize(chunkSize) {} std::shared_ptr createFile(mode_t mode) override; std::shared_ptr createDirectory(mode_t mode) override; const std::string getFileURL(const std::string& filePath); uint32_t getChunkSize(); }; - + class FetchFile : public ProxiedAsyncJSImplFile { std::string filePath; std::string fileUrl; @@ -81,6 +79,10 @@ class FetchDirectory : public MemoryDirectory { std::string getChildPath(const std::string& name) const { return dirPath + '/' + name; } + + std::shared_ptr getChild(const std::string& name) override { + return MemoryDirectory::getChild(name); + } }; std::shared_ptr FetchBackend::createFile(mode_t mode) { @@ -92,41 +94,38 @@ std::shared_ptr FetchBackend::createDirectory(mode_t mode) { } const std::string FetchBackend::getFileURL(const std::string& filePath) { - // TODO use manifest if(filePath == "") { return baseUrl; } return baseUrl + "/" + filePath; } + uint32_t FetchBackend::getChunkSize() { return chunkSize; } extern "C" { - backend_t wasmfs_create_fetch_backend(const char* base_url, uint32_t chunkSize /* TODO manifest */) { + backend_t wasmfs_create_fetch_backend(const char* base_url, uint32_t chunkSize) { // ProxyWorker cannot safely be synchronously spawned from the main browser // thread. See comment in thread_utils.h for more details. assert(!emscripten_is_main_browser_thread() && "Cannot safely create fetch backend on main browser thread"); return wasmFS.addBackend(std::make_unique( base_url ? base_url : "", - chunkSize != 0 ? chunkSize : DEFAULT_CHUNK_SIZE, - /* TODO manifest */ + chunkSize ? chunkSize : DEFAULT_CHUNK_SIZE, [](backend_t backend) { _wasmfs_create_fetch_backend_js(backend); })); } -const char* EMSCRIPTEN_KEEPALIVE _wasmfs_fetch_get_file_path(void* ptr) { - auto* file = reinterpret_cast(ptr); - return file ? file->getPath().data() : nullptr; -} -const char* EMSCRIPTEN_KEEPALIVE _wasmfs_fetch_get_file_url(void* ptr) { +const char* _wasmfs_fetch_get_file_url(void* ptr) { auto* file = reinterpret_cast(ptr); return file ? file->getURL().data() : nullptr; } -uint32_t EMSCRIPTEN_KEEPALIVE _wasmfs_fetch_get_chunk_size(void* ptr) { + +uint32_t _wasmfs_fetch_get_chunk_size(void* ptr) { auto* file = reinterpret_cast(ptr); return file ? file->getChunkSize() : DEFAULT_CHUNK_SIZE; } + } } // namespace wasmfs diff --git a/system/lib/wasmfs/backends/fetch_backend.h b/system/lib/wasmfs/backends/fetch_backend.h index b6136b1080f42..a5d76859c1451 100644 --- a/system/lib/wasmfs/backends/fetch_backend.h +++ b/system/lib/wasmfs/backends/fetch_backend.h @@ -6,6 +6,6 @@ #include "wasmfs.h" extern "C" { - // See library_wasmfs_fetch.js - void _wasmfs_create_fetch_backend_js(wasmfs::backend_t); +// See library_wasmfs_fetch.js +void _wasmfs_create_fetch_backend_js(wasmfs::backend_t); } diff --git a/test/common.py b/test/common.py index 56708abf27319..8c4e8473ca45a 100644 --- a/test/common.py +++ b/test/common.py @@ -2025,7 +2025,7 @@ def send_head(self): f = open(path, 'rb') fs = os.fstat(f.fileno()) except IOError: - self.send_error(404, "File not found: " + path) + self.send_error(404, f'File not found {path}') return None if self.path.endswith('.js'): self.send_response(200) @@ -2034,16 +2034,16 @@ def send_head(self): self.send_header('Content-length', fs[6]) self.end_headers() return f - elif self.headers.get("Range"): + elif self.headers.get('Range'): self.send_response(206) ctype = self.guess_type(path) self.send_header('Content-Type', ctype) - pieces = self.headers.get("Range").split("=")[1].split("-") + pieces = self.headers.get('Range').split('=')[1].split('-') start = int(pieces[0]) if pieces[0] != '' else 0 end = int(pieces[1]) if pieces[1] != '' else fs[6] - 1 end = min(fs[6] - 1, end) length = end - start + 1 - self.send_header('Content-Range', "bytes " + str(start) + "-" + str(end) + "/" + str(fs[6])) + self.send_header('Content-Range', f'bytes {start}-{end}/{fs[6]}') self.send_header('Content-Length', str(length)) self.end_headers() return f @@ -2147,19 +2147,19 @@ def do_GET(self): # Use SimpleHTTPServer default file serving operation for GET. if DEBUG: print('[simple HTTP serving:', unquote_plus(self.path), ']') - if self.headers.get("Range"): + if self.headers.get('Range'): self.send_response(206) path = self.translate_path(self.path) data = read_binary(path) ctype = self.guess_type(path) self.send_header('Content-type', ctype) - pieces = self.headers.get("range").split("=")[1].split("-") + pieces = self.headers.get('Range').split('=')[1].split('-') start = int(pieces[0]) if pieces[0] != '' else 0 end = int(pieces[1]) if pieces[1] != '' else len(data) - 1 end = min(len(data) - 1, end) length = end - start + 1 self.send_header('Content-Length', str(length)) - self.send_header('Content-Range', "bytes " + str(start) + "-" + str(end) + "/" + str(len(data))) + self.send_header('Content-Range', f'bytes {start}-{end}/{len(data)}') self.end_headers() self.wfile.write(data[start:end + 1]) else: diff --git a/test/wasmfs/wasmfs_fetch.c b/test/wasmfs/wasmfs_fetch.c index 95fb4c42f63f5..911438d81b6cd 100644 --- a/test/wasmfs/wasmfs_fetch.c +++ b/test/wasmfs/wasmfs_fetch.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include