add opfs support

.gitignore

@@ -17,6 +17,7 @@
 /.emscripten_cache
 .DS_Store
 compile_commands.json
+*.map
 
 /target
 

examples/esbuild-browser/bundle.mjs

@@ -12,8 +12,35 @@ function printErr(err) {
 
 fs.copyFile(path.resolve(DUCKDB_DIST, 'duckdb-mvp.wasm'), './duckdb-mvp.wasm', printErr);
 fs.copyFile(path.resolve(DUCKDB_DIST, 'duckdb-eh.wasm'), './duckdb-eh.wasm', printErr);
+fs.copyFile(path.resolve(DUCKDB_DIST, 'duckdb-coi.wasm'), './duckdb-coi.wasm', printErr);
 fs.copyFile(path.resolve(DUCKDB_DIST, 'duckdb-browser-mvp.worker.js'), './duckdb-browser-mvp.worker.js', printErr);
+fs.copyFile(
+    path.resolve(DUCKDB_DIST, 'duckdb-browser-mvp.worker.js.map'),
+    './duckdb-browser-mvp.worker.js.map',
+    printErr,
+);
 fs.copyFile(path.resolve(DUCKDB_DIST, 'duckdb-browser-eh.worker.js'), './duckdb-browser-eh.worker.js', printErr);
+fs.copyFile(
+    path.resolve(DUCKDB_DIST, 'duckdb-browser-eh.worker.js.map'),
+    './duckdb-browser-eh.worker.js.map',
+    printErr,
+);
+fs.copyFile(path.resolve(DUCKDB_DIST, 'duckdb-browser-coi.worker.js'), './duckdb-browser-coi.worker.js', printErr);
+fs.copyFile(
+    path.resolve(DUCKDB_DIST, 'duckdb-browser-coi.worker.js.map'),
+    './duckdb-browser-coi.worker.js.map',
+    printErr,
+);
+fs.copyFile(
+    path.resolve(DUCKDB_DIST, 'duckdb-browser-coi.pthread.worker.js'),
+    './duckdb-browser-coi.pthread.worker.js',
+    printErr,
+);
+fs.copyFile(
+    path.resolve(DUCKDB_DIST, 'duckdb-browser-coi.pthread.worker.js.map'),
+    './duckdb-browser-coi.pthread.worker.js.map',
+    printErr,
+);
 
 esbuild.build({
     entryPoints: ['./index.ts'],

examples/esbuild-browser/index.ts

@@ -12,16 +12,35 @@ import * as arrow from 'apache-arrow';
                 mainModule: './duckdb-eh.wasm',
                 mainWorker: './duckdb-browser-eh.worker.js',
             },
+            coi: {
+                mainModule: './duckdb-coi.wasm',
+                mainWorker: './duckdb-browser-coi.worker.js',
+                pthreadWorker: './duckdb-browser-coi.pthread.worker.js',
+            },
         });
 
         const logger = new duckdb.ConsoleLogger();
         const worker = new Worker(DUCKDB_CONFIG.mainWorker!);
         const db = new duckdb.AsyncDuckDB(logger, worker);
         await db.instantiate(DUCKDB_CONFIG.mainModule, DUCKDB_CONFIG.pthreadWorker);
 
+        // in-memory
         const conn = await db.connect();
         await conn.query<{ v: arrow.Int }>(`SELECT count(*)::INTEGER as v FROM generate_series(0, 100) t(v)`);
 
+        // opfs
+        // const opfsRoot = await navigator.storage.getDirectory();
+        // await opfsRoot.removeEntry('test.db').catch(e => {});
+        // await db.open({
+        //     path: 'opfs://test.db',
+        //     accessMode: duckdb.DuckDBAccessMode.READ_WRITE,
+        // });
+        // const conn = await db.connect();
+        // await conn.send(`CREATE TABLE integers(i INTEGER, j INTEGER);`);
+        // await conn.send(`INSERT INTO integers VALUES (3, 4), (5, 6);`);
+        // await conn.send(`CHECKPOINT;`);
+        // console.log(await conn.query(`SELECT * FROM integers;`));
+
         await conn.close();
         await db.terminate();
         await worker.terminate();

examples/esbuild-browser/package.json

@@ -10,10 +10,12 @@
     "devDependencies": {
         "esbuild": "^0.19.5",
         "http-server": "^14.1.1",
+        "serve": "^14.2.1",
         "typescript": "^5.2.2"
     },
     "scripts": {
         "build": "node ./bundle.mjs && tsc --noEmit",
-        "server": "http-server"
+        "server": "http-server",
+        "server-coi": "serve -p 8081"
     }
 }

examples/esbuild-browser/serve.json

@@ -0,0 +1,17 @@
+{
+  "headers": [
+    {
+      "source": "*",
+      "headers": [
+        {
+          "key": "Cross-Origin-Embedder-Policy",
+          "value": "require-corp"
+        },
+        {
+          "key": "Cross-Origin-Opener-Policy",
+          "value": "same-origin"
+        }
+      ]
+    }
+  ]
+}

lib/CMakeLists.txt

@@ -22,7 +22,7 @@ if(DEFINED ENV{DUCKDB_WASM_LOADABLE_EXTENSIONS})
 endif()
 
 
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -DDUCKDB_WASM=1")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -DDUCKDB_WASM=1 -DFSST_MUST_ALIGN")
 
 if(DUCKDB_WASM_LOADABLE_EXTENSIONS)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DWASM_LOADABLE_EXTENSIONS=1 -DDUCKDB_EXTENSION_AUTOLOAD_DEFAULT=1 -fPIC")
@@ -110,16 +110,11 @@ if(EMSCRIPTEN)
     endif()
   # Debug build
   elseif(CMAKE_BUILD_TYPE STREQUAL "Debug")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \
-      --profiling \
-      -gsource-map \
-      --source-map-base=file://${CMAKE_BINARY_DIR}/")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g")
     set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} \
       -sASSERTIONS=1 \
       -sSAFE_HEAP=1 \
-      -gsource-map \
-      --source-map-base=file://${CMAKE_BINARY_DIR}/ \
-      -sSEPARATE_DWARF_URL=file://${CMAKE_BINARY_DIR}/duckdb.wasm")
+      -g")
     # ... with fast linking
     if(WASM_FAST_LINKING)
       set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -O0")

lib/include/duckdb/web/config.h

@@ -79,6 +79,8 @@ struct WebDBConfig {
     std::optional<int8_t> access_mode = std::nullopt;
     /// The thread count
     uint32_t maximum_threads = 1;
+    /// The direct io flag
+    bool use_direct_io = false;
     /// The query config
     QueryConfig query = {
         .cast_bigint_to_double = std::nullopt,

lib/src/config.cc

@@ -69,6 +69,9 @@ WebDBConfig WebDBConfig::ReadFrom(std::string_view args_json) {
         if (doc.HasMember("maximumThreads") && doc["maximumThreads"].IsNumber()) {
             config.maximum_threads = doc["maximumThreads"].GetInt();
         }
+        if (doc.HasMember("useDirectIO") && doc["useDirectIO"].IsBool()) {
+            config.use_direct_io = doc["useDirectIO"].GetBool();
+        }
         if (doc.HasMember("query") && doc["query"].IsObject()) {
             auto q = doc["query"].GetObject();
             if (q.HasMember("queryPollingInterval") && q["queryPollingInterval"].IsNumber()) {

lib/src/io/web_filesystem.cc

@@ -226,6 +226,8 @@ WebFileSystem::DataProtocol WebFileSystem::inferDataProtocol(std::string_view ur
         proto = WebFileSystem::DataProtocol::HTTP;
     } else if (hasPrefix(url, "s3://")) {
         proto = WebFileSystem::DataProtocol::S3;
+    } else if (hasPrefix(url, "opfs://")) {
+        proto = WebFileSystem::DataProtocol::BROWSER_FSACCESS;
     } else if (hasPrefix(url, "file://")) {
         data_url = std::string_view{url}.substr(7);
         proto = default_data_protocol_;
@@ -778,7 +780,7 @@ void WebFileSystem::Write(duckdb::FileHandle &handle, void *buffer, int64_t nr_b
     auto file_size = file_hdl.file_->file_size_;
     auto writer = static_cast<char *>(buffer);
     file_hdl.position_ = location;
-    while (nr_bytes > 0 && location < file_size) {
+    while (nr_bytes > 0) {
         auto n = Write(handle, writer, nr_bytes);
         writer += n;
         nr_bytes -= n;

lib/src/webdb.cc

@@ -818,6 +818,7 @@ arrow::Status WebDB::Open(std::string_view args_json) {
         db_config.options.maximum_threads = config_->maximum_threads;
         db_config.options.use_temporary_directory = false;
         db_config.options.access_mode = access_mode;
+        db_config.options.use_direct_io = config_->use_direct_io;
         auto db = std::make_shared<duckdb::DuckDB>(config_->path, &db_config);
 #ifndef WASM_LOADABLE_EXTENSIONS
         duckdb_web_parquet_init(db.get());

packages/duckdb-wasm/src/bindings/bindings_base.ts

@@ -444,13 +444,32 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings {
         }
         dropResponseBuffers(this.mod);
     }
+    /** Prepare a file handle that could only be acquired aschronously */
+    public async prepareDBFileHandle(path: string, protocol: DuckDBDataProtocol): Promise<void> {
+        if (protocol === DuckDBDataProtocol.BROWSER_FSACCESS && this._runtime.prepareDBFileHandle) {
+            const list = await this._runtime.prepareDBFileHandle(path, DuckDBDataProtocol.BROWSER_FSACCESS);
+            for (const item of list) {
+                const { handle, path: filePath, fromCached } = item;
+                if (!fromCached && handle.getSize()) {
+                    await this.registerFileHandle(filePath, handle, DuckDBDataProtocol.BROWSER_FSACCESS, true);
+                }
+            }
+            return;
+        }
+        throw new Error(`prepareDBFileHandle: unsupported protocol ${protocol}`);
+    }
     /** Register a file object URL */
-    public registerFileHandle<HandleType>(
+    public async registerFileHandle<HandleType>(
         name: string,
         handle: HandleType,
         protocol: DuckDBDataProtocol,
         directIO: boolean,
-    ): void {
+    ): Promise<void> {
+        if (protocol === DuckDBDataProtocol.BROWSER_FSACCESS && handle instanceof FileSystemFileHandle) {
+            // handle is an async handle, should convert to sync handle
+            const fileHandle: FileSystemFileHandle = handle as any;
+            handle = (await fileHandle.createSyncAccessHandle()) as any;
+        }
         const [s, d, n] = callSRet(
             this.mod,
             'duckdb_web_fs_register_file_url',
@@ -462,6 +481,9 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings {
         }
         dropResponseBuffers(this.mod);
         globalThis.DUCKDB_RUNTIME._files = (globalThis.DUCKDB_RUNTIME._files || new Map()).set(name, handle);
+        if (globalThis.DUCKDB_RUNTIME._preparedHandles?.[name]) {
+            delete globalThis.DUCKDB_RUNTIME._preparedHandles[name];
+        }
         if (this.pthread) {
             for (const worker of this.pthread.runningWorkers) {
                 worker.postMessage({

packages/duckdb-wasm/src/bindings/bindings_interface.ts

@@ -41,7 +41,8 @@ export interface DuckDBBindings {
         handle: HandleType,
         protocol: DuckDBDataProtocol,
         directIO: boolean,
-    ): void;
+    ): Promise<void>;
+    prepareDBFileHandle(path: string, protocol: DuckDBDataProtocol): Promise<void>;
     globFiles(path: string): WebFile[];
     dropFile(name: string): void;
     dropFiles(): void;

packages/duckdb-wasm/src/bindings/config.ts

@@ -49,6 +49,10 @@ export interface DuckDBConfig {
      * Note that this will only work with cross-origin isolated sites since it requires SharedArrayBuffers.
      */
     maximumThreads?: number;
+    /**
+     * The direct io flag
+     */
+    useDirectIO?: boolean;
     /**
      * The query config
      */

packages/duckdb-wasm/src/bindings/runtime.ts

@@ -87,6 +87,12 @@ export interface DuckDBGlobalFileInfo {
     s3Config?: S3Config;
 }
 
+export interface PreparedDBFileHandle {
+    path: string;
+    handle: any;
+    fromCached: boolean;
+}
+
 /** Call a function with packed response buffer */
 export function callSRet(
     mod: DuckDBModule,
@@ -147,6 +153,8 @@ export interface DuckDBRuntime {
     checkFile(mod: DuckDBModule, pathPtr: number, pathLen: number): boolean;
     removeFile(mod: DuckDBModule, pathPtr: number, pathLen: number): void;
 
+    prepareDBFileHandle?: (path: string, protocol: DuckDBDataProtocol) => Promise<PreparedDBFileHandle[]>;
+
     // Call a scalar UDF function
     callScalarUDF(
         mod: DuckDBModule,