Skip to content

Commit c2d50c8

Browse files
simolus3simolus3
authored
Merge pull request #36 from powersync-ja/upstream-changes
Pull upstream changes
2 parents 968f936 + 8eba6ae commit c2d50c8

32 files changed

+494
-94
lines changed

‎.changeset/proud-moose-report.md

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@journeyapps/wa-sqlite': patch
3+
---
4+
5+
Fix issue where OPFS VFS could freeze in infinite loop

‎Makefile

+1
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ EMFLAGS_COMMON = \
8484
-s INVOKE_RUN \
8585
-s ENVIRONMENT="web,worker" \
8686
-s STACK_SIZE=512KB \
87+
-s WASM_BIGINT=0 \
8788
$(EMFLAGS_EXTRA)
8889

8990
EMFLAGS_DEBUG = \

‎demo/demo.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ window.addEventListener('DOMContentLoaded', async function() {
7777
let time = performance.now();
7878
worker.postMessage(queries);
7979
worker.addEventListener('message', async function(event) {
80+
timestamp.textContent += ` ${(performance.now() - time).toFixed(1)} milliseconds`;
8081
if (event.data.results) {
8182
// Format the results as tables.
8283
event.data.results
@@ -85,7 +86,6 @@ window.addEventListener('DOMContentLoaded', async function() {
8586
} else {
8687
output.innerHTML = `<pre>${event.data.error.message}</pre>`;
8788
}
88-
timestamp.textContent += ` ${Math.trunc(performance.now() - time) / 1000} seconds`;
8989
button.disabled = false;
9090
}, { once: true });
9191
});

‎demo/file/index.js

+5-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import * as VFS from "../../src/VFS.js";
2-
import { IDBBatchAtomicVFS } from "../../src/examples/IDBBatchAtomicVFS.js";
2+
import { IDBBatchAtomicVFS as MyVFS } from "../../src/examples/IDBBatchAtomicVFS.js";
3+
// import { IDBMirrorVFS as MyVFS } from "../../src/examples/IDBMirrorVFS.js";
34

45
const SEARCH_PARAMS = new URLSearchParams(location.search);
56
const IDB_NAME = SEARCH_PARAMS.get('idb') ?? 'sqlite-vfs';
@@ -43,7 +44,7 @@ document.getElementById('file-fetch').addEventListener('click', async () => {
4344
let vfs;
4445
try {
4546
log(`Importing to IndexedDB ${IDB_NAME}, path ${DB_NAME}`);
46-
vfs = await IDBBatchAtomicVFS.create(IDB_NAME, null);
47+
vfs = await MyVFS.create(IDB_NAME, null);
4748

4849
// @ts-ignore
4950
const importURL = document.getElementById('file-url').value;
@@ -69,7 +70,7 @@ document.getElementById('file-import').addEventListener('change', async event =>
6970
let vfs;
7071
try {
7172
log(`Importing to IndexedDB ${IDB_NAME}, path ${DB_NAME}`);
72-
vfs = await IDBBatchAtomicVFS.create(IDB_NAME, null);
73+
vfs = await MyVFS.create(IDB_NAME, null);
7374
// @ts-ignore
7475
await importDatabase(vfs, DB_NAME, event.target.files[0].stream());
7576
log('Import complete');
@@ -87,7 +88,7 @@ document.getElementById('file-import').addEventListener('change', async event =>
8788
});
8889

8990
/**
90-
* @param {IDBBatchAtomicVFS} vfs
91+
* @param {MyVFS} vfs
9192
* @param {string} path
9293
* @param {ReadableStream} stream
9394
*/

‎demo/file/service-worker.js

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import * as VFS from "../../src/VFS.js";
2-
import { IDBBatchAtomicVFS } from "../../src/examples/IDBBatchAtomicVFS.js";
2+
import { IDBBatchAtomicVFS as MyVFS } from "../../src/examples/IDBBatchAtomicVFS.js";
3+
// import { IDBMirrorVFS as MyVFS } from "../../src/examples/IDBMirrorVFS.js";
34

45
// Install the service worker as soon as possible.
56
globalThis.addEventListener('install', (/** @type {ExtendableEvent} */ event) => {
@@ -26,7 +27,7 @@ globalThis.addEventListener('fetch', async (/** @type {FetchEvent} */ event) =>
2627

2728
return event.respondWith((async () => {
2829
// Create the VFS and streaming source using the request parameters.
29-
const vfs = await IDBBatchAtomicVFS.create(url.searchParams.get('idb'), null);
30+
const vfs = await MyVFS.create(url.searchParams.get('idb'), null);
3031
const path = url.searchParams.get('db');
3132
const source = new DatabaseSource(vfs, path);
3233

‎demo/file/verifier.js

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import SQLiteESMFactory from '../../dist/wa-sqlite-async.mjs';
22
import * as SQLite from '../../src/sqlite-api.js';
3-
import { IDBBatchAtomicVFS } from '../../src/examples/IDBBatchAtomicVFS.js';
3+
import { IDBBatchAtomicVFS as MyVFS } from "../../src/examples/IDBBatchAtomicVFS.js";
4+
// import { IDBMirrorVFS as MyVFS } from "../../src/examples/IDBMirrorVFS.js";
45

56
const SEARCH_PARAMS = new URLSearchParams(location.search);
67
const IDB_NAME = SEARCH_PARAMS.get('idb') ?? 'sqlite-vfs';
@@ -10,7 +11,7 @@ const DB_NAME = SEARCH_PARAMS.get('db') ?? 'sqlite.db';
1011
const module = await SQLiteESMFactory();
1112
const sqlite3 = SQLite.Factory(module);
1213

13-
const vfs = await IDBBatchAtomicVFS.create(IDB_NAME, module);
14+
const vfs = await MyVFS.create(IDB_NAME, module);
1415
// @ts-ignore
1516
sqlite3.vfs_register(vfs, true);
1617

‎dist/mc-wa-sqlite-async.mjs

+2-3
Large diffs are not rendered by default.

‎dist/mc-wa-sqlite-async.wasm

-403 Bytes
Binary file not shown.

‎dist/mc-wa-sqlite-jspi.mjs

+2-3
Large diffs are not rendered by default.

‎dist/mc-wa-sqlite-jspi.wasm

41 Bytes
Binary file not shown.

‎dist/mc-wa-sqlite.mjs

+2-3
Large diffs are not rendered by default.

‎dist/mc-wa-sqlite.wasm

-71 Bytes
Binary file not shown.

‎dist/wa-sqlite-async-dynamic-main.mjs

+2-3
Large diffs are not rendered by default.

‎dist/wa-sqlite-async-dynamic-main.wasm

-61.4 KB
Binary file not shown.

‎dist/wa-sqlite-async.mjs

+2-3
Large diffs are not rendered by default.

‎dist/wa-sqlite-async.wasm

-398 Bytes
Binary file not shown.

‎dist/wa-sqlite-dynamic-main.mjs

+2-3
Large diffs are not rendered by default.

‎dist/wa-sqlite-dynamic-main.wasm

-61.9 KB
Binary file not shown.

‎dist/wa-sqlite-jspi.mjs

+2-3
Large diffs are not rendered by default.

‎dist/wa-sqlite-jspi.wasm

-25 Bytes
Binary file not shown.

‎dist/wa-sqlite.mjs

+2-3
Large diffs are not rendered by default.

‎dist/wa-sqlite.wasm

-127 Bytes
Binary file not shown.

‎package.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,8 @@
3838
"@changesets/cli": "^2.26.2",
3939
"@types/jasmine": "^5.1.4",
4040
"@web/dev-server": "^0.4.6",
41-
"@web/test-runner": "^0.18.2",
42-
"@web/test-runner-core": "^0.13.3",
41+
"@web/test-runner": "^0.20.0",
42+
"@web/test-runner-core": "^0.13.4",
4343
"comlink": "^4.4.1",
4444
"jasmine-core": "^4.5.0",
4545
"monaco-editor": "^0.34.1",

‎src/WebLocksMixin.js

+8-6
Original file line numberDiff line numberDiff line change
@@ -82,8 +82,10 @@ export const WebLocksMixin = superclass => class extends superclass {
8282
*/
8383
async jUnlock(fileId, lockType) {
8484
try {
85+
// SQLite can call xUnlock() without ever calling xLock() so
86+
// the state may not exist.
8587
const lockState = this.#mapIdToState.get(fileId);
86-
if (lockType >= lockState.type) return VFS.SQLITE_OK;
88+
if (!(lockType < lockState?.type)) return VFS.SQLITE_OK;
8789

8890
switch (this.#options.lockPolicy) {
8991
case 'exclusive':
@@ -122,17 +124,17 @@ export const WebLocksMixin = superclass => class extends superclass {
122124
}
123125

124126
/**
125-
* @param {number} pFile
127+
* @param {number} fileId
126128
* @param {number} op
127129
* @param {DataView} pArg
128130
* @returns {number|Promise<number>}
129131
*/
130-
jFileControl(pFile, op, pArg) {
131-
const lockState = this.#mapIdToState.get(pFile) ??
132+
jFileControl(fileId, op, pArg) {
133+
const lockState = this.#mapIdToState.get(fileId) ??
132134
(() => {
133135
// Call jLock() to create the lock state.
134-
this.jLock(pFile, VFS.SQLITE_LOCK_NONE);
135-
return this.#mapIdToState.get(pFile);
136+
this.jLock(fileId, VFS.SQLITE_LOCK_NONE);
137+
return this.#mapIdToState.get(fileId);
136138
})();
137139
if (op === WebLocksMixin.WRITE_HINT_OP_CODE &&
138140
this.#options.lockPolicy === 'shared+hint'){

‎src/examples/IDBBatchAtomicVFS.js

+33-26
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,11 @@ import { FacadeVFS } from '../FacadeVFS.js';
33
import * as VFS from '../VFS.js';
44
import { WebLocksMixin } from '../WebLocksMixin.js';
55

6+
const RETRYABLE_ERRORS = new Set([
7+
'TransactionInactiveError',
8+
'InvalidStateError'
9+
]);
10+
611
/**
712
* @typedef Metadata
813
* @property {string} name
@@ -482,25 +487,27 @@ export class IDBBatchAtomicVFS extends WebLocksMixin(FacadeVFS) {
482487
break;
483488
case VFS.SQLITE_FCNTL_SYNC:
484489
this.log?.('xFileControl', file.path, 'SYNC');
485-
const commitMetadata = Object.assign({}, file.metadata);
486-
const prevFileSize = file.rollback.fileSize
487-
this.#idb.q(({ metadata, blocks }) => {
488-
metadata.put(commitMetadata);
489-
490-
// Remove old page versions.
491-
for (const offset of file.changedPages) {
492-
if (offset < prevFileSize) {
493-
const range = IDBKeyRange.bound(
494-
[file.path, -offset, commitMetadata.version],
495-
[file.path, -offset, Infinity],
496-
true);
497-
blocks.delete(range);
490+
if (file.rollback) {
491+
const commitMetadata = Object.assign({}, file.metadata);
492+
const prevFileSize = file.rollback.fileSize
493+
this.#idb.q(({ metadata, blocks }) => {
494+
metadata.put(commitMetadata);
495+
496+
// Remove old page versions.
497+
for (const offset of file.changedPages) {
498+
if (offset < prevFileSize) {
499+
const range = IDBKeyRange.bound(
500+
[file.path, -offset, commitMetadata.version],
501+
[file.path, -offset, Infinity],
502+
true);
503+
blocks.delete(range);
504+
}
498505
}
499-
}
500-
file.changedPages.clear();
501-
}, 'rw', file.txOptions);
502-
file.needsMetadataSync = false;
503-
file.rollback = null;
506+
file.changedPages.clear();
507+
}, 'rw', file.txOptions);
508+
file.needsMetadataSync = false;
509+
file.rollback = null;
510+
}
504511
break;
505512
case VFS.SQLITE_FCNTL_BEGIN_ATOMIC_WRITE:
506513
// Every write transaction is atomic, so this is a no-op.
@@ -717,21 +724,21 @@ export class IDBContext {
717724
});
718725
}
719726

720-
// @ts-ignore
721-
// Create object store proxies.
722-
const objectStores = [...tx.objectStoreNames].map(name => {
723-
return [name, this.proxyStoreOrIndex(tx.objectStore(name))];
724-
});
725-
726727
try {
728+
// @ts-ignore
729+
// Create object store proxies.
730+
const objectStores = [...tx.objectStoreNames].map(name => {
731+
return [name, this.proxyStoreOrIndex(tx.objectStore(name))];
732+
});
733+
727734
// Execute the function.
728735
return await f(Object.fromEntries(objectStores));
729736
} catch (e) {
730737
// Use a new transaction if this one was inactive. This will
731738
// happen if the last request in the transaction completed
732739
// in a previous task but the transaction has not yet committed.
733-
if (!i && e.name === 'TransactionInactiveError') {
734-
this.log?.('TransactionInactiveError, retrying');
740+
if (!i && RETRYABLE_ERRORS.has(e.name)) {
741+
this.log?.(`${e.name}, retrying`);
735742
tx = null;
736743
continue;
737744
}

‎src/examples/IDBMirrorVFS.js

+21-7
Original file line numberDiff line numberDiff line change
@@ -332,13 +332,12 @@ export class IDBMirrorVFS extends FacadeVFS {
332332
const file = this.#mapIdToFile.get(fileId);
333333

334334
if (file.flags & VFS.SQLITE_OPEN_MAIN_DB) {
335-
if (!file.txActive) {
336-
file.txActive = {
337-
path: file.path,
338-
txId: file.viewTx.txId + 1,
339-
blocks: new Map(),
340-
fileSize: file.blockSize * file.blocks.size,
341-
};
335+
this.#requireTxActive(file);
336+
// SQLite is not necessarily written sequentially, so fill in the
337+
// unwritten blocks here.
338+
for (let fillOffset = file.txActive.fileSize;
339+
fillOffset < iOffset; fillOffset += pData.byteLength) {
340+
file.txActive.blocks.set(fillOffset, new Uint8Array(pData.byteLength));
342341
}
343342
file.txActive.blocks.set(iOffset, pData.slice());
344343
file.txActive.fileSize = Math.max(file.txActive.fileSize, iOffset + pData.byteLength);
@@ -375,6 +374,7 @@ export class IDBMirrorVFS extends FacadeVFS {
375374
const file = this.#mapIdToFile.get(fileId);
376375

377376
if (file.flags & VFS.SQLITE_OPEN_MAIN_DB) {
377+
this.#requireTxActive(file);
378378
file.txActive.fileSize = iSize;
379379
} else {
380380
// All files that are not main databases are stored in a single
@@ -717,6 +717,20 @@ export class IDBMirrorVFS extends FacadeVFS {
717717
file.txWriteHint = false;
718718
}
719719

720+
/**
721+
* @param {File} file
722+
*/
723+
#requireTxActive(file) {
724+
if (!file.txActive) {
725+
file.txActive = {
726+
path: file.path,
727+
txId: file.viewTx.txId + 1,
728+
blocks: new Map(),
729+
fileSize: file.blockSize * file.blocks.size,
730+
};
731+
}
732+
}
733+
720734
/**
721735
* @param {string} path
722736
* @returns {Promise}

‎src/examples/OPFSAdaptiveVFS.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -413,7 +413,7 @@ export class OPFSAdaptiveVFS extends WebLocksMixin(FacadeVFS) {
413413
this.lastError = e;
414414
return VFS.SQLITE_IOERR;
415415
}
416-
return VFS.SQLITE_NOTFOUND;
416+
return super.jFileControl(fileId, op, pArg);
417417
}
418418

419419
jGetLastError(zBuf) {

‎src/examples/OPFSCoopSyncVFS.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -437,7 +437,7 @@ export class OPFSCoopSyncVFS extends FacadeVFS {
437437
if (file.persistentFile.isHandleRequested) {
438438
// Another connection wants the access handle.
439439
this.#releaseAccessHandle(file);
440-
this.isHandleRequested = false;
440+
file.persistentFile.isHandleRequested = false;
441441
}
442442
file.persistentFile.isFileLocked = false;
443443
}

‎src/examples/OPFSPermutedVFS.js

+3
Original file line numberDiff line numberDiff line change
@@ -463,6 +463,9 @@ export class OPFSPermutedVFS extends FacadeVFS {
463463
const file = this.#mapIdToFile.get(fileId);
464464
if ((file.flags & VFS.SQLITE_OPEN_MAIN_DB) && !file.txIsOverwrite) {
465465
file.abortController.signal.throwIfAborted();
466+
if (!file.txActive) {
467+
this.#beginTx(file);
468+
}
466469
file.txActive.fileSize = iSize;
467470

468471
// Remove now obsolete pages from file.txActive.pages

‎src/examples/README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ Changing the page size after the database is created is not supported (this is a
1919
### IDBMirrorVFS
2020
This VFS keeps all files in memory, persisting database files to IndexedDB. It works on all contexts.
2121

22-
IDBBatchAtomicVFS can trade durability for performance by setting `PRAGMA synchronous=normal`.
22+
IDBMirrorVFS can trade durability for performance by setting `PRAGMA synchronous=normal`.
2323

2424
Changing the page size after the database is created is not supported.
2525

‎web-test-runner.config.mjs

+3
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,12 @@ export default /** @type {import("@web/test-runner").TestRunnerConfig} */ ({
88
defaultTimeoutInterval: 5 * 60 * 1000
99
},
1010
},
11+
browserLogs: true,
12+
browserStartTimeout: 60_000,
1113
nodeResolve: true,
1214
files: ['./test/*.test.js'],
1315
concurrency: 1,
16+
concurrentBrowsers: 1,
1417
browsers: [
1518
chromeLauncher({
1619
launchOptions: {

0 commit comments

Comments
 (0)