From 83763f17354a568e7c90e8c270cd72a15ab3757d Mon Sep 17 00:00:00 2001 From: Matt Glaman Date: Thu, 11 Jul 2024 12:25:25 -0500 Subject: [PATCH 1/8] Allow downloading sqlite DB --- playground/public/assets/export-db.php | 25 +++++++++++++++ playground/public/index.html | 42 +++++++++++++++++++++++--- 2 files changed, 62 insertions(+), 5 deletions(-) create mode 100644 playground/public/assets/export-db.php diff --git a/playground/public/assets/export-db.php b/playground/public/assets/export-db.php new file mode 100644 index 0000000..9c2deeb --- /dev/null +++ b/playground/public/assets/export-db.php @@ -0,0 +1,25 @@ +getAppRoot()}/{$dbinfo['default']['database']}"; + +print $dbfile; diff --git a/playground/public/index.html b/playground/public/index.html index f673f55..581af91 100644 --- a/playground/public/index.html +++ b/playground/public/index.html @@ -75,6 +75,10 @@

Browser compatibility warningsLoading... + @@ -92,6 +96,10 @@

Browser compatibility warningsLoading... + @@ -144,10 +152,13 @@

Browser compatibility warnings { const flavorCleanup = document.querySelector(`[data-cleanup][data-flavor="${flavor}"]`) const flavorLaunch = document.querySelector(`[data-launch][data-flavor="${flavor}"]`) + const flavorExportDb = document.querySelector(`[data-export-db][data-flavor="${flavor}"]`) if (checkWww.exists) { flavorCleanup.hidden = false flavorCleanup.disabled = false + flavorExportDb.hidden = false + flavorExportDb.disabled = false flavorLaunch.innerHTML = 'Launch' } else { @@ -163,27 +174,49 @@

Browser compatibility warnings { el.addEventListener('click', cleanup) }) + document.querySelectorAll('[data-export-db]').forEach(el => { + el.addEventListener('click', async event => { + const flavor = event.target.dataset.flavor; + + const php = new PhpWeb({ sharedLibs, persist: [{ mountPath: '/persist' }, { mountPath: '/config' }] }); + await php.binary; + + let filePath + php.addEventListener('output', event => filePath = event.detail[0]); + php.addEventListener('error', event => console.log(event.detail)); + + await sendMessage('writeFile', ['/config/flavor.txt', flavor]); + + const exportDbPhpCode = await (await fetch('/assets/export-db.php')).text(); + await php.run(exportDbPhpCode) + + const dbContents = await sendMessage('readFile', [filePath]); + const blob = new Blob([dbContents], {type:'application/x-sqlite3'}) + const link = document.createElement('a'); + link.href = URL.createObjectURL(blob); + link.click(); + }) + }) }); - async function cleanup() { + async function cleanup(event) { const flavor = event.target.dataset.flavor; const flavorCleanup = document.querySelector(`[data-cleanup][data-flavor="${flavor}"]`) - const flavorLaunch = document.querySelector(`[data-launch][data-flavor="${flavor}"]`) flavorCleanup.hidden = true; flavorCleanup.disabled = true; flavorCleanup.innerHTML = 'Cleaning up...' const openDb = indexedDB.open("/persist", 21); - openDb.onsuccess = event => { + openDb.onsuccess = () => { const db = openDb.result; const transaction = db.transaction(["FILE_DATA"], "readwrite"); const objectStore = transaction.objectStore("FILE_DATA"); // IDBKeyRange.bound trick found at https://stackoverflow.com/a/76714057/1949744 const objectStoreRequest = objectStore.delete(IDBKeyRange.bound(`/persist/${flavor}`, `/persist/${flavor}/\uffff`)); - objectStoreRequest.onsuccess = async (event) => { + objectStoreRequest.onsuccess = async () => { db.close(); console.log('Reloading after purging data...'); await sendMessage('refresh', []); @@ -197,7 +230,6 @@

Browser compatibility warnings Date: Thu, 11 Jul 2024 16:02:58 -0500 Subject: [PATCH 2/8] ensure a filename is set --- playground/public/index.html | 45 +++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/playground/public/index.html b/playground/public/index.html index 581af91..d765ec7 100644 --- a/playground/public/index.html +++ b/playground/public/index.html @@ -175,27 +175,7 @@

Browser compatibility warnings { - el.addEventListener('click', async event => { - const flavor = event.target.dataset.flavor; - - const php = new PhpWeb({ sharedLibs, persist: [{ mountPath: '/persist' }, { mountPath: '/config' }] }); - await php.binary; - - let filePath - php.addEventListener('output', event => filePath = event.detail[0]); - php.addEventListener('error', event => console.log(event.detail)); - - await sendMessage('writeFile', ['/config/flavor.txt', flavor]); - - const exportDbPhpCode = await (await fetch('/assets/export-db.php')).text(); - await php.run(exportDbPhpCode) - - const dbContents = await sendMessage('readFile', [filePath]); - const blob = new Blob([dbContents], {type:'application/x-sqlite3'}) - const link = document.createElement('a'); - link.href = URL.createObjectURL(blob); - link.click(); - }) + el.addEventListener('click', exportDb) }) }); @@ -282,6 +262,29 @@

Browser compatibility warnings filePath = event.detail[0]); + + await sendMessage('writeFile', ['/config/flavor.txt', flavor]); + + const exportDbPhpCode = await (await fetch('/assets/export-db.php')).text(); + await php.run(exportDbPhpCode) + + const dbContents = await sendMessage('readFile', [filePath]); + const blob = new Blob([dbContents], {type: 'application/vnd.sqlite3'}) + const link = document.createElement('a'); + link.href = URL.createObjectURL(blob); + link.download = 'drupal.db' + link.click(); + URL.revokeObjectURL(link.href); + } From 8d607b0c2422414cbc8c0ccb90eeec9fc17954df Mon Sep 17 00:00:00 2001 From: Matt Glaman Date: Thu, 11 Jul 2024 16:12:02 -0500 Subject: [PATCH 3/8] add an alert before beginning --- playground/public/index.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/playground/public/index.html b/playground/public/index.html index d765ec7..a7bbed8 100644 --- a/playground/public/index.html +++ b/playground/public/index.html @@ -264,6 +264,8 @@

Browser compatibility warningsBrowser compatibility warnings From 541037db0a75e4d7cbe54dd863c850bb46900bc7 Mon Sep 17 00:00:00 2001 From: Matt Glaman Date: Thu, 11 Jul 2024 17:09:21 -0500 Subject: [PATCH 4/8] instead of downloading DB, try creating a SQL dump --- playground/public/assets/export-db.php | 53 ++++++++++++++++++++++---- playground/public/index.html | 22 +++++++---- playground/public/worker.mjs | 20 +++++----- 3 files changed, 69 insertions(+), 26 deletions(-) diff --git a/playground/public/assets/export-db.php b/playground/public/assets/export-db.php index 9c2deeb..00f2eb5 100644 --- a/playground/public/assets/export-db.php +++ b/playground/public/assets/export-db.php @@ -1,12 +1,11 @@ getAppRoot()}/{$dbinfo['default']['database']}"; - -print $dbfile; +chdir($kernel->getAppRoot()); +$kernel->boot(); + +$sql = ''; + +// adapted from https://github.com/ephestione/php-sqlite-dump/blob/master/sqlite_dump.php +$database = \Drupal::database(); +$tables = $database->query('SELECT [name] FROM {sqlite_master} WHERE [type] = "table" AND [name] NOT LIKE "sqlite_%" ORDER BY name'); +foreach ($tables as $table) { + $sql .= '--' . PHP_EOL; + $sql .= "-- Table structure for table `{$table->name}`" . PHP_EOL; + $sql .= '--' . PHP_EOL; + $sql .= $database->query('SELECT [sql] FROM {sqlite_master} WHERE [name] = :name ', [ + ':name' => $table->name, + ])->fetchField(); + $sql .= PHP_EOL.PHP_EOL; + + $columns = array_map( + fn (object $row) => "`{$row->name}`", + $database->query("PRAGMA table_info({$table->name})")->fetchAll() + ); + $columns = implode(', ', $columns); + + $sql .= '--' . PHP_EOL; + $sql .= "-- Dumping data for table `{$table->name}`" . PHP_EOL; + $sql .= '--' . PHP_EOL; + $sql .= PHP_EOL; + + $sql .= "INSERT INTO {$table->name} ($columns) VALUES" . PHP_EOL; + + $rows = $database->select($table->name)->fields($table->name)->execute()->fetchAll(\PDO::FETCH_ASSOC); + if (count($rows) > 0) { + $last_row = implode(', ', array_pop($rows)); + foreach ($rows as $row) { + $values = implode(', ', array_values($row)); + $sql .= "($values),"; + } + $sql .= "($last_row);"; + } + + $sql .= PHP_EOL.PHP_EOL; +} + +file_put_contents('/persist/dump.sql', $sql); diff --git a/playground/public/index.html b/playground/public/index.html index a7bbed8..ba521e8 100644 --- a/playground/public/index.html +++ b/playground/public/index.html @@ -225,7 +225,10 @@

Browser compatibility warnings console.log(event.detail)); php.addEventListener('error', event => console.log(event.detail)); @@ -265,28 +268,31 @@

Browser compatibility warnings filePath = event.detail[0]); + php.addEventListener('output', event => console.log(event.detail)); await sendMessage('writeFile', ['/config/flavor.txt', flavor]); const exportDbPhpCode = await (await fetch('/assets/export-db.php')).text(); await php.run(exportDbPhpCode) - const dbContents = await sendMessage('readFile', [filePath]); - const blob = new Blob([dbContents], {type: 'application/vnd.sqlite3'}) + const dbContents = await sendMessage('readFile', ['/persist/dump.sql']); + const blob = new Blob([dbContents], {type: 'application/sql'}) const link = document.createElement('a'); link.href = URL.createObjectURL(blob); - link.download = 'drupal.db' + link.download = 'drupal.sql' link.click(); URL.revokeObjectURL(link.href); event.target.disabled = false + setTimeout(() => sendMessage('unlink', ['/persist/dump.sql']), 0); } diff --git a/playground/public/worker.mjs b/playground/public/worker.mjs index f278dd1..7698a5c 100644 --- a/playground/public/worker.mjs +++ b/playground/public/worker.mjs @@ -19,16 +19,16 @@ const notFound = (request) => { }; const sharedLibs = [ - `php\${PHP_VERSION}-zlib.so`, - `php\${PHP_VERSION}-zip.so`, - `php\${PHP_VERSION}-iconv.so`, - `php\${PHP_VERSION}-gd.so`, - `php\${PHP_VERSION}-dom.so`, - `php\${PHP_VERSION}-mbstring.so`, - `php\${PHP_VERSION}-sqlite.so`, - `php\${PHP_VERSION}-pdo-sqlite.so`, - `php\${PHP_VERSION}-xml.so`, - `php\${PHP_VERSION}-simplexml.so`, + `php\${PHP_VERSION}-zlib.so`, + `php\${PHP_VERSION}-zip.so`, + `php\${PHP_VERSION}-iconv.so`, + `php\${PHP_VERSION}-gd.so`, + `php\${PHP_VERSION}-dom.so`, + `php\${PHP_VERSION}-mbstring.so`, + `php\${PHP_VERSION}-sqlite.so`, + `php\${PHP_VERSION}-pdo-sqlite.so`, + `php\${PHP_VERSION}-xml.so`, + `php\${PHP_VERSION}-simplexml.so`, ]; const php = new PhpCgiWorker({ From d162d5b7baa51da99d1b5f09aa1b9d149f9ec543 Mon Sep 17 00:00:00 2001 From: Matt Glaman Date: Fri, 12 Jul 2024 10:41:41 -0500 Subject: [PATCH 5/8] add indexes, set sequences --- .gitignore | 1 + playground/public/assets/export-db.php | 35 ++++++++++++++------------ 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index a56201d..b7e6841 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /drupal-wasm-1.0.zip +*.sql diff --git a/playground/public/assets/export-db.php b/playground/public/assets/export-db.php index 00f2eb5..055194a 100644 --- a/playground/public/assets/export-db.php +++ b/playground/public/assets/export-db.php @@ -19,34 +19,24 @@ chdir($kernel->getAppRoot()); $kernel->boot(); -$sql = ''; +$sql = 'PRAGMA foreign_keys=OFF;' . PHP_EOL; +$sql .= 'BEGIN TRANSACTION;' . PHP_EOL; // adapted from https://github.com/ephestione/php-sqlite-dump/blob/master/sqlite_dump.php $database = \Drupal::database(); -$tables = $database->query('SELECT [name] FROM {sqlite_master} WHERE [type] = "table" AND [name] NOT LIKE "sqlite_%" ORDER BY name'); +$tables = $database->query('SELECT [name], [sql] FROM {sqlite_master} WHERE [type] = "table" AND [name] NOT LIKE "sqlite_%" ORDER BY name'); foreach ($tables as $table) { - $sql .= '--' . PHP_EOL; - $sql .= "-- Table structure for table `{$table->name}`" . PHP_EOL; - $sql .= '--' . PHP_EOL; - $sql .= $database->query('SELECT [sql] FROM {sqlite_master} WHERE [name] = :name ', [ - ':name' => $table->name, - ])->fetchField(); - $sql .= PHP_EOL.PHP_EOL; + $sql .= $table->sql . PHP_EOL; $columns = array_map( fn (object $row) => "`{$row->name}`", $database->query("PRAGMA table_info({$table->name})")->fetchAll() ); $columns = implode(', ', $columns); - - $sql .= '--' . PHP_EOL; - $sql .= "-- Dumping data for table `{$table->name}`" . PHP_EOL; - $sql .= '--' . PHP_EOL; $sql .= PHP_EOL; - $sql .= "INSERT INTO {$table->name} ($columns) VALUES" . PHP_EOL; - $rows = $database->select($table->name)->fields($table->name)->execute()->fetchAll(\PDO::FETCH_ASSOC); + $sql .= "INSERT INTO {$table->name} ($columns) VALUES" . PHP_EOL; if (count($rows) > 0) { $last_row = implode(', ', array_pop($rows)); foreach ($rows as $row) { @@ -56,7 +46,20 @@ $sql .= "($last_row);"; } - $sql .= PHP_EOL.PHP_EOL; + $sql .= PHP_EOL; +} + +$sql .= 'DELETE FROM sqlite_sequence;' . PHP_EOL; +$sequences = $database->query('SELECT [name], [seq] FROM sqlite_sequence'); +foreach ($sequences as $sequence) { + $sql .= "INSERT INTO sqlite_sequence VALUES('{$sequence->name}',{$sequence->seq});" . PHP_EOL; } +$indexes = $database->query('SELECT [sql] FROM {sqlite_master} WHERE [type] = "index" AND [name] NOT LIKE "sqlite_%" ORDER BY name'); +foreach ($indexes as $index) { + $sql .= $index->sql . PHP_EOL; +} + +$sql .= 'COMMIT;' . PHP_EOL; + file_put_contents('/persist/dump.sql', $sql); From 1ac5eb29b4479aaa8cf65dccaf9c3a0532f4557e Mon Sep 17 00:00:00 2001 From: Matt Glaman Date: Fri, 12 Jul 2024 10:42:57 -0500 Subject: [PATCH 6/8] CREATE TABLE to IF NOT EXISTS --- playground/public/assets/export-db.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/public/assets/export-db.php b/playground/public/assets/export-db.php index 055194a..943bc91 100644 --- a/playground/public/assets/export-db.php +++ b/playground/public/assets/export-db.php @@ -26,7 +26,7 @@ $database = \Drupal::database(); $tables = $database->query('SELECT [name], [sql] FROM {sqlite_master} WHERE [type] = "table" AND [name] NOT LIKE "sqlite_%" ORDER BY name'); foreach ($tables as $table) { - $sql .= $table->sql . PHP_EOL; + $sql .= str_replace('CREATE TABLE ', 'CREATE TABLE IF NOT EXISTS ', $table->sql) . PHP_EOL; $columns = array_map( fn (object $row) => "`{$row->name}`", From 1de39d63b2df72a08a34090c9ece1638af945117 Mon Sep 17 00:00:00 2001 From: Matt Glaman Date: Fri, 12 Jul 2024 12:06:05 -0500 Subject: [PATCH 7/8] reduce size and improve sql statements --- playground/public/assets/export-db.php | 64 +++++++++++++++++++++----- 1 file changed, 53 insertions(+), 11 deletions(-) diff --git a/playground/public/assets/export-db.php b/playground/public/assets/export-db.php index 943bc91..4e6defd 100644 --- a/playground/public/assets/export-db.php +++ b/playground/public/assets/export-db.php @@ -1,5 +1,6 @@ getAppRoot()); $kernel->boot(); +foreach (Cache::getBins() as $cache_backend) { + $cache_backend->deleteAll(); +} +\Drupal::service('plugin.cache_clearer')->clearCachedDefinitions(); +$kernel->invalidateContainer(); + $sql = 'PRAGMA foreign_keys=OFF;' . PHP_EOL; $sql .= 'BEGIN TRANSACTION;' . PHP_EOL; @@ -26,24 +33,54 @@ $database = \Drupal::database(); $tables = $database->query('SELECT [name], [sql] FROM {sqlite_master} WHERE [type] = "table" AND [name] NOT LIKE "sqlite_%" ORDER BY name'); foreach ($tables as $table) { - $sql .= str_replace('CREATE TABLE ', 'CREATE TABLE IF NOT EXISTS ', $table->sql) . PHP_EOL; + $sql .= str_replace('CREATE TABLE ', 'CREATE TABLE IF NOT EXISTS ', $table->sql) . ';' . PHP_EOL; $columns = array_map( - fn (object $row) => "`{$row->name}`", + static fn (object $row) => "`{$row->name}`", $database->query("PRAGMA table_info({$table->name})")->fetchAll() ); $columns = implode(', ', $columns); $sql .= PHP_EOL; $rows = $database->select($table->name)->fields($table->name)->execute()->fetchAll(\PDO::FETCH_ASSOC); - $sql .= "INSERT INTO {$table->name} ($columns) VALUES" . PHP_EOL; - if (count($rows) > 0) { - $last_row = implode(', ', array_pop($rows)); - foreach ($rows as $row) { - $values = implode(', ', array_values($row)); - $sql .= "($values),"; - } - $sql .= "($last_row);"; + foreach ($rows as $row) { + $values = implode( + ', ', + array: array_map( + static function ($value) { + if ($value === NULL) { + return 'NULL'; + } + if ($value === '') { + return "''"; + } + if (is_numeric($value)) { + return $value; + } + if (is_string($value)) { + // @todo: this str_contains isn't working correctly + if (str_contains(PHP_EOL, $value)) { + $value = str_replace([PHP_EOL, "\r", "\n", "\r\n"], "\\n", $value); + $value = "replace($value,'\\n',char(10))"; + } + $value = str_replace( + [ + "'", + // TODO there is a null byte in serialized code, replacing it breaks unserialize what is sqlite3 .dump doing? + chr(0)], + [ + "''", + '' + ], $value + ); + return "'$value'"; + } + return $value; + }, + array_values($row) + ) + ); + $sql .= "INSERT INTO {$table->name} VALUES($values);" . PHP_EOL; } $sql .= PHP_EOL; @@ -57,9 +94,14 @@ $indexes = $database->query('SELECT [sql] FROM {sqlite_master} WHERE [type] = "index" AND [name] NOT LIKE "sqlite_%" ORDER BY name'); foreach ($indexes as $index) { - $sql .= $index->sql . PHP_EOL; + $sql .= $index->sql . ';' . PHP_EOL; } + $sql .= 'COMMIT;' . PHP_EOL; +// TODO support this collation somehow +// see https://www.drupal.org/project/drupal/issues/3036487 +$sql = str_replace('NOCASE_UTF8', 'NOCASE', $sql); + file_put_contents('/persist/dump.sql', $sql); From dcdc4cb58300ec93315e3b0daa90173d9372ee91 Mon Sep 17 00:00:00 2001 From: Matt Glaman Date: Fri, 12 Jul 2024 14:43:14 -0500 Subject: [PATCH 8/8] handle serialized php strings and nul chars --- playground/public/assets/export-db.php | 28 +++++++++++--------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/playground/public/assets/export-db.php b/playground/public/assets/export-db.php index 4e6defd..5b15fe5 100644 --- a/playground/public/assets/export-db.php +++ b/playground/public/assets/export-db.php @@ -57,23 +57,19 @@ static function ($value) { if (is_numeric($value)) { return $value; } - if (is_string($value)) { - // @todo: this str_contains isn't working correctly - if (str_contains(PHP_EOL, $value)) { - $value = str_replace([PHP_EOL, "\r", "\n", "\r\n"], "\\n", $value); - $value = "replace($value,'\\n',char(10))"; - } - $value = str_replace( - [ - "'", - // TODO there is a null byte in serialized code, replacing it breaks unserialize what is sqlite3 .dump doing? - chr(0)], - [ - "''", - '' - ], $value + if ($value === 'b:0;') { + return false; + } + $is_serialized = $value[1] === ':' && str_ends_with($value, '}'); + + $value = str_replace(["'"], ["''"], $value); + $value = "'$value'"; + + if ($is_serialized) { + $value = sprintf( + "replace(%s, char(1), char(0))", + str_replace(chr(0), chr(1), $value), ); - return "'$value'"; } return $value; },