diff --git a/CHANGELOG-WIP.md b/CHANGELOG-WIP.md index 8d907e8b0bb..98c35beae9f 100644 --- a/CHANGELOG-WIP.md +++ b/CHANGELOG-WIP.md @@ -12,6 +12,8 @@ - Element conditions within field layout designers’ component settings now only list custom fields present in the current field layout. ([#14787](https://github.com/craftcms/cms/issues/14787)) - Improved the behavior of the URI input within Edit Route modals. ([#14884](https://github.com/craftcms/cms/issues/14884)) - Added the `asyncCsrfInputs` config setting. ([#14625](https://github.com/craftcms/cms/pull/14625)) +- Added the `backupCommandFormat` config setting. ([#14897](https://github.com/craftcms/cms/pull/14897)) +- The `backupCommand` config setting can now be set to a closure, which will be passed a `mikehaertl\shellcommand\Command` object. ([#14897](https://github.com/craftcms/cms/pull/14897)) - Added the `safeMode` config setting. ([#14734](https://github.com/craftcms/cms/pull/14734)) - `resave` commands now support an `--if-invalid` option. ([#14731](https://github.com/craftcms/cms/issues/14731)) - Improved the styling of conditional tabs and UI elements within field layout designers. diff --git a/src/config/GeneralConfig.php b/src/config/GeneralConfig.php index d6a8adca108..5cca92483d5 100644 --- a/src/config/GeneralConfig.php +++ b/src/config/GeneralConfig.php @@ -7,6 +7,7 @@ namespace craft\config; +use Closure; use Craft; use craft\helpers\ConfigHelper; use craft\helpers\DateTimeHelper; @@ -420,7 +421,7 @@ class GeneralConfig extends BaseConfig public bool $backupOnUpdate = true; /** - * @var string|null|false The shell command that Craft should execute to create a database backup. + * @var string|null|false|Closure The shell command that Craft should execute to create a database backup. * * When set to `null` (default), Craft will run `mysqldump` or `pg_dump`, provided that those libraries are in the `$PATH` variable * for the system user running the web server. @@ -448,7 +449,30 @@ class GeneralConfig extends BaseConfig * * @group Environment */ - public string|null|false $backupCommand = null; + public string|null|false|Closure $backupCommand = null; + + /** + * @var string|null The output format to pass to `pg_dump` when backing up the database. + * + * This setting has no effect with MySQL databases. + * + * Valid options are `custom`, `directory`, `tar`, or `plain`. + * When set to `null` (default), `pg_restore` will default to `plain` + * @see https://www.postgresql.org/docs/current/app-pgdump.html + * + * ::: code + * ```php Static Config + * ->backupCommandFormat('custom') + * ``` + * ```shell Environment Override + * CRAFT_BACKUP_COMMAND_FORMAT=custom + * ``` + * ::: + * + * @group Environment + * @since 4.9.0 + */ + public ?string $backupCommandFormat = null; /** * @var string|null The base URL Craft should use when generating control panel URLs. @@ -2475,7 +2499,7 @@ class GeneralConfig extends BaseConfig public string $resourceBaseUrl = '@web/cpresources'; /** - * @var string|null|false The shell command Craft should execute to restore a database backup. + * @var string|null|false|Closure The shell command Craft should execute to restore a database backup. * * By default Craft will run `mysql` or `psql`, provided those libraries are in the `$PATH` variable for the user the web server is running as. * @@ -2501,7 +2525,7 @@ class GeneralConfig extends BaseConfig * * @group Environment */ - public string|null|false $restoreCommand = null; + public string|null|false|Closure $restoreCommand = null; /** * @var bool Whether asset URLs should be revved so browsers don’t load cached versions when they’re modified. @@ -3632,17 +3656,38 @@ public function backupOnUpdate(bool $value = true): self * ``` * * @group Environment - * @param string|null|false $value + * @param string|null|false|Closure $value * @return self * @see $backupCommand * @since 4.2.0 */ - public function backupCommand(string|null|false $value): self + public function backupCommand(string|null|false|Closure $value): self { $this->backupCommand = $value; return $this; } + /** + * The output format to pass to `pg_dump` when backing up the database. + * + * This setting has no effect with MySQL databases. + * + * Valid options are `custom`, `directory`, `tar`, or `plain`. + * When set to `null` (default), `pg_restore` will default to `plain` + * @see https://www.postgresql.org/docs/current/app-pgdump.html + * + * @group Environment + * @param string $value + * @return self + * @see $backupCommandFormat + * @since 4.9.0 + */ + public function backupCommandFormat(string $value): self + { + $this->backupCommandFormat = $value; + return $this; + } + /** * The base URL Craft should use when generating control panel URLs. * @@ -5996,12 +6041,12 @@ public function resourceBaseUrl(string $value): self * ``` * * @group Environment - * @param string|null|false $value + * @param string|null|false|Closure $value * @return self * @see $restoreCommand * @since 4.2.0 */ - public function restoreCommand(string|null|false $value): self + public function restoreCommand(string|null|false|Closure $value): self { $this->restoreCommand = $value; return $this; diff --git a/src/console/controllers/DbController.php b/src/console/controllers/DbController.php index 98d4e525fc0..259e9ca6c67 100644 --- a/src/console/controllers/DbController.php +++ b/src/console/controllers/DbController.php @@ -214,7 +214,7 @@ public function actionBackup(?string $path = null): int $db->backupTo($path); if ($this->zip) { $zipPath = FileHelper::zip($path); - unlink($path); + FileHelper::unlink($path); $path = $zipPath; } } catch (Throwable $e) { diff --git a/src/db/Connection.php b/src/db/Connection.php index 08d1b91c827..40e4405102d 100644 --- a/src/db/Connection.php +++ b/src/db/Connection.php @@ -228,10 +228,11 @@ public function getBackupFilePath(): string $version = Craft::$app->getInfo()->version ?? Craft::$app->getVersion(); $filename = ($systemName ? "$systemName--" : '') . gmdate('Y-m-d-His') . "--v$version"; $backupPath = Craft::$app->getPath()->getDbBackupPath(); - $path = $backupPath . DIRECTORY_SEPARATOR . $filename . '.sql'; + $path = $backupPath . DIRECTORY_SEPARATOR . $filename . $this->_getDumpExtension(); + $i = 0; while (file_exists($path)) { - $path = $backupPath . DIRECTORY_SEPARATOR . $filename . '--' . ++$i . '.sql'; + $path = $backupPath . DIRECTORY_SEPARATOR . $filename . '--' . ++$i . $this->_getDumpExtension(); } return $path; } @@ -290,12 +291,10 @@ public function backupTo(string $filePath): void // Determine the command that should be executed $backupCommand = Craft::$app->getConfig()->getGeneral()->backupCommand; - if ($backupCommand === null) { - $backupCommand = $this->getSchema()->getDefaultBackupCommand($event->ignoreTables); - } - if ($backupCommand === false) { throw new Exception('Database not backed up because the backup command is false.'); + } elseif ($backupCommand === null || $backupCommand instanceof \Closure) { + $backupCommand = $this->getSchema()->getDefaultBackupCommand($event->ignoreTables); } // Create the shell command @@ -316,10 +315,10 @@ public function backupTo(string $filePath): void if ($generalConfig->maxBackups) { $backupPath = Craft::$app->getPath()->getDbBackupPath(); - // Grab all .sql files in the backup folder. + // Grab all .sql/.dump files in the backup folder. $files = array_merge( - glob($backupPath . DIRECTORY_SEPARATOR . '*.sql'), - glob($backupPath . DIRECTORY_SEPARATOR . '*.sql.zip'), + glob($backupPath . DIRECTORY_SEPARATOR . "*.{$this->_getDumpExtension()}"), + glob($backupPath . DIRECTORY_SEPARATOR . "*.{$this->_getDumpExtension()}.zip"), ); // Sort them by file modified time descending (newest first). @@ -356,12 +355,10 @@ public function restore(string $filePath): void // Determine the command that should be executed $restoreCommand = Craft::$app->getConfig()->getGeneral()->restoreCommand; - if ($restoreCommand === null) { - $restoreCommand = $this->getSchema()->getDefaultRestoreCommand(); - } - if ($restoreCommand === false) { throw new Exception('Database not restored because the restore command is false.'); + } elseif ($restoreCommand === null || $restoreCommand instanceof \Closure) { + $restoreCommand = $this->getSchema()->getDefaultRestoreCommand(); } // Create the shell command @@ -489,6 +486,11 @@ public function trigger($name, Event $event = null) parent::trigger($name, $event); } + private function _getDumpExtension(): string + { + return $this->getIsPgsql() && $this->getSchema()->usePgRestore() ? '.dump' : '.sql'; + } + /** * Generates a FK, index, or PK name. * diff --git a/src/db/mysql/Schema.php b/src/db/mysql/Schema.php index d0b902ef7d4..1a9b6872e90 100644 --- a/src/db/mysql/Schema.php +++ b/src/db/mysql/Schema.php @@ -182,76 +182,54 @@ public function createColumnSchemaBuilder($type, $length = null): ColumnSchemaBu */ public function getDefaultBackupCommand(?array $ignoreTables = null): string { - $useSingleTransaction = true; - $serverVersion = App::normalizeVersion($this->getServerVersion()); + $baseCommand = (new ShellCommand('mysqldump')) + ->addArg('--defaults-file=', $this->_createDumpConfigFile()) + ->addArg('--add-drop-table') + ->addArg('--comments') + ->addArg('--create-options') + ->addArg('--dump-date') + ->addArg('--no-autocommit') + ->addArg('--routines') + ->addArg('--default-character-set=', Craft::$app->getConfig()->getDb()->charset) + ->addArg('--set-charset') + ->addArg('--triggers') + ->addArg('--no-tablespaces'); + + $serverVersion = App::normalizeVersion(Craft::$app->getDb()->getServerVersion()); $isMySQL8 = version_compare($serverVersion, '8', '>='); + $ignoreTables = $ignoreTables ?? Craft::$app->getDb()->getIgnoredBackupTables(); + $commandFromConfig = Craft::$app->getConfig()->getGeneral()->backupCommand; // https://bugs.mysql.com/bug.php?id=109685 - if ($isMySQL8 && version_compare($serverVersion, '8.0.32', '>=')) { - $useSingleTransaction = false; - } - - $defaultArgs = - ' --defaults-file="' . $this->_createDumpConfigFile() . '"' . - ' --add-drop-table' . - ' --comments' . - ' --create-options' . - ' --dump-date' . - ' --no-autocommit' . - ' --routines' . - ' --default-character-set=' . Craft::$app->getConfig()->getDb()->getCharset() . - ' --set-charset' . - ' --triggers' . - ' --no-tablespaces'; + $useSingleTransaction = $isMySQL8 && version_compare($serverVersion, '8.0.32', '>='); if ($useSingleTransaction) { - $defaultArgs .= ' --single-transaction'; + $baseCommand->addArg('--single-transaction'); } - // Find out if the db/dump client supports column-statistics - $shellCommand = new ShellCommand(); - - if (App::isWindows()) { - $shellCommand->setCommand('mysqldump --help | findstr "column-statistics"'); - } else { - $shellCommand->setCommand('mysqldump --help | grep "column-statistics"'); + if ($this->supportsColumnStatistics()) { + $baseCommand->addArg('--column-statistics=', '0'); } - // If we don't have proc_open, maybe we've got exec - if (!function_exists('proc_open') && function_exists('exec')) { - $shellCommand->useExec = true; - } + $schemaDump = (clone $baseCommand) + ->addArg('--no-data') + ->addArg('--result-file=', '{file}') + ->addArg('{database}'); - $success = $shellCommand->execute(); - - // if there was output, then column-statistics is supported and we should disable it - if ($success && $shellCommand->getOutput()) { - $defaultArgs .= ' --column-statistics=0'; - } + $dataDump = (clone $baseCommand) + ->addArg('--no-create-info'); - if ($ignoreTables === null) { - $ignoreTables = $this->db->getIgnoredBackupTables(); - } - $ignoreTableArgs = []; foreach ($ignoreTables as $table) { $table = $this->getRawTableName($table); - $ignoreTableArgs[] = "--ignore-table={database}.$table"; + $dataDump->addArg('--ignore-table', "{schema}.$table"); } - $schemaDump = 'mysqldump' . - $defaultArgs . - ' --no-data' . - ' --result-file="{file}"' . - ' {database}'; - - $dataDump = 'mysqldump' . - $defaultArgs . - ' --no-create-info' . - ' ' . implode(' ', $ignoreTableArgs) . - ' {database}' . - ' >> "{file}"'; + if ($commandFromConfig instanceof \Closure) { + $schemaDump = $commandFromConfig($schemaDump); + $dataDump = $commandFromConfig($dataDump); + } - return $schemaDump . ' && ' . $dataDump; + return "{$schemaDump->getExecCommand()} && {$dataDump->getExecCommand()} >> {file}"; } /** @@ -262,10 +240,16 @@ public function getDefaultBackupCommand(?array $ignoreTables = null): string */ public function getDefaultRestoreCommand(): string { - return 'mysql' . - ' --defaults-file="' . $this->_createDumpConfigFile() . '"' . - ' {database}' . - ' < "{file}"'; + $commandFromConfig = Craft::$app->getConfig()->getGeneral()->restoreCommand; + $command = (new ShellCommand('mysql')) + ->addArg('--defaults-file=', $this->_createDumpConfigFile()) + ->addArg('{database}'); + + if ($commandFromConfig instanceof \Closure) { + $command = $commandFromConfig($command); + } + + return $command->getExecCommand() . ' < "{file}"'; } /** @@ -422,6 +406,28 @@ protected function findConstraints($table): void } } + protected function supportsColumnStatistics(): bool + { + // Find out if the db/dump client supports column-statistics + $shellCommand = new ShellCommand(); + + if (App::isWindows()) { + $shellCommand->setCommand('mysqldump --help | findstr "column-statistics"'); + } else { + $shellCommand->setCommand('mysqldump --help | grep "column-statistics"'); + } + + // If we don't have proc_open, maybe we've got exec + if (!function_exists('proc_open') && function_exists('exec')) { + $shellCommand->useExec = true; + } + + $success = $shellCommand->execute(); + + // if there was output, then column-statistics is supported + return $success && $shellCommand->getOutput(); + } + /** * Creates a temporary my.cnf file based on the DB config settings. * diff --git a/src/db/pgsql/Schema.php b/src/db/pgsql/Schema.php index 69bac6eafe6..d251b958e29 100644 --- a/src/db/pgsql/Schema.php +++ b/src/db/pgsql/Schema.php @@ -13,6 +13,7 @@ use craft\db\ExpressionInterface; use craft\db\TableSchema; use craft\helpers\App; +use mikehaertl\shellcommand\Command as ShellCommand; use yii\db\Exception; /** @@ -133,29 +134,37 @@ public function getLastInsertID($sequenceName = ''): string */ public function getDefaultBackupCommand(?array $ignoreTables = null): string { - if ($ignoreTables === null) { - $ignoreTables = $this->db->getIgnoredBackupTables(); - } - $ignoredTableArgs = []; + $command = (new ShellCommand('pg_dump')) + ->addArg('--dbname=', '{database}') + ->addArg('--host=', '{server}') + ->addArg('--port=', '{port}') + ->addArg('--username=', '{user}') + ->addArg('--if-exists') + ->addArg('--clean') + ->addArg('--no-owner') + ->addArg('--no-privileges') + ->addArg('--no-acl') + ->addArg('--file=', '{file}') + ->addArg('--schema=', '{schema}'); + + $ignoreTables = $ignoreTables ?? Craft::$app->getDb()->getIgnoredBackupTables(); + $format = Craft::$app->getConfig()->getGeneral()->backupCommandFormat; + $commandFromConfig = Craft::$app->getConfig()->getGeneral()->backupCommand; + foreach ($ignoreTables as $table) { $table = $this->getRawTableName($table); - $ignoredTableArgs[] = "--exclude-table-data '{schema}.$table'"; + $command->addArg('--exclude-table-data', "{schema}.$table"); } - return $this->_pgpasswordCommand() . - 'pg_dump' . - ' --dbname={database}' . - ' --host={server}' . - ' --port={port}' . - ' --username={user}' . - ' --if-exists' . - ' --clean' . - ' --no-owner' . - ' --no-privileges' . - ' --no-acl' . - ' --file="{file}"' . - ' --schema={schema}' . - ' ' . implode(' ', $ignoredTableArgs); + if ($format) { + $command->addArg('--format=', $format); + } + + if ($commandFromConfig instanceof \Closure) { + $command = $commandFromConfig($command); + } + + return $command->getExecCommand(); } /** @@ -165,14 +174,22 @@ public function getDefaultBackupCommand(?array $ignoreTables = null): string */ public function getDefaultRestoreCommand(): string { - return $this->_pgpasswordCommand() . - 'psql' . - ' --dbname={database}' . - ' --host={server}' . - ' --port={port}' . - ' --username={user}' . - ' --no-password' . - ' < "{file}"'; + $command = (new ShellCommand($this->usePgRestore() ? 'pg_restore' : 'psql')) + ->addArg('--dbname=', '{database}') + ->addArg('--host=', '{server}') + ->addArg('--port=', '{port}') + ->addArg('--username=', '{user}') + ->addArg('--no-password'); + + $commandFromConfig = Craft::$app->getConfig()->getGeneral()->restoreCommand; + + if ($commandFromConfig instanceof \Closure) { + $command = $commandFromConfig($command); + } + + return $this->_pgpasswordCommand() + . $command->getExecCommand() + . '< "{file}"'; } /** @@ -233,6 +250,20 @@ public function loadTableSchema($name): ?TableSchema return null; } + /** + * Whether `pg_restore` should be used by default for the backup command. + * + * @return bool + * @since 4.9.0 + */ + public function usePgRestore(): bool + { + return in_array(Craft::$app->getConfig()->getGeneral()->backupCommandFormat, [ + 'custom', + 'directory', + ], true); + } + /** * Collects extra foreign key information details for the given table. *