Skip to content

Commit

Permalink
New code for backups with lots of new features
Browse files Browse the repository at this point in the history
  • Loading branch information
@rvalitov
rvalitov authored and Till Brehm committed May 28, 2020
1 parent 2fa5ed2 commit 8bc6a8c
Show file tree
Hide file tree
Showing 75 changed files with 3,340 additions and 417 deletions.
12 changes: 12 additions & 0 deletions install/sql/incremental/upd_dev_collection.sql
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,15 @@
-- add new proxy_protocol column
ALTER TABLE `web_domain`
ADD COLUMN `proxy_protocol` ENUM('n','y') NOT NULL DEFAULT 'n' AFTER `log_retention`;

-- backup format
ALTER TABLE `web_domain` ADD `backup_format_web` VARCHAR( 255 ) NOT NULL default 'default' AFTER `backup_copies`;
ALTER TABLE `web_domain` ADD `backup_format_db` VARCHAR( 255 ) NOT NULL default 'gzip' AFTER `backup_format_web`;
-- end of backup format

-- backup encryption
ALTER TABLE `web_domain` ADD `backup_encrypt` enum('n','y') NOT NULL DEFAULT 'n' AFTER `backup_format_db`;
ALTER TABLE `web_domain` ADD `backup_password` VARCHAR( 255 ) NOT NULL DEFAULT '' AFTER `backup_encrypt`;
ALTER TABLE `web_backup` ADD `backup_format` VARCHAR( 64 ) NOT NULL DEFAULT '' AFTER `backup_mode`;
ALTER TABLE `web_backup` ADD `backup_password` VARCHAR( 255 ) NOT NULL DEFAULT '' AFTER `filesize`;
-- end of backup encryption
6 changes: 6 additions & 0 deletions install/sql/ispconfig3.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1922,9 +1922,11 @@ CREATE TABLE `web_backup` (
`parent_domain_id` int(10) unsigned NOT NULL DEFAULT '0',
`backup_type` enum('web','mysql','mongodb') NOT NULL DEFAULT 'web',
`backup_mode` varchar(64) NOT NULL DEFAULT '',
`backup_format` varchar(64) NOT NULL DEFAULT '',
`tstamp` int(10) unsigned NOT NULL DEFAULT '0',
`filename` varchar(255) NOT NULL DEFAULT '',
`filesize` VARCHAR(20) NOT NULL DEFAULT '',
`backup_password` VARCHAR(255) NOT NULL DEFAULT '',
PRIMARY KEY (`backup_id`)
) DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ;

Expand Down Expand Up @@ -2055,6 +2057,10 @@ CREATE TABLE `web_domain` (
`custom_php_ini` mediumtext,
`backup_interval` VARCHAR( 255 ) NOT NULL DEFAULT 'none',
`backup_copies` INT NOT NULL DEFAULT '1',
`backup_format_web` VARCHAR( 255 ) NOT NULL default 'default',
`backup_format_db` VARCHAR( 255 ) NOT NULL default 'gzip',
`backup_encrypt` enum('n','y') NOT NULL DEFAULT 'n',
`backup_password` VARCHAR( 255 ) NOT NULL DEFAULT '',
`backup_excludes` mediumtext,
`active` enum('n','y') NOT NULL default 'y',
`traffic_quota_lock` enum('n','y') NOT NULL default 'n',
Expand Down
2 changes: 2 additions & 0 deletions interface/lib/classes/aps_guicontroller.inc.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,8 @@ public function createDatabaseForPackageInstance(&$settings, $websrv) {
"remote_access" => $mysql_db_remote_access,
"remote_ips" => $mysql_db_remote_ips,
"backup_copies" => $websrv['backup_copies'],
"backup_format_web" => $websrv['backup_format_web'],
"backup_format_db" => $websrv['backup_format_db'],
"active" => 'y',
"backup_interval" => $websrv['backup_interval']
);
Expand Down
57 changes: 56 additions & 1 deletion interface/lib/classes/plugin_backuplist.inc.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,34 @@ class plugin_backuplist extends plugin_base {
var $formdef;
var $options;

/**
* Process request to make a backup. This request is triggered manually by the user in the ISPConfig interface.
* @param string $message
* @param string $error
* @param string[] $wb language text
* @author Ramil Valitov <[email protected]>
* @uses backup_plugin::make_backup_callback() this method is called later in the plugin to run the backup
*/
protected function makeBackup(&$message, &$error, $wb)
{
global $app;

$mode = $_GET['make_backup'];
$action_type = ($mode == 'web') ? 'backup_web_files' : 'backup_database';
$domain_id = intval($this->form->id);

$sql = "SELECT count(action_id) as number FROM sys_remoteaction WHERE action_state = 'pending' AND action_type = ? AND action_param = ?";
$tmp = $app->db->queryOneRecord($sql, $action_type, $domain_id);
if ($tmp['number'] == 0) {
$server_id = $this->form->dataRecord['server_id'];
$message .= $wb['backup_info_txt'];
$sql = "INSERT INTO sys_remoteaction (server_id, tstamp, action_type, action_param, action_state, response) VALUES (?, UNIX_TIMESTAMP(), ?, ?, 'pending', '')";
$app->db->query($sql, $server_id, $action_type, $domain_id);
} else {
$error .= $wb['backup_pending_txt'];
}
}

function onShow() {

global $app;
Expand All @@ -52,6 +80,10 @@ function onShow() {
$message = '';
$error = '';

if (isset($_GET['make_backup'])) {
$this->makeBackup($message, $error, $wb);
}

if(isset($_GET['backup_action'])) {
$backup_id = $app->functions->intval($_GET['backup_id']);

Expand Down Expand Up @@ -137,7 +169,30 @@ function onShow() {
$rec["bgcolor"] = $bgcolor;

$rec['date'] = date($app->lng('conf_format_datetime'), $rec['tstamp']);
$rec['backup_type'] = $wb[('backup_type_'.$rec['backup_type'])];
$backup_format = $rec['backup_format'];
if (empty($backup_format)) {
//We have a backup from old version of ISPConfig
switch ($rec['backup_type']) {
case 'mysql':
$backup_format = 'gzip';
break;
case 'web':
$backup_format = ($rec['backup_mode'] == 'userzip') ? 'zip' : 'tar_gzip';
break;
default:
$app->log('Unsupported backup type "' . $rec['backup_type'] . '" for backup id ' . $rec['backup_id'], LOGLEVEL_ERROR);
break;
}
}
$rec['backup_type'] = $wb[('backup_type_' . $rec['backup_type'])];
$backup_format = (!empty($backup_format)) ? $wb[('backup_format_' . $backup_format . '_txt')] : $wb["backup_format_unknown_txt"];
if (empty($backup_format))
$backup_format = $wb["backup_format_unknown_txt"];

$rec['backup_format'] = $backup_format;
$rec['backup_encrypted'] = empty($rec['backup_password']) ? $wb["no_txt"] : $wb["yes_txt"];
$backup_manual_prefix = 'manual-';
$rec['backup_job'] = (substr($rec['filename'], 0, strlen($backup_manual_prefix)) == $backup_manual_prefix) ? $wb["backup_job_manual_txt"] : $wb["backup_job_auto_txt"];

$rec['download_available'] = true;
if($rec['server_id'] != $web['server_id']) $rec['download_available'] = false;
Expand Down
8 changes: 6 additions & 2 deletions interface/lib/classes/remote.d/sites.inc.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,10 +132,12 @@ public function sites_database_add($session_id, $client_id, $params)
$app->sites_database_plugin->processDatabaseInsert($this);

// set correct values for backup_interval and backup_copies
if(isset($params['backup_interval']) || isset($params['backup_copies'])){
if(isset($params['backup_interval']) || isset($params['backup_copies']) || isset($params['backup_format_web']) || isset($params['backup_format_db'])){
$sql_set = array();
if(isset($params['backup_interval'])) $sql_set[] = "backup_interval = '".$app->db->quote($params['backup_interval'])."'";
if(isset($params['backup_copies'])) $sql_set[] = "backup_copies = ".$app->functions->intval($params['backup_copies']);
if(isset($params['backup_format_web'])) $sql_set[] = "backup_format_web = ".$app->functions->intval($params['backup_format_web']);
if(isset($params['backup_format_db'])) $sql_set[] = "backup_format_db = ".$app->functions->intval($params['backup_format_db']);
$this->updateQueryExecute("UPDATE web_database SET ".implode(', ', $sql_set)." WHERE database_id = ".$retval, $retval, $params);
}

Expand Down Expand Up @@ -165,10 +167,12 @@ public function sites_database_update($session_id, $client_id, $primary_id, $par
$retval = $this->updateQueryExecute($sql, $primary_id, $params);

// set correct values for backup_interval and backup_copies
if(isset($params['backup_interval']) || isset($params['backup_copies'])){
if(isset($params['backup_interval']) || isset($params['backup_copies']) || isset($params['backup_format_web']) || isset($params['backup_format_db'])){
$sql_set = array();
if(isset($params['backup_interval'])) $sql_set[] = "backup_interval = '".$app->db->quote($params['backup_interval'])."'";
if(isset($params['backup_copies'])) $sql_set[] = "backup_copies = ".$app->functions->intval($params['backup_copies']);
if(isset($params['backup_format_web'])) $sql_set[] = "backup_format_web = ".$app->functions->intval($params['backup_format_web']);
if(isset($params['backup_format_db'])) $sql_set[] = "backup_format_db = ".$app->functions->intval($params['backup_format_db']);
$this->updateQueryExecute("UPDATE web_database SET ".implode(', ', $sql_set)." WHERE database_id = ".$primary_id, $primary_id, $params);
}

Expand Down
6 changes: 4 additions & 2 deletions interface/lib/classes/sites_database_plugin.inc.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,12 @@ public function processDatabaseUpdate($form_page) {
//* The Database user shall be owned by the same group then the website
$sys_groupid = $app->functions->intval($web['sys_groupid']);
$backup_interval = $web['backup_interval'];
$backup_format_web = $web['backup_format_web'];
$backup_format_db = $web['backup_format_db'];
$backup_copies = $app->functions->intval($web['backup_copies']);

$sql = "UPDATE web_database SET sys_groupid = ?, backup_interval = ?, backup_copies = ? WHERE database_id = ?";
$app->db->query($sql, $sys_groupid, $backup_interval, $backup_copies, $form_page->id);
$sql = "UPDATE web_database SET sys_groupid = ?, backup_interval = ?, backup_copies = ?, backup_format_web = ?, backup_format_db = ? WHERE database_id = ?";
$app->db->query($sql, $sys_groupid, $backup_interval, $backup_copies, $backup_format_web, $backup_format_db, $form_page->id);
}
}

Expand Down
12 changes: 12 additions & 0 deletions interface/lib/classes/system.inc.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,5 +97,17 @@ public function system_safe($cmd) {
call_user_func_array(array($this, 'exec_safe'), func_get_args());
return implode("\n", $this->_last_exec_out);
}

//* Check if a application is installed
public function is_installed($appname) {
$this->exec_safe('which ? 2> /dev/null', $appname);
$out = $this->last_exec_out();
$returncode = $this->last_exec_retcode();
if(isset($out[0]) && stristr($out[0], $appname) && $returncode == 0) {
return true;
} else {
return false;
}
}

} //* End Class
8 changes: 6 additions & 2 deletions interface/lib/plugins/sites_web_vhost_domain_plugin.inc.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,18 +249,22 @@ function sites_web_vhost_domain_edit($event_name, $page_form) {
}

//* Change database backup options when web backup options have been changed
if(isset($page_form->dataRecord['backup_interval']) && ($page_form->dataRecord['backup_interval'] != $page_form->oldDataRecord['backup_interval'] || $page_form->dataRecord['backup_copies'] != $page_form->oldDataRecord['backup_copies'])) {
if(isset($page_form->dataRecord['backup_interval']) && ($page_form->dataRecord['backup_interval'] != $page_form->oldDataRecord['backup_interval'] || $page_form->dataRecord['backup_copies'] != $page_form->oldDataRecord['backup_copies'] || $page_form->dataRecord['backup_format_web'] != $page_form->oldDataRecord['backup_format_web'] || $page_form->dataRecord['backup_format_db'] != $page_form->oldDataRecord['backup_format_db'])) {
//* Update all databases
$backup_interval = $page_form->dataRecord['backup_interval'];
$backup_copies = $app->functions->intval($page_form->dataRecord['backup_copies']);
$backup_format_web = $page_form->dataRecord['backup_format_web'];
$backup_format_db = $page_form->dataRecord['backup_format_db'];
$records = $app->db->queryAllRecords("SELECT database_id FROM web_database WHERE parent_domain_id = ".$page_form->id);
foreach($records as $rec) {
$app->db->datalogUpdate('web_database', array("backup_interval" => $backup_interval, "backup_copies" => $backup_copies), 'database_id', $rec['database_id']);
$app->db->datalogUpdate('web_database', array("backup_interval" => $backup_interval, "backup_copies" => $backup_copies, "backup_format_web" => $backup_format_web, "backup_format_db" => $backup_format_db), 'database_id', $rec['database_id']);
}
unset($records);
unset($rec);
unset($backup_copies);
unset($backup_interval);
unset($backup_format_web);
unset($backup_format_db);
}

//* Change vhost subdomain and alias ip/ipv6 if domain ip/ipv6 has changed
Expand Down
76 changes: 75 additions & 1 deletion interface/web/sites/form/web_vhost_domain.tform.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@
if($client['limit_backup'] != 'y') $backup_available = false;
}

$app->uses('getconf');
$app->uses('getconf,system');
$web_config = $app->getconf->get_global_config('sites');

$form["tabs"]['domain'] = array (
Expand Down Expand Up @@ -649,6 +649,28 @@

//* Backup
if ($backup_available) {
$missing_utils = array();
$compressors_list = array(
'gzip',
'gunzip',
'zip',
'unzip',
'pigz',
'tar',
'bzip2',
'bunzip2',
'xz',
'unxz',
'7z',
'rar',
);
foreach ($compressors_list as $compressor) {
if (!$app->system->is_installed($compressor)) {
array_push($missing_utils, $compressor);
}
}
$app->tpl->setVar("missing_utils", implode(", ",$missing_utils), true);

$form["tabs"]['backup'] = array (
'title' => "Backup",
'width' => 100,
Expand Down Expand Up @@ -682,6 +704,58 @@
'width' => '30',
'maxlength' => '255'
),
'backup_format_web' => array (
'datatype' => 'VARCHAR',
'formtype' => 'SELECT',
'default' => '',
'value' => array(
'default' => 'backup_format_default_txt',
'zip' => 'backup_format_zip_txt',
'zip_bzip2' => 'backup_format_zip_bzip2_txt',
'tar_gzip' => 'backup_format_tar_gzip_txt',
'tar_bzip2' => 'backup_format_tar_bzip2_txt',
'tar_xz' => 'backup_format_tar_xz_txt',
'tar_7z_lzma2' => 'backup_format_tar_7z_lzma2_txt',
'tar_7z_lzma' => 'backup_format_tar_7z_lzma_txt',
'tar_7z_ppmd' => 'backup_format_tar_7z_ppmd_txt',
'tar_7z_bzip2' => 'backup_format_tar_7z_bzip2_txt',
'rar' => 'backup_format_rar_txt',
)
),
'backup_format_db' => array (
'datatype' => 'VARCHAR',
'formtype' => 'SELECT',
'default' => '',
'value' => array(
'zip' => 'backup_format_zip_txt',
'zip_bzip2' => 'backup_format_zip_bzip2_txt',
'gzip' => 'backup_format_gzip_txt',
'bzip2' => 'backup_format_bzip2_txt',
'xz' => 'backup_format_xz_txt',
'7z_lzma2' => 'backup_format_7z_lzma2_txt',
'7z_lzma' => 'backup_format_7z_lzma_txt',
'7z_ppmd' => 'backup_format_7z_ppmd_txt',
'7z_bzip2' => 'backup_format_7z_bzip2_txt',
'rar' => 'backup_format_rar_txt',
)
),
'backup_encrypt' => array (
'datatype' => 'VARCHAR',
'formtype' => 'CHECKBOX',
'default' => 'n',
'value' => array (
0 => 'n',
1 => 'y'
)
),
'backup_password' => array (
'datatype' => 'VARCHAR',
'formtype' => 'TEXT',
'default' => '',
'value' => '',
'width' => '30',
'maxlength' => '255'
),
//#################################
// END Datatable fields
//#################################
Expand Down
32 changes: 32 additions & 0 deletions interface/web/sites/lib/lang/ar_web_backup_list.lng
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,36 @@ $wb['backup_type_mysql'] = 'MySQL Database';
$wb['backup_type_web'] = 'Website files';
$wb['filesize_txt'] = 'Filesize';
$wb['backup_type_mongodb'] = 'MongoDB Database';
$wb['backup_pending_txt'] = 'There is already a pending backup job.';
$wb['error_txt'] = 'Error';
$wb['backup_info_txt'] = 'A backup process started. This action can take several minutes to complete.';
$wb["backup_format_txt"] = 'Backup format';
$wb["backup_format_unknown_txt"] = 'Unknown';
$wb["backup_job_txt"] = 'Scheduler';
$wb["backup_job_manual_txt"] = 'Manual';
$wb["backup_job_auto_txt"] = 'Auto';
$wb["manual_backup_title_txt"] = 'Manual backup';
$wb["make_backup_web_txt"] = 'Make backup of web files';
$wb["make_backup_database_txt"] = 'Make backup of databases';
$wb["make_backup_confirm_txt"] = 'You are about to start a manual backup process. Manual backups count towards the total number of allowed backup copies: therefore if the limit will be exceeded, then oldest backups may be deleted automatically. Proceed?';
$wb["yes_txt"] = 'Yes';
$wb["no_txt"] = 'No';
$wb["backup_is_encrypted_txt"] = "Encrypted";
$wb["backup_format_zip_txt"] = 'zip (deflate)';
$wb["backup_format_gzip_txt"] = 'gzip';
$wb["backup_format_bzip2_txt"] = 'bzip2';
$wb["backup_format_xz_txt"] = 'xz';
$wb["backup_format_tar_gzip_txt"] = 'tar (gzip)';
$wb["backup_format_tar_bzip2_txt"] = 'tar (bzip2)';
$wb["backup_format_tar_xz_txt"] = 'tar (xz)';
$wb["backup_format_zip_bzip2_txt"] = 'zip (bzip2)';
$wb["backup_format_7z_lzma_txt"] = '7z (LZMA)';
$wb["backup_format_7z_lzma2_txt"] = '7z (LZMA2)';
$wb["backup_format_7z_ppmd_txt"] = '7z (PPMd)';
$wb["backup_format_7z_bzip2_txt"] = '7z (BZip2)';
$wb["backup_format_tar_7z_lzma_txt"] = 'tar + 7z (LZMA)';
$wb["backup_format_tar_7z_lzma2_txt"] = 'tar + 7z (LZMA2)';
$wb["backup_format_tar_7z_ppmd_txt"] = 'tar + 7z (PPMd)';
$wb["backup_format_tar_7z_bzip2_txt"] = 'tar + 7z (BZip2)';
$wb["backup_format_rar_txt"] = 'RAR';
?>
27 changes: 27 additions & 0 deletions interface/web/sites/lib/lang/ar_web_vhost_domain.lng
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,4 +156,31 @@ $wb['log_retention_txt'] = 'Logfiles retention time';
$wb['log_retention_error_regex'] = 'Retention time in days (allowed values: min. 0 - max. 9999)';
$wb['limit_web_quota_not_0_txt'] = 'Harddisk Quota cannot be set to 0.';
$wb['proxy_protocol_txt'] = 'Enable PROXY Protocol';
$wb["backup_format_web_txt"] = 'Backup format for web files';
$wb["backup_format_db_txt"] = 'Backup format for database';
$wb["backup_format_web_note_txt"] = 'In "default" mode ISPConfig has the following behaviour: if backup mode is set as web user, then "zip" format is used for web files, otherwise backup mode is set as root user and "tar (gzip)" format is used; database is compressed in "gzip" format. Only "tar" based formats and "rar" preserve file ownership and permissions of web files and guarantee correct restore.';
$wb["backup_missing_utils_txt"] = 'Note: some utils are missing in the system that may prevent you from using some compression formats. Please, install the following utils to avoid possible backup problems: ';
$wb["backup_compression_options_txt"] = 'Compression options';
$wb["backup_encryption_note_txt"] = "Encryption is available only for the following backup formats: \"7z\", \"RAR\", \"zip\" (not secure). If any other format is used then encryption can't be applied and encryption settings are ignored. You can safely change the password at anytime: the \"Restore\" button will still work for old backups encrypted with a previous password. You don't need to type a password to restore backups using the \"Restore\" button. Password is required when you extract the archives manually. Please, don't forget your password, because it's impossible to restore it.";
$wb["backup_encryption_options_txt"] = 'Encryption options';
$wb["backup_enable_encryption_txt"] = 'Enable encryption';
$wb["backup_password_txt"] = 'Password';
$wb["backup_format_default_txt"] = 'Default: zip (deflate) or tar (gzip)';
$wb["backup_format_zip_txt"] = 'zip (deflate)';
$wb["backup_format_gzip_txt"] = 'gzip';
$wb["backup_format_bzip2_txt"] = 'bzip2';
$wb["backup_format_xz_txt"] = 'xz';
$wb["backup_format_zip_bzip2_txt"] = 'zip (bzip2)';
$wb["backup_format_7z_lzma_txt"] = '7z (LZMA)';
$wb["backup_format_7z_lzma2_txt"] = '7z (LZMA2)';
$wb["backup_format_7z_ppmd_txt"] = '7z (PPMd)';
$wb["backup_format_7z_bzip2_txt"] = '7z (BZip2)';
$wb["backup_format_tar_gzip_txt"] = 'tar (gzip)';
$wb["backup_format_tar_bzip2_txt"] = 'tar (bzip2)';
$wb["backup_format_tar_xz_txt"] = 'tar (xz)';
$wb["backup_format_rar_txt"] = 'RAR';
$wb["backup_format_tar_7z_lzma_txt"] = 'tar + 7z (LZMA)';
$wb["backup_format_tar_7z_lzma2_txt"] = 'tar + 7z (LZMA2)';
$wb["backup_format_tar_7z_ppmd_txt"] = 'tar + 7z (PPMd)';
$wb["backup_format_tar_7z_bzip2_txt"] = 'tar + 7z (BZip2)';
?>
Loading

0 comments on commit 8bc6a8c

Please sign in to comment.