Skip to content

Commit

Permalink
feat: added local backup option
Browse files Browse the repository at this point in the history
  • Loading branch information
lewislarsen committed Jul 6, 2024
1 parent 6ca5bf8 commit ebea39f
Show file tree
Hide file tree
Showing 15 changed files with 473 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ public function submit(): RedirectResponse|Redirector
{
$this->validate([
'label' => ['required', 'string'],
'type' => ['required', 'string', 'in:custom_s3,s3'],
'type' => ['required', 'string', 'in:custom_s3,s3,local'],
's3AccessKey' => ['nullable', 'required_if:type,custom_s3,s3'],
's3SecretKey' => ['nullable', 'required_if:type,custom_s3,s3'],
's3BucketName' => ['nullable', 'required_if:type,custom_s3,s3'],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ public function submit(): RedirectResponse|Redirector

$this->validate([
'label' => ['required', 'string'],
'type' => ['required', 'string', 'in:custom_s3,s3'],
'type' => ['required', 'string', 'in:custom_s3,s3,local'],
's3AccessKey' => ['nullable', 'required_if:type,custom_s3,s3'],
's3SecretKey' => ['nullable', 'required_if:type,custom_s3,s3'],
's3BucketName' => ['nullable', 'required_if:type,custom_s3,s3'],
Expand Down
11 changes: 11 additions & 0 deletions app/Models/BackupDestination.php
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ class BackupDestination extends Model

public const string TYPE_S3 = 's3';

public const string TYPE_LOCAL = 'local';

public const string STATUS_REACHABLE = 'reachable';

public const string STATUS_UNREACHABLE = 'unreachable';
Expand Down Expand Up @@ -57,6 +59,11 @@ public function isS3Connection(): bool
return $this->type === self::TYPE_S3 || $this->type === self::TYPE_CUSTOM_S3;
}

public function isLocalConnection(): bool
{
return $this->type === self::TYPE_LOCAL;
}

public function determineS3Region(): string
{
if ($this->type === BackupDestination::TYPE_CUSTOM_S3 && $this->custom_s3_region === null) {
Expand Down Expand Up @@ -107,6 +114,10 @@ public function type(): ?string
return 'Custom S3';
}

if ($this->type === self::TYPE_LOCAL) {
return 'Local';
}

return null;
}

Expand Down
3 changes: 2 additions & 1 deletion app/Rules/UniqueScheduledTimePerRemoteServer.php
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ class UniqueScheduledTimePerRemoteServer implements ValidationRule
public function __construct(
public int $remoteServerId,
public ?int $taskId = null,
) {}
) {
}

public function validate(string $attribute, mixed $value, Closure $fail): void
{
Expand Down
5 changes: 3 additions & 2 deletions app/Services/Backup/Backup.php
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
use App\Services\Backup\Adapters\SFTPAdapter;
use App\Services\Backup\Contracts\SFTPInterface;
use App\Services\Backup\Destinations\Contracts\BackupDestinationInterface;
use App\Services\Backup\Destinations\Local;
use App\Services\Backup\Destinations\S3;
use App\Services\Backup\Traits\BackupHelpers;
use Closure;
Expand Down Expand Up @@ -64,7 +65,8 @@ public function backupDestinationDriver(
$bucketName = $backupDestination->getAttribute('s3_bucket_name');

return (new S3($client, $bucketName))->streamFiles($sftp, $remotePath, $fileName, $storagePath);

case BackupConstants::DRIVER_LOCAL:
return (new Local($sftp, $storagePath))->streamFiles($sftp, $remotePath, $fileName, $storagePath);
default:
throw new RuntimeException("Unsupported destination driver: {$destinationDriver}");
}
Expand Down Expand Up @@ -474,7 +476,6 @@ public function createBackupDestinationInstance(BackupDestination $backupDestina
$client = $backupDestinationModel->getS3Client();

return new S3($client, $backupDestinationModel->getAttribute('s3_bucket_name'));

default:
throw new RuntimeException("Unsupported backup destination type: {$backupDestinationModel->getAttribute('type')}");
}
Expand Down
2 changes: 2 additions & 0 deletions app/Services/Backup/BackupConstants.php
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ class BackupConstants

public const string DRIVER_CUSTOM_S3 = 'custom_s3';

public const string DRIVER_LOCAL = 'local';

public const int ZIP_RETRY_MAX_ATTEMPTS = 3;

public const int ZIP_RETRY_DELAY_SECONDS = 5;
Expand Down
238 changes: 238 additions & 0 deletions app/Services/Backup/Destinations/Local.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,238 @@
<?php

declare(strict_types=1);

namespace App\Services\Backup\Destinations;

use App\Services\Backup\Contracts\SFTPInterface;
use App\Services\Backup\Destinations\Contracts\BackupDestinationInterface;
use App\Services\Backup\Traits\BackupHelpers;
use DateTime;
use Exception;
use Illuminate\Support\Facades\Log;
use RuntimeException;

class Local implements BackupDestinationInterface
{
use BackupHelpers;

public function __construct(
protected SFTPInterface $sftp,
protected string $storagePath
) {
}

/**
* @return array<string>
*/
public function listFiles(string $pattern): array
{
$files = $this->listDirectoryContents($this->storagePath);

return $this->filterAndSortFiles($files, $pattern);
}

public function deleteFile(string $filePath): void
{
$fullPath = $this->getFullPath($filePath);
if ($this->sftp->delete($fullPath)) {
Log::info('File deleted from remote local storage.', ['file_path' => $fullPath]);
} else {
Log::warning('Failed to delete file from remote local storage.', ['file_path' => $fullPath]);
}
}

/**
* @throws Exception
*/
public function streamFiles(
SFTPInterface $sftp,
string $remoteZipPath,
string $fileName,
?string $storagePath = null,
int $retries = 3,
int $delay = 5
): bool {
$fullPath = $this->getFullPath($fileName, $storagePath);

$this->logStartStreaming($remoteZipPath, $fileName, $fullPath);

return $this->retryCommand(
fn (): bool => $this->performFileStreaming($sftp, $remoteZipPath, $fullPath),
$retries,
$delay
);
}

public function getFullPath(string $fileName, ?string $storagePath = null): string
{
$basePath = $this->normalizePath($this->storagePath);

if ($storagePath) {
$storagePath = $this->normalizePath($storagePath);
if (str_starts_with($storagePath, $basePath)) {
$storagePath = substr($storagePath, strlen($basePath));
}
$relativePath = trim($storagePath, '/') . '/' . $fileName;
} else {
$relativePath = $fileName;
}

return $this->normalizePath($basePath . '/' . $relativePath);
}

/**
* Normalizes a file path by converting backslashes to forward slashes,
* removing duplicate slashes, and trimming trailing slashes.
*
* @param string $path The path to normalize
* @return string The normalized path
*/
protected function normalizePath(string $path): string
{
$path = str_replace('\\', '/', $path);
$path = (string) preg_replace('#/+#', '/', $path);

return rtrim($path, '/');
}

/**
* @param array<string> $files
* @return array<string>
*/
protected function filterAndSortFiles(array $files, string $pattern): array
{
return collect($files)
->map(fn ($file): string => $this->getRelativePath($file))
->filter(fn ($file): bool => str_contains($file, $pattern))
->sortByDesc(fn ($file): DateTime => $this->getLastModifiedDateTime($this->getFullPath($file)))
->values()
->all();
}

protected function getRelativePath(string $fullPath): string
{
return str_replace($this->storagePath . '/', '', $fullPath);
}

/**
* @throws Exception
*/
protected function performFileStreaming(SFTPInterface $sourceSftp, string $remoteZipPath, string $fullPath): bool
{
$tempFile = $this->downloadFileViaSFTP($sourceSftp, $remoteZipPath);
$content = $this->getFileContents($tempFile);

return $this->processAndUploadFile($content, $tempFile, $fullPath);
}

protected function getFileContents(string $filePath): string|false
{
return file_get_contents($filePath);
}

protected function processAndUploadFile(string|false $content, string $tempFile, string $fullPath): bool
{
if ($content === false) {
Log::error('Failed to read temporary file', ['temp_file' => $tempFile]);

return false;
}

// Ensure the directory exists
$directory = dirname($fullPath);
if (! $this->ensureDirectoryExists($directory)) {
Log::error('Failed to create directory structure', ['directory' => $directory]);

return false;
}

$result = $this->sftp->put($fullPath, $content);
$this->cleanUpTempFile($tempFile);

if ($result) {
$this->logSuccessfulStreaming($fullPath);

return true;
}

$lastError = $this->sftp->getLastError();
Log::error('Failed to stream file to remote local storage', [
'full_path' => $fullPath,
'sftp_error' => $lastError,
]);

return false;
}

public function ensureDirectoryExists(string $path): bool
{
$path = $this->normalizePath($path);

$result = $this->sftp->mkdir($path, 0755, true);

if ($result) {
Log::info('Directory created successfully', ['path' => $path]);

return true;
}

$listResult = $this->sftp->exec('ls -la ' . escapeshellarg($path));

if (! empty($listResult)) {
Log::info('Directory already exists', ['path' => $path]);

return true;
}

$lastError = $this->sftp->getLastError();
Log::error('Failed to create or access directory', [
'path' => $path,
'sftp_error' => $lastError,
]);

return false;
}

private function logStartStreaming(string $remoteZipPath, string $fileName, string $fullPath): void
{
Log::info('Starting to stream file to remote local storage.', [
'remote_zip_path' => $remoteZipPath,
'file_name' => $fileName,
'full_path' => $fullPath,
]);
}

private function logSuccessfulStreaming(string $fullPath): void
{
Log::info('File successfully streamed to remote local storage.', ['file_path' => $fullPath]);
}

/**
* @throws Exception
*/
private function getLastModifiedDateTime(string $file): DateTime
{
$stat = $this->sftp->stat($file);
if ($stat === false || ! isset($stat['mtime'])) {
throw new RuntimeException("Failed to get last modified time for file: {$file}");
}

return new DateTime("@{$stat['mtime']}");
}

/**
* @return array<string>
*/
private function listDirectoryContents(string $directory): array
{
$result = $this->sftp->exec("find {$directory} -type f");
if ($result === false) {
Log::error('Failed to list directory contents', ['directory' => $directory]);

return [];
}

return array_filter(explode("\n", $result));
}
}
3 changes: 2 additions & 1 deletion app/Services/Backup/Destinations/S3.php
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ class S3 implements BackupDestinationInterface
public function __construct(
protected S3Client $client,
protected string $bucketName
) {}
) {
}

/**
* @return array<string>
Expand Down
5 changes: 4 additions & 1 deletion app/Services/Backup/Tasks/DatabaseBackupTask.php
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,10 @@ protected function performBackup(): void

$this->backupTask->setScriptUpdateTime();

if ($this->backupTask->isRotatingBackups()) {
// NOTE: Vanguard doesn't support backup rotations on the local driver due to how the abstract class methods are set up.
// It is so heavily coupled to assuming it's a third party driver that it will be a nuisance to sort.
// It will need to be addressed.
if ($this->backupTask->isRotatingBackups() && ! $backupDestinationModel->isLocalConnection()) {
$backupDestination = $this->createBackupDestinationInstance($backupDestinationModel);
$this->rotateOldBackups($backupDestination, $this->backupTask->getAttribute('id'), $this->backupTask->getAttribute('maximum_backups_to_keep'), '.sql', 'backup_');
}
Expand Down
5 changes: 4 additions & 1 deletion app/Services/Backup/Tasks/FileBackupTask.php
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,10 @@ protected function performBackup(): void
throw new RuntimeException('Failed to upload the zip file to destination.');
}

if ($this->backupTask->isRotatingBackups()) {
// NOTE: Vanguard doesn't support backup rotations on the local driver due to how the abstract class methods are set up.
// It is so heavily coupled to assuming it's a third party driver that it will be a nuisance to sort.
// It will need to be addressed.
if ($this->backupTask->isRotatingBackups() && ! $backupDestinationModel->isLocalConnection()) {
$backupDestination = $this->createBackupDestinationInstance($backupDestinationModel);
$this->rotateOldBackups($backupDestination, $this->backupTask->getAttribute('id'), $this->backupTask->getAttribute('maximum_backups_to_keep'), '.zip', 'backup_');
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
<x-select id="type" class="block mt-1 w-full" wire:model.live="type" name="type">
<option value="s3">{{ __('Amazon S3') }}</option>
<option value="custom_s3">{{ __('Custom S3') }}</option>
<option value="local">{{ __('Local') }}</option>
</x-select>
<x-input-error :messages="$errors->get('type')" class="mt-2"/>
</div>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,10 @@
</x-table.body-item>
<x-table.body-item class="col-span-2">
<div class="flex justify-center space-x-2">
@if ($backupDestination->type !== \App\Models\BackupDestination::TYPE_LOCAL)
@livewire('backup-destinations.check-connection-button', ['backupDestination' => $backupDestination],
key($backupDestination->id))
@endif
<a href="{{ route('backup-destinations.edit', $backupDestination) }}" wire:navigate>
<x-secondary-button iconOnly>
<span class="sr-only">{{ __('Update Backup Destination') }}</span>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
<x-select id="type" class="block mt-1 w-full" wire:model.live="type" name="type">
<option value="s3">{{ __('Amazon S3') }}</option>
<option value="custom_s3">{{ __('Custom S3') }}</option>
<option value="local">{{ __('Local') }}</option>
</x-select>
<x-input-error :messages="$errors->get('type')" class="mt-2"/>
</div>
Expand Down
Loading

0 comments on commit ebea39f

Please sign in to comment.