From 7d39a5089c18539ab1e7286341b692b9bbaba474 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 16 Sep 2024 11:53:26 +0200 Subject: [PATCH 01/35] Feat: Add SSH Key fingerprint to DB --- ..._key_fingerprint_to_private_keys_table.php | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 database/migrations/2024_09_16_094127_add_ssh_key_fingerprint_to_private_keys_table.php diff --git a/database/migrations/2024_09_16_094127_add_ssh_key_fingerprint_to_private_keys_table.php b/database/migrations/2024_09_16_094127_add_ssh_key_fingerprint_to_private_keys_table.php new file mode 100644 index 0000000000..f64bb32c9c --- /dev/null +++ b/database/migrations/2024_09_16_094127_add_ssh_key_fingerprint_to_private_keys_table.php @@ -0,0 +1,22 @@ +string('fingerprint')->nullable()->unique()->after('private_key'); + }); + } + + public function down() + { + Schema::table('private_keys', function (Blueprint $table) { + $table->dropColumn('fingerprint'); + }); + } +} From f9b7841572d5845c8d4abe3f662bb6d371476513 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 16 Sep 2024 12:54:48 +0200 Subject: [PATCH 02/35] Feat: Add a fingerprint to every private key on save, create... --- app/Models/PrivateKey.php | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/app/Models/PrivateKey.php b/app/Models/PrivateKey.php index 45bc6bc847..8682103821 100644 --- a/app/Models/PrivateKey.php +++ b/app/Models/PrivateKey.php @@ -26,6 +26,7 @@ class PrivateKey extends BaseModel 'name', 'description', 'private_key', + 'fingerprint', 'is_git_related', 'team_id', ]; @@ -35,10 +36,10 @@ protected static function booted() static::saving(function ($key) { $privateKey = data_get($key, 'private_key'); if (substr($privateKey, -1) !== "\n") { - $key->private_key = $privateKey."\n"; + $key->private_key = $privateKey . "\n"; } + $key->fingerprint = $key->generateFingerprint(); }); - } public static function ownedByCurrentTeam(array $select = ['*']) @@ -85,4 +86,14 @@ public function gitlabApps() { return $this->hasMany(GitlabApp::class); } + + public function generateFingerprint() + { + try { + $key = PublicKeyLoader::load($this->private_key); + return $key->getPublicKey()->getFingerprint('sha256'); + } catch (\Throwable $e) { + return 'invalid_' . md5($this->private_key); // TODO: DO NOT ALLOW SAVING IF INVALID SSH KEYS SAY SSH KEY IS INVALID + } + } } From 02017334e50e8328fc7d1b009209ccec495182ae Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 16 Sep 2024 13:02:48 +0200 Subject: [PATCH 03/35] Fix: Make sure invalid private keys can not be added --- app/Livewire/Security/PrivateKey/Create.php | 26 +++++++++++++++------ app/Models/PrivateKey.php | 19 +++++++++------ 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/app/Livewire/Security/PrivateKey/Create.php b/app/Livewire/Security/PrivateKey/Create.php index 32a67bbea7..1a689076b5 100644 --- a/app/Livewire/Security/PrivateKey/Create.php +++ b/app/Livewire/Security/PrivateKey/Create.php @@ -59,13 +59,11 @@ public function updated($updateProperty) { if ($updateProperty === 'value') { try { - $this->publicKey = PublicKeyLoader::load($this->$updateProperty)->getPublicKey()->toString('OpenSSH', ['comment' => '']); + $key = PublicKeyLoader::load($this->$updateProperty); + $this->publicKey = $key->getPublicKey()->toString('OpenSSH', ['comment' => '']); } catch (\Throwable $e) { - if ($this->$updateProperty === '') { - $this->publicKey = ''; - } else { - $this->publicKey = 'Invalid private key'; - } + $this->publicKey = ''; + $this->addError('value', 'Invalid private key'); } } $this->validateOnly($updateProperty); @@ -73,7 +71,21 @@ public function updated($updateProperty) public function createPrivateKey() { - $this->validate(); + $this->validate([ + 'name' => 'required|string', + 'value' => [ + 'required', + 'string', + function ($attribute, $value, $fail) { + try { + PublicKeyLoader::load($value); + } catch (\Throwable $e) { + $fail('The private key is invalid.'); + } + }, + ], + ]); + try { $this->value = trim($this->value); if (! str_ends_with($this->value, "\n")) { diff --git a/app/Models/PrivateKey.php b/app/Models/PrivateKey.php index 8682103821..b047af6bb1 100644 --- a/app/Models/PrivateKey.php +++ b/app/Models/PrivateKey.php @@ -4,6 +4,7 @@ use OpenApi\Attributes as OA; use phpseclib3\Crypt\PublicKeyLoader; +use Illuminate\Validation\ValidationException; #[OA\Schema( description: 'Private Key model', @@ -38,7 +39,15 @@ protected static function booted() if (substr($privateKey, -1) !== "\n") { $key->private_key = $privateKey . "\n"; } - $key->fingerprint = $key->generateFingerprint(); + + try { + $publicKey = PublicKeyLoader::load($key->private_key)->getPublicKey(); + $key->fingerprint = $publicKey->getFingerprint('sha256'); + } catch (\Throwable $e) { + throw ValidationException::withMessages([ + 'private_key' => ['The private key is invalid.'], + ]); + } }); } @@ -89,11 +98,7 @@ public function gitlabApps() public function generateFingerprint() { - try { - $key = PublicKeyLoader::load($this->private_key); - return $key->getPublicKey()->getFingerprint('sha256'); - } catch (\Throwable $e) { - return 'invalid_' . md5($this->private_key); // TODO: DO NOT ALLOW SAVING IF INVALID SSH KEYS SAY SSH KEY IS INVALID - } + $key = PublicKeyLoader::load($this->private_key); + return $key->getPublicKey()->getFingerprint('sha256'); } } From 3aee8e030e6114d1c273eaa5d33502d839224c10 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 16 Sep 2024 13:17:39 +0200 Subject: [PATCH 04/35] Fix: Encrypt private SSH keys in the DB --- app/Models/PrivateKey.php | 4 ++++ ...6_111428_encrypt_existing_private_keys.php | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 database/migrations/2024_09_16_111428_encrypt_existing_private_keys.php diff --git a/app/Models/PrivateKey.php b/app/Models/PrivateKey.php index b047af6bb1..7cb58657c7 100644 --- a/app/Models/PrivateKey.php +++ b/app/Models/PrivateKey.php @@ -32,6 +32,10 @@ class PrivateKey extends BaseModel 'team_id', ]; + protected $casts = [ + 'private_key' => 'encrypted', + ]; + protected static function booted() { static::saving(function ($key) { diff --git a/database/migrations/2024_09_16_111428_encrypt_existing_private_keys.php b/database/migrations/2024_09_16_111428_encrypt_existing_private_keys.php new file mode 100644 index 0000000000..e2297cf375 --- /dev/null +++ b/database/migrations/2024_09_16_111428_encrypt_existing_private_keys.php @@ -0,0 +1,19 @@ +chunkById(100, function ($keys) { + foreach ($keys as $key) { + DB::table('private_keys') + ->where('id', $key->id) + ->update(['private_key' => Crypt::encryptString($key->private_key)]); + } + }); + } +} From ba636a95dc4ca5d27711e79756840e78cbd2d1a6 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 16 Sep 2024 17:24:42 +0200 Subject: [PATCH 05/35] Refactor SSH Keys --- app/Livewire/Security/PrivateKey/Create.php | 107 ++++++--------- app/Models/PrivateKey.php | 128 +++++++++++++++--- .../security/private-key/create.blade.php | 4 +- 3 files changed, 152 insertions(+), 87 deletions(-) diff --git a/app/Livewire/Security/PrivateKey/Create.php b/app/Livewire/Security/PrivateKey/Create.php index 1a689076b5..95f6a71bf1 100644 --- a/app/Livewire/Security/PrivateKey/Create.php +++ b/app/Livewire/Security/PrivateKey/Create.php @@ -3,22 +3,14 @@ namespace App\Livewire\Security\PrivateKey; use App\Models\PrivateKey; -use DanHarrin\LivewireRateLimiting\WithRateLimiting; use Livewire\Component; -use phpseclib3\Crypt\PublicKeyLoader; class Create extends Component { - use WithRateLimiting; - - public string $name; - - public string $value; - + public string $name = ''; + public string $value = ''; public ?string $from = null; - public ?string $description = null; - public ?string $publicKey = null; protected $rules = [ @@ -26,84 +18,69 @@ class Create extends Component 'value' => 'required|string', ]; - protected $validationAttributes = [ - 'name' => 'name', - 'value' => 'private Key', - ]; - public function generateNewRSAKey() { - try { - $this->rateLimit(10); - $this->name = generate_random_name(); - $this->description = 'Created by Coolify'; - ['private' => $this->value, 'public' => $this->publicKey] = generateSSHKey(); - } catch (\Throwable $e) { - return handleError($e, $this); - } + $this->generateNewKey('rsa'); } public function generateNewEDKey() { - try { - $this->rateLimit(10); - $this->name = generate_random_name(); - $this->description = 'Created by Coolify'; - ['private' => $this->value, 'public' => $this->publicKey] = generateSSHKey('ed25519'); - } catch (\Throwable $e) { - return handleError($e, $this); - } + $this->generateNewKey('ed25519'); } - public function updated($updateProperty) + private function generateNewKey($type) { - if ($updateProperty === 'value') { - try { - $key = PublicKeyLoader::load($this->$updateProperty); - $this->publicKey = $key->getPublicKey()->toString('OpenSSH', ['comment' => '']); - } catch (\Throwable $e) { - $this->publicKey = ''; - $this->addError('value', 'Invalid private key'); - } + $keyData = PrivateKey::generateNewKeyPair($type); + $this->setKeyData($keyData); + } + + public function updated($property) + { + if ($property === 'value') { + $this->validatePrivateKey(); } - $this->validateOnly($updateProperty); } public function createPrivateKey() { - $this->validate([ - 'name' => 'required|string', - 'value' => [ - 'required', - 'string', - function ($attribute, $value, $fail) { - try { - PublicKeyLoader::load($value); - } catch (\Throwable $e) { - $fail('The private key is invalid.'); - } - }, - ], - ]); + $this->validate(); try { - $this->value = trim($this->value); - if (! str_ends_with($this->value, "\n")) { - $this->value .= "\n"; - } - $private_key = PrivateKey::create([ + $privateKey = PrivateKey::createAndStore([ 'name' => $this->name, 'description' => $this->description, - 'private_key' => $this->value, + 'private_key' => trim($this->value) . "\n", 'team_id' => currentTeam()->id, ]); - if ($this->from === 'server') { - return redirect()->route('dashboard'); - } - return redirect()->route('security.private-key.show', ['private_key_uuid' => $private_key->uuid]); + return $this->redirectAfterCreation($privateKey); } catch (\Throwable $e) { return handleError($e, $this); } } + + private function setKeyData(array $keyData) + { + $this->name = $keyData['name']; + $this->description = $keyData['description']; + $this->value = $keyData['private_key']; + $this->publicKey = $keyData['public_key']; + } + + private function validatePrivateKey() + { + $validationResult = PrivateKey::validateAndExtractPublicKey($this->value); + $this->publicKey = $validationResult['publicKey']; + + if (!$validationResult['isValid']) { + $this->addError('value', 'Invalid private key'); + } + } + + private function redirectAfterCreation(PrivateKey $privateKey) + { + return $this->from === 'server' + ? redirect()->route('dashboard') + : redirect()->route('security.private-key.show', ['private_key_uuid' => $privateKey->uuid]); + } } diff --git a/app/Models/PrivateKey.php b/app/Models/PrivateKey.php index 7cb58657c7..a56e67ff60 100644 --- a/app/Models/PrivateKey.php +++ b/app/Models/PrivateKey.php @@ -3,8 +3,10 @@ namespace App\Models; use OpenApi\Attributes as OA; -use phpseclib3\Crypt\PublicKeyLoader; +use Illuminate\Support\Facades\Storage; use Illuminate\Validation\ValidationException; +use phpseclib3\Crypt\PublicKeyLoader; +use DanHarrin\LivewireRateLimiting\WithRateLimiting; #[OA\Schema( description: 'Private Key model', @@ -23,6 +25,8 @@ )] class PrivateKey extends BaseModel { + use WithRateLimiting; + protected $fillable = [ 'name', 'description', @@ -39,45 +43,127 @@ class PrivateKey extends BaseModel protected static function booted() { static::saving(function ($key) { - $privateKey = data_get($key, 'private_key'); - if (substr($privateKey, -1) !== "\n") { - $key->private_key = $privateKey . "\n"; - } + $key->private_key = rtrim($key->private_key) . "\n"; - try { - $publicKey = PublicKeyLoader::load($key->private_key)->getPublicKey(); - $key->fingerprint = $publicKey->getFingerprint('sha256'); - } catch (\Throwable $e) { + if (!self::validatePrivateKey($key->private_key)) { throw ValidationException::withMessages([ 'private_key' => ['The private key is invalid.'], ]); } + + $key->fingerprint = self::generateFingerprint($key->private_key); + }); + + static::deleted(function ($key) { + self::deleteFromStorage($key); }); } + public function getPublicKey() + { + return self::extractPublicKeyFromPrivate($this->private_key) ?? 'Error loading private key'; + } + + // For backwards compatibility + public function publicKey() + { + return $this->getPublicKey(); + } + public static function ownedByCurrentTeam(array $select = ['*']) { $selectArray = collect($select)->concat(['id']); + return self::whereTeamId(currentTeam()->id)->select($selectArray->all()); + } + + public static function validatePrivateKey($privateKey) + { + try { + PublicKeyLoader::load($privateKey); + return true; + } catch (\Throwable $e) { + return false; + } + } - return PrivateKey::whereTeamId(currentTeam()->id)->select($selectArray->all()); + public static function generateFingerprint($privateKey) + { + $key = PublicKeyLoader::load($privateKey); + return $key->getPublicKey()->getFingerprint('sha256'); } - public function publicKey() + public static function createAndStore(array $data) + { + $privateKey = new self($data); + $privateKey->save(); + $privateKey->storeInFileSystem(); + return $privateKey; + } + + public static function generateNewKeyPair($type = 'rsa') { try { - return PublicKeyLoader::load($this->private_key)->getPublicKey()->toString('OpenSSH', ['comment' => '']); + $instance = new self(); + $instance->rateLimit(10); + $name = generate_random_name(); + $description = 'Created by Coolify'; + ['private' => $privateKey, 'public' => $publicKey] = generateSSHKey($type === 'ed25519' ? 'ed25519' : 'rsa'); + + return [ + 'name' => $name, + 'description' => $description, + 'private_key' => $privateKey, + 'public_key' => $publicKey, + ]; } catch (\Throwable $e) { - return 'Error loading private key'; + throw new \Exception("Failed to generate new {$type} key: " . $e->getMessage()); } } - public function isEmpty() + public static function extractPublicKeyFromPrivate($privateKey) { - if ($this->servers()->count() === 0 && $this->applications()->count() === 0 && $this->githubApps()->count() === 0 && $this->gitlabApps()->count() === 0) { - return true; + try { + $key = PublicKeyLoader::load($privateKey); + return $key->getPublicKey()->toString('OpenSSH', ['comment' => '']); + } catch (\Throwable $e) { + return null; } + } + + public static function validateAndExtractPublicKey($privateKey) + { + $isValid = self::validatePrivateKey($privateKey); + $publicKey = $isValid ? self::extractPublicKeyFromPrivate($privateKey) : ''; + + return [ + 'isValid' => $isValid, + 'publicKey' => $publicKey, + ]; + } - return false; + public function storeInFileSystem() + { + $filename = "id_rsa@{$this->uuid}"; + Storage::disk('ssh-keys')->put($filename, $this->private_key); + return "/var/www/html/storage/app/ssh/keys/{$filename}"; + } + + public static function deleteFromStorage(self $privateKey) + { + $filename = "id_rsa@{$privateKey->uuid}"; + Storage::disk('ssh-keys')->delete($filename); + } + + public function getKeyLocation() + { + return "/var/www/html/storage/app/ssh/keys/id_rsa@{$this->uuid}"; + } + + public function updatePrivateKey(array $data) + { + $this->update($data); + $this->storeInFileSystem(); + return $this; } public function servers() @@ -100,9 +186,11 @@ public function gitlabApps() return $this->hasMany(GitlabApp::class); } - public function generateFingerprint() + public function isEmpty() { - $key = PublicKeyLoader::load($this->private_key); - return $key->getPublicKey()->getFingerprint('sha256'); + return $this->servers()->count() === 0 + && $this->applications()->count() === 0 + && $this->githubApps()->count() === 0 + && $this->gitlabApps()->count() === 0; } } diff --git a/resources/views/livewire/security/private-key/create.blade.php b/resources/views/livewire/security/private-key/create.blade.php index 1bace9f3a5..a57cfc8e7d 100644 --- a/resources/views/livewire/security/private-key/create.blade.php +++ b/resources/views/livewire/security/private-key/create.blade.php @@ -4,8 +4,8 @@
You should not use passphrase protected keys.
- Generate new RSA SSH Key - Generate new ED25519 SSH Key + Generate new RSA SSH Key + Generate new ED25519 SSH Key
From 54c03fae41fee089736ff60240409f3941b35332 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 16 Sep 2024 18:10:46 +0200 Subject: [PATCH 06/35] Remove ssh key fingerprint as we can just us uuid --- ..._key_fingerprint_to_private_keys_table.php | 22 ------------------- 1 file changed, 22 deletions(-) delete mode 100644 database/migrations/2024_09_16_094127_add_ssh_key_fingerprint_to_private_keys_table.php diff --git a/database/migrations/2024_09_16_094127_add_ssh_key_fingerprint_to_private_keys_table.php b/database/migrations/2024_09_16_094127_add_ssh_key_fingerprint_to_private_keys_table.php deleted file mode 100644 index f64bb32c9c..0000000000 --- a/database/migrations/2024_09_16_094127_add_ssh_key_fingerprint_to_private_keys_table.php +++ /dev/null @@ -1,22 +0,0 @@ -string('fingerprint')->nullable()->unique()->after('private_key'); - }); - } - - public function down() - { - Schema::table('private_keys', function (Blueprint $table) { - $table->dropColumn('fingerprint'); - }); - } -} From 95fcf38d45eeae59283bbdcb330416bdb752b17c Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 16 Sep 2024 18:11:16 +0200 Subject: [PATCH 07/35] Feat: Add is_sftp and is_server_ssh_key coloums --- ..._ssh_sftp_fields_to_private_keys_table.php | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 database/migrations/2024_09_16_160709_add_ssh_sftp_fields_to_private_keys_table.php diff --git a/database/migrations/2024_09_16_160709_add_ssh_sftp_fields_to_private_keys_table.php b/database/migrations/2024_09_16_160709_add_ssh_sftp_fields_to_private_keys_table.php new file mode 100644 index 0000000000..a12cd04e90 --- /dev/null +++ b/database/migrations/2024_09_16_160709_add_ssh_sftp_fields_to_private_keys_table.php @@ -0,0 +1,37 @@ +deleteDirectory(''); + Storage::disk('ssh-keys')->makeDirectory(''); + + Schema::table('private_keys', function (Blueprint $table) { + $table->boolean('is_server_ssh_key')->default(true); + $table->boolean('is_sftp_key')->default(false); + }); + + // Re-save SSH keys on server only for records with is_server_ssh_key = true + PrivateKey::where('is_server_ssh_key', true)->chunk(100, function ($keys) { + foreach ($keys as $key) { + $key->storeInFileSystem(); + } + }); + } + + public function down() + { + Schema::table('private_keys', function (Blueprint $table) { + $table->dropColumn('is_sftp_storage_key'); + $table->dropColumn('is_server_ssh_key'); + }); + } +}; \ No newline at end of file From b09017ea4672f5a0e5d033d8942e3e28244ac981 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 16 Sep 2024 18:11:37 +0200 Subject: [PATCH 08/35] Feat: new ssh key file name on disk --- app/Models/PrivateKey.php | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/app/Models/PrivateKey.php b/app/Models/PrivateKey.php index a56e67ff60..155ac2bdcb 100644 --- a/app/Models/PrivateKey.php +++ b/app/Models/PrivateKey.php @@ -31,7 +31,6 @@ class PrivateKey extends BaseModel 'name', 'description', 'private_key', - 'fingerprint', 'is_git_related', 'team_id', ]; @@ -50,8 +49,6 @@ protected static function booted() 'private_key' => ['The private key is invalid.'], ]); } - - $key->fingerprint = self::generateFingerprint($key->private_key); }); static::deleted(function ($key) { @@ -86,12 +83,6 @@ public static function validatePrivateKey($privateKey) } } - public static function generateFingerprint($privateKey) - { - $key = PublicKeyLoader::load($privateKey); - return $key->getPublicKey()->getFingerprint('sha256'); - } - public static function createAndStore(array $data) { $privateKey = new self($data); @@ -143,20 +134,20 @@ public static function validateAndExtractPublicKey($privateKey) public function storeInFileSystem() { - $filename = "id_rsa@{$this->uuid}"; + $filename = "ssh@{$this->uuid}"; Storage::disk('ssh-keys')->put($filename, $this->private_key); return "/var/www/html/storage/app/ssh/keys/{$filename}"; } public static function deleteFromStorage(self $privateKey) { - $filename = "id_rsa@{$privateKey->uuid}"; + $filename = "ssh@{$privateKey->uuid}"; Storage::disk('ssh-keys')->delete($filename); } public function getKeyLocation() { - return "/var/www/html/storage/app/ssh/keys/id_rsa@{$this->uuid}"; + return "/var/www/html/storage/app/ssh/keys/ssh@{$this->uuid}"; } public function updatePrivateKey(array $data) From 0bfdc1c5314b2fd672fa1fd2664707b5bdd80385 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 16 Sep 2024 18:45:08 +0200 Subject: [PATCH 09/35] Feat: Store all keys on disk by default --- ...709_add_ssh_sftp_fields_to_private_keys_table.php | 6 ++---- database/seeders/PrivateKeySeeder.php | 12 +++--------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/database/migrations/2024_09_16_160709_add_ssh_sftp_fields_to_private_keys_table.php b/database/migrations/2024_09_16_160709_add_ssh_sftp_fields_to_private_keys_table.php index a12cd04e90..8297ec75fd 100644 --- a/database/migrations/2024_09_16_160709_add_ssh_sftp_fields_to_private_keys_table.php +++ b/database/migrations/2024_09_16_160709_add_ssh_sftp_fields_to_private_keys_table.php @@ -10,7 +10,6 @@ { public function up() { - // Empty the SSH keys folder Storage::disk('ssh-keys')->deleteDirectory(''); Storage::disk('ssh-keys')->makeDirectory(''); @@ -19,7 +18,6 @@ public function up() $table->boolean('is_sftp_key')->default(false); }); - // Re-save SSH keys on server only for records with is_server_ssh_key = true PrivateKey::where('is_server_ssh_key', true)->chunk(100, function ($keys) { foreach ($keys as $key) { $key->storeInFileSystem(); @@ -30,8 +28,8 @@ public function up() public function down() { Schema::table('private_keys', function (Blueprint $table) { - $table->dropColumn('is_sftp_storage_key'); $table->dropColumn('is_server_ssh_key'); + $table->dropColumn('is_sftp_key'); }); } -}; \ No newline at end of file +}; diff --git a/database/seeders/PrivateKeySeeder.php b/database/seeders/PrivateKeySeeder.php index 8a70cf56d8..315c988a56 100644 --- a/database/seeders/PrivateKeySeeder.php +++ b/database/seeders/PrivateKeySeeder.php @@ -15,7 +15,7 @@ public function run(): void PrivateKey::create([ 'id' => 0, 'team_id' => 0, - 'name' => 'Testing-host', + 'name' => 'Testing Host Key', 'description' => 'This is a test docker container', 'private_key' => '-----BEGIN OPENSSH PRIVATE KEY----- b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW @@ -25,8 +25,9 @@ public function run(): void uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA== -----END OPENSSH PRIVATE KEY----- ', - + 'is_server_ssh_key' => true, ]); + PrivateKey::create([ 'id' => 1, 'team_id' => 0, @@ -61,12 +62,5 @@ public function run(): void -----END RSA PRIVATE KEY-----', 'is_git_related' => true, ]); - PrivateKey::create([ - 'id' => 2, - 'team_id' => 0, - 'name' => 'development-gitlab-app', - 'description' => 'This is the key for using the development Gitlab app', - 'private_key' => 'asdf', - ]); } } From b79b4015d7e226aaee74e29e86e6804585a968b0 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 16 Sep 2024 19:08:15 +0200 Subject: [PATCH 10/35] Feat: Populate SSH key folder --- ..._ssh_sftp_fields_to_private_keys_table.php | 35 ------------------- ..._16_170001_populate_ssh_keys_directory.php | 26 ++++++++++++++ database/seeders/PrivateKeySeeder.php | 1 - 3 files changed, 26 insertions(+), 36 deletions(-) delete mode 100644 database/migrations/2024_09_16_160709_add_ssh_sftp_fields_to_private_keys_table.php create mode 100644 database/migrations/2024_09_16_170001_populate_ssh_keys_directory.php diff --git a/database/migrations/2024_09_16_160709_add_ssh_sftp_fields_to_private_keys_table.php b/database/migrations/2024_09_16_160709_add_ssh_sftp_fields_to_private_keys_table.php deleted file mode 100644 index 8297ec75fd..0000000000 --- a/database/migrations/2024_09_16_160709_add_ssh_sftp_fields_to_private_keys_table.php +++ /dev/null @@ -1,35 +0,0 @@ -deleteDirectory(''); - Storage::disk('ssh-keys')->makeDirectory(''); - - Schema::table('private_keys', function (Blueprint $table) { - $table->boolean('is_server_ssh_key')->default(true); - $table->boolean('is_sftp_key')->default(false); - }); - - PrivateKey::where('is_server_ssh_key', true)->chunk(100, function ($keys) { - foreach ($keys as $key) { - $key->storeInFileSystem(); - } - }); - } - - public function down() - { - Schema::table('private_keys', function (Blueprint $table) { - $table->dropColumn('is_server_ssh_key'); - $table->dropColumn('is_sftp_key'); - }); - } -}; diff --git a/database/migrations/2024_09_16_170001_populate_ssh_keys_directory.php b/database/migrations/2024_09_16_170001_populate_ssh_keys_directory.php new file mode 100644 index 0000000000..61de17a076 --- /dev/null +++ b/database/migrations/2024_09_16_170001_populate_ssh_keys_directory.php @@ -0,0 +1,26 @@ +deleteDirectory(''); + Storage::disk('ssh-keys')->makeDirectory(''); + + PrivateKey::chunk(100, function ($keys) { + foreach ($keys as $key) { + $key->storeInFileSystem(); + } + }); + } + + public function down() + { + Storage::disk('ssh-keys')->deleteDirectory(''); + Storage::disk('ssh-keys')->makeDirectory(''); + } +}; diff --git a/database/seeders/PrivateKeySeeder.php b/database/seeders/PrivateKeySeeder.php index 315c988a56..afc8a1e816 100644 --- a/database/seeders/PrivateKeySeeder.php +++ b/database/seeders/PrivateKeySeeder.php @@ -25,7 +25,6 @@ public function run(): void uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA== -----END OPENSSH PRIVATE KEY----- ', - 'is_server_ssh_key' => true, ]); PrivateKey::create([ From a68fbefadb483655c2e78c236ccd8c27e968aacf Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 16 Sep 2024 19:34:46 +0200 Subject: [PATCH 11/35] Fix: Populate SSH keys in dev --- ..._16_170001_populate_ssh_keys_directory.php | 10 ++------- database/seeders/DatabaseSeeder.php | 1 + .../PopulateSshKeysDirectorySeeder.php | 22 +++++++++++++++++++ 3 files changed, 25 insertions(+), 8 deletions(-) create mode 100644 database/seeders/PopulateSshKeysDirectorySeeder.php diff --git a/database/migrations/2024_09_16_170001_populate_ssh_keys_directory.php b/database/migrations/2024_09_16_170001_populate_ssh_keys_directory.php index 61de17a076..33a5e695f6 100644 --- a/database/migrations/2024_09_16_170001_populate_ssh_keys_directory.php +++ b/database/migrations/2024_09_16_170001_populate_ssh_keys_directory.php @@ -4,7 +4,7 @@ use Illuminate\Support\Facades\Storage; use App\Models\PrivateKey; -return new class extends Migration +class PopulateSshKeysDirectory extends Migration { public function up() { @@ -17,10 +17,4 @@ public function up() } }); } - - public function down() - { - Storage::disk('ssh-keys')->deleteDirectory(''); - Storage::disk('ssh-keys')->makeDirectory(''); - } -}; +} diff --git a/database/seeders/DatabaseSeeder.php b/database/seeders/DatabaseSeeder.php index b3fac350f1..874762aef3 100644 --- a/database/seeders/DatabaseSeeder.php +++ b/database/seeders/DatabaseSeeder.php @@ -13,6 +13,7 @@ public function run(): void UserSeeder::class, TeamSeeder::class, PrivateKeySeeder::class, + PopulateSshKeysDirectorySeeder::class, ServerSeeder::class, ServerSettingSeeder::class, ProjectSeeder::class, diff --git a/database/seeders/PopulateSshKeysDirectorySeeder.php b/database/seeders/PopulateSshKeysDirectorySeeder.php new file mode 100644 index 0000000000..9fd6e5cfcb --- /dev/null +++ b/database/seeders/PopulateSshKeysDirectorySeeder.php @@ -0,0 +1,22 @@ +deleteDirectory(''); + Storage::disk('ssh-keys')->makeDirectory(''); + + PrivateKey::chunk(100, function ($keys) { + foreach ($keys as $key) { + $key->storeInFileSystem(); + } + }); + } +} From 451272bf11a7efb1ec11679f6c34c81dade5d169 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 16 Sep 2024 19:52:55 +0200 Subject: [PATCH 12/35] Fix: Use new function names and logic everywhere --- app/Livewire/Boarding/Index.php | 4 ++-- app/Livewire/Security/PrivateKey/Show.php | 2 +- app/Livewire/Server/ShowPrivateKey.php | 28 +++++++++++------------ 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/app/Livewire/Boarding/Index.php b/app/Livewire/Boarding/Index.php index af05ad767d..0ded72d7e2 100644 --- a/app/Livewire/Boarding/Index.php +++ b/app/Livewire/Boarding/Index.php @@ -141,7 +141,7 @@ public function setServerType(string $type) if (! $this->createdServer) { return $this->dispatch('error', 'Localhost server is not found. Something went wrong during installation. Please try to reinstall or contact support.'); } - $this->serverPublicKey = $this->createdServer->privateKey->publicKey(); + $this->serverPublicKey = $this->createdServer->privateKey->getPublicKey(); return $this->validateServer('localhost'); } elseif ($this->selectedServerType === 'remote') { @@ -175,7 +175,7 @@ public function selectExistingServer() return; } $this->selectedExistingPrivateKey = $this->createdServer->privateKey->id; - $this->serverPublicKey = $this->createdServer->privateKey->publicKey(); + $this->serverPublicKey = $this->createdServer->privateKey->getPublicKey(); $this->updateServerDetails(); $this->currentState = 'validate-server'; } diff --git a/app/Livewire/Security/PrivateKey/Show.php b/app/Livewire/Security/PrivateKey/Show.php index d86bd5d1e0..68e44d4082 100644 --- a/app/Livewire/Security/PrivateKey/Show.php +++ b/app/Livewire/Security/PrivateKey/Show.php @@ -35,7 +35,7 @@ public function mount() public function loadPublicKey() { - $this->public_key = $this->private_key->publicKey(); + $this->public_key = $this->private_key->getPublicKey(); } public function delete() diff --git a/app/Livewire/Server/ShowPrivateKey.php b/app/Livewire/Server/ShowPrivateKey.php index 578a089677..d5e20702d0 100644 --- a/app/Livewire/Server/ShowPrivateKey.php +++ b/app/Livewire/Server/ShowPrivateKey.php @@ -4,6 +4,7 @@ use App\Models\Server; use Livewire\Component; +use App\Models\PrivateKey; class ShowPrivateKey extends Component { @@ -13,25 +14,22 @@ class ShowPrivateKey extends Component public $parameters; - public function setPrivateKey($newPrivateKeyId) + public function setPrivateKey($privateKeyId) { try { - $oldPrivateKeyId = $this->server->private_key_id; - refresh_server_connection($this->server->privateKey); - $this->server->update([ - 'private_key_id' => $newPrivateKeyId, + $privateKey = PrivateKey::findOrFail($privateKeyId); + $this->server->update(['private_key_id' => $privateKeyId]); + $privateKey->storeInFileSystem(); + + $this->dispatch('notify', [ + 'type' => 'success', + 'message' => 'Private key updated and stored in the file system.', ]); - $this->server->refresh(); - refresh_server_connection($this->server->privateKey); - $this->checkConnection(); - } catch (\Throwable $e) { - $this->server->update([ - 'private_key_id' => $oldPrivateKeyId, + } catch (\Exception $e) { + $this->dispatch('notify', [ + 'type' => 'error', + 'message' => 'Failed to update private key: ' . $e->getMessage(), ]); - $this->server->refresh(); - refresh_server_connection($this->server->privateKey); - - return handleError($e, $this); } } From 70b757df5bcba7dc0bdf6248a25bc311215bc118 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 16 Sep 2024 19:53:45 +0200 Subject: [PATCH 13/35] remove old function --- app/Models/PrivateKey.php | 6 ------ 1 file changed, 6 deletions(-) diff --git a/app/Models/PrivateKey.php b/app/Models/PrivateKey.php index 155ac2bdcb..ee1a6b808f 100644 --- a/app/Models/PrivateKey.php +++ b/app/Models/PrivateKey.php @@ -61,12 +61,6 @@ public function getPublicKey() return self::extractPublicKeyFromPrivate($this->private_key) ?? 'Error loading private key'; } - // For backwards compatibility - public function publicKey() - { - return $this->getPublicKey(); - } - public static function ownedByCurrentTeam(array $select = ['*']) { $selectArray = collect($select)->concat(['id']); From 86722939cd56b6bda69b99f0abcda1b4efe38142 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 16 Sep 2024 21:34:27 +0200 Subject: [PATCH 14/35] Fix. Remove write to SSH key on every remote command execution --- app/Jobs/ApplicationDeploymentJob.php | 17 +--- app/Livewire/Boarding/Index.php | 27 +++-- app/Models/Server.php | 32 ++++-- bootstrap/helpers/remoteProcess.php | 140 +++++++++++--------------- routes/web.php | 2 +- 5 files changed, 105 insertions(+), 113 deletions(-) diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 718cea6390..205fdce15e 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -210,7 +210,6 @@ public function __construct(int $application_deployment_queue_id) } ray('New container name: ', $this->container_name)->green(); - savePrivateKeyToFs($this->server); $this->saved_outputs = collect(); // Set preview fqdn @@ -969,7 +968,7 @@ private function save_environment_variables() } } if ($this->application->environment_variables->where('key', 'COOLIFY_URL')->isEmpty()) { - $url = str($this->application->fqdn)->replace('http://', '')->replace('https://', ''); + $url = str($this->application->fqdn)->replace('http://', '').replace('https://', ''); if ($this->application->compose_parsing_version === '3') { $envs->push("COOLIFY_FQDN={$url}"); } else { @@ -1442,21 +1441,11 @@ private function check_git_if_build_needed() if ($this->pull_request_id !== 0) { $local_branch = "pull/{$this->pull_request_id}/head"; } - $private_key = data_get($this->application, 'private_key.private_key'); + $private_key = $this->application->privateKey->getKeyLocation(); if ($private_key) { - $private_key = base64_encode($private_key); $this->execute_remote_command( [ - executeInDocker($this->deployment_uuid, 'mkdir -p /root/.ssh'), - ], - [ - executeInDocker($this->deployment_uuid, "echo '{$private_key}' | base64 -d | tee /root/.ssh/id_rsa > /dev/null"), - ], - [ - executeInDocker($this->deployment_uuid, 'chmod 600 /root/.ssh/id_rsa'), - ], - [ - executeInDocker($this->deployment_uuid, "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" git ls-remote {$this->fullRepoUrl} {$local_branch}"), + executeInDocker($this->deployment_uuid, "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$private_key}\" git ls-remote {$this->fullRepoUrl} {$local_branch}"), 'hidden' => true, 'save' => 'git_commit_sha', ], diff --git a/app/Livewire/Boarding/Index.php b/app/Livewire/Boarding/Index.php index 0ded72d7e2..21dcda50fa 100644 --- a/app/Livewire/Boarding/Index.php +++ b/app/Livewire/Boarding/Index.php @@ -231,17 +231,24 @@ public function setPrivateKey(string $type) public function savePrivateKey() { $this->validate([ - 'privateKeyName' => 'required', - 'privateKey' => 'required', + 'privateKeyName' => 'required|string|max:255', + 'privateKeyDescription' => 'nullable|string|max:255', + 'privateKey' => 'required|string', ]); - $this->createdPrivateKey = PrivateKey::create([ - 'name' => $this->privateKeyName, - 'description' => $this->privateKeyDescription, - 'private_key' => $this->privateKey, - 'team_id' => currentTeam()->id, - ]); - $this->createdPrivateKey->save(); - $this->currentState = 'create-server'; + + try { + $privateKey = PrivateKey::createAndStore([ + 'name' => $this->privateKeyName, + 'description' => $this->privateKeyDescription, + 'private_key' => $this->privateKey, + 'team_id' => currentTeam()->id, + ]); + + $this->createdPrivateKey = $privateKey; + $this->currentState = 'create-server'; + } catch (\Exception $e) { + $this->addError('privateKey', 'Failed to save private key: ' . $e->getMessage()); + } } public function saveServer() diff --git a/app/Models/Server.php b/app/Models/Server.php index 65d70083fa..cae910257f 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -909,13 +909,15 @@ public function isProxyShouldRun() public function isFunctional() { - $isFunctional = $this->settings->is_reachable && $this->settings->is_usable && ! $this->settings->force_disabled; - ['private_key_filename' => $private_key_filename, 'mux_filename' => $mux_filename] = server_ssh_configuration($this); - if (! $isFunctional) { - Storage::disk('ssh-keys')->delete($private_key_filename); - Storage::disk('ssh-mux')->delete($mux_filename); + $isFunctional = $this->settings->is_reachable && $this->settings->is_usable && !$this->settings->force_disabled; + + if (!$isFunctional) { + if ($this->privateKey) { + PrivateKey::deleteFromStorage($this->privateKey); + } + Storage::disk('ssh-mux')->delete($this->muxFilename()); } - + return $isFunctional; } @@ -1115,4 +1117,22 @@ public function isBuildServer() { return $this->settings->is_build_server; } + + public static function createWithPrivateKey(array $data, PrivateKey $privateKey) + { + $server = new self($data); + $server->privateKey()->associate($privateKey); + $server->save(); + return $server; + } + + public function updateWithPrivateKey(array $data, PrivateKey $privateKey = null) + { + $this->update($data); + if ($privateKey) { + $this->privateKey()->associate($privateKey); + $this->save(); + } + return $this; + } } diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index 4ba378e675..0d21c20a0e 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -16,6 +16,7 @@ use Illuminate\Support\Str; use Spatie\Activitylog\Contracts\Activity; + function remote_process( Collection|array $command, Server $server, @@ -26,19 +27,18 @@ function remote_process( $callEventOnFinish = null, $callEventData = null ): Activity { - if (is_null($type)) { - $type = ActivityTypes::INLINE->value; - } - if ($command instanceof Collection) { - $command = $command->toArray(); - } + $type = $type ?? ActivityTypes::INLINE->value; + $command = $command instanceof Collection ? $command->toArray() : $command; + if ($server->isNonRoot()) { $command = parseCommandsByLineForSudo(collect($command), $server); } + $command_string = implode("\n", $command); + if (auth()->user()) { $teams = auth()->user()->teams->pluck('id'); - if (! $teams->contains($server->team_id) && ! $teams->contains(0)) { + if (!$teams->contains($server->team_id) && !$teams->contains(0)) { throw new \Exception('User is not part of the team that owns this server'); } } @@ -46,9 +46,7 @@ function remote_process( return resolve(PrepareCoolifyTask::class, [ 'remoteProcessArgs' => new CoolifyTaskArgs( server_uuid: $server->uuid, - command: <<privateKey->getKeyLocation(); + ray($sshKeyLocation); $mux_filename = '/var/www/html/storage/app/ssh/mux/'.$server->muxFilename(); return [ - 'location' => $location, - 'mux_filename' => $mux_filename, - 'private_key_filename' => $private_key_filename, + 'sshKeyLocation' => $sshKeyLocation, + 'muxFilename' => $mux_filename, ]; } -function savePrivateKeyToFs(Server $server) -{ - if (data_get($server, 'privateKey.private_key') === null) { - throw new \Exception("Server {$server->name} does not have a private key"); - } - ['location' => $location, 'private_key_filename' => $private_key_filename] = server_ssh_configuration($server); - Storage::disk('ssh-keys')->makeDirectory('.'); - Storage::disk('ssh-mux')->makeDirectory('.'); - Storage::disk('ssh-keys')->put($private_key_filename, $server->privateKey->private_key); - - return $location; -} function generateScpCommand(Server $server, string $source, string $dest) { + $sshConfig = server_ssh_configuration($server); + $sshKeyLocation = $sshConfig['sshKeyLocation']; + $muxSocket = $sshConfig['muxFilename']; + $user = $server->user; $port = $server->port; - $privateKeyLocation = savePrivateKeyToFs($server); $timeout = config('constants.ssh.command_timeout'); $connectionTimeout = config('constants.ssh.connection_timeout'); $serverInterval = config('constants.ssh.server_interval'); @@ -99,21 +84,20 @@ function generateScpCommand(Server $server, string $source, string $dest) $scp_command = "timeout $timeout scp "; $muxEnabled = config('constants.ssh.mux_enabled', true) && config('coolify.is_windows_docker_desktop') == false; - // ray('SSH Multiplexing Enabled:', $muxEnabled)->blue(); + ray('SSH Multiplexing Enabled:', $muxEnabled)->blue(); if ($muxEnabled) { - $muxSocket = "/var/www/html/storage/app/ssh/mux/{$server->muxFilename()}"; $scp_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; ensureMultiplexedConnection($server); - // ray('Using SSH Multiplexing')->green(); + ray('Using SSH Multiplexing')->green(); } else { - // ray('Not using SSH Multiplexing')->red(); + ray('Not using SSH Multiplexing')->red(); } if (data_get($server, 'settings.is_cloudflare_tunnel')) { $scp_command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" '; } - $scp_command .= "-i {$privateKeyLocation} " + $scp_command .= "-i {$sshKeyLocation} " .'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ' .'-o PasswordAuthentication=no ' ."-o ConnectTimeout=$connectionTimeout " @@ -126,6 +110,7 @@ function generateScpCommand(Server $server, string $source, string $dest) return $scp_command; } + function instant_scp(string $source, string $dest, Server $server, $throwError = true) { $timeout = config('constants.ssh.command_timeout'); @@ -146,48 +131,52 @@ function instant_scp(string $source, string $dest, Server $server, $throwError = return $output; } + function generateSshCommand(Server $server, string $command) { if ($server->settings->force_disabled) { throw new \RuntimeException('Server is disabled.'); } - $user = $server->user; - $port = $server->port; - $privateKeyLocation = savePrivateKeyToFs($server); + + $sshConfig = server_ssh_configuration($server); + $sshKeyLocation = $sshConfig['sshKeyLocation']; + $muxSocket = $sshConfig['muxFilename']; + $timeout = config('constants.ssh.command_timeout'); $connectionTimeout = config('constants.ssh.connection_timeout'); $serverInterval = config('constants.ssh.server_interval'); $muxPersistTime = config('constants.ssh.mux_persist_time'); + $muxEnabled = config('constants.ssh.mux_enabled') && !config('coolify.is_windows_docker_desktop'); $ssh_command = "timeout $timeout ssh "; - $muxEnabled = config('constants.ssh.mux_enabled') && config('coolify.is_windows_docker_desktop') == false; - // ray('SSH Multiplexing Enabled:', $muxEnabled)->blue(); + ray('SSH Multiplexing Enabled:', $muxEnabled)->blue(); + if ($muxEnabled) { - // Always use multiplexing when enabled - $muxSocket = "/var/www/html/storage/app/ssh/mux/{$server->muxFilename()}"; $ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; ensureMultiplexedConnection($server); - // ray('Using SSH Multiplexing')->green(); + ray('Using SSH Multiplexing')->green(); } else { - // ray('Not using SSH Multiplexing')->red(); + ray('Not using SSH Multiplexing')->red(); } if (data_get($server, 'settings.is_cloudflare_tunnel')) { $ssh_command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" '; } + $command = "PATH=\$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/local/sbin:/host/usr/local/bin:/host/usr/sbin:/host/usr/bin:/host/sbin:/host/bin && $command"; $delimiter = Hash::make($command); $command = str_replace($delimiter, '', $command); - $ssh_command .= "-i {$privateKeyLocation} " + + $ssh_command .= "-i {$sshKeyLocation} " .'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ' .'-o PasswordAuthentication=no ' ."-o ConnectTimeout=$connectionTimeout " ."-o ServerAliveInterval=$serverInterval " .'-o RequestTTY=no ' .'-o LogLevel=ERROR ' - ."-p {$port} " - ."{$user}@{$server->ip} " + ."-p {$server->port} " + ."{$server->user}@{$server->ip} " ." 'bash -se' << \\$delimiter".PHP_EOL .$command.PHP_EOL .$delimiter; @@ -197,52 +186,33 @@ function generateSshCommand(Server $server, string $command) function ensureMultiplexedConnection(Server $server) { - if (! (config('constants.ssh.mux_enabled') && config('coolify.is_windows_docker_desktop') == false)) { - return; - } - static $ensuredConnections = []; - if (isset($ensuredConnections[$server->id])) { - if (! shouldResetMultiplexedConnection($server)) { - // ray('Using Existing Multiplexed Connection')->green(); + $sshConfig = server_ssh_configuration($server); + $muxSocket = $sshConfig['muxFilename']; - return; - } - } - - $muxSocket = "/var/www/html/storage/app/ssh/mux/{$server->muxFilename()}"; - $checkCommand = "ssh -O check -o ControlPath=$muxSocket "; - if (data_get($server, 'settings.is_cloudflare_tunnel')) { - $checkCommand .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" '; + if (isset($ensuredConnections[$server->id]) && !shouldResetMultiplexedConnection($server)) { + return; } - $checkCommand .= " {$server->user}@{$server->ip}"; + $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip}"; $process = Process::run($checkCommand); if ($process->exitCode() === 0) { - // ray('Existing Multiplexed Connection is Valid')->green(); $ensuredConnections[$server->id] = [ 'timestamp' => now(), 'muxSocket' => $muxSocket, ]; - return; } - // ray('Establishing New Multiplexed Connection')->orange(); - - $privateKeyLocation = savePrivateKeyToFs($server); + $sshKeyLocation = $sshConfig['sshKeyLocation']; $connectionTimeout = config('constants.ssh.connection_timeout'); $serverInterval = config('constants.ssh.server_interval'); $muxPersistTime = config('constants.ssh.mux_persist_time'); - $establishCommand = "ssh -fNM -o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; - - if (data_get($server, 'settings.is_cloudflare_tunnel')) { - $establishCommand .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" '; - } - $establishCommand .= "-i {$privateKeyLocation} " + $establishCommand = "ssh -fNM -o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} " + ."-i {$sshKeyLocation} " .'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ' .'-o PasswordAuthentication=no ' ."-o ConnectTimeout=$connectionTimeout " @@ -262,8 +232,6 @@ function ensureMultiplexedConnection(Server $server) 'timestamp' => now(), 'muxSocket' => $muxSocket, ]; - - // ray('Established New Multiplexed Connection')->green(); } function shouldResetMultiplexedConnection(Server $server) @@ -320,7 +288,7 @@ function instant_remote_process(Collection|array $command, Server $server, bool $process = Process::timeout($timeout)->run($sshCommand); $end_time = microtime(true); - $execution_time = ($end_time - $start_time) * 1000; // Convert to milliseconds + // $execution_time = ($end_time - $start_time) * 1000; // Convert to milliseconds // ray('SSH command execution time:', $execution_time.' ms')->orange(); $output = trim($process->output()); @@ -339,6 +307,7 @@ function instant_remote_process(Collection|array $command, Server $server, bool return $output; } + function excludeCertainErrors(string $errorOutput, ?int $exitCode = null) { $ignoredErrors = collect([ @@ -358,6 +327,7 @@ function excludeCertainErrors(string $errorOutput, ?int $exitCode = null) } throw new \RuntimeException($errorOutput, $exitCode); } + function decode_remote_command_output(?ApplicationDeploymentQueue $application_deployment_queue = null): Collection { $application = Application::find(data_get($application_deployment_queue, 'application_id')); @@ -424,16 +394,20 @@ function decode_remote_command_output(?ApplicationDeploymentQueue $application_d return $formatted; } + function remove_iip($text) { $text = preg_replace('/x-access-token:.*?(?=@)/', 'x-access-token:'.REDACTED, $text); return preg_replace('/\x1b\[[0-9;]*m/', '', $text); } + function remove_mux_and_private_key(Server $server) { - $muxFilename = $server->muxFilename(); - $privateKeyLocation = savePrivateKeyToFs($server); + $sshConfig = server_ssh_configuration($server); + $privateKeyLocation = $sshConfig['sshKeyLocation']; + $muxFilename = basename($sshConfig['muxFilename']); + $closeCommand = "ssh -O exit -o ControlPath=/var/www/html/storage/app/ssh/mux/{$muxFilename} {$server->user}@{$server->ip}"; Process::run($closeCommand); @@ -441,13 +415,15 @@ function remove_mux_and_private_key(Server $server) Storage::disk('ssh-mux')->delete($muxFilename); Storage::disk('ssh-keys')->delete($privateKeyLocation); } + function refresh_server_connection(?PrivateKey $private_key = null) { if (is_null($private_key)) { return; } foreach ($private_key->servers as $server) { - $muxFilename = $server->muxFilename(); + $sshConfig = server_ssh_configuration($server); + $muxFilename = basename($sshConfig['muxFilename']); $closeCommand = "ssh -O exit -o ControlPath=/var/www/html/storage/app/ssh/mux/{$muxFilename} {$server->user}@{$server->ip}"; Process::run($closeCommand); Storage::disk('ssh-mux')->delete($muxFilename); diff --git a/routes/web.php b/routes/web.php index e2ccfc704a..98995dd996 100644 --- a/routes/web.php +++ b/routes/web.php @@ -264,7 +264,7 @@ } else { $server = $execution->scheduledDatabaseBackup->database->destination->server; } - $privateKeyLocation = savePrivateKeyToFs($server); + $privateKeyLocation = $server->privateKey->getKeyLocation(); $disk = Storage::build([ 'driver' => 'sftp', 'host' => $server->ip, From f9375f91ec1b4196211b8281da881cba209ffaf1 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 16 Sep 2024 22:33:43 +0200 Subject: [PATCH 15/35] Feat: Create a Multiplexing Helper --- app/Helpers/SshMultiplexingHelper.php | 223 ++++++++++++++++++++++++++ bootstrap/helpers/remoteProcess.php | 211 ++---------------------- 2 files changed, 232 insertions(+), 202 deletions(-) create mode 100644 app/Helpers/SshMultiplexingHelper.php diff --git a/app/Helpers/SshMultiplexingHelper.php b/app/Helpers/SshMultiplexingHelper.php new file mode 100644 index 0000000000..8b39d61e82 --- /dev/null +++ b/app/Helpers/SshMultiplexingHelper.php @@ -0,0 +1,223 @@ +privateKey->getKeyLocation(); + $muxFilename = '/var/www/html/storage/app/ssh/mux/' . $server->muxFilename(); + + return [ + 'sshKeyLocation' => $sshKeyLocation, + 'muxFilename' => $muxFilename, + ]; + } + + public static function ensureMultiplexedConnection(Server $server) + { + $sshConfig = self::serverSshConfiguration($server); + $muxSocket = $sshConfig['muxFilename']; + $sshKeyLocation = $sshConfig['sshKeyLocation']; + + if (!file_exists($sshKeyLocation)) { + throw new \RuntimeException("SSH key file not accessible: $sshKeyLocation"); + } + + if (isset(self::$ensuredConnections[$server->id]) && !self::shouldResetMultiplexedConnection($server)) { + return; + } + + $checkFileCommand = "ls $muxSocket 2>/dev/null"; + $fileCheckProcess = Process::run($checkFileCommand); + + if ($fileCheckProcess->exitCode() !== 0) { + self::establishNewMultiplexedConnection($server); + return; + } + + $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip}"; + $process = Process::run($checkCommand); + + if ($process->exitCode() === 0) { + self::$ensuredConnections[$server->id] = [ + 'timestamp' => now(), + 'muxSocket' => $muxSocket, + ]; + return; + } + + self::establishNewMultiplexedConnection($server); + } + + public static function establishNewMultiplexedConnection(Server $server) + { + $sshConfig = self::serverSshConfiguration($server); + $sshKeyLocation = $sshConfig['sshKeyLocation']; + $muxSocket = $sshConfig['muxFilename']; + + $connectionTimeout = config('constants.ssh.connection_timeout'); + $serverInterval = config('constants.ssh.server_interval'); + $muxPersistTime = config('constants.ssh.mux_persist_time'); + + $establishCommand = "ssh -fNM -o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} " + . "-i {$sshKeyLocation} " + . '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ' + . '-o PasswordAuthentication=no ' + . "-o ConnectTimeout=$connectionTimeout " + . "-o ServerAliveInterval=$serverInterval " + . '-o RequestTTY=no ' + . '-o LogLevel=ERROR ' + . "-p {$server->port} " + . "{$server->user}@{$server->ip}"; + + $establishProcess = Process::run($establishCommand); + + if ($establishProcess->exitCode() !== 0) { + throw new \RuntimeException('Failed to establish multiplexed connection: ' . $establishProcess->errorOutput()); + } + + $muxContent = "Multiplexed connection established at " . now()->toDateTimeString(); + Storage::disk('ssh-mux')->put(basename($muxSocket), $muxContent); + + self::$ensuredConnections[$server->id] = [ + 'timestamp' => now(), + 'muxSocket' => $muxSocket, + ]; + } + + public static function shouldResetMultiplexedConnection(Server $server) + { + if (!(config('constants.ssh.mux_enabled') && config('coolify.is_windows_docker_desktop') == false)) { + return false; + } + + if (!isset(self::$ensuredConnections[$server->id])) { + return true; + } + + $lastEnsured = self::$ensuredConnections[$server->id]['timestamp']; + $muxPersistTime = config('constants.ssh.mux_persist_time'); + $resetInterval = strtotime($muxPersistTime) - time(); + + return $lastEnsured->addSeconds($resetInterval)->isPast(); + } + + public static function removeMuxFile(Server $server) + { + $sshConfig = self::serverSshConfiguration($server); + $muxFilename = basename($sshConfig['muxFilename']); + + $closeCommand = "ssh -O exit -o ControlPath=/var/www/html/storage/app/ssh/mux/{$muxFilename} {$server->user}@{$server->ip}"; + Process::run($closeCommand); + + Storage::disk('ssh-mux')->delete($muxFilename); + } + + public static function generateScpCommand(Server $server, string $source, string $dest) + { + $sshConfig = self::serverSshConfiguration($server); + $sshKeyLocation = $sshConfig['sshKeyLocation']; + $muxSocket = $sshConfig['muxFilename']; + + $user = $server->user; + $port = $server->port; + $timeout = config('constants.ssh.command_timeout'); + $connectionTimeout = config('constants.ssh.connection_timeout'); + $serverInterval = config('constants.ssh.server_interval'); + $muxPersistTime = config('constants.ssh.mux_persist_time'); + + $scp_command = "timeout $timeout scp "; + $muxEnabled = config('constants.ssh.mux_enabled', true) && config('coolify.is_windows_docker_desktop') == false; + ray('SSH Multiplexing Enabled:', $muxEnabled)->blue(); + + if ($muxEnabled) { + $scp_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; + self::ensureMultiplexedConnection($server); + ray('Using SSH Multiplexing')->green(); + } else { + ray('Not using SSH Multiplexing')->red(); + } + + if (data_get($server, 'settings.is_cloudflare_tunnel')) { + $scp_command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" '; + } + $scp_command .= "-i {$sshKeyLocation} " + .'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ' + .'-o PasswordAuthentication=no ' + ."-o ConnectTimeout=$connectionTimeout " + ."-o ServerAliveInterval=$serverInterval " + .'-o RequestTTY=no ' + .'-o LogLevel=ERROR ' + ."-P {$port} " + ."{$source} " + ."{$user}@{$server->ip}:{$dest}"; + + return $scp_command; + } + + public static function generateSshCommand(Server $server, string $command) + { + if ($server->settings->force_disabled) { + throw new \RuntimeException('Server is disabled.'); + } + + $sshConfig = self::serverSshConfiguration($server); + $sshKeyLocation = $sshConfig['sshKeyLocation']; + $muxSocket = $sshConfig['muxFilename']; + + $timeout = config('constants.ssh.command_timeout'); + $connectionTimeout = config('constants.ssh.connection_timeout'); + $serverInterval = config('constants.ssh.server_interval'); + $muxPersistTime = config('constants.ssh.mux_persist_time'); + $muxEnabled = config('constants.ssh.mux_enabled') && !config('coolify.is_windows_docker_desktop'); + ray('Config MUX Enabled:', config('constants.ssh.mux_enabled')); + ray('Config Windows Docker Desktop:', config('coolify.is_windows_docker_desktop')); + ray('MUX Enabled:', $muxEnabled); + + $ssh_command = "timeout $timeout ssh "; + + ray('SSH Multiplexing Enabled:', $muxEnabled)->blue(); + + if ($muxEnabled) { + $ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; + self::ensureMultiplexedConnection($server); + ray('Using SSH Multiplexing')->green(); + } else { + ray('Not using SSH Multiplexing')->red(); + } + + if (data_get($server, 'settings.is_cloudflare_tunnel')) { + $ssh_command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" '; + } + + $command = "PATH=\$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/local/sbin:/host/usr/local/bin:/host/usr/sbin:/host/usr/bin:/host/sbin:/host/bin && $command"; + $delimiter = Hash::make($command); + $command = str_replace($delimiter, '', $command); + + $ssh_command .= "-i {$sshKeyLocation} " + .'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ' + .'-o PasswordAuthentication=no ' + ."-o ConnectTimeout=$connectionTimeout " + ."-o ServerAliveInterval=$serverInterval " + .'-o RequestTTY=no ' + .'-o LogLevel=ERROR ' + ."-p {$server->port} " + ."{$server->user}@{$server->ip} " + ." 'bash -se' << \\$delimiter".PHP_EOL + .$command.PHP_EOL + .$delimiter; + + return $ssh_command; + } +} \ No newline at end of file diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index 0d21c20a0e..988deaee3b 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -3,6 +3,7 @@ use App\Actions\CoolifyTask\PrepareCoolifyTask; use App\Data\CoolifyTaskArgs; use App\Enums\ActivityTypes; +use App\Helpers\SshMultiplexingHelper; use App\Models\Application; use App\Models\ApplicationDeploymentQueue; use App\Models\PrivateKey; @@ -10,13 +11,11 @@ use Carbon\Carbon; use Illuminate\Database\Eloquent\Model; use Illuminate\Support\Collection; -use Illuminate\Support\Facades\Hash; use Illuminate\Support\Facades\Process; -use Illuminate\Support\Facades\Storage; +use Illuminate\Support\Facades\Auth; use Illuminate\Support\Str; use Spatie\Activitylog\Contracts\Activity; - function remote_process( Collection|array $command, Server $server, @@ -43,6 +42,8 @@ function remote_process( } } + SshMultiplexingHelper::ensureMultiplexedConnection($server); + return resolve(PrepareCoolifyTask::class, [ 'remoteProcessArgs' => new CoolifyTaskArgs( server_uuid: $server->uuid, @@ -57,58 +58,9 @@ function remote_process( ])(); } -function server_ssh_configuration(Server $server) -{ - $sshKeyLocation = $server->privateKey->getKeyLocation(); - ray($sshKeyLocation); - $mux_filename = '/var/www/html/storage/app/ssh/mux/'.$server->muxFilename(); - - return [ - 'sshKeyLocation' => $sshKeyLocation, - 'muxFilename' => $mux_filename, - ]; -} - function generateScpCommand(Server $server, string $source, string $dest) { - $sshConfig = server_ssh_configuration($server); - $sshKeyLocation = $sshConfig['sshKeyLocation']; - $muxSocket = $sshConfig['muxFilename']; - - $user = $server->user; - $port = $server->port; - $timeout = config('constants.ssh.command_timeout'); - $connectionTimeout = config('constants.ssh.connection_timeout'); - $serverInterval = config('constants.ssh.server_interval'); - $muxPersistTime = config('constants.ssh.mux_persist_time'); - - $scp_command = "timeout $timeout scp "; - $muxEnabled = config('constants.ssh.mux_enabled', true) && config('coolify.is_windows_docker_desktop') == false; - ray('SSH Multiplexing Enabled:', $muxEnabled)->blue(); - - if ($muxEnabled) { - $scp_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; - ensureMultiplexedConnection($server); - ray('Using SSH Multiplexing')->green(); - } else { - ray('Not using SSH Multiplexing')->red(); - } - - if (data_get($server, 'settings.is_cloudflare_tunnel')) { - $scp_command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" '; - } - $scp_command .= "-i {$sshKeyLocation} " - .'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ' - .'-o PasswordAuthentication=no ' - ."-o ConnectTimeout=$connectionTimeout " - ."-o ServerAliveInterval=$serverInterval " - .'-o RequestTTY=no ' - .'-o LogLevel=ERROR ' - ."-P {$port} " - ."{$source} " - ."{$user}@{$server->ip}:{$dest}"; - - return $scp_command; + return SshMultiplexingHelper::generateScpCommand($server, $source, $dest); } function instant_scp(string $source, string $dest, Server $server, $throwError = true) @@ -134,139 +86,7 @@ function instant_scp(string $source, string $dest, Server $server, $throwError = function generateSshCommand(Server $server, string $command) { - if ($server->settings->force_disabled) { - throw new \RuntimeException('Server is disabled.'); - } - - $sshConfig = server_ssh_configuration($server); - $sshKeyLocation = $sshConfig['sshKeyLocation']; - $muxSocket = $sshConfig['muxFilename']; - - $timeout = config('constants.ssh.command_timeout'); - $connectionTimeout = config('constants.ssh.connection_timeout'); - $serverInterval = config('constants.ssh.server_interval'); - $muxPersistTime = config('constants.ssh.mux_persist_time'); - $muxEnabled = config('constants.ssh.mux_enabled') && !config('coolify.is_windows_docker_desktop'); - - $ssh_command = "timeout $timeout ssh "; - - ray('SSH Multiplexing Enabled:', $muxEnabled)->blue(); - - if ($muxEnabled) { - $ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; - ensureMultiplexedConnection($server); - ray('Using SSH Multiplexing')->green(); - } else { - ray('Not using SSH Multiplexing')->red(); - } - - if (data_get($server, 'settings.is_cloudflare_tunnel')) { - $ssh_command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" '; - } - - $command = "PATH=\$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/local/sbin:/host/usr/local/bin:/host/usr/sbin:/host/usr/bin:/host/sbin:/host/bin && $command"; - $delimiter = Hash::make($command); - $command = str_replace($delimiter, '', $command); - - $ssh_command .= "-i {$sshKeyLocation} " - .'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ' - .'-o PasswordAuthentication=no ' - ."-o ConnectTimeout=$connectionTimeout " - ."-o ServerAliveInterval=$serverInterval " - .'-o RequestTTY=no ' - .'-o LogLevel=ERROR ' - ."-p {$server->port} " - ."{$server->user}@{$server->ip} " - ." 'bash -se' << \\$delimiter".PHP_EOL - .$command.PHP_EOL - .$delimiter; - - return $ssh_command; -} - -function ensureMultiplexedConnection(Server $server) -{ - static $ensuredConnections = []; - - $sshConfig = server_ssh_configuration($server); - $muxSocket = $sshConfig['muxFilename']; - - if (isset($ensuredConnections[$server->id]) && !shouldResetMultiplexedConnection($server)) { - return; - } - - $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip}"; - $process = Process::run($checkCommand); - - if ($process->exitCode() === 0) { - $ensuredConnections[$server->id] = [ - 'timestamp' => now(), - 'muxSocket' => $muxSocket, - ]; - return; - } - - $sshKeyLocation = $sshConfig['sshKeyLocation']; - $connectionTimeout = config('constants.ssh.connection_timeout'); - $serverInterval = config('constants.ssh.server_interval'); - $muxPersistTime = config('constants.ssh.mux_persist_time'); - - $establishCommand = "ssh -fNM -o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} " - ."-i {$sshKeyLocation} " - .'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ' - .'-o PasswordAuthentication=no ' - ."-o ConnectTimeout=$connectionTimeout " - ."-o ServerAliveInterval=$serverInterval " - .'-o RequestTTY=no ' - .'-o LogLevel=ERROR ' - ."-p {$server->port} " - ."{$server->user}@{$server->ip}"; - - $establishProcess = Process::run($establishCommand); - - if ($establishProcess->exitCode() !== 0) { - throw new \RuntimeException('Failed to establish multiplexed connection: '.$establishProcess->errorOutput()); - } - - $ensuredConnections[$server->id] = [ - 'timestamp' => now(), - 'muxSocket' => $muxSocket, - ]; -} - -function shouldResetMultiplexedConnection(Server $server) -{ - if (! (config('constants.ssh.mux_enabled') && config('coolify.is_windows_docker_desktop') == false)) { - return false; - } - - static $ensuredConnections = []; - - if (! isset($ensuredConnections[$server->id])) { - return true; - } - - $lastEnsured = $ensuredConnections[$server->id]['timestamp']; - $muxPersistTime = config('constants.ssh.mux_persist_time'); - $resetInterval = strtotime($muxPersistTime) - time(); - - return $lastEnsured->addSeconds($resetInterval)->isPast(); -} - -function resetMultiplexedConnection(Server $server) -{ - if (! (config('constants.ssh.mux_enabled') && config('coolify.is_windows_docker_desktop') == false)) { - return; - } - - static $ensuredConnections = []; - - if (isset($ensuredConnections[$server->id])) { - $muxSocket = $ensuredConnections[$server->id]['muxSocket']; - $closeCommand = "ssh -O exit -o ControlPath=$muxSocket {$server->user}@{$server->ip}"; - Process::run($closeCommand); - unset($ensuredConnections[$server->id]); - } + return SshMultiplexingHelper::generateSshCommand($server, $command); } function instant_remote_process(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false): ?string @@ -402,18 +222,9 @@ function remove_iip($text) return preg_replace('/\x1b\[[0-9;]*m/', '', $text); } -function remove_mux_and_private_key(Server $server) +function remove_mux_file(Server $server) { - $sshConfig = server_ssh_configuration($server); - $privateKeyLocation = $sshConfig['sshKeyLocation']; - $muxFilename = basename($sshConfig['muxFilename']); - - - $closeCommand = "ssh -O exit -o ControlPath=/var/www/html/storage/app/ssh/mux/{$muxFilename} {$server->user}@{$server->ip}"; - Process::run($closeCommand); - - Storage::disk('ssh-mux')->delete($muxFilename); - Storage::disk('ssh-keys')->delete($privateKeyLocation); + SshMultiplexingHelper::removeMuxFile($server); } function refresh_server_connection(?PrivateKey $private_key = null) @@ -422,11 +233,7 @@ function refresh_server_connection(?PrivateKey $private_key = null) return; } foreach ($private_key->servers as $server) { - $sshConfig = server_ssh_configuration($server); - $muxFilename = basename($sshConfig['muxFilename']); - $closeCommand = "ssh -O exit -o ControlPath=/var/www/html/storage/app/ssh/mux/{$muxFilename} {$server->user}@{$server->ip}"; - Process::run($closeCommand); - Storage::disk('ssh-mux')->delete($muxFilename); + SshMultiplexingHelper::removeMuxFile($server); } } From 144508218e6339e8828d8a5001548b6e7e1b373b Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 17 Sep 2024 12:26:11 +0200 Subject: [PATCH 16/35] Fix: SSH multiplexing --- app/Actions/CoolifyTask/RunRemoteProcess.php | 3 +- app/Helpers/SshMultiplexingHelper.php | 126 +++++++++++-------- app/Jobs/ServerCheckJob.php | 7 +- app/Livewire/Project/Shared/GetLogs.php | 9 +- app/Models/Server.php | 7 +- app/Traits/ExecuteRemoteCommand.php | 3 +- bootstrap/helpers/remoteProcess.php | 26 +--- 7 files changed, 92 insertions(+), 89 deletions(-) diff --git a/app/Actions/CoolifyTask/RunRemoteProcess.php b/app/Actions/CoolifyTask/RunRemoteProcess.php index 63e3afe2f9..a5dfa92268 100644 --- a/app/Actions/CoolifyTask/RunRemoteProcess.php +++ b/app/Actions/CoolifyTask/RunRemoteProcess.php @@ -10,6 +10,7 @@ use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Process; use Spatie\Activitylog\Models\Activity; +use App\Helpers\SshMultiplexingHelper; class RunRemoteProcess { @@ -137,7 +138,7 @@ protected function getCommand(): string $command = $this->activity->getExtraProperty('command'); $server = Server::whereUuid($server_uuid)->firstOrFail(); - return generateSshCommand($server, $command); + return SshMultiplexingHelper::generateSshCommand($server, $command); } protected function handleOutput(string $type, string $output) diff --git a/app/Helpers/SshMultiplexingHelper.php b/app/Helpers/SshMultiplexingHelper.php index 8b39d61e82..077bd68db0 100644 --- a/app/Helpers/SshMultiplexingHelper.php +++ b/app/Helpers/SshMultiplexingHelper.php @@ -3,11 +3,10 @@ namespace App\Helpers; use App\Models\Server; +use App\Models\PrivateKey; use Illuminate\Support\Facades\Process; use Illuminate\Support\Facades\Storage; use Illuminate\Support\Facades\Hash; -use Illuminate\Support\Str; -use App\Models\PrivateKey; class SshMultiplexingHelper { @@ -15,7 +14,8 @@ class SshMultiplexingHelper public static function serverSshConfiguration(Server $server) { - $sshKeyLocation = $server->privateKey->getKeyLocation(); + $privateKey = PrivateKey::findOrFail($server->private_key_id); + $sshKeyLocation = $privateKey->getKeyLocation(); $muxFilename = '/var/www/html/storage/app/ssh/mux/' . $server->muxFilename(); return [ @@ -26,15 +26,21 @@ public static function serverSshConfiguration(Server $server) public static function ensureMultiplexedConnection(Server $server) { + if (!self::isMultiplexingEnabled()) { + ray('Multiplexing is disabled'); + return; + } + + ray('Ensuring multiplexed connection for server: ' . $server->id); + $sshConfig = self::serverSshConfiguration($server); $muxSocket = $sshConfig['muxFilename']; $sshKeyLocation = $sshConfig['sshKeyLocation']; - if (!file_exists($sshKeyLocation)) { - throw new \RuntimeException("SSH key file not accessible: $sshKeyLocation"); - } + self::validateSshKey($sshKeyLocation); if (isset(self::$ensuredConnections[$server->id]) && !self::shouldResetMultiplexedConnection($server)) { + ray('Existing connection is still valid'); return; } @@ -42,6 +48,7 @@ public static function ensureMultiplexedConnection(Server $server) $fileCheckProcess = Process::run($checkFileCommand); if ($fileCheckProcess->exitCode() !== 0) { + ray('Mux socket file not found, establishing new connection'); self::establishNewMultiplexedConnection($server); return; } @@ -49,19 +56,22 @@ public static function ensureMultiplexedConnection(Server $server) $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip}"; $process = Process::run($checkCommand); - if ($process->exitCode() === 0) { + if ($process->exitCode() !== 0) { + ray('Existing connection check failed, establishing new connection'); + self::establishNewMultiplexedConnection($server); + } else { + ray('Existing connection is valid'); self::$ensuredConnections[$server->id] = [ 'timestamp' => now(), 'muxSocket' => $muxSocket, ]; - return; } - - self::establishNewMultiplexedConnection($server); } public static function establishNewMultiplexedConnection(Server $server) { + ray('Establishing new multiplexed connection for server: ' . $server->id); + $sshConfig = self::serverSshConfiguration($server); $sshKeyLocation = $sshConfig['sshKeyLocation']; $muxSocket = $sshConfig['muxFilename']; @@ -84,9 +94,12 @@ public static function establishNewMultiplexedConnection(Server $server) $establishProcess = Process::run($establishCommand); if ($establishProcess->exitCode() !== 0) { + ray('Failed to establish multiplexed connection', $establishProcess->errorOutput()); throw new \RuntimeException('Failed to establish multiplexed connection: ' . $establishProcess->errorOutput()); } + ray('Multiplexed connection established successfully'); + $muxContent = "Multiplexed connection established at " . now()->toDateTimeString(); Storage::disk('ssh-mux')->put(basename($muxSocket), $muxContent); @@ -99,6 +112,7 @@ public static function establishNewMultiplexedConnection(Server $server) public static function shouldResetMultiplexedConnection(Server $server) { if (!(config('constants.ssh.mux_enabled') && config('coolify.is_windows_docker_desktop') == false)) { + ray('Multiplexing is disabled or running on Windows Docker Desktop'); return false; } @@ -110,7 +124,9 @@ public static function shouldResetMultiplexedConnection(Server $server) $muxPersistTime = config('constants.ssh.mux_persist_time'); $resetInterval = strtotime($muxPersistTime) - time(); - return $lastEnsured->addSeconds($resetInterval)->isPast(); + $shouldReset = $lastEnsured->addSeconds($resetInterval)->isPast(); + ray('Should reset multiplexed connection', ['server_id' => $server->id, 'should_reset' => $shouldReset]); + return $shouldReset; } public static function removeMuxFile(Server $server) @@ -130,38 +146,22 @@ public static function generateScpCommand(Server $server, string $source, string $sshKeyLocation = $sshConfig['sshKeyLocation']; $muxSocket = $sshConfig['muxFilename']; - $user = $server->user; - $port = $server->port; $timeout = config('constants.ssh.command_timeout'); $connectionTimeout = config('constants.ssh.connection_timeout'); $serverInterval = config('constants.ssh.server_interval'); - $muxPersistTime = config('constants.ssh.mux_persist_time'); $scp_command = "timeout $timeout scp "; - $muxEnabled = config('constants.ssh.mux_enabled', true) && config('coolify.is_windows_docker_desktop') == false; - ray('SSH Multiplexing Enabled:', $muxEnabled)->blue(); - if ($muxEnabled) { + if (self::isMultiplexingEnabled()) { + $muxPersistTime = config('constants.ssh.mux_persist_time'); $scp_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; self::ensureMultiplexedConnection($server); - ray('Using SSH Multiplexing')->green(); - } else { - ray('Not using SSH Multiplexing')->red(); } - if (data_get($server, 'settings.is_cloudflare_tunnel')) { - $scp_command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" '; - } - $scp_command .= "-i {$sshKeyLocation} " - .'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ' - .'-o PasswordAuthentication=no ' - ."-o ConnectTimeout=$connectionTimeout " - ."-o ServerAliveInterval=$serverInterval " - .'-o RequestTTY=no ' - .'-o LogLevel=ERROR ' - ."-P {$port} " - ."{$source} " - ."{$user}@{$server->ip}:{$dest}"; + self::addCloudflareProxyCommand($scp_command, $server); + + $scp_command .= self::getCommonSshOptions($server, $sshKeyLocation, $connectionTimeout, $serverInterval); + $scp_command .= "{$source} {$server->user}@{$server->ip}:{$dest}"; return $scp_command; } @@ -179,45 +179,61 @@ public static function generateSshCommand(Server $server, string $command) $timeout = config('constants.ssh.command_timeout'); $connectionTimeout = config('constants.ssh.connection_timeout'); $serverInterval = config('constants.ssh.server_interval'); - $muxPersistTime = config('constants.ssh.mux_persist_time'); - $muxEnabled = config('constants.ssh.mux_enabled') && !config('coolify.is_windows_docker_desktop'); - ray('Config MUX Enabled:', config('constants.ssh.mux_enabled')); - ray('Config Windows Docker Desktop:', config('coolify.is_windows_docker_desktop')); - ray('MUX Enabled:', $muxEnabled); $ssh_command = "timeout $timeout ssh "; - ray('SSH Multiplexing Enabled:', $muxEnabled)->blue(); - - if ($muxEnabled) { + if (self::isMultiplexingEnabled()) { + $muxPersistTime = config('constants.ssh.mux_persist_time'); $ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; self::ensureMultiplexedConnection($server); - ray('Using SSH Multiplexing')->green(); - } else { - ray('Not using SSH Multiplexing')->red(); } - if (data_get($server, 'settings.is_cloudflare_tunnel')) { - $ssh_command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" '; - } + self::addCloudflareProxyCommand($ssh_command, $server); + + $ssh_command .= self::getCommonSshOptions($server, $sshKeyLocation, $connectionTimeout, $serverInterval); $command = "PATH=\$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/local/sbin:/host/usr/local/bin:/host/usr/sbin:/host/usr/bin:/host/sbin:/host/bin && $command"; $delimiter = Hash::make($command); $command = str_replace($delimiter, '', $command); - $ssh_command .= "-i {$sshKeyLocation} " + $ssh_command .= "{$server->user}@{$server->ip} 'bash -se' << \\$delimiter".PHP_EOL + .$command.PHP_EOL + .$delimiter; + + return $ssh_command; + } + + private static function isMultiplexingEnabled(): bool + { + return config('constants.ssh.mux_enabled') && !config('coolify.is_windows_docker_desktop'); + } + + private static function validateSshKey(string $sshKeyLocation): void + { + $checkKeyCommand = "ls $sshKeyLocation 2>/dev/null"; + $keyCheckProcess = Process::run($checkKeyCommand); + + if ($keyCheckProcess->exitCode() !== 0) { + throw new \RuntimeException("SSH key file not accessible: $sshKeyLocation"); + } + } + + private static function addCloudflareProxyCommand(string &$command, Server $server): void + { + if (data_get($server, 'settings.is_cloudflare_tunnel')) { + $command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" '; + } + } + + private static function getCommonSshOptions(Server $server, string $sshKeyLocation, int $connectionTimeout, int $serverInterval): string + { + return "-i {$sshKeyLocation} " .'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ' .'-o PasswordAuthentication=no ' ."-o ConnectTimeout=$connectionTimeout " ."-o ServerAliveInterval=$serverInterval " .'-o RequestTTY=no ' .'-o LogLevel=ERROR ' - ."-p {$server->port} " - ."{$server->user}@{$server->ip} " - ." 'bash -se' << \\$delimiter".PHP_EOL - .$command.PHP_EOL - .$delimiter; - - return $ssh_command; + ."-p {$server->port} "; } } \ No newline at end of file diff --git a/app/Jobs/ServerCheckJob.php b/app/Jobs/ServerCheckJob.php index 5400853851..ed31a8c8f3 100644 --- a/app/Jobs/ServerCheckJob.php +++ b/app/Jobs/ServerCheckJob.php @@ -43,7 +43,7 @@ public function backoff(): int return isDev() ? 1 : 3; } - public function __construct(public Server $server) {} + public function __construct(public Server $server, public bool $isManualCheck = false) {} public function middleware(): array { @@ -58,6 +58,9 @@ public function uniqueId(): int public function handle() { try { + // Enable SSH multiplexing for autonomous checks, disable for manual checks + config()->set('constants.ssh.mux_enabled', !$this->isManualCheck); + $this->applications = $this->server->applications(); $this->databases = $this->server->databases(); $this->services = $this->server->services()->get(); @@ -93,7 +96,7 @@ public function handle() private function serverStatus() { - ['uptime' => $uptime] = $this->server->validateConnection(); + ['uptime' => $uptime] = $this->server->validateConnection($this->isManualCheck); if ($uptime) { if ($this->server->unreachable_notification_sent === true) { $this->server->update(['unreachable_notification_sent' => false]); diff --git a/app/Livewire/Project/Shared/GetLogs.php b/app/Livewire/Project/Shared/GetLogs.php index deccc875cd..b48ee7e231 100644 --- a/app/Livewire/Project/Shared/GetLogs.php +++ b/app/Livewire/Project/Shared/GetLogs.php @@ -17,6 +17,7 @@ use App\Models\StandaloneRedis; use Illuminate\Support\Facades\Process; use Livewire\Component; +use App\Helpers\SshMultiplexingHelper; class GetLogs extends Component { @@ -108,14 +109,14 @@ public function getLogs($refresh = false) $command = parseCommandsByLineForSudo(collect($command), $this->server); $command = $command[0]; } - $sshCommand = generateSshCommand($this->server, $command); + $sshCommand = SshMultiplexingHelper::generateSshCommand($this->server, $command); } else { $command = "docker logs -n {$this->numberOfLines} -t {$this->container}"; if ($this->server->isNonRoot()) { $command = parseCommandsByLineForSudo(collect($command), $this->server); $command = $command[0]; } - $sshCommand = generateSshCommand($this->server, $command); + $sshCommand = SshMultiplexingHelper::generateSshCommand($this->server, $command); } } else { if ($this->server->isSwarm()) { @@ -124,14 +125,14 @@ public function getLogs($refresh = false) $command = parseCommandsByLineForSudo(collect($command), $this->server); $command = $command[0]; } - $sshCommand = generateSshCommand($this->server, $command); + $sshCommand = SshMultiplexingHelper::generateSshCommand($this->server, $command); } else { $command = "docker logs -n {$this->numberOfLines} {$this->container}"; if ($this->server->isNonRoot()) { $command = parseCommandsByLineForSudo(collect($command), $this->server); $command = $command[0]; } - $sshCommand = generateSshCommand($this->server, $command); + $sshCommand = SshMultiplexingHelper::generateSshCommand($this->server, $command); } } if ($refresh) { diff --git a/app/Models/Server.php b/app/Models/Server.php index cae910257f..6eb9acf554 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -967,9 +967,10 @@ public function isSwarmWorker() return data_get($this, 'settings.is_swarm_worker'); } - public function validateConnection() + public function validateConnection($isManualCheck = true) { - config()->set('constants.ssh.mux_enabled', false); + // Set mux_enabled to true for automatic checks, false for manual checks + config()->set('constants.ssh.mux_enabled', !$isManualCheck); $server = Server::find($this->id); if (! $server) { @@ -979,7 +980,6 @@ public function validateConnection() return ['uptime' => false, 'error' => 'Server skipped.']; } try { - // EC2 does not have `uptime` command, lol instant_remote_process(['ls /'], $server); $server->settings()->update([ 'is_reachable' => true, @@ -988,7 +988,6 @@ public function validateConnection() 'unreachable_count' => 0, ]); if (data_get($server, 'unreachable_notification_sent') === true) { - // $server->team?->notify(new Revived($server)); $server->update(['unreachable_notification_sent' => false]); } diff --git a/app/Traits/ExecuteRemoteCommand.php b/app/Traits/ExecuteRemoteCommand.php index 9b58882eb1..03726f0958 100644 --- a/app/Traits/ExecuteRemoteCommand.php +++ b/app/Traits/ExecuteRemoteCommand.php @@ -7,6 +7,7 @@ use Carbon\Carbon; use Illuminate\Support\Collection; use Illuminate\Support\Facades\Process; +use App\Helpers\SshMultiplexingHelper; trait ExecuteRemoteCommand { @@ -42,7 +43,7 @@ public function execute_remote_command(...$commands) $command = parseLineForSudo($command, $this->server); } } - $remote_command = generateSshCommand($this->server, $command); + $remote_command = SshMultiplexingHelper::generateSshCommand($this->server, $command); $process = Process::timeout(3600)->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($command, $hidden, $customType, $append) { $output = str($output)->trim(); if ($output->startsWith('â•”')) { diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index 988deaee3b..5263ea9703 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -35,8 +35,8 @@ function remote_process( $command_string = implode("\n", $command); - if (auth()->user()) { - $teams = auth()->user()->teams->pluck('id'); + if (Auth::check()) { + $teams = Auth::user()->teams->pluck('id'); if (!$teams->contains($server->team_id) && !$teams->contains(0)) { throw new \Exception('User is not part of the team that owns this server'); } @@ -58,15 +58,10 @@ function remote_process( ])(); } -function generateScpCommand(Server $server, string $source, string $dest) -{ - return SshMultiplexingHelper::generateScpCommand($server, $source, $dest); -} - function instant_scp(string $source, string $dest, Server $server, $throwError = true) { $timeout = config('constants.ssh.command_timeout'); - $scp_command = generateScpCommand($server, $source, $dest); + $scp_command = SshMultiplexingHelper::generateScpCommand($server, $source, $dest); $process = Process::timeout($timeout)->run($scp_command); $output = trim($process->output()); $exitCode = $process->exitCode(); @@ -84,16 +79,8 @@ function instant_scp(string $source, string $dest, Server $server, $throwError = return $output; } -function generateSshCommand(Server $server, string $command) -{ - return SshMultiplexingHelper::generateSshCommand($server, $command); -} - function instant_remote_process(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false): ?string { - static $processCount = 0; - $processCount++; - $timeout = config('constants.ssh.command_timeout'); if ($command instanceof Collection) { $command = $command->toArray(); @@ -104,7 +91,7 @@ function instant_remote_process(Collection|array $command, Server $server, bool $command_string = implode("\n", $command); $start_time = microtime(true); - $sshCommand = generateSshCommand($server, $command_string); + $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string); $process = Process::timeout($timeout)->run($sshCommand); $end_time = microtime(true); @@ -222,11 +209,6 @@ function remove_iip($text) return preg_replace('/\x1b\[[0-9;]*m/', '', $text); } -function remove_mux_file(Server $server) -{ - SshMultiplexingHelper::removeMuxFile($server); -} - function refresh_server_connection(?PrivateKey $private_key = null) { if (is_null($private_key)) { From 52c4994d44e835f582cbd13d7841443fcf47c7b2 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 17 Sep 2024 12:44:59 +0200 Subject: [PATCH 17/35] Feat: remove unused code form multiplexing --- app/Helpers/SshMultiplexingHelper.php | 47 +-------------------------- 1 file changed, 1 insertion(+), 46 deletions(-) diff --git a/app/Helpers/SshMultiplexingHelper.php b/app/Helpers/SshMultiplexingHelper.php index 077bd68db0..2b8e0ad137 100644 --- a/app/Helpers/SshMultiplexingHelper.php +++ b/app/Helpers/SshMultiplexingHelper.php @@ -10,8 +10,6 @@ class SshMultiplexingHelper { - protected static $ensuredConnections = []; - public static function serverSshConfiguration(Server $server) { $privateKey = PrivateKey::findOrFail($server->private_key_id); @@ -39,32 +37,14 @@ public static function ensureMultiplexedConnection(Server $server) self::validateSshKey($sshKeyLocation); - if (isset(self::$ensuredConnections[$server->id]) && !self::shouldResetMultiplexedConnection($server)) { - ray('Existing connection is still valid'); - return; - } - - $checkFileCommand = "ls $muxSocket 2>/dev/null"; - $fileCheckProcess = Process::run($checkFileCommand); - - if ($fileCheckProcess->exitCode() !== 0) { - ray('Mux socket file not found, establishing new connection'); - self::establishNewMultiplexedConnection($server); - return; - } - $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip}"; $process = Process::run($checkCommand); if ($process->exitCode() !== 0) { - ray('Existing connection check failed, establishing new connection'); + ray('Existing connection check failed or not found, establishing new connection'); self::establishNewMultiplexedConnection($server); } else { ray('Existing connection is valid'); - self::$ensuredConnections[$server->id] = [ - 'timestamp' => now(), - 'muxSocket' => $muxSocket, - ]; } } @@ -102,31 +82,6 @@ public static function establishNewMultiplexedConnection(Server $server) $muxContent = "Multiplexed connection established at " . now()->toDateTimeString(); Storage::disk('ssh-mux')->put(basename($muxSocket), $muxContent); - - self::$ensuredConnections[$server->id] = [ - 'timestamp' => now(), - 'muxSocket' => $muxSocket, - ]; - } - - public static function shouldResetMultiplexedConnection(Server $server) - { - if (!(config('constants.ssh.mux_enabled') && config('coolify.is_windows_docker_desktop') == false)) { - ray('Multiplexing is disabled or running on Windows Docker Desktop'); - return false; - } - - if (!isset(self::$ensuredConnections[$server->id])) { - return true; - } - - $lastEnsured = self::$ensuredConnections[$server->id]['timestamp']; - $muxPersistTime = config('constants.ssh.mux_persist_time'); - $resetInterval = strtotime($muxPersistTime) - time(); - - $shouldReset = $lastEnsured->addSeconds($resetInterval)->isPast(); - ray('Should reset multiplexed connection', ['server_id' => $server->id, 'should_reset' => $shouldReset]); - return $shouldReset; } public static function removeMuxFile(Server $server) From 95070ab48d371c75effac2636b13e1a21bb220fc Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 17 Sep 2024 12:57:06 +0200 Subject: [PATCH 18/35] Feat: SSH Key cleanup job --- app/Console/Kernel.php | 5 +++++ app/Jobs/CleanupSshKeysJob.php | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 app/Jobs/CleanupSshKeysJob.php diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index b960a4a8bd..d28a399b9b 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -7,6 +7,7 @@ use App\Jobs\CleanupStaleMultiplexedConnections; use App\Jobs\DatabaseBackupJob; use App\Jobs\DockerCleanupJob; +use App\Jobs\CleanupSshKeysJob; use App\Jobs\PullHelperImageJob; use App\Jobs\PullSentinelImageJob; use App\Jobs\PullTemplatesFromCDN; @@ -43,6 +44,8 @@ protected function schedule(Schedule $schedule): void $schedule->command('uploads:clear')->everyTwoMinutes(); $schedule->command('telescope:prune')->daily(); + + $schedule->job(new CleanupSshKeysJob)->weekly()->onOneServer(); } else { // Instance Jobs $schedule->command('horizon:snapshot')->everyFiveMinutes(); @@ -59,6 +62,8 @@ protected function schedule(Schedule $schedule): void $schedule->command('cleanup:database --yes')->daily(); $schedule->command('uploads:clear')->everyTwoMinutes(); + + $schedule->job(new CleanupSshKeysJob)->weekly()->onOneServer(); } } diff --git a/app/Jobs/CleanupSshKeysJob.php b/app/Jobs/CleanupSshKeysJob.php new file mode 100644 index 0000000000..05e3a8837e --- /dev/null +++ b/app/Jobs/CleanupSshKeysJob.php @@ -0,0 +1,26 @@ +subWeek(); + + PrivateKey::where('created_at', '<', $oneWeekAgo) + ->whereDoesntHave('gitSources') + ->whereDoesntHave('servers') + ->delete(); + } +} \ No newline at end of file From 2d8bda4fa6ef495039008f8857c9c6cc5859b20a Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 17 Sep 2024 13:06:50 +0200 Subject: [PATCH 19/35] Fix: Private key with ID 2 on dev --- app/Livewire/Security/PrivateKey/Show.php | 20 +++++++++++--------- app/Models/PrivateKey.php | 17 +++++++++++++++++ database/seeders/GitlabAppSeeder.php | 14 -------------- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/app/Livewire/Security/PrivateKey/Show.php b/app/Livewire/Security/PrivateKey/Show.php index 68e44d4082..14d2ed767d 100644 --- a/app/Livewire/Security/PrivateKey/Show.php +++ b/app/Livewire/Security/PrivateKey/Show.php @@ -36,18 +36,19 @@ public function mount() public function loadPublicKey() { $this->public_key = $this->private_key->getPublicKey(); + if ($this->public_key === 'Error loading private key') { + $this->dispatch('error', 'Failed to load public key. The private key may be invalid.'); + } } public function delete() { try { - if ($this->private_key->isEmpty()) { - $this->private_key->delete(); - currentTeam()->privateKeys = PrivateKey::where('team_id', currentTeam()->id)->get(); - - return redirect()->route('security.private-key.index'); - } - $this->dispatch('error', 'This private key is in use and cannot be deleted. Please delete all servers, applications, and GitHub/GitLab apps that use this private key before deleting it.'); + $this->private_key->safeDelete(); + currentTeam()->privateKeys = PrivateKey::where('team_id', currentTeam()->id)->get(); + return redirect()->route('security.private-key.index'); + } catch (\Exception $e) { + $this->dispatch('error', $e->getMessage()); } catch (\Throwable $e) { return handleError($e, $this); } @@ -56,8 +57,9 @@ public function delete() public function changePrivateKey() { try { - $this->private_key->private_key = formatPrivateKey($this->private_key->private_key); - $this->private_key->save(); + $this->private_key->updatePrivateKey([ + 'private_key' => formatPrivateKey($this->private_key->private_key) + ]); refresh_server_connection($this->private_key); $this->dispatch('success', 'Private key updated.'); } catch (\Throwable $e) { diff --git a/app/Models/PrivateKey.php b/app/Models/PrivateKey.php index ee1a6b808f..e683e08b14 100644 --- a/app/Models/PrivateKey.php +++ b/app/Models/PrivateKey.php @@ -178,4 +178,21 @@ public function isEmpty() && $this->githubApps()->count() === 0 && $this->gitlabApps()->count() === 0; } + + public function isInUse() + { + return $this->servers()->exists() + || $this->applications()->exists() + || $this->githubApps()->exists() + || $this->gitlabApps()->exists(); + } + + public function safeDelete() + { + if ($this->isInUse()) { + throw new \Exception('This private key is in use and cannot be deleted.'); + } + + $this->delete(); + } } diff --git a/database/seeders/GitlabAppSeeder.php b/database/seeders/GitlabAppSeeder.php index af63f2ed72..ec2b7ec5e1 100644 --- a/database/seeders/GitlabAppSeeder.php +++ b/database/seeders/GitlabAppSeeder.php @@ -20,19 +20,5 @@ public function run(): void 'is_public' => true, 'team_id' => 0, ]); - GitlabApp::create([ - 'id' => 2, - 'name' => 'coolify-laravel-development-private-gitlab', - 'api_url' => 'https://gitlab.com/api/v4', - 'html_url' => 'https://gitlab.com', - 'app_id' => 1234, - 'app_secret' => '1234', - 'oauth_id' => 1234, - 'deploy_key_id' => '1234', - 'public_key' => 'dfjasiourj', - 'webhook_token' => '4u3928u4y392', - 'private_key_id' => 2, - 'team_id' => 0, - ]); } } From 871d09bd96602f91f12187d5b8fc40efb743030d Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 17 Sep 2024 13:20:27 +0200 Subject: [PATCH 20/35] Feat: Move more functions to the PrivateKey Model --- app/Models/PrivateKey.php | 48 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/app/Models/PrivateKey.php b/app/Models/PrivateKey.php index e683e08b14..79f36ccf85 100644 --- a/app/Models/PrivateKey.php +++ b/app/Models/PrivateKey.php @@ -7,6 +7,7 @@ use Illuminate\Validation\ValidationException; use phpseclib3\Crypt\PublicKeyLoader; use DanHarrin\LivewireRateLimiting\WithRateLimiting; +use Illuminate\Support\Facades\DB; #[OA\Schema( description: 'Private Key model', @@ -33,6 +34,7 @@ class PrivateKey extends BaseModel 'private_key', 'is_git_related', 'team_id', + 'fingerprint', ]; protected $casts = [ @@ -49,6 +51,14 @@ protected static function booted() 'private_key' => ['The private key is invalid.'], ]); } + + $key->fingerprint = self::generateFingerprint($key->private_key); + + if (self::fingerprintExists($key->fingerprint, $key->id)) { + throw ValidationException::withMessages([ + 'private_key' => ['This private key already exists.'], + ]); + } }); static::deleted(function ($key) { @@ -195,4 +205,42 @@ public function safeDelete() $this->delete(); } + + private static function privateKeyExists($key) + { + $publicKey = self::extractPublicKeyFromPrivate($key->private_key); + if (!$publicKey) { + return false; + } + + $existingKey = DB::table('private_keys') + ->where('team_id', $key->team_id) + ->where('id', '!=', $key->id) + ->whereRaw('? = (SELECT public_key FROM private_keys WHERE id = private_keys.id)', [$publicKey]) + ->exists(); + + return $existingKey; + } + + public static function generateFingerprint($privateKey) + { + try { + $key = PublicKeyLoader::load($privateKey); + $publicKey = $key->getPublicKey(); + return $publicKey->getFingerprint('sha256'); + } catch (\Throwable $e) { + return null; + } + } + + private static function fingerprintExists($fingerprint, $excludeId = null) + { + $query = self::where('fingerprint', $fingerprint); + + if ($excludeId) { + $query->where('id', '!=', $excludeId); + } + + return $query->exists(); + } } From 1c78067386c4902eda9db6da31c83882c9ea5d4a Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 17 Sep 2024 13:20:48 +0200 Subject: [PATCH 21/35] Feat: Add ssh key fingerprint and generate one for existing keys --- ..._key_fingerprint_to_private_keys_table.php | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 database/migrations/2024_09_17_111226_add_ssh_key_fingerprint_to_private_keys_table.php diff --git a/database/migrations/2024_09_17_111226_add_ssh_key_fingerprint_to_private_keys_table.php b/database/migrations/2024_09_17_111226_add_ssh_key_fingerprint_to_private_keys_table.php new file mode 100644 index 0000000000..15f56c76b5 --- /dev/null +++ b/database/migrations/2024_09_17_111226_add_ssh_key_fingerprint_to_private_keys_table.php @@ -0,0 +1,31 @@ +string('fingerprint')->after('private_key')->unique(); + }); + + PrivateKey::whereNull('fingerprint')->each(function ($key) { + $fingerprint = PrivateKey::generateFingerprint($key->private_key); + if ($fingerprint) { + $key->fingerprint = $fingerprint; + $key->save(); + } + }); + } + + public function down() + { + Schema::table('private_keys', function (Blueprint $table) { + $table->dropColumn('fingerprint'); + }); + } +} From 43895419ffbf67a70d048f56616fd0c7b7a8698a Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 17 Sep 2024 13:45:05 +0200 Subject: [PATCH 22/35] Remove unused code --- app/Models/PrivateKey.php | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/app/Models/PrivateKey.php b/app/Models/PrivateKey.php index 79f36ccf85..f2fa9c9cf7 100644 --- a/app/Models/PrivateKey.php +++ b/app/Models/PrivateKey.php @@ -7,7 +7,6 @@ use Illuminate\Validation\ValidationException; use phpseclib3\Crypt\PublicKeyLoader; use DanHarrin\LivewireRateLimiting\WithRateLimiting; -use Illuminate\Support\Facades\DB; #[OA\Schema( description: 'Private Key model', @@ -206,22 +205,6 @@ public function safeDelete() $this->delete(); } - private static function privateKeyExists($key) - { - $publicKey = self::extractPublicKeyFromPrivate($key->private_key); - if (!$publicKey) { - return false; - } - - $existingKey = DB::table('private_keys') - ->where('team_id', $key->team_id) - ->where('id', '!=', $key->id) - ->whereRaw('? = (SELECT public_key FROM private_keys WHERE id = private_keys.id)', [$publicKey]) - ->exists(); - - return $existingKey; - } - public static function generateFingerprint($privateKey) { try { From ccbbfd89087d948f83d99d8006aada81dcf906fc Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 17 Sep 2024 14:03:31 +0200 Subject: [PATCH 23/35] Fix: ID issues on dev seeders --- database/seeders/GithubAppSeeder.php | 2 +- database/seeders/PrivateKeySeeder.php | 2 -- database/seeders/ServerSeeder.php | 3 +-- database/seeders/ServerSettingSeeder.php | 2 +- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/database/seeders/GithubAppSeeder.php b/database/seeders/GithubAppSeeder.php index 4aa5ec7537..2ece7a05b9 100644 --- a/database/seeders/GithubAppSeeder.php +++ b/database/seeders/GithubAppSeeder.php @@ -31,7 +31,7 @@ public function run(): void 'client_id' => 'Iv1.220e564d2b0abd8c', 'client_secret' => '116d1d80289f378410dd70ab4e4b81dd8d2c52b6', 'webhook_secret' => '326a47b49054f03288f800d81247ec9414d0abf3', - 'private_key_id' => 1, + 'private_key_id' => 2, 'team_id' => 0, ]); } diff --git a/database/seeders/PrivateKeySeeder.php b/database/seeders/PrivateKeySeeder.php index afc8a1e816..6b44d0867d 100644 --- a/database/seeders/PrivateKeySeeder.php +++ b/database/seeders/PrivateKeySeeder.php @@ -13,7 +13,6 @@ class PrivateKeySeeder extends Seeder public function run(): void { PrivateKey::create([ - 'id' => 0, 'team_id' => 0, 'name' => 'Testing Host Key', 'description' => 'This is a test docker container', @@ -28,7 +27,6 @@ public function run(): void ]); PrivateKey::create([ - 'id' => 1, 'team_id' => 0, 'name' => 'development-github-app', 'description' => 'This is the key for using the development GitHub app', diff --git a/database/seeders/ServerSeeder.php b/database/seeders/ServerSeeder.php index 12594bcb94..ff0746028f 100644 --- a/database/seeders/ServerSeeder.php +++ b/database/seeders/ServerSeeder.php @@ -10,12 +10,11 @@ class ServerSeeder extends Seeder public function run(): void { Server::create([ - 'id' => 0, 'name' => 'localhost', 'description' => 'This is a test docker container in development mode', 'ip' => 'coolify-testing-host', 'team_id' => 0, - 'private_key_id' => 0, + 'private_key_id' => 1, ]); } } diff --git a/database/seeders/ServerSettingSeeder.php b/database/seeders/ServerSettingSeeder.php index af4a2694b6..72b2313692 100644 --- a/database/seeders/ServerSettingSeeder.php +++ b/database/seeders/ServerSettingSeeder.php @@ -12,7 +12,7 @@ class ServerSettingSeeder extends Seeder */ public function run(): void { - $server_2 = Server::find(0)->load(['settings']); + $server_2 = Server::find(1)->load(['settings']); $server_2->settings->wildcard_domain = 'http://127.0.0.1.sslip.io'; $server_2->settings->is_build_server = false; $server_2->settings->is_usable = true; From 2ec66fd146d07aa133dac24f9a2149154b4dc45c Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 17 Sep 2024 14:08:34 +0200 Subject: [PATCH 24/35] FIx: Server ID 0 --- database/seeders/ServerSeeder.php | 1 + database/seeders/ServerSettingSeeder.php | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/database/seeders/ServerSeeder.php b/database/seeders/ServerSeeder.php index ff0746028f..197a0b5cb0 100644 --- a/database/seeders/ServerSeeder.php +++ b/database/seeders/ServerSeeder.php @@ -10,6 +10,7 @@ class ServerSeeder extends Seeder public function run(): void { Server::create([ + 'id' => 0, 'name' => 'localhost', 'description' => 'This is a test docker container in development mode', 'ip' => 'coolify-testing-host', diff --git a/database/seeders/ServerSettingSeeder.php b/database/seeders/ServerSettingSeeder.php index 72b2313692..af4a2694b6 100644 --- a/database/seeders/ServerSettingSeeder.php +++ b/database/seeders/ServerSettingSeeder.php @@ -12,7 +12,7 @@ class ServerSettingSeeder extends Seeder */ public function run(): void { - $server_2 = Server::find(1)->load(['settings']); + $server_2 = Server::find(0)->load(['settings']); $server_2->settings->wildcard_domain = 'http://127.0.0.1.sslip.io'; $server_2->settings->is_build_server = false; $server_2->settings->is_usable = true; From 6a6b947fbaeaeec6fa37b621110b344a2e3b5b1d Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 17 Sep 2024 14:32:44 +0200 Subject: [PATCH 25/35] Fix: Make sure in use private keys are not deleted --- app/Models/PrivateKey.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/Models/PrivateKey.php b/app/Models/PrivateKey.php index f2fa9c9cf7..fef464823c 100644 --- a/app/Models/PrivateKey.php +++ b/app/Models/PrivateKey.php @@ -144,6 +144,10 @@ public function storeInFileSystem() public static function deleteFromStorage(self $privateKey) { + if ($privateKey->isInUse()) { + throw new \Exception('Cannot delete a private key that is in use.'); + } + $filename = "ssh@{$privateKey->uuid}"; Storage::disk('ssh-keys')->delete($filename); } From 845d32c94ca3e77b7350e80df7c61b89be5fd864 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 17 Sep 2024 14:33:04 +0200 Subject: [PATCH 26/35] Fix: Do not delete SSH Key from disk during server validation error --- app/Models/Server.php | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app/Models/Server.php b/app/Models/Server.php index 6eb9acf554..0834bf61b7 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -912,12 +912,9 @@ public function isFunctional() $isFunctional = $this->settings->is_reachable && $this->settings->is_usable && !$this->settings->force_disabled; if (!$isFunctional) { - if ($this->privateKey) { - PrivateKey::deleteFromStorage($this->privateKey); - } Storage::disk('ssh-mux')->delete($this->muxFilename()); } - + return $isFunctional; } From bdc0fc87f0e5ad877b064ee55b2c86b3e671a3be Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 17 Sep 2024 14:33:24 +0200 Subject: [PATCH 27/35] Fix: UI bug, do not write ssh key to disk in server dialog --- app/Livewire/Server/ShowPrivateKey.php | 15 ++++----------- .../livewire/server/show-private-key.blade.php | 3 ++- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/app/Livewire/Server/ShowPrivateKey.php b/app/Livewire/Server/ShowPrivateKey.php index d5e20702d0..8e9b591572 100644 --- a/app/Livewire/Server/ShowPrivateKey.php +++ b/app/Livewire/Server/ShowPrivateKey.php @@ -18,18 +18,11 @@ public function setPrivateKey($privateKeyId) { try { $privateKey = PrivateKey::findOrFail($privateKeyId); - $this->server->update(['private_key_id' => $privateKeyId]); - $privateKey->storeInFileSystem(); - - $this->dispatch('notify', [ - 'type' => 'success', - 'message' => 'Private key updated and stored in the file system.', - ]); + $this->server->update(['private_key_id' => $privateKey->id]); + $this->server->refresh(); + $this->dispatch('success', 'Private key updated successfully.'); } catch (\Exception $e) { - $this->dispatch('notify', [ - 'type' => 'error', - 'message' => 'Failed to update private key: ' . $e->getMessage(), - ]); + $this->dispatch('error', 'Failed to update private key: ' . $e->getMessage()); } } diff --git a/resources/views/livewire/server/show-private-key.blade.php b/resources/views/livewire/server/show-private-key.blade.php index 62b1c4614a..86bf2568eb 100644 --- a/resources/views/livewire/server/show-private-key.blade.php +++ b/resources/views/livewire/server/show-private-key.blade.php @@ -26,7 +26,8 @@

Choose another Key

@forelse ($privateKeys as $private_key) -
+
{{ $private_key->name }}
{{ $private_key->description }}
From 4ac2758d700b8f25cd2af2569e98cce069e61435 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 17 Sep 2024 14:35:08 +0200 Subject: [PATCH 28/35] Update .env.development.example --- .env.development.example | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env.development.example b/.env.development.example index 3023a21a68..89d3af930c 100644 --- a/.env.development.example +++ b/.env.development.example @@ -6,7 +6,7 @@ APP_KEY= APP_URL=http://localhost APP_PORT=8000 APP_DEBUG=true -SSH_MUX_ENABLED=false +SSH_MUX_ENABLED=true # PostgreSQL Database Configuration DB_DATABASE=coolify @@ -21,7 +21,7 @@ RAY_ENABLED=false # Set custom ray port RAY_PORT= -# Clockwork Configuration +# Clockwork Configuration (remove?) CLOCKWORK_ENABLED=false CLOCKWORK_QUEUE_COLLECT=true From 2bc74c75e12bd103a15d3e4f2b854468b35d5a96 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 17 Sep 2024 14:43:02 +0200 Subject: [PATCH 29/35] Remove duplicated code --- app/Jobs/CleanupSshKeysJob.php | 9 +++++---- app/Models/PrivateKey.php | 22 +++++----------------- 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/app/Jobs/CleanupSshKeysJob.php b/app/Jobs/CleanupSshKeysJob.php index 05e3a8837e..485bf99eba 100644 --- a/app/Jobs/CleanupSshKeysJob.php +++ b/app/Jobs/CleanupSshKeysJob.php @@ -19,8 +19,9 @@ public function handle() $oneWeekAgo = Carbon::now()->subWeek(); PrivateKey::where('created_at', '<', $oneWeekAgo) - ->whereDoesntHave('gitSources') - ->whereDoesntHave('servers') - ->delete(); + ->get() + ->each(function ($privateKey) { + $privateKey->safeDelete(); + }); } -} \ No newline at end of file +} diff --git a/app/Models/PrivateKey.php b/app/Models/PrivateKey.php index fef464823c..05310e413f 100644 --- a/app/Models/PrivateKey.php +++ b/app/Models/PrivateKey.php @@ -143,11 +143,7 @@ public function storeInFileSystem() } public static function deleteFromStorage(self $privateKey) - { - if ($privateKey->isInUse()) { - throw new \Exception('Cannot delete a private key that is in use.'); - } - + { $filename = "ssh@{$privateKey->uuid}"; Storage::disk('ssh-keys')->delete($filename); } @@ -184,14 +180,6 @@ public function gitlabApps() return $this->hasMany(GitlabApp::class); } - public function isEmpty() - { - return $this->servers()->count() === 0 - && $this->applications()->count() === 0 - && $this->githubApps()->count() === 0 - && $this->gitlabApps()->count() === 0; - } - public function isInUse() { return $this->servers()->exists() @@ -202,11 +190,11 @@ public function isInUse() public function safeDelete() { - if ($this->isInUse()) { - throw new \Exception('This private key is in use and cannot be deleted.'); + if (!$this->isInUse()) { + $this->delete(); + return true; } - - $this->delete(); + return false; } public static function generateFingerprint($privateKey) From 175f4b9ae1dbbc4af2aa58e2c4fe8334af4b6354 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 17 Sep 2024 14:47:02 +0200 Subject: [PATCH 30/35] use shared functions when possible --- app/Models/PrivateKey.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/Models/PrivateKey.php b/app/Models/PrivateKey.php index 05310e413f..1ad12cf362 100644 --- a/app/Models/PrivateKey.php +++ b/app/Models/PrivateKey.php @@ -43,7 +43,7 @@ class PrivateKey extends BaseModel protected static function booted() { static::saving(function ($key) { - $key->private_key = rtrim($key->private_key) . "\n"; + $key->private_key = formatPrivateKey($key->private_key); if (!self::validatePrivateKey($key->private_key)) { throw ValidationException::withMessages([ @@ -101,13 +101,13 @@ public static function generateNewKeyPair($type = 'rsa') $instance->rateLimit(10); $name = generate_random_name(); $description = 'Created by Coolify'; - ['private' => $privateKey, 'public' => $publicKey] = generateSSHKey($type === 'ed25519' ? 'ed25519' : 'rsa'); + $keyPair = generateSSHKey($type === 'ed25519' ? 'ed25519' : 'rsa'); return [ 'name' => $name, 'description' => $description, - 'private_key' => $privateKey, - 'public_key' => $publicKey, + 'private_key' => $keyPair['private'], + 'public_key' => $keyPair['public'], ]; } catch (\Throwable $e) { throw new \Exception("Failed to generate new {$type} key: " . $e->getMessage()); From ea3501ada6abd06b96599ce8cb14433a74e10cc6 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 17 Sep 2024 15:31:05 +0200 Subject: [PATCH 31/35] Fix: SSH Multiplexing for Jobs --- app/Actions/Proxy/CheckProxy.php | 2 +- app/Helpers/SshMultiplexingHelper.php | 38 +++++++++++++++++++++------ app/Jobs/ServerCheckJob.php | 7 ++--- app/Models/Server.php | 2 +- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/app/Actions/Proxy/CheckProxy.php b/app/Actions/Proxy/CheckProxy.php index 735b972aff..cf0f6015cb 100644 --- a/app/Actions/Proxy/CheckProxy.php +++ b/app/Actions/Proxy/CheckProxy.php @@ -26,7 +26,7 @@ public function handle(Server $server, $fromUI = false) if (is_null($proxyType) || $proxyType === 'NONE' || $server->proxy->force_stop) { return false; } - ['uptime' => $uptime, 'error' => $error] = $server->validateConnection(); + ['uptime' => $uptime, 'error' => $error] = $server->validateConnection(false); if (! $uptime) { throw new \Exception($error); } diff --git a/app/Helpers/SshMultiplexingHelper.php b/app/Helpers/SshMultiplexingHelper.php index 2b8e0ad137..b1507bd001 100644 --- a/app/Helpers/SshMultiplexingHelper.php +++ b/app/Helpers/SshMultiplexingHelper.php @@ -25,11 +25,12 @@ public static function serverSshConfiguration(Server $server) public static function ensureMultiplexedConnection(Server $server) { if (!self::isMultiplexingEnabled()) { - ray('Multiplexing is disabled'); + ray('SSH Multiplexing: DISABLED')->red(); return; } - ray('Ensuring multiplexed connection for server: ' . $server->id); + ray('SSH Multiplexing: ENABLED')->green(); + ray('Ensuring multiplexed connection for server:', $server->id); $sshConfig = self::serverSshConfiguration($server); $muxSocket = $sshConfig['muxFilename']; @@ -41,16 +42,17 @@ public static function ensureMultiplexedConnection(Server $server) $process = Process::run($checkCommand); if ($process->exitCode() !== 0) { - ray('Existing connection check failed or not found, establishing new connection'); + ray('SSH Multiplexing: Existing connection check failed or not found')->orange(); + ray('Establishing new connection'); self::establishNewMultiplexedConnection($server); } else { - ray('Existing connection is valid'); + ray('SSH Multiplexing: Existing connection is valid')->green(); } } public static function establishNewMultiplexedConnection(Server $server) { - ray('Establishing new multiplexed connection for server: ' . $server->id); + ray('SSH Multiplexing: Establishing new connection for server:', $server->id); $sshConfig = self::serverSshConfiguration($server); $sshKeyLocation = $sshConfig['sshKeyLocation']; @@ -74,11 +76,11 @@ public static function establishNewMultiplexedConnection(Server $server) $establishProcess = Process::run($establishCommand); if ($establishProcess->exitCode() !== 0) { - ray('Failed to establish multiplexed connection', $establishProcess->errorOutput()); + ray('SSH Multiplexing: Failed to establish connection', $establishProcess->errorOutput())->red(); throw new \RuntimeException('Failed to establish multiplexed connection: ' . $establishProcess->errorOutput()); } - ray('Multiplexed connection established successfully'); + ray('SSH Multiplexing: Connection established successfully')->green(); $muxContent = "Multiplexed connection established at " . now()->toDateTimeString(); Storage::disk('ssh-mux')->put(basename($muxSocket), $muxContent); @@ -108,9 +110,18 @@ public static function generateScpCommand(Server $server, string $source, string $scp_command = "timeout $timeout scp "; if (self::isMultiplexingEnabled()) { + ray('SSH Multiplexing: Enabled for SCP command')->green(); $muxPersistTime = config('constants.ssh.mux_persist_time'); $scp_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; self::ensureMultiplexedConnection($server); + + // Add this line to verify multiplexing is being used + ray('SSH Multiplexing: Verifying usage')->blue(); + $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip}"; + $checkProcess = Process::run($checkCommand); + ray('SSH Multiplexing: ' . ($checkProcess->exitCode() === 0 ? 'Active' : 'Not Active'))->color($checkProcess->exitCode() === 0 ? 'green' : 'red'); + } else { + ray('SSH Multiplexing: Disabled for SCP command')->orange(); } self::addCloudflareProxyCommand($scp_command, $server); @@ -138,9 +149,18 @@ public static function generateSshCommand(Server $server, string $command) $ssh_command = "timeout $timeout ssh "; if (self::isMultiplexingEnabled()) { + ray('SSH Multiplexing: Enabled for SSH command')->green(); $muxPersistTime = config('constants.ssh.mux_persist_time'); $ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; self::ensureMultiplexedConnection($server); + + // Add this line to verify multiplexing is being used + ray('SSH Multiplexing: Verifying usage')->blue(); + $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip}"; + $checkProcess = Process::run($checkCommand); + ray('SSH Multiplexing: ' . ($checkProcess->exitCode() === 0 ? 'Active' : 'Not Active'))->color($checkProcess->exitCode() === 0 ? 'green' : 'red'); + } else { + ray('SSH Multiplexing: Disabled for SSH command')->orange(); } self::addCloudflareProxyCommand($ssh_command, $server); @@ -160,7 +180,9 @@ public static function generateSshCommand(Server $server, string $command) private static function isMultiplexingEnabled(): bool { - return config('constants.ssh.mux_enabled') && !config('coolify.is_windows_docker_desktop'); + $isEnabled = config('constants.ssh.mux_enabled') && !config('coolify.is_windows_docker_desktop'); + ray('SSH Multiplexing Status:', $isEnabled ? 'ENABLED' : 'DISABLED')->color($isEnabled ? 'green' : 'red'); + return $isEnabled; } private static function validateSshKey(string $sshKeyLocation): void diff --git a/app/Jobs/ServerCheckJob.php b/app/Jobs/ServerCheckJob.php index ed31a8c8f3..be5ec20f9a 100644 --- a/app/Jobs/ServerCheckJob.php +++ b/app/Jobs/ServerCheckJob.php @@ -43,7 +43,7 @@ public function backoff(): int return isDev() ? 1 : 3; } - public function __construct(public Server $server, public bool $isManualCheck = false) {} + public function __construct(public Server $server) {} public function middleware(): array { @@ -58,9 +58,6 @@ public function uniqueId(): int public function handle() { try { - // Enable SSH multiplexing for autonomous checks, disable for manual checks - config()->set('constants.ssh.mux_enabled', !$this->isManualCheck); - $this->applications = $this->server->applications(); $this->databases = $this->server->databases(); $this->services = $this->server->services()->get(); @@ -96,7 +93,7 @@ public function handle() private function serverStatus() { - ['uptime' => $uptime] = $this->server->validateConnection($this->isManualCheck); + ['uptime' => $uptime] = $this->server->validateConnection(false); if ($uptime) { if ($this->server->unreachable_notification_sent === true) { $this->server->update(['unreachable_notification_sent' => false]); diff --git a/app/Models/Server.php b/app/Models/Server.php index 0834bf61b7..43045e1b03 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -966,8 +966,8 @@ public function isSwarmWorker() public function validateConnection($isManualCheck = true) { - // Set mux_enabled to true for automatic checks, false for manual checks config()->set('constants.ssh.mux_enabled', !$isManualCheck); + ray('Manual Check: ' . ($isManualCheck ? 'true' : 'false')); $server = Server::find($this->id); if (! $server) { From 283fcc87a5a9045309b8a45d708865e791d13cae Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 17 Sep 2024 15:31:44 +0200 Subject: [PATCH 32/35] Fix: SSH algorhytm text --- resources/views/livewire/security/private-key/create.blade.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/views/livewire/security/private-key/create.blade.php b/resources/views/livewire/security/private-key/create.blade.php index a57cfc8e7d..e45974b228 100644 --- a/resources/views/livewire/security/private-key/create.blade.php +++ b/resources/views/livewire/security/private-key/create.blade.php @@ -4,8 +4,8 @@
You should not use passphrase protected keys.
+ Generate new ED25519 SSH Key (Recommended, fastest and most secure) Generate new RSA SSH Key - Generate new ED25519 SSH Key
From 42ff7b19a46352acaf19f6879a15f566517a0353 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 17 Sep 2024 15:54:22 +0200 Subject: [PATCH 33/35] Fix: Few multiplexing things --- app/Helpers/SshMultiplexingHelper.php | 63 +++++++---------- .../CleanupStaleMultiplexedConnections.php | 66 +++++++++++++++--- app/Models/Server.php | 2 +- bootstrap/helpers/remoteProcess.php | 67 +++++-------------- config/constants.php | 5 +- 5 files changed, 99 insertions(+), 104 deletions(-) diff --git a/app/Helpers/SshMultiplexingHelper.php b/app/Helpers/SshMultiplexingHelper.php index b1507bd001..c5fe901682 100644 --- a/app/Helpers/SshMultiplexingHelper.php +++ b/app/Helpers/SshMultiplexingHelper.php @@ -14,7 +14,7 @@ public static function serverSshConfiguration(Server $server) { $privateKey = PrivateKey::findOrFail($server->private_key_id); $sshKeyLocation = $privateKey->getKeyLocation(); - $muxFilename = '/var/www/html/storage/app/ssh/mux/' . $server->muxFilename(); + $muxFilename = '/var/www/html/storage/app/ssh/mux/mux_' . $server->uuid; return [ 'sshKeyLocation' => $sshKeyLocation, @@ -25,12 +25,12 @@ public static function serverSshConfiguration(Server $server) public static function ensureMultiplexedConnection(Server $server) { if (!self::isMultiplexingEnabled()) { - ray('SSH Multiplexing: DISABLED')->red(); + // ray('SSH Multiplexing: DISABLED')->red(); return; } - ray('SSH Multiplexing: ENABLED')->green(); - ray('Ensuring multiplexed connection for server:', $server->id); + // ray('SSH Multiplexing: ENABLED')->green(); + // ray('Ensuring multiplexed connection for server:', $server); $sshConfig = self::serverSshConfiguration($server); $muxSocket = $sshConfig['muxFilename']; @@ -42,18 +42,16 @@ public static function ensureMultiplexedConnection(Server $server) $process = Process::run($checkCommand); if ($process->exitCode() !== 0) { - ray('SSH Multiplexing: Existing connection check failed or not found')->orange(); - ray('Establishing new connection'); + // ray('SSH Multiplexing: Existing connection check failed or not found')->orange(); + // ray('Establishing new connection'); self::establishNewMultiplexedConnection($server); } else { - ray('SSH Multiplexing: Existing connection is valid')->green(); + // ray('SSH Multiplexing: Existing connection is valid')->green(); } } public static function establishNewMultiplexedConnection(Server $server) { - ray('SSH Multiplexing: Establishing new connection for server:', $server->id); - $sshConfig = self::serverSshConfiguration($server); $sshKeyLocation = $sshConfig['sshKeyLocation']; $muxSocket = $sshConfig['muxFilename']; @@ -63,27 +61,20 @@ public static function establishNewMultiplexedConnection(Server $server) $muxPersistTime = config('constants.ssh.mux_persist_time'); $establishCommand = "ssh -fNM -o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} " - . "-i {$sshKeyLocation} " - . '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ' - . '-o PasswordAuthentication=no ' - . "-o ConnectTimeout=$connectionTimeout " - . "-o ServerAliveInterval=$serverInterval " - . '-o RequestTTY=no ' - . '-o LogLevel=ERROR ' - . "-p {$server->port} " + . self::getCommonSshOptions($server, $sshKeyLocation, $connectionTimeout, $serverInterval) . "{$server->user}@{$server->ip}"; $establishProcess = Process::run($establishCommand); if ($establishProcess->exitCode() !== 0) { - ray('SSH Multiplexing: Failed to establish connection', $establishProcess->errorOutput())->red(); throw new \RuntimeException('Failed to establish multiplexed connection: ' . $establishProcess->errorOutput()); } - ray('SSH Multiplexing: Connection established successfully')->green(); - $muxContent = "Multiplexed connection established at " . now()->toDateTimeString(); - Storage::disk('ssh-mux')->put(basename($muxSocket), $muxContent); + $muxFilename = basename($muxSocket); + if (!Storage::disk('ssh-mux')->put($muxFilename, $muxContent)) { + throw new \RuntimeException('Failed to write mux file to disk: ' . $muxFilename); + } } public static function removeMuxFile(Server $server) @@ -93,8 +84,6 @@ public static function removeMuxFile(Server $server) $closeCommand = "ssh -O exit -o ControlPath=/var/www/html/storage/app/ssh/mux/{$muxFilename} {$server->user}@{$server->ip}"; Process::run($closeCommand); - - Storage::disk('ssh-mux')->delete($muxFilename); } public static function generateScpCommand(Server $server, string $source, string $dest) @@ -104,29 +93,26 @@ public static function generateScpCommand(Server $server, string $source, string $muxSocket = $sshConfig['muxFilename']; $timeout = config('constants.ssh.command_timeout'); - $connectionTimeout = config('constants.ssh.connection_timeout'); - $serverInterval = config('constants.ssh.server_interval'); $scp_command = "timeout $timeout scp "; if (self::isMultiplexingEnabled()) { - ray('SSH Multiplexing: Enabled for SCP command')->green(); + // ray('SSH Multiplexing: Enabled for SCP command')->green(); $muxPersistTime = config('constants.ssh.mux_persist_time'); $scp_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; self::ensureMultiplexedConnection($server); - // Add this line to verify multiplexing is being used - ray('SSH Multiplexing: Verifying usage')->blue(); + // ray('SSH Multiplexing: Verifying usage')->blue(); $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip}"; $checkProcess = Process::run($checkCommand); - ray('SSH Multiplexing: ' . ($checkProcess->exitCode() === 0 ? 'Active' : 'Not Active'))->color($checkProcess->exitCode() === 0 ? 'green' : 'red'); + // ray('SSH Multiplexing: ' . ($checkProcess->exitCode() === 0 ? 'Active' : 'Not Active'))->color($checkProcess->exitCode() === 0 ? 'green' : 'red'); } else { - ray('SSH Multiplexing: Disabled for SCP command')->orange(); + // ray('SSH Multiplexing: Disabled for SCP command')->orange(); } self::addCloudflareProxyCommand($scp_command, $server); - $scp_command .= self::getCommonSshOptions($server, $sshKeyLocation, $connectionTimeout, $serverInterval); + $scp_command .= self::getCommonSshOptions($server, $sshKeyLocation, config('constants.ssh.connection_timeout'), config('constants.ssh.server_interval')); $scp_command .= "{$source} {$server->user}@{$server->ip}:{$dest}"; return $scp_command; @@ -143,29 +129,26 @@ public static function generateSshCommand(Server $server, string $command) $muxSocket = $sshConfig['muxFilename']; $timeout = config('constants.ssh.command_timeout'); - $connectionTimeout = config('constants.ssh.connection_timeout'); - $serverInterval = config('constants.ssh.server_interval'); $ssh_command = "timeout $timeout ssh "; if (self::isMultiplexingEnabled()) { - ray('SSH Multiplexing: Enabled for SSH command')->green(); + // ray('SSH Multiplexing: Enabled for SSH command')->green(); $muxPersistTime = config('constants.ssh.mux_persist_time'); $ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; self::ensureMultiplexedConnection($server); - // Add this line to verify multiplexing is being used - ray('SSH Multiplexing: Verifying usage')->blue(); + // ray('SSH Multiplexing: Verifying usage')->blue(); $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip}"; $checkProcess = Process::run($checkCommand); - ray('SSH Multiplexing: ' . ($checkProcess->exitCode() === 0 ? 'Active' : 'Not Active'))->color($checkProcess->exitCode() === 0 ? 'green' : 'red'); + // ray('SSH Multiplexing: ' . ($checkProcess->exitCode() === 0 ? 'Active' : 'Not Active'))->color($checkProcess->exitCode() === 0 ? 'green' : 'red'); } else { - ray('SSH Multiplexing: Disabled for SSH command')->orange(); + // ray('SSH Multiplexing: Disabled for SSH command')->orange(); } self::addCloudflareProxyCommand($ssh_command, $server); - $ssh_command .= self::getCommonSshOptions($server, $sshKeyLocation, $connectionTimeout, $serverInterval); + $ssh_command .= self::getCommonSshOptions($server, $sshKeyLocation, config('constants.ssh.connection_timeout'), config('constants.ssh.server_interval')); $command = "PATH=\$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/local/sbin:/host/usr/local/bin:/host/usr/sbin:/host/usr/bin:/host/sbin:/host/bin && $command"; $delimiter = Hash::make($command); @@ -181,7 +164,7 @@ public static function generateSshCommand(Server $server, string $command) private static function isMultiplexingEnabled(): bool { $isEnabled = config('constants.ssh.mux_enabled') && !config('coolify.is_windows_docker_desktop'); - ray('SSH Multiplexing Status:', $isEnabled ? 'ENABLED' : 'DISABLED')->color($isEnabled ? 'green' : 'red'); + // ray('SSH Multiplexing Status:', $isEnabled ? 'ENABLED' : 'DISABLED')->color($isEnabled ? 'green' : 'red'); return $isEnabled; } diff --git a/app/Jobs/CleanupStaleMultiplexedConnections.php b/app/Jobs/CleanupStaleMultiplexedConnections.php index bcca77c182..acb28c2f47 100644 --- a/app/Jobs/CleanupStaleMultiplexedConnections.php +++ b/app/Jobs/CleanupStaleMultiplexedConnections.php @@ -9,6 +9,8 @@ use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Facades\Process; +use Illuminate\Support\Facades\Storage; +use Carbon\Carbon; class CleanupStaleMultiplexedConnections implements ShouldQueue { @@ -16,22 +18,64 @@ class CleanupStaleMultiplexedConnections implements ShouldQueue public function handle() { - Server::chunk(100, function ($servers) { - foreach ($servers as $server) { - $this->cleanupStaleConnection($server); + $this->cleanupStaleConnections(); + $this->cleanupNonExistentServerConnections(); + } + + private function cleanupStaleConnections() + { + $muxFiles = Storage::disk('ssh-mux')->files(); + + foreach ($muxFiles as $muxFile) { + $serverUuid = $this->extractServerUuidFromMuxFile($muxFile); + $server = Server::where('uuid', $serverUuid)->first(); + + if (!$server) { + $this->removeMultiplexFile($muxFile); + continue; + } + + $muxSocket = "/var/www/html/storage/app/ssh/mux/{$muxFile}"; + $checkCommand = "ssh -O check -o ControlPath={$muxSocket} {$server->user}@{$server->ip} 2>/dev/null"; + $checkProcess = Process::run($checkCommand); + + if ($checkProcess->exitCode() !== 0) { + $this->removeMultiplexFile($muxFile); + } else { + $muxContent = Storage::disk('ssh-mux')->get($muxFile); + $establishedAt = Carbon::parse(substr($muxContent, 37)); + $expirationTime = $establishedAt->addSeconds(config('constants.ssh.mux_persist_time')); + + if (Carbon::now()->isAfter($expirationTime)) { + $this->removeMultiplexFile($muxFile); + } } - }); + } } - private function cleanupStaleConnection(Server $server) + private function cleanupNonExistentServerConnections() { - $muxSocket = "/tmp/mux_{$server->id}"; - $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip} 2>/dev/null"; - $checkProcess = Process::run($checkCommand); + $muxFiles = Storage::disk('ssh-mux')->files(); + $existingServerUuids = Server::pluck('uuid')->toArray(); - if ($checkProcess->exitCode() !== 0) { - $closeCommand = "ssh -O exit -o ControlPath=$muxSocket {$server->user}@{$server->ip} 2>/dev/null"; - Process::run($closeCommand); + foreach ($muxFiles as $muxFile) { + $serverUuid = $this->extractServerUuidFromMuxFile($muxFile); + if (!in_array($serverUuid, $existingServerUuids)) { + $this->removeMultiplexFile($muxFile); + } } } + + private function extractServerUuidFromMuxFile($muxFile) + { + return substr($muxFile, 4); + } + + private function removeMultiplexFile($muxFile) + { + $muxSocket = "/var/www/html/storage/app/ssh/mux/{$muxFile}"; + $closeCommand = "ssh -O exit -o ControlPath={$muxSocket} localhost 2>/dev/null"; + Process::run($closeCommand); + Storage::disk('ssh-mux')->delete($muxFile); + } } diff --git a/app/Models/Server.php b/app/Models/Server.php index 43045e1b03..363db32973 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -967,7 +967,7 @@ public function isSwarmWorker() public function validateConnection($isManualCheck = true) { config()->set('constants.ssh.mux_enabled', !$isManualCheck); - ray('Manual Check: ' . ($isManualCheck ? 'true' : 'false')); + // ray('Manual Check: ' . ($isManualCheck ? 'true' : 'false')); $server = Server::find($this->id); if (! $server) { diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index 5263ea9703..ebc8420c62 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -60,40 +60,28 @@ function remote_process( function instant_scp(string $source, string $dest, Server $server, $throwError = true) { - $timeout = config('constants.ssh.command_timeout'); $scp_command = SshMultiplexingHelper::generateScpCommand($server, $source, $dest); - $process = Process::timeout($timeout)->run($scp_command); + $process = Process::timeout(config('constants.ssh.command_timeout'))->run($scp_command); $output = trim($process->output()); $exitCode = $process->exitCode(); if ($exitCode !== 0) { - if (! $throwError) { - return null; - } - - return excludeCertainErrors($process->errorOutput(), $exitCode); - } - if ($output === 'null') { - $output = null; + return $throwError ? excludeCertainErrors($process->errorOutput(), $exitCode) : null; } - - return $output; + return $output === 'null' ? null : $output; } function instant_remote_process(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false): ?string { - $timeout = config('constants.ssh.command_timeout'); - if ($command instanceof Collection) { - $command = $command->toArray(); - } - if ($server->isNonRoot() && ! $no_sudo) { + $command = $command instanceof Collection ? $command->toArray() : $command; + if ($server->isNonRoot() && !$no_sudo) { $command = parseCommandsByLineForSudo(collect($command), $server); } $command_string = implode("\n", $command); - $start_time = microtime(true); + // $start_time = microtime(true); $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string); - $process = Process::timeout($timeout)->run($sshCommand); - $end_time = microtime(true); + $process = Process::timeout(config('constants.ssh.command_timeout'))->run($sshCommand); + // $end_time = microtime(true); // $execution_time = ($end_time - $start_time) * 1000; // Convert to milliseconds // ray('SSH command execution time:', $execution_time.' ms')->orange(); @@ -102,17 +90,9 @@ function instant_remote_process(Collection|array $command, Server $server, bool $exitCode = $process->exitCode(); if ($exitCode !== 0) { - if (! $throwError) { - return null; - } - - return excludeCertainErrors($process->errorOutput(), $exitCode); - } - if ($output === 'null') { - $output = null; + return $throwError ? excludeCertainErrors($process->errorOutput(), $exitCode) : null; } - - return $output; + return $output === 'null' ? null : $output; } function excludeCertainErrors(string $errorOutput, ?int $exitCode = null) @@ -121,13 +101,7 @@ function excludeCertainErrors(string $errorOutput, ?int $exitCode = null) 'Permission denied (publickey', 'Could not resolve hostname', ]); - $ignored = false; - foreach ($ignoredErrors as $ignoredError) { - if (Str::contains($errorOutput, $ignoredError)) { - $ignored = true; - break; - } - } + $ignored = $ignoredErrors->contains(fn($error) => Str::contains($errorOutput, $error)); if ($ignored) { // TODO: Create new exception and disable in sentry throw new \RuntimeException($errorOutput, $exitCode); @@ -137,11 +111,11 @@ function excludeCertainErrors(string $errorOutput, ?int $exitCode = null) function decode_remote_command_output(?ApplicationDeploymentQueue $application_deployment_queue = null): Collection { - $application = Application::find(data_get($application_deployment_queue, 'application_id')); - $is_debug_enabled = data_get($application, 'settings.is_debug_enabled'); if (is_null($application_deployment_queue)) { return collect([]); } + $application = Application::find(data_get($application_deployment_queue, 'application_id')); + $is_debug_enabled = data_get($application, 'settings.is_debug_enabled'); try { $decoded = json_decode( data_get($application_deployment_queue, 'logs'), @@ -153,20 +127,19 @@ function decode_remote_command_output(?ApplicationDeploymentQueue $application_d } $seenCommands = collect(); $formatted = collect($decoded); - if (! $is_debug_enabled) { + if (!$is_debug_enabled) { $formatted = $formatted->filter(fn ($i) => $i['hidden'] === false ?? false); } - $formatted = $formatted + return $formatted ->sortBy(fn ($i) => data_get($i, 'order')) ->map(function ($i) { data_set($i, 'timestamp', Carbon::parse(data_get($i, 'timestamp'))->format('Y-M-d H:i:s.u')); - return $i; }) ->reduce(function ($deploymentLogLines, $logItem) use ($seenCommands) { $command = data_get($logItem, 'command'); $isStderr = data_get($logItem, 'type') === 'stderr'; - $isNewCommand = ! is_null($command) && ! $seenCommands->first(function ($seenCommand) use ($logItem) { + $isNewCommand = !is_null($command) && !$seenCommands->first(function ($seenCommand) use ($logItem) { return data_get($seenCommand, 'command') === data_get($logItem, 'command') && data_get($seenCommand, 'batch') === data_get($logItem, 'batch'); }); @@ -198,14 +171,11 @@ function decode_remote_command_output(?ApplicationDeploymentQueue $application_d return $deploymentLogLines; }, collect()); - - return $formatted; } function remove_iip($text) { $text = preg_replace('/x-access-token:.*?(?=@)/', 'x-access-token:'.REDACTED, $text); - return preg_replace('/\x1b\[[0-9;]*m/', '', $text); } @@ -233,9 +203,8 @@ function checkRequiredCommands(Server $server) break; } $commandFound = instant_remote_process(["docker run --rm --privileged --net=host --pid=host --ipc=host --volume /:/host busybox chroot /host bash -c 'command -v {$command}'"], $server, false); - if ($commandFound) { - continue; + if (!$commandFound) { + break; } - break; } } diff --git a/config/constants.php b/config/constants.php index 906ef3ba2b..5792b358c4 100644 --- a/config/constants.php +++ b/config/constants.php @@ -6,9 +6,8 @@ 'contact' => 'https://coolify.io/docs/contact', ], 'ssh' => [ - // Using MUX - 'mux_enabled' => env('MUX_ENABLED', env('SSH_MUX_ENABLED', true), true), - 'mux_persist_time' => env('SSH_MUX_PERSIST_TIME', '1h'), + 'mux_enabled' => env('MUX_ENABLED', env('SSH_MUX_ENABLED', true)), + 'mux_persist_time' => env('SSH_MUX_PERSIST_TIME', 3600), 'connection_timeout' => 10, 'server_interval' => 20, 'command_timeout' => 7200, From d13e2c086541f8e2005ee5c2bc14da49baf419a2 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 17 Sep 2024 15:57:57 +0200 Subject: [PATCH 34/35] Fix: Clear mux directory --- ..._16_170001_populate_ssh_keys_and_clear_mux_directory.php} | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) rename database/migrations/{2024_09_16_170001_populate_ssh_keys_directory.php => 2024_09_16_170001_populate_ssh_keys_and_clear_mux_directory.php} (71%) diff --git a/database/migrations/2024_09_16_170001_populate_ssh_keys_directory.php b/database/migrations/2024_09_16_170001_populate_ssh_keys_and_clear_mux_directory.php similarity index 71% rename from database/migrations/2024_09_16_170001_populate_ssh_keys_directory.php rename to database/migrations/2024_09_16_170001_populate_ssh_keys_and_clear_mux_directory.php index 33a5e695f6..944b00e13b 100644 --- a/database/migrations/2024_09_16_170001_populate_ssh_keys_directory.php +++ b/database/migrations/2024_09_16_170001_populate_ssh_keys_and_clear_mux_directory.php @@ -4,13 +4,16 @@ use Illuminate\Support\Facades\Storage; use App\Models\PrivateKey; -class PopulateSshKeysDirectory extends Migration +class PopulateSshKeysAndClearMuxDirectory extends Migration { public function up() { Storage::disk('ssh-keys')->deleteDirectory(''); Storage::disk('ssh-keys')->makeDirectory(''); + Storage::disk('ssh-mux')->deleteDirectory(''); + Storage::disk('ssh-mux')->makeDirectory(''); + PrivateKey::chunk(100, function ($keys) { foreach ($keys as $key) { $key->storeInFileSystem(); From d9181bd00bf70b2db5550583c6ca60fb8a92b6ba Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 17 Sep 2024 16:22:53 +0200 Subject: [PATCH 35/35] Fix: Multiplexing do not write file manually --- app/Helpers/SshMultiplexingHelper.php | 59 +++++++++++++++------------ app/Models/PrivateKey.php | 6 +-- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/app/Helpers/SshMultiplexingHelper.php b/app/Helpers/SshMultiplexingHelper.php index c5fe901682..57d4c88a47 100644 --- a/app/Helpers/SshMultiplexingHelper.php +++ b/app/Helpers/SshMultiplexingHelper.php @@ -5,7 +5,6 @@ use App\Models\Server; use App\Models\PrivateKey; use Illuminate\Support\Facades\Process; -use Illuminate\Support\Facades\Storage; use Illuminate\Support\Facades\Hash; class SshMultiplexingHelper @@ -56,6 +55,10 @@ public static function establishNewMultiplexedConnection(Server $server) $sshKeyLocation = $sshConfig['sshKeyLocation']; $muxSocket = $sshConfig['muxFilename']; + // ray('Establishing new multiplexed connection')->blue(); + // ray('SSH Key Location:', $sshKeyLocation); + // ray('Mux Socket:', $muxSocket); + $connectionTimeout = config('constants.ssh.connection_timeout'); $serverInterval = config('constants.ssh.server_interval'); $muxPersistTime = config('constants.ssh.mux_persist_time'); @@ -64,26 +67,46 @@ public static function establishNewMultiplexedConnection(Server $server) . self::getCommonSshOptions($server, $sshKeyLocation, $connectionTimeout, $serverInterval) . "{$server->user}@{$server->ip}"; + // ray('Establish Command:', $establishCommand); + $establishProcess = Process::run($establishCommand); + // ray('Establish Process Exit Code:', $establishProcess->exitCode()); + // ray('Establish Process Output:', $establishProcess->output()); + // ray('Establish Process Error Output:', $establishProcess->errorOutput()); + if ($establishProcess->exitCode() !== 0) { + // ray('Failed to establish multiplexed connection')->red(); throw new \RuntimeException('Failed to establish multiplexed connection: ' . $establishProcess->errorOutput()); } - $muxContent = "Multiplexed connection established at " . now()->toDateTimeString(); - $muxFilename = basename($muxSocket); - if (!Storage::disk('ssh-mux')->put($muxFilename, $muxContent)) { - throw new \RuntimeException('Failed to write mux file to disk: ' . $muxFilename); + // ray('Successfully established multiplexed connection')->green(); + + // Check if the mux socket file was created + if (!file_exists($muxSocket)) { + // ray('Mux socket file not found after connection establishment')->orange(); } } public static function removeMuxFile(Server $server) { $sshConfig = self::serverSshConfiguration($server); - $muxFilename = basename($sshConfig['muxFilename']); + $muxSocket = $sshConfig['muxFilename']; + + $closeCommand = "ssh -O exit -o ControlPath=$muxSocket {$server->user}@{$server->ip}"; + $process = Process::run($closeCommand); - $closeCommand = "ssh -O exit -o ControlPath=/var/www/html/storage/app/ssh/mux/{$muxFilename} {$server->user}@{$server->ip}"; - Process::run($closeCommand); + // ray('Closing multiplexed connection')->blue(); + // ray('Close command:', $closeCommand); + // ray('Close process exit code:', $process->exitCode()); + // ray('Close process output:', $process->output()); + // ray('Close process error output:', $process->errorOutput()); + + if ($process->exitCode() !== 0) { + // ray('Failed to close multiplexed connection')->orange(); + } else { + // ray('Successfully closed multiplexed connection')->green(); + } } public static function generateScpCommand(Server $server, string $source, string $dest) @@ -97,17 +120,9 @@ public static function generateScpCommand(Server $server, string $source, string $scp_command = "timeout $timeout scp "; if (self::isMultiplexingEnabled()) { - // ray('SSH Multiplexing: Enabled for SCP command')->green(); $muxPersistTime = config('constants.ssh.mux_persist_time'); $scp_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; self::ensureMultiplexedConnection($server); - - // ray('SSH Multiplexing: Verifying usage')->blue(); - $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip}"; - $checkProcess = Process::run($checkCommand); - // ray('SSH Multiplexing: ' . ($checkProcess->exitCode() === 0 ? 'Active' : 'Not Active'))->color($checkProcess->exitCode() === 0 ? 'green' : 'red'); - } else { - // ray('SSH Multiplexing: Disabled for SCP command')->orange(); } self::addCloudflareProxyCommand($scp_command, $server); @@ -133,17 +148,9 @@ public static function generateSshCommand(Server $server, string $command) $ssh_command = "timeout $timeout ssh "; if (self::isMultiplexingEnabled()) { - // ray('SSH Multiplexing: Enabled for SSH command')->green(); $muxPersistTime = config('constants.ssh.mux_persist_time'); $ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; self::ensureMultiplexedConnection($server); - - // ray('SSH Multiplexing: Verifying usage')->blue(); - $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip}"; - $checkProcess = Process::run($checkCommand); - // ray('SSH Multiplexing: ' . ($checkProcess->exitCode() === 0 ? 'Active' : 'Not Active'))->color($checkProcess->exitCode() === 0 ? 'green' : 'red'); - } else { - // ray('SSH Multiplexing: Disabled for SSH command')->orange(); } self::addCloudflareProxyCommand($ssh_command, $server); @@ -163,9 +170,7 @@ public static function generateSshCommand(Server $server, string $command) private static function isMultiplexingEnabled(): bool { - $isEnabled = config('constants.ssh.mux_enabled') && !config('coolify.is_windows_docker_desktop'); - // ray('SSH Multiplexing Status:', $isEnabled ? 'ENABLED' : 'DISABLED')->color($isEnabled ? 'green' : 'red'); - return $isEnabled; + return config('constants.ssh.mux_enabled') && !config('coolify.is_windows_docker_desktop'); } private static function validateSshKey(string $sshKeyLocation): void diff --git a/app/Models/PrivateKey.php b/app/Models/PrivateKey.php index 1ad12cf362..6985ca536b 100644 --- a/app/Models/PrivateKey.php +++ b/app/Models/PrivateKey.php @@ -137,20 +137,20 @@ public static function validateAndExtractPublicKey($privateKey) public function storeInFileSystem() { - $filename = "ssh@{$this->uuid}"; + $filename = "ssh_key@{$this->uuid}"; Storage::disk('ssh-keys')->put($filename, $this->private_key); return "/var/www/html/storage/app/ssh/keys/{$filename}"; } public static function deleteFromStorage(self $privateKey) { - $filename = "ssh@{$privateKey->uuid}"; + $filename = "ssh_key@{$privateKey->uuid}"; Storage::disk('ssh-keys')->delete($filename); } public function getKeyLocation() { - return "/var/www/html/storage/app/ssh/keys/ssh@{$this->uuid}"; + return "/var/www/html/storage/app/ssh/keys/ssh_key@{$this->uuid}"; } public function updatePrivateKey(array $data)