-
-
Notifications
You must be signed in to change notification settings - Fork 1.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #3454 from peaklabs-dev/fix-ssh-keys
Fix: SSH Keys, Multiplexing issues and a lot of other small things for dev and prod
- Loading branch information
Showing
31 changed files
with
702 additions
and
485 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,204 @@ | ||
| <?php | ||
|
|
||
| namespace App\Helpers; | ||
|
|
||
| use App\Models\Server; | ||
| use App\Models\PrivateKey; | ||
| use Illuminate\Support\Facades\Process; | ||
| use Illuminate\Support\Facades\Hash; | ||
|
|
||
| class SshMultiplexingHelper | ||
| { | ||
| public static function serverSshConfiguration(Server $server) | ||
| { | ||
| $privateKey = PrivateKey::findOrFail($server->private_key_id); | ||
| $sshKeyLocation = $privateKey->getKeyLocation(); | ||
| $muxFilename = '/var/www/html/storage/app/ssh/mux/mux_' . $server->uuid; | ||
|
|
||
| return [ | ||
| 'sshKeyLocation' => $sshKeyLocation, | ||
| 'muxFilename' => $muxFilename, | ||
| ]; | ||
| } | ||
|
|
||
| public static function ensureMultiplexedConnection(Server $server) | ||
| { | ||
| if (!self::isMultiplexingEnabled()) { | ||
| // ray('SSH Multiplexing: DISABLED')->red(); | ||
| return; | ||
| } | ||
|
|
||
| // ray('SSH Multiplexing: ENABLED')->green(); | ||
| // ray('Ensuring multiplexed connection for server:', $server); | ||
|
|
||
| $sshConfig = self::serverSshConfiguration($server); | ||
| $muxSocket = $sshConfig['muxFilename']; | ||
| $sshKeyLocation = $sshConfig['sshKeyLocation']; | ||
|
|
||
| self::validateSshKey($sshKeyLocation); | ||
|
|
||
| $checkCommand = "ssh -O check -o ControlPath=$muxSocket {$server->user}@{$server->ip}"; | ||
| $process = Process::run($checkCommand); | ||
|
|
||
| if ($process->exitCode() !== 0) { | ||
| // ray('SSH Multiplexing: Existing connection check failed or not found')->orange(); | ||
| // ray('Establishing new connection'); | ||
| self::establishNewMultiplexedConnection($server); | ||
| } else { | ||
| // ray('SSH Multiplexing: Existing connection is valid')->green(); | ||
| } | ||
| } | ||
|
|
||
| public static function establishNewMultiplexedConnection(Server $server) | ||
| { | ||
| $sshConfig = self::serverSshConfiguration($server); | ||
| $sshKeyLocation = $sshConfig['sshKeyLocation']; | ||
| $muxSocket = $sshConfig['muxFilename']; | ||
|
|
||
| // ray('Establishing new multiplexed connection')->blue(); | ||
| // ray('SSH Key Location:', $sshKeyLocation); | ||
| // ray('Mux Socket:', $muxSocket); | ||
|
|
||
| $connectionTimeout = config('constants.ssh.connection_timeout'); | ||
| $serverInterval = config('constants.ssh.server_interval'); | ||
| $muxPersistTime = config('constants.ssh.mux_persist_time'); | ||
|
|
||
| $establishCommand = "ssh -fNM -o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} " | ||
| . self::getCommonSshOptions($server, $sshKeyLocation, $connectionTimeout, $serverInterval) | ||
| . "{$server->user}@{$server->ip}"; | ||
|
|
||
| // ray('Establish Command:', $establishCommand); | ||
|
|
||
| $establishProcess = Process::run($establishCommand); | ||
|
|
||
| // ray('Establish Process Exit Code:', $establishProcess->exitCode()); | ||
| // ray('Establish Process Output:', $establishProcess->output()); | ||
| // ray('Establish Process Error Output:', $establishProcess->errorOutput()); | ||
|
|
||
| if ($establishProcess->exitCode() !== 0) { | ||
| // ray('Failed to establish multiplexed connection')->red(); | ||
| throw new \RuntimeException('Failed to establish multiplexed connection: ' . $establishProcess->errorOutput()); | ||
| } | ||
|
|
||
| // ray('Successfully established multiplexed connection')->green(); | ||
|
|
||
| // Check if the mux socket file was created | ||
| if (!file_exists($muxSocket)) { | ||
| // ray('Mux socket file not found after connection establishment')->orange(); | ||
| } | ||
| } | ||
|
|
||
| public static function removeMuxFile(Server $server) | ||
| { | ||
| $sshConfig = self::serverSshConfiguration($server); | ||
| $muxSocket = $sshConfig['muxFilename']; | ||
|
|
||
| $closeCommand = "ssh -O exit -o ControlPath=$muxSocket {$server->user}@{$server->ip}"; | ||
| $process = Process::run($closeCommand); | ||
|
|
||
| // ray('Closing multiplexed connection')->blue(); | ||
| // ray('Close command:', $closeCommand); | ||
| // ray('Close process exit code:', $process->exitCode()); | ||
| // ray('Close process output:', $process->output()); | ||
| // ray('Close process error output:', $process->errorOutput()); | ||
|
|
||
| if ($process->exitCode() !== 0) { | ||
| // ray('Failed to close multiplexed connection')->orange(); | ||
| } else { | ||
| // ray('Successfully closed multiplexed connection')->green(); | ||
| } | ||
| } | ||
|
|
||
| public static function generateScpCommand(Server $server, string $source, string $dest) | ||
| { | ||
| $sshConfig = self::serverSshConfiguration($server); | ||
| $sshKeyLocation = $sshConfig['sshKeyLocation']; | ||
| $muxSocket = $sshConfig['muxFilename']; | ||
|
|
||
| $timeout = config('constants.ssh.command_timeout'); | ||
|
|
||
| $scp_command = "timeout $timeout scp "; | ||
|
|
||
| if (self::isMultiplexingEnabled()) { | ||
| $muxPersistTime = config('constants.ssh.mux_persist_time'); | ||
| $scp_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; | ||
| self::ensureMultiplexedConnection($server); | ||
| } | ||
|
|
||
| self::addCloudflareProxyCommand($scp_command, $server); | ||
|
|
||
| $scp_command .= self::getCommonSshOptions($server, $sshKeyLocation, config('constants.ssh.connection_timeout'), config('constants.ssh.server_interval')); | ||
| $scp_command .= "{$source} {$server->user}@{$server->ip}:{$dest}"; | ||
|
|
||
| return $scp_command; | ||
| } | ||
|
|
||
| public static function generateSshCommand(Server $server, string $command) | ||
| { | ||
| if ($server->settings->force_disabled) { | ||
| throw new \RuntimeException('Server is disabled.'); | ||
| } | ||
|
|
||
| $sshConfig = self::serverSshConfiguration($server); | ||
| $sshKeyLocation = $sshConfig['sshKeyLocation']; | ||
| $muxSocket = $sshConfig['muxFilename']; | ||
|
|
||
| $timeout = config('constants.ssh.command_timeout'); | ||
|
|
||
| $ssh_command = "timeout $timeout ssh "; | ||
|
|
||
| if (self::isMultiplexingEnabled()) { | ||
| $muxPersistTime = config('constants.ssh.mux_persist_time'); | ||
| $ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; | ||
| self::ensureMultiplexedConnection($server); | ||
| } | ||
|
|
||
| self::addCloudflareProxyCommand($ssh_command, $server); | ||
|
|
||
| $ssh_command .= self::getCommonSshOptions($server, $sshKeyLocation, config('constants.ssh.connection_timeout'), config('constants.ssh.server_interval')); | ||
|
|
||
| $command = "PATH=\$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/local/sbin:/host/usr/local/bin:/host/usr/sbin:/host/usr/bin:/host/sbin:/host/bin && $command"; | ||
| $delimiter = Hash::make($command); | ||
| $command = str_replace($delimiter, '', $command); | ||
|
|
||
| $ssh_command .= "{$server->user}@{$server->ip} 'bash -se' << \\$delimiter".PHP_EOL | ||
| .$command.PHP_EOL | ||
| .$delimiter; | ||
|
|
||
| return $ssh_command; | ||
| } | ||
|
|
||
| private static function isMultiplexingEnabled(): bool | ||
| { | ||
| return config('constants.ssh.mux_enabled') && !config('coolify.is_windows_docker_desktop'); | ||
| } | ||
|
|
||
| private static function validateSshKey(string $sshKeyLocation): void | ||
| { | ||
| $checkKeyCommand = "ls $sshKeyLocation 2>/dev/null"; | ||
| $keyCheckProcess = Process::run($checkKeyCommand); | ||
|
|
||
| if ($keyCheckProcess->exitCode() !== 0) { | ||
| throw new \RuntimeException("SSH key file not accessible: $sshKeyLocation"); | ||
| } | ||
| } | ||
|
|
||
| private static function addCloudflareProxyCommand(string &$command, Server $server): void | ||
| { | ||
| if (data_get($server, 'settings.is_cloudflare_tunnel')) { | ||
| $command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" '; | ||
| } | ||
| } | ||
|
|
||
| private static function getCommonSshOptions(Server $server, string $sshKeyLocation, int $connectionTimeout, int $serverInterval): string | ||
| { | ||
| return "-i {$sshKeyLocation} " | ||
| .'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ' | ||
| .'-o PasswordAuthentication=no ' | ||
| ."-o ConnectTimeout=$connectionTimeout " | ||
| ."-o ServerAliveInterval=$serverInterval " | ||
| .'-o RequestTTY=no ' | ||
| .'-o LogLevel=ERROR ' | ||
| ."-p {$server->port} "; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| <?php | ||
|
|
||
| namespace App\Jobs; | ||
|
|
||
| use App\Models\PrivateKey; | ||
| use Illuminate\Bus\Queueable; | ||
| use Illuminate\Contracts\Queue\ShouldQueue; | ||
| use Illuminate\Foundation\Bus\Dispatchable; | ||
| use Illuminate\Queue\InteractsWithQueue; | ||
| use Illuminate\Queue\SerializesModels; | ||
| use Carbon\Carbon; | ||
|
|
||
| class CleanupSshKeysJob implements ShouldQueue | ||
| { | ||
| use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; | ||
|
|
||
| public function handle() | ||
| { | ||
| $oneWeekAgo = Carbon::now()->subWeek(); | ||
|
|
||
| PrivateKey::where('created_at', '<', $oneWeekAgo) | ||
| ->get() | ||
| ->each(function ($privateKey) { | ||
| $privateKey->safeDelete(); | ||
| }); | ||
| } | ||
| } |
Oops, something went wrong.