diff --git a/cmd/limactl/copy.go b/cmd/limactl/copy.go
index 59c9476850b1..33eddb68a1a1 100644
--- a/cmd/limactl/copy.go
+++ b/cmd/limactl/copy.go
@@ -94,7 +94,7 @@ func copyAction(cmd *cobra.Command, args []string) error {
 			} else {
 				scpArgs = append(scpArgs, fmt.Sprintf("scp://%s@%s:%d/%s", u.Username, inst.SSHAddress, inst.SSHLocalPort, path[1]))
 			}
-			if inst.SSHAddress != "127.0.0.1" {
+			if !sshutil.IsLocalhost(inst.SSHAddress) {
 				instAddr = inst.SSHAddress
 				localhostOnly = false
 			}
diff --git a/pkg/sshutil/sshutil.go b/pkg/sshutil/sshutil.go
index 0bcbaeb2ee0a..860e9eae3fb0 100644
--- a/pkg/sshutil/sshutil.go
+++ b/pkg/sshutil/sshutil.go
@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io/fs"
+	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -121,6 +122,14 @@ var sshInfo struct {
 	openSSHVersion semver.Version
 }
 
+func IsLocalhost(address string) bool {
+	ip := net.ParseIP(address)
+	if ip == nil {
+		return false
+	}
+	return ip.IsLoopback()
+}
+
 // CommonOpts returns ssh option key-value pairs like {"IdentityFile=/path/to/id_foo"}.
 // The result may contain different values with the same key.
 //
@@ -238,7 +247,7 @@ func SSHOpts(instDir string, useDotSSH bool, hostAddress string, forwardAgent, f
 	if err != nil {
 		return nil, err
 	}
-	opts, err := CommonOpts(useDotSSH, hostAddress == "127.0.0.1")
+	opts, err := CommonOpts(useDotSSH, IsLocalhost(hostAddress))
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/sshutil/sshutil_test.go b/pkg/sshutil/sshutil_test.go
index ef2b754da23a..e452a160f1de 100644
--- a/pkg/sshutil/sshutil_test.go
+++ b/pkg/sshutil/sshutil_test.go
@@ -15,6 +15,10 @@ func TestDefaultPubKeys(t *testing.T) {
 	}
 }
 
+func TestIsLocalhost(t *testing.T) {
+	assert.Equal(t, IsLocalhost("127.0.0.1"), true)
+}
+
 func TestParseOpenSSHVersion(t *testing.T) {
 	assert.Check(t, ParseOpenSSHVersion([]byte("OpenSSH_8.4p1 Ubuntu")).Equal(
 		semver.Version{Major: 8, Minor: 4, Patch: 1, PreRelease: "", Metadata: ""}))