From cf950035edffa4462fd02d31cfb22de9b2336d27 Mon Sep 17 00:00:00 2001 From: James Lu Date: Thu, 9 Jan 2025 11:23:02 +0800 Subject: [PATCH 1/3] feat(crypto): allow v2 volume encryption in validator ref: longhorn/longhorn 7355 Signed-off-by: James Lu --- webhook/resources/volume/validator.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/webhook/resources/volume/validator.go b/webhook/resources/volume/validator.go index 31ab16f6b4..457deff1f7 100644 --- a/webhook/resources/volume/validator.go +++ b/webhook/resources/volume/validator.go @@ -167,9 +167,6 @@ func (v *volumeValidator) Create(request *admission.Request, newObj runtime.Obje // TODO: remove this check when we support the following features for SPDK volumes if types.IsDataEngineV2(volume.Spec.DataEngine) { - if volume.Spec.Encrypted { - return werror.NewInvalidError("encrypted volume is not supported for data engine v2", "") - } if types.IsDataFromVolume(volume.Spec.DataSource) { return werror.NewInvalidError("clone is not supported for data engine v2", "") } From 2af9783202cfeef60290697f04f66891b1b49996 Mon Sep 17 00:00:00 2001 From: James Lu Date: Fri, 10 Jan 2025 17:09:43 +0800 Subject: [PATCH 2/3] feat(crypto): add a suffix to v2 volume when opening and closing a luks2 format device. ref: longhorn/longhorn 7355 Signed-off-by: James Lu --- csi/crypto/crypto.go | 47 ++++++++++++++++++++++++++++++++------------ csi/node_server.go | 39 +++++++++++++++++++----------------- 2 files changed, 55 insertions(+), 31 deletions(-) diff --git a/csi/crypto/crypto.go b/csi/crypto/crypto.go index cfd3e0ee39..1cc52f3203 100644 --- a/csi/crypto/crypto.go +++ b/csi/crypto/crypto.go @@ -8,12 +8,16 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" + "github.com/longhorn/longhorn-manager/types" + lhns "github.com/longhorn/go-common-libs/ns" lhtypes "github.com/longhorn/go-common-libs/types" + longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ) const ( mapperFilePathPrefix = "/dev/mapper" + mapperV2VolumeSuffix = "-encrypted" CryptoKeyDefaultCipher = "aes-xts-plain64" CryptoKeyDefaultHash = "sha256" @@ -69,7 +73,13 @@ func (cp *EncryptParams) GetPBKDF() string { } // VolumeMapper returns the path for mapped encrypted device. -func VolumeMapper(volume string) string { +func VolumeMapper(volume, dataEngine string) string { + if types.IsDataEngineV2(longhorn.DataEngineType(dataEngine)) { + // v2 volume will use a dm device as default to control IO path when attaching. + // This dm device will be created with the same name as the volume name. + // The encrypted volume will be created with the volume name with "-encrypted" suffix to resolve the naming conflict. + return path.Join(mapperFilePathPrefix, getEncryptVolumeName(volume, dataEngine)) + } return path.Join(mapperFilePathPrefix, volume) } @@ -95,9 +105,10 @@ func EncryptVolume(devicePath, passphrase string, cryptoParams *EncryptParams) e } // OpenVolume opens volume so that it can be used by the client. -func OpenVolume(volume, devicePath, passphrase string) error { - if isOpen, _ := IsDeviceOpen(VolumeMapper(volume)); isOpen { - logrus.Infof("Device %s is already opened at %s", devicePath, VolumeMapper(volume)) +// devicePath is the path of the volume on the host that will be opened for instance '/dev/longhorn/volume1' +func OpenVolume(volume, dataEngine, devicePath, passphrase string) error { + if isOpen, _ := IsDeviceOpen(VolumeMapper(volume, dataEngine)); isOpen { + logrus.Infof("Device %s is already opened at %s", devicePath, VolumeMapper(volume, dataEngine)) return nil } @@ -107,29 +118,38 @@ func OpenVolume(volume, devicePath, passphrase string) error { return err } - logrus.Infof("Opening device %s with LUKS on %s", devicePath, volume) - _, err = nsexec.LuksOpen(volume, devicePath, passphrase, lhtypes.LuksTimeout) + encryptVolumeName := getEncryptVolumeName(volume, dataEngine) + logrus.Infof("Opening device %s with LUKS on %s", devicePath, encryptVolumeName) + _, err = nsexec.LuksOpen(encryptVolumeName, devicePath, passphrase, lhtypes.LuksTimeout) if err != nil { - logrus.WithError(err).Warnf("Failed to open LUKS device %s", devicePath) + logrus.WithError(err).Warnf("Failed to open LUKS device %s to %s", devicePath, encryptVolumeName) } return err } +func getEncryptVolumeName(volume, dataEngine string) string { + if types.IsDataEngineV2(longhorn.DataEngineType(dataEngine)) { + return volume + mapperV2VolumeSuffix + } + return volume +} + // CloseVolume closes encrypted volume so it can be detached. -func CloseVolume(volume string) error { +func CloseVolume(volume, dataEngine string) error { namespaces := []lhtypes.Namespace{lhtypes.NamespaceMnt, lhtypes.NamespaceIpc} nsexec, err := lhns.NewNamespaceExecutor(lhtypes.ProcessNone, lhtypes.HostProcDirectory, namespaces) if err != nil { return err } - logrus.Infof("Closing LUKS device %s", volume) - _, err = nsexec.LuksClose(volume, lhtypes.LuksTimeout) + encryptVolumeName := getEncryptVolumeName(volume, dataEngine) + logrus.Infof("Closing LUKS device %s", encryptVolumeName) + _, err = nsexec.LuksClose(encryptVolumeName, lhtypes.LuksTimeout) return err } -func ResizeEncryptoDevice(volume, passphrase string) error { - if isOpen, err := IsDeviceOpen(VolumeMapper(volume)); err != nil { +func ResizeEncryptoDevice(volume, dataEngine, passphrase string) error { + if isOpen, err := IsDeviceOpen(VolumeMapper(volume, dataEngine)); err != nil { return err } else if !isOpen { return fmt.Errorf("volume %v encrypto device is closed for resizing", volume) @@ -141,7 +161,8 @@ func ResizeEncryptoDevice(volume, passphrase string) error { return err } - _, err = nsexec.LuksResize(volume, passphrase, lhtypes.LuksTimeout) + encryptVolumeName := getEncryptVolumeName(volume, dataEngine) + _, err = nsexec.LuksResize(encryptVolumeName, passphrase, lhtypes.LuksTimeout) return err } diff --git a/csi/node_server.go b/csi/node_server.go index c51769c679..4fecb3b81f 100644 --- a/csi/node_server.go +++ b/csi/node_server.go @@ -488,7 +488,8 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol return nil, status.Errorf(codes.Internal, "failed to evaluate device filesystem %v format: %v", devicePath, err) } - log.Infof("Volume %v device %v contains filesystem of format %v", volumeID, devicePath, diskFormat) + dataEngine := volume.DataEngine + log.Infof("Volume %v (%v) device %v contains filesystem of format %v", volumeID, dataEngine, devicePath, diskFormat) if volume.Encrypted { secrets := req.GetSecrets() @@ -515,7 +516,7 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol } } - cryptoDevice := crypto.VolumeMapper(volumeID) + cryptoDevice := crypto.VolumeMapper(volumeID, dataEngine) log.Infof("Volume %s requires crypto device %s", volumeID, cryptoDevice) // check if the crypto device is open at the null path. @@ -525,12 +526,12 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol return nil, status.Errorf(codes.Internal, "failed to check if the crypto device %s for volume %s is mapped to the null path: %v", cryptoDevice, volumeID, err.Error()) } else if mappedToNullPath { log.Warnf("Closing active crypto device %s for volume %s since the volume is not closed properly before", cryptoDevice, volumeID) - if err := crypto.CloseVolume(volumeID); err != nil { + if err := crypto.CloseVolume(volumeID, dataEngine); err != nil { return nil, status.Errorf(codes.Internal, "failed to close active crypto device %s for volume %s: %v ", cryptoDevice, volumeID, err.Error()) } } - if err := crypto.OpenVolume(volumeID, devicePath, passphrase); err != nil { + if err := crypto.OpenVolume(volumeID, dataEngine, devicePath, passphrase); err != nil { return nil, status.Error(codes.Internal, err.Error()) } @@ -628,19 +629,20 @@ func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag // optionally try to retrieve the volume and check if it's an RWX volume // if it is we let the share-manager clean up the crypto device volume, _ := ns.apiClient.Volume.ById(volumeID) - if volume == nil || types.IsDataEngineV1(longhorn.DataEngineType(volume.DataEngine)) { - // Currently, only "RWO v1 volumes" and "block device with v1 volume.Migratable is true" supports encryption. - sharedAccess := requiresSharedAccess(volume, nil) - cleanupCryptoDevice := !sharedAccess || (sharedAccess && volume.Migratable) - if cleanupCryptoDevice { - cryptoDevice := crypto.VolumeMapper(volumeID) - if isOpen, err := crypto.IsDeviceOpen(cryptoDevice); err != nil { + dataEngine := string(longhorn.DataEngineTypeV1) + if volume != nil { + dataEngine = volume.DataEngine + } + sharedAccess := requiresSharedAccess(volume, nil) + cleanupCryptoDevice := !sharedAccess || (sharedAccess && volume.Migratable) + if cleanupCryptoDevice { + cryptoDevice := crypto.VolumeMapper(volumeID, dataEngine) + if isOpen, err := crypto.IsDeviceOpen(cryptoDevice); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } else if isOpen { + log.Infof("Volume %s closing active crypto device %s", volumeID, cryptoDevice) + if err := crypto.CloseVolume(volumeID, dataEngine); err != nil { return nil, status.Error(codes.Internal, err.Error()) - } else if isOpen { - log.Infof("Volume %s closing active crypto device %s", volumeID, cryptoDevice) - if err := crypto.CloseVolume(volumeID); err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } } } } @@ -820,6 +822,7 @@ func (ns *NodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandV return nil, fmt.Errorf("unknown filesystem type for volume %v node expansion", volumeID) } + dataEngine := volume.DataEngine devicePath, err = func() (string, error) { if !volume.Encrypted { return devicePath, nil @@ -827,7 +830,7 @@ func (ns *NodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandV if diskFormat != "crypto_LUKS" { return "", status.Errorf(codes.InvalidArgument, "unsupported disk encryption format %v", diskFormat) } - devicePath = crypto.VolumeMapper(volumeID) + devicePath = crypto.VolumeMapper(volumeID, dataEngine) // Need to enable feature gate in v1.25: // https://github.com/kubernetes/enhancements/issues/3107 @@ -847,7 +850,7 @@ func (ns *NodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandV } // blindly resize the encrypto device - if err := crypto.ResizeEncryptoDevice(volumeID, passphrase); err != nil { + if err := crypto.ResizeEncryptoDevice(volumeID, dataEngine, passphrase); err != nil { return "", status.Errorf(codes.InvalidArgument, "failed to resize crypto device %v for volume %v node expansion: %v", devicePath, volumeID, err) } From c2e6291ca81d7a87baf0bbd1e2764f1768558fe3 Mon Sep 17 00:00:00 2001 From: James Lu Date: Sat, 11 Jan 2025 14:57:27 +0800 Subject: [PATCH 3/3] feat(crypto): v2 RWX volume support ref: longhorn/longhorn 7355 Signed-off-by: James Lu --- controller/share_manager_controller.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/controller/share_manager_controller.go b/controller/share_manager_controller.go index c232d58f11..6e590e5a21 100644 --- a/controller/share_manager_controller.go +++ b/controller/share_manager_controller.go @@ -1298,7 +1298,7 @@ func (c *ShareManagerController) createShareManagerPod(sm *longhorn.ShareManager string(secret.Data[types.CryptoPBKDF])) } - manifest := c.createPodManifest(sm, annotations, tolerations, affinity, imagePullPolicy, nil, registrySecret, + manifest := c.createPodManifest(sm, volume.Spec.DataEngine, annotations, tolerations, affinity, imagePullPolicy, nil, registrySecret, priorityClass, nodeSelector, fsType, mountOptions, cryptoKey, cryptoParams, nfsConfig) storageNetwork, err := c.ds.GetSettingWithAutoFillingRO(types.SettingNameStorageNetwork) @@ -1411,13 +1411,13 @@ func (c *ShareManagerController) createLeaseManifest(sm *longhorn.ShareManager) return lease } -func (c *ShareManagerController) createPodManifest(sm *longhorn.ShareManager, annotations map[string]string, tolerations []corev1.Toleration, +func (c *ShareManagerController) createPodManifest(sm *longhorn.ShareManager, dataEngine longhorn.DataEngineType, annotations map[string]string, tolerations []corev1.Toleration, affinity *corev1.Affinity, pullPolicy corev1.PullPolicy, resourceReq *corev1.ResourceRequirements, registrySecret, priorityClass string, nodeSelector map[string]string, fsType string, mountOptions []string, cryptoKey string, cryptoParams *crypto.EncryptParams, nfsConfig *nfsServerConfig) *corev1.Pod { // command args for the share-manager - args := []string{"--debug", "daemon", "--volume", sm.Name} + args := []string{"--debug", "daemon", "--volume", sm.Name, "--data-engine", string(dataEngine)} if len(fsType) > 0 { args = append(args, "--fs", fsType)