rbd: implement pv key rotation

This patch implements the EncryptionKeyRotation spec for ceph-csi

Signed-off-by: Niraj Yadav <[email protected]>

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/ceph/ceph-csi/api v0.0.0-00010101000000-000000000000
 	github.com/ceph/go-ceph v0.28.0
 	github.com/container-storage-interface/spec v1.9.0
-	github.com/csi-addons/spec v0.2.1-0.20240619103729-12c61f25a2a5
+	github.com/csi-addons/spec v0.2.1-0.20240627093359-0dd74d521e67
 	github.com/gemalto/kmip-go v0.0.10
 	github.com/golang/protobuf v1.5.4
 	github.com/google/fscrypt v0.3.6-0.20240502174735-068b9f8f5dec

go.sum

@@ -918,8 +918,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
-github.com/csi-addons/spec v0.2.1-0.20240619103729-12c61f25a2a5 h1:/pXa+X+YKDPRI2JG8WEnxGKk6PcVZRhcLqdPks+bQa8=
-github.com/csi-addons/spec v0.2.1-0.20240619103729-12c61f25a2a5/go.mod h1:Mwq4iLiUV4s+K1bszcWU6aMsR5KPsbIYzzszJ6+56vI=
+github.com/csi-addons/spec v0.2.1-0.20240627093359-0dd74d521e67 h1:UAcAhE1pTkWaFBS0kvhHUcUsoEv5fsieD0tl8psQMCs=
+github.com/csi-addons/spec v0.2.1-0.20240627093359-0dd74d521e67/go.mod h1:Mwq4iLiUV4s+K1bszcWU6aMsR5KPsbIYzzszJ6+56vI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=

internal/csi-addons/rbd/encryptionkeyrotation.go

@@ -0,0 +1,76 @@
+/*
+Copyright 2024 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbd
+
+import (
+	"context"
+
+	ekr "github.com/csi-addons/spec/lib/go/encryptionkeyrotation"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
+	"github.com/ceph/ceph-csi/internal/rbd"
+	"github.com/ceph/ceph-csi/internal/util"
+)
+
+type EncryptionKeyRotationServer struct {
+	*ekr.UnimplementedEncryptionKeyRotationControllerServer
+}
+
+func NewEncryptionKeyRotationServer() *EncryptionKeyRotationServer {
+	return &EncryptionKeyRotationServer{}
+}
+
+func (ekrs *EncryptionKeyRotationServer) RegisterService(svc grpc.ServiceRegistrar) {
+	ekr.RegisterEncryptionKeyRotationControllerServer(svc, ekrs)
+}
+
+func (ekrs *EncryptionKeyRotationServer) RotateEncryptionKey(
+	ctx context.Context,
+	req *ekr.EncryptionKeyRotateRequest,
+) (*ekr.EncryptionKeyRotateResponse, error) {
+	// Get the volume ID from the request
+	volID := req.GetVolumeId()
+	if volID == "" {
+		return nil, status.Error(codes.InvalidArgument, "empty volume id in request")
+	}
+
+	// Get the credentials required to authenticate
+	// against a ceph cluster
+	creds, err := util.NewUserCredentials(req.GetSecrets())
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	defer creds.DeleteCredentials()
+
+	// Form the rbdVolume struct
+	rbdVol, err := rbd.GenVolFromVolID(ctx, volID, creds, req.GetSecrets())
+	if err != nil {
+		return nil, status.Errorf(codes.Aborted, "failed to find volume with ID %q: %s", volID, err.Error())
+	}
+	defer rbdVol.Destroy()
+
+	err = rbdVol.RotateEncryptionKey(ctx)
+	if err != nil {
+		return nil, status.Errorf(
+			codes.Aborted, "failed to rotate the key for volume with ID %q: %s", volID, err.Error())
+	}
+
+	// Success
+	return &ekr.EncryptionKeyRotateResponse{}, nil
+}

internal/csi-addons/rbd/identity.go

@@ -115,6 +115,13 @@ func (is *IdentityServer) GetCapabilities(
 						Type: identity.Capability_ReclaimSpace_ONLINE,
 					},
 				},
+			},
+			&identity.Capability{
+				Type: &identity.Capability_EncryptionKeyRotation_{
+					EncryptionKeyRotation: &identity.Capability_EncryptionKeyRotation{
+						Type: identity.Capability_EncryptionKeyRotation_ENCRYPTIONKEYROTATION,
+					},
+				},
 			})
 	}
 

internal/kms/vault.go

@@ -28,6 +28,8 @@ import (
 	"github.com/hashicorp/vault/api"
 	loss "github.com/libopenstorage/secrets"
 	"github.com/libopenstorage/secrets/vault"
+
+	"github.com/ceph/ceph-csi/internal/util/file"
 )
 
 const (
@@ -269,10 +271,12 @@ func (vc *vaultConnection) initCertificates(config map[string]interface{}, secre
 			return fmt.Errorf("missing vault CA in secret %s", vaultCAFromSecret)
 		}
 
-		vaultConfig[api.EnvVaultCACert], err = createTempFile("vault-ca-cert", []byte(caPEM))
+		tf, err := file.CreateTempFile("vault-ca-cert", caPEM)
 		if err != nil {
 			return fmt.Errorf("failed to create temporary file for Vault CA: %w", err)
 		}
+		vaultConfig[api.EnvVaultCACert] = tf.Name()
+
 		// update the existing config
 		for key, value := range vaultConfig {
 			vc.vaultConfig[key] = value
@@ -480,31 +484,3 @@ func detectAuthMountPath(path string) (string, error) {
 
 	return authMountPath, nil
 }
-
-// createTempFile writes data to a temporary file that contains the pattern in
-// the filename (see os.CreateTemp for details).
-func createTempFile(pattern string, data []byte) (string, error) {
-	t, err := os.CreateTemp("", pattern)
-	if err != nil {
-		return "", fmt.Errorf("failed to create temporary file: %w", err)
-	}
-
-	// delete the tmpfile on error
-	defer func() {
-		if err != nil {
-			// ignore error on failure to remove tmpfile (gosec complains)
-			_ = os.Remove(t.Name())
-		}
-	}()
-
-	s, err := t.Write(data)
-	if err != nil || s != len(data) {
-		return "", fmt.Errorf("failed to write temporary file: %w", err)
-	}
-	err = t.Close()
-	if err != nil {
-		return "", fmt.Errorf("failed to close temporary file: %w", err)
-	}
-
-	return t.Name(), nil
-}

internal/kms/vault_test.go

@@ -18,7 +18,6 @@ package kms
 
 import (
 	"errors"
-	"os"
 	"testing"
 
 	loss "github.com/libopenstorage/secrets"
@@ -44,23 +43,6 @@ func TestDetectAuthMountPath(t *testing.T) {
 	}
 }
 
-func TestCreateTempFile(t *testing.T) {
-	t.Parallel()
-	data := []byte("Hello World!")
-	tmpfile, err := createTempFile("my-file", data)
-	if err != nil {
-		t.Errorf("createTempFile() failed: %s", err)
-	}
-	if tmpfile == "" {
-		t.Errorf("createTempFile() returned an empty filename")
-	}
-
-	err = os.Remove(tmpfile)
-	if err != nil {
-		t.Errorf("failed to remove tmpfile (%s): %s", tmpfile, err)
-	}
-}
-
 func TestSetConfigString(t *testing.T) {
 	t.Parallel()
 	const defaultValue = "default-value"

internal/kms/vault_tokens.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"strconv"
 
+	"github.com/ceph/ceph-csi/internal/util/file"
 	"github.com/ceph/ceph-csi/internal/util/k8s"
 
 	"github.com/hashicorp/vault/api"
@@ -378,10 +379,11 @@ func (vtc *vaultTenantConnection) initCertificates(config map[string]interface{}
 				return fmt.Errorf("failed to get CA certificate from secret %s: %w", vaultCAFromSecret, cErr)
 			}
 		}
-		vaultConfig[api.EnvVaultCACert], err = createTempFile("vault-ca-cert", []byte(cert))
-		if err != nil {
-			return fmt.Errorf("failed to create temporary file for Vault CA: %w", err)
+		cer, ferr := file.CreateTempFile("vault-ca-cert", cert)
+		if ferr != nil {
+			return fmt.Errorf("failed to create temporary file for Vault CA: %w", ferr)
 		}
+		vaultConfig[api.EnvVaultCACert] = cer.Name()
 	}
 
 	vaultClientCertFromSecret := "" // optional
@@ -403,10 +405,11 @@ func (vtc *vaultTenantConnection) initCertificates(config map[string]interface{}
 				return fmt.Errorf("failed to get client certificate from secret %s: %w", vaultCAFromSecret, cErr)
 			}
 		}
-		vaultConfig[api.EnvVaultClientCert], err = createTempFile("vault-ca-cert", []byte(cert))
-		if err != nil {
-			return fmt.Errorf("failed to create temporary file for Vault client certificate: %w", err)
+		cer, ferr := file.CreateTempFile("vault-ca-cert", cert)
+		if ferr != nil {
+			return fmt.Errorf("failed to create temporary file for Vault client certificate: %w", ferr)
 		}
+		vaultConfig[api.EnvVaultClientCert] = cer.Name()
 	}
 
 	vaultClientCertKeyFromSecret := "" // optional
@@ -432,10 +435,11 @@ func (vtc *vaultTenantConnection) initCertificates(config map[string]interface{}
 				return fmt.Errorf("failed to get client certificate key from secret %s: %w", vaultCAFromSecret, err)
 			}
 		}
-		vaultConfig[api.EnvVaultClientKey], err = createTempFile("vault-client-cert-key", []byte(certKey))
+		ckey, err := file.CreateTempFile("vault-client-cert-key", certKey)
 		if err != nil {
 			return fmt.Errorf("failed to create temporary file for Vault client cert key: %w", err)
 		}
+		vaultConfig[api.EnvVaultClientKey] = ckey.Name()
 	}
 
 	for key, value := range vaultConfig {

internal/rbd/driver/driver.go

@@ -226,6 +226,9 @@ func (r *Driver) setupCSIAddonsServer(conf *util.Config) error {
 	if conf.IsNodeServer {
 		rs := casrbd.NewReclaimSpaceNodeServer()
 		r.cas.RegisterService(rs)
+
+		ekr := casrbd.NewEncryptionKeyRotationServer()
+		r.cas.RegisterService(ekr)
 	}
 
 	// start the server, this does not block, it runs a new go-routine

internal/rbd/encryption.go

@@ -437,3 +437,74 @@ func (ri *rbdImage) RemoveDEK(ctx context.Context, volumeID string) error {
 
 	return nil
 }
+
+// GetEncryptionPassphraseSize returns the value of `encryptionPassphraseSize`.
+func GetEncryptionPassphraseSize() int {
+	return encryptionPassphraseSize
+}
+
+// RotateKey processes the key rotation for the RBD Volume.
+func (rv *rbdVolume) RotateEncryptionKey(ctx context.Context) error {
+	if !rv.isBlockEncrypted() {
+		return errors.New("key rotation not supported for the encryption type")
+	}
+
+	// Verify that the underlying device has been setup for encryption
+	currState, err := rv.checkRbdImageEncrypted(ctx)
+	if err != nil {
+		return fmt.Errorf("error while checking encryption state: %w", err)
+	}
+
+	if currState != rbdImageEncrypted {
+		return errors.New("key rotation not supported for unencrypted device")
+	}
+
+	// Get the device path for the underlying image
+	useNbd := rv.Mounter == rbdNbdMounter && hasNBD
+	devicePath, found := waitForPath(ctx, rv.Pool, rv.RadosNamespace, rv.RbdImageName, 1, useNbd)
+	if !found {
+		return errors.New("unable to get the device path for the image")
+	}
+
+	// Step 1: Get the current passphrase
+	oldPassphrase, err := rv.blockEncryption.GetCryptoPassphrase(ctx, rv.VolID)
+	if err != nil {
+		return fmt.Errorf("error in fetching the current passphrase: %w", err)
+	}
+
+	// Step 2: Add current key to slot 1
+	err = util.LuksAddKey(devicePath, oldPassphrase, oldPassphrase, "1")
+	if err != nil {
+		return fmt.Errorf("error in adding curr key to slot 1: %w", err)
+	}
+
+	// Step 3: Generate new key and add it to slot 0
+	newPassphrase, err := rv.blockEncryption.GetNewCryptoPassphrase(
+		GetEncryptionPassphraseSize())
+	if err != nil {
+		return fmt.Errorf("error in generating a new passphrase: %w", err)
+	}
+
+	err = util.LuksAddKey(devicePath, oldPassphrase, newPassphrase, "0")
+	if err != nil {
+		return fmt.Errorf("error in adding the new key to slot 0: %w", err)
+	}
+
+	// Step 4: Add the new key to KMS
+	err = rv.blockEncryption.StoreCryptoPassphrase(ctx, rv.VolID, newPassphrase)
+	if err != nil {
+		return fmt.Errorf("failed to update the new key into the KMS: %w", err)
+	}
+
+	// Step 5: Remove the old key from slot 1
+	// We use the newPassphrase to authenticate LUKS here
+	err = util.LuksRemoveKey(devicePath, newPassphrase, "1")
+	if err != nil {
+		// FIXME: Discuss if we should return an error here
+		//nolint:nilerr // Intentional
+		return nil
+	}
+
+	// Return error accordingly.
+	return nil
+}

internal/util/crypto.go

@@ -237,6 +237,11 @@ func (ve *VolumeEncryption) GetCryptoPassphrase(ctx context.Context, volumeID st
 	return ve.KMS.DecryptDEK(ctx, volumeID, passphrase)
 }
 
+// GetNewCryptoPassphrase returns a random passphrase of given length.
+func (ve *VolumeEncryption) GetNewCryptoPassphrase(length int) (string, error) {
+	return generateNewEncryptionPassphrase(length)
+}
+
 // generateNewEncryptionPassphrase generates a random passphrase for encryption.
 func generateNewEncryptionPassphrase(length int) (string, error) {
 	bytesPassphrase := make([]byte, length)