fix(resize): delegate RWX filesystem resize to share-manager

Signed-off-by: James Munson <[email protected]>

csi/deployment.go

@@ -418,6 +418,14 @@ func NewPluginDeployment(namespace, serviceAccount, nodeDriverRegistrarImage, li
 									Name:  "CSI_ENDPOINT",
 									Value: GetCSIEndpoint(),
 								},
+								{
+									Name: "POD_NAMESPACE",
+									ValueFrom: &corev1.EnvVarSource{
+										FieldRef: &corev1.ObjectFieldSelector{
+											FieldPath: "metadata.namespace",
+										},
+									},
+								},
 							},
 							VolumeMounts: []corev1.VolumeMount{
 								{

csi/manager.go

@@ -31,7 +31,11 @@ func (m *Manager) Run(driverName, nodeID, endpoint, identityVersion, managerURL
 
 	// Create GRPC servers
 	m.ids = NewIdentityServer(driverName, identityVersion)
-	m.ns = NewNodeServer(apiClient, nodeID)
+	m.ns, err = NewNodeServer(apiClient, nodeID)
+	if err != nil {
+		return errors.Wrap(err, "Failed to create CSI node server ")
+	}
+
 	m.cs = NewControllerServer(apiClient, nodeID)
 	s := NewNonBlockingGRPCServer()
 	s.Start(endpoint, m.ids, m.cs, m.ns)

csi/node_server.go

@@ -16,18 +16,23 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
+	"k8s.io/client-go/rest"
 	"k8s.io/mount-utils"
 
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
 	utilexec "k8s.io/utils/exec"
 
 	"github.com/longhorn/longhorn-manager/csi/crypto"
+	"github.com/longhorn/longhorn-manager/engineapi"
 	"github.com/longhorn/longhorn-manager/types"
 
 	lhns "github.com/longhorn/go-common-libs/ns"
 
 	longhornclient "github.com/longhorn/longhorn-manager/client"
 	longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2"
+	lhclientset "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned"
 )
 
 const (
@@ -49,13 +54,36 @@ var supportedFs = map[string]fsParameters{
 
 type NodeServer struct {
 	csi.UnimplementedNodeServer
-	apiClient *longhornclient.RancherClient
-	nodeID    string
-	caps      []*csi.NodeServiceCapability
-	log       *logrus.Entry
+	apiClient   *longhornclient.RancherClient
+	nodeID      string
+	caps        []*csi.NodeServiceCapability
+	log         *logrus.Entry
+	lhNamespace string
+	kubeClient  *clientset.Clientset
+	lhClient    *lhclientset.Clientset
 }
 
-func NewNodeServer(apiClient *longhornclient.RancherClient, nodeID string) *NodeServer {
+func NewNodeServer(apiClient *longhornclient.RancherClient, nodeID string) (*NodeServer, error) {
+	lhNamespace := os.Getenv(types.EnvPodNamespace)
+	if lhNamespace == "" {
+		return nil, fmt.Errorf("failed to detect pod namespace, environment variable %v is missing", types.EnvPodNamespace)
+	}
+
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get client config")
+	}
+
+	kubeClient, err := clientset.NewForConfig(config)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get k8s client")
+	}
+
+	lhClient, err := lhclientset.NewForConfig(config)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get longhorn clientset")
+	}
+
 	return &NodeServer{
 		apiClient: apiClient,
 		nodeID:    nodeID,
@@ -65,8 +93,11 @@ func NewNodeServer(apiClient *longhornclient.RancherClient, nodeID string) *Node
 				csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME,
 				csi.NodeServiceCapability_RPC_EXPAND_VOLUME,
 			}),
-		log: logrus.StandardLogger().WithField("component", "csi-node-server"),
-	}
+		log:         logrus.StandardLogger().WithField("component", "csi-node-server"),
+		lhNamespace: lhNamespace,
+		kubeClient:  kubeClient,
+		lhClient:    lhClient,
+	}, nil
 }
 
 // NodePublishVolume will mount the volume /dev/longhorn/<volume_name> to target_path
@@ -690,11 +721,46 @@ func (ns *NodeServer) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVo
 	}, nil
 }
 
+// NodeExpandShared Volume is designed to expand the file system in an RWX volume for ONLINE expansion.
+// It does so with a gRPC call into the share-manager pod.
+func (ns *NodeServer) NodeExpandSharedVolume(volumeName string) error {
+	log := ns.log.WithFields(logrus.Fields{"function": "NodeExpandSharedVolume"})
+
+	sm, err := ns.lhClient.LonghornV1beta2().ShareManagers(ns.lhNamespace).Get(context.TODO(), volumeName, metav1.GetOptions{})
+	if err != nil {
+		return errors.Wrap(err, "failed to get ShareManager CR")
+	}
+
+	podName := types.GetShareManagerPodNameFromShareManagerName(sm.Name)
+	pod, err := ns.kubeClient.CoreV1().Pods(ns.lhNamespace).Get(context.TODO(), podName, metav1.GetOptions{})
+	if err != nil {
+		return errors.Wrap(err, "failed to get ShareManager pod")
+	}
+
+	client, err := engineapi.NewShareManagerClient(sm, pod)
+	if err != nil {
+		return errors.Wrapf(err, "failed to launch gRPC client for share manager before resizing volume %v", volumeName)
+	}
+	defer client.Close()
+
+	// Each node with a workload pod will send an RPC request.  The first will win, and the others are no-ops.
+	err = client.FilesystemResize()
+	if status.Code(err) == codes.Unimplemented {
+		// This is a downrev longhorn-share-manager image.  It will be necessary either to kill the share-manager pod
+		// and let it restart with the new image, or scale the workload down and back up to accomplish the same thing.
+		// It might be tempting to delete the pod here, but there will be one of these calls for each workload pod,
+		// and it would be messy to have multiple kill requests at once.  So the kill will need to be done by the user.
+		log.WithError(err).Warn("Share Manager image is down-rev, does not implement RPC FilesystemResize.  Share Manager pod must be restarted with current image.")
+	}
+
+	return errors.Wrapf(err, "failed to expand shared volume %v", volumeName)
+}
+
 // NodeExpandVolume is designed to expand the file system for ONLINE expansion,
 func (ns *NodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) {
 	log := ns.log.WithFields(logrus.Fields{"function": "NodeExpandVolume"})
 
-	log.Infof("NodeNodeExpandVolume is called with req %+v", req)
+	log.Infof("NodeExpandVolume is called with req %+v", req)
 
 	if req.CapacityRange == nil {
 		return nil, status.Error(codes.InvalidArgument, "capacity range missing in request")
@@ -729,6 +795,20 @@ func (ns *NodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandV
 	if volume.State != string(longhorn.VolumeStateAttached) {
 		return nil, status.Errorf(codes.FailedPrecondition, "invalid state %v for volume %v node expansion", volume.State, volumeID)
 	}
+
+	if requiresSharedAccess(volume, volumeCapability) && !volume.Migratable {
+		if volume.AccessMode != string(longhorn.AccessModeReadWriteMany) {
+			return nil, status.Errorf(codes.FailedPrecondition, "volume %s requires shared access but is not marked for shared use", volumeID)
+		}
+
+		if err := ns.NodeExpandSharedVolume(volumeID); err != nil {
+			log.WithError(err).Errorf("failed to expand shared volume %v", volumeID)
+			return nil, err
+		}
+
+		return &csi.NodeExpandVolumeResponse{CapacityBytes: requestedSize}, nil
+	}
+
 	devicePath := volume.Controllers[0].Endpoint
 
 	mounter := &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: utilexec.New()}

engineapi/share_manager.go

@@ -39,6 +39,10 @@ func (c *ShareManagerClient) FilesystemTrim(encryptedDevice bool) error {
 	return c.grpcClient.FilesystemTrim(encryptedDevice)
 }
 
+func (c *ShareManagerClient) FilesystemResize() error {
+	return c.grpcClient.FilesystemResize()
+}
+
 func (c *ShareManagerClient) Unmount() error {
 	return c.grpcClient.Unmount()
 }

go.mod

@@ -1,6 +1,6 @@
 module github.com/longhorn/longhorn-manager
 
-go 1.22.7
+go 1.23
 
 toolchain go1.23.3
 
@@ -67,7 +67,7 @@ require (
 	github.com/longhorn/go-spdk-helper v0.0.0-20241130163407-e912304fab8b
 	github.com/longhorn/longhorn-engine v1.8.0-dev-20241201
 	github.com/longhorn/longhorn-instance-manager v1.8.0-dev-20241201
-	github.com/longhorn/longhorn-share-manager v1.7.0-rc1
+	github.com/longhorn/longhorn-share-manager v1.8.0-dev-20241117.0.20241125222022-2a7cb41cd62a
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.20.5
 	github.com/rancher/dynamiclistener v0.6.1
@@ -88,7 +88,7 @@ require (
 	k8s.io/apiextensions-apiserver v0.31.3
 	k8s.io/apimachinery v0.31.3
 	k8s.io/cli-runtime v0.31.3
-	k8s.io/client-go v0.31.3
+	k8s.io/client-go v1.5.2
 	k8s.io/kubernetes v1.31.3
 	k8s.io/metrics v0.31.3
 	k8s.io/mount-utils v0.31.3
@@ -102,25 +102,24 @@ require (
 	github.com/distribution/reference v0.5.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
-	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/jonboulle/clockwork v0.4.0 // indirect
 	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/longhorn/types v0.0.0-20241123075624-48c550af4eab // indirect
 	github.com/mitchellh/go-ps v1.0.0 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
 	github.com/moby/term v0.5.0 // indirect
-	github.com/opencontainers/runc v1.1.14 // indirect
-	github.com/opencontainers/runtime-spec v1.1.0 // indirect
+	github.com/opencontainers/runc v1.2.2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
 	github.com/shirou/gopsutil/v3 v3.24.5 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect
 	golang.org/x/tools v0.27.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241118233622-e639e219e697 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
@@ -139,16 +138,16 @@ require (
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.12.1 // indirect
 	github.com/evanphx/json-patch v5.9.0+incompatible // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/fsnotify/fsnotify v1.8.0 // indirect
 	github.com/gammazero/deque v0.2.1 // indirect
 	github.com/gammazero/workerpool v1.1.3 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
-	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/jsonreference v0.21.0 // indirect
 	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -168,7 +167,7 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
-	github.com/moby/sys/mountinfo v0.7.1 // indirect
+	github.com/moby/sys/mountinfo v0.7.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
@@ -202,7 +201,7 @@ require (
 	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.26.0 // indirect
 	golang.org/x/crypto v0.29.0 // indirect
-	golang.org/x/oauth2 v0.23.0 // indirect
+	golang.org/x/oauth2 v0.24.0 // indirect
 	golang.org/x/sync v0.9.0
 	golang.org/x/term v0.26.0 // indirect
 	golang.org/x/text v0.20.0
@@ -219,10 +218,10 @@ require (
 	k8s.io/klog/v2 v2.130.1 // indirect
 	k8s.io/kms v0.31.3 // indirect
 	k8s.io/kube-aggregator v0.31.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
+	k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect
 	k8s.io/kubelet v0.0.0 // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 // indirect
-	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.3 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )