From 38a249d36343308a02b9491f894238b84dc9ef7a Mon Sep 17 00:00:00 2001 From: Christian Schlatter Date: Wed, 21 Jul 2021 08:06:52 +0200 Subject: [PATCH 01/22] Add ProjectID and DomainID for creating volumes --- pkg/cloud/cloud.go | 8 ++++++-- pkg/cloud/config.go | 2 ++ pkg/cloud/project.go | 18 ++++++++++++++++++ pkg/cloud/volumes.go | 4 +++- pkg/driver/controller.go | 12 +++++++++++- 5 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 pkg/cloud/project.go diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index 03af562..238e66f 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -16,9 +16,12 @@ type Interface interface { ListZonesID(ctx context.Context) ([]string, error) + GetDomainID(ctx context.Context) (string, error) + GetProjectID() string + GetVolumeByID(ctx context.Context, volumeID string) (*Volume, error) GetVolumeByName(ctx context.Context, name string) (*Volume, error) - CreateVolume(ctx context.Context, diskOfferingID, zoneID, name string, sizeInGB int64) (string, error) + CreateVolume(ctx context.Context, diskOfferingID, projectID, domainID, zoneID, name string, sizeInGB int64) (string, error) DeleteVolume(ctx context.Context, id string) error AttachVolume(ctx context.Context, volumeID, vmID string) (string, error) DetachVolume(ctx context.Context, volumeID string) error @@ -54,10 +57,11 @@ var ( // client is the implementation of Interface. type client struct { *cloudstack.CloudStackClient + ProjectID string } // New creates a new cloud connector, given its configuration. func New(config *Config) Interface { csClient := cloudstack.NewAsyncClient(config.APIURL, config.APIKey, config.SecretKey, config.VerifySSL) - return &client{csClient} + return &client{csClient, config.ProjectID} } diff --git a/pkg/cloud/config.go b/pkg/cloud/config.go index cb851bd..fa9ff66 100644 --- a/pkg/cloud/config.go +++ b/pkg/cloud/config.go @@ -12,6 +12,7 @@ type Config struct { APIKey string SecretKey string VerifySSL bool + ProjectID string } // csConfig wraps the config for the CloudStack cloud provider. @@ -42,5 +43,6 @@ func ReadConfig(configFilePath string) (*Config, error) { APIKey: cfg.Global.APIKey, SecretKey: cfg.Global.SecretKey, VerifySSL: cfg.Global.SSLNoVerify, + ProjectID: cfg.Global.ProjectID, }, nil } diff --git a/pkg/cloud/project.go b/pkg/cloud/project.go new file mode 100644 index 0000000..94c2fa8 --- /dev/null +++ b/pkg/cloud/project.go @@ -0,0 +1,18 @@ +package cloud + +import ( + "context" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" +) + +func (c *client) GetDomainID(ctx context.Context) (string, error) { + p, _, err := c.Project.GetProjectByID(c.ProjectID) + ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "GetProjectByID", "params", map[string]string{ + "projectID": c.ProjectID, + }) + return p.Domainid, err +} + +func (c *client) GetProjectID() string { + return c.ProjectID +} diff --git a/pkg/cloud/volumes.go b/pkg/cloud/volumes.go index 394be39..ee95d35 100644 --- a/pkg/cloud/volumes.go +++ b/pkg/cloud/volumes.go @@ -66,12 +66,14 @@ func (c *client) GetVolumeByName(ctx context.Context, name string) (*Volume, err return &v, nil } -func (c *client) CreateVolume(ctx context.Context, diskOfferingID, zoneID, name string, sizeInGB int64) (string, error) { +func (c *client) CreateVolume(ctx context.Context, diskOfferingID, projectID, domainID, zoneID, name string, sizeInGB int64) (string, error) { p := c.Volume.NewCreateVolumeParams() p.SetDiskofferingid(diskOfferingID) p.SetZoneid(zoneID) p.SetName(name) p.SetSize(sizeInGB) + p.SetDomainid(domainID) + p.SetProjectid(projectID) ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "CreateVolume", "params", map[string]string{ "diskofferingid": diskOfferingID, "zoneid": zoneID, diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index 4cc3f14..3116317 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -114,7 +114,17 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol zoneID = t.ZoneID } - volID, err := cs.connector.CreateVolume(ctx, diskOfferingID, zoneID, name, sizeInGB) + projectID := cs.connector.GetProjectID() + domaindID := "" + + if projectID != "" { + domaindID, err = cs.connector.GetDomainID(ctx) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + } + + volID, err := cs.connector.CreateVolume(ctx, diskOfferingID, projectID, domaindID, zoneID, name, sizeInGB) if err != nil { return nil, status.Errorf(codes.Internal, "Cannot create volume %s: %v", name, err.Error()) } From 5fe6f87e912e68363800f4c55fcf038ae73e3a4a Mon Sep 17 00:00:00 2001 From: Christian Schlatter Date: Wed, 21 Jul 2021 08:45:20 +0200 Subject: [PATCH 02/22] update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 14f029d..13a9625 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ api-url = api-key = secret-key = ssl-no-verify = +project-id = ``` Create a secret named `cloudstack-secret` in namespace `kube-system`: From 0d02f0799e39d8a5d62584067faccc6665f4bfc9 Mon Sep 17 00:00:00 2001 From: Christian Schlatter Date: Wed, 4 Aug 2021 01:43:54 +0200 Subject: [PATCH 03/22] add hypervisor to volumes, add rescan with script, find disk by path instead of volume, fix vmware device id, add eudev and bash dependencies, add rescan script for scsi bus --- cmd/cloudstack-csi-driver/Dockerfile | 6 +- pkg/cloud/cloud.go | 1 + pkg/cloud/fake/fake.go | 10 +- pkg/cloud/volumes.go | 3 + pkg/driver/controller.go | 6 +- pkg/driver/node.go | 19 +- pkg/mount/fake.go | 2 +- pkg/mount/mount.go | 77 +- rescan-scsi-bus.sh | 1190 ++++++++++++++++++++++++++ 9 files changed, 1269 insertions(+), 45 deletions(-) create mode 100755 rescan-scsi-bus.sh diff --git a/cmd/cloudstack-csi-driver/Dockerfile b/cmd/cloudstack-csi-driver/Dockerfile index a5eaccc..7268f17 100644 --- a/cmd/cloudstack-csi-driver/Dockerfile +++ b/cmd/cloudstack-csi-driver/Dockerfile @@ -11,7 +11,11 @@ RUN apk add --no-cache \ # Provides mkfs.xfs xfsprogs \ # Provides blkid, also used by k8s.io/mount-utils - blkid + blkid \ + eudev \ + bash COPY ./bin/cloudstack-csi-driver /cloudstack-csi-driver +COPY rescan-scsi-bus.sh /usr/bin/ +RUN chmod +x /usr/bin/rescan-scsi-bus.sh ENTRYPOINT ["/cloudstack-csi-driver"] \ No newline at end of file diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index 238e66f..2424294 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -40,6 +40,7 @@ type Volume struct { VirtualMachineID string DeviceID string + Hypervisor string } // VM represents a CloudStack Virtual Machine. diff --git a/pkg/cloud/fake/fake.go b/pkg/cloud/fake/fake.go index 23c7e12..5fcc1ad 100644 --- a/pkg/cloud/fake/fake.go +++ b/pkg/cloud/fake/fake.go @@ -72,7 +72,7 @@ func (f *fakeConnector) GetVolumeByName(ctx context.Context, name string) (*clou return nil, cloud.ErrNotFound } -func (f *fakeConnector) CreateVolume(ctx context.Context, diskOfferingID, zoneID, name string, sizeInGB int64) (string, error) { +func (f *fakeConnector) CreateVolume(ctx context.Context, diskOfferingID, projectID, domainID, zoneID, name string, sizeInGB int64) (string, error) { id, _ := uuid.GenerateUUID() vol := cloud.Volume{ ID: id, @@ -100,3 +100,11 @@ func (f *fakeConnector) AttachVolume(ctx context.Context, volumeID, vmID string) } func (f *fakeConnector) DetachVolume(ctx context.Context, volumeID string) error { return nil } + +func (f *fakeConnector) GetDomainID(ctx context.Context) (string, error) { + return "domain", nil +} + +func (f *fakeConnector) GetProjectID() string { + return "test" +} diff --git a/pkg/cloud/volumes.go b/pkg/cloud/volumes.go index ee95d35..f27c18e 100644 --- a/pkg/cloud/volumes.go +++ b/pkg/cloud/volumes.go @@ -32,6 +32,7 @@ func (c *client) GetVolumeByID(ctx context.Context, volumeID string) (*Volume, e DiskOfferingID: vol.Diskofferingid, ZoneID: vol.Zoneid, VirtualMachineID: vol.Virtualmachineid, + Hypervisor: vol.Hypervisor, DeviceID: strconv.FormatInt(vol.Deviceid, 10), } return &v, nil @@ -62,7 +63,9 @@ func (c *client) GetVolumeByName(ctx context.Context, name string) (*Volume, err ZoneID: vol.Zoneid, VirtualMachineID: vol.Virtualmachineid, DeviceID: strconv.FormatInt(vol.Deviceid, 10), + Hypervisor: vol.Hypervisor, } + return &v, nil } diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index 3116317..6498a2e 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -3,6 +3,7 @@ package driver import ( "context" "fmt" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "math/rand" "github.com/apalia/cloudstack-csi-driver/pkg/cloud" @@ -33,7 +34,6 @@ func NewControllerServer(connector cloud.Interface) csi.ControllerServer { } func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { - // Check arguments if req.GetName() == "" { @@ -269,6 +269,8 @@ func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *cs return nil, status.Errorf(codes.Internal, "Cannot attach volume %s: %s", volumeID, err.Error()) } + ctxzap.Extract(ctx).Sugar().Debugf("volume %s attached successfully on node %s", volumeID, nodeID) + publishContext := map[string]string{ deviceIDContextKey: deviceID, } @@ -319,6 +321,8 @@ func (cs *controllerServer) ControllerUnpublishVolume(ctx context.Context, req * return nil, status.Errorf(codes.Internal, "Cannot detach volume %s: %s", volumeID, err.Error()) } + ctxzap.Extract(ctx).Sugar().Debugf("volume %s detached successfully on node %s", volumeID, nodeID) + return &csi.ControllerUnpublishVolumeResponse{}, nil } diff --git a/pkg/driver/node.go b/pkg/driver/node.go index dcb96e3..2c07af8 100644 --- a/pkg/driver/node.go +++ b/pkg/driver/node.go @@ -60,11 +60,14 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol return nil, status.Error(codes.InvalidArgument, "Volume capability not supported") } + ctxzap.Extract(ctx).Sugar().Infow("#### Mount 1 target: " + target) // Now, find the device path + v, _ := ns.connector.GetVolumeByID(ctx, volumeID) + deviceID := req.PublishContext[deviceIDContextKey] - devicePath, err := ns.mounter.GetDevicePath(ctx, volumeID) + devicePath, err := ns.mounter.GetDevicePath(ctx, v.DeviceID, v.Hypervisor) if err != nil { return nil, status.Errorf(codes.Internal, "Cannot find device path for volume %s: %s", volumeID, err.Error()) } @@ -113,6 +116,7 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol return nil, status.Error(codes.Internal, err.Error()) } } + ctxzap.Extract(ctx).Sugar().Debugf("Staged volume device %s on %s on target %s successfully", volumeID, devicePath, target) return &csi.NodeStageVolumeResponse{}, nil } @@ -166,6 +170,8 @@ func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag return nil, status.Errorf(codes.Internal, "Could not unmount target %q: %v", target, err) } + ctxzap.Extract(ctx).Sugar().Debugf("NodeUnstageVolume: unmounted %d on target %d", dev, target) + return &csi.NodeUnstageVolumeResponse{}, nil } @@ -191,6 +197,10 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis if req.GetStagingTargetPath() == "" { return nil, status.Error(codes.InvalidArgument, "Staging target path missing in request") } + v, err := ns.connector.GetVolumeByID(ctx, volumeID) + if err != nil { + + } readOnly := req.GetReadonly() options := []string{"bind"} @@ -236,12 +246,13 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis if err := ns.mounter.Mount(source, targetPath, fsType, options); err != nil { return nil, status.Errorf(codes.Internal, "failed to mount %s at %s: %s", source, targetPath, err.Error()) } + ctxzap.Extract(ctx).Sugar().Debugf("mount volume %s from source %s on target %s ", volumeID, source, targetPath) } if req.GetVolumeCapability().GetBlock() != nil { volumeID := req.GetVolumeId() - devicePath, err := ns.mounter.GetDevicePath(ctx, volumeID) + devicePath, err := ns.mounter.GetDevicePath(ctx, v.DeviceID, v.Hypervisor) if err != nil { return nil, status.Errorf(codes.Internal, "Cannot find device path for volume %s: %s", volumeID, err.Error()) } @@ -268,12 +279,14 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis if err := ns.mounter.Mount(devicePath, targetPath, "", options); err != nil { return nil, status.Errorf(codes.Internal, "failed to mount %s at %s: %s", devicePath, targetPath, err.Error()) } + ctxzap.Extract(ctx).Sugar().Infow("### mount volume on devicePath: " + devicePath + " and targetPath: " + targetPath) } return &csi.NodePublishVolumeResponse{}, nil } func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { + if req.GetVolumeId() == "" { return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") } @@ -290,6 +303,7 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu return nil, status.Errorf(codes.Internal, "Error %v", err) } + ctxzap.Extract(ctx).Sugar().Debugw("node unpublish (call unmount) volume", "id", volumeID, "targetPath", targetPath) err := ns.mounter.Unmount(targetPath) if err != nil { return nil, status.Errorf(codes.Internal, "Unmount of targetpath %s failed with error %v", targetPath, err) @@ -298,6 +312,7 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu if err != nil && !os.IsNotExist(err) { return nil, status.Errorf(codes.Internal, "Deleting %s failed with error %v", targetPath, err) } + ctxzap.Extract(ctx).Sugar().Debugw("sucessfully unpublish volume", "id", volumeID, "targetPath", targetPath) return &csi.NodeUnpublishVolumeResponse{}, nil } diff --git a/pkg/mount/fake.go b/pkg/mount/fake.go index 985b920..9cae82c 100644 --- a/pkg/mount/fake.go +++ b/pkg/mount/fake.go @@ -26,7 +26,7 @@ func NewFake() Interface { } } -func (m *fakeMounter) GetDevicePath(ctx context.Context, volumeID string) (string, error) { +func (m *fakeMounter) GetDevicePath(ctx context.Context, volumeID string, hypervisor string) (string, error) { return "/dev/sdb", nil } diff --git a/pkg/mount/mount.go b/pkg/mount/mount.go index 7a9f79b..a4a4bec 100644 --- a/pkg/mount/mount.go +++ b/pkg/mount/mount.go @@ -5,20 +5,19 @@ package mount import ( "context" "fmt" - "io/ioutil" - "os" - "path/filepath" - "strings" - "time" - "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/mount-utils" "k8s.io/utils/exec" + "os" + "path/filepath" + "strconv" + "strings" + "time" ) const ( - diskIDPath = "/dev/disk/by-id" + diskIDPath = "/dev/disk/by-path" ) // Interface defines the set of methods to allow for @@ -29,7 +28,7 @@ type Interface interface { FormatAndMount(source string, target string, fstype string, options []string) error - GetDevicePath(ctx context.Context, volumeID string) (string, error) + GetDevicePath(ctx context.Context, volumeID string, hypervisor string) (string, error) GetDeviceName(mountPath string) (string, int, error) ExistsPath(filename string) (bool, error) MakeDir(pathname string) error @@ -52,7 +51,23 @@ func New() Interface { } } -func (m *mounter) GetDevicePath(ctx context.Context, volumeID string) (string, error) { +func (m *mounter) GetDevicePath(ctx context.Context, deviceID string, hypervisor string) (string, error) { + + ctxzap.Extract(ctx).Sugar().Debugf("device id: '%s' (Hypervisor: %s)", deviceID, hypervisor) + + if strings.ToLower(hypervisor) == "vmware" { + ctxzap.Extract(ctx).Sugar().Warnf("volume hypervisor is VMWare, try to correct SCSI ID") + idInt, _ := strconv.Atoi(deviceID) + if idInt > 3 { + idInt-- + deviceID = fmt.Sprintf("%d", idInt) + ctxzap.Extract(ctx).Sugar().Warnf("new device id: %s", deviceID) + } + } + + deviceID = fmt.Sprintf("pci-0000:00:10.0-scsi-0:0:%s:0", deviceID) + ctxzap.Extract(ctx).Sugar().Debugf("device path: %s/%s", diskIDPath, deviceID) + backoff := wait.Backoff{ Duration: 1 * time.Second, Factor: 1.1, @@ -61,12 +76,13 @@ func (m *mounter) GetDevicePath(ctx context.Context, volumeID string) (string, e var devicePath string err := wait.ExponentialBackoffWithContext(ctx, backoff, func() (bool, error) { - path, err := m.getDevicePathBySerialID(volumeID) + path, err := m.getDevicePathBySerialID(deviceID) if err != nil { return false, err } if path != "" { devicePath = path + ctxzap.Extract(ctx).Sugar().Debugf("device path found: %s", path) return true, nil } m.probeVolume(ctx) @@ -74,25 +90,21 @@ func (m *mounter) GetDevicePath(ctx context.Context, volumeID string) (string, e }) if err == wait.ErrWaitTimeout { - return "", fmt.Errorf("Failed to find device for the volumeID: %q within the alloted time", volumeID) + return "", fmt.Errorf("Failed to find device for the deviceID: %q within the alloted time", deviceID) } else if devicePath == "" { - return "", fmt.Errorf("Device path was empty for volumeID: %q", volumeID) + return "", fmt.Errorf("Device path was empty for deviceID: %q", deviceID) } return devicePath, nil } func (m *mounter) getDevicePathBySerialID(volumeID string) (string, error) { - sourcePathPrefixes := []string{"virtio-", "scsi-", "scsi-0QEMU_QEMU_HARDDISK_"} - serial := diskUUIDToSerial(volumeID) - for _, prefix := range sourcePathPrefixes { - source := filepath.Join(diskIDPath, prefix+serial) - _, err := os.Stat(source) - if err == nil { - return source, nil - } - if !os.IsNotExist(err) { - return "", err - } + source := filepath.Join(diskIDPath, volumeID) + _, err := os.Stat(source) + if err == nil { + return source, nil + } + if !os.IsNotExist(err) { + return "", err } return "", nil } @@ -101,24 +113,11 @@ func (m *mounter) probeVolume(ctx context.Context) { log := ctxzap.Extract(ctx).Sugar() log.Debug("Scaning SCSI host...") - scsiPath := "/sys/class/scsi_host/" - if dirs, err := ioutil.ReadDir(scsiPath); err == nil { - for _, f := range dirs { - name := scsiPath + f.Name() + "/scan" - data := []byte("- - -") - if err = ioutil.WriteFile(name, data, 0666); err != nil { - log.Warnf("Failed to rescan scsi host %s", name) - } - } - } else { - log.Warnf("Failed to read %s, err %v", scsiPath, err) - } - - args := []string{"trigger"} - cmd := m.Exec.Command("udevadm", args...) + args := []string{"-a"} + cmd := m.Exec.Command("rescan-scsi-bus.sh", args...) _, err := cmd.CombinedOutput() if err != nil { - log.Warnf("Error running udevadm trigger %v\n", err) + log.Warnf("Error running rescan-scsi-bus.sh -a %v\n", err) } } diff --git a/rescan-scsi-bus.sh b/rescan-scsi-bus.sh new file mode 100755 index 0000000..8bb1227 --- /dev/null +++ b/rescan-scsi-bus.sh @@ -0,0 +1,1190 @@ +#!/bin/bash +# Skript to rescan SCSI bus, using the +# scsi add-single-device mechanism +# (c) 1998--2010 Kurt Garloff , GNU GPL v2 or v3 +# (c) 2006--2013 Hannes Reinecke, GNU GPL v2 or later +# $Id: rescan-scsi-bus.sh,v 1.57 2012/03/31 14:08:48 garloff Exp $ + +SCAN_WILD_CARD=4294967295 + +setcolor () +{ + red="\e[0;31m" + green="\e[0;32m" + yellow="\e[0;33m" + bold="\e[0;1m" + norm="\e[0;0m" +} + +unsetcolor () +{ + red=""; green="" + yellow=""; norm="" +} + +echo_debug() +{ + if [ $debug -eq 1 ] ; then + echo "$1" + fi +} + +# Output some text and return cursor to previous position +# (only works for simple strings) +# Stores length of string in LN and returns it +print_and_scroll_back () +{ + STRG="$1" + LN=${#STRG} + BK="" + declare -i cntr=0 + while test $cntr -lt $LN; do BK="$BK\e[D"; let cntr+=1; done + echo -en "$STRG$BK" + return $LN +} + +# Overwrite a text of length $1 (fallback to $LN) with whitespace +white_out () +{ + BK=""; WH="" + if test -n "$1"; then LN=$1; fi + declare -i cntr=0 + while test $cntr -lt $LN; do BK="$BK\e[D"; WH="$WH "; let cntr+=1; done + echo -en "$WH$BK" +} + +# Return hosts. sysfs must be mounted +findhosts_26 () +{ + hosts=`find /sys/class/scsi_host/host* -maxdepth 4 -type d -o -type l 2> /dev/null | awk -F'/' '{print $5}' | sed -e 's~host~~' | sort -nu` + scsi_host_data=`echo "$hosts" | sed -e 's~^~/sys/class/scsi_host/host~'` + for hostdir in $scsi_host_data; do + hostno=${hostdir#/sys/class/scsi_host/host} + if [ -f $hostdir/isp_name ] ; then + hostname="qla2xxx" + elif [ -f $hostdir/lpfc_drvr_version ] ; then + hostname="lpfc" + else + hostname=`cat $hostdir/proc_name` + fi + #hosts="$hosts $hostno" + echo_debug "Host adapter $hostno ($hostname) found." + done + if [ -z "$hosts" ] ; then + echo "No SCSI host adapters found in sysfs" + exit 1; + fi + # Not necessary just use double quotes around variable to preserve new lines + #hosts=`echo $hosts | tr ' ' '\n'` +} + +# Return hosts. /proc/scsi/HOSTADAPTER/? must exist +findhosts () +{ + hosts= + for driverdir in /proc/scsi/*; do + driver=${driverdir#/proc/scsi/} + if test $driver = scsi -o $driver = sg -o $driver = dummy -o $driver = device_info; then continue; fi + for hostdir in $driverdir/*; do + name=${hostdir#/proc/scsi/*/} + if test $name = add_map -o $name = map -o $name = mod_parm; then continue; fi + num=$name + driverinfo=$driver + if test -r $hostdir/status; then + num=$(printf '%d\n' `sed -n 's/SCSI host number://p' $hostdir/status`) + driverinfo="$driver:$name" + fi + hosts="$hosts $num" + echo "Host adapter $num ($driverinfo) found." + done + done +} + +printtype () +{ + local type=$1 + + case "$type" in + 0) echo "Direct-Access " ;; + 1) echo "Sequential-Access" ;; + 2) echo "Printer " ;; + 3) echo "Processor " ;; + 4) echo "WORM " ;; + 5) echo "CD-ROM " ;; + 6) echo "Scanner " ;; + 7) echo "Optical Device " ;; + 8) echo "Medium Changer " ;; + 9) echo "Communications " ;; + 10) echo "Unknown " ;; + 11) echo "Unknown " ;; + 12) echo "RAID " ;; + 13) echo "Enclosure " ;; + 14) echo "Direct-Access-RBC" ;; + *) echo "Unknown " ;; + esac +} + +print02i() +{ + if [ "$1" = "*" ] ; then + echo "00" + else + printf "%02i" "$1" + fi +} + +# Get /proc/scsi/scsi info for device $host:$channel:$id:$lun +# Optional parameter: Number of lines after first (default = 2), +# result in SCSISTR, return code 1 means empty. +procscsiscsi () +{ + if test -z "$1"; then LN=2; else LN=$1; fi + CHANNEL=`print02i "$channel"` + ID=`print02i "$id"` + LUN=`print02i "$lun"` + if [ -d /sys/class/scsi_device ]; then + SCSIPATH="/sys/class/scsi_device/${host}:${channel}:${id}:${lun}" + if [ -d "$SCSIPATH" ] ; then + SCSISTR="Host: scsi${host} Channel: $CHANNEL Id: $ID Lun: $LUN" + if [ "$LN" -gt 0 ] ; then + IVEND=$(cat ${SCSIPATH}/device/vendor) + IPROD=$(cat ${SCSIPATH}/device/model) + IPREV=$(cat ${SCSIPATH}/device/rev) + SCSIDEV=$(printf ' Vendor: %-08s Model: %-16s Rev: %-4s' "$IVEND" "$IPROD" "$IPREV") + SCSISTR="$SCSISTR +$SCSIDEV" + fi + if [ "$LN" -gt 1 ] ; then + ILVL=$(cat ${SCSIPATH}/device/scsi_level) + type=$(cat ${SCSIPATH}/device/type) + ITYPE=$(printtype $type) + SCSITMP=$(printf ' Type: %-16s ANSI SCSI revision: %02d' "$ITYPE" "$((ILVL - 1))") + SCSISTR="$SCSISTR +$SCSITMP" + fi + else + return 1 + fi + else + grepstr="scsi$host Channel: $CHANNEL Id: $ID Lun: $LUN" + SCSISTR=`cat /proc/scsi/scsi | grep -A$LN -e"$grepstr"` + fi + if test -z "$SCSISTR"; then return 1; else return 0; fi +} + +# Find sg device with 2.6 sysfs support +sgdevice26 () +{ + if test -e /sys/class/scsi_device/$host\:$channel\:$id\:$lun/device/generic; then + SGDEV=`readlink /sys/class/scsi_device/$host\:$channel\:$id\:$lun/device/generic` + SGDEV=`basename $SGDEV` + else + for SGDEV in /sys/class/scsi_generic/sg*; do + DEV=`readlink $SGDEV/device` + if test "${DEV##*/}" = "$host:$channel:$id:$lun"; then + SGDEV=`basename $SGDEV`; return + fi + done + SGDEV="" + fi +} + +# Find sg device with 2.4 report-devs extensions +sgdevice24 () +{ + if procscsiscsi 3; then + SGDEV=`echo "$SCSISTR" | grep 'Attached drivers:' | sed 's/^ *Attached drivers: \(sg[0-9]*\).*/\1/'` + fi +} + +# Find sg device that belongs to SCSI device $host $channel $id $lun +# and return in SGDEV +sgdevice () +{ + SGDEV= + if test -d /sys/class/scsi_device; then + sgdevice26 + else + DRV=`grep 'Attached drivers:' /proc/scsi/scsi 2>/dev/null` + repdevstat=$((1-$?)) + if [ $repdevstat = 0 ]; then + echo "scsi report-devs 1" >/proc/scsi/scsi + DRV=`grep 'Attached drivers:' /proc/scsi/scsi 2>/dev/null` + if [ $? = 1 ]; then return; fi + fi + if ! `echo $DRV | grep 'drivers: sg' >/dev/null`; then + modprobe sg + fi + sgdevice24 + if [ $repdevstat = 0 ]; then + echo "scsi report-devs 0" >/proc/scsi/scsi + fi + fi +} + +# Test if SCSI device is still responding to commands +# Return values: +# 0 device is present +# 1 device has changed +# 2 device has been removed +testonline () +{ + : testonline + RC=0 + if test ! -x /usr/bin/sg_turs; then return 0; fi + sgdevice + if test -z "$SGDEV"; then return 0; fi + sg_turs /dev/$SGDEV &>/dev/null + RC=$? + # Handle in progress of becoming ready and unit attention -- wait at max 11s + declare -i ctr=0 + if test $RC = 2 -o $RC = 6; then + RMB=`sg_inq /dev/$SGDEV | grep 'RMB=' | sed 's/^.*RMB=\(.\).*$/\1/'` + print_and_scroll_back "$host:$channel:$id:$lun $SGDEV ($RMB) " + fi + while test $RC = 2 -o $RC = 6 && test $ctr -le 8; do + if test $RC = 2 -a "$RMB" != "1"; then echo -n "."; let LN+=1; sleep 1 + else usleep 20000; fi + let ctr+=1 + sg_turs /dev/$SGDEV &>/dev/null + RC=$? + done + if test $ctr != 0; then white_out; fi + # echo -e "\e[A\e[A\e[A${yellow}Test existence of $SGDEV = $RC ${norm} \n\n\n" + if test $RC = 1; then return $RC; fi + # Reset RC (might be !=0 for passive paths) + RC=0 + # OK, device online, compare INQUIRY string + INQ=`sg_inq $sg_len_arg /dev/$SGDEV 2>/dev/null` + IVEND=`echo "$INQ" | grep 'Vendor identification:' | sed 's/^[^:]*: \(.*\)$/\1/'` + IPROD=`echo "$INQ" | grep 'Product identification:' | sed 's/^[^:]*: \(.*\)$/\1/'` + IPREV=`echo "$INQ" | grep 'Product revision level:' | sed 's/^[^:]*: \(.*\)$/\1/'` + STR=`printf " Vendor: %-08s Model: %-16s Rev: %-4s" "$IVEND" "$IPROD" "$IPREV"` + IPTYPE=`echo "$INQ" | sed -n 's/.* Device_type=\([0-9]*\) .*/\1/p'` + IPQUAL=`echo "$INQ" | sed -n 's/ *PQual=\([0-9]*\) Device.*/\1/p'` + if [ "$IPQUAL" != 0 ] ; then + echo -e "\e[A\e[A\e[A\e[A${red}$SGDEV changed: ${bold}LU not available (PQual $IPQUAL)${norm} \n\n\n" + return 2 + fi + + TYPE=$(printtype $IPTYPE) + if ! procscsiscsi ; then + echo -e "\e[A\e[A\e[A\e[A${red}$SGDEV removed.\n\n\n" + return 2 + fi + TMPSTR=`echo "$SCSISTR" | grep 'Vendor:'` + if [ "$TMPSTR" != "$STR" ]; then + echo -e "\e[A\e[A\e[A\e[A${red}$SGDEV changed: ${bold}\nfrom:${SCSISTR#* } \nto: $STR ${norm} \n\n\n" + return 1 + fi + TMPSTR=`echo "$SCSISTR" | sed -n 's/.*Type: *\(.*\) *ANSI.*/\1/p'` + NTMPSTR="$(sed -e 's/[[:space:]]*$//' -e 's/^[[:space:]]*//' <<<${TMPSTR})" + NTYPE="$(sed -e 's/[[:space:]]*$//' -e 's/^[[:space:]]*//' <<<${TYPE})" + if [ "$NTMPSTR" != "$NTYPE" ] ; then + echo -e "\e[A\e[A\e[A\e[A${red}$SGDEV changed: ${bold}\nfrom:${TMPSTR} \nto: $TYPE ${norm} \n\n\n" + return 1 + fi + return $RC +} + +# Test if SCSI device $host $channen $id $lun exists +# Outputs description from /proc/scsi/scsi (unless arg passed) +# Returns SCSISTR (empty if no dev) +testexist () +{ + : testexist + SCSISTR= + if procscsiscsi && test -z "$1"; then + echo "$SCSISTR" | head -n1 + echo "$SCSISTR" | tail -n2 | pr -o4 -l1 + fi +} + +# Returns the list of existing channels per host +chanlist () +{ + local hcil + local cil + local chan + local tmpchan + + for dev in /sys/class/scsi_device/${host}:* ; do + [ -d $dev ] || continue; + hcil=${dev##*/} + cil=${hcil#*:} + chan=${cil%%:*} + for tmpchan in $channelsearch ; do + if test "$chan" -eq $tmpchan ; then + chan= + fi + done + if test -n "$chan" ; then + channelsearch="$channelsearch $chan" + fi + done + if test -z "$channelsearch"; then channelsearch="0"; fi +} + +# Returns the list of existing targets per host +idlist () +{ + local hcil + local target + local tmpid + local newid + + idsearch=$(ls /sys/class/scsi_device/ | sed -n "s/${host}:${channel}:\([0-9]*\):[0-9]*/\1/p" | uniq) + echo "${channel} - -" > /sys/class/scsi_host/host${host}/scan + # Rescan to check if we found new targets + newsearch=$(ls /sys/class/scsi_device/ | sed -n "s/${host}:${channel}:\([0-9]*\):[0-9]*/\1/p" | uniq) + for id in $newsearch ; do + newid=$id + for tmpid in $idsearch ; do + if test $id -eq $tmpid ; then + newid= + break + fi + done + if test -n "$newid" ; then + id=$newid + for dev in /sys/class/scsi_device/${host}:${channel}:${newid}:* ; do + [ -d $dev ] || continue; + hcil=${dev##*/} + lun=${hcil##*:} + printf "\r${green}NEW: $norm" + testexist + if test "$SCSISTR" ; then + incrfound "$hcil" + fi + done + fi + done +} + +# Returns the list of existing LUNs from device $host $channel $id $lun +# and returns list to stdout +getluns() +{ + sgdevice + if test -z "$SGDEV"; then return 1; fi + if test ! -x /usr/bin/sg_luns; then echo 0; return 1; fi + LLUN=`sg_luns /dev/$SGDEV 2>/dev/null | sed -n 's/ \(.*\)/\1/p'` + # Added -z $LLUN condition because $? gets the RC from sed, not sg_luns + if test $? != 0 -o -z "$LLUN"; then echo 0; return 1; fi + for lun in $LLUN ; do + # Swap LUN number + l0=$(printf '%u' 0x$lun) + l1=$(( ($l0 >> 48) & 0xffff )) + l2=$(( ($l0 >> 32) & 0xffff )) + l3=$(( ($l0 >> 16) & 0xffff )) + l4=$(( $l0 & 0xffff )) + l0=$(( ( ( ($l4 * 0xffff) + $l3 ) * 0xffff + $l2 ) * 0xffff + $l1 )) + printf "%u\n" $l0 + done + return 0 +} + +# Wait for udev to settle (create device nodes etc.) +udevadm_settle() +{ + if test -x /sbin/udevadm; then + print_and_scroll_back " Calling udevadm settle (can take a while) " + /sbin/udevadm settle + white_out + elif test -x /sbin/udevsettle; then + print_and_scroll_back " Calling udevsettle (can take a while) " + /sbin/udevsettle + white_out + else + usleep 20000 + fi +} + +# Perform scan on a single lun $host $channel $id $lun +dolunscan() +{ + local remappedlun0= + SCSISTR= + devnr="$host $channel $id $lun" + echo -e " Scanning for device $devnr ... " + printf "${yellow}OLD: $norm" + testexist + # Device exists: Test whether it's still online + # (testonline returns 2 if it's gone and 1 if it has changed) + if test "$SCSISTR" ; then + testonline + RC=$? + # Well known lun transition case. Only for Direct-Access devs (type 0) + # If block directory exists && and PQUAL != 0, we unmapped lun0 and just have a well-known lun + # If block directory doesn't exist && PQUAL == 0, we mapped a real lun0 + if test $lun -eq 0 -a $IPTYPE -eq 0 ; then + if test $RC = 2 ; then + if test -e /sys/class/scsi_device/${host}:${channel}:${id}:${lun}/device; then + if test -d /sys/class/scsi_device/${host}:${channel}:${id}:${lun}/device/block ; then + remappedlun0=2 # Transition from real lun 0 to well-known + else + RC=0 # Set this so the system leaves the existing well known lun alone. This is a lun 0 with no block directory + fi + fi + elif test $RC = 0 -a $IPTYPE -eq 0; then + if test -e /sys/class/scsi_device/${host}:${channel}:${id}:${lun}/device; then + if test ! -d /sys/class/scsi_device/${host}:${channel}:${id}:${lun}/device/block ; then + remappedlun0=1 # Transition from well-known to real lun 0 + fi + fi + fi + fi + fi + + # Special case: lun 0 just got added (for reportlunscan), + # so make sure we correctly treat it as new + if test "$lun" = "0" -a "$1" = "1" -a -z "$remappedlun0"; then + SCSISTR="" + printf "\r\e[A\e[A\e[A" + fi + + : f $remove s $SCSISTR + if test "$remove" -a "$SCSISTR" -o "$remappedlun0" = "1"; then + if test $RC != 0 -o ! -z "$forceremove" -o -n "$remappedlun0"; then + if test "$remappedlun0" != "1" ; then + echo -en "\r\e[A\e[A\e[A${red}REM: " + echo "$SCSISTR" | head -n1 + echo -e "${norm}\e[B\e[B" + fi + if test -e /sys/class/scsi_device/${host}:${channel}:${id}:${lun}/device; then + # have to preemptively do this so we can figure out the mpath device + # Don't do this if we're deleting a well known lun to replace it + if test "$remappedlun0" != "1" ; then + incrrmvd "$host:$channel:$id:$lun" + fi + echo 1 > /sys/class/scsi_device/${host}:${channel}:${id}:${lun}/device/delete + usleep 20000 + else + echo "scsi remove-single-device $devnr" > /proc/scsi/scsi + if test $RC -eq 1 -o $lun -eq 0 ; then + # Try readding, should fail if device is gone + echo "scsi add-single-device $devnr" > /proc/scsi/scsi + fi + fi + fi + if test $RC = 0 -o "$forcerescan" ; then + if test -e /sys/class/scsi_device/${host}:${channel}:${id}:${lun}/device; then + echo 1 > /sys/class/scsi_device/${host}:${channel}:${id}:${lun}/device/rescan + fi + fi + printf "\r\e[A\e[A\e[A${yellow}OLD: $norm" + testexist + if test -z "$SCSISTR" -a $RC != 1 -a "$remappedlun0" != "1"; then + printf "\r${red}DEL: $norm\r\n\n" + # In the event we're replacing with a well known node, we need to let it continue, to create the replacement node + test "$remappedlun0" != "2" && return 1 + fi + fi + if test -z "$SCSISTR" -o -n "$remappedlun0"; then + if test "$remappedlun0" != "2" ; then + # Device does not exist, try to add + printf "\r${green}NEW: $norm" + fi + if test -e /sys/class/scsi_host/host${host}/scan; then + echo "$channel $id $lun" > /sys/class/scsi_host/host${host}/scan 2> /dev/null + else + echo "scsi add-single-device $devnr" > /proc/scsi/scsi + fi + testexist + if test -z "$SCSISTR"; then + # Device not present + printf "\r\e[A"; + # Optimization: if lun==0, stop here (only if in non-remove mode) + if test $lun = 0 -a -z "$remove" -a $optscan = 1; then + break; + fi + else + if test "$remappedlun0" != "2" ; then + incrfound "$host:$channel:$id:$lun" + fi + fi + fi +} + +# Perform report lun scan on $host $channel $id using REPORT_LUNS +doreportlun() +{ + lun=0 + SCSISTR= + devnr="$host $channel $id $lun" + echo -en " Scanning for device $devnr ...\r" + lun0added= + #printf "${yellow}OLD: $norm" + # Phase one: If LUN0 does not exist, try to add + testexist -q + if test -z "$SCSISTR"; then + # Device does not exist, try to add + #printf "\r${green}NEW: $norm" + if test -e /sys/class/scsi_host/host${host}/scan; then + echo "$channel $id $lun" > /sys/class/scsi_host/host${host}/scan 2> /dev/null + udevadm_settle + else + echo "scsi add-single-device $devnr" > /proc/scsi/scsi + fi + testexist -q + if test -n "$SCSISTR"; then + lun0added=1 + #testonline + else + # Device not present + # return + # Find alternative LUN to send getluns to + for dev in /sys/class/scsi_device/${host}:${channel}:${id}:*; do + [ -d "$dev" ] || continue + lun=${dev##*:} + break + done + fi + fi + targetluns=`getluns` + REPLUNSTAT=$? + lunremove= + #echo "getluns reports " $targetluns + olddev=`find /sys/class/scsi_device/ -name $host:$channel:$id:* 2>/dev/null` + oldluns=`echo "$olddev" | awk -F'/' '{print $5}' | awk -F':' '{print $4}'` + oldtargets="$targetluns" + # OK -- if we don't have a LUN to send a REPORT_LUNS to, we could + # fall back to wildcard scanning. Same thing if the device does not + # support REPORT_LUNS + # TODO: We might be better off to ALWAYS use wildcard scanning if + # it works + if test "$REPLUNSTAT" = "1"; then + if test -e /sys/class/scsi_host/host${host}/scan; then + echo "$channel $id -" > /sys/class/scsi_host/host${host}/scan 2> /dev/null + udevadm_settle + else + echo "scsi add-single-device $host $channel $id $SCAN_WILD_CARD" > /proc/scsi/scsi + fi + targetluns=`find /sys/class/scsi_device/ -name $host:$channel:$id:* 2>/dev/null | awk -F'/' '{print $5}' | awk -F':' '{print $4}' | sort -n` + let found+=`echo "$targetluns" | wc -l` + let found-=`echo "$olddev" | wc -l` + fi + if test -z "$targetluns"; then targetluns="$oldtargets"; fi + # Check existing luns + for dev in $olddev; do + [ -d "$dev" ] || continue + lun=${dev##*:} + newsearch= + inlist= + # OK, is existing $lun (still) in reported list + for tmplun in $targetluns; do + if test $tmplun -eq $lun ; then + inlist=1 + dolunscan $lun0added + else + newsearch="$newsearch $tmplun" + fi + done + # OK, we have now done a lunscan on $lun and + # $newsearch is the old $targetluns without $lun + if [ -z "$inlist" ]; then + # Stale lun + lunremove="$lunremove $lun" + fi + # $lun removed from $lunsearch (echo for whitespace cleanup) + targetluns=`echo $newsearch` + done + # Add new ones and check stale ones + for lun in $targetluns $lunremove; do + dolunscan $lun0added + done +} + +# Perform search (scan $host) +dosearch () +{ + if test -z "$channelsearch" ; then + chanlist + fi + for channel in $channelsearch; do + if test -z "$idsearch" ; then + idlist + fi + for id in $idsearch; do + if test -z "$lunsearch" ; then + doreportlun + else + for lun in $lunsearch; do + dolunscan + done + fi + done + done +} + +expandlist () +{ + list=$1 + result="" + first=${list%%,*} + rest=${list#*,} + while test ! -z "$first"; do + beg=${first%%-*}; + if test "$beg" = "$first"; then + result="$result $beg"; + else + end=${first#*-} + result="$result `seq $beg $end`" + fi + test "$rest" = "$first" && rest="" + first=${rest%%,*} + rest=${rest#*,} + done + echo $result +} + +searchexisting() +{ + local tmpch; + local tmpid + local match=0 + local targets=`ls -d /sys/class/scsi_device/$host:* 2> /dev/null | egrep -o $host:[0-9]+:[0-9]+ | sort | uniq` + + # Nothing came back on this host, so we should skip it + test -z "$targets" && return + + local target=; + for target in $targets ; do + channel=`echo $target | cut -d":" -f2` + id=`echo $target | cut -d":" -f 3` + if [ -n "$channelsearch" ] ; then + for tmpch in $channelsearch ; do + test $tmpch -eq $channel && match=1 + done + else + match=1 + fi + + test $match -eq 0 && continue + match=0 + + if [ $filter_ids -eq 1 ] ; then + for tmpid in $idsearch ; do + if [ $tmpid -eq $id ] ; then + match=1 + fi + done + else + match=1 + fi + + test $match -eq 0 && continue + + if [ -z "$lunsearch" ] ; then + doreportlun + else + for lun in $lunsearch ; do + dolunscan + done + fi + done +} + +# Go through all of the existing devices and figure out any that have been remapped +findremapped() +{ + local hctl=; + local devs=`ls /sys/class/scsi_device/` + local sddev= + local id_serial= + local id_serial_old= + local sysfs_devpath= + mpaths="" + local tmpfile="/tmp/rescan-scsi-bus-`date +s`" + + test -f $tmpfile && rm $tmpfile + + # Get all of the ID_SERIAL attributes, after finding their sd node + for hctl in $devs ; do + if [ -d /sys/class/scsi_device/$hctl/device/block ] ; then + sddev=`ls /sys/class/scsi_device/$hctl/device/block` + id_serial_old=`udevadm info -q all -n $sddev | grep "ID_SERIAL=" | cut -d"=" -f2` + [ -z "$id_serial_old" ] && id_serial_old="none" + echo "$hctl $sddev $id_serial_old" >> $tmpfile + fi + done + + # Trigger udev to update the info + echo -n "Triggering udev to update device information... " + /sbin/udevadm trigger + udevadm_settle &>/dev/null + echo "Done" + + # See what changed and reload the respective multipath device if applicable + while read hctl sddev id_serial_old ; do + id_serial=`udevadm info -q all -n $sddev | grep "ID_SERIAL=" | cut -d"=" -f2` + [ -z "$id_serial" ] && id_serial="none" + if [ "$id_serial_old" != "$id_serial" ] ; then + if [ "$id_serial" = "1" ] ; then + continue # the lun was unmapped and is getting the blank scsi id (1) + fi + printf "${yellow}REMAPPED: $norm" + host=`echo $hctl | cut -d":" -f1` + channel=`echo $hctl | cut -d":" -f2` + id=`echo $hctl | cut -d":" -f3` + lun=`echo $hctl | cut -d":" -f4` + procscsiscsi + echo "$SCSISTR" + incrchgd "$hctl" + fi + done < $tmpfile + rm $tmpfile &>/dev/null + + if test -n "$mp_enable" -a -n "$mpaths" ; then + echo "Updating multipath device mappings" + flushmpaths + $MULTIPATH | grep "create:" 2> /dev/null + fi +} + +incrfound() +{ + local hctl="$1" + if test -n "$hctl" ; then + let found+=1 + FOUNDDEVS="$FOUNDDEVS\t[$hctl]\n" + else + return + fi +} + +incrchgd() +{ + local hctl="$1" + if test -n "$hctl" ; then + let updated+=1 + CHGDEVS="$CHGDEVS\t[$hctl]\n" + else + return + fi + + if test -n "$mp_enable" ; then + local sdev="`findsddev \"$hctl\"`" + if test -n "$sdev" ; then + findmultipath "$sdev" + fi + fi +} + +incrrmvd() +{ + local hctl="$1" + if test -n "$hctl" ; then + let rmvd+=1; + RMVDDEVS="$RMVDDEVS\t[$hctl]\n" + else + return + fi + + if test -n "$mp_enable" ; then + local sdev="`findsddev \"$hctl\"`" + if test -n "$sdev" ; then + findmultipath "$sdev" + fi + fi +} + +findsddev() +{ + local hctl="$1" + local sddev= + + if test ! -e /sys/class/scsi_device/$hctl/device/block ; then + return 1 + fi + + sddev=`ls /sys/class/scsi_device/$hctl/device/block` + echo $sddev + + return 0 +} + +findmultipath() +{ + local dev="$1" + local mp= + local mp2= + + # Need a sdev, and executable multipath and dmsetup command here + if [ -z "$dev" ] || [ ! -x $DMSETUP ] ; then + return 1 + fi + + local maj_min=`cat /sys/block/$dev/dev` + for mp in $($DMSETUP ls --target=multipath | cut -f 1) ; do + if $($DMSETUP status $mp | grep -q " $maj_min ") ; then + for mp2 in $mpaths ; do + # mp device is already there, return + if [ "$mp2" = "$mp" ] ; then + return + fi + done + mpaths="$mpaths $mp" + return + fi + done +} + +reloadmpaths() +{ + local mpath + if [ ! -x $MULTIPATH ] ; then + echo "no -x multipath" + return + fi + + if [ "$1" = "1" ] ; then + echo "Reloading all multipath devices" + $MULTIPATH -r &>/dev/null + return + fi + + # Reload the multipath devices + for mpath in $mpaths ; do + echo "Reloading multipath device $mpath" + $MULTIPATH -r $mpath &>/dev/null + done +} + +flushmpaths() +{ + local mpath + # Mode: flush only failed + if test -n "$1" ; then + for mpath in $($DMSETUP ls --target=multipath | cut -f 1) ; do + num=$($DMSETUP status $mpath | awk 'BEGIN{RS=" ";active=0}/[0-9]+:[0-9]+/{dev=1}/A/{if (dev == 1) active++; dev=0} END{ print active }') + if [ $num -eq 0 ] ; then + echo -n "Flushing multipath device $mpath... " + $DMSETUP message $mpath 0 fail_if_no_path &>/dev/null + $MULTIPATH -f $mpath &>/dev/null + $DMSETUP status $mpath &>/dev/null + if test "$?" = "1" ; then + echo "Done" + else + echo "Fail" + fi + fi + done + return + fi + # Flush all the devs specified in $mpaths + for mpath in $mpaths ; do + echo -n "Flushing multipath device $mpath... " + $DMSETUP message $mpath 0 fail_if_no_path &>/dev/null + $MULTIPATH -f $mpath &>/dev/null + $DMSETUP status $mpath &>/dev/null + if test "$?" = "1" ; then + echo "Done" + else + echo "Fail" + fi + done +} + + +# Find resized luns +findresized() +{ + local devs=`ls /sys/class/scsi_device/` + local size= + local new_size= + local sysfs_path= + local sddev= + + for hctl in $devs ; do + sysfs_path="/sys/class/scsi_device/$hctl/device" + if [ -d "$sysfs_path/block" ] ; then + sddev=`ls $sysfs_path/block` + size=`cat $sysfs_path/block/$sddev/size` + + echo 1 > $sysfs_path/rescan + new_size=`cat $sysfs_path/block/$sddev/size` + + if [ "$size" != "$new_size" ] && [ "$size" != "0" ] && [ "$new_size" != "0" ] ; then + printf "${yellow}RESIZED: $norm" + host=`echo $hctl | cut -d":" -f1` + channel=`echo $hctl | cut -d":" -f2` + id=`echo $hctl | cut -d":" -f3` + lun=`echo $hctl | cut -d":" -f4` + + procscsiscsi + echo "$SCSISTR" + incrchgd "$hctl" + fi + fi + done + + if test -n "$mp_enable" -a -n "$mpaths" ; then + reloadmpaths + fi +} + +FOUNDDEVS="" +CHGDEVS="" +RMVDDEVS="" + +# main +if test @$1 = @--help -o @$1 = @-h -o @$1 = @-?; then + echo "Usage: rescan-scsi-bus.sh [options] [host [host ...]]" + echo "Options:" + echo " -a scan all targets, not just currently existing [default: disabled]" + echo " -d enable debug [default: 0]" + echo " -l activates scanning for LUNs 0--7 [default: 0]" + echo " -L NUM activates scanning for LUNs 0--NUM [default: 0]" + echo " -w scan for target device IDs 0--15 [default: 0--7]" + echo " -c enables scanning of channels 0 1 [default: 0 / all detected ones]" + echo " -m update multipath devices [default: disabled]" + echo " -r enables removing of devices [default: disabled]" + echo " -f flush failed multipath devices [default: disabled]" + echo " -i issue a FibreChannel LIP reset [default: disabled]" + echo " -u look for existing disks that have been remapped" + echo " -s look for resized disks and reload associated multipath devices, if applicable" + echo "--alltargets: same as -a" + echo "--remove: same as -r" + echo "--flush: same as -f" + echo "--issue-lip: same as -i" + echo "--wide: same as -w" + echo "--multipath: same as -m" + echo "--forceremove: Remove stale devices (DANGEROUS)" + echo "--forcerescan: Remove and readd existing devices (DANGEROUS)" + echo "--nooptscan: don't stop looking for LUNs if 0 is not found" + echo "--color: use coloured prefixes OLD/NEW/DEL" + echo "--hosts=LIST: Scan only host(s) in LIST" + echo "--channels=LIST: Scan only channel(s) in LIST" + echo "--ids=LIST: Scan only target ID(s) in LIST" + echo "--luns=LIST: Scan only lun(s) in LIST" + echo "--sync/nosync: Issue a sync / no sync [default: sync if remove]" + echo "--attachpq3: Tell kernel to attach sg to LUN 0 that reports PQ=3" + echo "--reportlun2: Tell kernel to try REPORT_LUN even on SCSI2 devices" + echo "--largelun: Tell kernel to support LUNs > 7 even on SCSI2 devs" + echo "--sparselun: Tell kernel to support sparse LUN numbering" + echo "--update: same as -u" + echo "--resize: same as -s" + echo " Host numbers may thus be specified either directly on cmd line (deprecated) or" + echo " or with the --hosts=LIST parameter (recommended)." + echo "LIST: A[-B][,C[-D]]... is a comma separated list of single values and ranges" + echo " (No spaces allowed.)" + exit 0 +fi + +if test ! -d /sys/class/scsi_host/ -a ! -d /proc/scsi/; then + echo "Error: SCSI subsystem not active" + exit 1 +fi + +# Make sure sg is there +modprobe sg &>/dev/null + +if test -x /usr/bin/sg_inq; then + sg_version=$(sg_inq -V 2>&1 | cut -d " " -f 3) + if test -n "$sg_version"; then + sg_ver_maj=${sg_version:0:1} + sg_version=${sg_version##?.} + let sg_version+=$((100*$sg_ver_maj)) + fi + sg_version=${sg_version##0.} + #echo "\"$sg_version\"" + if [ -z "$sg_version" -o "$sg_version" -lt 70 ] ; then + sg_len_arg="-36" + else + sg_len_arg="--len=36" + fi +else + echo "WARN: /usr/bin/sg_inq not present -- please install sg3_utils" + echo " or rescan-scsi-bus.sh might not fully work." +fi + +# defaults +unsetcolor +debug=0 +lunsearch= +opt_idsearch=`seq 0 7` +filter_ids=0 +opt_channelsearch= +remove= +updated=0 +update=0 +resize=0 +forceremove= +optscan=1 +sync=1 +existing_targets=1 +mp_enable= +declare -i scan_flags=0 + +# Scan options +opt="$1" +while test ! -z "$opt" -a -z "${opt##-*}"; do + opt=${opt#-} + case "$opt" in + a) existing_targets=;; #Scan ALL targets when specified + d) debug=1 ;; + f) flush=1 ;; + l) lunsearch=`seq 0 7` ;; + L) lunsearch=`seq 0 $2`; shift ;; + m) mp_enable=1 ;; + w) opt_idsearch=`seq 0 15` ;; + c) opt_channelsearch="0 1" ;; + r) remove=1 ;; + s) resize=1 ;; + i) lipreset=1 ;; + u) update=1 ;; + -alltargets) existing_targets=;; + -flush) flush=1 ;; + -remove) remove=1 ;; + -forcerescan) remove=1; forcerescan=1 ;; + -forceremove) remove=1; forceremove=1 ;; + -hosts=*) arg=${opt#-hosts=}; hosts=`expandlist $arg` ;; + -channels=*) arg=${opt#-channels=};opt_channelsearch=`expandlist $arg` ;; + -ids=*) arg=${opt#-ids=}; opt_idsearch=`expandlist $arg` ; filter_ids=1;; + -luns=*) arg=${opt#-luns=}; lunsearch=`expandlist $arg` ;; + -color) setcolor ;; + -nooptscan) optscan=0 ;; + -issue-lip) lipreset=1 ;; + -sync) sync=2 ;; + -nosync) sync=0 ;; + -multipath) mp_enable=1 ;; + -attachpq3) scan_flags=$(($scan_flags|0x1000000)) ;; + -reportlun2) scan_flags=$(($scan_flags|0x20000)) ;; + -resize) resize=1;; + -largelun) scan_flags=$(($scan_flags|0x200)) ;; + -sparselun) scan_flags=$((scan_flags|0x40)) ;; + -update) update=1;; + -wide) opt_idsearch=`seq 0 15` ;; + *) echo "Unknown option -$opt !" ;; + esac + shift + opt="$1" +done + +if [ -z "$hosts" ] ; then + if test -d /sys/class/scsi_host; then + findhosts_26 + else + findhosts + fi +fi + +if [ -d /sys/class/scsi_host -a ! -w /sys/class/scsi_host ]; then + echo "You need to run scsi-rescan-bus.sh as root" + exit 2 +fi +if test "$sync" = 1 -a "$remove" = 1; then sync=2; fi +if test "$sync" = 2; then echo "Syncing file systems"; sync; fi +if test -w /sys/module/scsi_mod/parameters/default_dev_flags -a $scan_flags != 0; then + OLD_SCANFLAGS=`cat /sys/module/scsi_mod/parameters/default_dev_flags` + NEW_SCANFLAGS=$(($OLD_SCANFLAGS|$scan_flags)) + if test "$OLD_SCANFLAGS" != "$NEW_SCANFLAGS"; then + echo -n "Temporarily setting kernel scanning flags from " + printf "0x%08x to 0x%08x\n" $OLD_SCANFLAGS $NEW_SCANFLAGS + echo $NEW_SCANFLAGS > /sys/module/scsi_mod/parameters/default_dev_flags + else + unset OLD_SCANFLAGS + fi +fi +DMSETUP=$(which dmsetup) +[ -z "$DMSETUP" ] && flush= && mp_enable= +MULTIPATH=$(which multipath) +[ -z "$MULTIPATH" ] && flush= && mp_enable= + +echo -n "Scanning SCSI subsystem for new devices" +test -z "$flush" || echo -n ", flush failed multipath devices," +test -z "$remove" || echo -n " and remove devices that have disappeared" +echo +declare -i found=0 +declare -i updated=0 +declare -i rmvd=0 + +if [ -n "$flush" ] ; then + if [ -x $MULTIPATH ] ; then + flushmpaths 1 + fi +fi + +# Update existing mappings +if [ $update -eq 1 ] ; then + echo "Searching for remapped LUNs" + findremapped +# Search for resized LUNs +elif [ $resize -eq 1 ] ; then + echo "Searching for resized LUNs" + findresized +# Normal rescan mode +else + for host in $hosts; do + echo -n "Scanning host $host " + if test -e /sys/class/fc_host/host$host ; then + # It's pointless to do a target scan on FC + if test -n "$lipreset" ; then + echo 1 > /sys/class/fc_host/host$host/issue_lip 2> /dev/null; + udevadm_settle + fi + channelsearch= + idsearch= + else + channelsearch=$opt_channelsearch + idsearch=$opt_idsearch + fi + [ -n "$channelsearch" ] && echo -n "channels $channelsearch " + echo -n "for " + if [ -n "$idsearch" ] ; then + echo -n " SCSI target IDs " $idsearch + else + echo -n " all SCSI target IDs" + fi + if [ -n "$lunsearch" ] ; then + echo ", LUNs " $lunsearch + else + echo ", all LUNs" + fi + + if [ -n "$existing_targets" ] ; then + searchexisting + else + dosearch + fi + done + if test -n "$OLD_SCANFLAGS"; then + echo $OLD_SCANFLAGS > /sys/module/scsi_mod/parameters/default_dev_flags + fi +fi + +let rmvd_found=$rmvd+$found +if test -n "$mp_enable" -a $rmvd_found -gt 0 ; then + echo "Attempting to update multipath devices..." + if test $rmvd -gt 0 ; then + /sbin/udevadm trigger + udevadm_settle + echo "Removing multipath mappings for removed devices if all paths are now failed... " + flushmpaths 1 + fi + if test $found -gt 0 ; then + /sbin/udevadm trigger + udevadm_settle + echo "Trying to discover new multipath mappings for newly discovered devices... " + $MULTIPATH | grep "create:" 2> /dev/null + fi +fi + +echo "$found new or changed device(s) found. " +if test ! -z "$FOUNDDEVS" ; then + printf "$FOUNDDEVS" +fi +echo "$updated remapped or resized device(s) found. " +if test ! -z "$CHGDEVS" ; then + printf "$CHGDEVS" +fi +echo "$rmvd device(s) removed. " +if test ! -z "$RMVDDEVS" ; then + printf "$RMVDDEVS" +fi + +# Local Variables: +# sh-basic-offset: 2 +# End: + From 9f2b2b57a4ef01719c59f77862740c0c2b286268 Mon Sep 17 00:00:00 2001 From: Christian Schlatter Date: Wed, 4 Aug 2021 02:58:59 +0200 Subject: [PATCH 04/22] clean scsi disks on bus in unstage volume --- pkg/driver/node.go | 5 ++++- pkg/mount/fake.go | 4 ++++ pkg/mount/mount.go | 25 +++++++++++++++++++++++-- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/pkg/driver/node.go b/pkg/driver/node.go index 2c07af8..766a0ff 100644 --- a/pkg/driver/node.go +++ b/pkg/driver/node.go @@ -172,6 +172,8 @@ func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag ctxzap.Extract(ctx).Sugar().Debugf("NodeUnstageVolume: unmounted %d on target %d", dev, target) + ns.mounter.CleanScsi(ctx) + return &csi.NodeUnstageVolumeResponse{}, nil } @@ -199,7 +201,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis } v, err := ns.connector.GetVolumeByID(ctx, volumeID) if err != nil { - + return nil, status.Error(codes.InvalidArgument, "No volume found") } readOnly := req.GetReadonly() @@ -313,6 +315,7 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu return nil, status.Errorf(codes.Internal, "Deleting %s failed with error %v", targetPath, err) } ctxzap.Extract(ctx).Sugar().Debugw("sucessfully unpublish volume", "id", volumeID, "targetPath", targetPath) + return &csi.NodeUnpublishVolumeResponse{}, nil } diff --git a/pkg/mount/fake.go b/pkg/mount/fake.go index 9cae82c..9b5ebd9 100644 --- a/pkg/mount/fake.go +++ b/pkg/mount/fake.go @@ -51,3 +51,7 @@ func (*fakeMounter) MakeDir(pathname string) error { func (*fakeMounter) MakeFile(pathname string) error { return nil } + +func (m *fakeMounter) CleanScsi(ctx context.Context) { + //Do nothing +} diff --git a/pkg/mount/mount.go b/pkg/mount/mount.go index a4a4bec..8d39a65 100644 --- a/pkg/mount/mount.go +++ b/pkg/mount/mount.go @@ -11,6 +11,7 @@ import ( "k8s.io/utils/exec" "os" "path/filepath" + "regexp" "strconv" "strings" "time" @@ -28,6 +29,8 @@ type Interface interface { FormatAndMount(source string, target string, fstype string, options []string) error + CleanScsi(ctx context.Context) + GetDevicePath(ctx context.Context, volumeID string, hypervisor string) (string, error) GetDeviceName(mountPath string) (string, int, error) ExistsPath(filename string) (bool, error) @@ -85,7 +88,7 @@ func (m *mounter) GetDevicePath(ctx context.Context, deviceID string, hypervisor ctxzap.Extract(ctx).Sugar().Debugf("device path found: %s", path) return true, nil } - m.probeVolume(ctx) + m.rescanScsi(ctx) return false, nil }) @@ -109,7 +112,7 @@ func (m *mounter) getDevicePathBySerialID(volumeID string) (string, error) { return "", nil } -func (m *mounter) probeVolume(ctx context.Context) { +func (m *mounter) rescanScsi(ctx context.Context) { log := ctxzap.Extract(ctx).Sugar() log.Debug("Scaning SCSI host...") @@ -121,6 +124,24 @@ func (m *mounter) probeVolume(ctx context.Context) { } } +func (m *mounter) CleanScsi(ctx context.Context) { + log := ctxzap.Extract(ctx).Sugar() + log.Debug("removing unmounted SCSI devices...") + + args := []string{"-r"} + cmd := m.Exec.Command("rescan-scsi-bus.sh", args...) + out, err := cmd.CombinedOutput() + if err != nil { + log.Warnf("Error running rescan-scsi-bus.sh -r %v\n", err) + } + + r, _ := regexp.Compile("(\\d)* device\\(s\\) removed\\.") + subs := r.FindAllStringSubmatch(string(out), -1) + + ctxzap.Extract(ctx).Sugar().Debugf("%+v", subs) + +} + func (m *mounter) GetDeviceName(mountPath string) (string, int, error) { return mount.GetDeviceNameFromMount(m, mountPath) } From 99ed16394323b0ada080cae68f2a9597ce794067 Mon Sep 17 00:00:00 2001 From: Christian Schlatter Date: Wed, 4 Aug 2021 03:02:40 +0200 Subject: [PATCH 05/22] clean scsi disks on bus in unpublish volume on node --- pkg/driver/node.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/driver/node.go b/pkg/driver/node.go index 766a0ff..9e91f1b 100644 --- a/pkg/driver/node.go +++ b/pkg/driver/node.go @@ -314,7 +314,9 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu if err != nil && !os.IsNotExist(err) { return nil, status.Errorf(codes.Internal, "Deleting %s failed with error %v", targetPath, err) } - ctxzap.Extract(ctx).Sugar().Debugw("sucessfully unpublish volume", "id", volumeID, "targetPath", targetPath) + ctxzap.Extract(ctx).Sugar().Debugf("NodeUnpublishVolume: successfully unpublish volume %s on node %s", volumeID, targetPath) + + ns.mounter.CleanScsi(ctx) return &csi.NodeUnpublishVolumeResponse{}, nil } From c95cdfbb4a33df68f5166450a6859205335fdb82 Mon Sep 17 00:00:00 2001 From: Christian Schlatter Date: Wed, 4 Aug 2021 11:11:17 +0200 Subject: [PATCH 06/22] add max allowed volumes check --- pkg/cloud/cloud.go | 1 + pkg/cloud/project.go | 7 ++++++- pkg/cloud/vms.go | 1 + pkg/cloud/volumes.go | 43 ++++++++++++++++++++++++++++++++++++++-- pkg/driver/controller.go | 21 +++++++++++++++----- pkg/driver/node.go | 4 +++- 6 files changed, 68 insertions(+), 9 deletions(-) diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index 2424294..aab1b5a 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -21,6 +21,7 @@ type Interface interface { GetVolumeByID(ctx context.Context, volumeID string) (*Volume, error) GetVolumeByName(ctx context.Context, name string) (*Volume, error) + ListVolumesForVM(ctx context.Context, virtualMachineID string) ([]*Volume, error) CreateVolume(ctx context.Context, diskOfferingID, projectID, domainID, zoneID, name string, sizeInGB int64) (string, error) DeleteVolume(ctx context.Context, id string) error AttachVolume(ctx context.Context, volumeID, vmID string) (string, error) diff --git a/pkg/cloud/project.go b/pkg/cloud/project.go index 94c2fa8..1304267 100644 --- a/pkg/cloud/project.go +++ b/pkg/cloud/project.go @@ -10,7 +10,12 @@ func (c *client) GetDomainID(ctx context.Context) (string, error) { ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "GetProjectByID", "params", map[string]string{ "projectID": c.ProjectID, }) - return p.Domainid, err + + if err != nil { + return "", err + } + + return p.Domainid, nil } func (c *client) GetProjectID() string { diff --git a/pkg/cloud/vms.go b/pkg/cloud/vms.go index 29124c3..7a08c92 100644 --- a/pkg/cloud/vms.go +++ b/pkg/cloud/vms.go @@ -23,6 +23,7 @@ func (c *client) GetVMByID(ctx context.Context, vmID string) (*VM, error) { return nil, ErrTooManyResults } vm := l.VirtualMachines[0] + return &VM{ ID: vm.Id, ZoneID: vm.Zoneid, diff --git a/pkg/cloud/volumes.go b/pkg/cloud/volumes.go index f27c18e..68c49a1 100644 --- a/pkg/cloud/volumes.go +++ b/pkg/cloud/volumes.go @@ -75,8 +75,14 @@ func (c *client) CreateVolume(ctx context.Context, diskOfferingID, projectID, do p.SetZoneid(zoneID) p.SetName(name) p.SetSize(sizeInGB) - p.SetDomainid(domainID) - p.SetProjectid(projectID) + if domainID != "" { + p.SetDomainid(domainID) + } + + if projectID != "" { + p.SetProjectid(projectID) + } + ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "CreateVolume", "params", map[string]string{ "diskofferingid": diskOfferingID, "zoneid": zoneID, @@ -125,3 +131,36 @@ func (c *client) DetachVolume(ctx context.Context, volumeID string) error { _, err := c.Volume.DetachVolume(p) return err } + +func (c *client) ListVolumesForVM(ctx context.Context, virtualMachineID string) ([]*Volume, error) { + ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "ListVolumes", "params", map[string]string{ + "virtualmachineid": virtualMachineID, + }) + p := c.Volume.NewListVolumesParams() + p.SetVirtualmachineid(virtualMachineID) + l, err := c.Volume.ListVolumes(p) + if err != nil { + return nil, err + } + if l.Count == 0 { + return nil, ErrNotFound + } + + volumes := make([]*Volume, len(l.Volumes)) + for i, _ := range l.Volumes { + vol := l.Volumes[i] + v := &Volume{ + ID: vol.Id, + Name: vol.Name, + Size: vol.Size, + DiskOfferingID: vol.Diskofferingid, + ZoneID: vol.Zoneid, + VirtualMachineID: vol.Virtualmachineid, + Hypervisor: vol.Hypervisor, + DeviceID: strconv.FormatInt(vol.Deviceid, 10), + } + volumes[i] = v + } + + return volumes, nil +} diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index 6498a2e..0f76ca6 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -115,16 +115,17 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol } projectID := cs.connector.GetProjectID() - domaindID := "" + domainID := "" if projectID != "" { - domaindID, err = cs.connector.GetDomainID(ctx) + domainID, err = cs.connector.GetDomainID(ctx) if err != nil { - return nil, status.Error(codes.Internal, err.Error()) + ctxzap.Extract(ctx).Sugar().Debugf("could not get a domainID for project %s, create volume without DomainID", projectID) + //return nil, status.Errorf(codes.Internal, "Cannot create volume %s in project %s due error in GetDomainID: %v", name, projectID, err.Error()) } } - volID, err := cs.connector.CreateVolume(ctx, diskOfferingID, projectID, domaindID, zoneID, name, sizeInGB) + volID, err := cs.connector.CreateVolume(ctx, diskOfferingID, projectID, domainID, zoneID, name, sizeInGB) if err != nil { return nil, status.Errorf(codes.Internal, "Cannot create volume %s: %v", name, err.Error()) } @@ -248,13 +249,23 @@ func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *cs return nil, status.Error(codes.AlreadyExists, "Volume already assigned") } - if _, err := cs.connector.GetVMByID(ctx, nodeID); err == cloud.ErrNotFound { + if vm, err := cs.connector.GetVMByID(ctx, nodeID); err == cloud.ErrNotFound { return nil, status.Errorf(codes.NotFound, "VM %v not found", volumeID) } else if err != nil { // Error with CloudStack return nil, status.Errorf(codes.Internal, "Error %v", err) } + vols, err := cs.connector.ListVolumesForVM(ctx, nodeID) + if err != nil { + return nil, status.Errorf(codes.Internal, "Error %v", err) + } + + //Check max volumes + if len(vols) >= maxAllowedBlockVolumesPerNode { + return nil, status.Errorf(codes.ResourceExhausted, "Maximum allowed volumes per node reached: %v", err) + } + if vol.VirtualMachineID == nodeID { // volume already attached diff --git a/pkg/driver/node.go b/pkg/driver/node.go index 9e91f1b..1516057 100644 --- a/pkg/driver/node.go +++ b/pkg/driver/node.go @@ -16,7 +16,8 @@ import ( const ( // default file system type to be used when it is not provided - defaultFsType = "ext4" + defaultFsType = "ext4" + maxAllowedBlockVolumesPerNode = 10 ) type nodeServer struct { @@ -341,6 +342,7 @@ func (ns *nodeServer) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoReque return &csi.NodeGetInfoResponse{ NodeId: vm.ID, AccessibleTopology: topology.ToCSI(), + MaxVolumesPerNode: maxAllowedBlockVolumesPerNode, }, nil } From 89a40cbbb7d9601d620a95d5f9046c067d3e3a4d Mon Sep 17 00:00:00 2001 From: Christian Schlatter Date: Wed, 4 Aug 2021 11:11:44 +0200 Subject: [PATCH 07/22] fix error --- pkg/driver/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index 0f76ca6..91f4345 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -249,7 +249,7 @@ func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *cs return nil, status.Error(codes.AlreadyExists, "Volume already assigned") } - if vm, err := cs.connector.GetVMByID(ctx, nodeID); err == cloud.ErrNotFound { + if _, err := cs.connector.GetVMByID(ctx, nodeID); err == cloud.ErrNotFound { return nil, status.Errorf(codes.NotFound, "VM %v not found", volumeID) } else if err != nil { // Error with CloudStack From 7d2a8a3b849835fc3740deb441bc3884fcb80606 Mon Sep 17 00:00:00 2001 From: Christian Schlatter Date: Wed, 4 Aug 2021 13:44:13 +0200 Subject: [PATCH 08/22] better log statements, add project id for listing volumes --- pkg/cloud/cloud.go | 2 +- pkg/cloud/volumes.go | 9 ++++----- pkg/driver/controller.go | 8 +++++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index aab1b5a..502e7a4 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -21,7 +21,7 @@ type Interface interface { GetVolumeByID(ctx context.Context, volumeID string) (*Volume, error) GetVolumeByName(ctx context.Context, name string) (*Volume, error) - ListVolumesForVM(ctx context.Context, virtualMachineID string) ([]*Volume, error) + ListVolumesForVM(ctx context.Context, virtualMachineID, projectID string) ([]*Volume, error) CreateVolume(ctx context.Context, diskOfferingID, projectID, domainID, zoneID, name string, sizeInGB int64) (string, error) DeleteVolume(ctx context.Context, id string) error AttachVolume(ctx context.Context, volumeID, vmID string) (string, error) diff --git a/pkg/cloud/volumes.go b/pkg/cloud/volumes.go index 68c49a1..335dda7 100644 --- a/pkg/cloud/volumes.go +++ b/pkg/cloud/volumes.go @@ -132,19 +132,18 @@ func (c *client) DetachVolume(ctx context.Context, volumeID string) error { return err } -func (c *client) ListVolumesForVM(ctx context.Context, virtualMachineID string) ([]*Volume, error) { +func (c *client) ListVolumesForVM(ctx context.Context, virtualMachineID, projectID string) ([]*Volume, error) { ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "ListVolumes", "params", map[string]string{ - "virtualmachineid": virtualMachineID, + "virtualmachineid": virtualMachineID, "projectid": projectID, }) p := c.Volume.NewListVolumesParams() p.SetVirtualmachineid(virtualMachineID) + p.SetProjectid(projectID) + l, err := c.Volume.ListVolumes(p) if err != nil { return nil, err } - if l.Count == 0 { - return nil, ErrNotFound - } volumes := make([]*Volume, len(l.Volumes)) for i, _ := range l.Volumes { diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index 91f4345..f113c8b 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -256,14 +256,16 @@ func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *cs return nil, status.Errorf(codes.Internal, "Error %v", err) } - vols, err := cs.connector.ListVolumesForVM(ctx, nodeID) + projectID := cs.connector.GetProjectID() + + vols, err := cs.connector.ListVolumesForVM(ctx, nodeID, projectID) if err != nil { - return nil, status.Errorf(codes.Internal, "Error %v", err) + return nil, status.Errorf(codes.Internal, "Error no volumes found for nodeID %s: %v", nodeID, err) } //Check max volumes if len(vols) >= maxAllowedBlockVolumesPerNode { - return nil, status.Errorf(codes.ResourceExhausted, "Maximum allowed volumes per node reached: %v", err) + return nil, status.Errorf(codes.ResourceExhausted, "Maximum allowed volumes (%s/%s) per node reached", len(vols), maxAllowedBlockVolumesPerNode) } if vol.VirtualMachineID == nodeID { From ffb27620343b3bb9e98bab5c912a596aeb067674 Mon Sep 17 00:00:00 2001 From: Christian Schlatter Date: Thu, 5 Aug 2021 22:57:00 +0200 Subject: [PATCH 09/22] add rescan with echo command, better log statements, check max volumes on controller, add clean scsi command --- pkg/driver/controller.go | 18 +- pkg/driver/node.go | 18 +- pkg/mount/fake.go | 2 +- pkg/mount/mount.go | 59 +- rescan-scsi-bus.sh | 1192 +------------------------------------- 5 files changed, 61 insertions(+), 1228 deletions(-) diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index f113c8b..baa6365 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -246,7 +246,7 @@ func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *cs } if vol.VirtualMachineID != "" && vol.VirtualMachineID != nodeID { - return nil, status.Error(codes.AlreadyExists, "Volume already assigned") + return nil, status.Error(codes.AlreadyExists, "Volume already assigned to another node") } if _, err := cs.connector.GetVMByID(ctx, nodeID); err == cloud.ErrNotFound { @@ -263,13 +263,9 @@ func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *cs return nil, status.Errorf(codes.Internal, "Error no volumes found for nodeID %s: %v", nodeID, err) } - //Check max volumes - if len(vols) >= maxAllowedBlockVolumesPerNode { - return nil, status.Errorf(codes.ResourceExhausted, "Maximum allowed volumes (%s/%s) per node reached", len(vols), maxAllowedBlockVolumesPerNode) - } - if vol.VirtualMachineID == nodeID { // volume already attached + ctxzap.Extract(ctx).Sugar().Debugf("volume %s is already attached on node %s", volumeID, nodeID) publishContext := map[string]string{ deviceIDContextKey: vol.DeviceID, @@ -277,8 +273,15 @@ func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *cs return &csi.ControllerPublishVolumeResponse{PublishContext: publishContext}, nil } + //Check max volumes + if len(vols) >= maxAllowedBlockVolumesPerNode { + return nil, status.Errorf(codes.ResourceExhausted, "Maximum allowed volumes (%d/%d) per node reached. Could not attach volume %s", len(vols), maxAllowedBlockVolumesPerNode, volumeID) + } + deviceID, err := cs.connector.AttachVolume(ctx, volumeID, nodeID) if err != nil { + ctxzap.Extract(ctx).Sugar().Errorf("volume %s failed node %s (may attached on node %s)", volumeID, nodeID, vol.VirtualMachineID) + return nil, status.Errorf(codes.Internal, "Cannot attach volume %s: %s", volumeID, err.Error()) } @@ -303,6 +306,7 @@ func (cs *controllerServer) ControllerUnpublishVolume(ctx context.Context, req * } nodeID := req.GetNodeId() + ctxzap.Extract(ctx).Sugar().Debugf("ControllerUnpublishVolume: try to unpublish volume %s on node %s", volumeID, nodeID) // Check volume if vol, err := cs.connector.GetVolumeByID(ctx, volumeID); err == cloud.ErrNotFound { return nil, status.Errorf(codes.NotFound, "Volume %v not found", volumeID) @@ -334,7 +338,7 @@ func (cs *controllerServer) ControllerUnpublishVolume(ctx context.Context, req * return nil, status.Errorf(codes.Internal, "Cannot detach volume %s: %s", volumeID, err.Error()) } - ctxzap.Extract(ctx).Sugar().Debugf("volume %s detached successfully on node %s", volumeID, nodeID) + ctxzap.Extract(ctx).Sugar().Debugf("ControllerUnpublishVolume: volume %s detached successfully on node %s", volumeID, nodeID) return &csi.ControllerUnpublishVolumeResponse{}, nil } diff --git a/pkg/driver/node.go b/pkg/driver/node.go index 1516057..06c647a 100644 --- a/pkg/driver/node.go +++ b/pkg/driver/node.go @@ -17,7 +17,7 @@ import ( const ( // default file system type to be used when it is not provided defaultFsType = "ext4" - maxAllowedBlockVolumesPerNode = 10 + maxAllowedBlockVolumesPerNode = 4 ) type nodeServer struct { @@ -61,7 +61,7 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol return nil, status.Error(codes.InvalidArgument, "Volume capability not supported") } - ctxzap.Extract(ctx).Sugar().Infow("#### Mount 1 target: " + target) + ctxzap.Extract(ctx).Sugar().Infof("mount stage volume on target: %s", target) // Now, find the device path v, _ := ns.connector.GetVolumeByID(ctx, volumeID) @@ -171,9 +171,11 @@ func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag return nil, status.Errorf(codes.Internal, "Could not unmount target %q: %v", target, err) } - ctxzap.Extract(ctx).Sugar().Debugf("NodeUnstageVolume: unmounted %d on target %d", dev, target) + ctxzap.Extract(ctx).Sugar().Debugf("NodeUnstageVolume: unmounted %s on target %s", dev, target) - ns.mounter.CleanScsi(ctx) + v, _ := ns.connector.GetVolumeByID(ctx, volumeID) + + ns.mounter.CleanScsi(ctx, volumeID, v.Hypervisor) return &csi.NodeUnstageVolumeResponse{}, nil } @@ -299,7 +301,9 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu targetPath := req.GetTargetPath() volumeID := req.GetVolumeId() - if _, err := ns.connector.GetVolumeByID(ctx, volumeID); err == cloud.ErrNotFound { + ctxzap.Extract(ctx).Sugar().Debugf("NodeUnpublishVolume: unpublish volume %s on node %s", volumeID, targetPath) + v, err := ns.connector.GetVolumeByID(ctx, volumeID) + if err == cloud.ErrNotFound { return nil, status.Errorf(codes.NotFound, "Volume %v not found", volumeID) } else if err != nil { // Error with CloudStack @@ -307,7 +311,7 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu } ctxzap.Extract(ctx).Sugar().Debugw("node unpublish (call unmount) volume", "id", volumeID, "targetPath", targetPath) - err := ns.mounter.Unmount(targetPath) + err = ns.mounter.Unmount(targetPath) if err != nil { return nil, status.Errorf(codes.Internal, "Unmount of targetpath %s failed with error %v", targetPath, err) } @@ -317,7 +321,7 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu } ctxzap.Extract(ctx).Sugar().Debugf("NodeUnpublishVolume: successfully unpublish volume %s on node %s", volumeID, targetPath) - ns.mounter.CleanScsi(ctx) + ns.mounter.CleanScsi(ctx, v.DeviceID, v.Hypervisor) return &csi.NodeUnpublishVolumeResponse{}, nil } diff --git a/pkg/mount/fake.go b/pkg/mount/fake.go index 9b5ebd9..4c75dbc 100644 --- a/pkg/mount/fake.go +++ b/pkg/mount/fake.go @@ -52,6 +52,6 @@ func (*fakeMounter) MakeFile(pathname string) error { return nil } -func (m *fakeMounter) CleanScsi(ctx context.Context) { +func (m *fakeMounter) CleanScsi(ctx context.Context, deviceID, hypervisor string) { //Do nothing } diff --git a/pkg/mount/mount.go b/pkg/mount/mount.go index 8d39a65..e4eba9b 100644 --- a/pkg/mount/mount.go +++ b/pkg/mount/mount.go @@ -11,7 +11,6 @@ import ( "k8s.io/utils/exec" "os" "path/filepath" - "regexp" "strconv" "strings" "time" @@ -29,7 +28,7 @@ type Interface interface { FormatAndMount(source string, target string, fstype string, options []string) error - CleanScsi(ctx context.Context) + CleanScsi(ctx context.Context, deviceID, hypervisor string) GetDevicePath(ctx context.Context, volumeID string, hypervisor string) (string, error) GetDeviceName(mountPath string) (string, int, error) @@ -56,17 +55,7 @@ func New() Interface { func (m *mounter) GetDevicePath(ctx context.Context, deviceID string, hypervisor string) (string, error) { - ctxzap.Extract(ctx).Sugar().Debugf("device id: '%s' (Hypervisor: %s)", deviceID, hypervisor) - - if strings.ToLower(hypervisor) == "vmware" { - ctxzap.Extract(ctx).Sugar().Warnf("volume hypervisor is VMWare, try to correct SCSI ID") - idInt, _ := strconv.Atoi(deviceID) - if idInt > 3 { - idInt-- - deviceID = fmt.Sprintf("%d", idInt) - ctxzap.Extract(ctx).Sugar().Warnf("new device id: %s", deviceID) - } - } + deviceID = CorrectDeviceId(ctx, deviceID, hypervisor) deviceID = fmt.Sprintf("pci-0000:00:10.0-scsi-0:0:%s:0", deviceID) ctxzap.Extract(ctx).Sugar().Debugf("device path: %s/%s", diskIDPath, deviceID) @@ -100,6 +89,22 @@ func (m *mounter) GetDevicePath(ctx context.Context, deviceID string, hypervisor return devicePath, nil } +func CorrectDeviceId(ctx context.Context, deviceID, hypervisor string) string { + ctxzap.Extract(ctx).Sugar().Debugf("device id: '%s' (Hypervisor: %s)", deviceID, hypervisor) + + if strings.ToLower(hypervisor) == "vmware" { + ctxzap.Extract(ctx).Sugar().Warnf("volume hypervisor is VMWare, try to correct SCSI ID between ID 3-7") + idInt, _ := strconv.Atoi(deviceID) + if idInt > 3 && idInt <= 7 { + idInt-- + deviceID = fmt.Sprintf("%d", idInt) + ctxzap.Extract(ctx).Sugar().Warnf("new device id: %s", deviceID) + } + } + + return deviceID +} + func (m *mounter) getDevicePathBySerialID(volumeID string) (string, error) { source := filepath.Join(diskIDPath, volumeID) _, err := os.Stat(source) @@ -116,30 +121,36 @@ func (m *mounter) rescanScsi(ctx context.Context) { log := ctxzap.Extract(ctx).Sugar() log.Debug("Scaning SCSI host...") - args := []string{"-a"} + args := []string{} cmd := m.Exec.Command("rescan-scsi-bus.sh", args...) _, err := cmd.CombinedOutput() if err != nil { - log.Warnf("Error running rescan-scsi-bus.sh -a %v\n", err) + log.Warnf("Error running rescan-scsi-bus.sh: %v\n", err) } } -func (m *mounter) CleanScsi(ctx context.Context) { +func (m *mounter) CleanScsi(ctx context.Context, deviceID, hypervisor string) { log := ctxzap.Extract(ctx).Sugar() - log.Debug("removing unmounted SCSI devices...") - args := []string{"-r"} - cmd := m.Exec.Command("rescan-scsi-bus.sh", args...) + deviceID = CorrectDeviceId(ctx, deviceID, hypervisor) + + devicePath := fmt.Sprintf("/sys/class/scsi_device/0':'0':'%s':'0/device/delete", deviceID) + log.Debugf("removing SCSI devices on %s", devicePath) + cmds := "ls " + strings.TrimSuffix(devicePath, "/delete") + cmd := m.Exec.Command(cmds) out, err := cmd.CombinedOutput() if err != nil { - log.Warnf("Error running rescan-scsi-bus.sh -r %v\n", err) + log.Warnf("Error running echo 1 > %s: %v\n", devicePath, err) } - r, _ := regexp.Compile("(\\d)* device\\(s\\) removed\\.") - subs := r.FindAllStringSubmatch(string(out), -1) - - ctxzap.Extract(ctx).Sugar().Debugf("%+v", subs) + cmds = "echo 1 > " + devicePath + cmd = m.Exec.Command(cmds) + out, err = cmd.CombinedOutput() + if err != nil { + log.Warnf("Error running echo 1 > %s: %v\n", devicePath, err) + } + fmt.Println(string(out)) } func (m *mounter) GetDeviceName(mountPath string) (string, int, error) { diff --git a/rescan-scsi-bus.sh b/rescan-scsi-bus.sh index 8bb1227..c1285e4 100755 --- a/rescan-scsi-bus.sh +++ b/rescan-scsi-bus.sh @@ -1,1190 +1,4 @@ -#!/bin/bash -# Skript to rescan SCSI bus, using the -# scsi add-single-device mechanism -# (c) 1998--2010 Kurt Garloff , GNU GPL v2 or v3 -# (c) 2006--2013 Hannes Reinecke, GNU GPL v2 or later -# $Id: rescan-scsi-bus.sh,v 1.57 2012/03/31 14:08:48 garloff Exp $ - -SCAN_WILD_CARD=4294967295 - -setcolor () -{ - red="\e[0;31m" - green="\e[0;32m" - yellow="\e[0;33m" - bold="\e[0;1m" - norm="\e[0;0m" -} - -unsetcolor () -{ - red=""; green="" - yellow=""; norm="" -} - -echo_debug() -{ - if [ $debug -eq 1 ] ; then - echo "$1" - fi -} - -# Output some text and return cursor to previous position -# (only works for simple strings) -# Stores length of string in LN and returns it -print_and_scroll_back () -{ - STRG="$1" - LN=${#STRG} - BK="" - declare -i cntr=0 - while test $cntr -lt $LN; do BK="$BK\e[D"; let cntr+=1; done - echo -en "$STRG$BK" - return $LN -} - -# Overwrite a text of length $1 (fallback to $LN) with whitespace -white_out () -{ - BK=""; WH="" - if test -n "$1"; then LN=$1; fi - declare -i cntr=0 - while test $cntr -lt $LN; do BK="$BK\e[D"; WH="$WH "; let cntr+=1; done - echo -en "$WH$BK" -} - -# Return hosts. sysfs must be mounted -findhosts_26 () -{ - hosts=`find /sys/class/scsi_host/host* -maxdepth 4 -type d -o -type l 2> /dev/null | awk -F'/' '{print $5}' | sed -e 's~host~~' | sort -nu` - scsi_host_data=`echo "$hosts" | sed -e 's~^~/sys/class/scsi_host/host~'` - for hostdir in $scsi_host_data; do - hostno=${hostdir#/sys/class/scsi_host/host} - if [ -f $hostdir/isp_name ] ; then - hostname="qla2xxx" - elif [ -f $hostdir/lpfc_drvr_version ] ; then - hostname="lpfc" - else - hostname=`cat $hostdir/proc_name` - fi - #hosts="$hosts $hostno" - echo_debug "Host adapter $hostno ($hostname) found." - done - if [ -z "$hosts" ] ; then - echo "No SCSI host adapters found in sysfs" - exit 1; - fi - # Not necessary just use double quotes around variable to preserve new lines - #hosts=`echo $hosts | tr ' ' '\n'` -} - -# Return hosts. /proc/scsi/HOSTADAPTER/? must exist -findhosts () -{ - hosts= - for driverdir in /proc/scsi/*; do - driver=${driverdir#/proc/scsi/} - if test $driver = scsi -o $driver = sg -o $driver = dummy -o $driver = device_info; then continue; fi - for hostdir in $driverdir/*; do - name=${hostdir#/proc/scsi/*/} - if test $name = add_map -o $name = map -o $name = mod_parm; then continue; fi - num=$name - driverinfo=$driver - if test -r $hostdir/status; then - num=$(printf '%d\n' `sed -n 's/SCSI host number://p' $hostdir/status`) - driverinfo="$driver:$name" - fi - hosts="$hosts $num" - echo "Host adapter $num ($driverinfo) found." - done - done -} - -printtype () -{ - local type=$1 - - case "$type" in - 0) echo "Direct-Access " ;; - 1) echo "Sequential-Access" ;; - 2) echo "Printer " ;; - 3) echo "Processor " ;; - 4) echo "WORM " ;; - 5) echo "CD-ROM " ;; - 6) echo "Scanner " ;; - 7) echo "Optical Device " ;; - 8) echo "Medium Changer " ;; - 9) echo "Communications " ;; - 10) echo "Unknown " ;; - 11) echo "Unknown " ;; - 12) echo "RAID " ;; - 13) echo "Enclosure " ;; - 14) echo "Direct-Access-RBC" ;; - *) echo "Unknown " ;; - esac -} - -print02i() -{ - if [ "$1" = "*" ] ; then - echo "00" - else - printf "%02i" "$1" - fi -} - -# Get /proc/scsi/scsi info for device $host:$channel:$id:$lun -# Optional parameter: Number of lines after first (default = 2), -# result in SCSISTR, return code 1 means empty. -procscsiscsi () -{ - if test -z "$1"; then LN=2; else LN=$1; fi - CHANNEL=`print02i "$channel"` - ID=`print02i "$id"` - LUN=`print02i "$lun"` - if [ -d /sys/class/scsi_device ]; then - SCSIPATH="/sys/class/scsi_device/${host}:${channel}:${id}:${lun}" - if [ -d "$SCSIPATH" ] ; then - SCSISTR="Host: scsi${host} Channel: $CHANNEL Id: $ID Lun: $LUN" - if [ "$LN" -gt 0 ] ; then - IVEND=$(cat ${SCSIPATH}/device/vendor) - IPROD=$(cat ${SCSIPATH}/device/model) - IPREV=$(cat ${SCSIPATH}/device/rev) - SCSIDEV=$(printf ' Vendor: %-08s Model: %-16s Rev: %-4s' "$IVEND" "$IPROD" "$IPREV") - SCSISTR="$SCSISTR -$SCSIDEV" - fi - if [ "$LN" -gt 1 ] ; then - ILVL=$(cat ${SCSIPATH}/device/scsi_level) - type=$(cat ${SCSIPATH}/device/type) - ITYPE=$(printtype $type) - SCSITMP=$(printf ' Type: %-16s ANSI SCSI revision: %02d' "$ITYPE" "$((ILVL - 1))") - SCSISTR="$SCSISTR -$SCSITMP" - fi - else - return 1 - fi - else - grepstr="scsi$host Channel: $CHANNEL Id: $ID Lun: $LUN" - SCSISTR=`cat /proc/scsi/scsi | grep -A$LN -e"$grepstr"` - fi - if test -z "$SCSISTR"; then return 1; else return 0; fi -} - -# Find sg device with 2.6 sysfs support -sgdevice26 () -{ - if test -e /sys/class/scsi_device/$host\:$channel\:$id\:$lun/device/generic; then - SGDEV=`readlink /sys/class/scsi_device/$host\:$channel\:$id\:$lun/device/generic` - SGDEV=`basename $SGDEV` - else - for SGDEV in /sys/class/scsi_generic/sg*; do - DEV=`readlink $SGDEV/device` - if test "${DEV##*/}" = "$host:$channel:$id:$lun"; then - SGDEV=`basename $SGDEV`; return - fi - done - SGDEV="" - fi -} - -# Find sg device with 2.4 report-devs extensions -sgdevice24 () -{ - if procscsiscsi 3; then - SGDEV=`echo "$SCSISTR" | grep 'Attached drivers:' | sed 's/^ *Attached drivers: \(sg[0-9]*\).*/\1/'` - fi -} - -# Find sg device that belongs to SCSI device $host $channel $id $lun -# and return in SGDEV -sgdevice () -{ - SGDEV= - if test -d /sys/class/scsi_device; then - sgdevice26 - else - DRV=`grep 'Attached drivers:' /proc/scsi/scsi 2>/dev/null` - repdevstat=$((1-$?)) - if [ $repdevstat = 0 ]; then - echo "scsi report-devs 1" >/proc/scsi/scsi - DRV=`grep 'Attached drivers:' /proc/scsi/scsi 2>/dev/null` - if [ $? = 1 ]; then return; fi - fi - if ! `echo $DRV | grep 'drivers: sg' >/dev/null`; then - modprobe sg - fi - sgdevice24 - if [ $repdevstat = 0 ]; then - echo "scsi report-devs 0" >/proc/scsi/scsi - fi - fi -} - -# Test if SCSI device is still responding to commands -# Return values: -# 0 device is present -# 1 device has changed -# 2 device has been removed -testonline () -{ - : testonline - RC=0 - if test ! -x /usr/bin/sg_turs; then return 0; fi - sgdevice - if test -z "$SGDEV"; then return 0; fi - sg_turs /dev/$SGDEV &>/dev/null - RC=$? - # Handle in progress of becoming ready and unit attention -- wait at max 11s - declare -i ctr=0 - if test $RC = 2 -o $RC = 6; then - RMB=`sg_inq /dev/$SGDEV | grep 'RMB=' | sed 's/^.*RMB=\(.\).*$/\1/'` - print_and_scroll_back "$host:$channel:$id:$lun $SGDEV ($RMB) " - fi - while test $RC = 2 -o $RC = 6 && test $ctr -le 8; do - if test $RC = 2 -a "$RMB" != "1"; then echo -n "."; let LN+=1; sleep 1 - else usleep 20000; fi - let ctr+=1 - sg_turs /dev/$SGDEV &>/dev/null - RC=$? - done - if test $ctr != 0; then white_out; fi - # echo -e "\e[A\e[A\e[A${yellow}Test existence of $SGDEV = $RC ${norm} \n\n\n" - if test $RC = 1; then return $RC; fi - # Reset RC (might be !=0 for passive paths) - RC=0 - # OK, device online, compare INQUIRY string - INQ=`sg_inq $sg_len_arg /dev/$SGDEV 2>/dev/null` - IVEND=`echo "$INQ" | grep 'Vendor identification:' | sed 's/^[^:]*: \(.*\)$/\1/'` - IPROD=`echo "$INQ" | grep 'Product identification:' | sed 's/^[^:]*: \(.*\)$/\1/'` - IPREV=`echo "$INQ" | grep 'Product revision level:' | sed 's/^[^:]*: \(.*\)$/\1/'` - STR=`printf " Vendor: %-08s Model: %-16s Rev: %-4s" "$IVEND" "$IPROD" "$IPREV"` - IPTYPE=`echo "$INQ" | sed -n 's/.* Device_type=\([0-9]*\) .*/\1/p'` - IPQUAL=`echo "$INQ" | sed -n 's/ *PQual=\([0-9]*\) Device.*/\1/p'` - if [ "$IPQUAL" != 0 ] ; then - echo -e "\e[A\e[A\e[A\e[A${red}$SGDEV changed: ${bold}LU not available (PQual $IPQUAL)${norm} \n\n\n" - return 2 - fi - - TYPE=$(printtype $IPTYPE) - if ! procscsiscsi ; then - echo -e "\e[A\e[A\e[A\e[A${red}$SGDEV removed.\n\n\n" - return 2 - fi - TMPSTR=`echo "$SCSISTR" | grep 'Vendor:'` - if [ "$TMPSTR" != "$STR" ]; then - echo -e "\e[A\e[A\e[A\e[A${red}$SGDEV changed: ${bold}\nfrom:${SCSISTR#* } \nto: $STR ${norm} \n\n\n" - return 1 - fi - TMPSTR=`echo "$SCSISTR" | sed -n 's/.*Type: *\(.*\) *ANSI.*/\1/p'` - NTMPSTR="$(sed -e 's/[[:space:]]*$//' -e 's/^[[:space:]]*//' <<<${TMPSTR})" - NTYPE="$(sed -e 's/[[:space:]]*$//' -e 's/^[[:space:]]*//' <<<${TYPE})" - if [ "$NTMPSTR" != "$NTYPE" ] ; then - echo -e "\e[A\e[A\e[A\e[A${red}$SGDEV changed: ${bold}\nfrom:${TMPSTR} \nto: $TYPE ${norm} \n\n\n" - return 1 - fi - return $RC -} - -# Test if SCSI device $host $channen $id $lun exists -# Outputs description from /proc/scsi/scsi (unless arg passed) -# Returns SCSISTR (empty if no dev) -testexist () -{ - : testexist - SCSISTR= - if procscsiscsi && test -z "$1"; then - echo "$SCSISTR" | head -n1 - echo "$SCSISTR" | tail -n2 | pr -o4 -l1 - fi -} - -# Returns the list of existing channels per host -chanlist () -{ - local hcil - local cil - local chan - local tmpchan - - for dev in /sys/class/scsi_device/${host}:* ; do - [ -d $dev ] || continue; - hcil=${dev##*/} - cil=${hcil#*:} - chan=${cil%%:*} - for tmpchan in $channelsearch ; do - if test "$chan" -eq $tmpchan ; then - chan= - fi - done - if test -n "$chan" ; then - channelsearch="$channelsearch $chan" - fi - done - if test -z "$channelsearch"; then channelsearch="0"; fi -} - -# Returns the list of existing targets per host -idlist () -{ - local hcil - local target - local tmpid - local newid - - idsearch=$(ls /sys/class/scsi_device/ | sed -n "s/${host}:${channel}:\([0-9]*\):[0-9]*/\1/p" | uniq) - echo "${channel} - -" > /sys/class/scsi_host/host${host}/scan - # Rescan to check if we found new targets - newsearch=$(ls /sys/class/scsi_device/ | sed -n "s/${host}:${channel}:\([0-9]*\):[0-9]*/\1/p" | uniq) - for id in $newsearch ; do - newid=$id - for tmpid in $idsearch ; do - if test $id -eq $tmpid ; then - newid= - break - fi - done - if test -n "$newid" ; then - id=$newid - for dev in /sys/class/scsi_device/${host}:${channel}:${newid}:* ; do - [ -d $dev ] || continue; - hcil=${dev##*/} - lun=${hcil##*:} - printf "\r${green}NEW: $norm" - testexist - if test "$SCSISTR" ; then - incrfound "$hcil" - fi +#!/bin/sh +for host in /sys/class/scsi_host/host*/scan ; do + echo "- - -" > $host done - fi - done -} - -# Returns the list of existing LUNs from device $host $channel $id $lun -# and returns list to stdout -getluns() -{ - sgdevice - if test -z "$SGDEV"; then return 1; fi - if test ! -x /usr/bin/sg_luns; then echo 0; return 1; fi - LLUN=`sg_luns /dev/$SGDEV 2>/dev/null | sed -n 's/ \(.*\)/\1/p'` - # Added -z $LLUN condition because $? gets the RC from sed, not sg_luns - if test $? != 0 -o -z "$LLUN"; then echo 0; return 1; fi - for lun in $LLUN ; do - # Swap LUN number - l0=$(printf '%u' 0x$lun) - l1=$(( ($l0 >> 48) & 0xffff )) - l2=$(( ($l0 >> 32) & 0xffff )) - l3=$(( ($l0 >> 16) & 0xffff )) - l4=$(( $l0 & 0xffff )) - l0=$(( ( ( ($l4 * 0xffff) + $l3 ) * 0xffff + $l2 ) * 0xffff + $l1 )) - printf "%u\n" $l0 - done - return 0 -} - -# Wait for udev to settle (create device nodes etc.) -udevadm_settle() -{ - if test -x /sbin/udevadm; then - print_and_scroll_back " Calling udevadm settle (can take a while) " - /sbin/udevadm settle - white_out - elif test -x /sbin/udevsettle; then - print_and_scroll_back " Calling udevsettle (can take a while) " - /sbin/udevsettle - white_out - else - usleep 20000 - fi -} - -# Perform scan on a single lun $host $channel $id $lun -dolunscan() -{ - local remappedlun0= - SCSISTR= - devnr="$host $channel $id $lun" - echo -e " Scanning for device $devnr ... " - printf "${yellow}OLD: $norm" - testexist - # Device exists: Test whether it's still online - # (testonline returns 2 if it's gone and 1 if it has changed) - if test "$SCSISTR" ; then - testonline - RC=$? - # Well known lun transition case. Only for Direct-Access devs (type 0) - # If block directory exists && and PQUAL != 0, we unmapped lun0 and just have a well-known lun - # If block directory doesn't exist && PQUAL == 0, we mapped a real lun0 - if test $lun -eq 0 -a $IPTYPE -eq 0 ; then - if test $RC = 2 ; then - if test -e /sys/class/scsi_device/${host}:${channel}:${id}:${lun}/device; then - if test -d /sys/class/scsi_device/${host}:${channel}:${id}:${lun}/device/block ; then - remappedlun0=2 # Transition from real lun 0 to well-known - else - RC=0 # Set this so the system leaves the existing well known lun alone. This is a lun 0 with no block directory - fi - fi - elif test $RC = 0 -a $IPTYPE -eq 0; then - if test -e /sys/class/scsi_device/${host}:${channel}:${id}:${lun}/device; then - if test ! -d /sys/class/scsi_device/${host}:${channel}:${id}:${lun}/device/block ; then - remappedlun0=1 # Transition from well-known to real lun 0 - fi - fi - fi - fi - fi - - # Special case: lun 0 just got added (for reportlunscan), - # so make sure we correctly treat it as new - if test "$lun" = "0" -a "$1" = "1" -a -z "$remappedlun0"; then - SCSISTR="" - printf "\r\e[A\e[A\e[A" - fi - - : f $remove s $SCSISTR - if test "$remove" -a "$SCSISTR" -o "$remappedlun0" = "1"; then - if test $RC != 0 -o ! -z "$forceremove" -o -n "$remappedlun0"; then - if test "$remappedlun0" != "1" ; then - echo -en "\r\e[A\e[A\e[A${red}REM: " - echo "$SCSISTR" | head -n1 - echo -e "${norm}\e[B\e[B" - fi - if test -e /sys/class/scsi_device/${host}:${channel}:${id}:${lun}/device; then - # have to preemptively do this so we can figure out the mpath device - # Don't do this if we're deleting a well known lun to replace it - if test "$remappedlun0" != "1" ; then - incrrmvd "$host:$channel:$id:$lun" - fi - echo 1 > /sys/class/scsi_device/${host}:${channel}:${id}:${lun}/device/delete - usleep 20000 - else - echo "scsi remove-single-device $devnr" > /proc/scsi/scsi - if test $RC -eq 1 -o $lun -eq 0 ; then - # Try readding, should fail if device is gone - echo "scsi add-single-device $devnr" > /proc/scsi/scsi - fi - fi - fi - if test $RC = 0 -o "$forcerescan" ; then - if test -e /sys/class/scsi_device/${host}:${channel}:${id}:${lun}/device; then - echo 1 > /sys/class/scsi_device/${host}:${channel}:${id}:${lun}/device/rescan - fi - fi - printf "\r\e[A\e[A\e[A${yellow}OLD: $norm" - testexist - if test -z "$SCSISTR" -a $RC != 1 -a "$remappedlun0" != "1"; then - printf "\r${red}DEL: $norm\r\n\n" - # In the event we're replacing with a well known node, we need to let it continue, to create the replacement node - test "$remappedlun0" != "2" && return 1 - fi - fi - if test -z "$SCSISTR" -o -n "$remappedlun0"; then - if test "$remappedlun0" != "2" ; then - # Device does not exist, try to add - printf "\r${green}NEW: $norm" - fi - if test -e /sys/class/scsi_host/host${host}/scan; then - echo "$channel $id $lun" > /sys/class/scsi_host/host${host}/scan 2> /dev/null - else - echo "scsi add-single-device $devnr" > /proc/scsi/scsi - fi - testexist - if test -z "$SCSISTR"; then - # Device not present - printf "\r\e[A"; - # Optimization: if lun==0, stop here (only if in non-remove mode) - if test $lun = 0 -a -z "$remove" -a $optscan = 1; then - break; - fi - else - if test "$remappedlun0" != "2" ; then - incrfound "$host:$channel:$id:$lun" - fi - fi - fi -} - -# Perform report lun scan on $host $channel $id using REPORT_LUNS -doreportlun() -{ - lun=0 - SCSISTR= - devnr="$host $channel $id $lun" - echo -en " Scanning for device $devnr ...\r" - lun0added= - #printf "${yellow}OLD: $norm" - # Phase one: If LUN0 does not exist, try to add - testexist -q - if test -z "$SCSISTR"; then - # Device does not exist, try to add - #printf "\r${green}NEW: $norm" - if test -e /sys/class/scsi_host/host${host}/scan; then - echo "$channel $id $lun" > /sys/class/scsi_host/host${host}/scan 2> /dev/null - udevadm_settle - else - echo "scsi add-single-device $devnr" > /proc/scsi/scsi - fi - testexist -q - if test -n "$SCSISTR"; then - lun0added=1 - #testonline - else - # Device not present - # return - # Find alternative LUN to send getluns to - for dev in /sys/class/scsi_device/${host}:${channel}:${id}:*; do - [ -d "$dev" ] || continue - lun=${dev##*:} - break - done - fi - fi - targetluns=`getluns` - REPLUNSTAT=$? - lunremove= - #echo "getluns reports " $targetluns - olddev=`find /sys/class/scsi_device/ -name $host:$channel:$id:* 2>/dev/null` - oldluns=`echo "$olddev" | awk -F'/' '{print $5}' | awk -F':' '{print $4}'` - oldtargets="$targetluns" - # OK -- if we don't have a LUN to send a REPORT_LUNS to, we could - # fall back to wildcard scanning. Same thing if the device does not - # support REPORT_LUNS - # TODO: We might be better off to ALWAYS use wildcard scanning if - # it works - if test "$REPLUNSTAT" = "1"; then - if test -e /sys/class/scsi_host/host${host}/scan; then - echo "$channel $id -" > /sys/class/scsi_host/host${host}/scan 2> /dev/null - udevadm_settle - else - echo "scsi add-single-device $host $channel $id $SCAN_WILD_CARD" > /proc/scsi/scsi - fi - targetluns=`find /sys/class/scsi_device/ -name $host:$channel:$id:* 2>/dev/null | awk -F'/' '{print $5}' | awk -F':' '{print $4}' | sort -n` - let found+=`echo "$targetluns" | wc -l` - let found-=`echo "$olddev" | wc -l` - fi - if test -z "$targetluns"; then targetluns="$oldtargets"; fi - # Check existing luns - for dev in $olddev; do - [ -d "$dev" ] || continue - lun=${dev##*:} - newsearch= - inlist= - # OK, is existing $lun (still) in reported list - for tmplun in $targetluns; do - if test $tmplun -eq $lun ; then - inlist=1 - dolunscan $lun0added - else - newsearch="$newsearch $tmplun" - fi - done - # OK, we have now done a lunscan on $lun and - # $newsearch is the old $targetluns without $lun - if [ -z "$inlist" ]; then - # Stale lun - lunremove="$lunremove $lun" - fi - # $lun removed from $lunsearch (echo for whitespace cleanup) - targetluns=`echo $newsearch` - done - # Add new ones and check stale ones - for lun in $targetluns $lunremove; do - dolunscan $lun0added - done -} - -# Perform search (scan $host) -dosearch () -{ - if test -z "$channelsearch" ; then - chanlist - fi - for channel in $channelsearch; do - if test -z "$idsearch" ; then - idlist - fi - for id in $idsearch; do - if test -z "$lunsearch" ; then - doreportlun - else - for lun in $lunsearch; do - dolunscan - done - fi - done - done -} - -expandlist () -{ - list=$1 - result="" - first=${list%%,*} - rest=${list#*,} - while test ! -z "$first"; do - beg=${first%%-*}; - if test "$beg" = "$first"; then - result="$result $beg"; - else - end=${first#*-} - result="$result `seq $beg $end`" - fi - test "$rest" = "$first" && rest="" - first=${rest%%,*} - rest=${rest#*,} - done - echo $result -} - -searchexisting() -{ - local tmpch; - local tmpid - local match=0 - local targets=`ls -d /sys/class/scsi_device/$host:* 2> /dev/null | egrep -o $host:[0-9]+:[0-9]+ | sort | uniq` - - # Nothing came back on this host, so we should skip it - test -z "$targets" && return - - local target=; - for target in $targets ; do - channel=`echo $target | cut -d":" -f2` - id=`echo $target | cut -d":" -f 3` - if [ -n "$channelsearch" ] ; then - for tmpch in $channelsearch ; do - test $tmpch -eq $channel && match=1 - done - else - match=1 - fi - - test $match -eq 0 && continue - match=0 - - if [ $filter_ids -eq 1 ] ; then - for tmpid in $idsearch ; do - if [ $tmpid -eq $id ] ; then - match=1 - fi - done - else - match=1 - fi - - test $match -eq 0 && continue - - if [ -z "$lunsearch" ] ; then - doreportlun - else - for lun in $lunsearch ; do - dolunscan - done - fi - done -} - -# Go through all of the existing devices and figure out any that have been remapped -findremapped() -{ - local hctl=; - local devs=`ls /sys/class/scsi_device/` - local sddev= - local id_serial= - local id_serial_old= - local sysfs_devpath= - mpaths="" - local tmpfile="/tmp/rescan-scsi-bus-`date +s`" - - test -f $tmpfile && rm $tmpfile - - # Get all of the ID_SERIAL attributes, after finding their sd node - for hctl in $devs ; do - if [ -d /sys/class/scsi_device/$hctl/device/block ] ; then - sddev=`ls /sys/class/scsi_device/$hctl/device/block` - id_serial_old=`udevadm info -q all -n $sddev | grep "ID_SERIAL=" | cut -d"=" -f2` - [ -z "$id_serial_old" ] && id_serial_old="none" - echo "$hctl $sddev $id_serial_old" >> $tmpfile - fi - done - - # Trigger udev to update the info - echo -n "Triggering udev to update device information... " - /sbin/udevadm trigger - udevadm_settle &>/dev/null - echo "Done" - - # See what changed and reload the respective multipath device if applicable - while read hctl sddev id_serial_old ; do - id_serial=`udevadm info -q all -n $sddev | grep "ID_SERIAL=" | cut -d"=" -f2` - [ -z "$id_serial" ] && id_serial="none" - if [ "$id_serial_old" != "$id_serial" ] ; then - if [ "$id_serial" = "1" ] ; then - continue # the lun was unmapped and is getting the blank scsi id (1) - fi - printf "${yellow}REMAPPED: $norm" - host=`echo $hctl | cut -d":" -f1` - channel=`echo $hctl | cut -d":" -f2` - id=`echo $hctl | cut -d":" -f3` - lun=`echo $hctl | cut -d":" -f4` - procscsiscsi - echo "$SCSISTR" - incrchgd "$hctl" - fi - done < $tmpfile - rm $tmpfile &>/dev/null - - if test -n "$mp_enable" -a -n "$mpaths" ; then - echo "Updating multipath device mappings" - flushmpaths - $MULTIPATH | grep "create:" 2> /dev/null - fi -} - -incrfound() -{ - local hctl="$1" - if test -n "$hctl" ; then - let found+=1 - FOUNDDEVS="$FOUNDDEVS\t[$hctl]\n" - else - return - fi -} - -incrchgd() -{ - local hctl="$1" - if test -n "$hctl" ; then - let updated+=1 - CHGDEVS="$CHGDEVS\t[$hctl]\n" - else - return - fi - - if test -n "$mp_enable" ; then - local sdev="`findsddev \"$hctl\"`" - if test -n "$sdev" ; then - findmultipath "$sdev" - fi - fi -} - -incrrmvd() -{ - local hctl="$1" - if test -n "$hctl" ; then - let rmvd+=1; - RMVDDEVS="$RMVDDEVS\t[$hctl]\n" - else - return - fi - - if test -n "$mp_enable" ; then - local sdev="`findsddev \"$hctl\"`" - if test -n "$sdev" ; then - findmultipath "$sdev" - fi - fi -} - -findsddev() -{ - local hctl="$1" - local sddev= - - if test ! -e /sys/class/scsi_device/$hctl/device/block ; then - return 1 - fi - - sddev=`ls /sys/class/scsi_device/$hctl/device/block` - echo $sddev - - return 0 -} - -findmultipath() -{ - local dev="$1" - local mp= - local mp2= - - # Need a sdev, and executable multipath and dmsetup command here - if [ -z "$dev" ] || [ ! -x $DMSETUP ] ; then - return 1 - fi - - local maj_min=`cat /sys/block/$dev/dev` - for mp in $($DMSETUP ls --target=multipath | cut -f 1) ; do - if $($DMSETUP status $mp | grep -q " $maj_min ") ; then - for mp2 in $mpaths ; do - # mp device is already there, return - if [ "$mp2" = "$mp" ] ; then - return - fi - done - mpaths="$mpaths $mp" - return - fi - done -} - -reloadmpaths() -{ - local mpath - if [ ! -x $MULTIPATH ] ; then - echo "no -x multipath" - return - fi - - if [ "$1" = "1" ] ; then - echo "Reloading all multipath devices" - $MULTIPATH -r &>/dev/null - return - fi - - # Reload the multipath devices - for mpath in $mpaths ; do - echo "Reloading multipath device $mpath" - $MULTIPATH -r $mpath &>/dev/null - done -} - -flushmpaths() -{ - local mpath - # Mode: flush only failed - if test -n "$1" ; then - for mpath in $($DMSETUP ls --target=multipath | cut -f 1) ; do - num=$($DMSETUP status $mpath | awk 'BEGIN{RS=" ";active=0}/[0-9]+:[0-9]+/{dev=1}/A/{if (dev == 1) active++; dev=0} END{ print active }') - if [ $num -eq 0 ] ; then - echo -n "Flushing multipath device $mpath... " - $DMSETUP message $mpath 0 fail_if_no_path &>/dev/null - $MULTIPATH -f $mpath &>/dev/null - $DMSETUP status $mpath &>/dev/null - if test "$?" = "1" ; then - echo "Done" - else - echo "Fail" - fi - fi - done - return - fi - # Flush all the devs specified in $mpaths - for mpath in $mpaths ; do - echo -n "Flushing multipath device $mpath... " - $DMSETUP message $mpath 0 fail_if_no_path &>/dev/null - $MULTIPATH -f $mpath &>/dev/null - $DMSETUP status $mpath &>/dev/null - if test "$?" = "1" ; then - echo "Done" - else - echo "Fail" - fi - done -} - - -# Find resized luns -findresized() -{ - local devs=`ls /sys/class/scsi_device/` - local size= - local new_size= - local sysfs_path= - local sddev= - - for hctl in $devs ; do - sysfs_path="/sys/class/scsi_device/$hctl/device" - if [ -d "$sysfs_path/block" ] ; then - sddev=`ls $sysfs_path/block` - size=`cat $sysfs_path/block/$sddev/size` - - echo 1 > $sysfs_path/rescan - new_size=`cat $sysfs_path/block/$sddev/size` - - if [ "$size" != "$new_size" ] && [ "$size" != "0" ] && [ "$new_size" != "0" ] ; then - printf "${yellow}RESIZED: $norm" - host=`echo $hctl | cut -d":" -f1` - channel=`echo $hctl | cut -d":" -f2` - id=`echo $hctl | cut -d":" -f3` - lun=`echo $hctl | cut -d":" -f4` - - procscsiscsi - echo "$SCSISTR" - incrchgd "$hctl" - fi - fi - done - - if test -n "$mp_enable" -a -n "$mpaths" ; then - reloadmpaths - fi -} - -FOUNDDEVS="" -CHGDEVS="" -RMVDDEVS="" - -# main -if test @$1 = @--help -o @$1 = @-h -o @$1 = @-?; then - echo "Usage: rescan-scsi-bus.sh [options] [host [host ...]]" - echo "Options:" - echo " -a scan all targets, not just currently existing [default: disabled]" - echo " -d enable debug [default: 0]" - echo " -l activates scanning for LUNs 0--7 [default: 0]" - echo " -L NUM activates scanning for LUNs 0--NUM [default: 0]" - echo " -w scan for target device IDs 0--15 [default: 0--7]" - echo " -c enables scanning of channels 0 1 [default: 0 / all detected ones]" - echo " -m update multipath devices [default: disabled]" - echo " -r enables removing of devices [default: disabled]" - echo " -f flush failed multipath devices [default: disabled]" - echo " -i issue a FibreChannel LIP reset [default: disabled]" - echo " -u look for existing disks that have been remapped" - echo " -s look for resized disks and reload associated multipath devices, if applicable" - echo "--alltargets: same as -a" - echo "--remove: same as -r" - echo "--flush: same as -f" - echo "--issue-lip: same as -i" - echo "--wide: same as -w" - echo "--multipath: same as -m" - echo "--forceremove: Remove stale devices (DANGEROUS)" - echo "--forcerescan: Remove and readd existing devices (DANGEROUS)" - echo "--nooptscan: don't stop looking for LUNs if 0 is not found" - echo "--color: use coloured prefixes OLD/NEW/DEL" - echo "--hosts=LIST: Scan only host(s) in LIST" - echo "--channels=LIST: Scan only channel(s) in LIST" - echo "--ids=LIST: Scan only target ID(s) in LIST" - echo "--luns=LIST: Scan only lun(s) in LIST" - echo "--sync/nosync: Issue a sync / no sync [default: sync if remove]" - echo "--attachpq3: Tell kernel to attach sg to LUN 0 that reports PQ=3" - echo "--reportlun2: Tell kernel to try REPORT_LUN even on SCSI2 devices" - echo "--largelun: Tell kernel to support LUNs > 7 even on SCSI2 devs" - echo "--sparselun: Tell kernel to support sparse LUN numbering" - echo "--update: same as -u" - echo "--resize: same as -s" - echo " Host numbers may thus be specified either directly on cmd line (deprecated) or" - echo " or with the --hosts=LIST parameter (recommended)." - echo "LIST: A[-B][,C[-D]]... is a comma separated list of single values and ranges" - echo " (No spaces allowed.)" - exit 0 -fi - -if test ! -d /sys/class/scsi_host/ -a ! -d /proc/scsi/; then - echo "Error: SCSI subsystem not active" - exit 1 -fi - -# Make sure sg is there -modprobe sg &>/dev/null - -if test -x /usr/bin/sg_inq; then - sg_version=$(sg_inq -V 2>&1 | cut -d " " -f 3) - if test -n "$sg_version"; then - sg_ver_maj=${sg_version:0:1} - sg_version=${sg_version##?.} - let sg_version+=$((100*$sg_ver_maj)) - fi - sg_version=${sg_version##0.} - #echo "\"$sg_version\"" - if [ -z "$sg_version" -o "$sg_version" -lt 70 ] ; then - sg_len_arg="-36" - else - sg_len_arg="--len=36" - fi -else - echo "WARN: /usr/bin/sg_inq not present -- please install sg3_utils" - echo " or rescan-scsi-bus.sh might not fully work." -fi - -# defaults -unsetcolor -debug=0 -lunsearch= -opt_idsearch=`seq 0 7` -filter_ids=0 -opt_channelsearch= -remove= -updated=0 -update=0 -resize=0 -forceremove= -optscan=1 -sync=1 -existing_targets=1 -mp_enable= -declare -i scan_flags=0 - -# Scan options -opt="$1" -while test ! -z "$opt" -a -z "${opt##-*}"; do - opt=${opt#-} - case "$opt" in - a) existing_targets=;; #Scan ALL targets when specified - d) debug=1 ;; - f) flush=1 ;; - l) lunsearch=`seq 0 7` ;; - L) lunsearch=`seq 0 $2`; shift ;; - m) mp_enable=1 ;; - w) opt_idsearch=`seq 0 15` ;; - c) opt_channelsearch="0 1" ;; - r) remove=1 ;; - s) resize=1 ;; - i) lipreset=1 ;; - u) update=1 ;; - -alltargets) existing_targets=;; - -flush) flush=1 ;; - -remove) remove=1 ;; - -forcerescan) remove=1; forcerescan=1 ;; - -forceremove) remove=1; forceremove=1 ;; - -hosts=*) arg=${opt#-hosts=}; hosts=`expandlist $arg` ;; - -channels=*) arg=${opt#-channels=};opt_channelsearch=`expandlist $arg` ;; - -ids=*) arg=${opt#-ids=}; opt_idsearch=`expandlist $arg` ; filter_ids=1;; - -luns=*) arg=${opt#-luns=}; lunsearch=`expandlist $arg` ;; - -color) setcolor ;; - -nooptscan) optscan=0 ;; - -issue-lip) lipreset=1 ;; - -sync) sync=2 ;; - -nosync) sync=0 ;; - -multipath) mp_enable=1 ;; - -attachpq3) scan_flags=$(($scan_flags|0x1000000)) ;; - -reportlun2) scan_flags=$(($scan_flags|0x20000)) ;; - -resize) resize=1;; - -largelun) scan_flags=$(($scan_flags|0x200)) ;; - -sparselun) scan_flags=$((scan_flags|0x40)) ;; - -update) update=1;; - -wide) opt_idsearch=`seq 0 15` ;; - *) echo "Unknown option -$opt !" ;; - esac - shift - opt="$1" -done - -if [ -z "$hosts" ] ; then - if test -d /sys/class/scsi_host; then - findhosts_26 - else - findhosts - fi -fi - -if [ -d /sys/class/scsi_host -a ! -w /sys/class/scsi_host ]; then - echo "You need to run scsi-rescan-bus.sh as root" - exit 2 -fi -if test "$sync" = 1 -a "$remove" = 1; then sync=2; fi -if test "$sync" = 2; then echo "Syncing file systems"; sync; fi -if test -w /sys/module/scsi_mod/parameters/default_dev_flags -a $scan_flags != 0; then - OLD_SCANFLAGS=`cat /sys/module/scsi_mod/parameters/default_dev_flags` - NEW_SCANFLAGS=$(($OLD_SCANFLAGS|$scan_flags)) - if test "$OLD_SCANFLAGS" != "$NEW_SCANFLAGS"; then - echo -n "Temporarily setting kernel scanning flags from " - printf "0x%08x to 0x%08x\n" $OLD_SCANFLAGS $NEW_SCANFLAGS - echo $NEW_SCANFLAGS > /sys/module/scsi_mod/parameters/default_dev_flags - else - unset OLD_SCANFLAGS - fi -fi -DMSETUP=$(which dmsetup) -[ -z "$DMSETUP" ] && flush= && mp_enable= -MULTIPATH=$(which multipath) -[ -z "$MULTIPATH" ] && flush= && mp_enable= - -echo -n "Scanning SCSI subsystem for new devices" -test -z "$flush" || echo -n ", flush failed multipath devices," -test -z "$remove" || echo -n " and remove devices that have disappeared" -echo -declare -i found=0 -declare -i updated=0 -declare -i rmvd=0 - -if [ -n "$flush" ] ; then - if [ -x $MULTIPATH ] ; then - flushmpaths 1 - fi -fi - -# Update existing mappings -if [ $update -eq 1 ] ; then - echo "Searching for remapped LUNs" - findremapped -# Search for resized LUNs -elif [ $resize -eq 1 ] ; then - echo "Searching for resized LUNs" - findresized -# Normal rescan mode -else - for host in $hosts; do - echo -n "Scanning host $host " - if test -e /sys/class/fc_host/host$host ; then - # It's pointless to do a target scan on FC - if test -n "$lipreset" ; then - echo 1 > /sys/class/fc_host/host$host/issue_lip 2> /dev/null; - udevadm_settle - fi - channelsearch= - idsearch= - else - channelsearch=$opt_channelsearch - idsearch=$opt_idsearch - fi - [ -n "$channelsearch" ] && echo -n "channels $channelsearch " - echo -n "for " - if [ -n "$idsearch" ] ; then - echo -n " SCSI target IDs " $idsearch - else - echo -n " all SCSI target IDs" - fi - if [ -n "$lunsearch" ] ; then - echo ", LUNs " $lunsearch - else - echo ", all LUNs" - fi - - if [ -n "$existing_targets" ] ; then - searchexisting - else - dosearch - fi - done - if test -n "$OLD_SCANFLAGS"; then - echo $OLD_SCANFLAGS > /sys/module/scsi_mod/parameters/default_dev_flags - fi -fi - -let rmvd_found=$rmvd+$found -if test -n "$mp_enable" -a $rmvd_found -gt 0 ; then - echo "Attempting to update multipath devices..." - if test $rmvd -gt 0 ; then - /sbin/udevadm trigger - udevadm_settle - echo "Removing multipath mappings for removed devices if all paths are now failed... " - flushmpaths 1 - fi - if test $found -gt 0 ; then - /sbin/udevadm trigger - udevadm_settle - echo "Trying to discover new multipath mappings for newly discovered devices... " - $MULTIPATH | grep "create:" 2> /dev/null - fi -fi - -echo "$found new or changed device(s) found. " -if test ! -z "$FOUNDDEVS" ; then - printf "$FOUNDDEVS" -fi -echo "$updated remapped or resized device(s) found. " -if test ! -z "$CHGDEVS" ; then - printf "$CHGDEVS" -fi -echo "$rmvd device(s) removed. " -if test ! -z "$RMVDDEVS" ; then - printf "$RMVDDEVS" -fi - -# Local Variables: -# sh-basic-offset: 2 -# End: - From 702e2768d6c25178e4e12ec8c5fea744dff31f42 Mon Sep 17 00:00:00 2001 From: Christian Schlatter Date: Fri, 6 Aug 2021 09:37:00 +0200 Subject: [PATCH 10/22] use script for cleaning scsi bus --- clean-scsi-bus.sh | 2 ++ cmd/cloudstack-csi-driver/Dockerfile | 4 +++- pkg/driver/node.go | 7 +++++-- pkg/mount/mount.go | 13 +++---------- 4 files changed, 13 insertions(+), 13 deletions(-) create mode 100755 clean-scsi-bus.sh diff --git a/clean-scsi-bus.sh b/clean-scsi-bus.sh new file mode 100755 index 0000000..466c475 --- /dev/null +++ b/clean-scsi-bus.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /sys/class/scsi_device/0\:0\:$1\:0/device/delete \ No newline at end of file diff --git a/cmd/cloudstack-csi-driver/Dockerfile b/cmd/cloudstack-csi-driver/Dockerfile index 7268f17..de2bd40 100644 --- a/cmd/cloudstack-csi-driver/Dockerfile +++ b/cmd/cloudstack-csi-driver/Dockerfile @@ -2,7 +2,7 @@ FROM alpine:3.14.0 LABEL \ org.opencontainers.image.description="CloudStack CSI driver" \ - org.opencontainers.image.source="https://github.com/apalia/cloudstack-csi-driver/" + org.opencontainers.image.source="https://github.com/swisstxt/cloudstack-csi-driver/" RUN apk add --no-cache \ ca-certificates \ @@ -18,4 +18,6 @@ RUN apk add --no-cache \ COPY ./bin/cloudstack-csi-driver /cloudstack-csi-driver COPY rescan-scsi-bus.sh /usr/bin/ RUN chmod +x /usr/bin/rescan-scsi-bus.sh +COPY clean-scsi-bus.sh /usr/bin/ +RUN chmod +x /usr/bin/clean-scsi-bus.sh ENTRYPOINT ["/cloudstack-csi-driver"] \ No newline at end of file diff --git a/pkg/driver/node.go b/pkg/driver/node.go index 06c647a..cf0a0fc 100644 --- a/pkg/driver/node.go +++ b/pkg/driver/node.go @@ -173,9 +173,12 @@ func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag ctxzap.Extract(ctx).Sugar().Debugf("NodeUnstageVolume: unmounted %s on target %s", dev, target) - v, _ := ns.connector.GetVolumeByID(ctx, volumeID) + v, err := ns.connector.GetVolumeByID(ctx, volumeID) + if err != nil { + return nil, status.Errorf(codes.Internal, "Could not find volume %s: %v", volumeID, err) + } - ns.mounter.CleanScsi(ctx, volumeID, v.Hypervisor) + ns.mounter.CleanScsi(ctx, v.DeviceID, v.Hypervisor) return &csi.NodeUnstageVolumeResponse{}, nil } diff --git a/pkg/mount/mount.go b/pkg/mount/mount.go index e4eba9b..0bbf599 100644 --- a/pkg/mount/mount.go +++ b/pkg/mount/mount.go @@ -134,22 +134,15 @@ func (m *mounter) CleanScsi(ctx context.Context, deviceID, hypervisor string) { deviceID = CorrectDeviceId(ctx, deviceID, hypervisor) - devicePath := fmt.Sprintf("/sys/class/scsi_device/0':'0':'%s':'0/device/delete", deviceID) + devicePath := fmt.Sprintf("/sys/class/scsi_device/0:0:%s:0/device/delete", deviceID) log.Debugf("removing SCSI devices on %s", devicePath) - cmds := "ls " + strings.TrimSuffix(devicePath, "/delete") - cmd := m.Exec.Command(cmds) + args := []string{deviceID} + cmd := m.Exec.Command("clean-scsi-bus.sh", args...) out, err := cmd.CombinedOutput() if err != nil { log.Warnf("Error running echo 1 > %s: %v\n", devicePath, err) } - cmds = "echo 1 > " + devicePath - cmd = m.Exec.Command(cmds) - out, err = cmd.CombinedOutput() - if err != nil { - log.Warnf("Error running echo 1 > %s: %v\n", devicePath, err) - } - fmt.Println(string(out)) } From 685b37bbf231c79c1c8972ed63938f32fe3a5f6d Mon Sep 17 00:00:00 2001 From: Christian Schlatter Date: Fri, 6 Aug 2021 10:12:24 +0200 Subject: [PATCH 11/22] add mutex to avoid race conditions while attaching volume --- pkg/driver/controller.go | 12 ++++++++++++ pkg/mount/mount.go | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index baa6365..f911516 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "math/rand" + "sync" "github.com/apalia/cloudstack-csi-driver/pkg/cloud" "github.com/apalia/cloudstack-csi-driver/pkg/util" @@ -24,12 +25,14 @@ var onlyVolumeCapAccessMode = csi.VolumeCapability_AccessMode{ type controllerServer struct { csi.UnimplementedControllerServer connector cloud.Interface + locks map[string]*sync.Mutex } // NewControllerServer creates a new Controller gRPC server. func NewControllerServer(connector cloud.Interface) csi.ControllerServer { return &controllerServer{ connector: connector, + locks: make(map[string]*sync.Mutex), } } @@ -225,6 +228,15 @@ func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *cs } nodeID := req.GetNodeId() + //Ensure only one node is processing at same time + lock, ok := cs.locks[nodeID] + if !ok { + lock = &sync.Mutex{} + cs.locks[nodeID] = lock + } + lock.Lock() + defer lock.Unlock() + if req.GetReadonly() { return nil, status.Error(codes.InvalidArgument, "Readonly not possible") } diff --git a/pkg/mount/mount.go b/pkg/mount/mount.go index 0bbf599..ffaee54 100644 --- a/pkg/mount/mount.go +++ b/pkg/mount/mount.go @@ -89,6 +89,11 @@ func (m *mounter) GetDevicePath(ctx context.Context, deviceID string, hypervisor return devicePath, nil } +//Corrects the device id on the node. the scsi id may not match th id which is set from te cloudstack controller +//1. ClousStack assumes that SCSI ID 3 is always the CD-ROM and is ignoring this id. +//https://github.com/apache/cloudstack/blob/98d42750cc21dfce5a8dd6d1880e09a621e0152e/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java#L3442 +//2. SCSI ID 7 is reserved for the Virtual SCSI Controller +//https://docs.vmware.com/en/VMware-vSphere/6.0/com.vmware.vsphere.hostclient.doc/GUID-5872D173-A076-42FE-8D0B-9DB0EB0E7362_copy.html func CorrectDeviceId(ctx context.Context, deviceID, hypervisor string) string { ctxzap.Extract(ctx).Sugar().Debugf("device id: '%s' (Hypervisor: %s)", deviceID, hypervisor) From dd9d074e827fd3ee3e2e2c301f27c4c593dcf2f9 Mon Sep 17 00:00:00 2001 From: Christian Schlatter Date: Fri, 6 Aug 2021 12:04:10 +0200 Subject: [PATCH 12/22] add volume stats capability --- deploy/k8s/node-daemonset.yaml | 137 ++++++++++++++++++++------------- pkg/driver/node.go | 53 +++++++++++++ pkg/mount/fake.go | 8 ++ pkg/mount/mount.go | 63 +++++++++++++++ 4 files changed, 206 insertions(+), 55 deletions(-) diff --git a/deploy/k8s/node-daemonset.yaml b/deploy/k8s/node-daemonset.yaml index d65231f..4cb9c3e 100644 --- a/deploy/k8s/node-daemonset.yaml +++ b/deploy/k8s/node-daemonset.yaml @@ -2,101 +2,128 @@ apiVersion: apps/v1 kind: DaemonSet metadata: name: cloudstack-csi-node - namespace: kube-system spec: + revisionHistoryLimit: 10 selector: matchLabels: app.kubernetes.io/name: cloudstack-csi-node - updateStrategy: - type: RollingUpdate template: metadata: labels: app.kubernetes.io/name: cloudstack-csi-node app.kubernetes.io/part-of: cloudstack-csi-driver spec: - nodeSelector: - kubernetes.io/os: linux - tolerations: - - effect: NoExecute - operator: Exists - - effect: NoSchedule - operator: Exists - containers: - - name: cloudstack-csi-node - image: cloudstack-csi-driver - imagePullPolicy: Always - args: - - "-endpoint=$(CSI_ENDPOINT)" - - "-cloudstackconfig=/etc/cloudstack-csi-driver/cloudstack.ini" - - "-nodeName=$(NODE_NAME)" - - "-debug" + - args: + - -endpoint=$(CSI_ENDPOINT) + - -cloudstackconfig=/etc/cloudstack-csi-driver/cloudstack.ini + - -nodeName=$(NODE_NAME) + - -debug env: - name: CSI_ENDPOINT value: unix:///csi/csi.sock - name: NODE_NAME valueFrom: fieldRef: + apiVersion: v1 fieldPath: spec.nodeName + image: registry.puzzle.ch/cschlatter/cloudstack-csi-driver:v1.0.0 + imagePullPolicy: Always + name: cloudstack-csi-node + resources: {} securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - all privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: kubelet-dir - mountPath: /var/lib/kubelet - # needed so that any mounts setup inside this container are - # propagated back to the host machine. + - mountPath: /csi + name: plugin-dir + - mountPath: /var/lib/kubelet mountPropagation: Bidirectional - - name: device-dir - mountPath: /dev - - name: cloud-init-dir - mountPath: /run/cloud-init/ - - name: cloudstack-conf - mountPath: /etc/cloudstack-csi-driver - - - name: node-driver-registrar - image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1 - imagePullPolicy: IfNotPresent - args: - - "--csi-address=$(ADDRESS)" - - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" - - "--v=5" + name: kubelet-dir + - mountPath: /dev + name: device-dir + - mountPath: /run/cloud-init/ + name: cloud-init-dir + - mountPath: /etc/cloudstack-csi-driver + name: cloudstack-conf + - mountPath: /sys/class/scsi_host/ + name: scsi-host-dir + - mountPath: /sys/devices/ + name: scsi-device-dir + - args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=5 env: - name: ADDRESS value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/csi.cloudstack.apache.org/csi.sock + image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1 + imagePullPolicy: IfNotPresent + name: node-driver-registrar + resources: {} securityContext: privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration - + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + dnsPolicy: ClusterFirst + nodeSelector: + kubernetes.io/os: linux + restartPolicy: Always + schedulerName: default-scheduler + serviceAccount: cloudstack-csi-node + serviceAccountName: cloudstack-csi-node + terminationGracePeriodSeconds: 30 + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists volumes: - - name: plugin-dir - hostPath: + - hostPath: path: /var/lib/kubelet/plugins/csi.cloudstack.apache.org/ type: DirectoryOrCreate - - name: kubelet-dir - hostPath: + name: plugin-dir + - hostPath: path: /var/lib/kubelet type: Directory - - name: device-dir - hostPath: + name: kubelet-dir + - hostPath: path: /dev type: Directory - - name: registration-dir - hostPath: + name: device-dir + - hostPath: path: /var/lib/kubelet/plugins_registry type: Directory - - name: cloud-init-dir - hostPath: + name: registration-dir + - hostPath: path: /run/cloud-init/ type: Directory + name: cloud-init-dir + - hostPath: + path: /sys/class/scsi_host/ + type: Directory + name: scsi-host-dir + - hostPath: + path: /sys/devices/ + type: "" + name: scsi-device-dir - name: cloudstack-conf secret: - secretName: cloudstack-secret + defaultMode: 420 + secretName: csi-cloudstack-secret + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate diff --git a/pkg/driver/node.go b/pkg/driver/node.go index cf0a0fc..3002ca3 100644 --- a/pkg/driver/node.go +++ b/pkg/driver/node.go @@ -353,6 +353,54 @@ func (ns *nodeServer) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoReque }, nil } +func (ns *nodeServer) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { + if req.GetVolumeId() == "" { + return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") + } + volumeID := req.GetVolumeId() + + volumePath := req.VolumePath + if volumePath == "" { + return nil, status.Error(codes.InvalidArgument, "NodeGetVolumeStats Volume Path must be provided") + } + + ctxzap.Extract(ctx).Sugar().Debugf("NodeGetVolumeStats: for volume %s", volumeID) + _, err := ns.connector.GetVolumeByID(ctx, volumeID) + if err == cloud.ErrNotFound { + return nil, status.Errorf(codes.NotFound, "Volume %v not found", volumeID) + } else if err != nil { + // Error with CloudStack + return nil, status.Errorf(codes.Internal, "Error %v", err) + } + + _, err = ns.mounter.IsBlockDevice(volumePath) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to determine if %q is block device: %s", volumePath, err) + } + + stats, err := ns.mounter.GetStatistics(volumePath) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to retrieve capacity statistics for volume path %q: %s", volumePath, err) + } + + return &csi.NodeGetVolumeStatsResponse{ + Usage: []*csi.VolumeUsage{ + &csi.VolumeUsage{ + Available: stats.AvailableBytes, + Total: stats.TotalBytes, + Used: stats.UsedBytes, + Unit: csi.VolumeUsage_BYTES, + }, + &csi.VolumeUsage{ + Available: stats.AvailableInodes, + Total: stats.TotalInodes, + Used: stats.UsedInodes, + Unit: csi.VolumeUsage_INODES, + }, + }, + }, nil +} + func (ns *nodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { return &csi.NodeGetCapabilitiesResponse{ Capabilities: []*csi.NodeServiceCapability{ @@ -363,6 +411,11 @@ func (ns *nodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetC }, }, }, + { + Type: &csi.NodeServiceCapability_Rpc{ + Rpc: &csi.NodeServiceCapability_RPC{ + Type: csi.NodeServiceCapability_RPC_GET_VOLUME_STATS}}, + }, }, }, nil } diff --git a/pkg/mount/fake.go b/pkg/mount/fake.go index 4c75dbc..2e9dd69 100644 --- a/pkg/mount/fake.go +++ b/pkg/mount/fake.go @@ -14,6 +14,14 @@ type fakeMounter struct { utilsexec.Interface } +func (m *fakeMounter) GetStatistics(volumePath string) (volumeStatistics, error) { + return volumeStatistics{}, nil +} + +func (m *fakeMounter) IsBlockDevice(devicePath string) (bool, error) { + return true, nil +} + // NewFake creates an fake implementation of the // mount.Interface, to be used in tests. func NewFake() Interface { diff --git a/pkg/mount/mount.go b/pkg/mount/mount.go index ffaee54..ee83872 100644 --- a/pkg/mount/mount.go +++ b/pkg/mount/mount.go @@ -6,6 +6,7 @@ import ( "context" "fmt" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "golang.org/x/sys/unix" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/mount-utils" "k8s.io/utils/exec" @@ -30,6 +31,9 @@ type Interface interface { CleanScsi(ctx context.Context, deviceID, hypervisor string) + GetStatistics(volumePath string) (volumeStatistics, error) + IsBlockDevice(devicePath string) (bool, error) + GetDevicePath(ctx context.Context, volumeID string, hypervisor string) (string, error) GetDeviceName(mountPath string) (string, int, error) ExistsPath(filename string) (bool, error) @@ -42,6 +46,11 @@ type mounter struct { exec.Interface } +type volumeStatistics struct { + AvailableBytes, TotalBytes, UsedBytes int64 + AvailableInodes, TotalInodes, UsedInodes int64 +} + // New creates an implementation of the mount.Interface. func New() Interface { return &mounter{ @@ -199,3 +208,57 @@ func (*mounter) MakeFile(pathname string) error { } return nil } + +//Copy Pasta from https://github.com/digitalocean/csi-digitalocean/blob/db266f4044178a96c5aa9e2420efae8723af75f4/driver/mounter.go +func (m *mounter) GetStatistics(volumePath string) (volumeStatistics, error) { + isBlock, err := m.IsBlockDevice(volumePath) + if err != nil { + return volumeStatistics{}, fmt.Errorf("failed to determine if volume %s is block device: %v", volumePath, err) + } + + if isBlock { + // See http://man7.org/linux/man-pages/man8/blockdev.8.html for details + output, err := m.Exec.Command("blockdev", "getsize64", volumePath).CombinedOutput() + if err != nil { + return volumeStatistics{}, fmt.Errorf("error when getting size of block volume at path %s: output: %s, err: %v", volumePath, string(output), err) + } + strOut := strings.TrimSpace(string(output)) + gotSizeBytes, err := strconv.ParseInt(strOut, 10, 64) + if err != nil { + return volumeStatistics{}, fmt.Errorf("failed to parse size %s into int", strOut) + } + + return volumeStatistics{ + TotalBytes: gotSizeBytes, + }, nil + } + + var statfs unix.Statfs_t + // See http://man7.org/linux/man-pages/man2/statfs.2.html for details. + err = unix.Statfs(volumePath, &statfs) + if err != nil { + return volumeStatistics{}, err + } + + volStats := volumeStatistics{ + AvailableBytes: int64(statfs.Bavail) * int64(statfs.Bsize), + TotalBytes: int64(statfs.Blocks) * int64(statfs.Bsize), + UsedBytes: (int64(statfs.Blocks) - int64(statfs.Bfree)) * int64(statfs.Bsize), + + AvailableInodes: int64(statfs.Ffree), + TotalInodes: int64(statfs.Files), + UsedInodes: int64(statfs.Files) - int64(statfs.Ffree), + } + + return volStats, nil +} + +func (m *mounter) IsBlockDevice(devicePath string) (bool, error) { + var stat unix.Stat_t + err := unix.Stat(devicePath, &stat) + if err != nil { + return false, err + } + + return (stat.Mode & unix.S_IFMT) == unix.S_IFBLK, nil +} From f57d5b34175f26b7b4527e46aef0550f3f0638fe Mon Sep 17 00:00:00 2001 From: Christian Schlatter Date: Fri, 6 Aug 2021 14:56:53 +0200 Subject: [PATCH 13/22] add env. variable for hypervisor --- go.mod | 1 + pkg/driver/controller.go | 1 - pkg/driver/node.go | 32 ++++++++++++++++++++++---------- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 1664885..ea8ba8c 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/kubernetes-csi/csi-test/v4 v4.1.0 github.com/xanzy/go-cloudstack/v2 v2.9.0 go.uber.org/zap v1.16.0 + golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d golang.org/x/text v0.3.5 google.golang.org/grpc v1.36.0 gopkg.in/gcfg.v1 v1.2.3 diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index f911516..5baae63 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -217,7 +217,6 @@ func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { // Check arguments - if req.GetVolumeId() == "" { return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") } diff --git a/pkg/driver/node.go b/pkg/driver/node.go index 3002ca3..3a15f97 100644 --- a/pkg/driver/node.go +++ b/pkg/driver/node.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/apalia/cloudstack-csi-driver/pkg/cloud" "github.com/apalia/cloudstack-csi-driver/pkg/mount" @@ -22,20 +23,31 @@ const ( type nodeServer struct { csi.UnimplementedNodeServer - connector cloud.Interface - mounter mount.Interface - nodeName string + connector cloud.Interface + mounter mount.Interface + nodeName string + hypervisor string } // NewNodeServer creates a new Node gRPC server. func NewNodeServer(connector cloud.Interface, mounter mount.Interface, nodeName string) csi.NodeServer { + hypervisor, ok := os.LookupEnv("NODE_HYPERVISOR") + if !ok { + panic("Environment variable NODE_HYPERVISOR must be set") + } + + if strings.ToLower(hypervisor) != "vmware" && strings.ToLower(hypervisor) == "kvm" { + panic("Environment variable NODE_HYPERVISOR must be 'vmware' or 'kvm'") + } + if mounter == nil { mounter = mount.New() } return &nodeServer{ - connector: connector, - mounter: mounter, - nodeName: nodeName, + connector: connector, + mounter: mounter, + nodeName: nodeName, + hypervisor: hypervisor, } } @@ -68,7 +80,7 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol deviceID := req.PublishContext[deviceIDContextKey] - devicePath, err := ns.mounter.GetDevicePath(ctx, v.DeviceID, v.Hypervisor) + devicePath, err := ns.mounter.GetDevicePath(ctx, v.DeviceID, ns.hypervisor) if err != nil { return nil, status.Errorf(codes.Internal, "Cannot find device path for volume %s: %s", volumeID, err.Error()) } @@ -178,7 +190,7 @@ func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag return nil, status.Errorf(codes.Internal, "Could not find volume %s: %v", volumeID, err) } - ns.mounter.CleanScsi(ctx, v.DeviceID, v.Hypervisor) + ns.mounter.CleanScsi(ctx, v.DeviceID, ns.hypervisor) return &csi.NodeUnstageVolumeResponse{}, nil } @@ -260,7 +272,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis if req.GetVolumeCapability().GetBlock() != nil { volumeID := req.GetVolumeId() - devicePath, err := ns.mounter.GetDevicePath(ctx, v.DeviceID, v.Hypervisor) + devicePath, err := ns.mounter.GetDevicePath(ctx, v.DeviceID, ns.hypervisor) if err != nil { return nil, status.Errorf(codes.Internal, "Cannot find device path for volume %s: %s", volumeID, err.Error()) } @@ -324,7 +336,7 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu } ctxzap.Extract(ctx).Sugar().Debugf("NodeUnpublishVolume: successfully unpublish volume %s on node %s", volumeID, targetPath) - ns.mounter.CleanScsi(ctx, v.DeviceID, v.Hypervisor) + ns.mounter.CleanScsi(ctx, v.DeviceID, ns.hypervisor) return &csi.NodeUnpublishVolumeResponse{}, nil } From 2b1420e2f98ff29a71278a99e7cd8b3f49b99d91 Mon Sep 17 00:00:00 2001 From: Christian Schlatter Date: Fri, 6 Aug 2021 15:08:51 +0200 Subject: [PATCH 14/22] update README.md --- README.md | 6 ++++++ deploy/k8s/node-daemonset.yaml | 2 ++ 2 files changed, 8 insertions(+) diff --git a/README.md b/README.md index 13a9625..33c67c9 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,12 @@ kubectl create secret generic \ cloudstack-secret ``` +Set the correct hypervisor in the DaemonSet Env Vars: +``` + - name: NODE_HYPERVISOR + value: vmware +``` + If you have also deployed the [CloudStack Kubernetes Provider](https://github.com/apache/cloudstack-kubernetes-provider), you may use the same secret for both tools. diff --git a/deploy/k8s/node-daemonset.yaml b/deploy/k8s/node-daemonset.yaml index 4cb9c3e..63d1f00 100644 --- a/deploy/k8s/node-daemonset.yaml +++ b/deploy/k8s/node-daemonset.yaml @@ -22,6 +22,8 @@ spec: env: - name: CSI_ENDPOINT value: unix:///csi/csi.sock + - name: NODE_HYPERVISOR + value: vmware - name: NODE_NAME valueFrom: fieldRef: From 812f683013a51628111c9a1ab66bf9f7d9ad6d64 Mon Sep 17 00:00:00 2001 From: Christian Schlatter Date: Mon, 9 Aug 2021 08:46:27 +0200 Subject: [PATCH 15/22] update github actions to swisstxt values --- .github/workflows/release.yaml | 18 +++---- .idea/cloudstack-csi.iml | 8 +++ .idea/modules.xml | 8 +++ .idea/vcs.xml | 6 +++ .idea/workspace.xml | 99 ++++++++++++++++++++++++++++++++++ 5 files changed, 130 insertions(+), 9 deletions(-) create mode 100644 .idea/cloudstack-csi.iml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml create mode 100644 .idea/workspace.xml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 7577929..1feef4c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -34,16 +34,16 @@ jobs: - name: Log into registry uses: docker/login-action@v1 with: - registry: quay.io - username: apalia+github_cloudstack_csi_driver - password: ${{ secrets.QUAY_TOKEN }} + registry: ${{ secrets.REGISTRY_CARGO_SYSTEM_URL }} + username: ${{ secrets.REGISTRY_CARGO_SYSTEM_USERNAME }} + password: ${{ secrets.REGISTRY_CARGO_SYSTEM_PASSWORD }} - name: Push master if: github.ref == 'refs/heads/master' run: | for img in $IMAGES; do - docker tag ${img} ${REGISTRY_NAME}/${img}:master - docker push ${REGISTRY_NAME}/${img}:master + docker tag ${img} ${{ secrets.REGISTRY_CARGO_URL}}/${img}:master + docker push ${{ secrets.REGISTRY_CARGO_URL}}/${img}:master done - name: Push tagged release @@ -53,8 +53,8 @@ jobs: VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,' | sed -e 's/^v//') for img in $IMAGES; do - docker tag ${img} ${REGISTRY_NAME}/${img}:${VERSION} - docker push ${REGISTRY_NAME}/${img}:${VERSION} + docker tag ${img} ${{ secrets.REGISTRY_CARGO_URL}}/${img}:${VERSION} + docker push ${{ secrets.REGISTRY_CARGO_URL}}/${img}:${VERSION} done - name: Upload cloudstack-csi-sc-syncer artifact @@ -87,9 +87,9 @@ jobs: echo "---" >> manifest.yaml cat deploy/k8s/csidriver.yaml >> manifest.yaml echo "---" >> manifest.yaml - sed -E "s|image: +cloudstack-csi-driver|image: ${REGISTRY_NAME}/cloudstack-csi-driver:${VERSION}|" deploy/k8s/controller-deployment.yaml >> manifest.yaml + sed -E "s|image: +cloudstack-csi-driver|image: ${{ secrets.REGISTRY_CARGO_URL}}/cloudstack-csi-driver:${VERSION}|" deploy/k8s/controller-deployment.yaml >> manifest.yaml echo "---" >> manifest.yaml - sed -E "s|image: +cloudstack-csi-driver|image: ${REGISTRY_NAME}/cloudstack-csi-driver:${VERSION}|" deploy/k8s/node-daemonset.yaml >> manifest.yaml + sed -E "s|image: +cloudstack-csi-driver|image: ${{ secrets.REGISTRY_CARGO_URL}}/cloudstack-csi-driver:${VERSION}|" deploy/k8s/node-daemonset.yaml >> manifest.yaml - name: Create Release id: create_release diff --git a/.idea/cloudstack-csi.iml b/.idea/cloudstack-csi.iml new file mode 100644 index 0000000..c956989 --- /dev/null +++ b/.idea/cloudstack-csi.iml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..a40cac5 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..f00f7d5 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,99 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 237862e4a120660fdee70304a85b41735b6ba50f Mon Sep 17 00:00:00 2001 From: Christian Schlatter Date: Mon, 9 Aug 2021 09:47:36 +0200 Subject: [PATCH 16/22] remove .idea folder --- .gitignore | 1 + .idea/cloudstack-csi.iml | 8 ---- .idea/modules.xml | 8 ---- .idea/vcs.xml | 6 --- .idea/workspace.xml | 99 ---------------------------------------- 5 files changed, 1 insertion(+), 121 deletions(-) delete mode 100644 .idea/cloudstack-csi.iml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/vcs.xml delete mode 100644 .idea/workspace.xml diff --git a/.gitignore b/.gitignore index 68ff0d9..3b67dd7 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /test/e2e/e2e.test /test/e2e/ginkgo /cloudstack.ini +/.idea \ No newline at end of file diff --git a/.idea/cloudstack-csi.iml b/.idea/cloudstack-csi.iml deleted file mode 100644 index c956989..0000000 --- a/.idea/cloudstack-csi.iml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index a40cac5..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml deleted file mode 100644 index f00f7d5..0000000 --- a/.idea/workspace.xml +++ /dev/null @@ -1,99 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From 44db10a3a43c2209bd6dd338af310c491d95a661 Mon Sep 17 00:00:00 2001 From: Christian Schlatter Date: Mon, 9 Aug 2021 14:56:11 +0200 Subject: [PATCH 17/22] update readme for reusing volumes --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 33c67c9..92b0b90 100644 --- a/README.md +++ b/README.md @@ -95,11 +95,19 @@ disk offerings to Kubernetes storage classes. Example: -``` +```bash kubectl apply -f ./examples/k8s/pvc.yaml kubectl apply -f ./examples/k8s/pod.yaml ``` +#### Reusing volumes + +1. Patch PV `reclaimPolicy` with `kubectl patch pv my-pv-name -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}'` +2. Delete Old Pod and PVC +3. Patch PV `claimRef` with `kubectl patch pv my-pv-name -p '{"spec":{"claimRef": null}}'` +4. Create new Pod and PVC with existing claimName `.spec.claimRef.name = my-pv-name` + + ## Building To build the driver binary: From d78034a5fd26630ce2a9ef72f924082229a0cf82 Mon Sep 17 00:00:00 2001 From: Christian Schlatter Date: Sat, 14 Aug 2021 02:15:36 +0200 Subject: [PATCH 18/22] make max volumes per node configurable --- pkg/driver/controller.go | 2 +- pkg/driver/node.go | 41 +++++++++++++++++++++++++++++++--------- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index 5baae63..720c55c 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -285,7 +285,7 @@ func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *cs } //Check max volumes - if len(vols) >= maxAllowedBlockVolumesPerNode { + if len(vols) >= getMaxAllowedVolumes() { return nil, status.Errorf(codes.ResourceExhausted, "Maximum allowed volumes (%d/%d) per node reached. Could not attach volume %s", len(vols), maxAllowedBlockVolumesPerNode, volumeID) } diff --git a/pkg/driver/node.go b/pkg/driver/node.go index 3a15f97..77395aa 100644 --- a/pkg/driver/node.go +++ b/pkg/driver/node.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strconv" "strings" "github.com/apalia/cloudstack-csi-driver/pkg/cloud" @@ -18,15 +19,16 @@ import ( const ( // default file system type to be used when it is not provided defaultFsType = "ext4" - maxAllowedBlockVolumesPerNode = 4 + maxAllowedBlockVolumesPerNode = 10 ) type nodeServer struct { csi.UnimplementedNodeServer - connector cloud.Interface - mounter mount.Interface - nodeName string - hypervisor string + connector cloud.Interface + mounter mount.Interface + nodeName string + hypervisor string + maxAllowedBlockVolumesPerNode int } // NewNodeServer creates a new Node gRPC server. @@ -40,14 +42,23 @@ func NewNodeServer(connector cloud.Interface, mounter mount.Interface, nodeName panic("Environment variable NODE_HYPERVISOR must be 'vmware' or 'kvm'") } + maxVolumesStr, ok := os.LookupEnv("NODE_MAX_BLOCK_VOLUMES") + if ok { + _, err := strconv.Atoi(maxVolumesStr) + if err != nil { + panic("Environment variable NODE_MAX_BLOCK_VOLUMES must be of type integer: " + err.Error()) + } + } + if mounter == nil { mounter = mount.New() } return &nodeServer{ - connector: connector, - mounter: mounter, - nodeName: nodeName, - hypervisor: hypervisor, + connector: connector, + mounter: mounter, + nodeName: nodeName, + hypervisor: hypervisor, + maxAllowedBlockVolumesPerNode: getMaxAllowedVolumes(), } } @@ -431,3 +442,15 @@ func (ns *nodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetC }, }, nil } + +func getMaxAllowedVolumes() int { + maxVolumes := maxAllowedBlockVolumesPerNode + maxVolumesStr, ok := os.LookupEnv("NODE_MAX_BLOCK_VOLUMES") + if ok { + max, err := strconv.Atoi(maxVolumesStr) + if err != nil { + maxVolumes = max + } + } + return maxVolumes +} From 3a22f7efc1c4e67abefa3e4859fdfd0a9ce2c945 Mon Sep 17 00:00:00 2001 From: Christian Schlatter Date: Thu, 19 Aug 2021 09:05:25 +0200 Subject: [PATCH 19/22] update readme and daemonset, get max vol per node from env. var. if set --- README.md | 6 ++++ deploy/k8s/node-daemonset.yaml | 54 ++++++++++++++++++---------------- pkg/driver/node.go | 2 +- 3 files changed, 36 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 92b0b90..b30a4a0 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,12 @@ Set the correct hypervisor in the DaemonSet Env Vars: value: vmware ``` +You can manually set the maximal attachable number of block volumes per node: +``` + - name: NODE_MAX_BLOCK_VOLUMES + value: "15" #Default value is 10 volumes per node +``` + If you have also deployed the [CloudStack Kubernetes Provider](https://github.com/apache/cloudstack-kubernetes-provider), you may use the same secret for both tools. diff --git a/deploy/k8s/node-daemonset.yaml b/deploy/k8s/node-daemonset.yaml index 63d1f00..14ec1a2 100644 --- a/deploy/k8s/node-daemonset.yaml +++ b/deploy/k8s/node-daemonset.yaml @@ -24,39 +24,39 @@ spec: value: unix:///csi/csi.sock - name: NODE_HYPERVISOR value: vmware + - name: NODE_MAX_BLOCK_VOLUMES + value: "10" - name: NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - image: registry.puzzle.ch/cschlatter/cloudstack-csi-driver:v1.0.0 + image: registry.swisstxt.ch/stxt-proj-cargo-system/cloudstack-csi-driver:latest imagePullPolicy: Always name: cloudstack-csi-node resources: {} securityContext: - allowPrivilegeEscalation: true - capabilities: - add: - - all privileged: true terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - - mountPath: /csi - name: plugin-dir - - mountPath: /var/lib/kubelet - mountPropagation: Bidirectional - name: kubelet-dir - - mountPath: /dev - name: device-dir - - mountPath: /run/cloud-init/ - name: cloud-init-dir - - mountPath: /etc/cloudstack-csi-driver - name: cloudstack-conf - - mountPath: /sys/class/scsi_host/ - name: scsi-host-dir - - mountPath: /sys/devices/ - name: scsi-device-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /dev + name: device-dir + - mountPath: /run/cloud-init/ + name: cloud-init-dir + - mountPath: /etc/cloudstack-csi-driver + name: cloudstack-conf + - mountPath: /sys/class/scsi_host/ + name: sys-class-scsi-host-dir + - mountPath: /sys/class/scsi_device/ + name: sys-class-scsi-device-dir + - mountPath: /sys/devices + name: sys-devices - args: - --csi-address=$(ADDRESS) - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) @@ -114,13 +114,17 @@ spec: type: Directory name: cloud-init-dir - hostPath: - path: /sys/class/scsi_host/ + path: /sys/class/scsi_host type: Directory - name: scsi-host-dir + name: sys-class-scsi-host-dir - hostPath: - path: /sys/devices/ - type: "" - name: scsi-device-dir + path: /sys/class/scsi_device + type: Directory + name: sys-class-scsi-device-dir + - hostPath: + path: /sys/devices + type: Directory + name: sys-devices - name: cloudstack-conf secret: defaultMode: 420 diff --git a/pkg/driver/node.go b/pkg/driver/node.go index 77395aa..aecb436 100644 --- a/pkg/driver/node.go +++ b/pkg/driver/node.go @@ -372,7 +372,7 @@ func (ns *nodeServer) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoReque return &csi.NodeGetInfoResponse{ NodeId: vm.ID, AccessibleTopology: topology.ToCSI(), - MaxVolumesPerNode: maxAllowedBlockVolumesPerNode, + MaxVolumesPerNode: int64(getMaxAllowedVolumes()), }, nil } From 15d20c041fe1f8f1c806eb4998018801f7ebe0f2 Mon Sep 17 00:00:00 2001 From: Christian Schlatter Date: Tue, 29 Mar 2022 07:52:50 +0200 Subject: [PATCH 20/22] reverts swisstxt specific parts --- .github/workflows/release.yaml | 18 +++++++++--------- cmd/cloudstack-csi-driver/Dockerfile | 2 +- deploy/k8s/node-daemonset.yaml | 2 +- go.mod | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1feef4c..c65373e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -34,16 +34,16 @@ jobs: - name: Log into registry uses: docker/login-action@v1 with: - registry: ${{ secrets.REGISTRY_CARGO_SYSTEM_URL }} - username: ${{ secrets.REGISTRY_CARGO_SYSTEM_USERNAME }} - password: ${{ secrets.REGISTRY_CARGO_SYSTEM_PASSWORD }} + registry: ${{ secrets.quay.io }} + username: ${{ secrets.apalia+github_cloudstack_csi_driver }} + password: ${{ secrets.QUAY_TOKEN }} - name: Push master if: github.ref == 'refs/heads/master' run: | for img in $IMAGES; do - docker tag ${img} ${{ secrets.REGISTRY_CARGO_URL}}/${img}:master - docker push ${{ secrets.REGISTRY_CARGO_URL}}/${img}:master + docker tag ${img} ${{ secrets.REGISTRY_NAME}}/${img}:master + docker push ${{ secrets.REGISTRY_NAME}}/${img}:master done - name: Push tagged release @@ -53,8 +53,8 @@ jobs: VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,' | sed -e 's/^v//') for img in $IMAGES; do - docker tag ${img} ${{ secrets.REGISTRY_CARGO_URL}}/${img}:${VERSION} - docker push ${{ secrets.REGISTRY_CARGO_URL}}/${img}:${VERSION} + docker tag ${img} ${{ secrets.REGISTRY_NAME}}/${img}:${VERSION} + docker push ${{ secrets.REGISTRY_NAME}}/${img}:${VERSION} done - name: Upload cloudstack-csi-sc-syncer artifact @@ -87,9 +87,9 @@ jobs: echo "---" >> manifest.yaml cat deploy/k8s/csidriver.yaml >> manifest.yaml echo "---" >> manifest.yaml - sed -E "s|image: +cloudstack-csi-driver|image: ${{ secrets.REGISTRY_CARGO_URL}}/cloudstack-csi-driver:${VERSION}|" deploy/k8s/controller-deployment.yaml >> manifest.yaml + sed -E "s|image: +cloudstack-csi-driver|image: ${{ secrets.REGISTRY_NAME}}/cloudstack-csi-driver:${VERSION}|" deploy/k8s/controller-deployment.yaml >> manifest.yaml echo "---" >> manifest.yaml - sed -E "s|image: +cloudstack-csi-driver|image: ${{ secrets.REGISTRY_CARGO_URL}}/cloudstack-csi-driver:${VERSION}|" deploy/k8s/node-daemonset.yaml >> manifest.yaml + sed -E "s|image: +cloudstack-csi-driver|image: ${{ secrets.REGISTRY_NAME}}/cloudstack-csi-driver:${VERSION}|" deploy/k8s/node-daemonset.yaml >> manifest.yaml - name: Create Release id: create_release diff --git a/cmd/cloudstack-csi-driver/Dockerfile b/cmd/cloudstack-csi-driver/Dockerfile index de2bd40..4d4f3f9 100644 --- a/cmd/cloudstack-csi-driver/Dockerfile +++ b/cmd/cloudstack-csi-driver/Dockerfile @@ -2,7 +2,7 @@ FROM alpine:3.14.0 LABEL \ org.opencontainers.image.description="CloudStack CSI driver" \ - org.opencontainers.image.source="https://github.com/swisstxt/cloudstack-csi-driver/" + org.opencontainers.image.source="https://github.com/apalia/cloudstack-csi-driver/" RUN apk add --no-cache \ ca-certificates \ diff --git a/deploy/k8s/node-daemonset.yaml b/deploy/k8s/node-daemonset.yaml index 14ec1a2..4027745 100644 --- a/deploy/k8s/node-daemonset.yaml +++ b/deploy/k8s/node-daemonset.yaml @@ -31,7 +31,7 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - image: registry.swisstxt.ch/stxt-proj-cargo-system/cloudstack-csi-driver:latest + image: cloudstack-csi-driver imagePullPolicy: Always name: cloudstack-csi-node resources: {} diff --git a/go.mod b/go.mod index 2a4b667..7163562 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/hashicorp/go-uuid v1.0.2 github.com/kubernetes-csi/csi-test/v4 v4.2.0 go.uber.org/zap v1.16.0 - golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d + golang.org/x/sys v0.0.0-20210510120138-977fb7262007 golang.org/x/text v0.3.6 google.golang.org/genproto v0.0.0-20210726200206-e7812ac95cc0 // indirect google.golang.org/grpc v1.39.0 From 9e19bb9d90ed1861e987ee29a9f683ecbe31e40f Mon Sep 17 00:00:00 2001 From: Christian Schlatter Date: Tue, 29 Mar 2022 07:57:38 +0200 Subject: [PATCH 21/22] reverts swisstxt specific parts --- .github/workflows/release.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c65373e..f25efd2 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -34,16 +34,16 @@ jobs: - name: Log into registry uses: docker/login-action@v1 with: - registry: ${{ secrets.quay.io }} - username: ${{ secrets.apalia+github_cloudstack_csi_driver }} + registry: quay.io + username: apalia+github_cloudstack_csi_driver password: ${{ secrets.QUAY_TOKEN }} - name: Push master if: github.ref == 'refs/heads/master' run: | for img in $IMAGES; do - docker tag ${img} ${{ secrets.REGISTRY_NAME}}/${img}:master - docker push ${{ secrets.REGISTRY_NAME}}/${img}:master + docker tag ${img} ${REGISTRY_NAME}/${img}:master + docker push ${REGISTRY_NAME}/${img}:master done - name: Push tagged release @@ -53,8 +53,8 @@ jobs: VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,' | sed -e 's/^v//') for img in $IMAGES; do - docker tag ${img} ${{ secrets.REGISTRY_NAME}}/${img}:${VERSION} - docker push ${{ secrets.REGISTRY_NAME}}/${img}:${VERSION} + docker tag ${img} ${REGISTRY_NAME}/${img}:${VERSION} + docker push ${REGISTRY_NAME}/${img}:${VERSION} done - name: Upload cloudstack-csi-sc-syncer artifact From 63c8c5301d1946dac13476cd80131f0f70cfaab6 Mon Sep 17 00:00:00 2001 From: Christian Schlatter Date: Tue, 29 Mar 2022 08:00:11 +0200 Subject: [PATCH 22/22] reverts swisstxt specific parts --- .github/workflows/release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f25efd2..7577929 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -87,9 +87,9 @@ jobs: echo "---" >> manifest.yaml cat deploy/k8s/csidriver.yaml >> manifest.yaml echo "---" >> manifest.yaml - sed -E "s|image: +cloudstack-csi-driver|image: ${{ secrets.REGISTRY_NAME}}/cloudstack-csi-driver:${VERSION}|" deploy/k8s/controller-deployment.yaml >> manifest.yaml + sed -E "s|image: +cloudstack-csi-driver|image: ${REGISTRY_NAME}/cloudstack-csi-driver:${VERSION}|" deploy/k8s/controller-deployment.yaml >> manifest.yaml echo "---" >> manifest.yaml - sed -E "s|image: +cloudstack-csi-driver|image: ${{ secrets.REGISTRY_NAME}}/cloudstack-csi-driver:${VERSION}|" deploy/k8s/node-daemonset.yaml >> manifest.yaml + sed -E "s|image: +cloudstack-csi-driver|image: ${REGISTRY_NAME}/cloudstack-csi-driver:${VERSION}|" deploy/k8s/node-daemonset.yaml >> manifest.yaml - name: Create Release id: create_release