From 6225e8a47e666238f876c1083db4cae42c9c4e1b Mon Sep 17 00:00:00 2001 From: Hongmei Parkin Date: Wed, 20 Sep 2023 13:56:26 -0700 Subject: [PATCH 1/6] Added storageclass allowedTopologies support --- cmd/cloudstack-csi-sc-syncer/main.go | 2 ++ pkg/syncer/run.go | 19 +++++++++++++ pkg/syncer/syncer.go | 41 +++++++++++++++++++++------- 3 files changed, 52 insertions(+), 10 deletions(-) diff --git a/cmd/cloudstack-csi-sc-syncer/main.go b/cmd/cloudstack-csi-sc-syncer/main.go index cf3edfc..09d8bf6 100644 --- a/cmd/cloudstack-csi-sc-syncer/main.go +++ b/cmd/cloudstack-csi-sc-syncer/main.go @@ -19,6 +19,7 @@ var ( cloudstackconfig = flag.String("cloudstackconfig", "./cloud-config", "CloudStack configuration file") kubeconfig = flag.String("kubeconfig", path.Join(os.Getenv("HOME"), ".kube/config"), "Kubernetes configuration file. Use \"-\" to use in-cluster configuration.") label = flag.String("label", "app.kubernetes.io/managed-by="+agent, "") + nodeName = flag.String("nodeName", "", "Node name") namePrefix = flag.String("namePrefix", "cloudstack-", "") delete = flag.Bool("delete", false, "Delete") showVersion = flag.Bool("version", false, "Show version") @@ -41,6 +42,7 @@ func main() { CloudStackConfig: *cloudstackconfig, KubeConfig: *kubeconfig, Label: *label, + NodeName: *nodeName, NamePrefix: *namePrefix, Delete: *delete, }) diff --git a/pkg/syncer/run.go b/pkg/syncer/run.go index d8262d6..2fd25ec 100644 --- a/pkg/syncer/run.go +++ b/pkg/syncer/run.go @@ -107,6 +107,14 @@ func (s syncer) syncOffering(ctx context.Context, offering *cloudstack.DiskOffer } log.Printf("Storage class name: %s", name) + var zoneID string + vm, err := s.csConnector.GetNodeInfo(ctx, s.nodeName) + if err != nil { + log.Printf("GetNodeinfo failed: %s", err.Error()) + } else { + zoneID = vm.ZoneID + } + sc, err := s.k8sClient.StorageV1().StorageClasses().Get(ctx, name, metav1.GetOptions{}) if err != nil { if k8serrors.IsNotFound(err) { @@ -127,7 +135,18 @@ func (s syncer) syncOffering(ctx context.Context, offering *cloudstack.DiskOffer Parameters: map[string]string{ driver.DiskOfferingKey: offering.Id, }, + AllowedTopologies: []corev1.TopologySelectorTerm{ + { + MatchLabelExpressions: []corev1.TopologySelectorLabelRequirement{ + { + Key: "topology." + driver.DriverName + "/zone", + Values: []string{zoneID}, + }, + }, + }, + }, } + _, err = s.k8sClient.StorageV1().StorageClasses().Create(ctx, newSc, metav1.CreateOptions{}) return name, err } diff --git a/pkg/syncer/syncer.go b/pkg/syncer/syncer.go index df936e9..b1ecdcb 100644 --- a/pkg/syncer/syncer.go +++ b/pkg/syncer/syncer.go @@ -24,6 +24,7 @@ type Config struct { CloudStackConfig string KubeConfig string Label string + NodeName string NamePrefix string Delete bool } @@ -36,11 +37,13 @@ type Syncer interface { // syncer is Syncer implementation. type syncer struct { - k8sClient *kubernetes.Clientset - csClient *cloudstack.CloudStackClient - labelsSet labels.Set - namePrefix string - delete bool + k8sClient *kubernetes.Clientset + csClient *cloudstack.CloudStackClient + csConnector cloud.Interface + nodeName string + labelsSet labels.Set + namePrefix string + delete bool } func createK8sClient(kubeconfig, agent string) (*kubernetes.Clientset, error) { @@ -70,6 +73,17 @@ func createCloudStackClient(cloudstackconfig string) (*cloudstack.CloudStackClie return client, nil } +func createCSConnector(cloudstackconfig string) (cloud.Interface, error) { + config, err := cloud.ReadConfig(cloudstackconfig) + if err != nil { + return nil, err + } + + csConnector := cloud.New(config) + + return csConnector, nil +} + func createLabelsSet(label string) labels.Set { m := make(map[string]string) if len(label) > 0 { @@ -95,11 +109,18 @@ func New(config Config) (Syncer, error) { return nil, fmt.Errorf("cannot create CloudStack client: %w", err) } + csConnector, err := createCSConnector(config.CloudStackConfig) + if err != nil { + return nil, fmt.Errorf("cannot create CS connector interface: %w", err) + } + return syncer{ - k8sClient: k8sClient, - csClient: csClient, - labelsSet: createLabelsSet(config.Label), - namePrefix: config.NamePrefix, - delete: config.Delete, + k8sClient: k8sClient, + csClient: csClient, + csConnector: csConnector, + nodeName: config.NodeName, + labelsSet: createLabelsSet(config.Label), + namePrefix: config.NamePrefix, + delete: config.Delete, }, nil } From a6a241ecdfdd3e062e8ab3fefcf6c1cdddcd6b1b Mon Sep 17 00:00:00 2001 From: Hongmei Parkin Date: Tue, 26 Sep 2023 08:09:21 -0700 Subject: [PATCH 2/6] Added command line option addAllowedTopologies to make it an optional feature --- cmd/cloudstack-csi-sc-syncer/main.go | 30 ++++++++++--------- pkg/syncer/run.go | 25 +++++++++------- pkg/syncer/syncer.go | 45 +++++++++++++++------------- 3 files changed, 55 insertions(+), 45 deletions(-) diff --git a/cmd/cloudstack-csi-sc-syncer/main.go b/cmd/cloudstack-csi-sc-syncer/main.go index 09d8bf6..197a96d 100644 --- a/cmd/cloudstack-csi-sc-syncer/main.go +++ b/cmd/cloudstack-csi-sc-syncer/main.go @@ -16,13 +16,14 @@ import ( const agent = "cloudstack-csi-sc-syncer" var ( - cloudstackconfig = flag.String("cloudstackconfig", "./cloud-config", "CloudStack configuration file") - kubeconfig = flag.String("kubeconfig", path.Join(os.Getenv("HOME"), ".kube/config"), "Kubernetes configuration file. Use \"-\" to use in-cluster configuration.") - label = flag.String("label", "app.kubernetes.io/managed-by="+agent, "") - nodeName = flag.String("nodeName", "", "Node name") - namePrefix = flag.String("namePrefix", "cloudstack-", "") - delete = flag.Bool("delete", false, "Delete") - showVersion = flag.Bool("version", false, "Show version") + cloudstackconfig = flag.String("cloudstackconfig", "./cloud-config", "CloudStack configuration file") + kubeconfig = flag.String("kubeconfig", path.Join(os.Getenv("HOME"), ".kube/config"), "Kubernetes configuration file. Use \"-\" to use in-cluster configuration.") + label = flag.String("label", "app.kubernetes.io/managed-by="+agent, "") + nodeName = flag.String("nodeName", "", "Node name") + addAllowedTopology = flag.Bool("addAllowedTopology", false, "Add allowed topology to storageclass") + namePrefix = flag.String("namePrefix", "cloudstack-", "") + delete = flag.Bool("delete", false, "Delete") + showVersion = flag.Bool("version", false, "Show version") // Version is set by the build process version = "" @@ -38,13 +39,14 @@ func main() { } s, err := syncer.New(syncer.Config{ - Agent: agent, - CloudStackConfig: *cloudstackconfig, - KubeConfig: *kubeconfig, - Label: *label, - NodeName: *nodeName, - NamePrefix: *namePrefix, - Delete: *delete, + Agent: agent, + CloudStackConfig: *cloudstackconfig, + KubeConfig: *kubeconfig, + Label: *label, + NodeName: *nodeName, + AddAllowedTopology: *addAllowedTopology, + NamePrefix: *namePrefix, + Delete: *delete, }) if err != nil { log.Fatalf("Error: %v", err) diff --git a/pkg/syncer/run.go b/pkg/syncer/run.go index 2fd25ec..8fa6343 100644 --- a/pkg/syncer/run.go +++ b/pkg/syncer/run.go @@ -107,14 +107,6 @@ func (s syncer) syncOffering(ctx context.Context, offering *cloudstack.DiskOffer } log.Printf("Storage class name: %s", name) - var zoneID string - vm, err := s.csConnector.GetNodeInfo(ctx, s.nodeName) - if err != nil { - log.Printf("GetNodeinfo failed: %s", err.Error()) - } else { - zoneID = vm.ZoneID - } - sc, err := s.k8sClient.StorageV1().StorageClasses().Get(ctx, name, metav1.GetOptions{}) if err != nil { if k8serrors.IsNotFound(err) { @@ -135,7 +127,19 @@ func (s syncer) syncOffering(ctx context.Context, offering *cloudstack.DiskOffer Parameters: map[string]string{ driver.DiskOfferingKey: offering.Id, }, - AllowedTopologies: []corev1.TopologySelectorTerm{ + } + + //Add AllowedTopologies if the addAllowedTopology flag is true + if s.addAllowedTopology { + var zoneID string + vm, err := s.csConnector.GetNodeInfo(ctx, s.nodeName) + if err != nil { + log.Printf("GetNodeinfo failed: %s", err.Error()) + } else { + zoneID = vm.ZoneID + } + + allowedtopology := []corev1.TopologySelectorTerm{ { MatchLabelExpressions: []corev1.TopologySelectorLabelRequirement{ { @@ -144,7 +148,8 @@ func (s syncer) syncOffering(ctx context.Context, offering *cloudstack.DiskOffer }, }, }, - }, + } + newSc.AllowedTopologies = allowedtopology } _, err = s.k8sClient.StorageV1().StorageClasses().Create(ctx, newSc, metav1.CreateOptions{}) diff --git a/pkg/syncer/syncer.go b/pkg/syncer/syncer.go index b1ecdcb..1ef3515 100644 --- a/pkg/syncer/syncer.go +++ b/pkg/syncer/syncer.go @@ -20,13 +20,14 @@ import ( // Config holds the syncer tool configuration. type Config struct { - Agent string - CloudStackConfig string - KubeConfig string - Label string - NodeName string - NamePrefix string - Delete bool + Agent string + CloudStackConfig string + KubeConfig string + Label string + NodeName string + AddAllowedTopology bool + NamePrefix string + Delete bool } // Syncer has a function Run which synchronizes CloudStack @@ -37,13 +38,14 @@ type Syncer interface { // syncer is Syncer implementation. type syncer struct { - k8sClient *kubernetes.Clientset - csClient *cloudstack.CloudStackClient - csConnector cloud.Interface - nodeName string - labelsSet labels.Set - namePrefix string - delete bool + k8sClient *kubernetes.Clientset + csClient *cloudstack.CloudStackClient + csConnector cloud.Interface + nodeName string + addAllowedTopology bool + labelsSet labels.Set + namePrefix string + delete bool } func createK8sClient(kubeconfig, agent string) (*kubernetes.Clientset, error) { @@ -115,12 +117,13 @@ func New(config Config) (Syncer, error) { } return syncer{ - k8sClient: k8sClient, - csClient: csClient, - csConnector: csConnector, - nodeName: config.NodeName, - labelsSet: createLabelsSet(config.Label), - namePrefix: config.NamePrefix, - delete: config.Delete, + k8sClient: k8sClient, + csClient: csClient, + csConnector: csConnector, + nodeName: config.NodeName, + addAllowedTopology: config.AddAllowedTopology, + labelsSet: createLabelsSet(config.Label), + namePrefix: config.NamePrefix, + delete: config.Delete, }, nil } From fd12acfde73572514ea3304396488a4e181b7381 Mon Sep 17 00:00:00 2001 From: Hongmei Parkin Date: Tue, 26 Sep 2023 13:55:21 -0700 Subject: [PATCH 3/6] Updated the syncer job to include nodeName --- cmd/cloudstack-csi-sc-syncer/README.md | 218 +++++++++++++------------ 1 file changed, 115 insertions(+), 103 deletions(-) diff --git a/cmd/cloudstack-csi-sc-syncer/README.md b/cmd/cloudstack-csi-sc-syncer/README.md index 3c4cbba..48597a9 100644 --- a/cmd/cloudstack-csi-sc-syncer/README.md +++ b/cmd/cloudstack-csi-sc-syncer/README.md @@ -1,103 +1,115 @@ -# cloudstack-csi-sc-syncer - -`cloudstack-csi-sc-syncer` connects to CloudStack (using the same CloudStack -configuration file as `cloudstack-csi-driver`), lists all disk offerings -suitable for usage in Kubernetes (currently: checks they have a custom size), -and creates corresponding Storage Classes in Kubernetes if needed. - -It also adds a label to the Storage Classes it creates. - -If option `-delete=true` is passed, it may also delete Kubernetes Storage -Classes, when they have its label and their corresponding CloudStack disk -offering has been deleted. - -## Usage - -You may use it locally or as a Kubernetes Job. - -### Locally - -You must have a CloudStack configuration file and a Kubernetes `kubeconfig` -file. - -1. Download `cloudstack-csi-sc-syncer` from [latest release](https://github.com/apalia/cloudstack-csi-driver/releases/latest/); - -1. Set the execution permission: - - ``` - chmod +x ./cloudstack-csi-sc-syncer - ``` - -1. Then simply execute the tool: - - ``` - ./cloudstack-csi-sc-syncer - ``` - -Run `./cloudstack-csi-sc-syncer -h` to get the complete list of options and their default values. - -### As a Kubernetes Job - -You may run `cloudstack-csi-sc-syncer` as a Kubernetes Job. In that case, it -re-uses the CloudStack configuration file in Secret `cloudstack-secret`, and use -in-cluster Kubernetes authentification, using a ServiceAccount. - -```sh -export version=... - -kubectl apply -f - < + ``` + +Run `./cloudstack-csi-sc-syncer -h` to get the complete list of options and their default values. + +### As a Kubernetes Job + +You may run `cloudstack-csi-sc-syncer` as a Kubernetes Job. In that case, it +re-uses the CloudStack configuration file in Secret `cloudstack-secret`, and use +in-cluster Kubernetes authentification, using a ServiceAccount. + +```sh +export version=... + +kubectl apply -f - < Date: Tue, 26 Sep 2023 14:10:34 -0700 Subject: [PATCH 4/6] Revert "Updated the syncer job to include nodeName" This reverts commit fd12acfde73572514ea3304396488a4e181b7381. --- cmd/cloudstack-csi-sc-syncer/README.md | 218 ++++++++++++------------- 1 file changed, 103 insertions(+), 115 deletions(-) diff --git a/cmd/cloudstack-csi-sc-syncer/README.md b/cmd/cloudstack-csi-sc-syncer/README.md index 48597a9..3c4cbba 100644 --- a/cmd/cloudstack-csi-sc-syncer/README.md +++ b/cmd/cloudstack-csi-sc-syncer/README.md @@ -1,115 +1,103 @@ -# cloudstack-csi-sc-syncer - -`cloudstack-csi-sc-syncer` connects to CloudStack (using the same CloudStack -configuration file as `cloudstack-csi-driver`), lists all disk offerings -suitable for usage in Kubernetes (currently: checks they have a custom size), -and creates corresponding Storage Classes in Kubernetes if needed. - -It also adds a label to the Storage Classes it creates. - -If option `-delete=true` is passed, it may also delete Kubernetes Storage -Classes, when they have its label and their corresponding CloudStack disk -offering has been deleted. - -## Usage - -You may use it locally or as a Kubernetes Job. - -### Locally - -You must have a CloudStack configuration file and a Kubernetes `kubeconfig` -file. - -1. Download `cloudstack-csi-sc-syncer` from [latest release](https://github.com/apalia/cloudstack-csi-driver/releases/latest/); - -1. Set the execution permission: - - ``` - chmod +x ./cloudstack-csi-sc-syncer - ``` - -1. Then simply execute the tool: - - ``` - ./cloudstack-csi-sc-syncer - ``` - -Run `./cloudstack-csi-sc-syncer -h` to get the complete list of options and their default values. - -### As a Kubernetes Job - -You may run `cloudstack-csi-sc-syncer` as a Kubernetes Job. In that case, it -re-uses the CloudStack configuration file in Secret `cloudstack-secret`, and use -in-cluster Kubernetes authentification, using a ServiceAccount. - -```sh -export version=... - -kubectl apply -f - < + ``` + +Run `./cloudstack-csi-sc-syncer -h` to get the complete list of options and their default values. + +### As a Kubernetes Job + +You may run `cloudstack-csi-sc-syncer` as a Kubernetes Job. In that case, it +re-uses the CloudStack configuration file in Secret `cloudstack-secret`, and use +in-cluster Kubernetes authentification, using a ServiceAccount. + +```sh +export version=... + +kubectl apply -f - < Date: Mon, 2 Oct 2023 10:58:25 -0700 Subject: [PATCH 5/6] Added NodeName and cloud-init directory mount to syncer --- cmd/cloudstack-csi-sc-syncer/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cmd/cloudstack-csi-sc-syncer/README.md b/cmd/cloudstack-csi-sc-syncer/README.md index 3c4cbba..87b948e 100644 --- a/cmd/cloudstack-csi-sc-syncer/README.md +++ b/cmd/cloudstack-csi-sc-syncer/README.md @@ -89,14 +89,26 @@ spec: args: - "-cloudstackconfig=/etc/cloudstack-csi-driver/cloud-config" - "-kubeconfig=-" + - "-nodeName=$(NODE_NAME)" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName volumeMounts: - name: cloudstack-conf mountPath: /etc/cloudstack-csi-driver + - name: cloud-init-dir + mountPath: /run/cloud-init/ restartPolicy: Never volumes: - name: cloudstack-conf secret: secretName: cloudstack-secret + - name: cloud-init-dir + hostPath: + path: /run/cloud-init/ + type: DirectoryOrCreate E0F ``` From fb7a55a483a12ffc93a84a40daa97a164a6e8867 Mon Sep 17 00:00:00 2001 From: PARKIN Date: Thu, 12 Oct 2023 10:42:32 -0700 Subject: [PATCH 6/6] Add zoneID to storageclass AddallowedTopologies --- cmd/cloudstack-csi-sc-syncer/README.md | 2 +- pkg/syncer/run.go | 77 +++++++++++++++++++------- 2 files changed, 58 insertions(+), 21 deletions(-) diff --git a/cmd/cloudstack-csi-sc-syncer/README.md b/cmd/cloudstack-csi-sc-syncer/README.md index 87b948e..d628467 100644 --- a/cmd/cloudstack-csi-sc-syncer/README.md +++ b/cmd/cloudstack-csi-sc-syncer/README.md @@ -108,7 +108,7 @@ spec: - name: cloud-init-dir hostPath: path: /run/cloud-init/ - type: DirectoryOrCreate + type: Directory E0F ``` diff --git a/pkg/syncer/run.go b/pkg/syncer/run.go index 8fa6343..c1e60b8 100644 --- a/pkg/syncer/run.go +++ b/pkg/syncer/run.go @@ -20,6 +20,7 @@ var ( volBindingMode = storagev1.VolumeBindingWaitForFirstConsumer reclaimPolicy = corev1.PersistentVolumeReclaimDelete allowVolumeExpansion = false + zoneID string ) func (s syncer) Run(ctx context.Context) error { @@ -91,6 +92,22 @@ func (s syncer) Run(ctx context.Context) error { return combinedError(errs) } +func getAllowedTopologies() []corev1.TopologySelectorTerm { + if zoneID != "" { + return []corev1.TopologySelectorTerm{ + { + MatchLabelExpressions: []corev1.TopologySelectorLabelRequirement{ + { + Key: "topology." + driver.DriverName + "/zone", + Values: []string{zoneID}, + }, + }, + }, + } + } + return nil +} + func (s syncer) syncOffering(ctx context.Context, offering *cloudstack.DiskOffering) (string, error) { offeringName := offering.Name custom := offering.Iscustomized @@ -107,6 +124,8 @@ func (s syncer) syncOffering(ctx context.Context, offering *cloudstack.DiskOffer } log.Printf("Storage class name: %s", name) + zoneID = s.getZoneID(ctx) + sc, err := s.k8sClient.StorageV1().StorageClasses().Get(ctx, name, metav1.GetOptions{}) if err != nil { if k8serrors.IsNotFound(err) { @@ -131,25 +150,9 @@ func (s syncer) syncOffering(ctx context.Context, offering *cloudstack.DiskOffer //Add AllowedTopologies if the addAllowedTopology flag is true if s.addAllowedTopology { - var zoneID string - vm, err := s.csConnector.GetNodeInfo(ctx, s.nodeName) - if err != nil { - log.Printf("GetNodeinfo failed: %s", err.Error()) - } else { - zoneID = vm.ZoneID - } - - allowedtopology := []corev1.TopologySelectorTerm{ - { - MatchLabelExpressions: []corev1.TopologySelectorLabelRequirement{ - { - Key: "topology." + driver.DriverName + "/zone", - Values: []string{zoneID}, - }, - }, - }, + if getAllowedTopologies() != nil { + newSc.AllowedTopologies = getAllowedTopologies() } - newSc.AllowedTopologies = allowedtopology } _, err = s.k8sClient.StorageV1().StorageClasses().Create(ctx, newSc, metav1.CreateOptions{}) @@ -160,7 +163,7 @@ func (s syncer) syncOffering(ctx context.Context, offering *cloudstack.DiskOffer // Storage class already exists - err = checkStorageClass(sc, offering.Id) + err = s.checkStorageClass(sc, offering.Id) if err != nil { // Updates to provisioner, reclaimpolicy, volumeBindingMode and parameters are forbidden log.Printf("Storage class %s exists but it not compatible.", name) @@ -183,7 +186,32 @@ func (s syncer) syncOffering(ctx context.Context, offering *cloudstack.DiskOffer return name, nil } -func checkStorageClass(sc *storagev1.StorageClass, expectedOfferingID string) error { +func getExistingZoneID(terms []corev1.TopologySelectorTerm) string { + prefix := "topology." + driver.DriverName + "/zone" + for _, term := range terms { + for _, exp := range term.MatchLabelExpressions { + if exp.Key == prefix { + if len(exp.Values) > 0 { + return exp.Values[0] + } + } + } + } + return "" +} + +// get ZoneID of the node where syncer is running +func (s syncer) getZoneID(ctx context.Context) string { + vm, err := s.csConnector.GetNodeInfo(ctx, s.nodeName) + if err != nil { + log.Printf("GetNodeinfo failed: %s", err.Error()) + } else { + return vm.ZoneID + } + return "" +} + +func (s syncer) checkStorageClass(sc *storagev1.StorageClass, expectedOfferingID string) error { errs := make([]error, 0) diskOfferingID, ok := sc.Parameters[driver.DiskOfferingKey] if !ok { @@ -201,6 +229,15 @@ func checkStorageClass(sc *storagev1.StorageClass, expectedOfferingID string) er if sc.AllowVolumeExpansion == nil || *sc.AllowVolumeExpansion != allowVolumeExpansion { errs = append(errs, errors.New("wrong AllowVolumeExpansion")) } + if s.addAllowedTopology { + if sc.AllowedTopologies == nil { + errs = append(errs, errors.New("allowedtopology flag is true but missing allowedtopologies")) + } else if sc.AllowedTopologies != nil { + if zoneID != getExistingZoneID(sc.AllowedTopologies) { + errs = append(errs, errors.New("allowedtopology flag is true but zoneID is not the same with desired zoneID: "+zoneID)) + } + } + } if len(errs) > 0 { return combinedError(errs)