diff --git a/cmd/directpv/main.go b/cmd/directpv/main.go index 60902ad2..a1289ef8 100644 --- a/cmd/directpv/main.go +++ b/cmd/directpv/main.go @@ -25,10 +25,10 @@ import ( "syscall" "time" + "github.com/minio/directpv/pkg/admin/installer" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/consts" - "github.com/minio/directpv/pkg/installer" "github.com/spf13/cobra" "github.com/spf13/viper" "k8s.io/klog/v2" diff --git a/cmd/kubectl-directpv/clean.go b/cmd/kubectl-directpv/clean.go index c94d4e5e..889e1fea 100644 --- a/cmd/kubectl-directpv/clean.go +++ b/cmd/kubectl-directpv/clean.go @@ -131,7 +131,7 @@ func validateCleanCmd() error { } func cleanMain(ctx context.Context) { - if err := admin.Clean(ctx, admin.CleanArgs{ + if err := adminClient.Clean(ctx, admin.CleanArgs{ Nodes: nodesArgs, Drives: drivesArgs, DriveIDs: driveIDArgs, diff --git a/cmd/kubectl-directpv/cordon.go b/cmd/kubectl-directpv/cordon.go index 3d0d95f5..2d9da48a 100644 --- a/cmd/kubectl-directpv/cordon.go +++ b/cmd/kubectl-directpv/cordon.go @@ -111,7 +111,7 @@ func validateCordonCmd() error { } func cordonMain(ctx context.Context) { - if err := admin.Cordon(ctx, admin.CordonArgs{ + if err := adminClient.Cordon(ctx, admin.CordonArgs{ Nodes: nodesArgs, Drives: drivesArgs, Status: driveStatusSelectors, diff --git a/cmd/kubectl-directpv/discover.go b/cmd/kubectl-directpv/discover.go index 86f0991e..5610c07b 100644 --- a/cmd/kubectl-directpv/discover.go +++ b/cmd/kubectl-directpv/discover.go @@ -166,7 +166,7 @@ func writeInitConfig(config admin.InitConfig) error { } func discoverMain(ctx context.Context) { - resultMap, err := admin.DiscoverDevices(ctx, admin.DiscoverArgs{ + resultMap, err := adminClient.DiscoverDevices(ctx, admin.DiscoverArgs{ Nodes: nodesArgs, Drives: drivesArgs, PrintProgress: !quietFlag, diff --git a/cmd/kubectl-directpv/info.go b/cmd/kubectl-directpv/info.go index bdf1bea4..e7838fbf 100644 --- a/cmd/kubectl-directpv/info.go +++ b/cmd/kubectl-directpv/info.go @@ -24,7 +24,6 @@ import ( "github.com/dustin/go-humanize" "github.com/fatih/color" "github.com/jedib0t/go-pretty/v6/table" - "github.com/minio/directpv/pkg/admin" "github.com/minio/directpv/pkg/consts" "github.com/minio/directpv/pkg/utils" "github.com/spf13/cobra" @@ -41,7 +40,7 @@ var infoCmd = &cobra.Command{ } func infoMain(ctx context.Context) { - nodeInfoMap, err := admin.Info(ctx) + nodeInfoMap, err := adminClient.Info(ctx) if err != nil { utils.Eprintf(quietFlag, true, "%v\n", err) os.Exit(1) diff --git a/cmd/kubectl-directpv/init.go b/cmd/kubectl-directpv/init.go index c64fe37c..16087a5d 100644 --- a/cmd/kubectl-directpv/init.go +++ b/cmd/kubectl-directpv/init.go @@ -144,7 +144,7 @@ func initMain(ctx context.Context, inputFile string) { utils.Eprintf(quietFlag, true, "unable to read the input file; %v", err.Error()) os.Exit(1) } - results, err := admin.InitDevices(ctx, admin.InitDevicesArgs{ + results, err := adminClient.InitDevices(ctx, admin.InitDevicesArgs{ InitConfig: initConfig, PrintProgress: !quietFlag, ListTimeout: initRequestListTimeout, diff --git a/cmd/kubectl-directpv/install.go b/cmd/kubectl-directpv/install.go index ab6718ec..a87327b4 100644 --- a/cmd/kubectl-directpv/install.go +++ b/cmd/kubectl-directpv/install.go @@ -25,8 +25,8 @@ import ( "github.com/fatih/color" "github.com/jedib0t/go-pretty/v6/table" "github.com/minio/directpv/pkg/admin" + "github.com/minio/directpv/pkg/admin/installer" "github.com/minio/directpv/pkg/consts" - "github.com/minio/directpv/pkg/installer" "github.com/minio/directpv/pkg/k8s" "github.com/minio/directpv/pkg/utils" "github.com/spf13/cobra" @@ -146,7 +146,7 @@ func installMain(ctx context.Context) { pluginVersion = Version } enableProgress := dryRunPrinter == nil && !declarativeFlag && !quietFlag - installedComponents, err := admin.Install(ctx, admin.InstallArgs{ + installedComponents, err := adminClient.Install(ctx, admin.InstallArgs{ Image: image, Registry: registry, Org: org, diff --git a/cmd/kubectl-directpv/label_drives.go b/cmd/kubectl-directpv/label_drives.go index 7a7b13e0..6e0bc3c6 100644 --- a/cmd/kubectl-directpv/label_drives.go +++ b/cmd/kubectl-directpv/label_drives.go @@ -93,7 +93,7 @@ func init() { } func labelDrivesMain(ctx context.Context) { - if err := admin.LabelDrives(ctx, admin.LabelDriveArgs{ + if err := adminClient.LabelDrives(ctx, admin.LabelDriveArgs{ Nodes: nodesArgs, Drives: drivesArgs, DriveStatus: driveStatusSelectors, diff --git a/cmd/kubectl-directpv/label_volumes.go b/cmd/kubectl-directpv/label_volumes.go index 101a9c60..6dad051e 100644 --- a/cmd/kubectl-directpv/label_volumes.go +++ b/cmd/kubectl-directpv/label_volumes.go @@ -102,7 +102,7 @@ func init() { } func labelVolumesMain(ctx context.Context) { - if err := admin.LabelVolumes(ctx, admin.LabelVolumeArgs{ + if err := adminClient.LabelVolumes(ctx, admin.LabelVolumeArgs{ Nodes: nodesArgs, Drives: drivesArgs, DriveIDs: driveIDArgs, diff --git a/cmd/kubectl-directpv/list_drives.go b/cmd/kubectl-directpv/list_drives.go index e5c305fd..9b8c8fbd 100644 --- a/cmd/kubectl-directpv/list_drives.go +++ b/cmd/kubectl-directpv/list_drives.go @@ -24,7 +24,6 @@ import ( "github.com/jedib0t/go-pretty/v6/table" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" - "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/consts" "github.com/minio/directpv/pkg/types" "github.com/minio/directpv/pkg/utils" @@ -117,7 +116,7 @@ func validateListDrivesArgs() error { } func listDrivesMain(ctx context.Context) { - drives, err := client.NewDriveLister(). + drives, err := adminClient.NewDriveLister(). NodeSelector(utils.ToLabelValues(nodesArgs)). DriveNameSelector(utils.ToLabelValues(drivesArgs)). StatusSelector(driveStatusSelectors). diff --git a/cmd/kubectl-directpv/list_volumes.go b/cmd/kubectl-directpv/list_volumes.go index 83800f86..d613d16b 100644 --- a/cmd/kubectl-directpv/list_volumes.go +++ b/cmd/kubectl-directpv/list_volumes.go @@ -23,7 +23,6 @@ import ( "github.com/jedib0t/go-pretty/v6/table" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" - "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/consts" "github.com/minio/directpv/pkg/k8s" "github.com/minio/directpv/pkg/types" @@ -155,7 +154,7 @@ func getPVCName(ctx context.Context, volume types.Volume) string { } func listVolumesMain(ctx context.Context) { - volumes, err := client.NewVolumeLister(). + volumes, err := adminClient.NewVolumeLister(). NodeSelector(utils.ToLabelValues(nodesArgs)). DriveNameSelector(utils.ToLabelValues(drivesArgs)). DriveIDSelector(utils.ToLabelValues(driveIDArgs)). diff --git a/cmd/kubectl-directpv/main.go b/cmd/kubectl-directpv/main.go index 26a47025..c8d94b68 100644 --- a/cmd/kubectl-directpv/main.go +++ b/cmd/kubectl-directpv/main.go @@ -23,11 +23,13 @@ import ( "os/signal" "syscall" - "github.com/minio/directpv/pkg/client" + "github.com/minio/directpv/pkg/admin" "github.com/minio/directpv/pkg/consts" + "github.com/minio/directpv/pkg/k8s" "github.com/minio/directpv/pkg/utils" "github.com/spf13/cobra" "github.com/spf13/viper" + "k8s.io/client-go/rest" "k8s.io/klog/v2" ) @@ -35,7 +37,10 @@ import ( // e.g. $ go build -ldflags="-X main.Version=v4.0.1" var Version string -var disableInit bool +var ( + disableInit bool + adminClient *admin.Client +) var mainCmd = &cobra.Command{ Use: consts.AppName, @@ -48,7 +53,15 @@ var mainCmd = &cobra.Command{ Version: Version, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { if !disableInit { - client.Init() + kubeConfig, err := k8s.GetKubeConfig() + if err != nil { + klog.Fatalf("unable to get kubernetes configuration; %v", err) + } + kubeConfig.WarningHandler = rest.NoWarnings{} + adminClient, err = admin.NewClient(kubeConfig) + if err != nil { + klog.Fatalf("unable to create admin client; %v", err) + } } return nil }, diff --git a/cmd/kubectl-directpv/migrate.go b/cmd/kubectl-directpv/migrate.go index 113333e7..d50a5d5d 100644 --- a/cmd/kubectl-directpv/migrate.go +++ b/cmd/kubectl-directpv/migrate.go @@ -18,13 +18,12 @@ package main import ( "context" - "fmt" "os" "strings" "time" + "github.com/minio/directpv/pkg/admin" "github.com/minio/directpv/pkg/consts" - "github.com/minio/directpv/pkg/installer" "github.com/minio/directpv/pkg/utils" "github.com/spf13/cobra" ) @@ -53,41 +52,14 @@ func init() { } func migrateMain(ctx context.Context) { - if err := installer.Migrate(ctx, &installer.Args{ - Quiet: quietFlag, - Legacy: true, - }, false); err != nil { - utils.Eprintf(quietFlag, true, "migration failed; %v", err) - os.Exit(1) - } - - if !quietFlag { - fmt.Println("Migration successful; Please restart the pods in '" + consts.AppName + "' namespace.") - } - - if retainFlag { - return - } - suffix := time.Now().Format(time.RFC3339) - - drivesBackupFile := "directcsidrives-" + suffix + ".yaml" - backupCreated, err := installer.RemoveLegacyDrives(ctx, drivesBackupFile) - if err != nil { - utils.Eprintf(quietFlag, true, "unable to remove legacy drive CRDs; %v", err) - os.Exit(1) - } - if backupCreated && !quietFlag { - fmt.Println("Legacy drive CRDs backed up to", drivesBackupFile) - } - - volumesBackupFile := "directcsivolumes-" + suffix + ".yaml" - backupCreated, err = installer.RemoveLegacyVolumes(ctx, volumesBackupFile) - if err != nil { - utils.Eprintf(quietFlag, true, "unable to remove legacy volume CRDs; %v", err) + if err := adminClient.Migrate(ctx, admin.MigrateArgs{ + Quiet: quietFlag, + Retain: retainFlag, + DrivesBackupFile: "directcsidrives-" + suffix + ".yaml", + VolumesBackupFile: "directcsivolumes-" + suffix + ".yaml", + }); err != nil { + utils.Eprintf(quietFlag, true, "migration failed; %v", err) os.Exit(1) } - if backupCreated && !quietFlag { - fmt.Println("Legacy volume CRDs backed up to", volumesBackupFile) - } } diff --git a/cmd/kubectl-directpv/move.go b/cmd/kubectl-directpv/move.go index 63ebda97..fb3d1b63 100644 --- a/cmd/kubectl-directpv/move.go +++ b/cmd/kubectl-directpv/move.go @@ -63,7 +63,7 @@ var moveCmd = &cobra.Command{ } func moveMain(ctx context.Context, src, dest directpvtypes.DriveID) { - if err := admin.Move(ctx, admin.MoveArgs{ + if err := adminClient.Move(ctx, admin.MoveArgs{ Source: src, Destination: dest, Quiet: quietFlag, diff --git a/cmd/kubectl-directpv/remove.go b/cmd/kubectl-directpv/remove.go index 7e046a52..de4c77aa 100644 --- a/cmd/kubectl-directpv/remove.go +++ b/cmd/kubectl-directpv/remove.go @@ -112,7 +112,7 @@ func validateRemoveCmd() error { } func removeMain(ctx context.Context) { - if err := admin.Remove(ctx, admin.RemoveArgs{ + if err := adminClient.Remove(ctx, admin.RemoveArgs{ Nodes: nodesArgs, Drives: drivesArgs, DriveStatus: driveStatusSelectors, diff --git a/cmd/kubectl-directpv/resume_drives.go b/cmd/kubectl-directpv/resume_drives.go index 76921221..b54bd9f6 100644 --- a/cmd/kubectl-directpv/resume_drives.go +++ b/cmd/kubectl-directpv/resume_drives.go @@ -87,7 +87,7 @@ func validateResumeDrivesCmd() error { } func resumeDrivesMain(ctx context.Context) { - if err := admin.ResumeDrives(ctx, admin.ResumeDriveArgs{ + if err := adminClient.ResumeDrives(ctx, admin.ResumeDriveArgs{ Nodes: nodesArgs, Drives: drivesArgs, DriveIDSelectors: driveIDSelectors, diff --git a/cmd/kubectl-directpv/resume_volumes.go b/cmd/kubectl-directpv/resume_volumes.go index 6c712763..c226964b 100644 --- a/cmd/kubectl-directpv/resume_volumes.go +++ b/cmd/kubectl-directpv/resume_volumes.go @@ -97,7 +97,7 @@ func validateResumeVolumesCmd() error { } func resumeVolumesMain(ctx context.Context) { - if err := admin.ResumeVolumes(ctx, admin.ResumeVolumeArgs{ + if err := adminClient.ResumeVolumes(ctx, admin.ResumeVolumeArgs{ Nodes: nodesArgs, Drives: drivesArgs, PodNames: podNameArgs, diff --git a/cmd/kubectl-directpv/suspend_drives.go b/cmd/kubectl-directpv/suspend_drives.go index 5fc26eb4..a1a82697 100644 --- a/cmd/kubectl-directpv/suspend_drives.go +++ b/cmd/kubectl-directpv/suspend_drives.go @@ -94,7 +94,7 @@ func validateSuspendDrivesCmd() error { } func suspendDrivesMain(ctx context.Context) { - if err := admin.SuspendDrives(ctx, admin.SuspendDriveArgs{ + if err := adminClient.SuspendDrives(ctx, admin.SuspendDriveArgs{ Nodes: nodesArgs, Drives: drivesArgs, DriveIDSelectors: driveIDSelectors, diff --git a/cmd/kubectl-directpv/suspend_volumes.go b/cmd/kubectl-directpv/suspend_volumes.go index 0b508ac2..5fbd0e49 100644 --- a/cmd/kubectl-directpv/suspend_volumes.go +++ b/cmd/kubectl-directpv/suspend_volumes.go @@ -104,7 +104,7 @@ func validateSuspendVolumesCmd() error { } func suspendVolumesMain(ctx context.Context) { - if err := admin.SuspendVolumes(ctx, admin.SuspendVolumeArgs{ + if err := adminClient.SuspendVolumes(ctx, admin.SuspendVolumeArgs{ Nodes: nodesArgs, Drives: drivesArgs, PodNames: podNameArgs, diff --git a/cmd/kubectl-directpv/uncordon.go b/cmd/kubectl-directpv/uncordon.go index d24f638b..ef7ec22c 100644 --- a/cmd/kubectl-directpv/uncordon.go +++ b/cmd/kubectl-directpv/uncordon.go @@ -111,7 +111,7 @@ func validateUncordonCmd() error { } func uncordonMain(ctx context.Context) { - if err := admin.Uncordon(ctx, admin.UncordonArgs{ + if err := adminClient.Uncordon(ctx, admin.UncordonArgs{ Nodes: nodesArgs, Drives: drivesArgs, Status: driveStatusSelectors, diff --git a/cmd/kubectl-directpv/uninstall.go b/cmd/kubectl-directpv/uninstall.go index 3ea933df..d02adbcb 100644 --- a/cmd/kubectl-directpv/uninstall.go +++ b/cmd/kubectl-directpv/uninstall.go @@ -45,7 +45,7 @@ func init() { } func uninstallMain(ctx context.Context) { - if err := admin.Uninstall(ctx, admin.UninstallArgs{ + if err := adminClient.Uninstall(ctx, admin.UninstallArgs{ Quiet: quietFlag, Dangerous: dangerousFlag, }); err != nil { diff --git a/codegen.sh b/codegen.sh index c1b3b904..5835a39f 100755 --- a/codegen.sh +++ b/codegen.sh @@ -107,8 +107,8 @@ client-gen \ --input-base "${REPOSITORY}/pkg/apis" echo "Running controller-gen ..." -controller-gen crd:crdVersions=v1 paths=./... output:dir=pkg/installer -rm -f pkg/installer/direct.csi.min.io_directcsidrives.yaml pkg/installer/direct.csi.min.io_directcsivolumes.yaml +controller-gen crd:crdVersions=v1 paths=./... output:dir=pkg/admin/installer +rm -f pkg/admin/installer/direct.csi.min.io_directcsidrives.yaml pkg/admin/installer/direct.csi.min.io_directcsivolumes.yaml echo "Running conversion-gen ..." conversion-gen \ diff --git a/pkg/admin/clean.go b/pkg/admin/clean.go index db8d18ca..3c335076 100644 --- a/pkg/admin/clean.go +++ b/pkg/admin/clean.go @@ -21,8 +21,6 @@ import ( "fmt" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" - "github.com/minio/directpv/pkg/client" - "github.com/minio/directpv/pkg/k8s" "github.com/minio/directpv/pkg/types" "github.com/minio/directpv/pkg/utils" corev1 "k8s.io/api/core/v1" @@ -44,7 +42,7 @@ type CleanArgs struct { } // Clean removes the stale/abandoned volumes -func Clean(ctx context.Context, args CleanArgs) error { +func (client *Client) Clean(ctx context.Context, args CleanArgs) error { ctx, cancelFunc := context.WithCancel(ctx) defer cancelFunc() @@ -59,7 +57,7 @@ func Clean(ctx context.Context, args CleanArgs) error { List(ctx) matchFunc := func(volume *types.Volume) bool { - pv, err := k8s.KubeClient().CoreV1().PersistentVolumes().Get(ctx, volume.Name, metav1.GetOptions{}) + pv, err := client.Kube().CoreV1().PersistentVolumes().Get(ctx, volume.Name, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { return true @@ -86,12 +84,12 @@ func Clean(ctx context.Context, args CleanArgs) error { if args.DryRun { continue } - if _, err := client.VolumeClient().Update(ctx, &result.Volume, metav1.UpdateOptions{ + if _, err := client.Volume().Update(ctx, &result.Volume, metav1.UpdateOptions{ TypeMeta: types.NewVolumeTypeMeta(), }); err != nil { return err } - if err := client.VolumeClient().Delete(ctx, result.Volume.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + if err := client.Volume().Delete(ctx, result.Volume.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { return err } if !args.Quiet { diff --git a/pkg/admin/client.go b/pkg/admin/client.go new file mode 100644 index 00000000..3a3057f9 --- /dev/null +++ b/pkg/admin/client.go @@ -0,0 +1,38 @@ +// This file is part of MinIO DirectPV +// Copyright (c) 2024 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package admin + +import ( + "github.com/minio/directpv/pkg/client" + "k8s.io/client-go/rest" +) + +// Client represents the admin clientset +type Client struct { + *client.Client +} + +// NewClient returns a new admin client +func NewClient(c *rest.Config) (*Client, error) { + directpvClientSet, err := client.NewClient(c) + if err != nil { + return nil, err + } + return &Client{ + Client: directpvClientSet, + }, nil +} diff --git a/pkg/admin/cordon.go b/pkg/admin/cordon.go index 036d0391..b6083f3b 100644 --- a/pkg/admin/cordon.go +++ b/pkg/admin/cordon.go @@ -21,7 +21,6 @@ import ( "fmt" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" - "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -37,7 +36,7 @@ type CordonArgs struct { } // Cordon makes a drive unschedulable -func Cordon(ctx context.Context, args CordonArgs) error { +func (client *Client) Cordon(ctx context.Context, args CordonArgs) error { var processed bool ctx, cancelFunc := context.WithCancel(ctx) @@ -75,7 +74,7 @@ func Cordon(ctx context.Context, args CordonArgs) error { result.Drive.Unschedulable() if !args.DryRun { - if _, err := client.DriveClient().Update(ctx, &result.Drive, metav1.UpdateOptions{}); err != nil { + if _, err := client.Drive().Update(ctx, &result.Drive, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("unable to cordon drive %v; %v", result.Drive.GetDriveID(), err) } } diff --git a/pkg/admin/discovery.go b/pkg/admin/discovery.go index 6ec3b957..6acce224 100644 --- a/pkg/admin/discovery.go +++ b/pkg/admin/discovery.go @@ -26,7 +26,6 @@ import ( tea "github.com/charmbracelet/bubbletea" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" - "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/types" "github.com/minio/directpv/pkg/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -44,7 +43,7 @@ type DiscoverArgs struct { } // DiscoverDevices discovers and fetches the devices present in the cluster -func DiscoverDevices(ctx context.Context, args DiscoverArgs) (map[directpvtypes.NodeID][]types.Device, error) { +func (client *Client) DiscoverDevices(ctx context.Context, args DiscoverArgs) (map[directpvtypes.NodeID][]types.Device, error) { if err := client.SyncNodes(ctx); err != nil { return nil, err } @@ -77,7 +76,7 @@ func DiscoverDevices(ctx context.Context, args DiscoverArgs) (map[directpvtypes. } }() } - resultMap, err := discoverDevices(ctx, nodes, args.Drives, teaProgram) + resultMap, err := client.discoverDevices(ctx, nodes, args.Drives, teaProgram) if err != nil { return nil, err } @@ -91,9 +90,9 @@ func DiscoverDevices(ctx context.Context, args DiscoverArgs) (map[directpvtypes. return resultMap, nil } -func discoverDevices(ctx context.Context, nodes []types.Node, drives []string, teaProgram *tea.Program) (devices map[directpvtypes.NodeID][]types.Device, err error) { +func (client *Client) discoverDevices(ctx context.Context, nodes []types.Node, drives []string, teaProgram *tea.Program) (devices map[directpvtypes.NodeID][]types.Device, err error) { var nodeNames []string - nodeClient := client.NodeClient() + nodeClient := client.Node() totalNodeCount := len(nodes) discoveryProgressMap := make(map[string]progressLog, totalNodeCount) for i := range nodes { diff --git a/pkg/admin/examples/clean.go b/pkg/admin/examples/clean.go index e2f6154b..66b69d1b 100644 --- a/pkg/admin/examples/clean.go +++ b/pkg/admin/examples/clean.go @@ -28,7 +28,6 @@ import ( "path/filepath" "github.com/minio/directpv/pkg/admin" - "github.com/minio/directpv/pkg/client" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -59,12 +58,11 @@ func main() { if err != nil { log.Fatalf("unable to get kubeconfig; %v", err) } - - if err := client.InitWithConfig(kubeConfig); err != nil { + adminClient, err := admin.NewClient(kubeConfig) + if err != nil { log.Fatalf("unable to initialize client; %v", err) } - - if err := admin.Clean(context.Background(), admin.CleanArgs{ + if err := adminClient.Clean(context.Background(), admin.CleanArgs{ Nodes: []string{"praveen-thinkpad-x1-carbon-6th"}, Drives: []string{"dm-0"}, }); err != nil { diff --git a/pkg/admin/examples/cordon.go b/pkg/admin/examples/cordon.go index 010336c6..f39e3b91 100644 --- a/pkg/admin/examples/cordon.go +++ b/pkg/admin/examples/cordon.go @@ -27,7 +27,6 @@ import ( "path/filepath" "github.com/minio/directpv/pkg/admin" - "github.com/minio/directpv/pkg/client" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -58,12 +57,11 @@ func main() { if err != nil { log.Fatalf("unable to get kubeconfig; %v", err) } - - if err := client.InitWithConfig(kubeConfig); err != nil { + adminClient, err := admin.NewClient(kubeConfig) + if err != nil { log.Fatalf("unable to initialize client; %v", err) } - - if err := admin.Cordon(context.Background(), admin.CordonArgs{ + if err := adminClient.Cordon(context.Background(), admin.CordonArgs{ Drives: []string{"dm-1"}, }); err != nil { log.Fatalf("unable to cordon the drive; %v", err) diff --git a/pkg/admin/examples/discovery.go b/pkg/admin/examples/discovery.go index 4df08181..ce45ef54 100644 --- a/pkg/admin/examples/discovery.go +++ b/pkg/admin/examples/discovery.go @@ -28,7 +28,6 @@ import ( "path/filepath" "github.com/minio/directpv/pkg/admin" - "github.com/minio/directpv/pkg/client" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -59,12 +58,11 @@ func main() { if err != nil { log.Fatalf("unable to get kubeconfig; %v", err) } - - if err := client.InitWithConfig(kubeConfig); err != nil { + adminClient, err := admin.NewClient(kubeConfig) + if err != nil { log.Fatalf("unable to initialize client; %v", err) } - - resultMap, err := admin.DiscoverDevices(context.Background(), admin.DiscoverArgs{ + resultMap, err := adminClient.DiscoverDevices(context.Background(), admin.DiscoverArgs{ PrintProgress: true, Nodes: nil, Drives: nil, diff --git a/pkg/admin/examples/info.go b/pkg/admin/examples/info.go index 5bd5a2ab..6fa7594b 100644 --- a/pkg/admin/examples/info.go +++ b/pkg/admin/examples/info.go @@ -29,7 +29,6 @@ import ( "github.com/dustin/go-humanize" "github.com/minio/directpv/pkg/admin" - "github.com/minio/directpv/pkg/client" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -60,12 +59,11 @@ func main() { if err != nil { log.Fatalf("unable to get kubeconfig; %v", err) } - - if err := client.InitWithConfig(kubeConfig); err != nil { + adminClient, err := admin.NewClient(kubeConfig) + if err != nil { log.Fatalf("unable to initialize client; %v", err) } - - nodeInfoMap, err := admin.Info(context.Background()) + nodeInfoMap, err := adminClient.Info(context.Background()) if err != nil { log.Fatalf("unable to get info; %v", err) } diff --git a/pkg/admin/examples/init.go b/pkg/admin/examples/init.go index d5edb498..03b36d80 100644 --- a/pkg/admin/examples/init.go +++ b/pkg/admin/examples/init.go @@ -32,7 +32,6 @@ import ( "strings" "github.com/minio/directpv/pkg/admin" - "github.com/minio/directpv/pkg/client" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -136,17 +135,15 @@ func main() { if err != nil { log.Fatalf("unable to get kubeconfig; %v", err) } - - if err := client.InitWithConfig(kubeConfig); err != nil { + adminClient, err := admin.NewClient(kubeConfig) + if err != nil { log.Fatalf("unable to initialize client; %v", err) } - initConfig, err := admin.ParseInitConfig(strings.NewReader(initConfigJson)) if err != nil { log.Fatalf("unable to parse init config; %v", err) } - - results, err := admin.InitDevices(context.Background(), admin.InitDevicesArgs{ + results, err := adminClient.InitDevices(context.Background(), admin.InitDevicesArgs{ InitConfig: initConfig, }) if err != nil { diff --git a/pkg/admin/examples/install.go b/pkg/admin/examples/install.go index 56f9f989..7bb20545 100644 --- a/pkg/admin/examples/install.go +++ b/pkg/admin/examples/install.go @@ -28,7 +28,6 @@ import ( "github.com/fatih/color" "github.com/minio/directpv/pkg/admin" - "github.com/minio/directpv/pkg/client" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -59,12 +58,11 @@ func main() { if err != nil { log.Fatalf("unable to get kubeconfig; %v", err) } - - if err := client.InitWithConfig(kubeConfig); err != nil { + adminClient, err := admin.NewClient(kubeConfig) + if err != nil { log.Fatalf("unable to initialize client; %v", err) } - - if _, err := admin.Install(context.Background(), admin.InstallArgs{ + if _, err := adminClient.Install(context.Background(), admin.InstallArgs{ Image: "directpv:v4.0.10", Registry: "quay.io", Org: "minio", diff --git a/pkg/admin/examples/label_drives.go b/pkg/admin/examples/label_drives.go index 855450fd..344a37d7 100644 --- a/pkg/admin/examples/label_drives.go +++ b/pkg/admin/examples/label_drives.go @@ -29,7 +29,6 @@ import ( "github.com/minio/directpv/pkg/admin" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" - "github.com/minio/directpv/pkg/client" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -60,24 +59,21 @@ func main() { if err != nil { log.Fatalf("unable to get kubeconfig; %v", err) } - - if err := client.InitWithConfig(kubeConfig); err != nil { + adminClient, err := admin.NewClient(kubeConfig) + if err != nil { log.Fatalf("unable to initialize client; %v", err) } - labels := []admin.Label{ { Key: directpvtypes.LabelKey("example-key"), Value: directpvtypes.LabelValue("example-value"), }, } - - if err := admin.LabelDrives(context.Background(), admin.LabelDriveArgs{ + if err := adminClient.LabelDrives(context.Background(), admin.LabelDriveArgs{ Nodes: []string{"praveen-thinkpad-x1-carbon-6th"}, Drives: []string{"dm-0"}, }, labels); err != nil { log.Fatalf("unable to label the drive; %v", err) } - fmt.Println("successfully labelled the drive(s)") } diff --git a/pkg/admin/examples/label_volumes.go b/pkg/admin/examples/label_volumes.go index fac3cc31..b9f106bf 100644 --- a/pkg/admin/examples/label_volumes.go +++ b/pkg/admin/examples/label_volumes.go @@ -29,7 +29,6 @@ import ( "github.com/minio/directpv/pkg/admin" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" - "github.com/minio/directpv/pkg/client" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -60,24 +59,21 @@ func main() { if err != nil { log.Fatalf("unable to get kubeconfig; %v", err) } - - if err := client.InitWithConfig(kubeConfig); err != nil { + adminClient, err := admin.NewClient(kubeConfig) + if err != nil { log.Fatalf("unable to initialize client; %v", err) } - labels := []admin.Label{ { Key: directpvtypes.LabelKey("example-key"), Value: directpvtypes.LabelValue("example-value"), }, } - - if err := admin.LabelVolumes(context.Background(), admin.LabelVolumeArgs{ + if err := adminClient.LabelVolumes(context.Background(), admin.LabelVolumeArgs{ Nodes: []string{"praveen-thinkpad-x1-carbon-6th"}, Drives: []string{"dm-0"}, }, labels); err != nil { log.Fatalf("unable to label the volume; %v", err) } - fmt.Println("successfully labelled the volume(s)") } diff --git a/pkg/admin/examples/move.go b/pkg/admin/examples/move.go index 78eeba14..f6f96dfc 100644 --- a/pkg/admin/examples/move.go +++ b/pkg/admin/examples/move.go @@ -32,7 +32,6 @@ import ( "github.com/minio/directpv/pkg/admin" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" - "github.com/minio/directpv/pkg/client" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -63,12 +62,11 @@ func main() { if err != nil { log.Fatalf("unable to get kubeconfig; %v", err) } - - if err := client.InitWithConfig(kubeConfig); err != nil { + adminClient, err := admin.NewClient(kubeConfig) + if err != nil { log.Fatalf("unable to initialize client; %v", err) } - - if err := admin.Move(context.Background(), admin.MoveArgs{ + if err := adminClient.Move(context.Background(), admin.MoveArgs{ Source: directpvtypes.DriveID("2786de98-2a84-40d4-8cee-8f73686928f8"), Destination: directpvtypes.DriveID("b35f1f8e-6bf3-4747-9976-192b23c1a019"), }); err != nil { diff --git a/pkg/admin/examples/remove.go b/pkg/admin/examples/remove.go index 3b925fd4..4a81a2a0 100644 --- a/pkg/admin/examples/remove.go +++ b/pkg/admin/examples/remove.go @@ -28,7 +28,6 @@ import ( "path/filepath" "github.com/minio/directpv/pkg/admin" - "github.com/minio/directpv/pkg/client" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -59,12 +58,11 @@ func main() { if err != nil { log.Fatalf("unable to get kubeconfig; %v", err) } - - if err := client.InitWithConfig(kubeConfig); err != nil { + adminClient, err := admin.NewClient(kubeConfig) + if err != nil { log.Fatalf("unable to initialize client; %v", err) } - - if err := admin.Remove(context.Background(), admin.RemoveArgs{ + if err := adminClient.Remove(context.Background(), admin.RemoveArgs{ Nodes: []string{"praveen-thinkpad-x1-carbon-6th"}, Drives: []string{"dm-0"}, }); err != nil { diff --git a/pkg/admin/examples/resume_drives.go b/pkg/admin/examples/resume_drives.go index e8676500..ebabf19d 100644 --- a/pkg/admin/examples/resume_drives.go +++ b/pkg/admin/examples/resume_drives.go @@ -28,7 +28,6 @@ import ( "path/filepath" "github.com/minio/directpv/pkg/admin" - "github.com/minio/directpv/pkg/client" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -59,12 +58,11 @@ func main() { if err != nil { log.Fatalf("unable to get kubeconfig; %v", err) } - - if err := client.InitWithConfig(kubeConfig); err != nil { + adminClient, err := admin.NewClient(kubeConfig) + if err != nil { log.Fatalf("unable to initialize client; %v", err) } - - if err := admin.ResumeDrives(context.Background(), admin.SuspendDriveArgs{ + if err := adminClient.ResumeDrives(context.Background(), admin.SuspendDriveArgs{ Nodes: []string{"praveen-thinkpad-x1-carbon-6th"}, Drives: []string{"dm-0"}, }); err != nil { diff --git a/pkg/admin/examples/resume_volumes.go b/pkg/admin/examples/resume_volumes.go index 2f29613c..c10d00eb 100644 --- a/pkg/admin/examples/resume_volumes.go +++ b/pkg/admin/examples/resume_volumes.go @@ -28,7 +28,6 @@ import ( "path/filepath" "github.com/minio/directpv/pkg/admin" - "github.com/minio/directpv/pkg/client" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -59,12 +58,11 @@ func main() { if err != nil { log.Fatalf("unable to get kubeconfig; %v", err) } - - if err := client.InitWithConfig(kubeConfig); err != nil { + adminClient, err := admin.NewClient(kubeConfig) + if err != nil { log.Fatalf("unable to initialize client; %v", err) } - - if err := admin.ResumeVolumes(context.Background(), admin.ResumeVolumeArgs{ + if err := adminClient.ResumeVolumes(context.Background(), admin.ResumeVolumeArgs{ Nodes: []string{"praveen-thinkpad-x1-carbon-6th"}, Drives: []string{"dm-0"}, }); err != nil { diff --git a/pkg/admin/examples/suspend_drives.go b/pkg/admin/examples/suspend_drives.go index 6a85792b..5e8f8c1d 100644 --- a/pkg/admin/examples/suspend_drives.go +++ b/pkg/admin/examples/suspend_drives.go @@ -28,7 +28,6 @@ import ( "path/filepath" "github.com/minio/directpv/pkg/admin" - "github.com/minio/directpv/pkg/client" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -59,11 +58,11 @@ func main() { if err != nil { log.Fatalf("unable to get kubeconfig; %v", err) } - if err := client.InitWithConfig(kubeConfig); err != nil { + adminClient, err := admin.NewClient(kubeConfig) + if err != nil { log.Fatalf("unable to initialize client; %v", err) } - - if err := admin.SuspendDrives(context.Background(), admin.SuspendDriveArgs{ + if err := adminClient.SuspendDrives(context.Background(), admin.SuspendDriveArgs{ Nodes: []string{"praveen-thinkpad-x1-carbon-6th"}, Drives: []string{"dm-0"}, }); err != nil { diff --git a/pkg/admin/examples/suspend_volumes.go b/pkg/admin/examples/suspend_volumes.go index 30f86323..cdacc55b 100644 --- a/pkg/admin/examples/suspend_volumes.go +++ b/pkg/admin/examples/suspend_volumes.go @@ -28,7 +28,6 @@ import ( "path/filepath" "github.com/minio/directpv/pkg/admin" - "github.com/minio/directpv/pkg/client" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -59,12 +58,11 @@ func main() { if err != nil { log.Fatalf("unable to get kubeconfig; %v", err) } - - if err := client.InitWithConfig(kubeConfig); err != nil { + adminClient, err := admin.NewClient(kubeConfig) + if err != nil { log.Fatalf("unable to initialize client; %v", err) } - - if err := admin.SuspendVolumes(context.Background(), admin.SuspendVolumeArgs{ + if err := adminClient.SuspendVolumes(context.Background(), admin.SuspendVolumeArgs{ Nodes: []string{"praveen-thinkpad-x1-carbon-6th"}, Drives: []string{"dm-0"}, }); err != nil { diff --git a/pkg/admin/examples/uncordon.go b/pkg/admin/examples/uncordon.go index 48c6f910..2e503b1e 100644 --- a/pkg/admin/examples/uncordon.go +++ b/pkg/admin/examples/uncordon.go @@ -27,7 +27,6 @@ import ( "path/filepath" "github.com/minio/directpv/pkg/admin" - "github.com/minio/directpv/pkg/client" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -58,12 +57,11 @@ func main() { if err != nil { log.Fatalf("unable to get kubeconfig; %v", err) } - - if err := client.InitWithConfig(kubeConfig); err != nil { + adminClient, err := admin.NewClient(kubeConfig) + if err != nil { log.Fatalf("unable to initialize client; %v", err) } - - if err := admin.Uncordon(context.Background(), admin.UncordonArgs{ + if err := adminClient.Uncordon(context.Background(), admin.UncordonArgs{ Drives: []string{"dm-1"}, }); err != nil { log.Fatalf("unable to uncordon the drive; %v", err) diff --git a/pkg/admin/examples/uninstall.go b/pkg/admin/examples/uninstall.go index 848b153e..726e1f34 100644 --- a/pkg/admin/examples/uninstall.go +++ b/pkg/admin/examples/uninstall.go @@ -28,7 +28,6 @@ import ( "path/filepath" "github.com/minio/directpv/pkg/admin" - "github.com/minio/directpv/pkg/client" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -59,12 +58,11 @@ func main() { if err != nil { log.Fatalf("unable to get kubeconfig; %v", err) } - - if err := client.InitWithConfig(kubeConfig); err != nil { + adminClient, err := admin.NewClient(kubeConfig) + if err != nil { log.Fatalf("unable to initialize client; %v", err) } - - if err := admin.Uninstall(context.Background(), admin.UninstallArgs{}); err != nil { + if err := adminClient.Uninstall(context.Background(), admin.UninstallArgs{}); err != nil { log.Fatalf("unable to uninstall directpv; %v", err) } fmt.Println("\nDirectPV uninstalled successfully") diff --git a/pkg/admin/info.go b/pkg/admin/info.go index 5a27fcdd..b746d209 100644 --- a/pkg/admin/info.go +++ b/pkg/admin/info.go @@ -21,9 +21,7 @@ import ( "fmt" "strings" - "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/consts" - "github.com/minio/directpv/pkg/k8s" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -36,8 +34,8 @@ type NodeLevelInfo struct { } // Info returns the overall info of the directpv installation -func Info(ctx context.Context) (map[string]NodeLevelInfo, error) { - crds, err := k8s.CRDClient().List(ctx, metav1.ListOptions{}) +func (client *Client) Info(ctx context.Context) (map[string]NodeLevelInfo, error) { + crds, err := client.CRD().List(ctx, metav1.ListOptions{}) if err != nil { return nil, fmt.Errorf("unable to list CRDs; %v", err) } @@ -54,7 +52,7 @@ func Info(ctx context.Context) (map[string]NodeLevelInfo, error) { if !drivesFound || !volumesFound { return nil, fmt.Errorf("%v installation not found", consts.AppPrettyName) } - nodeList, err := k8s.GetCSINodes(ctx) + nodeList, err := client.K8s().GetCSINodes(ctx) if err != nil { return nil, fmt.Errorf("unable to get CSI nodes; %v", err) } diff --git a/pkg/admin/init.go b/pkg/admin/init.go index c110b8d6..748d196c 100644 --- a/pkg/admin/init.go +++ b/pkg/admin/init.go @@ -25,7 +25,6 @@ import ( tea "github.com/charmbracelet/bubbletea" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" - "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/types" "github.com/minio/directpv/pkg/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -64,7 +63,7 @@ func (args *InitDevicesArgs) Validate() error { } // InitDevices creates InitRequest objects and waits until it gets initialized -func InitDevices(ctx context.Context, args InitDevicesArgs) ([]InitResult, error) { +func (client *Client) InitDevices(ctx context.Context, args InitDevicesArgs) ([]InitResult, error) { if err := args.Validate(); err != nil { return nil, fmt.Errorf("unable to validate args; %v", err) } @@ -76,7 +75,7 @@ func InitDevices(ctx context.Context, args InitDevicesArgs) ([]InitResult, error labelMap := map[directpvtypes.LabelKey][]directpvtypes.LabelValue{ directpvtypes.RequestIDLabelKey: utils.ToLabelValues([]string{requestID}), } - client.InitRequestClient().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{ + client.InitRequest().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{ LabelSelector: directpvtypes.ToLabelSelector(labelMap), }) }() @@ -94,7 +93,7 @@ func InitDevices(ctx context.Context, args InitDevicesArgs) ([]InitResult, error } }() } - results, err := initDevices(ctx, initRequests, requestID, teaProgram, args.ListTimeout) + results, err := client.initDevices(ctx, initRequests, requestID, teaProgram, args.ListTimeout) if err != nil && teaProgram == nil { return nil, err } @@ -108,13 +107,13 @@ func InitDevices(ctx context.Context, args InitDevicesArgs) ([]InitResult, error return results, nil } -func initDevices(ctx context.Context, initRequests []types.InitRequest, requestID string, teaProgram *tea.Program, listTimeout time.Duration) (results []InitResult, err error) { +func (client *Client) initDevices(ctx context.Context, initRequests []types.InitRequest, requestID string, teaProgram *tea.Program, listTimeout time.Duration) (results []InitResult, err error) { totalReqCount := len(initRequests) totalTasks := totalReqCount * 2 var completedTasks int initProgressMap := make(map[string]progressLog, totalReqCount) for i := range initRequests { - initReq, err := client.InitRequestClient().Create(ctx, &initRequests[i], metav1.CreateOptions{TypeMeta: types.NewInitRequestTypeMeta()}) + initReq, err := client.InitRequest().Create(ctx, &initRequests[i], metav1.CreateOptions{TypeMeta: types.NewInitRequestTypeMeta()}) if err != nil { return nil, err } diff --git a/pkg/admin/init_config.go b/pkg/admin/init_config.go index b8e84be1..c2c07246 100644 --- a/pkg/admin/init_config.go +++ b/pkg/admin/init_config.go @@ -100,21 +100,6 @@ func ToInitConfig(resultMap map[directpvtypes.NodeID][]types.Device) InitConfig } // ToInitRequestObjects converts initConfig to init request objects. -// -// NOTE: After initrequest object creation, use the requestID for the cleanup -// -// Example :- -// -// ``` -// defer func() { -// labelMap := map[directpvtypes.LabelKey][]directpvtypes.LabelValue{ -// directpvtypes.RequestIDLabelKey: toLabelValues([]string{requestID}), -// } -// client.InitRequestClient().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{ -// LabelSelector: directpvtypes.ToLabelSelector(labelMap), -// }) -// }() -// ``` func (config *InitConfig) ToInitRequestObjects() (initRequests []types.InitRequest, requestID string) { requestID = uuid.New().String() for _, node := range config.Nodes { diff --git a/pkg/admin/install.go b/pkg/admin/install.go index 7db3af17..552914ce 100644 --- a/pkg/admin/install.go +++ b/pkg/admin/install.go @@ -27,16 +27,16 @@ import ( tea "github.com/charmbracelet/bubbletea" "github.com/fatih/color" + "github.com/minio/directpv/pkg/admin/installer" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" - "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/consts" - "github.com/minio/directpv/pkg/installer" legacyclient "github.com/minio/directpv/pkg/legacy/client" "github.com/minio/directpv/pkg/utils" "github.com/mitchellh/go-homedir" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/version" + versionpkg "k8s.io/apimachinery/pkg/util/version" + "k8s.io/klog/v2" ) // ErrInstallationIncomplete denotes that the installation couldn't complete @@ -69,7 +69,7 @@ type InstallArgs struct { // Quiet enables quiet mode Quiet bool // KubeVersion is required for declarative and dryrun manifests - KubeVersion *version.Version + KubeVersion *versionpkg.Version // DryRun when set, runs in dryrun mode and generates the manifests DryRun bool // OutputFormat denotes the output format (yaml|json) for the manifests; to be used for DryRun @@ -95,7 +95,7 @@ func (args *InstallArgs) Validate() error { } // Install - installs directpv with the provided arguments -func Install(ctx context.Context, args InstallArgs) ([]installer.Component, error) { +func (client *Client) Install(ctx context.Context, args InstallArgs) ([]installer.Component, error) { if err := args.Validate(); err != nil { return nil, err } @@ -137,7 +137,7 @@ func Install(ctx context.Context, args InstallArgs) ([]installer.Component, erro installerArgs.AppArmorProfile = args.AppArmorProfile installerArgs.Quiet = args.Quiet installerArgs.KubeVersion = args.KubeVersion - installerArgs.Legacy = isLegacyEnabled(ctx, args) + installerArgs.Legacy = client.isLegacyEnabled(ctx, args) installerArgs.PluginVersion = version if file != nil { installerArgs.ObjectWriter = file @@ -153,6 +153,21 @@ func Install(ctx context.Context, args InstallArgs) ([]installer.Component, erro return utils.ToJSON(obj) } } + if installerArgs.KubeVersion == nil { + // default higher version + if installerArgs.KubeVersion, err = versionpkg.ParseSemantic("1.27.0"); err != nil { + klog.Fatalf("this should not happen; %v", err) + } + } + } else { + major, minor, err := client.K8s().GetKubeVersion() + if err != nil { + return nil, err + } + installerArgs.KubeVersion, err = versionpkg.ParseSemantic(fmt.Sprintf("%v.%v.0", major, minor)) + if err != nil { + klog.Fatalf("this should not happen; %v", err) + } } installerArgs.Declarative = args.Declarative installerArgs.Openshift = args.Openshift @@ -160,6 +175,11 @@ func Install(ctx context.Context, args InstallArgs) ([]installer.Component, erro var failed bool var installedComponents []installer.Component var wg sync.WaitGroup + legacyClient, err := legacyclient.NewClient(client.K8s()) + if err != nil { + return nil, err + } + installerTasks := installer.GetTasks(client.Client, legacyClient) if args.PrintProgress { m := newProgressModel(true) teaProgram := tea.NewProgram(m) @@ -174,7 +194,7 @@ func Install(ctx context.Context, args InstallArgs) ([]installer.Component, erro wg.Add(1) var totalSteps, step, completedTasks int var currentPercent float64 - totalTasks := len(installer.Tasks) + totalTasks := len(installerTasks) weightagePerTask := 1.0 / totalTasks progressCh := make(chan installer.Message) go func() { @@ -244,7 +264,7 @@ func Install(ctx context.Context, args InstallArgs) ([]installer.Component, erro }() installerArgs.ProgressCh = progressCh } - if err := installer.Install(ctx, installerArgs); err != nil && installerArgs.ProgressCh == nil { + if err := installer.Install(ctx, installerArgs, installerTasks); err != nil && installerArgs.ProgressCh == nil { return installedComponents, err } if installerArgs.ProgressCh != nil { @@ -256,7 +276,7 @@ func Install(ctx context.Context, args InstallArgs) ([]installer.Component, erro return installedComponents, nil } -func isLegacyEnabled(ctx context.Context, args InstallArgs) bool { +func (client Client) isLegacyEnabled(ctx context.Context, args InstallArgs) bool { if args.DryRun { return args.EnableLegacy } @@ -281,9 +301,13 @@ func isLegacyEnabled(ctx context.Context, args InstallArgs) bool { return true } - legacyclient.Init() + legacyClient, err := legacyclient.NewClient(client.K8s()) + if err != nil { + utils.Eprintf(args.Quiet, true, "unable to create legacy client; %v", err) + return false + } - for result := range legacyclient.ListVolumes(ctx) { + for result := range legacyClient.ListVolumes(ctx) { if result.Err != nil { utils.Eprintf(args.Quiet, true, "unable to get legacy volumes; %v", result.Err) break diff --git a/pkg/installer/args.go b/pkg/admin/installer/args.go similarity index 98% rename from pkg/installer/args.go rename to pkg/admin/installer/args.go index 31bde459..10f4867d 100644 --- a/pkg/installer/args.go +++ b/pkg/admin/installer/args.go @@ -117,6 +117,10 @@ func (args *Args) validate() error { return errors.New("object converter must be provided") } + if args.KubeVersion == nil { + return errors.New("kubeversion is not set") + } + return nil } diff --git a/pkg/installer/consts.go b/pkg/admin/installer/consts.go similarity index 100% rename from pkg/installer/consts.go rename to pkg/admin/installer/consts.go diff --git a/pkg/installer/crd.go b/pkg/admin/installer/crd.go similarity index 77% rename from pkg/installer/crd.go rename to pkg/admin/installer/crd.go index 3bbb4061..5a69013f 100644 --- a/pkg/installer/crd.go +++ b/pkg/admin/installer/crd.go @@ -24,7 +24,6 @@ import ( directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/consts" - "github.com/minio/directpv/pkg/k8s" "k8s.io/apiextensions-apiserver/pkg/apihelpers" apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -45,7 +44,9 @@ var nodesYAML []byte //go:embed directpv.min.io_directpvinitrequests.yaml var initrequestsYAML []byte -type crdTask struct{} +type crdTask struct { + client *client.Client +} func (crdTask) Name() string { return "CRD" @@ -65,12 +66,12 @@ func (crdTask) End(ctx context.Context, args *Args, err error) error { return nil } -func (crdTask) Execute(ctx context.Context, args *Args) error { - return createCRDs(ctx, args) +func (t crdTask) Execute(ctx context.Context, args *Args) error { + return t.createCRDs(ctx, args) } -func (c crdTask) Delete(ctx context.Context, args *Args) error { - return deleteCRDs(ctx, args.ForceUninstall) +func (t crdTask) Delete(ctx context.Context, args *Args) error { + return t.deleteCRDs(ctx, args.ForceUninstall) } func setNoneConversionStrategy(crd *apiextensions.CustomResourceDefinition) { @@ -144,7 +145,7 @@ func updateCRD( return existingCRD, false, nil } -func createCRDs(ctx context.Context, args *Args) (err error) { +func (t crdTask) createCRDs(ctx context.Context, args *Args) (err error) { register := func(data []byte, step int) error { object := map[string]interface{}{} if err := yaml.Unmarshal(data, &object); err != nil { @@ -167,7 +168,7 @@ func createCRDs(ctx context.Context, args *Args) (err error) { return args.writeObject(&crd) } - existingCRD, err := k8s.CRDClient().Get(ctx, crd.Name, metav1.GetOptions{}) + existingCRD, err := t.client.CRD().Get(ctx, crd.Name, metav1.GetOptions{}) if err != nil { if !apierrors.IsNotFound(err) { return err @@ -178,7 +179,7 @@ func createCRDs(ctx context.Context, args *Args) (err error) { } if !args.Declarative { - _, err := k8s.CRDClient().Create(ctx, &crd, metav1.CreateOptions{}) + _, err := t.client.CRD().Create(ctx, &crd, metav1.CreateOptions{}) if err != nil { return err } @@ -200,7 +201,7 @@ func createCRDs(ctx context.Context, args *Args) (err error) { } if !args.Declarative && !isLatest { - updatedCRD, err = k8s.CRDClient().Update(ctx, updatedCRD, metav1.UpdateOptions{}) + updatedCRD, err = t.client.CRD().Update(ctx, updatedCRD, metav1.UpdateOptions{}) if err != nil { return err } @@ -229,11 +230,11 @@ func createCRDs(ctx context.Context, args *Args) (err error) { return register(initrequestsYAML, 4) } -func removeVolumes(ctx context.Context) error { +func (t crdTask) removeVolumes(ctx context.Context) error { ctx, cancelFunc := context.WithCancel(ctx) defer cancelFunc() - for result := range client.NewVolumeLister().List(ctx) { + for result := range t.client.NewVolumeLister().List(ctx) { if result.Err != nil { if apierrors.IsNotFound(result.Err) { break @@ -244,12 +245,12 @@ func removeVolumes(ctx context.Context) error { result.Volume.RemovePVProtection() result.Volume.RemovePurgeProtection() - _, err := client.VolumeClient().Update(ctx, &result.Volume, metav1.UpdateOptions{}) + _, err := t.client.Volume().Update(ctx, &result.Volume, metav1.UpdateOptions{}) if err != nil { return err } - err = client.VolumeClient().Delete(ctx, result.Volume.Name, metav1.DeleteOptions{}) + err = t.client.Volume().Delete(ctx, result.Volume.Name, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { return err } @@ -258,11 +259,11 @@ func removeVolumes(ctx context.Context) error { return nil } -func removeDrives(ctx context.Context) error { +func (t crdTask) removeDrives(ctx context.Context) error { ctx, cancelFunc := context.WithCancel(ctx) defer cancelFunc() - for result := range client.NewDriveLister().List(ctx) { + for result := range t.client.NewDriveLister().List(ctx) { if result.Err != nil { if apierrors.IsNotFound(result.Err) { break @@ -270,12 +271,12 @@ func removeDrives(ctx context.Context) error { return result.Err } result.Drive.Finalizers = []string{} - _, err := client.DriveClient().Update(ctx, &result.Drive, metav1.UpdateOptions{}) + _, err := t.client.Drive().Update(ctx, &result.Drive, metav1.UpdateOptions{}) if err != nil && !apierrors.IsNotFound(err) { return err } - err = client.DriveClient().Delete(ctx, result.Drive.Name, metav1.DeleteOptions{}) + err = t.client.Drive().Delete(ctx, result.Drive.Name, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { return err } @@ -284,11 +285,11 @@ func removeDrives(ctx context.Context) error { return nil } -func removeNodes(ctx context.Context) error { +func (t crdTask) removeNodes(ctx context.Context) error { ctx, cancelFunc := context.WithCancel(ctx) defer cancelFunc() - for result := range client.NewNodeLister().List(ctx) { + for result := range t.client.NewNodeLister().List(ctx) { if result.Err != nil { if apierrors.IsNotFound(result.Err) { break @@ -296,11 +297,11 @@ func removeNodes(ctx context.Context) error { return result.Err } result.Node.Finalizers = []string{} - _, err := client.NodeClient().Update(ctx, &result.Node, metav1.UpdateOptions{}) + _, err := t.client.Node().Update(ctx, &result.Node, metav1.UpdateOptions{}) if err != nil && !apierrors.IsNotFound(err) { return err } - err = client.NodeClient().Delete(ctx, result.Node.Name, metav1.DeleteOptions{}) + err = t.client.Node().Delete(ctx, result.Node.Name, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { return err } @@ -309,11 +310,11 @@ func removeNodes(ctx context.Context) error { return nil } -func removeInitRequests(ctx context.Context) error { +func (t crdTask) removeInitRequests(ctx context.Context) error { ctx, cancelFunc := context.WithCancel(ctx) defer cancelFunc() - for result := range client.NewInitRequestLister().List(ctx) { + for result := range t.client.NewInitRequestLister().List(ctx) { if result.Err != nil { if apierrors.IsNotFound(result.Err) { break @@ -321,11 +322,11 @@ func removeInitRequests(ctx context.Context) error { return result.Err } result.InitRequest.Finalizers = []string{} - _, err := client.InitRequestClient().Update(ctx, &result.InitRequest, metav1.UpdateOptions{}) + _, err := t.client.InitRequest().Update(ctx, &result.InitRequest, metav1.UpdateOptions{}) if err != nil && !apierrors.IsNotFound(err) { return err } - err = client.InitRequestClient().Delete(ctx, result.InitRequest.Name, metav1.DeleteOptions{}) + err = t.client.InitRequest().Delete(ctx, result.InitRequest.Name, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { return err } @@ -334,47 +335,47 @@ func removeInitRequests(ctx context.Context) error { return nil } -func deleteCRDs(ctx context.Context, force bool) error { +func (t crdTask) deleteCRDs(ctx context.Context, force bool) error { if !force { return nil } - if err := removeVolumes(ctx); err != nil { + if err := t.removeVolumes(ctx); err != nil { return err } - if err := removeDrives(ctx); err != nil { + if err := t.removeDrives(ctx); err != nil { return err } - if err := removeNodes(ctx); err != nil { + if err := t.removeNodes(ctx); err != nil { return err } - if err := removeInitRequests(ctx); err != nil { + if err := t.removeInitRequests(ctx); err != nil { return err } driveCRDName := consts.DriveResource + "." + consts.GroupName - err := k8s.CRDClient().Delete(ctx, driveCRDName, metav1.DeleteOptions{}) + err := t.client.CRD().Delete(ctx, driveCRDName, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { return err } volumeCRDName := consts.VolumeResource + "." + consts.GroupName - err = k8s.CRDClient().Delete(ctx, volumeCRDName, metav1.DeleteOptions{}) + err = t.client.CRD().Delete(ctx, volumeCRDName, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { return err } nodeCRDName := consts.NodeResource + "." + consts.GroupName - err = k8s.CRDClient().Delete(ctx, nodeCRDName, metav1.DeleteOptions{}) + err = t.client.CRD().Delete(ctx, nodeCRDName, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { return err } initRequestCRDName := consts.InitRequestResource + "." + consts.GroupName - err = k8s.CRDClient().Delete(ctx, initRequestCRDName, metav1.DeleteOptions{}) + err = t.client.CRD().Delete(ctx, initRequestCRDName, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { return err } diff --git a/pkg/installer/csidriver.go b/pkg/admin/installer/csidriver.go similarity index 74% rename from pkg/installer/csidriver.go rename to pkg/admin/installer/csidriver.go index ba888cdd..7cce21ac 100644 --- a/pkg/installer/csidriver.go +++ b/pkg/admin/installer/csidriver.go @@ -22,8 +22,8 @@ import ( "fmt" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" + "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/consts" - "github.com/minio/directpv/pkg/k8s" legacyclient "github.com/minio/directpv/pkg/legacy/client" storagev1 "k8s.io/api/storage/v1" storagev1beta1 "k8s.io/api/storage/v1beta1" @@ -31,7 +31,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -type csiDriverTask struct{} +type csiDriverTask struct { + client *client.Client +} func (csiDriverTask) Name() string { return "CSIDriver" @@ -55,17 +57,17 @@ func (csiDriverTask) End(ctx context.Context, args *Args, err error) error { return nil } -func (csiDriverTask) Execute(ctx context.Context, args *Args) error { - return createCSIDriver(ctx, args) +func (t csiDriverTask) Execute(ctx context.Context, args *Args) error { + return t.createCSIDriver(ctx, args) } -func (csiDriverTask) Delete(ctx context.Context, _ *Args) error { - return deleteCSIDriver(ctx) +func (t csiDriverTask) Delete(ctx context.Context, _ *Args) error { + return t.deleteCSIDriver(ctx) } var errCSIDriverVersionUnsupported = errors.New("unsupported CSIDriver version found") -func doCreateCSIDriver(ctx context.Context, args *Args, version string, legacy bool, step int) (err error) { +func (t csiDriverTask) doCreateCSIDriver(ctx context.Context, args *Args, version string, legacy bool, step int) (err error) { name := consts.Identity if legacy { name = legacyclient.Identity @@ -109,7 +111,7 @@ func doCreateCSIDriver(ctx context.Context, args *Args, version string, legacy b } if !args.DryRun && !args.Declarative { - _, err := k8s.KubeClient().StorageV1().CSIDrivers().Create(ctx, csiDriver, metav1.CreateOptions{}) + _, err := t.client.Kube().StorageV1().CSIDrivers().Create(ctx, csiDriver, metav1.CreateOptions{}) if err != nil && !apierrors.IsAlreadyExists(err) { return err } @@ -142,7 +144,7 @@ func doCreateCSIDriver(ctx context.Context, args *Args, version string, legacy b } if !args.DryRun && !args.Declarative { - _, err := k8s.KubeClient().StorageV1beta1().CSIDrivers().Create(ctx, csiDriver, metav1.CreateOptions{}) + _, err := t.client.Kube().StorageV1beta1().CSIDrivers().Create(ctx, csiDriver, metav1.CreateOptions{}) if err != nil && !apierrors.IsAlreadyExists(err) { return err } @@ -155,26 +157,26 @@ func doCreateCSIDriver(ctx context.Context, args *Args, version string, legacy b } } -func createCSIDriver(ctx context.Context, args *Args) (err error) { +func (t csiDriverTask) createCSIDriver(ctx context.Context, args *Args) (err error) { version := "v1" if args.DryRun { if args.KubeVersion.Major() >= 1 && args.KubeVersion.Minor() < 19 { version = "v1beta1" } } else { - gvk, err := k8s.GetGroupVersionKind("storage.k8s.io", "CSIDriver", "v1", "v1beta1") + gvk, err := t.client.K8s().GetGroupVersionKind("storage.k8s.io", "CSIDriver", "v1", "v1beta1") if err != nil { return err } version = gvk.Version } - if err := doCreateCSIDriver(ctx, args, version, false, 1); err != nil { + if err := t.doCreateCSIDriver(ctx, args, version, false, 1); err != nil { return err } if args.Legacy { - if err := doCreateCSIDriver(ctx, args, version, true, 2); err != nil { + if err := t.doCreateCSIDriver(ctx, args, version, true, 2); err != nil { return err } } @@ -182,14 +184,14 @@ func createCSIDriver(ctx context.Context, args *Args) (err error) { return nil } -func doDeleteCSIDriver(ctx context.Context, version, name string) (err error) { +func (t csiDriverTask) doDeleteCSIDriver(ctx context.Context, version, name string) (err error) { switch version { case "v1": - err = k8s.KubeClient().StorageV1().CSIDrivers().Delete( + err = t.client.Kube().StorageV1().CSIDrivers().Delete( ctx, name, metav1.DeleteOptions{}, ) case "v1beta1": - err = k8s.KubeClient().StorageV1beta1().CSIDrivers().Delete( + err = t.client.Kube().StorageV1beta1().CSIDrivers().Delete( ctx, name, metav1.DeleteOptions{}, ) default: @@ -203,15 +205,15 @@ func doDeleteCSIDriver(ctx context.Context, version, name string) (err error) { return nil } -func deleteCSIDriver(ctx context.Context) error { - gvk, err := k8s.GetGroupVersionKind("storage.k8s.io", "CSIDriver", "v1", "v1beta1") +func (t csiDriverTask) deleteCSIDriver(ctx context.Context) error { + gvk, err := t.client.K8s().GetGroupVersionKind("storage.k8s.io", "CSIDriver", "v1", "v1beta1") if err != nil { return err } - if err = doDeleteCSIDriver(ctx, gvk.Version, consts.Identity); err != nil { + if err = t.doDeleteCSIDriver(ctx, gvk.Version, consts.Identity); err != nil { return err } - return doDeleteCSIDriver(ctx, gvk.Version, legacyclient.Identity) + return t.doDeleteCSIDriver(ctx, gvk.Version, legacyclient.Identity) } diff --git a/pkg/installer/daemonset.go b/pkg/admin/installer/daemonset.go similarity index 92% rename from pkg/installer/daemonset.go rename to pkg/admin/installer/daemonset.go index f3fd048a..4ca0fbae 100644 --- a/pkg/installer/daemonset.go +++ b/pkg/admin/installer/daemonset.go @@ -21,8 +21,8 @@ import ( "fmt" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" + "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/consts" - "github.com/minio/directpv/pkg/k8s" legacyclient "github.com/minio/directpv/pkg/legacy/client" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -48,7 +48,9 @@ const ( totalDaemonsetSteps = 2 ) -type daemonsetTask struct{} +type daemonsetTask struct { + client *client.Client +} func (daemonsetTask) Name() string { return "Daemonset" @@ -72,12 +74,12 @@ func (daemonsetTask) End(ctx context.Context, args *Args, err error) error { return nil } -func (daemonsetTask) Execute(ctx context.Context, args *Args) error { - return createDaemonset(ctx, args) +func (t daemonsetTask) Execute(ctx context.Context, args *Args) error { + return t.createDaemonset(ctx, args) } -func (daemonsetTask) Delete(ctx context.Context, _ *Args) error { - return deleteDaemonset(ctx) +func (t daemonsetTask) Delete(ctx context.Context, _ *Args) error { + return t.deleteDaemonset(ctx) } func newSecurityContext(seccompProfile string) *corev1.SecurityContext { @@ -249,7 +251,7 @@ func newDaemonset(podSpec corev1.PodSpec, name, selectorValue string, args *Args } } -func doCreateDaemonset(ctx context.Context, args *Args) (err error) { +func (t daemonsetTask) doCreateDaemonset(ctx context.Context, args *Args) (err error) { securityContext := newSecurityContext(args.SeccompProfile) pluginSocketDir := newPluginsSocketDir(kubeletDirPath, consts.Identity) volumes, volumeMounts := getVolumesAndMounts(pluginSocketDir) @@ -286,7 +288,7 @@ func doCreateDaemonset(ctx context.Context, args *Args) (err error) { var selectorValue string if !args.DryRun { - daemonset, err := k8s.KubeClient().AppsV1().DaemonSets(namespace).Get( + daemonset, err := t.client.Kube().AppsV1().DaemonSets(namespace).Get( ctx, consts.NodeServerName, metav1.GetOptions{}, ) if err != nil && !apierrors.IsNotFound(err) { @@ -309,7 +311,7 @@ func doCreateDaemonset(ctx context.Context, args *Args) (err error) { daemonset := newDaemonset(podSpec, consts.NodeServerName, selectorValue, args) if !args.DryRun && !args.Declarative { - _, err = k8s.KubeClient().AppsV1().DaemonSets(namespace).Create( + _, err = t.client.Kube().AppsV1().DaemonSets(namespace).Create( ctx, daemonset, metav1.CreateOptions{}, ) if err != nil && !apierrors.IsAlreadyExists(err) { @@ -320,7 +322,7 @@ func doCreateDaemonset(ctx context.Context, args *Args) (err error) { return args.writeObject(daemonset) } -func doCreateLegacyDaemonset(ctx context.Context, args *Args) (err error) { +func (t daemonsetTask) doCreateLegacyDaemonset(ctx context.Context, args *Args) (err error) { securityContext := newSecurityContext(args.SeccompProfile) pluginSocketDir := newPluginsSocketDir(kubeletDirPath, legacyclient.Identity) volumes, volumeMounts := getVolumesAndMounts(pluginSocketDir) @@ -349,7 +351,7 @@ func doCreateLegacyDaemonset(ctx context.Context, args *Args) (err error) { var selectorValue string if !args.DryRun { - daemonset, err := k8s.KubeClient().AppsV1().DaemonSets(namespace).Get( + daemonset, err := t.client.Kube().AppsV1().DaemonSets(namespace).Get( ctx, consts.LegacyNodeServerName, metav1.GetOptions{}, ) if err != nil && !apierrors.IsNotFound(err) { @@ -372,7 +374,7 @@ func doCreateLegacyDaemonset(ctx context.Context, args *Args) (err error) { daemonset := newDaemonset(podSpec, consts.LegacyNodeServerName, selectorValue, args) if !args.DryRun && !args.Declarative { - _, err = k8s.KubeClient().AppsV1().DaemonSets(namespace).Create( + _, err = t.client.Kube().AppsV1().DaemonSets(namespace).Create( ctx, daemonset, metav1.CreateOptions{}, ) if err != nil && !apierrors.IsAlreadyExists(err) { @@ -383,11 +385,11 @@ func doCreateLegacyDaemonset(ctx context.Context, args *Args) (err error) { return args.writeObject(daemonset) } -func createDaemonset(ctx context.Context, args *Args) (err error) { +func (t daemonsetTask) createDaemonset(ctx context.Context, args *Args) (err error) { if !sendProgressMessage(ctx, args.ProgressCh, fmt.Sprintf("Creating %s Daemonset", consts.NodeServerName), 1, nil) { return errSendProgress } - if err := doCreateDaemonset(ctx, args); err != nil { + if err := t.doCreateDaemonset(ctx, args); err != nil { return err } if !sendProgressMessage(ctx, args.ProgressCh, fmt.Sprintf("Created %s Daemonset", consts.NodeServerName), 1, daemonsetComponent(consts.NodeServerName)) { @@ -401,7 +403,7 @@ func createDaemonset(ctx context.Context, args *Args) (err error) { if !sendProgressMessage(ctx, args.ProgressCh, fmt.Sprintf("Creating %s Daemonset", consts.LegacyNodeServerName), 2, nil) { return errSendProgress } - if err := doCreateLegacyDaemonset(ctx, args); err != nil { + if err := t.doCreateLegacyDaemonset(ctx, args); err != nil { return err } if !sendProgressMessage(ctx, args.ProgressCh, fmt.Sprintf("Created %s Daemonset", consts.LegacyNodeServerName), 2, daemonsetComponent(consts.LegacyNodeServerName)) { @@ -411,15 +413,15 @@ func createDaemonset(ctx context.Context, args *Args) (err error) { return nil } -func deleteDaemonset(ctx context.Context) error { - err := k8s.KubeClient().AppsV1().DaemonSets(namespace).Delete( +func (t daemonsetTask) deleteDaemonset(ctx context.Context) error { + err := t.client.Kube().AppsV1().DaemonSets(namespace).Delete( ctx, consts.NodeServerName, metav1.DeleteOptions{}, ) if err != nil && !apierrors.IsNotFound(err) { return err } - err = k8s.KubeClient().AppsV1().DaemonSets(namespace).Delete( + err = t.client.Kube().AppsV1().DaemonSets(namespace).Delete( ctx, consts.LegacyNodeServerName, metav1.DeleteOptions{}, ) if err != nil && !apierrors.IsNotFound(err) { diff --git a/pkg/installer/deployment.go b/pkg/admin/installer/deployment.go similarity index 87% rename from pkg/installer/deployment.go rename to pkg/admin/installer/deployment.go index 2229e9c8..b6d74d44 100644 --- a/pkg/installer/deployment.go +++ b/pkg/admin/installer/deployment.go @@ -21,8 +21,8 @@ import ( "fmt" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" + "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/consts" - "github.com/minio/directpv/pkg/k8s" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -31,7 +31,9 @@ import ( const deploymentFinalizer = consts.Identity + "/delete-protection" -type deploymentTask struct{} +type deploymentTask struct { + client *client.Client +} func (deploymentTask) Name() string { return "Deployment" @@ -55,15 +57,15 @@ func (deploymentTask) End(ctx context.Context, args *Args, err error) error { return nil } -func (deploymentTask) Execute(ctx context.Context, args *Args) error { - return createDeployment(ctx, args) +func (t deploymentTask) Execute(ctx context.Context, args *Args) error { + return t.createDeployment(ctx, args) } -func (deploymentTask) Delete(ctx context.Context, _ *Args) error { - return deleteDeployment(ctx) +func (t deploymentTask) Delete(ctx context.Context, _ *Args) error { + return t.deleteDeployment(ctx) } -func doCreateDeployment(ctx context.Context, args *Args, legacy bool, step int) (err error) { +func (t deploymentTask) doCreateDeployment(ctx context.Context, args *Args, legacy bool, step int) (err error) { name := consts.ControllerServerName containerArgs := []string{name, fmt.Sprintf("--identity=%s", consts.Identity)} if legacy { @@ -179,7 +181,7 @@ func doCreateDeployment(ctx context.Context, args *Args, legacy bool, step int) var selectorValue string if !args.DryRun { - deployment, err := k8s.KubeClient().AppsV1().Deployments(namespace).Get( + deployment, err := t.client.Kube().AppsV1().Deployments(namespace).Get( ctx, name, metav1.GetOptions{}, ) if err != nil && !apierrors.IsNotFound(err) { @@ -232,7 +234,7 @@ func doCreateDeployment(ctx context.Context, args *Args, legacy bool, step int) } if !args.DryRun && !args.Declarative { - _, err = k8s.KubeClient().AppsV1().Deployments(namespace).Create( + _, err = t.client.Kube().AppsV1().Deployments(namespace).Create( ctx, deployment, metav1.CreateOptions{}, ) if err != nil && !apierrors.IsAlreadyExists(err) { @@ -243,13 +245,13 @@ func doCreateDeployment(ctx context.Context, args *Args, legacy bool, step int) return args.writeObject(deployment) } -func createDeployment(ctx context.Context, args *Args) (err error) { - if err := doCreateDeployment(ctx, args, false, 1); err != nil { +func (t deploymentTask) createDeployment(ctx context.Context, args *Args) (err error) { + if err := t.doCreateDeployment(ctx, args, false, 1); err != nil { return err } if args.Legacy { - if err := doCreateDeployment(ctx, args, true, 2); err != nil { + if err := t.doCreateDeployment(ctx, args, true, 2); err != nil { return err } } @@ -271,8 +273,8 @@ func removeFinalizer(objectMeta *metav1.ObjectMeta, finalizer string) []string { return finalizers } -func doDeleteDeployment(ctx context.Context, name string) error { - deploymentClient := k8s.KubeClient().AppsV1().Deployments(namespace) +func (t deploymentTask) doDeleteDeployment(ctx context.Context, name string) error { + deploymentClient := t.client.Kube().AppsV1().Deployments(namespace) deployment, err := deploymentClient.Get(ctx, name, metav1.GetOptions{}) if err != nil { @@ -294,10 +296,10 @@ func doDeleteDeployment(ctx context.Context, name string) error { return nil } -func deleteDeployment(ctx context.Context) error { - if err := doDeleteDeployment(ctx, consts.ControllerServerName); err != nil { +func (t deploymentTask) deleteDeployment(ctx context.Context) error { + if err := t.doDeleteDeployment(ctx, consts.ControllerServerName); err != nil { return err } - return doDeleteDeployment(ctx, consts.LegacyControllerServerName) + return t.doDeleteDeployment(ctx, consts.LegacyControllerServerName) } diff --git a/pkg/installer/directpv.min.io_directpvdrives.yaml b/pkg/admin/installer/directpv.min.io_directpvdrives.yaml similarity index 100% rename from pkg/installer/directpv.min.io_directpvdrives.yaml rename to pkg/admin/installer/directpv.min.io_directpvdrives.yaml diff --git a/pkg/installer/directpv.min.io_directpvinitrequests.yaml b/pkg/admin/installer/directpv.min.io_directpvinitrequests.yaml similarity index 100% rename from pkg/installer/directpv.min.io_directpvinitrequests.yaml rename to pkg/admin/installer/directpv.min.io_directpvinitrequests.yaml diff --git a/pkg/installer/directpv.min.io_directpvnodes.yaml b/pkg/admin/installer/directpv.min.io_directpvnodes.yaml similarity index 100% rename from pkg/installer/directpv.min.io_directpvnodes.yaml rename to pkg/admin/installer/directpv.min.io_directpvnodes.yaml diff --git a/pkg/installer/directpv.min.io_directpvvolumes.yaml b/pkg/admin/installer/directpv.min.io_directpvvolumes.yaml similarity index 100% rename from pkg/installer/directpv.min.io_directpvvolumes.yaml rename to pkg/admin/installer/directpv.min.io_directpvvolumes.yaml diff --git a/pkg/installer/installer.go b/pkg/admin/installer/installer.go similarity index 51% rename from pkg/installer/installer.go rename to pkg/admin/installer/installer.go index 1afde102..8f54307b 100644 --- a/pkg/installer/installer.go +++ b/pkg/admin/installer/installer.go @@ -18,61 +18,30 @@ package installer import ( "context" - "fmt" - "strconv" - "strings" "github.com/fatih/color" - "github.com/minio/directpv/pkg/k8s" + "github.com/minio/directpv/pkg/client" + legacyclient "github.com/minio/directpv/pkg/legacy/client" "github.com/minio/directpv/pkg/utils" - "k8s.io/apimachinery/pkg/util/version" - "k8s.io/klog/v2" ) -// Tasks is a list of tasks to performed during installation and uninstallation -var Tasks = []Task{ - namespaceTask{}, - rbacTask{}, - pspTask{}, - crdTask{}, - migrateTask{}, - csiDriverTask{}, - storageClassTask{}, - daemonsetTask{}, - deploymentTask{}, -} - -func getKubeVersion() (major, minor uint, err error) { - versionInfo, err := k8s.DiscoveryClient().ServerVersion() - if err != nil { - return 0, 0, err - } - - var u64 uint64 - if u64, err = strconv.ParseUint(versionInfo.Major, 10, 64); err != nil { - return 0, 0, fmt.Errorf("unable to parse major version %v; %v", versionInfo.Major, err) - } - major = uint(u64) - - minorString := versionInfo.Minor - if strings.Contains(versionInfo.GitVersion, "-eks-") { - // Do trimming only for EKS. - // Refer https://github.com/aws/containers-roadmap/issues/1404 - i := strings.IndexFunc(minorString, func(r rune) bool { return r < '0' || r > '9' }) - if i > -1 { - minorString = minorString[:i] - } +// GetTasks returns the installer tasks to be run +func GetTasks(client *client.Client, legacyClient *legacyclient.Client) []Task { + return []Task{ + namespaceTask{client}, + rbacTask{client}, + pspTask{client}, + crdTask{client}, + migrateTask{client, legacyClient}, + csiDriverTask{client}, + storageClassTask{client}, + daemonsetTask{client}, + deploymentTask{client}, } - if u64, err = strconv.ParseUint(minorString, 10, 64); err != nil { - return 0, 0, fmt.Errorf("unable to parse minor version %v; %v", minor, err) - } - minor = uint(u64) - - return major, minor, nil } // Install performs DirectPV installation on kubernetes. -func Install(ctx context.Context, args *Args) (err error) { +func Install(ctx context.Context, args *Args, tasks []Task) (err error) { defer func() { if !sendDoneMessage(ctx, args.ProgressCh, err) { err = errSendProgress @@ -84,25 +53,6 @@ func Install(ctx context.Context, args *Args) (err error) { return err } - switch { - case args.DryRun: - if args.KubeVersion == nil { - // default higher version - if args.KubeVersion, err = version.ParseSemantic("1.27.0"); err != nil { - klog.Fatalf("this should not happen; %v", err) - } - } - default: - major, minor, err := getKubeVersion() - if err != nil { - return err - } - args.KubeVersion, err = version.ParseSemantic(fmt.Sprintf("%v.%v.0", major, minor)) - if err != nil { - klog.Fatalf("this should not happen; %v", err) - } - } - if args.KubeVersion.Major() == 1 { if args.KubeVersion.Minor() < 20 { args.csiProvisionerImage = csiProvisionerImageV2_2_0 @@ -127,7 +77,7 @@ func Install(ctx context.Context, args *Args) (err error) { } } - for _, task := range Tasks { + for _, task := range tasks { if err := task.Start(ctx, args); err != nil { return err } @@ -141,12 +91,12 @@ func Install(ctx context.Context, args *Args) (err error) { } // Uninstall removes DirectPV from kubernetes. -func Uninstall(ctx context.Context, quiet, force bool) (err error) { +func Uninstall(ctx context.Context, quiet, force bool, tasks []Task) (err error) { args := &Args{ ForceUninstall: force, Quiet: quiet, } - for _, task := range Tasks { + for _, task := range tasks { if err := task.Delete(ctx, args); err != nil { return err } diff --git a/pkg/installer/installer_test.go b/pkg/admin/installer/installer_test.go similarity index 60% rename from pkg/installer/installer_test.go rename to pkg/admin/installer/installer_test.go index bdbae0d0..75c7d78a 100644 --- a/pkg/installer/installer_test.go +++ b/pkg/admin/installer/installer_test.go @@ -23,12 +23,15 @@ import ( "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/k8s" + legacyclient "github.com/minio/directpv/pkg/legacy/client" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + versionpkg "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/version" ) func init() { client.FakeInit() + legacyclient.FakeInit() } var ( @@ -93,57 +96,15 @@ func getDiscoveryGroupsAndMethods() ([]*metav1.APIGroup, []*metav1.APIResourceLi return apiGroups, apiResourceList, nil } -func TestGetKubeVersion(t *testing.T) { - testCases := []struct { - info version.Info - major uint - minor uint - expectErr bool - }{ - {version.Info{Major: "a", Minor: "0"}, 0, 0, true}, // invalid major - {version.Info{Major: "-1", Minor: "0"}, 0, 0, true}, // invalid major - {version.Info{Major: "0", Minor: "a"}, 0, 0, true}, // invalid minor - {version.Info{Major: "0", Minor: "-1"}, 0, 0, true}, // invalid minor - {version.Info{Major: "0", Minor: "-1", GitVersion: "commit-eks-id"}, 0, 0, true}, // invalid minor for eks - {version.Info{Major: "0", Minor: "incompat", GitVersion: "commit-eks-"}, 0, 0, true}, // invalid minor for eks - {version.Info{Major: "0", Minor: "0"}, 0, 0, false}, - {version.Info{Major: "1", Minor: "0"}, 1, 0, false}, - {version.Info{Major: "0", Minor: "1"}, 0, 1, false}, - {version.Info{Major: "1", Minor: "18"}, 1, 18, false}, - {version.Info{Major: "1", Minor: "18+", GitVersion: "commit-eks-id"}, 1, 18, false}, - {version.Info{Major: "1", Minor: "18-", GitVersion: "commit-eks-id"}, 1, 18, false}, - {version.Info{Major: "1", Minor: "18incompat", GitVersion: "commit-eks-id"}, 1, 18, false}, - {version.Info{Major: "1", Minor: "18-incompat", GitVersion: "commit-eks-id"}, 1, 18, false}, - } - - for i, testCase := range testCases { - k8s.SetDiscoveryInterface(getDiscoveryGroupsAndMethods, &testCase.info) - major, minor, err := getKubeVersion() - if testCase.expectErr { - if err == nil { - t.Fatalf("case %v: expected error, but succeeded", i+1) - } - continue - } - - if err != nil { - t.Fatalf("case %v: unexpected error: %v", i+1, err) - } - - if major != testCase.major { - t.Fatalf("case %v: major: expected: %v, got: %v", i+1, testCase.major, major) - } - - if minor != testCase.minor { - t.Fatalf("case %v: minor: expected: %v, got: %v", i+1, testCase.minor, minor) - } - } -} - func TestInstallUinstall(t *testing.T) { + kversion, err := versionpkg.ParseSemantic("1.26.0") + if err != nil { + t.Fatalf("unable to parse version; %v", err) + } args := Args{ image: "directpv-0.0.0dev0", ObjectWriter: io.Discard, + KubeVersion: kversion, } testVersions := []version.Info{ @@ -161,16 +122,18 @@ func TestInstallUinstall(t *testing.T) { } for i, testVersion := range testVersions { - k8s.SetDiscoveryInterface(getDiscoveryGroupsAndMethods, &testVersion) + client := client.GetClient() + legacyClient := legacyclient.GetClient() + client.K8sClient.DiscoveryClient = k8s.NewFakeDiscovery(getDiscoveryGroupsAndMethods, &testVersion) ctx := context.TODO() args := args - if err := Install(ctx, &args); err != nil { + tasks := GetTasks(client, legacyClient) + if err := Install(ctx, &args, tasks); err != nil { t.Fatalf("case %v: unexpected error; %v", i+1, err) } - if err := Uninstall(ctx, false, true); err != nil { + if err := Uninstall(ctx, false, true, tasks); err != nil { t.Fatalf("csae %v: unexpected error; %v", i+1, err) } - _, err := k8s.KubeClient().CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) if err == nil { t.Fatalf("case %v: uninstall on kube version v%v.%v not removed namespace", i+1, testVersion.Major, testVersion.Minor) diff --git a/pkg/installer/migrate.go b/pkg/admin/installer/migrate.go similarity index 74% rename from pkg/installer/migrate.go rename to pkg/admin/installer/migrate.go index bc53f77b..8b3b1a83 100644 --- a/pkg/installer/migrate.go +++ b/pkg/admin/installer/migrate.go @@ -19,7 +19,6 @@ package installer import ( "context" "fmt" - "os" "strings" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" @@ -47,7 +46,10 @@ const ( legacyPurgeProtection = legacyclient.GroupName + "/purge-protection" ) -type migrateTask struct{} +type migrateTask struct { + client *client.Client + legacyClient *legacyclient.Client +} func (migrateTask) Name() string { return "Migration" @@ -67,15 +69,15 @@ func (migrateTask) End(ctx context.Context, args *Args, err error) error { return nil } -func (migrateTask) Execute(ctx context.Context, args *Args) error { - return Migrate(ctx, args, true) +func (t migrateTask) Execute(ctx context.Context, args *Args) error { + return t.migrate(ctx, args, true) } func (migrateTask) Delete(_ context.Context, _ *Args) error { return nil } -func migrateDrives(ctx context.Context, dryRun bool, progressCh chan<- Message) (driveMap map[string]string, legacyDriveErrors, driveErrors map[string]error, err error) { +func (t migrateTask) migrateDrives(ctx context.Context, dryRun bool, progressCh chan<- Message) (driveMap map[string]string, legacyDriveErrors, driveErrors map[string]error, err error) { ctx, cancelFunc := context.WithCancel(ctx) defer cancelFunc() @@ -84,7 +86,7 @@ func migrateDrives(ctx context.Context, dryRun bool, progressCh chan<- Message) driveErrors = map[string]error{} fsUUIDs := make(utils.StringSet) - for result := range legacyclient.ListDrives(ctx) { + for result := range t.legacyClient.ListDrives(ctx) { if result.Err != nil { return nil, legacyDriveErrors, driveErrors, fmt.Errorf( "unable to get legacy drives; %v", result.Err, @@ -181,13 +183,13 @@ func migrateDrives(ctx context.Context, dryRun bool, progressCh chan<- Message) } } - existingDrive, err := client.DriveClient().Get(ctx, string(driveID), metav1.GetOptions{}) + existingDrive, err := t.client.Drive().Get(ctx, string(driveID), metav1.GetOptions{}) if err != nil { switch { case apierrors.IsNotFound(err): if !dryRun { sendProgressMessage(ctx, progressCh, fmt.Sprintf("Migrating directcsidrive %s to directpvdrive %s", result.Drive.Name, drive.Name), 1, nil) - _, err = client.DriveClient().Create(ctx, drive, metav1.CreateOptions{}) + _, err = t.client.Drive().Create(ctx, drive, metav1.CreateOptions{}) if err != nil { legacyDriveErrors[result.Drive.Name] = fmt.Errorf( "unable to create drive %v by migrating legacy drive %v; %w", @@ -219,14 +221,14 @@ func migrateDrives(ctx context.Context, dryRun bool, progressCh chan<- Message) return driveMap, legacyDriveErrors, driveErrors, nil } -func migrateVolumes(ctx context.Context, driveMap map[string]string, dryRun bool, progressCh chan<- Message) (legacyVolumeErrors, volumeErrors map[string]error, err error) { +func (t migrateTask) migrateVolumes(ctx context.Context, driveMap map[string]string, dryRun bool, progressCh chan<- Message) (legacyVolumeErrors, volumeErrors map[string]error, err error) { ctx, cancelFunc := context.WithCancel(ctx) defer cancelFunc() legacyVolumeErrors = map[string]error{} volumeErrors = map[string]error{} - for result := range legacyclient.ListVolumes(ctx) { + for result := range t.legacyClient.ListVolumes(ctx) { if result.Err != nil { return legacyVolumeErrors, volumeErrors, fmt.Errorf( "unable to get legacy volumes; %v", result.Err, @@ -296,13 +298,13 @@ func migrateVolumes(ctx context.Context, driveMap map[string]string, dryRun bool } } - existingVolume, err := client.VolumeClient().Get(ctx, name, metav1.GetOptions{}) + existingVolume, err := t.client.Volume().Get(ctx, name, metav1.GetOptions{}) if err != nil { switch { case apierrors.IsNotFound(err): if !dryRun { sendProgressMessage(ctx, progressCh, fmt.Sprintf("Migrating directcsivolume %s to directpvvolume %s", result.Volume.Name, volume.Name), 2, nil) - _, err = client.VolumeClient().Create(ctx, volume, metav1.CreateOptions{}) + _, err = t.client.Volume().Create(ctx, volume, metav1.CreateOptions{}) if err != nil { legacyVolumeErrors[result.Volume.Name] = fmt.Errorf( "unable to create volume %v by migrating legacy volume %v; %w", @@ -332,14 +334,14 @@ func migrateVolumes(ctx context.Context, driveMap map[string]string, dryRun bool } // Migrate migrates legacy drives and volumes. -func Migrate(ctx context.Context, args *Args, installer bool) (err error) { +func (t migrateTask) migrate(ctx context.Context, args *Args, installer bool) (err error) { if (installer && args.DryRun) || args.Declarative || !args.Legacy { return nil } legacyclient.Init() - version, _, err := legacyclient.GetGroupVersion("DirectCSIDrive") + version, _, err := legacyclient.GetGroupVersion(t.legacyClient.K8sClient, "DirectCSIDrive") if err != nil { return fmt.Errorf("unable to probe DirectCSIDrive version; %w", err) } @@ -350,7 +352,7 @@ func Migrate(ctx context.Context, args *Args, installer bool) (err error) { return fmt.Errorf("migration does not support DirectCSIDrive version %v", version) } - version, _, err = legacyclient.GetGroupVersion("DirectCSIVolume") + version, _, err = legacyclient.GetGroupVersion(t.legacyClient.K8sClient, "DirectCSIVolume") if err != nil { return fmt.Errorf("unable to probe DirectCSIVolume version; %w", err) } @@ -361,14 +363,14 @@ func Migrate(ctx context.Context, args *Args, installer bool) (err error) { return fmt.Errorf("migration does not support DirectCSIVolume version %v", version) } - driveMap, legacyDriveErrors, driveErrors, err := migrateDrives(ctx, args.DryRun, args.ProgressCh) + driveMap, legacyDriveErrors, driveErrors, err := t.migrateDrives(ctx, args.DryRun, args.ProgressCh) if err != nil { return err } if !sendProgressMessage(ctx, args.ProgressCh, "Migrated the drives", 1, nil) { return errSendProgress } - legacyVolumeErrors, volumeErrors, err := migrateVolumes(ctx, driveMap, args.DryRun, args.ProgressCh) + legacyVolumeErrors, volumeErrors, err := t.migrateVolumes(ctx, driveMap, args.DryRun, args.ProgressCh) if err != nil { return err } @@ -403,84 +405,7 @@ func Migrate(ctx context.Context, args *Args, installer bool) (err error) { return nil } -// RemoveLegacyDrives removes legacy drive CRDs. -func RemoveLegacyDrives(ctx context.Context, backupFile string) (backupCreated bool, err error) { - var drives []directv1beta5.DirectCSIDrive - for result := range legacyclient.ListDrives(ctx) { - if result.Err != nil { - return false, fmt.Errorf("unable to get legacy drives; %w", result.Err) - } - drives = append(drives, result.Drive) - } - if len(drives) == 0 { - return false, nil - } - - data, err := utils.ToYAML(directv1beta5.DirectCSIDriveList{ - TypeMeta: metav1.TypeMeta{ - Kind: "List", - APIVersion: "v1", - }, - Items: drives, - }) - if err != nil { - return false, fmt.Errorf("unable to generate legacy drives YAML; %w", err) - } - - if err = os.WriteFile(backupFile, data, os.ModePerm); err != nil { - return false, fmt.Errorf("unable to write legacy drives YAML; %w", err) - } - - for _, drive := range drives { - drive.Finalizers = []string{} - if _, err := legacyclient.DriveClient().Update(ctx, &drive, metav1.UpdateOptions{}); err != nil { - return false, fmt.Errorf("unable to update legacy drive %v; %w", drive.Name, err) - } - if err := legacyclient.DriveClient().Delete(ctx, drive.Name, metav1.DeleteOptions{}); err != nil { - return false, fmt.Errorf("unable to remove legacy drive %v; %w", drive.Name, err) - } - } - - return true, nil -} - -// RemoveLegacyVolumes removes legacy volume CRDs. -func RemoveLegacyVolumes(ctx context.Context, backupFile string) (backupCreated bool, err error) { - var volumes []directv1beta5.DirectCSIVolume - for result := range legacyclient.ListVolumes(ctx) { - if result.Err != nil { - return false, fmt.Errorf("unable to get legacy volumes; %w", result.Err) - } - volumes = append(volumes, result.Volume) - } - if len(volumes) == 0 { - return false, nil - } - - data, err := utils.ToYAML(directv1beta5.DirectCSIVolumeList{ - TypeMeta: metav1.TypeMeta{ - Kind: "List", - APIVersion: "v1", - }, - Items: volumes, - }) - if err != nil { - return false, fmt.Errorf("unable to generate legacy volumes YAML; %w", err) - } - - if err = os.WriteFile(backupFile, data, os.ModePerm); err != nil { - return false, fmt.Errorf("unable to write legacy volumes YAML; %w", err) - } - - for _, volume := range volumes { - volume.Finalizers = nil - if _, err := legacyclient.VolumeClient().Update(ctx, &volume, metav1.UpdateOptions{}); err != nil { - return false, fmt.Errorf("unable to update legacy volume %v; %w", volume.Name, err) - } - if err := legacyclient.VolumeClient().Delete(ctx, volume.Name, metav1.DeleteOptions{}); err != nil { - return false, fmt.Errorf("unable to remove legacy volume %v; %w", volume.Name, err) - } - } - - return true, nil +// Migrate migrates the resources using the provided clients +func Migrate(ctx context.Context, args *Args, client *client.Client, legacyClient *legacyclient.Client) error { + return migrateTask{client, legacyClient}.migrate(ctx, args, false) } diff --git a/pkg/installer/migrate_test.go b/pkg/admin/installer/migrate_test.go similarity index 94% rename from pkg/installer/migrate_test.go rename to pkg/admin/installer/migrate_test.go index 5aa2ab0e..c52e62f3 100644 --- a/pkg/installer/migrate_test.go +++ b/pkg/admin/installer/migrate_test.go @@ -41,7 +41,8 @@ func TestMigrateDrivesError(t *testing.T) { } legacyclient.SetDriveClient(legacyclientsetfake.NewSimpleClientset(drive)) - driveMap, legacyDriveErrors, driveErrors, err := migrateDrives(context.TODO(), false, nil) + task := migrateTask{client.GetClient(), legacyclient.GetClient()} + driveMap, legacyDriveErrors, driveErrors, err := task.migrateDrives(context.TODO(), false, nil) if len(driveMap) == 0 && len(legacyDriveErrors) == 0 && len(driveErrors) == 0 && err == nil { t.Fatalf("expected error, but succeeded\n") } @@ -55,7 +56,7 @@ func TestMigrateDrivesError(t *testing.T) { }, } legacyclient.SetDriveClient(legacyclientsetfake.NewSimpleClientset(drive)) - driveMap, legacyDriveErrors, driveErrors, err = migrateDrives(context.TODO(), false, nil) + driveMap, legacyDriveErrors, driveErrors, err = task.migrateDrives(context.TODO(), false, nil) if len(driveMap) == 0 && len(legacyDriveErrors) == 0 && len(driveErrors) == 0 && err == nil { t.Fatalf("expected error, but succeeded\n") } @@ -78,7 +79,7 @@ func TestMigrateDrivesError(t *testing.T) { }, } legacyclient.SetDriveClient(legacyclientsetfake.NewSimpleClientset(drive1, drive2)) - driveMap, legacyDriveErrors, driveErrors, err = migrateDrives(context.TODO(), false, nil) + driveMap, legacyDriveErrors, driveErrors, err = task.migrateDrives(context.TODO(), false, nil) if len(driveMap) == 0 && len(legacyDriveErrors) == 0 && len(driveErrors) == 0 && err == nil { t.Fatalf("expected error, but succeeded\n") } @@ -88,8 +89,8 @@ func TestMigrateNoDrives(t *testing.T) { legacyclient.SetDriveClient(legacyclientsetfake.NewSimpleClientset()) clientset := types.NewExtFakeClientset(clientsetfake.NewSimpleClientset()) client.SetDriveInterface(clientset.DirectpvLatest().DirectPVDrives()) - - _, legacyDriveErrors, driveErrors, err := migrateDrives(context.TODO(), false, nil) + task := migrateTask{client.GetClient(), legacyclient.GetClient()} + _, legacyDriveErrors, driveErrors, err := task.migrateDrives(context.TODO(), false, nil) if len(legacyDriveErrors) != 0 || len(driveErrors) != 0 || err != nil { t.Fatalf("unexpected error; %v\n", err) } @@ -115,7 +116,7 @@ func TestMigrateNoDrives(t *testing.T) { clientset = types.NewExtFakeClientset(clientsetfake.NewSimpleClientset()) client.SetDriveInterface(clientset.DirectpvLatest().DirectPVDrives()) - _, legacyDriveErrors, driveErrors, err = migrateDrives(context.TODO(), false, nil) + _, legacyDriveErrors, driveErrors, err = task.migrateDrives(context.TODO(), false, nil) if len(legacyDriveErrors) != 0 || len(driveErrors) != 0 || err != nil { t.Fatalf("unexpected error; %v\n", err) } @@ -204,8 +205,8 @@ func TestMigrateReadyDrive(t *testing.T) { legacyclient.SetDriveClient(legacyclientsetfake.NewSimpleClientset(drive)) clientset := types.NewExtFakeClientset(clientsetfake.NewSimpleClientset()) client.SetDriveInterface(clientset.DirectpvLatest().DirectPVDrives()) - - driveMap, legacyDriveErrors, driveErrors, err := migrateDrives(context.TODO(), false, nil) + task := migrateTask{client.GetClient(), legacyclient.GetClient()} + driveMap, legacyDriveErrors, driveErrors, err := task.migrateDrives(context.TODO(), false, nil) if len(legacyDriveErrors) != 0 || len(driveErrors) != 0 || err != nil { t.Fatalf("unexpected error; %v, %v, %v\n", legacyDriveErrors, driveErrors, err) } @@ -340,8 +341,8 @@ func TestMigrateInUseDrive(t *testing.T) { legacyclient.SetDriveClient(legacyclientsetfake.NewSimpleClientset(drive)) clientset := types.NewExtFakeClientset(clientsetfake.NewSimpleClientset()) client.SetDriveInterface(clientset.DirectpvLatest().DirectPVDrives()) - - driveMap, legacyDriveErrors, driveErrors, err := migrateDrives(context.TODO(), false, nil) + task := migrateTask{client.GetClient(), legacyclient.GetClient()} + driveMap, legacyDriveErrors, driveErrors, err := task.migrateDrives(context.TODO(), false, nil) if len(legacyDriveErrors) != 0 || len(driveErrors) != 0 || err != nil { t.Fatalf("unexpected error; %v, %v, %v\n", legacyDriveErrors, driveErrors, err) } @@ -451,8 +452,8 @@ func TestMigrateVolumes(t *testing.T) { legacyclient.SetVolumeClient(legacyclientsetfake.NewSimpleClientset(volume)) clientset := types.NewExtFakeClientset(clientsetfake.NewSimpleClientset()) client.SetVolumeInterface(clientset.DirectpvLatest().DirectPVVolumes()) - - legacyVolumeErrors, volumeErrors, err := migrateVolumes( + task := migrateTask{client.GetClient(), legacyclient.GetClient()} + legacyVolumeErrors, volumeErrors, err := task.migrateVolumes( context.TODO(), map[string]string{"08450612-7ab3-40f9-ab83-38645fba6d29": "a9908089-96dd-4e8b-8f06-0c0b5e391f39"}, false, @@ -492,7 +493,7 @@ func TestMigrateVolumes(t *testing.T) { // no fsuuid found error legacyclient.SetVolumeClient(legacyclientsetfake.NewSimpleClientset(volume)) - legacyVolumeErrors, volumeErrors, err = migrateVolumes(context.TODO(), map[string]string{}, false, nil) + legacyVolumeErrors, volumeErrors, err = task.migrateVolumes(context.TODO(), map[string]string{}, false, nil) if len(legacyVolumeErrors) == 0 && len(volumeErrors) == 0 && err == nil { t.Fatalf("expected error; but succeeded\n") } diff --git a/pkg/installer/namespace.go b/pkg/admin/installer/namespace.go similarity index 82% rename from pkg/installer/namespace.go rename to pkg/admin/installer/namespace.go index feeb624d..5e7a1fb8 100644 --- a/pkg/installer/namespace.go +++ b/pkg/admin/installer/namespace.go @@ -20,14 +20,16 @@ import ( "context" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" - "github.com/minio/directpv/pkg/k8s" + "github.com/minio/directpv/pkg/client" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" podsecurityadmissionapi "k8s.io/pod-security-admission/api" ) -type namespaceTask struct{} +type namespaceTask struct { + client *client.Client +} func (namespaceTask) Name() string { return "Namespace" @@ -40,8 +42,8 @@ func (namespaceTask) Start(ctx context.Context, args *Args) error { return nil } -func (namespaceTask) Execute(ctx context.Context, args *Args) error { - return createNamespace(ctx, args) +func (t namespaceTask) Execute(ctx context.Context, args *Args) error { + return t.createNamespace(ctx, args) } func (namespaceTask) End(ctx context.Context, args *Args, err error) error { @@ -51,11 +53,11 @@ func (namespaceTask) End(ctx context.Context, args *Args, err error) error { return nil } -func (namespaceTask) Delete(ctx context.Context, _ *Args) error { - return deleteNamespace(ctx) +func (t namespaceTask) Delete(ctx context.Context, _ *Args) error { + return t.deleteNamespace(ctx) } -func createNamespace(ctx context.Context, args *Args) (err error) { +func (t namespaceTask) createNamespace(ctx context.Context, args *Args) (err error) { if !sendProgressMessage(ctx, args.ProgressCh, "Creating namespace", 1, nil) { return errSendProgress } @@ -99,7 +101,7 @@ func createNamespace(ctx context.Context, args *Args) (err error) { } if !args.DryRun && !args.Declarative { - _, err = k8s.KubeClient().CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + _, err = t.client.Kube().CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) if err != nil && !apierrors.IsAlreadyExists(err) { return err } @@ -108,9 +110,9 @@ func createNamespace(ctx context.Context, args *Args) (err error) { return args.writeObject(ns) } -func deleteNamespace(ctx context.Context) error { +func (t namespaceTask) deleteNamespace(ctx context.Context) error { propagationPolicy := metav1.DeletePropagationForeground - err := k8s.KubeClient().CoreV1().Namespaces().Delete( + err := t.client.Kube().CoreV1().Namespaces().Delete( ctx, namespace, metav1.DeleteOptions{PropagationPolicy: &propagationPolicy}, ) if err != nil { diff --git a/pkg/installer/progress.go b/pkg/admin/installer/progress.go similarity index 100% rename from pkg/installer/progress.go rename to pkg/admin/installer/progress.go diff --git a/pkg/installer/psp.go b/pkg/admin/installer/psp.go similarity index 82% rename from pkg/installer/psp.go rename to pkg/admin/installer/psp.go index 9af06d56..9efa3e77 100644 --- a/pkg/installer/psp.go +++ b/pkg/admin/installer/psp.go @@ -21,8 +21,8 @@ import ( "errors" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" + "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/consts" - "github.com/minio/directpv/pkg/k8s" corev1 "k8s.io/api/core/v1" policy "k8s.io/api/policy/v1beta1" rbac "k8s.io/api/rbac/v1" @@ -35,7 +35,9 @@ const pspClusterRoleBindingName = "psp-" + consts.Identity var errPSPUnsupported = errors.New("pod security policy is not supported in your kubernetes version") -type pspTask struct{} +type pspTask struct { + client *client.Client +} func (pspTask) Name() string { return "PSP" @@ -55,12 +57,12 @@ func (pspTask) End(ctx context.Context, args *Args, err error) error { return nil } -func (pspTask) Execute(ctx context.Context, args *Args) error { - return createPSP(ctx, args) +func (t pspTask) Execute(ctx context.Context, args *Args) error { + return t.createPSP(ctx, args) } -func (pspTask) Delete(ctx context.Context, _ *Args) error { - major, minor, err := getKubeVersion() +func (t pspTask) Delete(ctx context.Context, _ *Args) error { + major, minor, err := t.client.K8s().GetKubeVersion() if err != nil { return err } @@ -68,10 +70,10 @@ func (pspTask) Delete(ctx context.Context, _ *Args) error { if podSecurityAdmission { return nil } - return deletePSP(ctx) + return t.deletePSP(ctx) } -func createPSPClusterRoleBinding(ctx context.Context, args *Args) (err error) { +func (t pspTask) createPSPClusterRoleBinding(ctx context.Context, args *Args) (err error) { if !sendProgressMessage(ctx, args.ProgressCh, "Creating psp cluster role binding", 2, nil) { return errSendProgress } @@ -110,7 +112,7 @@ func createPSPClusterRoleBinding(ctx context.Context, args *Args) (err error) { } if !args.DryRun && !args.Declarative { - _, err = k8s.KubeClient().RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{}) + _, err = t.client.Kube().RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{}) if err != nil && !apierrors.IsAlreadyExists(err) { return err } @@ -119,7 +121,7 @@ func createPSPClusterRoleBinding(ctx context.Context, args *Args) (err error) { return args.writeObject(crb) } -func createPodSecurityPolicy(ctx context.Context, args *Args) (err error) { +func (t pspTask) createPodSecurityPolicy(ctx context.Context, args *Args) (err error) { if !sendProgressMessage(ctx, args.ProgressCh, "Creating pod security policy", 1, nil) { return errSendProgress } @@ -173,7 +175,7 @@ func createPodSecurityPolicy(ctx context.Context, args *Args) (err error) { } if !args.DryRun && !args.Declarative { - _, err = k8s.KubeClient().PolicyV1beta1().PodSecurityPolicies().Create( + _, err = t.client.Kube().PolicyV1beta1().PodSecurityPolicies().Create( ctx, psp, metav1.CreateOptions{}, ) if err != nil && !apierrors.IsAlreadyExists(err) { @@ -184,14 +186,14 @@ func createPodSecurityPolicy(ctx context.Context, args *Args) (err error) { return args.writeObject(psp) } -func createPSP(ctx context.Context, args *Args) error { +func (t pspTask) createPSP(ctx context.Context, args *Args) error { if args.podSecurityAdmission { return nil } var gvk *schema.GroupVersionKind if !args.DryRun { var err error - if gvk, err = k8s.GetGroupVersionKind("policy", "PodSecurityPolicy", "v1beta1"); err != nil { + if gvk, err = t.client.K8s().GetGroupVersionKind("policy", "PodSecurityPolicy", "v1beta1"); err != nil { return err } } else { @@ -199,24 +201,24 @@ func createPSP(ctx context.Context, args *Args) error { } if gvk.Version == "v1beta1" { - if err := createPodSecurityPolicy(ctx, args); err != nil { + if err := t.createPodSecurityPolicy(ctx, args); err != nil { return err } - return createPSPClusterRoleBinding(ctx, args) + return t.createPSPClusterRoleBinding(ctx, args) } return errPSPUnsupported } -func deletePSP(ctx context.Context) error { - err := k8s.KubeClient().RbacV1().ClusterRoleBindings().Delete( +func (t pspTask) deletePSP(ctx context.Context) error { + err := t.client.Kube().RbacV1().ClusterRoleBindings().Delete( ctx, pspClusterRoleBindingName, metav1.DeleteOptions{}, ) if err != nil && !apierrors.IsNotFound(err) { return err } - err = k8s.KubeClient().PolicyV1beta1().PodSecurityPolicies().Delete( + err = t.client.Kube().PolicyV1beta1().PodSecurityPolicies().Delete( ctx, consts.Identity, metav1.DeleteOptions{}, ) if err != nil && !apierrors.IsNotFound(err) { diff --git a/pkg/installer/rbac.go b/pkg/admin/installer/rbac.go similarity index 85% rename from pkg/installer/rbac.go rename to pkg/admin/installer/rbac.go index 190b653c..4bd1773b 100644 --- a/pkg/installer/rbac.go +++ b/pkg/admin/installer/rbac.go @@ -20,8 +20,8 @@ import ( "context" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" + "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/consts" - "github.com/minio/directpv/pkg/k8s" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -50,7 +50,9 @@ func newPolicyRule(resources []string, apiGroups []string, verbs ...string) rbac } } -type rbacTask struct{} +type rbacTask struct { + client *client.Client +} func (rbacTask) Name() string { return "RBAC" @@ -70,15 +72,15 @@ func (rbacTask) End(ctx context.Context, args *Args, err error) error { return nil } -func (rbacTask) Execute(ctx context.Context, args *Args) error { - return createRBAC(ctx, args) +func (t rbacTask) Execute(ctx context.Context, args *Args) error { + return t.createRBAC(ctx, args) } -func (rbacTask) Delete(ctx context.Context, _ *Args) error { - return deleteRBAC(ctx) +func (t rbacTask) Delete(ctx context.Context, _ *Args) error { + return t.deleteRBAC(ctx) } -func createServiceAccount(ctx context.Context, args *Args) (err error) { +func (t rbacTask) createServiceAccount(ctx context.Context, args *Args) (err error) { if !sendProgressMessage(ctx, args.ProgressCh, "Creating service account", 1, nil) { return errSendProgress } @@ -108,7 +110,7 @@ func createServiceAccount(ctx context.Context, args *Args) (err error) { } if !args.DryRun && !args.Declarative { - _, err = k8s.KubeClient().CoreV1().ServiceAccounts(namespace).Create( + _, err = t.client.Kube().CoreV1().ServiceAccounts(namespace).Create( ctx, serviceAccount, metav1.CreateOptions{}, ) if err != nil && !apierrors.IsAlreadyExists(err) { @@ -119,7 +121,7 @@ func createServiceAccount(ctx context.Context, args *Args) (err error) { return args.writeObject(serviceAccount) } -func createClusterRole(ctx context.Context, args *Args) (err error) { +func (t rbacTask) createClusterRole(ctx context.Context, args *Args) (err error) { if !sendProgressMessage(ctx, args.ProgressCh, "Creating cluster role", 2, nil) { return errSendProgress } @@ -175,7 +177,7 @@ func createClusterRole(ctx context.Context, args *Args) (err error) { } if !args.DryRun && !args.Declarative { - _, err = k8s.KubeClient().RbacV1().ClusterRoles().Create( + _, err = t.client.Kube().RbacV1().ClusterRoles().Create( ctx, clusterRole, metav1.CreateOptions{}, ) if err != nil && !apierrors.IsAlreadyExists(err) { @@ -186,7 +188,7 @@ func createClusterRole(ctx context.Context, args *Args) (err error) { return args.writeObject(clusterRole) } -func createClusterRoleBinding(ctx context.Context, args *Args) (err error) { +func (t rbacTask) createClusterRoleBinding(ctx context.Context, args *Args) (err error) { if !sendProgressMessage(ctx, args.ProgressCh, "Creating cluster role binding", 3, nil) { return errSendProgress } @@ -226,7 +228,7 @@ func createClusterRoleBinding(ctx context.Context, args *Args) (err error) { } if !args.DryRun && !args.Declarative { - _, err = k8s.KubeClient().RbacV1().ClusterRoleBindings().Create( + _, err = t.client.Kube().RbacV1().ClusterRoleBindings().Create( ctx, clusterRoleBinding, metav1.CreateOptions{}, ) if err != nil && !apierrors.IsAlreadyExists(err) { @@ -237,7 +239,7 @@ func createClusterRoleBinding(ctx context.Context, args *Args) (err error) { return args.writeObject(clusterRoleBinding) } -func createRole(ctx context.Context, args *Args) (err error) { +func (t rbacTask) createRole(ctx context.Context, args *Args) (err error) { if !sendProgressMessage(ctx, args.ProgressCh, "Creating role", 4, nil) { return errSendProgress } @@ -268,7 +270,7 @@ func createRole(ctx context.Context, args *Args) (err error) { } if !args.DryRun && !args.Declarative { - _, err = k8s.KubeClient().RbacV1().Roles(namespace).Create( + _, err = t.client.Kube().RbacV1().Roles(namespace).Create( ctx, role, metav1.CreateOptions{}, ) if err != nil && !apierrors.IsAlreadyExists(err) { @@ -279,7 +281,7 @@ func createRole(ctx context.Context, args *Args) (err error) { return args.writeObject(role) } -func createRoleBinding(ctx context.Context, args *Args) (err error) { +func (t rbacTask) createRoleBinding(ctx context.Context, args *Args) (err error) { if !sendProgressMessage(ctx, args.ProgressCh, "Creating role binding", 5, nil) { return errSendProgress } @@ -319,7 +321,7 @@ func createRoleBinding(ctx context.Context, args *Args) (err error) { } if !args.DryRun && !args.Declarative { - _, err = k8s.KubeClient().RbacV1().RoleBindings(namespace).Create( + _, err = t.client.Kube().RbacV1().RoleBindings(namespace).Create( ctx, roleBinding, metav1.CreateOptions{}, ) if err != nil && !apierrors.IsAlreadyExists(err) { @@ -330,52 +332,52 @@ func createRoleBinding(ctx context.Context, args *Args) (err error) { return args.writeObject(roleBinding) } -func createRBAC(ctx context.Context, args *Args) (err error) { - if err = createServiceAccount(ctx, args); err != nil { +func (t rbacTask) createRBAC(ctx context.Context, args *Args) (err error) { + if err = t.createServiceAccount(ctx, args); err != nil { return err } - if err = createClusterRole(ctx, args); err != nil { + if err = t.createClusterRole(ctx, args); err != nil { return err } - if err = createClusterRoleBinding(ctx, args); err != nil { + if err = t.createClusterRoleBinding(ctx, args); err != nil { return err } - if err = createRole(ctx, args); err != nil { + if err = t.createRole(ctx, args); err != nil { return err } - return createRoleBinding(ctx, args) + return t.createRoleBinding(ctx, args) } -func deleteRBAC(ctx context.Context) error { - err := k8s.KubeClient().CoreV1().ServiceAccounts(namespace).Delete( +func (t rbacTask) deleteRBAC(ctx context.Context) error { + err := t.client.Kube().CoreV1().ServiceAccounts(namespace).Delete( ctx, consts.Identity, metav1.DeleteOptions{}, ) if err != nil && !apierrors.IsNotFound(err) { return err } - err = k8s.KubeClient().RbacV1().ClusterRoles().Delete( + err = t.client.Kube().RbacV1().ClusterRoles().Delete( ctx, consts.Identity, metav1.DeleteOptions{}, ) if err != nil && !apierrors.IsNotFound(err) { return err } - err = k8s.KubeClient().RbacV1().ClusterRoleBindings().Delete( + err = t.client.Kube().RbacV1().ClusterRoleBindings().Delete( ctx, consts.Identity, metav1.DeleteOptions{}, ) if err != nil && !apierrors.IsNotFound(err) { return err } - err = k8s.KubeClient().RbacV1().Roles(namespace).Delete( + err = t.client.Kube().RbacV1().Roles(namespace).Delete( ctx, consts.Identity, metav1.DeleteOptions{}, ) if err != nil && !apierrors.IsNotFound(err) { return err } - err = k8s.KubeClient().RbacV1().RoleBindings(namespace).Delete( + err = t.client.Kube().RbacV1().RoleBindings(namespace).Delete( ctx, consts.Identity, metav1.DeleteOptions{}, ) if err != nil && !apierrors.IsNotFound(err) { diff --git a/pkg/installer/storageclass.go b/pkg/admin/installer/storageclass.go similarity index 77% rename from pkg/installer/storageclass.go rename to pkg/admin/installer/storageclass.go index d25bf600..8c4fce60 100644 --- a/pkg/installer/storageclass.go +++ b/pkg/admin/installer/storageclass.go @@ -22,8 +22,8 @@ import ( "fmt" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" + "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/consts" - "github.com/minio/directpv/pkg/k8s" legacyclient "github.com/minio/directpv/pkg/legacy/client" corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" @@ -34,7 +34,9 @@ import ( var errStorageClassVersionUnsupported = errors.New("unsupported StorageClass version found") -type storageClassTask struct{} +type storageClassTask struct { + client *client.Client +} func (storageClassTask) Name() string { return "StorageClass" @@ -58,15 +60,15 @@ func (storageClassTask) End(ctx context.Context, args *Args, err error) error { return nil } -func (storageClassTask) Execute(ctx context.Context, args *Args) error { - return createStorageClass(ctx, args) +func (t storageClassTask) Execute(ctx context.Context, args *Args) error { + return t.createStorageClass(ctx, args) } -func (storageClassTask) Delete(ctx context.Context, _ *Args) error { - return deleteStorageClass(ctx) +func (t storageClassTask) Delete(ctx context.Context, _ *Args) error { + return t.deleteStorageClass(ctx) } -func doCreateStorageClass(ctx context.Context, args *Args, version string, legacy bool, step int) (err error) { +func (t storageClassTask) doCreateStorageClass(ctx context.Context, args *Args, version string, legacy bool, step int) (err error) { name := consts.Identity if legacy { name = legacyclient.Identity @@ -118,7 +120,7 @@ func doCreateStorageClass(ctx context.Context, args *Args, version string, legac } if !args.DryRun && !args.Declarative { - _, err := k8s.KubeClient().StorageV1().StorageClasses().Create( + _, err := t.client.Kube().StorageV1().StorageClasses().Create( ctx, storageClass, metav1.CreateOptions{}, ) if err != nil && !apierrors.IsAlreadyExists(err) { @@ -152,7 +154,7 @@ func doCreateStorageClass(ctx context.Context, args *Args, version string, legac } if !args.DryRun && !args.Declarative { - _, err := k8s.KubeClient().StorageV1beta1().StorageClasses().Create( + _, err := t.client.Kube().StorageV1beta1().StorageClasses().Create( ctx, storageClass, metav1.CreateOptions{}, ) if err != nil && !apierrors.IsAlreadyExists(err) { @@ -167,7 +169,7 @@ func doCreateStorageClass(ctx context.Context, args *Args, version string, legac } } -func createStorageClass(ctx context.Context, args *Args) (err error) { +func (t storageClassTask) createStorageClass(ctx context.Context, args *Args) (err error) { version := "v1" switch { case args.DryRun: @@ -175,19 +177,19 @@ func createStorageClass(ctx context.Context, args *Args) (err error) { version = "v1beta1" } default: - gvk, err := k8s.GetGroupVersionKind("storage.k8s.io", "CSIDriver", "v1", "v1beta1") + gvk, err := t.client.K8s().GetGroupVersionKind("storage.k8s.io", "CSIDriver", "v1", "v1beta1") if err != nil { return err } version = gvk.Version } - if err := doCreateStorageClass(ctx, args, version, false, 1); err != nil { + if err := t.doCreateStorageClass(ctx, args, version, false, 1); err != nil { return err } if args.Legacy { - if err := doCreateStorageClass(ctx, args, version, true, 2); err != nil { + if err := t.doCreateStorageClass(ctx, args, version, true, 2); err != nil { return err } } @@ -195,14 +197,14 @@ func createStorageClass(ctx context.Context, args *Args) (err error) { return nil } -func doDeleteStorageClass(ctx context.Context, version, name string) (err error) { +func (t storageClassTask) doDeleteStorageClass(ctx context.Context, version, name string) (err error) { switch version { case "v1": - err = k8s.KubeClient().StorageV1().StorageClasses().Delete( + err = t.client.Kube().StorageV1().StorageClasses().Delete( ctx, name, metav1.DeleteOptions{}, ) case "v1beta1": - err = k8s.KubeClient().StorageV1beta1().StorageClasses().Delete( + err = t.client.Kube().StorageV1beta1().StorageClasses().Delete( ctx, name, metav1.DeleteOptions{}, ) default: @@ -216,15 +218,15 @@ func doDeleteStorageClass(ctx context.Context, version, name string) (err error) return nil } -func deleteStorageClass(ctx context.Context) error { - gvk, err := k8s.GetGroupVersionKind("storage.k8s.io", "CSIDriver", "v1", "v1beta1") +func (t storageClassTask) deleteStorageClass(ctx context.Context) error { + gvk, err := t.client.K8s().GetGroupVersionKind("storage.k8s.io", "CSIDriver", "v1", "v1beta1") if err != nil { return err } - if err = doDeleteStorageClass(ctx, gvk.Version, consts.StorageClassName); err != nil { + if err = t.doDeleteStorageClass(ctx, gvk.Version, consts.StorageClassName); err != nil { return err } - return doDeleteStorageClass(ctx, gvk.Version, legacyclient.Identity) + return t.doDeleteStorageClass(ctx, gvk.Version, legacyclient.Identity) } diff --git a/pkg/installer/utils.go b/pkg/admin/installer/utils.go similarity index 100% rename from pkg/installer/utils.go rename to pkg/admin/installer/utils.go diff --git a/pkg/installer/vars.go b/pkg/admin/installer/vars.go similarity index 100% rename from pkg/installer/vars.go rename to pkg/admin/installer/vars.go diff --git a/pkg/admin/label_drives.go b/pkg/admin/label_drives.go index bf11a99c..ccd435c8 100644 --- a/pkg/admin/label_drives.go +++ b/pkg/admin/label_drives.go @@ -21,7 +21,6 @@ import ( "fmt" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" - "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/retry" @@ -53,7 +52,7 @@ type LabelDriveArgs struct { } // LabelDrives sets/removes labels on/from the drives -func LabelDrives(ctx context.Context, args LabelDriveArgs, labels []Label) error { +func (client *Client) LabelDrives(ctx context.Context, args LabelDriveArgs, labels []Label) error { var processed bool ctx, cancelFunc := context.WithCancel(ctx) defer cancelFunc() @@ -86,7 +85,7 @@ func LabelDrives(ctx context.Context, args LabelDriveArgs, labels []Label) error verb = "set on" } if !args.DryRun { - drive, err = client.DriveClient().Update(ctx, drive, metav1.UpdateOptions{}) + drive, err = client.Drive().Update(ctx, drive, metav1.UpdateOptions{}) } if err != nil { utils.Eprintf(args.Quiet, true, "%v/%v: %v\n", drive.GetNodeID(), drive.GetDriveName(), err) diff --git a/pkg/admin/label_volumes.go b/pkg/admin/label_volumes.go index 7306e985..9774bfb9 100644 --- a/pkg/admin/label_volumes.go +++ b/pkg/admin/label_volumes.go @@ -21,7 +21,6 @@ import ( "fmt" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" - "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/retry" @@ -42,7 +41,7 @@ type LabelVolumeArgs struct { } // LabelVolumes sets/removes labels on/from the volumes -func LabelVolumes(ctx context.Context, args LabelVolumeArgs, labels []Label) error { +func (client *Client) LabelVolumes(ctx context.Context, args LabelVolumeArgs, labels []Label) error { ctx, cancelFunc := context.WithCancel(ctx) defer cancelFunc() @@ -78,7 +77,7 @@ func LabelVolumes(ctx context.Context, args LabelVolumeArgs, labels []Label) err verb = "set on" } if !args.DryRun { - volume, err = client.VolumeClient().Update(ctx, volume, metav1.UpdateOptions{}) + volume, err = client.Volume().Update(ctx, volume, metav1.UpdateOptions{}) } if err != nil { utils.Eprintf(args.Quiet, true, "%v: %v\n", volume.Name, err) diff --git a/pkg/admin/migrate.go b/pkg/admin/migrate.go new file mode 100644 index 00000000..96977a75 --- /dev/null +++ b/pkg/admin/migrate.go @@ -0,0 +1,78 @@ +// This file is part of MinIO DirectPV +// Copyright (c) 2024 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package admin + +import ( + "context" + "errors" + "fmt" + + "github.com/minio/directpv/pkg/admin/installer" + "github.com/minio/directpv/pkg/consts" + legacyclient "github.com/minio/directpv/pkg/legacy/client" +) + +// MigrateArgs denotest the migrate arguments +type MigrateArgs struct { + Quiet bool + Retain bool + DrivesBackupFile string + VolumesBackupFile string +} + +// Migrate migrates the directpv resources +func (client *Client) Migrate(ctx context.Context, args MigrateArgs) error { + if !args.Retain { + if args.DrivesBackupFile == "" || args.VolumesBackupFile == "" { + return errors.New("backup file should not be empty") + } + if args.DrivesBackupFile == args.VolumesBackupFile { + return errors.New("backup filenames are same") + } + } + legacyClient, err := legacyclient.NewClient(client.K8s()) + if err != nil { + return fmt.Errorf("unable to create legacy client; %v", err) + } + if err := installer.Migrate(ctx, &installer.Args{ + Quiet: args.Quiet, + Legacy: true, + }, client.Client, legacyClient); err != nil { + return fmt.Errorf("migration failed; %v", err) + } + if !args.Quiet { + fmt.Println("Migration successful; Please restart the pods in '" + consts.AppName + "' namespace.") + } + if args.Retain { + return nil + } + backupCreated, err := legacyClient.RemoveAllDrives(ctx, args.DrivesBackupFile) + if err != nil { + return fmt.Errorf("unable to remove legacy drive CRDs; %v", err) + } + if backupCreated && !args.Quiet { + fmt.Println("Legacy drive CRDs backed up to", args.DrivesBackupFile) + } + backupCreated, err = legacyClient.RemoveAllVolumes(ctx, args.VolumesBackupFile) + if err != nil { + return fmt.Errorf("unable to remove legacy volume CRDs; %v", err) + } + if backupCreated && !args.Quiet { + fmt.Println("Legacy volume CRDs backed up to", args.VolumesBackupFile) + } + return nil +} diff --git a/pkg/admin/move.go b/pkg/admin/move.go index 68eea2a9..a1bc6b91 100644 --- a/pkg/admin/move.go +++ b/pkg/admin/move.go @@ -23,7 +23,6 @@ import ( "github.com/dustin/go-humanize" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" - "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/types" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -36,12 +35,12 @@ type MoveArgs struct { } // Move - moves the volume references from source to destination -func Move(ctx context.Context, args MoveArgs) error { +func (client *Client) Move(ctx context.Context, args MoveArgs) error { if args.Source == args.Destination { return errors.New("source and destination drives are same") } - srcDrive, err := client.DriveClient().Get(ctx, string(args.Source), metav1.GetOptions{}) + srcDrive, err := client.Drive().Get(ctx, string(args.Source), metav1.GetOptions{}) if err != nil { return fmt.Errorf("unable to get source drive; %v", err) } @@ -72,7 +71,7 @@ func Move(ctx context.Context, args MoveArgs) error { return fmt.Errorf("no volumes found in source drive %v", args.Source) } - destDrive, err := client.DriveClient().Get(ctx, string(args.Destination), metav1.GetOptions{}) + destDrive, err := client.Drive().Get(ctx, string(args.Destination), metav1.GetOptions{}) if err != nil { return fmt.Errorf("unable to get destination drive %v; %v", args.Destination, err) } @@ -107,7 +106,7 @@ func Move(ctx context.Context, args MoveArgs) error { } } destDrive.Status.Status = directpvtypes.DriveStatusMoving - _, err = client.DriveClient().Update( + _, err = client.Drive().Update( ctx, destDrive, metav1.UpdateOptions{TypeMeta: types.NewDriveTypeMeta()}, ) if err != nil { @@ -121,7 +120,7 @@ func Move(ctx context.Context, args MoveArgs) error { } srcDrive.ResetFinalizers() - _, err = client.DriveClient().Update( + _, err = client.Drive().Update( ctx, srcDrive, metav1.UpdateOptions{TypeMeta: types.NewDriveTypeMeta()}, ) if err != nil { diff --git a/pkg/admin/remove.go b/pkg/admin/remove.go index 4820a66d..256cc26e 100644 --- a/pkg/admin/remove.go +++ b/pkg/admin/remove.go @@ -22,7 +22,6 @@ import ( "fmt" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" - "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -38,7 +37,7 @@ type RemoveArgs struct { } // Remove removes the initialized drive(s) -func Remove(ctx context.Context, args RemoveArgs) error { +func (client *Client) Remove(ctx context.Context, args RemoveArgs) error { var processed bool var failed bool @@ -68,7 +67,7 @@ func Remove(ctx context.Context, args RemoveArgs) error { result.Drive.Status.Status = directpvtypes.DriveStatusRemoved var err error if !args.DryRun { - _, err = client.DriveClient().Update(ctx, &result.Drive, metav1.UpdateOptions{}) + _, err = client.Drive().Update(ctx, &result.Drive, metav1.UpdateOptions{}) } if err != nil { failed = true diff --git a/pkg/admin/resume_drives.go b/pkg/admin/resume_drives.go index 4228e56c..6092ef58 100644 --- a/pkg/admin/resume_drives.go +++ b/pkg/admin/resume_drives.go @@ -20,7 +20,6 @@ import ( "context" "fmt" - "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/retry" @@ -30,7 +29,7 @@ import ( type ResumeDriveArgs = SuspendDriveArgs // ResumeDrives will resume the suspended drives -func ResumeDrives(ctx context.Context, args ResumeDriveArgs) error { +func (client *Client) ResumeDrives(ctx context.Context, args ResumeDriveArgs) error { var processed bool ctx, cancelFunc := context.WithCancel(ctx) @@ -50,7 +49,7 @@ func ResumeDrives(ctx context.Context, args ResumeDriveArgs) error { // only suspended drives can be resumed. continue } - driveClient := client.DriveClient() + driveClient := client.Drive() updateFunc := func() error { drive, err := driveClient.Get(ctx, result.Drive.Name, metav1.GetOptions{}) if err != nil { diff --git a/pkg/admin/resume_volumes.go b/pkg/admin/resume_volumes.go index 62e15299..c21e47b9 100644 --- a/pkg/admin/resume_volumes.go +++ b/pkg/admin/resume_volumes.go @@ -20,7 +20,6 @@ import ( "context" "fmt" - "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/retry" @@ -30,7 +29,7 @@ import ( type ResumeVolumeArgs = SuspendVolumeArgs // ResumeVolumes will resume the suspended volumes -func ResumeVolumes(ctx context.Context, args ResumeVolumeArgs) error { +func (client *Client) ResumeVolumes(ctx context.Context, args ResumeVolumeArgs) error { var processed bool ctx, cancelFunc := context.WithCancel(ctx) @@ -52,7 +51,7 @@ func ResumeVolumes(ctx context.Context, args ResumeVolumeArgs) error { // only suspended drives can be resumed. continue } - volumeClient := client.VolumeClient() + volumeClient := client.Volume() updateFunc := func() error { volume, err := volumeClient.Get(ctx, result.Volume.Name, metav1.GetOptions{}) if err != nil { diff --git a/pkg/admin/suspend_drives.go b/pkg/admin/suspend_drives.go index 192a0db5..acb7f37c 100644 --- a/pkg/admin/suspend_drives.go +++ b/pkg/admin/suspend_drives.go @@ -22,7 +22,6 @@ import ( "fmt" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" - "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/retry" @@ -41,7 +40,7 @@ type SuspendDriveArgs struct { } // SuspendDrives suspends the drive -func SuspendDrives(ctx context.Context, args SuspendDriveArgs) error { +func (client *Client) SuspendDrives(ctx context.Context, args SuspendDriveArgs) error { var processed bool ctx, cancelFunc := context.WithCancel(ctx) @@ -63,7 +62,7 @@ func SuspendDrives(ctx context.Context, args SuspendDriveArgs) error { continue } - driveClient := client.DriveClient() + driveClient := client.Drive() updateFunc := func() error { drive, err := driveClient.Get(ctx, result.Drive.Name, metav1.GetOptions{}) if err != nil { diff --git a/pkg/admin/suspend_volumes.go b/pkg/admin/suspend_volumes.go index 434c91dd..e3a50e18 100644 --- a/pkg/admin/suspend_volumes.go +++ b/pkg/admin/suspend_volumes.go @@ -20,7 +20,6 @@ import ( "context" "fmt" - "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/retry" @@ -38,7 +37,7 @@ type SuspendVolumeArgs struct { } // SuspendVolumes suspends the volume -func SuspendVolumes(ctx context.Context, args SuspendVolumeArgs) error { +func (client *Client) SuspendVolumes(ctx context.Context, args SuspendVolumeArgs) error { var processed bool ctx, cancelFunc := context.WithCancel(ctx) @@ -59,7 +58,7 @@ func SuspendVolumes(ctx context.Context, args SuspendVolumeArgs) error { if result.Volume.IsSuspended() { continue } - volumeClient := client.VolumeClient() + volumeClient := client.Volume() updateFunc := func() error { volume, err := volumeClient.Get(ctx, result.Volume.Name, metav1.GetOptions{}) if err != nil { diff --git a/pkg/admin/uncordon.go b/pkg/admin/uncordon.go index cfa2aada..575cff51 100644 --- a/pkg/admin/uncordon.go +++ b/pkg/admin/uncordon.go @@ -20,7 +20,6 @@ import ( "context" "fmt" - "github.com/minio/directpv/pkg/client" "github.com/minio/directpv/pkg/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -29,7 +28,7 @@ import ( type UncordonArgs = CordonArgs // Uncordon makes the drive schedulable again -func Uncordon(ctx context.Context, args UncordonArgs) error { +func (client *Client) Uncordon(ctx context.Context, args UncordonArgs) error { var processed bool ctx, cancelFunc := context.WithCancel(ctx) @@ -55,7 +54,7 @@ func Uncordon(ctx context.Context, args UncordonArgs) error { result.Drive.Schedulable() var err error if !args.DryRun { - _, err = client.DriveClient().Update(ctx, &result.Drive, metav1.UpdateOptions{}) + _, err = client.Drive().Update(ctx, &result.Drive, metav1.UpdateOptions{}) } if err != nil { diff --git a/pkg/admin/uninstall.go b/pkg/admin/uninstall.go index 7e69dd2c..b87bfd17 100644 --- a/pkg/admin/uninstall.go +++ b/pkg/admin/uninstall.go @@ -19,7 +19,8 @@ package admin import ( "context" - "github.com/minio/directpv/pkg/installer" + "github.com/minio/directpv/pkg/admin/installer" + legacyclient "github.com/minio/directpv/pkg/legacy/client" ) // UninstallArgs represents the args to uninstall @@ -29,6 +30,10 @@ type UninstallArgs struct { } // Uninstall uninstalls directpv -func Uninstall(ctx context.Context, args UninstallArgs) error { - return installer.Uninstall(ctx, args.Quiet, args.Dangerous) +func (client Client) Uninstall(ctx context.Context, args UninstallArgs) error { + legacyClient, err := legacyclient.NewClient(client.K8s()) + if err != nil { + return err + } + return installer.Uninstall(ctx, args.Quiet, args.Dangerous, installer.GetTasks(client.Client, legacyClient)) } diff --git a/pkg/client/clients.go b/pkg/client/clients.go index 4179b83e..6913e3c5 100644 --- a/pkg/client/clients.go +++ b/pkg/client/clients.go @@ -22,36 +22,51 @@ import ( ) var ( - initialized int32 - clientsetInterface types.ExtClientsetInterface - restClient rest.Interface - driveClient types.LatestDriveInterface - volumeClient types.LatestVolumeInterface - nodeClient types.LatestNodeInterface - initRequestClient types.LatestInitRequestInterface + initialized int32 + client *Client ) // RESTClient gets latest versioned REST client. func RESTClient() rest.Interface { - return restClient + return client.REST() } // DriveClient gets latest versioned drive interface. func DriveClient() types.LatestDriveInterface { - return driveClient + return client.Drive() } // VolumeClient gets latest versioned volume interface. func VolumeClient() types.LatestVolumeInterface { - return volumeClient + return client.Volume() } // NodeClient gets latest versioned node interface. func NodeClient() types.LatestNodeInterface { - return nodeClient + return client.Node() } // InitRequestClient gets latest versioned init request interface. func InitRequestClient() types.LatestInitRequestInterface { - return initRequestClient + return client.InitRequest() +} + +// NewDriveLister returns the new drive lister +func NewDriveLister() *DriveLister { + return client.NewDriveLister() +} + +// NewVolumeLister returns the new volume lister +func NewVolumeLister() *VolumeLister { + return client.NewVolumeLister() +} + +// NewNodeLister returns the new node lister +func NewNodeLister() *NodeLister { + return client.NewNodeLister() +} + +// NewInitRequestLister returns the new initrequest lister +func NewInitRequestLister() *InitRequestLister { + return client.NewInitRequestLister() } diff --git a/pkg/client/discovery.go b/pkg/client/discovery.go index 76e113b5..f0020865 100644 --- a/pkg/client/discovery.go +++ b/pkg/client/discovery.go @@ -1,5 +1,5 @@ // This file is part of MinIO DirectPV -// Copyright (c) 2023 MinIO, Inc. +// Copyright (c) 2024 MinIO, Inc. // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by @@ -21,7 +21,6 @@ import ( "fmt" directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types" - "github.com/minio/directpv/pkg/k8s" "github.com/minio/directpv/pkg/types" "github.com/minio/directpv/pkg/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -29,13 +28,12 @@ import ( // SyncNodes compares the csinodes with directpvnode list and syncs them // It adds the missing nodes and deletes the non-existing nodes -func SyncNodes(ctx context.Context) (err error) { - csiNodes, err := k8s.GetCSINodes(ctx) +func (c Client) SyncNodes(ctx context.Context) (err error) { + csiNodes, err := c.K8s().GetCSINodes(ctx) if err != nil { return fmt.Errorf("unable to get CSI nodes; %w", err) } - - nodes, err := NewNodeLister().Get(ctx) + nodes, err := c.NewNodeLister().Get(ctx) if err != nil { return fmt.Errorf("unable to get nodes; %w", err) } @@ -50,7 +48,7 @@ func SyncNodes(ctx context.Context) (err error) { if !utils.Contains(nodeNames, csiNode) { node := types.NewNode(directpvtypes.NodeID(csiNode), nil) node.Spec.Refresh = true - if _, err = NodeClient().Create(ctx, node, metav1.CreateOptions{}); err != nil { + if _, err = c.Node().Create(ctx, node, metav1.CreateOptions{}); err != nil { return fmt.Errorf("unable to create node %v; %w", csiNode, err) } } @@ -59,11 +57,10 @@ func SyncNodes(ctx context.Context) (err error) { // Remove non-existing nodes. for _, nodeName := range nodeNames { if !utils.Contains(csiNodes, nodeName) { - if err = NodeClient().Delete(ctx, nodeName, metav1.DeleteOptions{}); err != nil { + if err = c.Node().Delete(ctx, nodeName, metav1.DeleteOptions{}); err != nil { return fmt.Errorf("unable to remove non-existing node %v; %w", nodeName, err) } } } - return nil } diff --git a/pkg/client/drive_lister.go b/pkg/client/drive_lister.go index 017e7eb5..20f6d448 100644 --- a/pkg/client/drive_lister.go +++ b/pkg/client/drive_lister.go @@ -1,5 +1,5 @@ // This file is part of MinIO DirectPV -// Copyright (c) 2021, 2022 MinIO, Inc. +// Copyright (c) 2021-2024 MinIO, Inc. // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by @@ -43,12 +43,14 @@ type DriveLister struct { labels map[directpvtypes.LabelKey]directpvtypes.LabelValue maxObjects int64 ignoreNotFound bool + driveClient types.LatestDriveInterface } // NewDriveLister creates new drive lister. -func NewDriveLister() *DriveLister { +func (client Client) NewDriveLister() *DriveLister { return &DriveLister{ - maxObjects: k8s.MaxThreadCount, + maxObjects: k8s.MaxThreadCount, + driveClient: client.Drive(), } } @@ -132,7 +134,7 @@ func (lister *DriveLister) List(ctx context.Context) <-chan ListDriveResult { LabelSelector: labelSelector, } for { - result, err := DriveClient().List(ctx, options) + result, err := lister.driveClient.List(ctx, options) if err != nil { if apierrors.IsNotFound(err) && lister.ignoreNotFound { break @@ -169,7 +171,7 @@ func (lister *DriveLister) List(ctx context.Context) <-chan ListDriveResult { } for _, driveID := range lister.driveIDs { - drive, err := DriveClient().Get(ctx, string(driveID), metav1.GetOptions{}) + drive, err := lister.driveClient.Get(ctx, string(driveID), metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) && lister.ignoreNotFound { continue diff --git a/pkg/client/drive_lister_test.go b/pkg/client/drive_lister_test.go index 5dfff0be..27aa4f18 100644 --- a/pkg/client/drive_lister_test.go +++ b/pkg/client/drive_lister_test.go @@ -30,7 +30,7 @@ import ( func TestGetDriveList(t *testing.T) { clientset := types.NewExtFakeClientset(clientsetfake.NewSimpleClientset()) SetDriveInterface(clientset.DirectpvLatest().DirectPVDrives()) - drives, err := NewDriveLister().Get(context.TODO()) + drives, err := client.NewDriveLister().Get(context.TODO()) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -47,7 +47,7 @@ func TestGetDriveList(t *testing.T) { clientset = types.NewExtFakeClientset(clientsetfake.NewSimpleClientset(objects...)) SetDriveInterface(clientset.DirectpvLatest().DirectPVDrives()) - drives, err = NewDriveLister().Get(context.TODO()) + drives, err = client.NewDriveLister().Get(context.TODO()) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/pkg/client/dynamic.go b/pkg/client/dynamic.go index 2e994191..8a0e90a2 100644 --- a/pkg/client/dynamic.go +++ b/pkg/client/dynamic.go @@ -31,7 +31,6 @@ import ( apimachinerytypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/dynamic" - "k8s.io/client-go/rest" "k8s.io/klog/v2" ) @@ -40,12 +39,12 @@ type dynamicInterface struct { groupVersion schema.GroupVersion } -func dynamicInterfaceForConfig(config *rest.Config, kind, resource string) (*dynamicInterface, error) { - gvk, err := k8s.GetGroupVersionKind(consts.GroupName, kind, types.Versions...) +func dynamicInterfaceForConfig(k8sClient *k8s.Client, kind, resource string) (*dynamicInterface, error) { + gvk, err := k8sClient.GetGroupVersionKind(consts.GroupName, kind, types.Versions...) if err != nil && !meta.IsNoMatchError(err) { return nil, err } - resourceInterface, err := dynamic.NewForConfig(config) + resourceInterface, err := dynamic.NewForConfig(k8sClient.KubeConfig) if err != nil { return nil, err } @@ -229,8 +228,8 @@ type latestDriveClient struct { } // latestDriveClientForConfig creates new dynamic drive interface. -func latestDriveClientForConfig(config *rest.Config) (*latestDriveClient, error) { - inter, err := dynamicInterfaceForConfig(config, consts.DriveKind, consts.DriveResource) +func latestDriveClientForConfig(k8sClient *k8s.Client) (*latestDriveClient, error) { + inter, err := dynamicInterfaceForConfig(k8sClient, consts.DriveKind, consts.DriveResource) if err != nil { return nil, err } @@ -351,8 +350,8 @@ type latestVolumeClient struct { } // latestVolumeClientForConfig creates new dynamic volume interface. -func latestVolumeClientForConfig(config *rest.Config) (*latestVolumeClient, error) { - inter, err := dynamicInterfaceForConfig(config, consts.VolumeKind, consts.VolumeResource) +func latestVolumeClientForConfig(k8sClient *k8s.Client) (*latestVolumeClient, error) { + inter, err := dynamicInterfaceForConfig(k8sClient, consts.VolumeKind, consts.VolumeResource) if err != nil { return nil, err } @@ -473,8 +472,8 @@ type latestNodeClient struct { } // latestNodeClientForConfig creates new dynamic node interface. -func latestNodeClientForConfig(config *rest.Config) (*latestNodeClient, error) { - inter, err := dynamicInterfaceForConfig(config, consts.NodeKind, consts.NodeResource) +func latestNodeClientForConfig(k8sClient *k8s.Client) (*latestNodeClient, error) { + inter, err := dynamicInterfaceForConfig(k8sClient, consts.NodeKind, consts.NodeResource) if err != nil { return nil, err } @@ -597,8 +596,8 @@ type latestInitRequestClient struct { } // latestInitRequestClientForConfig creates new dynamic initrequest interface. -func latestInitRequestClientForConfig(config *rest.Config) (*latestInitRequestClient, error) { - inter, err := dynamicInterfaceForConfig(config, consts.InitRequestKind, consts.InitRequestResource) +func latestInitRequestClientForConfig(k8sClient *k8s.Client) (*latestInitRequestClient, error) { + inter, err := dynamicInterfaceForConfig(k8sClient, consts.InitRequestKind, consts.InitRequestResource) if err != nil { return nil, err } diff --git a/pkg/client/fake.go b/pkg/client/fake.go index 09c48e13..360419db 100644 --- a/pkg/client/fake.go +++ b/pkg/client/fake.go @@ -25,36 +25,46 @@ import ( // FakeInit initializes fake clients. func FakeInit() { k8s.FakeInit() + k8sClient := k8s.GetClient() + clientsetInterface := types.NewExtFakeClientset(fake.NewSimpleClientset()) + driveClient := clientsetInterface.DirectpvLatest().DirectPVDrives() + volumeClient := clientsetInterface.DirectpvLatest().DirectPVVolumes() + nodeClient := clientsetInterface.DirectpvLatest().DirectPVNodes() + initRequestClient := clientsetInterface.DirectpvLatest().DirectPVInitRequests() + restClient := clientsetInterface.DirectpvLatest().RESTClient() - clientsetInterface = types.NewExtFakeClientset(fake.NewSimpleClientset()) - driveClient = clientsetInterface.DirectpvLatest().DirectPVDrives() - volumeClient = clientsetInterface.DirectpvLatest().DirectPVVolumes() - nodeClient = clientsetInterface.DirectpvLatest().DirectPVNodes() - initRequestClient = clientsetInterface.DirectpvLatest().DirectPVInitRequests() - - initEvent(k8s.KubeClient()) + initEvent(k8sClient.KubeClient) + client = &Client{ + K8sClient: k8sClient, + ClientsetInterface: clientsetInterface, + RESTClient: restClient, + DriveClient: driveClient, + VolumeClient: volumeClient, + NodeClient: nodeClient, + InitRequestClient: initRequestClient, + } } // SetDriveInterface sets latest drive interface. // Note: To be used for writing test cases only func SetDriveInterface(i types.LatestDriveInterface) { - driveClient = i + client.DriveClient = i } // SetVolumeInterface sets the latest volume interface. // Note: To be used for writing test cases only func SetVolumeInterface(i types.LatestVolumeInterface) { - volumeClient = i + client.VolumeClient = i } // SetNodeInterface sets latest node interface. // Note: To be used for writing test cases only func SetNodeInterface(i types.LatestNodeInterface) { - nodeClient = i + client.NodeClient = i } // SetInitRequestInterface sets latest initrequest interface. // Note: To be used for writing test cases only func SetInitRequestInterface(i types.LatestInitRequestInterface) { - initRequestClient = i + client.InitRequestClient = i } diff --git a/pkg/client/init.go b/pkg/client/init.go index abc88d71..b66435ce 100644 --- a/pkg/client/init.go +++ b/pkg/client/init.go @@ -18,14 +18,21 @@ package client import ( "fmt" - "log" "sync/atomic" "github.com/minio/directpv/pkg/clientset" "github.com/minio/directpv/pkg/k8s" "github.com/minio/directpv/pkg/types" + apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1" + "k8s.io/client-go/discovery" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/klog/v2" + + // support gcp, azure, and oidc client auth + _ "k8s.io/client-go/plugin/pkg/client/auth/azure" + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" ) // Init initializes various clients. @@ -33,45 +40,128 @@ func Init() { if atomic.AddInt32(&initialized, 1) != 1 { return } - if err := k8s.Init(); err != nil { - log.Fatalf("unable to initialize k8s clients; %v", err) + var err error + if err = k8s.Init(); err != nil { + klog.Fatalf("unable to initialize k8s clients; %v", err) } - if err := InitWithConfig(k8s.KubeConfig()); err != nil { - klog.Fatalf("unable to initialize; %v", err) + client, err = newClient(k8s.GetClient()) + if err != nil { + klog.Fatalf("unable to initialize client; %v", err) } + initEvent(k8s.KubeClient()) } -// InitWithConfig initializes the clients with k8s config provided -func InitWithConfig(c *rest.Config) error { - cs, err := clientset.NewForConfig(c) - if err != nil { - return fmt.Errorf("unable to create new clientset interface; %v", err) - } +// Client represents the directpv client set +type Client struct { + ClientsetInterface types.ExtClientsetInterface + RESTClient rest.Interface + DriveClient types.LatestDriveInterface + VolumeClient types.LatestVolumeInterface + NodeClient types.LatestNodeInterface + InitRequestClient types.LatestInitRequestInterface + K8sClient *k8s.Client +} - if err := k8s.Init(); err != nil { - return err - } +// GetClient returns the global client +func GetClient() *Client { + return client +} - clientsetInterface = types.NewExtClientset(cs) +// REST returns the REST client +func (c Client) REST() rest.Interface { + return c.RESTClient +} - restClient = clientsetInterface.DirectpvLatest().RESTClient() +// Drive returns the DirectPV Drive interface +func (c Client) Drive() types.LatestDriveInterface { + return c.DriveClient +} - if driveClient, err = latestDriveClientForConfig(k8s.KubeConfig()); err != nil { - return fmt.Errorf("unable to create new drive interface; %v", err) - } +// Volume returns the DirectPV Volume interface +func (c Client) Volume() types.LatestVolumeInterface { + return c.VolumeClient +} - if volumeClient, err = latestVolumeClientForConfig(k8s.KubeConfig()); err != nil { - return fmt.Errorf("unable to create new volume interface; %v", err) - } +// Node returns the DirectPV Node interface +func (c Client) Node() types.LatestNodeInterface { + return c.NodeClient +} - if nodeClient, err = latestNodeClientForConfig(k8s.KubeConfig()); err != nil { - return fmt.Errorf("unable to create new node interface; %v", err) - } +// InitRequest returns the DirectPV InitRequest interface +func (c Client) InitRequest() types.LatestInitRequestInterface { + return c.InitRequestClient +} - if initRequestClient, err = latestInitRequestClientForConfig(k8s.KubeConfig()); err != nil { - return fmt.Errorf("unable to create new initrequest interface; %v", err) +// K8s returns the kubernetes client +func (c Client) K8s() *k8s.Client { + return c.K8sClient +} + +// KubeConfig returns the kubeconfig +func (c Client) KubeConfig() *rest.Config { + return c.K8sClient.KubeConfig +} + +// Kube returns the kube client +func (c Client) Kube() kubernetes.Interface { + return c.K8sClient.KubeClient +} + +// APIextensions returns the APIextensionsClient +func (c Client) APIextensions() apiextensions.ApiextensionsV1Interface { + return c.K8sClient.APIextensionsClient +} + +// CRD returns the CRD client +func (c Client) CRD() apiextensions.CustomResourceDefinitionInterface { + return c.K8sClient.CRDClient +} + +// Discovery returns the discovery client +func (c Client) Discovery() discovery.DiscoveryInterface { + return c.K8sClient.DiscoveryClient +} + +// NewClient returns the directpv client +func NewClient(c *rest.Config) (*Client, error) { + k8sClient, err := k8s.NewClient(c) + if err != nil { + return nil, fmt.Errorf("unable to create kubernetes client; %v", err) } + return newClient(k8sClient) +} - initEvent(k8s.KubeClient()) - return nil +// newClient returns the directpv client +func newClient(k8sClient *k8s.Client) (*Client, error) { + cs, err := clientset.NewForConfig(k8sClient.KubeConfig) + if err != nil { + return nil, fmt.Errorf("unable to create new clientset interface; %v", err) + } + clientsetInterface := types.NewExtClientset(cs) + restClient := clientsetInterface.DirectpvLatest().RESTClient() + driveClient, err := latestDriveClientForConfig(k8sClient) + if err != nil { + return nil, fmt.Errorf("unable to create new drive interface; %v", err) + } + volumeClient, err := latestVolumeClientForConfig(k8sClient) + if err != nil { + return nil, fmt.Errorf("unable to create new volume interface; %v", err) + } + nodeClient, err := latestNodeClientForConfig(k8sClient) + if err != nil { + return nil, fmt.Errorf("unable to create new node interface; %v", err) + } + initRequestClient, err := latestInitRequestClientForConfig(k8sClient) + if err != nil { + return nil, fmt.Errorf("unable to create new initrequest interface; %v", err) + } + return &Client{ + ClientsetInterface: clientsetInterface, + RESTClient: restClient, + DriveClient: driveClient, + VolumeClient: volumeClient, + NodeClient: nodeClient, + InitRequestClient: initRequestClient, + K8sClient: k8sClient, + }, nil } diff --git a/pkg/client/initrequest_lister.go b/pkg/client/initrequest_lister.go index eeed12ae..213529aa 100644 --- a/pkg/client/initrequest_lister.go +++ b/pkg/client/initrequest_lister.go @@ -37,17 +37,19 @@ type ListInitRequestResult struct { // InitRequestLister is initRequest lister. type InitRequestLister struct { - nodes []directpvtypes.LabelValue - requestIDs []directpvtypes.LabelValue - initRequestNames []string - maxObjects int64 - ignoreNotFound bool + nodes []directpvtypes.LabelValue + requestIDs []directpvtypes.LabelValue + initRequestNames []string + maxObjects int64 + ignoreNotFound bool + initRequestClient types.LatestInitRequestInterface } // NewInitRequestLister creates new volume lister. -func NewInitRequestLister() *InitRequestLister { +func (client Client) NewInitRequestLister() *InitRequestLister { return &InitRequestLister{ - maxObjects: k8s.MaxThreadCount, + maxObjects: k8s.MaxThreadCount, + initRequestClient: client.InitRequest(), } } @@ -112,7 +114,7 @@ func (lister *InitRequestLister) List(ctx context.Context) <-chan ListInitReques LabelSelector: labelSelector, } for { - result, err := InitRequestClient().List(ctx, options) + result, err := lister.initRequestClient.List(ctx, options) if err != nil { send(ListInitRequestResult{Err: err}) return @@ -146,7 +148,7 @@ func (lister *InitRequestLister) List(ctx context.Context) <-chan ListInitReques } for _, initRequestName := range lister.initRequestNames { - initRequest, err := InitRequestClient().Get(ctx, initRequestName, metav1.GetOptions{}) + initRequest, err := lister.initRequestClient.Get(ctx, initRequestName, metav1.GetOptions{}) if err != nil { send(ListInitRequestResult{Err: err}) return @@ -182,7 +184,7 @@ func (lister *InitRequestLister) Watch(ctx context.Context) (<-chan WatchEvent[* directpvtypes.NodeLabelKey: lister.nodes, directpvtypes.RequestIDLabelKey: lister.requestIDs, } - initRequestWatchInterface, err := InitRequestClient().Watch(ctx, metav1.ListOptions{ + initRequestWatchInterface, err := lister.initRequestClient.Watch(ctx, metav1.ListOptions{ LabelSelector: directpvtypes.ToLabelSelector(labelMap), }) if err != nil { diff --git a/pkg/client/node_lister.go b/pkg/client/node_lister.go index 8e7a0a4f..9ce5edac 100644 --- a/pkg/client/node_lister.go +++ b/pkg/client/node_lister.go @@ -42,12 +42,14 @@ type NodeLister struct { nodeNames []string maxObjects int64 ignoreNotFound bool + nodeClient types.LatestNodeInterface } // NewNodeLister creates new volume lister. -func NewNodeLister() *NodeLister { +func (client Client) NewNodeLister() *NodeLister { return &NodeLister{ maxObjects: k8s.MaxThreadCount, + nodeClient: client.Node(), } } @@ -104,7 +106,7 @@ func (lister *NodeLister) List(ctx context.Context) <-chan ListNodeResult { LabelSelector: labelSelector, } for { - result, err := NodeClient().List(ctx, options) + result, err := lister.nodeClient.List(ctx, options) if err != nil { send(ListNodeResult{Err: err}) return @@ -138,7 +140,7 @@ func (lister *NodeLister) List(ctx context.Context) <-chan ListNodeResult { } for _, nodeName := range lister.nodeNames { - node, err := NodeClient().Get(ctx, nodeName, metav1.GetOptions{}) + node, err := lister.nodeClient.Get(ctx, nodeName, metav1.GetOptions{}) if err != nil { send(ListNodeResult{Err: err}) return @@ -183,7 +185,7 @@ func (lister *NodeLister) Watch(ctx context.Context) (<-chan WatchEvent[*types.N labelMap := map[directpvtypes.LabelKey][]directpvtypes.LabelValue{ directpvtypes.NodeLabelKey: lister.nodes, } - nodeWatchInterface, err := NodeClient().Watch(ctx, metav1.ListOptions{ + nodeWatchInterface, err := lister.nodeClient.Watch(ctx, metav1.ListOptions{ LabelSelector: directpvtypes.ToLabelSelector(labelMap), }) if err != nil { diff --git a/pkg/client/volume_lister.go b/pkg/client/volume_lister.go index 766b3ed9..cf9aadf9 100644 --- a/pkg/client/volume_lister.go +++ b/pkg/client/volume_lister.go @@ -45,12 +45,14 @@ type VolumeLister struct { labels map[directpvtypes.LabelKey]directpvtypes.LabelValue maxObjects int64 ignoreNotFound bool + volumeClient types.LatestVolumeInterface } // NewVolumeLister creates new volume lister. -func NewVolumeLister() *VolumeLister { +func (client Client) NewVolumeLister() *VolumeLister { return &VolumeLister{ - maxObjects: k8s.MaxThreadCount, + maxObjects: k8s.MaxThreadCount, + volumeClient: client.Volume(), } } @@ -157,7 +159,7 @@ func (lister *VolumeLister) List(ctx context.Context) <-chan ListVolumeResult { } for { - result, err := VolumeClient().List(ctx, options) + result, err := lister.volumeClient.List(ctx, options) if err != nil { if apierrors.IsNotFound(err) && lister.ignoreNotFound { break @@ -195,7 +197,7 @@ func (lister *VolumeLister) List(ctx context.Context) <-chan ListVolumeResult { } for _, volumeName := range lister.volumeNames { - volume, err := VolumeClient().Get(ctx, volumeName, metav1.GetOptions{}) + volume, err := lister.volumeClient.Get(ctx, volumeName, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) && lister.ignoreNotFound { continue diff --git a/pkg/client/volume_lister_test.go b/pkg/client/volume_lister_test.go index 42a6fa93..529df48c 100644 --- a/pkg/client/volume_lister_test.go +++ b/pkg/client/volume_lister_test.go @@ -32,7 +32,7 @@ func TestGetVolumeList(t *testing.T) { SetDriveInterface(clientset.DirectpvLatest().DirectPVDrives()) SetVolumeInterface(clientset.DirectpvLatest().DirectPVVolumes()) - volumes, err := NewVolumeLister().Get(context.TODO()) + volumes, err := client.NewVolumeLister().Get(context.TODO()) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -51,7 +51,7 @@ func TestGetVolumeList(t *testing.T) { SetDriveInterface(clientset.DirectpvLatest().DirectPVDrives()) SetVolumeInterface(clientset.DirectpvLatest().DirectPVVolumes()) - volumes, err = NewVolumeLister().Get(context.TODO()) + volumes, err = client.NewVolumeLister().Get(context.TODO()) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -65,7 +65,7 @@ func TestGetSortedVolumeList(t *testing.T) { SetDriveInterface(clientset.DirectpvLatest().DirectPVDrives()) SetVolumeInterface(clientset.DirectpvLatest().DirectPVVolumes()) - volumes, err := NewVolumeLister().Get(context.TODO()) + volumes, err := client.NewVolumeLister().Get(context.TODO()) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -94,7 +94,7 @@ func TestGetSortedVolumeList(t *testing.T) { SetDriveInterface(clientset.DirectpvLatest().DirectPVDrives()) SetVolumeInterface(clientset.DirectpvLatest().DirectPVVolumes()) - volumes, err = NewVolumeLister().Get(context.TODO()) + volumes, err = client.NewVolumeLister().Get(context.TODO()) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/pkg/csi/node/server_test.go b/pkg/csi/node/server_test.go index 2ac51731..8794458e 100644 --- a/pkg/csi/node/server_test.go +++ b/pkg/csi/node/server_test.go @@ -27,6 +27,10 @@ import ( "github.com/minio/directpv/pkg/xfs" ) +func init() { + client.FakeInit() +} + func TestNodeExpandVolume(t *testing.T) { volumeID := "volume-id-1" volume := types.NewVolume(volumeID, "fsuuid1", "node-1", "drive-1", "sda", 100*MiB) diff --git a/pkg/k8s/clients.go b/pkg/k8s/clients.go index 372d5e09..0ade6408 100644 --- a/pkg/k8s/clients.go +++ b/pkg/k8s/clients.go @@ -27,30 +27,31 @@ import ( const MaxThreadCount = 200 var ( - initialized int32 - kubeConfig *rest.Config - kubeClient kubernetes.Interface - apiextensionsClient apiextensions.ApiextensionsV1Interface - crdClient apiextensions.CustomResourceDefinitionInterface - discoveryClient discovery.DiscoveryInterface + initialized int32 + client *Client ) +// GetClient returns kubernetes client. +func GetClient() *Client { + return client +} + // KubeConfig gets kubernetes client configuration. func KubeConfig() *rest.Config { - return kubeConfig + return client.KubeConfig } // KubeClient gets kubernetes client. func KubeClient() kubernetes.Interface { - return kubeClient + return client.KubeClient } // CRDClient gets kubernetes CRD client. func CRDClient() apiextensions.CustomResourceDefinitionInterface { - return crdClient + return client.CRDClient } // DiscoveryClient gets kubernetes discovery client. func DiscoveryClient() discovery.DiscoveryInterface { - return discoveryClient + return client.DiscoveryClient } diff --git a/pkg/k8s/fake.go b/pkg/k8s/fake.go index cbbf1e0a..4917c0d9 100644 --- a/pkg/k8s/fake.go +++ b/pkg/k8s/fake.go @@ -47,28 +47,34 @@ func (fd *FakeDiscovery) ServerGroupsAndResources() ([]*metav1.APIGroup, []*meta // FakeInit initializes fake clients. func FakeInit() { + var kubeClient kubernetes.Interface kubeClient = kubernetesfake.NewSimpleClientset() - crdClient = &apiextensionsv1fake.FakeCustomResourceDefinitions{ + crdClient := &apiextensionsv1fake.FakeCustomResourceDefinitions{ Fake: &apiextensionsv1fake.FakeApiextensionsV1{ Fake: &kubeClient.(*kubernetesfake.Clientset).Fake, }, } - discoveryClient = &discoveryfake.FakeDiscovery{} + discoveryClient := &discoveryfake.FakeDiscovery{} scheme := runtime.NewScheme() _ = metav1.AddMetaToScheme(scheme) + client = &Client{ + KubeClient: kubeClient, + CRDClient: crdClient, + DiscoveryClient: discoveryClient, + } } // SetKubeInterface sets the given kube interface // Note: To be used for writing test cases only func SetKubeInterface(i kubernetes.Interface) { - kubeClient = i + client.KubeClient = i } -// SetDiscoveryInterface sets the fake discovery interface +// NewFakeDiscovery creates a fake discovery interface // Note: To be used for writing test cases only -func SetDiscoveryInterface(groupsAndMethodsFn fakeServerGroupsAndResourcesMethod, serverVersionInfo *version.Info) { - discoveryClient = &FakeDiscovery{ - FakeDiscovery: discoveryfake.FakeDiscovery{Fake: &kubeClient.(*kubernetesfake.Clientset).Fake}, +func NewFakeDiscovery(groupsAndMethodsFn fakeServerGroupsAndResourcesMethod, serverVersionInfo *version.Info) *FakeDiscovery { + return &FakeDiscovery{ + FakeDiscovery: discoveryfake.FakeDiscovery{Fake: &client.KubeClient.(*kubernetesfake.Clientset).Fake}, fakeServerGroupsAndResourcesMethod: groupsAndMethodsFn, versionInfo: serverVersionInfo, } diff --git a/pkg/k8s/init.go b/pkg/k8s/init.go index db6eb5a1..f2ed2e8f 100644 --- a/pkg/k8s/init.go +++ b/pkg/k8s/init.go @@ -18,12 +18,15 @@ package k8s import ( "fmt" + "strconv" + "strings" "sync/atomic" apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1" "k8s.io/client-go/discovery" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/klog/v2" // support gcp, azure, and oidc client auth _ "k8s.io/client-go/plugin/pkg/client/auth/azure" @@ -36,25 +39,76 @@ func Init() error { if atomic.AddInt32(&initialized, 1) != 1 { return nil } + kubeConfig, err := GetKubeConfig() + if err != nil { + klog.Fatalf("unable to get kubernetes configuration; %v", err) + } + kubeConfig.WarningHandler = rest.NoWarnings{} + client, err = NewClient(kubeConfig) + if err != nil { + klog.Fatalf("unable to create new kubernetes client interface; %v", err) + } + return nil +} - var err error +// Client represents the kubernetes client set. +type Client struct { + KubeConfig *rest.Config + KubeClient kubernetes.Interface + APIextensionsClient apiextensions.ApiextensionsV1Interface + CRDClient apiextensions.CustomResourceDefinitionInterface + DiscoveryClient discovery.DiscoveryInterface +} - if kubeConfig, err = GetKubeConfig(); err != nil { - return fmt.Errorf("unable to get kubernetes configuration; %v", err) +// GetKubeVersion returns the k8s version info +func (client Client) GetKubeVersion() (major, minor uint, err error) { + versionInfo, err := client.DiscoveryClient.ServerVersion() + if err != nil { + return 0, 0, err } - kubeConfig.WarningHandler = rest.NoWarnings{} - if kubeClient, err = kubernetes.NewForConfig(kubeConfig); err != nil { - return fmt.Errorf("unable to create new kubernetes client interface; %v", err) + var u64 uint64 + if u64, err = strconv.ParseUint(versionInfo.Major, 10, 64); err != nil { + return 0, 0, fmt.Errorf("unable to parse major version %v; %v", versionInfo.Major, err) } + major = uint(u64) - if apiextensionsClient, err = apiextensions.NewForConfig(kubeConfig); err != nil { - return fmt.Errorf("unable to create new API extensions client interface; %v", err) + minorString := versionInfo.Minor + if strings.Contains(versionInfo.GitVersion, "-eks-") { + // Do trimming only for EKS. + // Refer https://github.com/aws/containers-roadmap/issues/1404 + i := strings.IndexFunc(minorString, func(r rune) bool { return r < '0' || r > '9' }) + if i > -1 { + minorString = minorString[:i] + } + } + if u64, err = strconv.ParseUint(minorString, 10, 64); err != nil { + return 0, 0, fmt.Errorf("unable to parse minor version %v; %v", minor, err) } - crdClient = apiextensionsClient.CustomResourceDefinitions() + minor = uint(u64) + return major, minor, nil +} - if discoveryClient, err = discovery.NewDiscoveryClientForConfig(kubeConfig); err != nil { - return fmt.Errorf("unable to create new discovery client interface; %v", err) +// NewClient initializes the client with the provided kube config. +func NewClient(kubeConfig *rest.Config) (*Client, error) { + kubeClient, err := kubernetes.NewForConfig(kubeConfig) + if err != nil { + return nil, fmt.Errorf("unable to create new kubernetes client interface; %v", err) } - return nil + apiextensionsClient, err := apiextensions.NewForConfig(kubeConfig) + if err != nil { + return nil, fmt.Errorf("unable to create new API extensions client interface; %v", err) + } + crdClient := apiextensionsClient.CustomResourceDefinitions() + discoveryClient, err := discovery.NewDiscoveryClientForConfig(kubeConfig) + if err != nil { + return nil, fmt.Errorf("unable to create new discovery client interface; %v", err) + } + return &Client{ + KubeConfig: kubeConfig, + KubeClient: kubeClient, + APIextensionsClient: apiextensionsClient, + CRDClient: crdClient, + DiscoveryClient: discoveryClient, + }, nil } diff --git a/pkg/k8s/k8s.go b/pkg/k8s/k8s.go index 780e9755..d93fc874 100644 --- a/pkg/k8s/k8s.go +++ b/pkg/k8s/k8s.go @@ -64,8 +64,8 @@ func GetKubeConfig() (*rest.Config, error) { } // GetGroupVersionKind gets group/version/kind of given versions. -func GetGroupVersionKind(group, kind string, versions ...string) (*schema.GroupVersionKind, error) { - apiGroupResources, err := restmapper.GetAPIGroupResources(discoveryClient) +func (client Client) GetGroupVersionKind(group, kind string, versions ...string) (*schema.GroupVersionKind, error) { + apiGroupResources, err := restmapper.GetAPIGroupResources(client.DiscoveryClient) if err != nil { klog.ErrorS(err, "unable to get API group resources") return nil, err @@ -90,13 +90,13 @@ func GetGroupVersionKind(group, kind string, versions ...string) (*schema.GroupV } // GetClientForNonCoreGroupVersionKind gets client for group/kind of given versions. -func GetClientForNonCoreGroupVersionKind(group, kind string, versions ...string) (rest.Interface, *schema.GroupVersionKind, error) { - gvk, err := GetGroupVersionKind(group, kind, versions...) +func (client Client) GetClientForNonCoreGroupVersionKind(group, kind string, versions ...string) (rest.Interface, *schema.GroupVersionKind, error) { + gvk, err := client.GetGroupVersionKind(group, kind, versions...) if err != nil { return nil, nil, err } - config := *kubeConfig + config := *client.KubeConfig config.GroupVersion = &schema.GroupVersion{ Group: gvk.Group, Version: gvk.Version, @@ -107,12 +107,12 @@ func GetClientForNonCoreGroupVersionKind(group, kind string, versions ...string) config.UserAgent = rest.DefaultKubernetesUserAgent() } - client, err := rest.RESTClientFor(&config) + restClient, err := rest.RESTClientFor(&config) if err != nil { return nil, nil, err } - return client, gvk, nil + return restClient, gvk, nil } // IsCondition checks whether type/status/reason/message in conditions or not. @@ -199,8 +199,8 @@ func SanitizeResourceName(name string) string { } // GetCSINodes fetches the CSI Node list -func GetCSINodes(ctx context.Context) (nodes []string, err error) { - storageClient, gvk, err := GetClientForNonCoreGroupVersionKind("storage.k8s.io", "CSINode", "v1", "v1beta1", "v1alpha1") +func (client Client) GetCSINodes(ctx context.Context) (nodes []string, err error) { + storageClient, gvk, err := client.GetClientForNonCoreGroupVersionKind("storage.k8s.io", "CSINode", "v1", "v1beta1", "v1alpha1") if err != nil { return nil, err } diff --git a/pkg/k8s/k8s_test.go b/pkg/k8s/k8s_test.go index d0ad8068..9dde4b14 100644 --- a/pkg/k8s/k8s_test.go +++ b/pkg/k8s/k8s_test.go @@ -21,8 +21,75 @@ import ( "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/version" ) +func init() { + FakeInit() +} + +var ( + apiGroups = []*metav1.APIGroup{ + { + Name: "policy", + Versions: []metav1.GroupVersionForDiscovery{ + { + GroupVersion: "policy/v1beta1", + Version: "v1beta1", + }, + }, + }, + { + Name: "storage.k8s.io", + Versions: []metav1.GroupVersionForDiscovery{ + { + GroupVersion: "storage.k8s.io/v1", + Version: "v1", + }, + }, + }, + } + + apiResourceList = []*metav1.APIResourceList{ + { + TypeMeta: metav1.TypeMeta{ + APIVersion: "policy/v1beta1", + Kind: "PodSecurityPolicy", + }, + GroupVersion: "policy/v1beta1", + APIResources: []metav1.APIResource{ + { + Name: "policy", + Group: "policy", + Version: "v1beta1", + Namespaced: false, + Kind: "PodSecurityPolicy", + }, + }, + }, + { + TypeMeta: metav1.TypeMeta{ + APIVersion: "storage.k8s.io/v1", + Kind: "CSIDriver", + }, + GroupVersion: "storage.k8s.io/v1", + APIResources: []metav1.APIResource{ + { + Name: "CSIDriver", + Group: "storage.k8s.io", + Version: "v1", + Namespaced: false, + Kind: "CSIDriver", + }, + }, + }, + } +) + +func getDiscoveryGroupsAndMethods() ([]*metav1.APIGroup, []*metav1.APIResourceList, error) { + return apiGroups, apiResourceList, nil +} + func TestVolumeStatusTransitions(t *testing.T) { statusList := []metav1.Condition{ { @@ -73,3 +140,50 @@ func TestVolumeStatusTransitions(t *testing.T) { }) } } + +func TestGetKubeVersion(t *testing.T) { + testCases := []struct { + info version.Info + major uint + minor uint + expectErr bool + }{ + {version.Info{Major: "a", Minor: "0"}, 0, 0, true}, // invalid major + {version.Info{Major: "-1", Minor: "0"}, 0, 0, true}, // invalid major + {version.Info{Major: "0", Minor: "a"}, 0, 0, true}, // invalid minor + {version.Info{Major: "0", Minor: "-1"}, 0, 0, true}, // invalid minor + {version.Info{Major: "0", Minor: "-1", GitVersion: "commit-eks-id"}, 0, 0, true}, // invalid minor for eks + {version.Info{Major: "0", Minor: "incompat", GitVersion: "commit-eks-"}, 0, 0, true}, // invalid minor for eks + {version.Info{Major: "0", Minor: "0"}, 0, 0, false}, + {version.Info{Major: "1", Minor: "0"}, 1, 0, false}, + {version.Info{Major: "0", Minor: "1"}, 0, 1, false}, + {version.Info{Major: "1", Minor: "18"}, 1, 18, false}, + {version.Info{Major: "1", Minor: "18+", GitVersion: "commit-eks-id"}, 1, 18, false}, + {version.Info{Major: "1", Minor: "18-", GitVersion: "commit-eks-id"}, 1, 18, false}, + {version.Info{Major: "1", Minor: "18incompat", GitVersion: "commit-eks-id"}, 1, 18, false}, + {version.Info{Major: "1", Minor: "18-incompat", GitVersion: "commit-eks-id"}, 1, 18, false}, + } + + for i, testCase := range testCases { + client.DiscoveryClient = NewFakeDiscovery(getDiscoveryGroupsAndMethods, &testCase.info) + major, minor, err := client.GetKubeVersion() + if testCase.expectErr { + if err == nil { + t.Fatalf("case %v: expected error, but succeeded", i+1) + } + continue + } + + if err != nil { + t.Fatalf("case %v: unexpected error: %v", i+1, err) + } + + if major != testCase.major { + t.Fatalf("case %v: major: expected: %v, got: %v", i+1, testCase.major, major) + } + + if minor != testCase.minor { + t.Fatalf("case %v: minor: expected: %v, got: %v", i+1, testCase.minor, minor) + } + } +} diff --git a/pkg/legacy/client/client.go b/pkg/legacy/client/client.go index 4eb52ece..2629ed0c 100644 --- a/pkg/legacy/client/client.go +++ b/pkg/legacy/client/client.go @@ -17,21 +17,46 @@ package client import ( + "context" + "fmt" + "os" + "github.com/minio/directpv/pkg/k8s" directcsi "github.com/minio/directpv/pkg/legacy/apis/direct.csi.min.io/v1beta5" + directv1beta5 "github.com/minio/directpv/pkg/legacy/apis/direct.csi.min.io/v1beta5" typeddirectcsi "github.com/minio/directpv/pkg/legacy/clientset/typed/direct.csi.min.io/v1beta5" + "github.com/minio/directpv/pkg/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/restmapper" - "k8s.io/klog/v2" + "k8s.io/client-go/discovery" ) var ( - initialized int32 - driveClient typeddirectcsi.DirectCSIDriveInterface - volumeClient typeddirectcsi.DirectCSIVolumeInterface + initialized int32 + client *Client ) +// Client represents the legacy client +type Client struct { + DriveClient typeddirectcsi.DirectCSIDriveInterface + VolumeClient typeddirectcsi.DirectCSIVolumeInterface + K8sClient *k8s.Client +} + +// Discovery returns the discovery client +func (client Client) Discovery() discovery.DiscoveryInterface { + return client.K8sClient.DiscoveryClient +} + +// Drive returns the legacy drive client +func (client Client) Drive() typeddirectcsi.DirectCSIDriveInterface { + return client.DriveClient +} + +// Volume returns the volume client +func (client Client) Volume() typeddirectcsi.DirectCSIVolumeInterface { + return client.VolumeClient +} + // DirectCSI group and identity names. const ( GroupName = "direct.csi.min.io" @@ -57,38 +82,99 @@ func DirectCSIVolumeTypeMeta() metav1.TypeMeta { } } -// GetGroupKindVersions gets group/version/kind of given versions. -func GetGroupKindVersions(group, kind string, versions ...string) (*schema.GroupVersionKind, error) { - apiGroupResources, err := restmapper.GetAPIGroupResources(k8s.DiscoveryClient()) - if err != nil { - klog.V(3).Infof("could not obtain API group resources: %v", err) - return nil, err +// DriveClient gets latest versioned drive interface. +func DriveClient() typeddirectcsi.DirectCSIDriveInterface { + return client.DriveClient +} + +// VolumeClient gets latest versioned volume interface. +func VolumeClient() typeddirectcsi.DirectCSIVolumeInterface { + return client.VolumeClient +} + +// GetClient returns the client +func GetClient() *Client { + return client +} + +// RemoveAllDrives removes legacy drive CRDs. +func (client Client) RemoveAllDrives(ctx context.Context, backupFile string) (backupCreated bool, err error) { + var drives []directv1beta5.DirectCSIDrive + for result := range client.ListDrives(ctx) { + if result.Err != nil { + return false, fmt.Errorf("unable to get legacy drives; %w", result.Err) + } + drives = append(drives, result.Drive) } - restMapper := restmapper.NewDiscoveryRESTMapper(apiGroupResources) - gk := schema.GroupKind{ - Group: group, - Kind: kind, + if len(drives) == 0 { + return false, nil } - mapper, err := restMapper.RESTMapping(gk, versions...) + + data, err := utils.ToYAML(directv1beta5.DirectCSIDriveList{ + TypeMeta: metav1.TypeMeta{ + Kind: "List", + APIVersion: "v1", + }, + Items: drives, + }) if err != nil { - klog.V(3).Infof("could not find valid restmapping: %v", err) - return nil, err + return false, fmt.Errorf("unable to generate legacy drives YAML; %w", err) } - gvk := &schema.GroupVersionKind{ - Group: mapper.Resource.Group, - Version: mapper.Resource.Version, - Kind: mapper.Resource.Resource, + if err = os.WriteFile(backupFile, data, os.ModePerm); err != nil { + return false, fmt.Errorf("unable to write legacy drives YAML; %w", err) } - return gvk, nil -} -// DriveClient gets latest versioned drive interface. -func DriveClient() typeddirectcsi.DirectCSIDriveInterface { - return driveClient + for _, drive := range drives { + drive.Finalizers = []string{} + if _, err := client.Drive().Update(ctx, &drive, metav1.UpdateOptions{}); err != nil { + return false, fmt.Errorf("unable to update legacy drive %v; %w", drive.Name, err) + } + if err := client.Drive().Delete(ctx, drive.Name, metav1.DeleteOptions{}); err != nil { + return false, fmt.Errorf("unable to remove legacy drive %v; %w", drive.Name, err) + } + } + + return true, nil } -// VolumeClient gets latest versioned volume interface. -func VolumeClient() typeddirectcsi.DirectCSIVolumeInterface { - return volumeClient +// RemoveAllVolumes removes legacy volume CRDs. +func (client Client) RemoveAllVolumes(ctx context.Context, backupFile string) (backupCreated bool, err error) { + var volumes []directv1beta5.DirectCSIVolume + for result := range client.ListVolumes(ctx) { + if result.Err != nil { + return false, fmt.Errorf("unable to get legacy volumes; %w", result.Err) + } + volumes = append(volumes, result.Volume) + } + if len(volumes) == 0 { + return false, nil + } + + data, err := utils.ToYAML(directv1beta5.DirectCSIVolumeList{ + TypeMeta: metav1.TypeMeta{ + Kind: "List", + APIVersion: "v1", + }, + Items: volumes, + }) + if err != nil { + return false, fmt.Errorf("unable to generate legacy volumes YAML; %w", err) + } + + if err = os.WriteFile(backupFile, data, os.ModePerm); err != nil { + return false, fmt.Errorf("unable to write legacy volumes YAML; %w", err) + } + + for _, volume := range volumes { + volume.Finalizers = nil + if _, err := client.Volume().Update(ctx, &volume, metav1.UpdateOptions{}); err != nil { + return false, fmt.Errorf("unable to update legacy volume %v; %w", volume.Name, err) + } + if err := client.Volume().Delete(ctx, volume.Name, metav1.DeleteOptions{}); err != nil { + return false, fmt.Errorf("unable to remove legacy volume %v; %w", volume.Name, err) + } + } + + return true, nil } diff --git a/pkg/legacy/client/fake.go b/pkg/legacy/client/fake.go index c2f89246..9e949930 100644 --- a/pkg/legacy/client/fake.go +++ b/pkg/legacy/client/fake.go @@ -24,20 +24,21 @@ import ( // FakeInit initializes fake clients. func FakeInit() { k8s.FakeInit() - fakeClientset := legacyclientsetfake.NewSimpleClientset() - driveClient = fakeClientset.DirectV1beta5().DirectCSIDrives() - volumeClient = fakeClientset.DirectV1beta5().DirectCSIVolumes() + client = &Client{ + DriveClient: fakeClientset.DirectV1beta5().DirectCSIDrives(), + VolumeClient: fakeClientset.DirectV1beta5().DirectCSIVolumes(), + } } // SetDriveClient sets drive interface from fake clientset. // Note: To be used for writing test cases only func SetDriveClient(clientset *legacyclientsetfake.Clientset) { - driveClient = clientset.DirectV1beta5().DirectCSIDrives() + client.DriveClient = clientset.DirectV1beta5().DirectCSIDrives() } // SetVolumeClient sets volume interface from fake clientset. // Note: To be used for writing test cases only func SetVolumeClient(clientset *legacyclientsetfake.Clientset) { - volumeClient = clientset.DirectV1beta5().DirectCSIVolumes() + client.VolumeClient = clientset.DirectV1beta5().DirectCSIVolumes() } diff --git a/pkg/legacy/client/init.go b/pkg/legacy/client/init.go index 4b64f924..3140b421 100644 --- a/pkg/legacy/client/init.go +++ b/pkg/legacy/client/init.go @@ -17,7 +17,7 @@ package client import ( - "log" + "fmt" "sync/atomic" "github.com/minio/directpv/pkg/k8s" @@ -29,17 +29,29 @@ func Init() { if atomic.AddInt32(&initialized, 1) != 1 { return } - - if err := k8s.Init(); err != nil { - log.Fatalf("unable to initialize k8s clients; %v", err) - } - var err error - if driveClient, err = DirectCSIDriveInterfaceForConfig(k8s.KubeConfig()); err != nil { - klog.Fatalf("unable to create new DirectCSI drive interface; %v", err) + if err = k8s.Init(); err != nil { + klog.Fatalf("unable to initialize k8s clients; %v", err) } + client, err = NewClient(k8s.GetClient()) + if err != nil { + klog.Fatalf("unable to create legacy client; %v", err) + } +} - if volumeClient, err = DirectCSIVolumeInterfaceForConfig(k8s.KubeConfig()); err != nil { - klog.Fatalf("unable to create new DirectCSI volume interface; %v", err) +// NewClient creates a legacy client +func NewClient(k8sClient *k8s.Client) (*Client, error) { + driveClient, err := DirectCSIDriveInterfaceForConfig(k8sClient) + if err != nil { + return nil, fmt.Errorf("unable to create new DirectCSI drive interface; %v", err) + } + volumeClient, err := DirectCSIVolumeInterfaceForConfig(k8sClient) + if err != nil { + return nil, fmt.Errorf("unable to create new DirectCSI volume interface; %v", err) } + return &Client{ + DriveClient: driveClient, + VolumeClient: volumeClient, + K8sClient: k8sClient, + }, nil } diff --git a/pkg/legacy/client/interface.go b/pkg/legacy/client/interface.go index 34fae095..d35ea75e 100644 --- a/pkg/legacy/client/interface.go +++ b/pkg/legacy/client/interface.go @@ -42,8 +42,8 @@ import ( ) // GetGroupVersion probes group and version of given resource kind. -func GetGroupVersion(kind string) (version, group string, err error) { - gvk, err := GetGroupKindVersions( +func GetGroupVersion(k8sClient *k8s.Client, kind string) (version, group string, err error) { + gvk, err := k8sClient.GetGroupVersionKind( directcsi.Group, kind, directcsi.Version, @@ -56,7 +56,6 @@ func GetGroupVersion(kind string) (version, group string, err error) { if err != nil && !meta.IsNoMatchError(err) { return "", "", err } - version = directcsi.Version if gvk != nil { version = gvk.Version @@ -65,13 +64,12 @@ func GetGroupVersion(kind string) (version, group string, err error) { if gvk != nil { group = gvk.Group } - return version, group, nil } // GetLatestDirectCSIRESTClient gets REST client of the latest direct-csi. -func GetLatestDirectCSIRESTClient() rest.Interface { - directClientset, err := clientset.NewForConfig(k8s.KubeConfig()) +func GetLatestDirectCSIRESTClient(k8sClient *k8s.Client) rest.Interface { + directClientset, err := clientset.NewForConfig(k8sClient.KubeConfig) if err != nil { panic(err) } @@ -100,13 +98,13 @@ type directCSIInterface struct { groupVersion schema.GroupVersion } -func directCSIInterfaceForConfig(config *rest.Config, kind, resource string) (*directCSIInterface, error) { - version, group, err := GetGroupVersion(kind) +func directCSIInterfaceForConfig(k8sClient *k8s.Client, kind, resource string) (*directCSIInterface, error) { + version, group, err := GetGroupVersion(k8sClient, kind) if err != nil { return nil, err } - resourceInterface, err := dynamic.NewForConfig(config) + resourceInterface, err := dynamic.NewForConfig(k8sClient.KubeConfig) if err != nil { return nil, err } @@ -284,8 +282,8 @@ type DirectCSIDriveInterface struct { } // DirectCSIDriveInterfaceForConfig provides a dynamic client interface for DirectCSIDrives -func DirectCSIDriveInterfaceForConfig(config *rest.Config) (*DirectCSIDriveInterface, error) { - inter, err := directCSIInterfaceForConfig(config, "DirectCSIDrive", "directcsidrives") +func DirectCSIDriveInterfaceForConfig(k8sClient *k8s.Client) (*DirectCSIDriveInterface, error) { + inter, err := directCSIInterfaceForConfig(k8sClient, "DirectCSIDrive", "directcsidrives") if err != nil { return nil, err } @@ -398,8 +396,8 @@ type DirectCSIVolumeInterface struct { } // DirectCSIVolumeInterfaceForConfig provides a dynamic client interface for DirectCSIVolumes -func DirectCSIVolumeInterfaceForConfig(config *rest.Config) (*DirectCSIVolumeInterface, error) { - inter, err := directCSIInterfaceForConfig(config, "DirectCSIVolume", "directcsivolumes") +func DirectCSIVolumeInterfaceForConfig(k8sClient *k8s.Client) (*DirectCSIVolumeInterface, error) { + inter, err := directCSIInterfaceForConfig(k8sClient, "DirectCSIVolume", "directcsivolumes") if err != nil { return nil, err } diff --git a/pkg/legacy/client/list.go b/pkg/legacy/client/list.go index 66b6cced..76fcefaf 100644 --- a/pkg/legacy/client/list.go +++ b/pkg/legacy/client/list.go @@ -31,7 +31,7 @@ type ListDriveResult struct { } // ListDrives returns channel to loop through drive items. -func ListDrives(ctx context.Context) <-chan ListDriveResult { +func (client Client) ListDrives(ctx context.Context) <-chan ListDriveResult { resultCh := make(chan ListDriveResult) go func() { defer close(resultCh) @@ -47,7 +47,7 @@ func ListDrives(ctx context.Context) <-chan ListDriveResult { options := metav1.ListOptions{Limit: 1000} for { - result, err := DriveClient().List(ctx, options) + result, err := client.Drive().List(ctx, options) if err != nil { if !apierrors.IsNotFound(err) { send(ListDriveResult{Err: err}) @@ -79,7 +79,7 @@ type ListVolumeResult struct { } // ListVolumes returns channel to loop through volume items. -func ListVolumes(ctx context.Context) <-chan ListVolumeResult { +func (client Client) ListVolumes(ctx context.Context) <-chan ListVolumeResult { resultCh := make(chan ListVolumeResult) go func() { defer close(resultCh) @@ -95,7 +95,7 @@ func ListVolumes(ctx context.Context) <-chan ListVolumeResult { options := metav1.ListOptions{Limit: 1000} for { - result, err := VolumeClient().List(ctx, options) + result, err := client.Volume().List(ctx, options) if err != nil { if !apierrors.IsNotFound(err) { send(ListVolumeResult{Err: err}) diff --git a/pkg/metrics/collector.go b/pkg/metrics/collector.go index 2567eabd..716e0601 100644 --- a/pkg/metrics/collector.go +++ b/pkg/metrics/collector.go @@ -100,7 +100,7 @@ func (c *metricsCollector) Collect(ch chan<- prometheus.Metric) { ctx, cancelFunc := context.WithCancel(context.Background()) defer cancelFunc() - resultCh := client.NewVolumeLister(). + resultCh := client.GetClient().NewVolumeLister(). NodeSelector([]directpvtypes.LabelValue{directpvtypes.ToLabelValue(string(c.nodeID))}). List(ctx) for result := range resultCh { diff --git a/pkg/metrics/collector_test.go b/pkg/metrics/collector_test.go index 75714b0e..5b002787 100644 --- a/pkg/metrics/collector_test.go +++ b/pkg/metrics/collector_test.go @@ -55,6 +55,7 @@ func init() { volumes[0].Status.TargetPath = "/path/targetpath" volumes[1].Status.UsedCapacity = 20 * MiB volumes[1].Status.TargetPath = "/path/targetpath" + client.FakeInit() } func createFakeMetricsCollector() *metricsCollector { diff --git a/pkg/volume/event_test.go b/pkg/volume/event_test.go index ce77c7e3..693a0f70 100644 --- a/pkg/volume/event_test.go +++ b/pkg/volume/event_test.go @@ -30,6 +30,10 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +func init() { + client.FakeInit() +} + const MiB = 1024 * 1024 func createFakeVolumeEventListener(nodeID directpvtypes.NodeID) *volumeEventHandler {