From a3fcd02b5a93bcaefb1272cebf61980562798fad Mon Sep 17 00:00:00 2001 From: jun <2456868764@qq.com> Date: Wed, 8 Nov 2023 14:19:33 +0800 Subject: [PATCH] fix: fix hgctl in high kubenetes version problem by auto detecting k8s version and adding helm lookup function --- pkg/cmd/hgctl/helm/render.go | 27 +++++++++- pkg/cmd/hgctl/installer/component.go | 9 ++++ pkg/cmd/hgctl/installer/gateway_api.go | 7 ++- pkg/cmd/hgctl/installer/higress.go | 10 +++- pkg/cmd/hgctl/installer/installer_k8s.go | 31 +++++++++-- pkg/cmd/hgctl/installer/istio.go | 9 +++- pkg/cmd/hgctl/installer/server_info.go | 66 ++++++++++++++++++++++++ pkg/cmd/hgctl/kubernetes/client.go | 8 +++ pkg/cmd/hgctl/uninstall.go | 17 +++--- 9 files changed, 167 insertions(+), 17 deletions(-) create mode 100644 pkg/cmd/hgctl/installer/server_info.go diff --git a/pkg/cmd/hgctl/helm/render.go b/pkg/cmd/hgctl/helm/render.go index 3e72184414..6d6a9487b3 100644 --- a/pkg/cmd/hgctl/helm/render.go +++ b/pkg/cmd/hgctl/helm/render.go @@ -38,6 +38,7 @@ import ( "helm.sh/helm/v3/pkg/engine" "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/repo" + "k8s.io/client-go/rest" "sigs.k8s.io/yaml" ) @@ -134,6 +135,12 @@ type RendererOptions struct { // fields for RemoteRenderer Version string RepoURL string + + // Capabilities + Capabilities *chartutil.Capabilities + + // rest config + restConfig *rest.Config } type RendererOption func(*RendererOptions) @@ -174,6 +181,18 @@ func WithRepoURL(repo string) RendererOption { } } +func WithCapabilities(capabilities *chartutil.Capabilities) RendererOption { + return func(opts *RendererOptions) { + opts.Capabilities = capabilities + } +} + +func WithRestConfig(config *rest.Config) RendererOption { + return func(opts *RendererOptions) { + opts.restConfig = config + } +} + // LocalFileRenderer load yaml files from local file system type LocalFileRenderer struct { Opts *RendererOptions @@ -419,7 +438,11 @@ func renderManifest(valsYaml string, cht *chart.Chart, builtIn bool, opts *Rende Namespace: opts.Namespace, } // TODO need to specify k8s version - caps := chartutil.DefaultCapabilities + var caps *chartutil.Capabilities + caps = opts.Capabilities + if caps == nil { + caps = chartutil.DefaultCapabilities + } // maybe we need a configuration to change this caps resVals, err := chartutil.ToRenderValues(cht, valsMap, RelOpts, caps) if err != nil { @@ -428,7 +451,7 @@ func renderManifest(valsYaml string, cht *chart.Chart, builtIn bool, opts *Rende if builtIn { resVals["Values"].(chartutil.Values)["enabled"] = true } - filesMap, err := engine.Render(cht, resVals) + filesMap, err := engine.RenderWithClient(cht, resVals, opts.restConfig) if err != nil { return "", fmt.Errorf("Render chart failed err: %s", err) } diff --git a/pkg/cmd/hgctl/installer/component.go b/pkg/cmd/hgctl/installer/component.go index 5f03973e9a..9230fd89f7 100644 --- a/pkg/cmd/hgctl/installer/component.go +++ b/pkg/cmd/hgctl/installer/component.go @@ -17,6 +17,7 @@ package installer import ( "github.com/alibaba/higress/pkg/cmd/hgctl/helm" "github.com/alibaba/higress/pkg/cmd/hgctl/util" + "helm.sh/helm/v3/pkg/chartutil" "sigs.k8s.io/yaml" ) @@ -49,6 +50,8 @@ type ComponentOptions struct { ChartName string Version string Quiet bool + // Capabilities + Capabilities *chartutil.Capabilities } type ComponentOption func(*ComponentOptions) @@ -83,6 +86,12 @@ func WithComponentVersion(version string) ComponentOption { } } +func WithComponentCapabilities(capabilities *chartutil.Capabilities) ComponentOption { + return func(opts *ComponentOptions) { + opts.Capabilities = capabilities + } +} + func WithQuiet() ComponentOption { return func(opts *ComponentOptions) { opts.Quiet = true diff --git a/pkg/cmd/hgctl/installer/gateway_api.go b/pkg/cmd/hgctl/installer/gateway_api.go index b2bcc64de0..eb159c708f 100644 --- a/pkg/cmd/hgctl/installer/gateway_api.go +++ b/pkg/cmd/hgctl/installer/gateway_api.go @@ -21,6 +21,7 @@ import ( "strings" "github.com/alibaba/higress/pkg/cmd/hgctl/helm" + "github.com/alibaba/higress/pkg/cmd/hgctl/kubernetes" "github.com/alibaba/higress/pkg/cmd/hgctl/manifests" ) @@ -34,9 +35,10 @@ type GatewayAPIComponent struct { opts *ComponentOptions renderer helm.Renderer writer io.Writer + kubeCli kubernetes.CLIClient } -func NewGatewayAPIComponent(profile *helm.Profile, writer io.Writer, opts ...ComponentOption) (Component, error) { +func NewGatewayAPIComponent(kubeCli kubernetes.CLIClient, profile *helm.Profile, writer io.Writer, opts ...ComponentOption) (Component, error) { newOpts := &ComponentOptions{} for _, opt := range opts { opt(newOpts) @@ -55,6 +57,8 @@ func NewGatewayAPIComponent(profile *helm.Profile, writer io.Writer, opts ...Com helm.WithVersion(newOpts.Version), helm.WithFS(manifests.BuiltinOrDir("")), helm.WithDir(chartDir), + helm.WithCapabilities(newOpts.Capabilities), + helm.WithRestConfig(kubeCli.RESTConfig()), ) if err != nil { return nil, err @@ -65,6 +69,7 @@ func NewGatewayAPIComponent(profile *helm.Profile, writer io.Writer, opts ...Com renderer: renderer, opts: newOpts, writer: writer, + kubeCli: kubeCli, } return gatewayAPIComponent, nil } diff --git a/pkg/cmd/hgctl/installer/higress.go b/pkg/cmd/hgctl/installer/higress.go index 70a098ae6a..90060a4799 100644 --- a/pkg/cmd/hgctl/installer/higress.go +++ b/pkg/cmd/hgctl/installer/higress.go @@ -17,8 +17,10 @@ package installer import ( "errors" "fmt" - "github.com/alibaba/higress/pkg/cmd/hgctl/helm" "io" + + "github.com/alibaba/higress/pkg/cmd/hgctl/helm" + "github.com/alibaba/higress/pkg/cmd/hgctl/kubernetes" ) const ( @@ -31,6 +33,7 @@ type HigressComponent struct { opts *ComponentOptions renderer helm.Renderer writer io.Writer + kubeCli kubernetes.CLIClient } func (h *HigressComponent) ComponentName() ComponentName { @@ -89,7 +92,7 @@ func (h *HigressComponent) RenderManifest() (string, error) { return manifest, nil } -func NewHigressComponent(profile *helm.Profile, writer io.Writer, opts ...ComponentOption) (Component, error) { +func NewHigressComponent(kubeCli kubernetes.CLIClient, profile *helm.Profile, writer io.Writer, opts ...ComponentOption) (Component, error) { newOpts := &ComponentOptions{} for _, opt := range opts { opt(newOpts) @@ -105,6 +108,8 @@ func NewHigressComponent(profile *helm.Profile, writer io.Writer, opts ...Compon helm.WithNamespace(newOpts.Namespace), helm.WithRepoURL(newOpts.RepoURL), helm.WithVersion(newOpts.Version), + helm.WithCapabilities(newOpts.Capabilities), + helm.WithRestConfig(kubeCli.RESTConfig()), ) if err != nil { return nil, err @@ -115,6 +120,7 @@ func NewHigressComponent(profile *helm.Profile, writer io.Writer, opts ...Compon renderer: renderer, opts: newOpts, writer: writer, + kubeCli: kubeCli, } return higressComponent, nil } diff --git a/pkg/cmd/hgctl/installer/installer_k8s.go b/pkg/cmd/hgctl/installer/installer_k8s.go index 2b3f8d9646..e891d834dc 100644 --- a/pkg/cmd/hgctl/installer/installer_k8s.go +++ b/pkg/cmd/hgctl/installer/installer_k8s.go @@ -19,6 +19,7 @@ import ( "fmt" "io" "os" + "path/filepath" "github.com/alibaba/higress/pkg/cmd/hgctl/helm" "github.com/alibaba/higress/pkg/cmd/hgctl/helm/object" @@ -202,6 +203,19 @@ func (o *K8sInstaller) DeleteManifests(manifestMap map[ComponentName]string) err return nil } +// WriteManifests write component manifests to local files +func (o *K8sInstaller) WriteManifests(manifestMap map[ComponentName]string) error { + if o.kubeCli == nil { + return errors.New("no injected k8s cli into K8sInstaller") + } + rootPath, _ := os.Getwd() + for name, manifest := range manifestMap { + fileName := filepath.Join(rootPath, string(name)+".yaml") + util.WriteFileString(fileName, manifest, 0o644) + } + return nil +} + // deleteManifest delete manifest to certain namespace func (o *K8sInstaller) deleteManifest(manifest string, ns string) error { objs, err := object.ParseK8sObjectsFromYAMLManifest(manifest) @@ -239,6 +253,14 @@ func NewK8sInstaller(profile *helm.Profile, cli kubernetes.CLIClient, writer io. if profile == nil { return nil, errors.New("install profile is empty") } + // initialize server info + serverInfo, _ := NewServerInfo(cli) + fmt.Fprintf(writer, "\n⌛️ Detecting kubernetes version ... ") + capabilities, err := serverInfo.GetCapabilities() + if err != nil { + return nil, err + } + fmt.Fprintf(writer, "%s\n", capabilities.KubeVersion.Version) // initialize components components := make(map[ComponentName]Component) opts := []ComponentOption{ @@ -247,11 +269,12 @@ func NewK8sInstaller(profile *helm.Profile, cli kubernetes.CLIClient, writer io. WithComponentVersion(profile.Charts.Higress.Version), WithComponentRepoURL(profile.Charts.Higress.Url), WithComponentChartName(profile.Charts.Higress.Name), + WithComponentCapabilities(capabilities), } if quiet { opts = append(opts, WithQuiet()) } - higressComponent, err := NewHigressComponent(profile, writer, opts...) + higressComponent, err := NewHigressComponent(cli, profile, writer, opts...) if err != nil { return nil, fmt.Errorf("NewHigressComponent failed, err: %s", err) } @@ -267,12 +290,13 @@ func NewK8sInstaller(profile *helm.Profile, cli kubernetes.CLIClient, writer io. WithComponentVersion("1.18.2"), WithComponentRepoURL("embed://istiobase"), WithComponentChartName("istio"), + WithComponentCapabilities(capabilities), } if quiet { opts = append(opts, WithQuiet()) } - istioCRDComponent, err := NewIstioCRDComponent(profile, writer, opts...) + istioCRDComponent, err := NewIstioCRDComponent(cli, profile, writer, opts...) if err != nil { return nil, fmt.Errorf("NewIstioCRDComponent failed, err: %s", err) } @@ -285,12 +309,13 @@ func NewK8sInstaller(profile *helm.Profile, cli kubernetes.CLIClient, writer io. WithComponentVersion("1.0.0"), WithComponentRepoURL("embed://gatewayapi"), WithComponentChartName("gatewayAPI"), + WithComponentCapabilities(capabilities), } if quiet { opts = append(opts, WithQuiet()) } - gatewayAPIComponent, err := NewGatewayAPIComponent(profile, writer, opts...) + gatewayAPIComponent, err := NewGatewayAPIComponent(cli, profile, writer, opts...) if err != nil { return nil, fmt.Errorf("NewGatewayAPIComponent failed, err: %s", err) } diff --git a/pkg/cmd/hgctl/installer/istio.go b/pkg/cmd/hgctl/installer/istio.go index 52a8744305..07e5f07dda 100644 --- a/pkg/cmd/hgctl/installer/istio.go +++ b/pkg/cmd/hgctl/installer/istio.go @@ -20,6 +20,7 @@ import ( "strings" "github.com/alibaba/higress/pkg/cmd/hgctl/helm" + "github.com/alibaba/higress/pkg/cmd/hgctl/kubernetes" "github.com/alibaba/higress/pkg/cmd/hgctl/manifests" ) @@ -33,9 +34,10 @@ type IstioCRDComponent struct { opts *ComponentOptions renderer helm.Renderer writer io.Writer + kubeCli kubernetes.CLIClient } -func NewIstioCRDComponent(profile *helm.Profile, writer io.Writer, opts ...ComponentOption) (Component, error) { +func NewIstioCRDComponent(kubeCli kubernetes.CLIClient, profile *helm.Profile, writer io.Writer, opts ...ComponentOption) (Component, error) { newOpts := &ComponentOptions{} for _, opt := range opts { opt(newOpts) @@ -54,6 +56,8 @@ func NewIstioCRDComponent(profile *helm.Profile, writer io.Writer, opts ...Compo helm.WithVersion(newOpts.Version), helm.WithFS(manifests.BuiltinOrDir("")), helm.WithDir(chartDir), + helm.WithCapabilities(newOpts.Capabilities), + helm.WithRestConfig(kubeCli.RESTConfig()), ) if err != nil { return nil, err @@ -64,6 +68,8 @@ func NewIstioCRDComponent(profile *helm.Profile, writer io.Writer, opts ...Compo helm.WithNamespace(newOpts.Namespace), helm.WithRepoURL(newOpts.RepoURL), helm.WithVersion(newOpts.Version), + helm.WithCapabilities(newOpts.Capabilities), + helm.WithRestConfig(kubeCli.RESTConfig()), ) if err != nil { return nil, err @@ -75,6 +81,7 @@ func NewIstioCRDComponent(profile *helm.Profile, writer io.Writer, opts ...Compo renderer: renderer, opts: newOpts, writer: writer, + kubeCli: kubeCli, } return istioComponent, nil } diff --git a/pkg/cmd/hgctl/installer/server_info.go b/pkg/cmd/hgctl/installer/server_info.go new file mode 100644 index 0000000000..1760354f6c --- /dev/null +++ b/pkg/cmd/hgctl/installer/server_info.go @@ -0,0 +1,66 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installer + +import ( + "github.com/alibaba/higress/pkg/cmd/hgctl/kubernetes" + "github.com/pkg/errors" + "helm.sh/helm/v3/pkg/action" + "helm.sh/helm/v3/pkg/chartutil" + "k8s.io/client-go/discovery" +) + +type ServerInfo struct { + kubeCli kubernetes.CLIClient +} + +func (c *ServerInfo) GetCapabilities() (*chartutil.Capabilities, error) { + // force a discovery cache invalidation to always fetch the latest server version/capabilities. + dc := c.kubeCli.KubernetesInterface().Discovery() + + kubeVersion, err := dc.ServerVersion() + if err != nil { + return nil, errors.Wrap(err, "could not get server version from Kubernetes") + } + // Issue #6361: + // Client-Go emits an error when an API service is registered but unimplemented. + // We trap that error here and print a warning. But since the discovery client continues + // building the API object, it is correctly populated with all valid APIs. + // See https://github.com/kubernetes/kubernetes/issues/72051#issuecomment-521157642 + apiVersions, err := action.GetVersionSet(dc) + if err != nil { + if discovery.IsGroupDiscoveryFailedError(err) { + } else { + return nil, errors.Wrap(err, "could not get apiVersions from Kubernetes") + } + } + capabilities := &chartutil.Capabilities{ + APIVersions: apiVersions, + KubeVersion: chartutil.KubeVersion{ + Version: kubeVersion.GitVersion, + Major: kubeVersion.Major, + Minor: kubeVersion.Minor, + }, + HelmVersion: chartutil.DefaultCapabilities.HelmVersion, + } + return capabilities, nil +} + +func NewServerInfo(kubCli kubernetes.CLIClient) (*ServerInfo, error) { + serverInfo := &ServerInfo{ + kubeCli: kubCli, + } + return serverInfo, nil +} diff --git a/pkg/cmd/hgctl/kubernetes/client.go b/pkg/cmd/hgctl/kubernetes/client.go index 2a7ab0e9d8..b1776ddc4c 100644 --- a/pkg/cmd/hgctl/kubernetes/client.go +++ b/pkg/cmd/hgctl/kubernetes/client.go @@ -57,6 +57,9 @@ type CLIClient interface { // CreateNamespace create namespace CreateNamespace(namespace string) error + + // KubernetesInterface get kubernetes interface + KubernetesInterface() kubernetes.Interface } var _ CLIClient = &client{} @@ -246,3 +249,8 @@ func (c *client) CreateNamespace(namespace string) error { return nil } + +// KubernetesInterface get kubernetes interface +func (c *client) KubernetesInterface() kubernetes.Interface { + return c.kube +} diff --git a/pkg/cmd/hgctl/uninstall.go b/pkg/cmd/hgctl/uninstall.go index 9f9fbff212..bcb9e1c788 100644 --- a/pkg/cmd/hgctl/uninstall.go +++ b/pkg/cmd/hgctl/uninstall.go @@ -26,13 +26,13 @@ import ( ) type uninstallArgs struct { - // purgeIstioCRD delete all of Istio resources. - purgeIstioCRD bool + // purgeResources delete all of installed resources. + purgeResources bool } func addUninstallFlags(cmd *cobra.Command, args *uninstallArgs) { - cmd.PersistentFlags().BoolVarP(&args.purgeIstioCRD, "purge-istio-crd", "", false, - "Delete all of Istio resources") + cmd.PersistentFlags().BoolVarP(&args.purgeResources, "purge-resources", "", false, + "Delete all of IstioAPI,GatewayAPI resources") } // newUninstallCmd command uninstalls Istio from a cluster @@ -42,11 +42,11 @@ func newUninstallCmd() *cobra.Command { Use: "uninstall", Short: "Uninstall higress from a cluster", Long: "The uninstall command uninstalls higress from a cluster or local environment", - Example: ` # Uninstall higress + Example: `# Uninstall higress hgctl uninstal - # Uninstall higress and istio CRD from a cluster - hgctl uninstall --purge-istio-crd + # Uninstall higress, istioAPI and GatewayAPI from a cluster + hgctl uninstall --purge-resources `, RunE: func(cmd *cobra.Command, args []string) error { return uninstall(cmd.OutOrStdout(), uiArgs) @@ -82,7 +82,8 @@ func uninstall(writer io.Writer, uiArgs *uninstallArgs) error { } if profile.Global.Install == helm.InstallK8s || profile.Global.Install == helm.InstallLocalK8s { - profile.Global.EnableIstioAPI = uiArgs.purgeIstioCRD + profile.Global.EnableIstioAPI = uiArgs.purgeResources + profile.Global.EnableGatewayAPI = uiArgs.purgeResources } err = uninstallManifests(profile, writer, uiArgs)