From c0bff5be47a6b691d501e224480bc806af581c89 Mon Sep 17 00:00:00 2001 From: Miskiewicz Date: Tue, 10 Sep 2024 15:24:50 +0200 Subject: [PATCH 1/5] Use Runtime CR to create a kubeconfig --- internal/kubeconfig/builder.go | 48 +++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/internal/kubeconfig/builder.go b/internal/kubeconfig/builder.go index eb6e35626b..a2b305ce93 100644 --- a/internal/kubeconfig/builder.go +++ b/internal/kubeconfig/builder.go @@ -2,12 +2,14 @@ package kubeconfig import ( "bytes" + "context" "fmt" + imv1 "github.com/kyma-project/infrastructure-manager/api/v1" + "github.com/kyma-project/kyma-environment-broker/internal/provisioner" + "sigs.k8s.io/controller-runtime/pkg/client" "text/template" "github.com/kyma-project/kyma-environment-broker/internal" - "github.com/kyma-project/kyma-environment-broker/internal/provisioner" - "gopkg.in/yaml.v2" ) @@ -16,18 +18,20 @@ type Config struct { } type Builder struct { - provisionerClient provisioner.Client kubeconfigProvider kubeconfigProvider + kcpClient client.Client + provisionerClient provisioner.Client } type kubeconfigProvider interface { KubeconfigForRuntimeID(runtimeID string) ([]byte, error) } -func NewBuilder(provisionerClient provisioner.Client, provider kubeconfigProvider) *Builder { +func NewBuilder(provisionerClient provisioner.Client, kcpClient client.Client, provider kubeconfigProvider) *Builder { return &Builder{ - provisionerClient: provisionerClient, + kcpClient: kcpClient, kubeconfigProvider: provider, + provisionerClient: provisionerClient, } } @@ -43,9 +47,12 @@ func (b *Builder) BuildFromAdminKubeconfig(instance *internal.Instance, adminKub if instance.RuntimeID == "" { return "", fmt.Errorf("RuntimeID must not be empty") } - status, err := b.provisionerClient.RuntimeStatus(instance.GlobalAccountID, instance.RuntimeID) + issuerURL, clientID, err := b.getOidcDataFromRuntimeResource(instance.RuntimeID) + if IsNotFound(err) { + issuerURL, clientID, err = b.getOidcDataFromProvisioner(instance) + } if err != nil { - return "", fmt.Errorf("while fetching runtime status from provisioner: %w", err) + return "", fmt.Errorf("while fetching oidc data: %w", err) } var kubeCfg kubeconfig @@ -70,8 +77,8 @@ func (b *Builder) BuildFromAdminKubeconfig(instance *internal.Instance, adminKub ContextName: kubeCfg.CurrentContext, CAData: kubeCfg.Clusters[0].Cluster.CertificateAuthorityData, ServerURL: kubeCfg.Clusters[0].Cluster.Server, - OIDCIssuerURL: status.RuntimeConfiguration.ClusterConfig.OidcConfig.IssuerURL, - OIDCClientID: status.RuntimeConfiguration.ClusterConfig.OidcConfig.ClientID, + OIDCIssuerURL: issuerURL, + OIDCClientID: clientID, }) } @@ -126,3 +133,26 @@ func (b *Builder) validKubeconfig(kc kubeconfig) error { return nil } + +func (b *Builder) getOidcDataFromRuntimeResource(id string) (string, string, error) { + var runtime imv1.Runtime + err := b.kcpClient.Get(context.Background(), client.ObjectKey{Name: id, Namespace: kcpNamespace}, &runtime) + if err != nil { + return "", "", err + } + if runtime.Spec.Shoot.Kubernetes.KubeAPIServer.OidcConfig.IssuerURL == nil { + return "", "", fmt.Errorf("Runtime Resource contains an empty OIDC issuer URL") + } + if runtime.Spec.Shoot.Kubernetes.KubeAPIServer.OidcConfig.ClientID == nil { + return "", "", fmt.Errorf("Runtime Resource contains an empty OIDC client ID") + } + return *runtime.Spec.Shoot.Kubernetes.KubeAPIServer.OidcConfig.IssuerURL, *runtime.Spec.Shoot.Kubernetes.KubeAPIServer.OidcConfig.ClientID, nil +} + +func (b *Builder) getOidcDataFromProvisioner(instance *internal.Instance) (string, string, error) { + status, err := b.provisionerClient.RuntimeStatus(instance.GlobalAccountID, instance.RuntimeID) + if err != nil { + return "", "", err + } + return status.RuntimeConfiguration.ClusterConfig.OidcConfig.IssuerURL, status.RuntimeConfiguration.ClusterConfig.OidcConfig.ClientID, nil +} From d0f579f69fb9a3e863e871392de8f8e7d7e3a7fa Mon Sep 17 00:00:00 2001 From: Miskiewicz Date: Tue, 10 Sep 2024 16:16:43 +0200 Subject: [PATCH 2/5] tests --- internal/kubeconfig/builder.go | 3 +- internal/kubeconfig/builder_test.go | 62 ++++++++++++++++++++++++++--- 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/internal/kubeconfig/builder.go b/internal/kubeconfig/builder.go index a2b305ce93..89a9c03c7a 100644 --- a/internal/kubeconfig/builder.go +++ b/internal/kubeconfig/builder.go @@ -11,6 +11,7 @@ import ( "github.com/kyma-project/kyma-environment-broker/internal" "gopkg.in/yaml.v2" + "k8s.io/apimachinery/pkg/api/errors" ) type Config struct { @@ -48,7 +49,7 @@ func (b *Builder) BuildFromAdminKubeconfig(instance *internal.Instance, adminKub return "", fmt.Errorf("RuntimeID must not be empty") } issuerURL, clientID, err := b.getOidcDataFromRuntimeResource(instance.RuntimeID) - if IsNotFound(err) { + if errors.IsNotFound(err) { issuerURL, clientID, err = b.getOidcDataFromProvisioner(instance) } if err != nil { diff --git a/internal/kubeconfig/builder_test.go b/internal/kubeconfig/builder_test.go index d2e605ca30..ec0918e745 100644 --- a/internal/kubeconfig/builder_test.go +++ b/internal/kubeconfig/builder_test.go @@ -2,6 +2,10 @@ package kubeconfig import ( "fmt" + imv1 "github.com/kyma-project/infrastructure-manager/api/v1" + "github.com/stretchr/testify/assert" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "testing" schema "github.com/kyma-project/control-plane/components/provisioner/pkg/gqlschema" @@ -19,7 +23,11 @@ const ( clientID = "c1id" ) -func TestBuilder_Build(t *testing.T) { +func TestBuilder_BuildFromProvisioner(t *testing.T) { + err := imv1.AddToScheme(scheme.Scheme) + assert.NoError(t, err) + kcpClient := fake.NewClientBuilder().Build() + t.Run("new kubeconfig was build properly", func(t *testing.T) { // given provisionerClient := &automock.Client{} @@ -40,7 +48,7 @@ func TestBuilder_Build(t *testing.T) { }, nil) defer provisionerClient.AssertExpectations(t) - builder := NewBuilder(provisionerClient, NewFakeKubeconfigProvider(skrKubeconfig())) + builder := NewBuilder(provisionerClient, kcpClient, NewFakeKubeconfigProvider(skrKubeconfig())) instance := &internal.Instance{ RuntimeID: runtimeID, @@ -61,7 +69,7 @@ func TestBuilder_Build(t *testing.T) { provisionerClient.On("RuntimeStatus", globalAccountID, runtimeID).Return(schema.RuntimeStatus{}, fmt.Errorf("cannot return kubeconfig")) defer provisionerClient.AssertExpectations(t) - builder := NewBuilder(provisionerClient, NewFakeKubeconfigProvider(skrKubeconfig())) + builder := NewBuilder(provisionerClient, kcpClient, NewFakeKubeconfigProvider(skrKubeconfig())) instance := &internal.Instance{ RuntimeID: runtimeID, GlobalAccountID: globalAccountID, @@ -72,11 +80,55 @@ func TestBuilder_Build(t *testing.T) { //then require.Error(t, err) - require.Contains(t, err.Error(), "while fetching runtime status from provisioner: cannot return kubeconfig") + require.Contains(t, err.Error(), "while fetching oidc data") + }) +} + +func TestBuilder_BuildFromRuntimeResource(t *testing.T) { + err := imv1.AddToScheme(scheme.Scheme) + assert.NoError(t, err) + kcpClient := fake.NewClientBuilder().Build() + + t.Run("new kubeconfig was build properly", func(t *testing.T) { + // given + provisionerClient := &automock.Client{} + provisionerClient.On("RuntimeStatus", globalAccountID, runtimeID).Return(schema.RuntimeStatus{ + RuntimeConfiguration: &schema.RuntimeConfig{ + Kubeconfig: skrKubeconfig(), + ClusterConfig: &schema.GardenerConfig{ + OidcConfig: &schema.OIDCConfig{ + ClientID: clientID, + GroupsClaim: "gclaim", + IssuerURL: issuerURL, + SigningAlgs: nil, + UsernameClaim: "uclaim", + UsernamePrefix: "-", + }, + }, + }, + }, nil) + defer provisionerClient.AssertExpectations(t) + + builder := NewBuilder(provisionerClient, kcpClient, NewFakeKubeconfigProvider(skrKubeconfig())) + + instance := &internal.Instance{ + RuntimeID: runtimeID, + GlobalAccountID: globalAccountID, + } + + // when + kubeconfig, err := builder.Build(instance) + + //then + require.NoError(t, err) + require.Equal(t, kubeconfig, newKubeconfig()) }) } func TestBuilder_BuildFromAdminKubeconfig(t *testing.T) { + err := imv1.AddToScheme(scheme.Scheme) + assert.NoError(t, err) + kcpClient := fake.NewClientBuilder().Build() t.Run("new kubeconfig was build properly", func(t *testing.T) { // given provisionerClient := &automock.Client{} @@ -97,7 +149,7 @@ func TestBuilder_BuildFromAdminKubeconfig(t *testing.T) { }, nil) defer provisionerClient.AssertExpectations(t) - builder := NewBuilder(provisionerClient, NewFakeKubeconfigProvider(skrKubeconfig())) + builder := NewBuilder(provisionerClient, kcpClient, NewFakeKubeconfigProvider(skrKubeconfig())) instance := &internal.Instance{ RuntimeID: runtimeID, From 0f12721855a6f4eadc542178e22ba898f265aa8f Mon Sep 17 00:00:00 2001 From: Miskiewicz Date: Wed, 11 Sep 2024 08:41:43 +0200 Subject: [PATCH 3/5] wip --- cmd/broker/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/broker/main.go b/cmd/broker/main.go index 1d14157fe3..e7f46d7862 100644 --- a/cmd/broker/main.go +++ b/cmd/broker/main.go @@ -337,7 +337,7 @@ func main() { fatalOnError(err, logs) // create kubeconfig builder - kcBuilder := kubeconfig.NewBuilder(provisionerClient, skrK8sClientProvider) + kcBuilder := kubeconfig.NewBuilder(provisionerClient, cli, skrK8sClientProvider) // create server router := mux.NewRouter() From 0cb1e216ebfb24d18639acacc4e66b1136097f29 Mon Sep 17 00:00:00 2001 From: Miskiewicz Date: Wed, 11 Sep 2024 08:56:27 +0200 Subject: [PATCH 4/5] wip --- internal/kubeconfig/builder.go | 3 ++- internal/kubeconfig/builder_test.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/kubeconfig/builder.go b/internal/kubeconfig/builder.go index 89a9c03c7a..5a5ad91a33 100644 --- a/internal/kubeconfig/builder.go +++ b/internal/kubeconfig/builder.go @@ -4,10 +4,11 @@ import ( "bytes" "context" "fmt" + "text/template" + imv1 "github.com/kyma-project/infrastructure-manager/api/v1" "github.com/kyma-project/kyma-environment-broker/internal/provisioner" "sigs.k8s.io/controller-runtime/pkg/client" - "text/template" "github.com/kyma-project/kyma-environment-broker/internal" "gopkg.in/yaml.v2" diff --git a/internal/kubeconfig/builder_test.go b/internal/kubeconfig/builder_test.go index ec0918e745..9833b0eb6b 100644 --- a/internal/kubeconfig/builder_test.go +++ b/internal/kubeconfig/builder_test.go @@ -2,11 +2,12 @@ package kubeconfig import ( "fmt" + "testing" + imv1 "github.com/kyma-project/infrastructure-manager/api/v1" "github.com/stretchr/testify/assert" "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "testing" schema "github.com/kyma-project/control-plane/components/provisioner/pkg/gqlschema" "github.com/kyma-project/kyma-environment-broker/internal" From 7aa96a7d66c2abda67dff12e1f8b33ff8bc6af24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Mi=C5=9Bkiewicz?= Date: Wed, 11 Sep 2024 09:30:08 +0200 Subject: [PATCH 5/5] Update internal/kubeconfig/builder_test.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jarosław Pieszka --- internal/kubeconfig/builder_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/kubeconfig/builder_test.go b/internal/kubeconfig/builder_test.go index 9833b0eb6b..3d6f2cc13c 100644 --- a/internal/kubeconfig/builder_test.go +++ b/internal/kubeconfig/builder_test.go @@ -90,7 +90,7 @@ func TestBuilder_BuildFromRuntimeResource(t *testing.T) { assert.NoError(t, err) kcpClient := fake.NewClientBuilder().Build() - t.Run("new kubeconfig was build properly", func(t *testing.T) { + t.Run("new kubeconfig was built properly", func(t *testing.T) { // given provisionerClient := &automock.Client{} provisionerClient.On("RuntimeStatus", globalAccountID, runtimeID).Return(schema.RuntimeStatus{