Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support Karmada installed by operator to use karmadactl register #5755

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
160 changes: 160 additions & 0 deletions operator/pkg/karmadaresource/rbac/manifest.go
Original file line number Diff line number Diff line change
Expand Up @@ -195,4 +195,164 @@ subjects:
- kind: User
name: "system:admin"
`

// ClusterInfoRole defines a role with permission to get the cluster-info configmap
ClusterInfoRole = `
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
labels:
karmada.io/bootstrapping: rbac-defaults
name: system:karmada:bootstrap-signer-clusterinfo
namespace: kube-public
rules:
- apiGroups:
- ""
resourceNames:
- cluster-info
resources:
- configmaps
verbs:
- get
`

// ClusterInfoRoleBinding authorizes system:anonymous to get the cluster-info configmap
ClusterInfoRoleBinding = `
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
karmada.io/bootstrapping: rbac-defaults
name: system:karmada:bootstrap-signer-clusterinfo
namespace: kube-public
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: system:karmada:bootstrap-signer-clusterinfo
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: system:anonymous
`

// CSRAutoApproverClusterRole defines a ClusterRole with permissions to automatically approve the agent CSRs when the agentcsrapproving controller is enabled by karmada-controller-manager
CSRAutoApproverClusterRole = `
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
karmada.io/bootstrapping: rbac-defaults
name: system:karmada:certificatesigningrequest:autoapprover
rules:
- apiGroups:
- certificates.k8s.io
resources:
- certificatesigningrequests/clusteragent
verbs:
- create
`

// CSRAutoApproverClusterRoleBinding authorizes Group system:bootstrappers:karmada:default-cluster-token to auto approve the agent CSRs
CSRAutoApproverClusterRoleBinding = `
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
karmada.io/bootstrapping: rbac-defaults
name: system:karmada:agent-autoapprove-bootstrap
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:karmada:certificatesigningrequest:autoapprover
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:bootstrappers:karmada:default-cluster-token
`

// AgentBootstrapClusterRoleBinding authorizes Group system:bootstrappers:karmada:default-cluster-token to obtain the CSRs.
AgentBootstrapClusterRoleBinding = `
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
karmada.io/bootstrapping: rbac-defaults
name: system:karmada:agent-bootstrap
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:node-bootstrapper
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:bootstrappers:karmada:default-cluster-token
`

// CSRSelfAutoApproverClusterRole defines a ClusterRole with permissions to automatically approve the agent CSRs
CSRSelfAutoApproverClusterRole = `
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
karmada.io/bootstrapping: rbac-defaults
name: system:karmada:certificatesigningrequest:selfautoapprover
rules:
- apiGroups:
- certificates.k8s.io
resources:
- certificatesigningrequests/selfclusteragent
verbs:
- create
`

// CSRSelfAutoApproverClusterRoleBinding authorizes Group system:karmada:agents to automatically approve the agent CSRs
CSRSelfAutoApproverClusterRoleBinding = `
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
karmada.io/bootstrapping: rbac-defaults
name: system:karmada:agent-autoapprove-certificate-rotation
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:karmada:certificatesigningrequest:selfautoapprover
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:karmada:agents
`

// AgentRBACGeneratorClusterRole is not used for the connection between the karmada-agent and the control plane,
// but is used by karmadactl register to generate the RBAC resources required by the karmada-agent.
AgentRBACGeneratorClusterRole = `
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
karmada.io/bootstrapping: rbac-defaults
name: system:karmada:agent-rbac-generator
rules:
- apiGroups: ['*']
resources: ['*']
verbs: ['*']
`

// AgentRBACGeneratorClusterRoleBinding User `system:karmada:agent:rbac-generator` is specifically used during the `karmadactl register` process to generate restricted RBAC resources for the `karmada-agent`
AgentRBACGeneratorClusterRoleBinding = `
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
karmada.io/bootstrapping: rbac-defaults
name: system:karmada:agent-rbac-generator
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:karmada:agent-rbac-generator
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: system:karmada:agent:rbac-generator
`
)
96 changes: 72 additions & 24 deletions operator/pkg/karmadaresource/rbac/rbac.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,41 +36,89 @@ func EnsureKarmadaRBAC(client clientset.Interface) error {
if err := grantKarmadaResourceViewClusterRole(client); err != nil {
return err
}
return grantKarmadaResourceEditClusterRole(client)
if err := grantKarmadaResourceEditClusterRole(client); err != nil {
return err
}
return grantPullModeRBAC(client)
}

func grantClusterProxyAdminRBAC(client clientset.Interface) error {
role := &rbacv1.ClusterRole{}
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), []byte(ClusterProxyAdminClusterRole), role); err != nil {
return fmt.Errorf("err when decoding ClusterProxyAdmin ClusterRole: %w", err)
if err := grantRBACClusterRole(client, ClusterProxyAdminClusterRole); err != nil {
return err
}

return grantRBACClusterRoleBinding(client, ClusterProxyAdminClusterRoleBinding)
}

func grantKarmadaResourceViewClusterRole(client clientset.Interface) error {
return grantRBACClusterRole(client, KarmadaResourceViewClusterRole)
}

func grantKarmadaResourceEditClusterRole(client clientset.Interface) error {
return grantRBACClusterRole(client, KarmadaResourceEditClusterRole)
}

func grantPullModeRBAC(client clientset.Interface) error {
if err := grantRBACRole(client, ClusterInfoRole); err != nil {
return err
}
if err := grantRBACRoleBinding(client, ClusterInfoRoleBinding); err != nil {
return err
}
if err := grantRBACClusterRole(client, CSRAutoApproverClusterRole); err != nil {
return err
}
util.MergeLabel(role, util.KarmadaSystemLabel, util.KarmadaSystemLabelValue)
if err := apiclient.CreateOrUpdateClusterRole(client, role); err != nil {
return fmt.Errorf("failed to create or update ClusterRole: %w", err)
if err := grantRBACClusterRoleBinding(client, CSRAutoApproverClusterRoleBinding); err != nil {
return err
}
if err := grantRBACClusterRoleBinding(client, AgentBootstrapClusterRoleBinding); err != nil {
return err
}
if err := grantRBACClusterRole(client, CSRSelfAutoApproverClusterRole); err != nil {
return err
}
if err := grantRBACClusterRoleBinding(client, CSRSelfAutoApproverClusterRoleBinding); err != nil {
return err
}
if err := grantRBACClusterRole(client, AgentRBACGeneratorClusterRole); err != nil {
return err
}

return grantRBACClusterRoleBinding(client, AgentRBACGeneratorClusterRoleBinding)
}

roleBinding := &rbacv1.ClusterRoleBinding{}
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), []byte(ClusterProxyAdminClusterRoleBinding), roleBinding); err != nil {
return fmt.Errorf("err when decoding ClusterProxyAdmin ClusterRoleBinding: %w", err)
func grantRBACRole(client clientset.Interface, role string) error {
roleObject := &rbacv1.Role{}
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), []byte(role), roleObject); err != nil {
return fmt.Errorf("err when decoding into Role object: %w", err)
}
util.MergeLabel(role, util.KarmadaSystemLabel, util.KarmadaSystemLabelValue)
return apiclient.CreateOrUpdateClusterRoleBinding(client, roleBinding)
util.MergeLabel(roleObject, util.KarmadaSystemLabel, util.KarmadaSystemLabelValue)
return apiclient.CreateOrUpdateRole(client, roleObject)
}

func grantKarmadaResourceViewClusterRole(client clientset.Interface) error {
role := &rbacv1.ClusterRole{}
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), []byte(KarmadaResourceViewClusterRole), role); err != nil {
return fmt.Errorf("err when decoding Karmada view ClusterRole: %w", err)
func grantRBACRoleBinding(client clientset.Interface, roleBinding string) error {
roleBindingObject := &rbacv1.RoleBinding{}
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), []byte(roleBinding), roleBindingObject); err != nil {
return fmt.Errorf("err when decoding into RoleBinding object: %w", err)
}
util.MergeLabel(role, util.KarmadaSystemLabel, util.KarmadaSystemLabelValue)
return apiclient.CreateOrUpdateClusterRole(client, role)
util.MergeLabel(roleBindingObject, util.KarmadaSystemLabel, util.KarmadaSystemLabelValue)
return apiclient.CreateOrUpdateRoleBinding(client, roleBindingObject)
}

func grantKarmadaResourceEditClusterRole(client clientset.Interface) error {
role := &rbacv1.ClusterRole{}
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), []byte(KarmadaResourceEditClusterRole), role); err != nil {
return fmt.Errorf("err when decoding Karmada edit ClusterRole: %w", err)
func grantRBACClusterRole(client clientset.Interface, clusterRole string) error {
clusterRoleObject := &rbacv1.ClusterRole{}
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), []byte(clusterRole), clusterRoleObject); err != nil {
return fmt.Errorf("err when decoding into ClusterRole object: %w", err)
}
util.MergeLabel(clusterRoleObject, util.KarmadaSystemLabel, util.KarmadaSystemLabelValue)
return apiclient.CreateOrUpdateClusterRole(client, clusterRoleObject)
}

func grantRBACClusterRoleBinding(client clientset.Interface, clusterRoleBinding string) error {
clusterRoleBindingObject := &rbacv1.ClusterRoleBinding{}
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), []byte(clusterRoleBinding), clusterRoleBindingObject); err != nil {
return fmt.Errorf("err when decoding into RoleBinding object: %w", err)
}
util.MergeLabel(role, util.KarmadaSystemLabel, util.KarmadaSystemLabelValue)
return apiclient.CreateOrUpdateClusterRole(client, role)
util.MergeLabel(clusterRoleBindingObject, util.KarmadaSystemLabel, util.KarmadaSystemLabelValue)
return apiclient.CreateOrUpdateClusterRoleBinding(client, clusterRoleBindingObject)
}
4 changes: 2 additions & 2 deletions operator/pkg/karmadaresource/rbac/rbac_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ func TestEnsureKarmadaRBAC(t *testing.T) {
}

actions := fakeClient.Actions()
if len(actions) != 4 {
t.Fatalf("expected 4 actions, but got %d", len(actions))
if len(actions) != 13 {
t.Fatalf("expected 13 actions, but got %d", len(actions))
}
}

Expand Down
59 changes: 59 additions & 0 deletions operator/pkg/tasks/init/karmadaresource.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package tasks

import (
"context"
"encoding/base64"
"encoding/json"
"errors"
Expand All @@ -29,6 +30,10 @@ import (
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
crdsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
bootstrapapi "k8s.io/cluster-bootstrap/token/api"
"k8s.io/klog/v2"

"github.com/karmada-io/karmada/operator/pkg/constants"
Expand All @@ -37,6 +42,7 @@ import (
"github.com/karmada-io/karmada/operator/pkg/util"
"github.com/karmada-io/karmada/operator/pkg/util/apiclient"
"github.com/karmada-io/karmada/operator/pkg/workflow"
cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util"
)

// NewKarmadaResourcesTask init KarmadaResources task
Expand All @@ -62,6 +68,10 @@ func NewKarmadaResourcesTask() workflow.Task {
Name: "APIService",
Run: runAPIService,
},
{
Name: "BootstrapConfigMap",
Run: runBootstrapConfigMap,
},
},
}
}
Expand Down Expand Up @@ -221,6 +231,55 @@ func runAPIService(r workflow.RunData) error {
return nil
}

func runBootstrapConfigMap(r workflow.RunData) error {
data, ok := r.(InitData)
if !ok {
return errors.New("BootstrapConfigMap task invoked with an invalid data struct")
}

clientSet, err := kubernetes.NewForConfig(data.ControlplaneConfig())
if err != nil {
return err
}

adminConfigSecret, err := data.RemoteClient().CoreV1().Secrets(data.GetNamespace()).Get(context.Background(), util.AdminKubeconfigSecretName(data.GetName()), metav1.GetOptions{})
if err != nil {
return err
}

adminConfig, err := clientcmd.Load(adminConfigSecret.Data["kubeconfig"])
if err != nil {
return err
}

if err = clientcmdapi.FlattenConfig(adminConfig); err != nil {
return err
}

adminCluster := adminConfig.Contexts[adminConfig.CurrentContext].Cluster
// Copy the cluster to the bootstrap kubeconfig, contains the CA cert and the server URL
bootstrapConfig := &clientcmdapi.Config{
Clusters: map[string]*clientcmdapi.Cluster{
"": adminConfig.Clusters[adminCluster],
},
}
bootstrapBytes, err := clientcmd.Write(*bootstrapConfig)
if err != nil {
return err
}

// Create or update the ConfigMap in the kube-public namespace
return cmdutil.CreateOrUpdateConfigMap(clientSet, &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: bootstrapapi.ConfigMapClusterInfo,
Namespace: metav1.NamespacePublic,
},
Data: map[string]string{
bootstrapapi.KubeConfigKey: string(bootstrapBytes),
},
})
}

func splitToCrdNameFormFile(file string, start, end string) string {
index := strings.LastIndex(file, start)
crdName := file[index+1:]
Expand Down
Loading
Loading