Skip to content

Commit

Permalink
Configurable service CIDR (#189)
Browse files Browse the repository at this point in the history
* initial work for customizable service CIDR

* add service cidr to interactive bootstrap; add first ip4 service ip to sans; test service cidrs

* handle ip6 service cidrs in addition to ip4

* Update src/k8s/pkg/utils/cidr.go

Co-authored-by: Angelos Kolaitis <[email protected]>

* rework invalid tests into subtests

* Update src/k8s/pkg/k8sd/types/cluster_config_test.go

Co-authored-by: Angelos Kolaitis <[email protected]>

* lets tell some stories

---------

Co-authored-by: Angelos Kolaitis <[email protected]>
  • Loading branch information
kwmonroe and neoaggelos authored Mar 5, 2024
1 parent 43966cb commit 004a1af
Show file tree
Hide file tree
Showing 9 changed files with 179 additions and 24 deletions.
3 changes: 3 additions & 0 deletions src/k8s/api/v1/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ type BootstrapConfig struct {
Components []string `yaml:"components"`
// ClusterCIDR is the CIDR of the cluster.
ClusterCIDR string `yaml:"cluster-cidr"`
// ServiceCIDR is the CIDR of the cluster services.
ServiceCIDR string `yaml:"service-cidr"`
// EnableRBAC determines if RBAC will be enabled; *bool to know true/false/unset.
EnableRBAC *bool `yaml:"enable-rbac"`
K8sDqlitePort int `yaml:"k8s-dqlite-port"`
Expand All @@ -22,6 +24,7 @@ type BootstrapConfig struct {
func (b *BootstrapConfig) SetDefaults() {
b.Components = []string{"dns", "metrics-server", "network"}
b.ClusterCIDR = "10.1.0.0/16"
b.ServiceCIDR = "10.152.183.0/24"
b.EnableRBAC = vals.Pointer(true)
b.K8sDqlitePort = 9000
}
Expand Down
2 changes: 1 addition & 1 deletion src/k8s/cmd/k8s/k8s_bootstrap.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ func getConfigInteractively() apiv1.BootstrapConfig {
config.Components = strings.Split(components, ",")

config.ClusterCIDR = askQuestion("Please set the Cluster CIDR:", nil, config.ClusterCIDR, nil)

config.ServiceCIDR = askQuestion("Please set the Service CIDR:", nil, config.ServiceCIDR, nil)
rbac := askBool("Enable Role Based Access Control (RBAC)?", []string{"yes", "no"}, "yes")
*config.EnableRBAC = rbac
return config
Expand Down
20 changes: 14 additions & 6 deletions src/k8s/pkg/k8sd/app/hooks_bootstrap.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/canonical/k8s/pkg/k8sd/types"
"github.com/canonical/k8s/pkg/snap"
snaputil "github.com/canonical/k8s/pkg/snap/util"
"github.com/canonical/k8s/pkg/utils"
"github.com/canonical/k8s/pkg/utils/k8s"
"github.com/canonical/k8s/pkg/utils/vals"
"github.com/canonical/microcluster/state"
Expand Down Expand Up @@ -165,19 +166,26 @@ func onBootstrapControlPlane(s *state.State, initConfig map[string]string) error
if nodeIP == nil {
return fmt.Errorf("failed to parse node IP address %q", s.Address().Hostname())
}
certificates := pki.NewControlPlanePKI(pki.ControlPlanePKIOpts{
Hostname: s.Name(),
IPSANs: []net.IP{nodeIP},
Years: 10,
AllowSelfSignedCA: true,
})

// Create directories
if err := setup.EnsureAllDirectories(snap); err != nil {
return fmt.Errorf("failed to create directories: %w", err)
}

// cfg.Network.ServiceCIDR may be "IPv4CIDR[,IPv6CIDR]". get the first ip from CIDR(s).
serviceIPs, err := utils.GetKubernetesServiceIPsFromServiceCIDRs(cfg.Network.ServiceCIDR)
if err != nil {
return fmt.Errorf("failed to get IP address(es) from ServiceCIDR %q: %w", cfg.Network.ServiceCIDR, err)
}

// Certificates
certificates := pki.NewControlPlanePKI(pki.ControlPlanePKIOpts{
Hostname: s.Name(),
IPSANs: append([]net.IP{nodeIP}, serviceIPs...),
Years: 10,
AllowSelfSignedCA: true,
})

if err := certificates.CompleteCertificates(); err != nil {
return fmt.Errorf("failed to initialize cluster certificates: %w", err)
}
Expand Down
8 changes: 7 additions & 1 deletion src/k8s/pkg/k8sd/app/hooks_join.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,16 @@ func onPostJoin(s *state.State, initConfig map[string]string) error {
return fmt.Errorf("failed to create directories: %w", err)
}

// cfg.Network.ServiceCIDR may be "IPv4CIDR[,IPv6CIDR]". get the first ip from CIDR(s).
serviceIPs, err := utils.GetKubernetesServiceIPsFromServiceCIDRs(cfg.Network.ServiceCIDR)
if err != nil {
return fmt.Errorf("failed to get IP address(es) from ServiceCIDR %q: %w", cfg.Network.ServiceCIDR, err)
}

// Certificates
certificates := pki.NewControlPlanePKI(pki.ControlPlanePKIOpts{
Hostname: s.Name(),
IPSANs: []net.IP{nodeIP},
IPSANs: append([]net.IP{nodeIP}, serviceIPs...),
Years: 10,
})

Expand Down
3 changes: 1 addition & 2 deletions src/k8s/pkg/k8sd/pki/control_plane.go
Original file line number Diff line number Diff line change
Expand Up @@ -202,12 +202,11 @@ func (c *ControlPlanePKI) CompleteCertificates() error {
return fmt.Errorf("using an external kubernetes CA without providing the apiserver certificate is not possible")
}

// TODO(neoaggelos): we also need to specify the kubernetes service IP here, not hardcode 10.152.183.1
template, err := generateCertificate(
pkix.Name{CommonName: "kube-apiserver"},
c.years,
false,
append(c.dnsSANs, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"), append(c.ipSANs, net.IP{10, 152, 183, 1}, net.IP{127, 0, 0, 1}))
append(c.dnsSANs, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"), append(c.ipSANs, net.IP{127, 0, 0, 1}))
if err != nil {
return fmt.Errorf("failed to generate apiserver certificate: %w", err)
}
Expand Down
9 changes: 7 additions & 2 deletions src/k8s/pkg/k8sd/types/cluster_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,8 +106,12 @@ func (c *ClusterConfig) Validate() error {
if len(clusterCIDRs) != 1 && len(clusterCIDRs) != 2 {
return fmt.Errorf("invalid number of cluster CIDRs: %d", len(clusterCIDRs))
}
serviceCIDRs := strings.Split(c.Network.ServiceCIDR, ",")
if len(serviceCIDRs) != 1 && len(serviceCIDRs) != 2 {
return fmt.Errorf("invalid number of service CIDRs: %d", len(serviceCIDRs))
}

for _, cidr := range clusterCIDRs {
for _, cidr := range append(clusterCIDRs, serviceCIDRs...) {
_, _, err := net.ParseCIDR(cidr)
if err != nil {
return fmt.Errorf("invalid CIDR: %w", err)
Expand Down Expand Up @@ -170,7 +174,8 @@ func ClusterConfigFromBootstrapConfig(b *apiv1.BootstrapConfig) ClusterConfig {
AuthorizationMode: authzMode,
},
Network: Network{
PodCIDR: b.ClusterCIDR,
PodCIDR: b.ClusterCIDR,
ServiceCIDR: b.ServiceCIDR,
},
K8sDqlite: K8sDqlite{
Port: b.K8sDqlitePort,
Expand Down
49 changes: 37 additions & 12 deletions src/k8s/pkg/k8sd/types/cluster_config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ func TestClusterConfigFromBootstrapConfig(t *testing.T) {
g := NewWithT(t)
bootstrapConfig := apiv1.BootstrapConfig{
ClusterCIDR: "10.1.0.0/16",
ServiceCIDR: "10.152.183.0/24",
Components: []string{"dns", "network"},
EnableRBAC: vals.Pointer(true),
K8sDqlitePort: 12345,
Expand All @@ -24,8 +25,9 @@ func TestClusterConfigFromBootstrapConfig(t *testing.T) {
AuthorizationMode: "Node,RBAC",
},
Network: types.Network{
Enabled: vals.Pointer(true),
PodCIDR: "10.1.0.0/16",
Enabled: vals.Pointer(true),
PodCIDR: "10.1.0.0/16",
ServiceCIDR: "10.152.183.0/24",
},
K8sDqlite: types.K8sDqlite{
Port: 12345,
Expand All @@ -43,21 +45,44 @@ func TestValidateCIDR(t *testing.T) {
// Create a new BootstrapConfig with default values
validConfig := types.ClusterConfig{
Network: types.Network{
PodCIDR: "10.1.0.0/16,2001:0db8::/32",
PodCIDR: "10.1.0.0/16,2001:0db8::/32",
ServiceCIDR: "10.152.183.0/16",
},
}

err := validConfig.Validate()
g.Expect(err).To(BeNil())

// Create a new BootstrapConfig with invalid CIDR
invalidConfig := types.ClusterConfig{
Network: types.Network{
PodCIDR: "bananas",
},
}
err = invalidConfig.Validate()
g.Expect(err).ToNot(BeNil())
t.Run("InvalidCIDR", func(t *testing.T) {
for _, tc := range []struct {
cidr string
}{
{cidr: "bananas"},
{cidr: "fd01::/64,fd02::/64,fd03::/64"},
} {
t.Run(tc.cidr, func(t *testing.T) {
t.Run("Pod", func(t *testing.T) {
g := NewWithT(t)
config := types.ClusterConfig{
Network: types.Network{
PodCIDR: tc.cidr,
},
}
err := config.Validate()
g.Expect(err).ToNot(BeNil())
})
t.Run("Service", func(t *testing.T) {
g := NewWithT(t)
config := types.ClusterConfig{
Network: types.Network{
ServiceCIDR: tc.cidr,
},
}
err := config.Validate()
g.Expect(err).ToNot(BeNil())
})
})
}
})
}

func TestUnsetRBAC(t *testing.T) {
Expand Down
39 changes: 39 additions & 0 deletions src/k8s/pkg/utils/cidr.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package utils

import (
"fmt"
"math/big"
"net"
"strings"
)

// GetFirstIP returns the first IP address of a subnet. Use big.Int so that it can handle both IPv4 and IPv6 addreses.
func GetFirstIP(subnet string) (net.IP, error) {
_, cidr, err := net.ParseCIDR(subnet)
if err != nil {
return nil, fmt.Errorf("%q is not a valid subnet CIDR: %w", subnet, err)
}
r := big.NewInt(0).Add(
big.NewInt(0).SetBytes(cidr.IP.To16()),
big.NewInt(1),
).Bytes()
r = append(make([]byte, 16), r...)
return net.IP(r[len(r)-16:]), nil
}

// GetKubernetesServiceIPsFromServiceCIDRs returns a list of the first IP addrs from a given service cidr string.
func GetKubernetesServiceIPsFromServiceCIDRs(serviceCIDR string) ([]net.IP, error) {
var firstIPs []net.IP
cidrs := strings.Split(serviceCIDR, ",")
if v := len(cidrs); v != 1 && v != 2 {
return nil, fmt.Errorf("invalid ServiceCIDR value: %v", cidrs)
}
for _, cidr := range cidrs {
ip, err := GetFirstIP(cidr)
if err != nil {
return nil, fmt.Errorf("could not get IP from CIDR %q: %w", cidr, err)
}
firstIPs = append(firstIPs, ip)
}
return firstIPs, nil
}
70 changes: 70 additions & 0 deletions src/k8s/pkg/utils/cidr_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
package utils_test

import (
"testing"

"github.com/canonical/k8s/pkg/utils"
. "github.com/onsi/gomega"
)

func TestGetFirstIP(t *testing.T) {
for _, tc := range []struct {
cidr string
ip string
}{
{cidr: "10.152.183.0/24", ip: "10.152.183.1"},
{cidr: "10.152.183.10/24", ip: "10.152.183.1"},
{cidr: "10.100.0.0/16", ip: "10.100.0.1"},
{cidr: "fd01::/64", ip: "fd01::1"},
// TODO: do we need more test cases?
} {
t.Run(tc.cidr, func(t *testing.T) {
g := NewWithT(t)
ip, err := utils.GetFirstIP(tc.cidr)
g.Expect(err).To(BeNil())
g.Expect(ip.String()).To(Equal(tc.ip))
})
}
}

func TestGetKubernetesServiceIPsFromServiceCIDRs(t *testing.T) {
// Test valid subnet cidr strings
t.Run("ValidCIDR", func(t *testing.T) {
for _, tc := range []struct {
cidr string
ips []string
}{
{cidr: "10.152.183.0/24", ips: []string{"10.152.183.1"}},
{cidr: "fd01::/64", ips: []string{"fd01::1"}},
{cidr: "10.152.183.0/24,fd01::/64", ips: []string{"10.152.183.1", "fd01::1"}},
} {
t.Run(tc.cidr, func(t *testing.T) {
g := NewWithT(t)
i, err := utils.GetKubernetesServiceIPsFromServiceCIDRs(tc.cidr)
ips := make([]string, len(i))
for idx, v := range i {
ips[idx] = v.String()
}

g.Expect(err).To(BeNil())
g.Expect(ips).To(Equal(tc.ips))
})
}
})

t.Run("InvalidCIDR", func(t *testing.T) {
for _, tc := range []struct {
cidr string
}{
{cidr: "fd01::/64,fd02::/64,fd03::/64"},
{cidr: "bananas"},
} {
t.Run(tc.cidr, func(t *testing.T) {
g := NewWithT(t)
_, err := utils.GetKubernetesServiceIPsFromServiceCIDRs(tc.cidr)

g.Expect(err).ToNot(BeNil())
})
}
})
}

0 comments on commit 004a1af

Please sign in to comment.