From 142d149f7e75896a2746459c6347d916eae8b3d5 Mon Sep 17 00:00:00 2001 From: Michael Schmidt Date: Wed, 20 Dec 2017 17:16:35 +0100 Subject: [PATCH] Upgrade to Kubernetes 1.8.5 (#132) * Upgrade to Kubernetes 1.8.5 This commit upgrades the helm chart and ignition template to use 1.8.5. Currently, this assumes that klusters without version information in the spec are on 1.7.5. All new klusters must provide a version. With 1.8.5 the default behaviour of the Openstack LoadBalancer creation changed. Instead of defaulting to an internal LoadBalancer it now creates an external LB by default. This requires the external network to be configured in the cloud-provider config. Fixes #62 * removes v from versions * fixes logging in case of error while creating kluster struct * uses one version for new klusters only --- charts/kube-master/Chart.yaml | 2 +- .../templates/_openstack.config.tpl | 1 + charts/kube-master/templates/api.yaml | 6 +- charts/kube-master/values.yaml | 8 +- pkg/api/handlers/create_cluster.go | 8 + pkg/api/models/kluster_spec.go | 23 ++ pkg/api/models/openstack_spec.go | 3 + pkg/api/spec/embedded_spec.go | 9 + pkg/apis/kubernikus/factory.go | 13 + pkg/client/openstack/client.go | 7 +- pkg/controller/ground.go | 20 +- pkg/templates/ignition.go | 15 +- pkg/templates/ignition_test.go | 5 +- pkg/templates/{node.go => node_1.7.go} | 2 +- pkg/templates/node_1.8.go | 378 ++++++++++++++++++ pkg/util/constants.go | 3 + pkg/util/helm/helm.go | 46 ++- swagger.yml | 7 + 18 files changed, 524 insertions(+), 32 deletions(-) rename pkg/templates/{node.go => node_1.7.go} (99%) create mode 100644 pkg/templates/node_1.8.go diff --git a/charts/kube-master/Chart.yaml b/charts/kube-master/Chart.yaml index 8cf04a6e65..1f310a6999 100644 --- a/charts/kube-master/Chart.yaml +++ b/charts/kube-master/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: A Helm chart for Kubernetes name: kube-master -version: 0.1.0 +version: 1.8.5-kubernikus.0 diff --git a/charts/kube-master/templates/_openstack.config.tpl b/charts/kube-master/templates/_openstack.config.tpl index 27bf9a25ca..e5ba82cf48 100644 --- a/charts/kube-master/templates/_openstack.config.tpl +++ b/charts/kube-master/templates/_openstack.config.tpl @@ -13,6 +13,7 @@ region = {{ .Values.openstack.region }} [LoadBalancer] lb-version=v2 subnet-id= {{ required "missing openstack.lbSubnetID" .Values.openstack.lbSubnetID }} +floating-network-id= {{ required "missing openstack.lbFloatingNetworkID" .Values.openstack.lbFloatingNetworkID }} create-monitor = yes monitor-delay = 1m monitor-timeout = 30s diff --git a/charts/kube-master/templates/api.yaml b/charts/kube-master/templates/api.yaml index 0a75d86d6e..d2010a3391 100644 --- a/charts/kube-master/templates/api.yaml +++ b/charts/kube-master/templates/api.yaml @@ -107,9 +107,9 @@ spec: - --authorization-mode=Node,RBAC - --cloud-config=/etc/kubernetes/cloudprovider/openstack.config - --cloud-provider=openstack - - --experimental-bootstrap-token-auth=true + - --enable-bootstrap-token-auth=true + - --external-hostname={{ required "missing .api.apiserverHost" .Values.api.apiserverHost }} - --token-auth-file=/etc/kubernetes/bootstrap/token.csv - - --runtime-config=rbac.authorization.k8s.io/v1alpha1,extensions/v1beta1=true,extensions/v1beta1/thirdpartyresources=true - --service-cluster-ip-range={{ .Values.serviceCIDR }} - --kubelet-preferred-address-types=InternalIP #Cert Spratz @@ -144,7 +144,7 @@ spec: resources: {{ toYaml .Values.api.resources | indent 12 }} - name: wormhole - image: sapcc/kubernikus:{{ required ".version missing" .Values.version }} + image: sapcc/kubernikus:{{ required ".version.kubernikus missing" .Values.version.kubernikus }} args: - wormhole - server diff --git a/charts/kube-master/values.yaml b/charts/kube-master/values.yaml index d6341be16c..43bc0a93ec 100644 --- a/charts/kube-master/values.yaml +++ b/charts/kube-master/values.yaml @@ -3,7 +3,7 @@ # Declare variables to be passed into your templates. image: repository: quay.io/coreos/hyperkube - tag: v1.7.5_coreos.0 + tag: v1.8.5_coreos.0 pullPolicy: IfNotPresent # Settings for the openstack cloudprovider @@ -15,6 +15,7 @@ openstack: {} #projectID: #region: #lbSubnetID: + #lbFloatingNetworkID #routerID: #region @@ -29,7 +30,10 @@ clusterCIDR: 198.19.0.0/16 serviceCIDR: 198.18.128.0/17 advertiseAddress: 198.18.128.1 #bootstrapToken -#version: + +version: +# kubernikus: + kubernetes: 1.8.5 api: replicaCount: 1 diff --git a/pkg/api/handlers/create_cluster.go b/pkg/api/handlers/create_cluster.go index dec21c15e8..c71707ea1c 100644 --- a/pkg/api/handlers/create_cluster.go +++ b/pkg/api/handlers/create_cluster.go @@ -39,6 +39,14 @@ func (d *createCluster) Handle(params operations.CreateClusterParams, principal } } kluster, err := kubernikus.NewKlusterFactory().KlusterFor(spec) + if err != nil { + logger.Log( + "msg", "failed to create cluster", + "kluster", name, + "project", principal.Account, + "err", err) + return NewErrorResponse(&operations.CreateClusterDefault{}, 400, err.Error()) + } kluster.ObjectMeta = metav1.ObjectMeta{ Name: qualifiedName(name, principal.Account), diff --git a/pkg/api/models/kluster_spec.go b/pkg/api/models/kluster_spec.go index 81425ca4a2..5519d7846c 100644 --- a/pkg/api/models/kluster_spec.go +++ b/pkg/api/models/kluster_spec.go @@ -43,6 +43,11 @@ type KlusterSpec struct { // CIDR Range for Services in the cluster. Can not be updated. // Pattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ ServiceCIDR string `json:"serviceCIDR,omitempty"` + + // version + // Read Only: true + // Pattern: ^[0-9]+\.[0-9]+\.[0-9]+$ + Version string `json:"version,omitempty"` } // Validate validates this kluster spec @@ -69,6 +74,11 @@ func (m *KlusterSpec) Validate(formats strfmt.Registry) error { res = append(res, err) } + if err := m.validateVersion(formats); err != nil { + // prop + res = append(res, err) + } + if len(res) > 0 { return errors.CompositeValidationError(res...) } @@ -126,6 +136,19 @@ func (m *KlusterSpec) validateServiceCIDR(formats strfmt.Registry) error { return nil } +func (m *KlusterSpec) validateVersion(formats strfmt.Registry) error { + + if swag.IsZero(m.Version) { // not required + return nil + } + + if err := validate.Pattern("version", "body", string(m.Version), `^[0-9]+\.[0-9]+\.[0-9]+$`); err != nil { + return err + } + + return nil +} + // MarshalBinary interface implementation func (m *KlusterSpec) MarshalBinary() ([]byte, error) { if m == nil { diff --git a/pkg/api/models/openstack_spec.go b/pkg/api/models/openstack_spec.go index 1e3639bd4e..77188c504f 100644 --- a/pkg/api/models/openstack_spec.go +++ b/pkg/api/models/openstack_spec.go @@ -16,6 +16,9 @@ import ( // swagger:model OpenstackSpec type OpenstackSpec struct { + // l b floating network ID + LBFloatingNetworkID string `json:"lbFloatingNetworkID,omitempty"` + // l b subnet ID LBSubnetID string `json:"lbSubnetID,omitempty"` diff --git a/pkg/api/spec/embedded_spec.go b/pkg/api/spec/embedded_spec.go index 06809e369b..69a16e255e 100644 --- a/pkg/api/spec/embedded_spec.go +++ b/pkg/api/spec/embedded_spec.go @@ -433,6 +433,11 @@ func init() { "default": "198.18.128.0/17", "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\\/([0-9]|[1-2][0-9]|3[0-2]))$", "x-nullable": false + }, + "version": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$", + "readOnly": true } }, "x-nullable": false @@ -630,6 +635,10 @@ func init() { "OpenstackSpec": { "type": "object", "properties": { + "lbFloatingNetworkID": { + "type": "string", + "x-go-name": "LBFloatingNetworkID" + }, "lbSubnetID": { "type": "string", "x-go-name": "LBSubnetID" diff --git a/pkg/apis/kubernikus/factory.go b/pkg/apis/kubernikus/factory.go index a93194dc3f..0d0a73829e 100644 --- a/pkg/apis/kubernikus/factory.go +++ b/pkg/apis/kubernikus/factory.go @@ -8,6 +8,7 @@ import ( "github.com/sapcc/kubernikus/pkg/api/spec" "github.com/sapcc/kubernikus/pkg/apis/kubernikus/v1" "github.com/sapcc/kubernikus/pkg/controller/ground/bootstrap/dns" + "github.com/sapcc/kubernikus/pkg/util" "github.com/sapcc/kubernikus/pkg/util/ip" "github.com/sapcc/kubernikus/pkg/version" ) @@ -74,6 +75,18 @@ func (klusterFactory) KlusterFor(spec models.KlusterSpec) (*v1.Kluster, error) { k.Spec.DNSDomain = dns.DEFAULT_DOMAIN } + if k.Spec.Version == "" { + k.Spec.Version = util.DEFAULT_KUBERNETES_VERSION + } + + if k.Spec.Version == "" { + k.Spec.Version = util.DEFAULT_KUBERNETES_VERSION + } + + if k.Spec.Version != util.DEFAULT_KUBERNETES_VERSION { + return nil, fmt.Errorf("Unabled to create cluster. Unsupported Kubernetes version.") + } + if k.ObjectMeta.Name == "" { k.ObjectMeta.Name = spec.Name } diff --git a/pkg/client/openstack/client.go b/pkg/client/openstack/client.go index 63aa49f65b..f35a52f39a 100644 --- a/pkg/client/openstack/client.go +++ b/pkg/client/openstack/client.go @@ -75,8 +75,9 @@ type Project struct { } type Router struct { - ID string - Networks []Network + ID string + ExternalNetworkID string + Networks []Network } type Network struct { @@ -312,7 +313,7 @@ func (c *client) GetRouters(project_id string) ([]Router, error) { return false, err } for _, router := range routers { - resultRouter := Router{ID: router.ID} + resultRouter := Router{ID: router.ID, ExternalNetworkID: router.GatewayInfo.NetworkID} networkIDs, err := getRouterNetworks(networkClient, router.ID) if err != nil { return false, err diff --git a/pkg/controller/ground.go b/pkg/controller/ground.go index 95e52181c7..a6b8b94946 100644 --- a/pkg/controller/ground.go +++ b/pkg/controller/ground.go @@ -449,7 +449,8 @@ func (op *GroundControl) requiresOpenstackInfo(kluster *v1.Kluster) bool { return kluster.Spec.Openstack.ProjectID == "" || kluster.Spec.Openstack.RouterID == "" || kluster.Spec.Openstack.NetworkID == "" || - kluster.Spec.Openstack.LBSubnetID == "" + kluster.Spec.Openstack.LBSubnetID == "" || + kluster.Spec.Openstack.LBFloatingNetworkID == "" } func (op *GroundControl) requiresKubernikusInfo(kluster *v1.Kluster) bool { @@ -594,6 +595,23 @@ func (op *GroundControl) discoverOpenstackInfo(kluster *v1.Kluster) error { } } + if floatingNetworkID := copy.Spec.Openstack.LBFloatingNetworkID; floatingNetworkID != "" { + if selectedRouter.ExternalNetworkID != "" && floatingNetworkID != selectedRouter.ExternalNetworkID { + return fmt.Errorf("External network missmatch. Router is configured with %s but config specifies %s", selectedRouter.ExternalNetworkID, floatingNetworkID) + } + } else { + if selectedRouter.ExternalNetworkID == "" { + return fmt.Errorf("Selected router %s doesn't have an external network ID set", selectedRouter.ID) + } else { + copy.Spec.Openstack.LBFloatingNetworkID = selectedRouter.ExternalNetworkID + op.Logger.Log( + "msg", "discovered LBFloatingNetworkID", + "id", copy.Spec.Openstack.LBFloatingNetworkID, + "kluster", kluster.GetName(), + "project", kluster.Account()) + } + } + if securityGroupID := copy.Spec.Openstack.SecurityGroupID; securityGroupID != "" { //TODO: Validate that the securitygroup id exists diff --git a/pkg/templates/ignition.go b/pkg/templates/ignition.go index 2b2356eacd..837bac2ddc 100644 --- a/pkg/templates/ignition.go +++ b/pkg/templates/ignition.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "strings" "text/template" "github.com/Masterminds/sprig" @@ -36,6 +37,15 @@ var Ignition = &ignition{ }, } +func (i *ignition) getIgnitionTemplate(kluster *kubernikusv1.Kluster) string { + switch { + case strings.HasPrefix(kluster.Spec.Version, "1.8"): + return Node_1_8 + default: + return Node_1_7 + } +} + func (i *ignition) GenerateNode(kluster *kubernikusv1.Kluster, secret *v1.Secret, logger log.Logger) ([]byte, error) { for _, field := range i.requiredNodeSecrets { if _, ok := secret.Data[field]; !ok { @@ -43,7 +53,8 @@ func (i *ignition) GenerateNode(kluster *kubernikusv1.Kluster, secret *v1.Secret } } - tmpl, err := template.New("node").Funcs(sprig.TxtFuncMap()).Parse(Node) + ignition := i.getIgnitionTemplate(kluster) + tmpl, err := template.New("node").Funcs(sprig.TxtFuncMap()).Parse(ignition) if err != nil { return nil, err } @@ -65,6 +76,7 @@ func (i *ignition) GenerateNode(kluster *kubernikusv1.Kluster, secret *v1.Secret OpenstackDomain string OpenstackRegion string OpenstackLBSubnetID string + OpenstackLBFloatingNetworkID string OpenstackRouterID string KubernikusImage string KubernikusImageTag string @@ -85,6 +97,7 @@ func (i *ignition) GenerateNode(kluster *kubernikusv1.Kluster, secret *v1.Secret OpenstackDomain: string(secret.Data["openstack-domain-name"]), OpenstackRegion: string(secret.Data["openstack-region"]), OpenstackLBSubnetID: kluster.Spec.Openstack.LBSubnetID, + OpenstackLBFloatingNetworkID: kluster.Spec.Openstack.LBFloatingNetworkID, OpenstackRouterID: kluster.Spec.Openstack.RouterID, KubernikusImage: "sapcc/kubernikus", KubernikusImageTag: version.GitCommit, diff --git a/pkg/templates/ignition_test.go b/pkg/templates/ignition_test.go index 9c12b91f79..20345dc660 100644 --- a/pkg/templates/ignition_test.go +++ b/pkg/templates/ignition_test.go @@ -26,8 +26,9 @@ func TestGenerateNode(t *testing.T) { DNSAddress: "2.2.2.2", DNSDomain: "cluster.local", Openstack: models.OpenstackSpec{ - LBSubnetID: "lb-id", - RouterID: "router-id", + LBSubnetID: "lb-id", + LBFloatingNetworkID: "lb-fipid", + RouterID: "router-id", }, }, Status: models.KlusterStatus{ diff --git a/pkg/templates/node.go b/pkg/templates/node_1.7.go similarity index 99% rename from pkg/templates/node.go rename to pkg/templates/node_1.7.go index 043da9c8d4..05b2ebca81 100644 --- a/pkg/templates/node.go +++ b/pkg/templates/node_1.7.go @@ -2,7 +2,7 @@ package templates -var Node = ` +var Node_1_7 = ` passwd: users: - name: core diff --git a/pkg/templates/node_1.8.go b/pkg/templates/node_1.8.go new file mode 100644 index 0000000000..7368bd07b4 --- /dev/null +++ b/pkg/templates/node_1.8.go @@ -0,0 +1,378 @@ +/* vim: set filetype=yaml : */ + +package templates + +var Node_1_8 = ` +passwd: + users: + - name: core + password_hash: xyTGJkB462ewk + ssh_authorized_keys: + - "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAvFapuevZeHFpFn438XMjvEQYd0wt7+tzUdAkMiSd007Tx1h79Xm9ZziDDUe4W6meinVOq93MAS/ER27hoVWGo2H/vn/Cz5M8xr2j5rQODnrF3RmfrJTbZAWaDN0JTq2lFjmCHhZJNhr+VQP1uw4z2ofMBP6MLybnLmm9ukzxFYZqCCyfEEUTCMA9SWywtTpGQp8VLM4INCxzBSCuyt3SO6PBvJSo4HoKg/sLvmRwpCVZth48PI0EUbJ72wp88Cw3bv8CLce2TOkLMwkE6NRN55w2aOyqP1G3vixHa6YcVaLlkQhJoJsBwE3rX5603y2KjOhMomqHfXxXn/3GKTWlsQ== michael.j.schmidt@gmail.com" + +locksmith: + reboot_strategy: "reboot" + +systemd: + units: + - name: iptables-restore.service + enable: true + - name: ccloud-metadata.service + contents: | + [Unit] + Description=Converged Cloud Metadata Agent + + [Service] + Type=oneshot + ExecStart=/usr/bin/coreos-metadata --provider=openstack-metadata --attributes=/run/metadata/coreos --ssh-keys=core --hostname=/etc/hostname + - name: ccloud-metadata-hostname.service + enable: true + contents: | + [Unit] + Description=Workaround for coreos-metadata hostname bug + Requires=ccloud-metadata.service + After=ccloud-metadata.service + + [Service] + Type=oneshot + EnvironmentFile=/run/metadata/coreos + ExecStart=/usr/bin/hostnamectl set-hostname ${COREOS_OPENSTACK_HOSTNAME} + + [Install] + WantedBy=multi-user.target + - name: docker.service + enable: true + dropins: + - name: 20-docker-opts.conf + contents: | + [Service] + Environment="DOCKER_OPTS=--log-opt max-size=5m --log-opt max-file=5 --ip-masq=false --iptables=false --bridge=none" + - name: kubelet.service + enable: true + contents: | + [Unit] + Description=Kubelet via Hyperkube ACI + + [Service] + Environment="RKT_RUN_ARGS=--uuid-file-save=/var/run/kubelet-pod.uuid \ + --inherit-env \ + --dns=host \ + --net=host \ + --volume var-lib-cni,kind=host,source=/var/lib/cni \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-lib-cni,target=/var/lib/cni \ + --mount volume=var-log,target=/var/log" + Environment="KUBELET_IMAGE_TAG=v1.8.5_coreos.0" + Environment="KUBELET_IMAGE_URL=quay.io/coreos/hyperkube" + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /var/lib/cni + ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid + ExecStart=/usr/lib/coreos/kubelet-wrapper \ + --cert-dir=/var/lib/kubelet/pki \ + --cloud-config=/etc/kubernetes/openstack/openstack.config \ + --cloud-provider=openstack \ + --require-kubeconfig \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap/kubeconfig \ + --network-plugin=kubenet \ + --lock-file=/var/run/lock/kubelet.lock \ + --exit-on-lock-contention \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged \ + --cluster-dns={{ .ClusterDNSAddress }} \ + --cluster-domain={{ .ClusterDomain }} \ + --client-ca-file=/etc/kubernetes/certs/kubelet-clients-ca.pem \ + --non-masquerade-cidr=0.0.0.0/0 \ + --anonymous-auth=false + ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid + Restart=always + RestartSec=10 + + [Install] + WantedBy=multi-user.target + - name: wormhole.service + contents: | + [Unit] + Description=Kubernikus Wormhole + Requires=network-online.target + After=network-online.target + [Service] + Slice=machine.slice + ExecStartPre=/usr/bin/rkt fetch --insecure-options=image --pull-policy=new docker://{{ .KubernikusImage }}:{{ .KubernikusImageTag }} + ExecStart=/usr/bin/rkt run \ + --inherit-env \ + --net=host \ + --dns=host \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,readOnly=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume etc-kubernetes-certs,kind=host,source=/etc/kubernetes/certs,readOnly=true \ + --mount volume=etc-kubernetes-certs,target=/etc/kubernetes/certs \ + docker://{{ .KubernikusImage }}:{{ .KubernikusImageTag }} \ + --exec wormhole -- client --listen {{ .ApiserverIP }}:6443 --kubeconfig=/var/lib/kubelet/kubeconfig + ExecStopPost=/usr/bin/rkt gc --mark-only + KillMode=mixed + Restart=always + RestartSec=10s + - name: wormhole.path + enable: true + contents: | + [Path] + PathExists=/var/lib/kubelet/kubeconfig + [Install] + WantedBy=multi-user.target + - name: kube-proxy.service + enable: true + contents: | + [Unit] + Description=Kube-Proxy + Requires=network-online.target + After=network-online.target + [Service] + Slice=machine.slice + ExecStart=/usr/bin/rkt run \ + --trust-keys-from-https \ + --inherit-env \ + --net=host \ + --dns=host \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --stage1-from-dir=stage1-fly.aci \ + quay.io/coreos/hyperkube:v1.8.5_coreos.0 \ + --exec=hyperkube \ + -- \ + proxy \ + --config=/etc/kubernetes/kube-proxy/config + ExecStopPost=/usr/bin/rkt gc --mark-only + KillMode=mixed + Restart=always + RestartSec=10s + [Install] + WantedBy=multi-user.target + - name: updatecertificates.service + command: start + enable: true + contents: | + [Unit] + Description=Update the certificates w/ self-signed root CAs + ConditionPathIsSymbolicLink=!/etc/ssl/certs/48b11003.0 + Before=early-docker.service docker.service + [Service] + ExecStart=/usr/sbin/update-ca-certificates + RemainAfterExit=yes + Type=oneshot + [Install] + WantedBy=multi-user.target + +networkd: + units: + - name: 50-kubernikus.netdev + contents: | + [NetDev] + Description=Kubernikus Dummy Interface + Name=kubernikus + Kind=dummy + - name: 51-kubernikus.network + contents: | + [Match] + Name=kubernikus + [Network] + DHCP=no + Address={{ .ApiserverIP }}/32 + +storage: + files: + - path: /etc/ssl/certs/SAPNetCA_G2.pem + filesystem: root + mode: 0644 + contents: + inline: |- + -----BEGIN CERTIFICATE----- + MIIGPTCCBCWgAwIBAgIKYQ4GNwAAAAAADDANBgkqhkiG9w0BAQsFADBOMQswCQYD + VQQGEwJERTERMA8GA1UEBwwIV2FsbGRvcmYxDzANBgNVBAoMBlNBUCBBRzEbMBkG + A1UEAwwSU0FQIEdsb2JhbCBSb290IENBMB4XDTE1MDMxNzA5MjQ1MVoXDTI1MDMx + NzA5MzQ1MVowRDELMAkGA1UEBhMCREUxETAPBgNVBAcMCFdhbGxkb3JmMQwwCgYD + VQQKDANTQVAxFDASBgNVBAMMC1NBUE5ldENBX0cyMIICIjANBgkqhkiG9w0BAQEF + AAOCAg8AMIICCgKCAgEAjuP7Hj/1nVWfsCr8M/JX90s88IhdTLaoekrxpLNJ1W27 + ECUQogQF6HCu/RFD4uIoanH0oGItbmp2p8I0XVevHXnisxQGxBdkjz+a6ZyOcEVk + cEGTcXev1i0R+MxM8Y2WW/LGDKKkYOoVRvA5ChhTLtX2UXnBLcRdf2lMMvEHd/nn + KWEQ47ENC+uXd6UPxzE+JqVSVaVN+NNbXBJrI1ddNdEE3/++PSAmhF7BSeNWscs7 + w0MoPwHAGMvMHe9pas1xD3RsRFQkV01XiJqqUbf1OTdYAoUoXo9orPPrO7FMfXjZ + RbzwzFtdKRlAFnKZOVf95MKlSo8WzhffKf7pQmuabGSLqSSXzIuCpxuPlNy7kwCX + j5m8U1xGN7L2vlalKEG27rCLx/n6ctXAaKmQo3FM+cHim3ko/mOy+9GDwGIgToX3 + 5SQPnmCSR19H3nYscT06ff5lgWfBzSQmBdv//rjYkk2ZeLnTMqDNXsgT7ac6LJlj + WXAdfdK2+gvHruf7jskio29hYRb2//ti5jD3NM6LLyovo1GOVl0uJ0NYLsmjDUAJ + dqqNzBocy/eV3L2Ky1L6DvtcQ1otmyvroqsL5JxziP0/gRTj/t170GC/aTxjUnhs + 7vDebVOT5nffxFsZwmolzTIeOsvM4rAnMu5Gf4Mna/SsMi9w/oeXFFc/b1We1a0C + AwEAAaOCASUwggEhMAsGA1UdDwQEAwIBBjAdBgNVHQ4EFgQUOCSvjXUS/Dg/N4MQ + r5A8/BshWv8wHwYDVR0jBBgwFoAUg8dB/Q4mTynBuHmOhnrhv7XXagMwSwYDVR0f + BEQwQjBAoD6gPIY6aHR0cDovL2NkcC5wa2kuY28uc2FwLmNvbS9jZHAvU0FQJTIw + R2xvYmFsJTIwUm9vdCUyMENBLmNybDBWBggrBgEFBQcBAQRKMEgwRgYIKwYBBQUH + MAKGOmh0dHA6Ly9haWEucGtpLmNvLnNhcC5jb20vYWlhL1NBUCUyMEdsb2JhbCUy + MFJvb3QlMjBDQS5jcnQwGQYJKwYBBAGCNxQCBAweCgBTAHUAYgBDAEEwEgYDVR0T + AQH/BAgwBgEB/wIBADANBgkqhkiG9w0BAQsFAAOCAgEAGdBNALO509FQxcPhMCwE + /eymAe9f2u6hXq0hMlQAuuRbpnxr0+57lcw/1eVFsT4slceh7+CHGCTCVHK1ELAd + XQeibeQovsVx80BkugEG9PstCJpHnOAoWGjlZS2uWz89Y4O9nla+L9SCuK7tWI5Y + +QuVhyGCD6FDIUCMlVADOLQV8Ffcm458q5S6eGViVa8Y7PNpvMyFfuUTLcUIhrZv + eh4yjPSpz5uvQs7p/BJLXilEf3VsyXX5Q4ssibTS2aH2z7uF8gghfMvbLi7sS7oj + XBEylxyaegwOBLtlmcbII8PoUAEAGJzdZ4kFCYjqZBMgXK9754LMpvkXDTVzy4OP + emK5Il+t+B0VOV73T4yLamXG73qqt8QZndJ3ii7NGutv4SWhVYQ4s7MfjRwbFYlB + z/N5eH3veBx9lJbV6uXHuNX3liGS8pNVNKPycfwlaGEbD2qZE0aZRU8OetuH1kVp + jGqvWloPjj45iCGSCbG7FcY1gPVTEAreLjyINVH0pPve1HXcrnCV4PALT6HvoZoF + bCuBKVgkSSoGgmasxjjjVIfMiOhkevDya52E5m0WnM1LD3ZoZzavsDSYguBP6MOV + ViWNsVHocptphbEgdwvt3B75CDN4kf6MNZg2/t8bRhEQyK1FRy8NMeBnbRFnnEPe + 7HJNBB1ZTjnrxJAgCQgNBIQ= + -----END CERTIFICATE----- + - path: /var/lib/iptables/rules-save + filesystem: root + mode: 0644 + contents: + inline: | + *nat + :PREROUTING ACCEPT [0:0] + :INPUT ACCEPT [0:0] + :OUTPUT ACCEPT [0:0] + :POSTROUTING ACCEPT [0:0] + -A POSTROUTING -p tcp ! -d {{ .ClusterCIDR }} -m addrtype ! --dst-type LOCAL -j MASQUERADE --to-ports 32000-65000 + -A POSTROUTING -p udp ! -d {{ .ClusterCIDR }} -m addrtype ! --dst-type LOCAL -j MASQUERADE --to-ports 32000-65000 + -A POSTROUTING -p icmp ! -d {{ .ClusterCIDR }} -m addrtype ! --dst-type LOCAL -j MASQUERADE + COMMIT + - path: /etc/sysctl.d/10-enable-icmp-redirects + filesystem: root + mode: 0644 + contents: + inline: |- + net.ipv4.conf.all.accept_redirects=1 + - path: /etc/kubernetes/certs/kubelet-clients-ca.pem + filesystem: root + mode: 0644 + contents: + inline: |- +{{ .KubeletClientsCA | indent 10 }} + - path: /etc/kubernetes/certs/apiserver-clients-system-kube-proxy-key.pem + filesystem: root + mode: 0644 + contents: + inline: |- +{{ .ApiserverClientsSystemKubeProxyKey | indent 10 }} + - path: /etc/kubernetes/certs/apiserver-clients-system-kube-proxy.pem + filesystem: root + mode: 0644 + contents: + inline: |- +{{ .ApiserverClientsSystemKubeProxy | indent 10 }} + - path: /etc/kubernetes/certs/tls-ca.pem + filesystem: root + mode: 0644 + contents: + inline: |- +{{ .TLSCA | indent 10 }} + - path: /etc/kubernetes/bootstrap/kubeconfig + filesystem: root + mode: 0644 + contents: + inline: |- + apiVersion: v1 + kind: Config + clusters: + - name: local + cluster: + certificate-authority: /etc/kubernetes/certs/tls-ca.pem + server: {{ .ApiserverURL }} + contexts: + - name: local + context: + cluster: local + user: local + current-context: local + users: + - name: local + user: + token: {{ .BootstrapToken }} + - path: /etc/kubernetes/kube-proxy/kubeconfig + filesystem: root + mode: 0644 + contents: + inline: |- + apiVersion: v1 + kind: Config + clusters: + - name: local + cluster: + certificate-authority: /etc/kubernetes/certs/tls-ca.pem + server: {{ .ApiserverURL }} + contexts: + - name: local + context: + cluster: local + user: local + current-context: local + users: + - name: local + user: + client-certificate: /etc/kubernetes/certs/apiserver-clients-system-kube-proxy.pem + client-key: /etc/kubernetes/certs/apiserver-clients-system-kube-proxy-key.pem + - path: /etc/kubernetes/kube-proxy/config + filesystem: root + mode: 0644 + contents: + inline: |- + apiVersion: componentconfig/v1alpha1 + kind: KubeProxyConfiguration + bindAddress: 0.0.0.0 + clientConnection: + acceptContentTypes: "" + burst: 10 + contentType: application/vnd.kubernetes.protobuf + kubeconfig: "/etc/kubernetes/kube-proxy/kubeconfig" + qps: 5 + clusterCIDR: "{{ .ClusterCIDR }}" + configSyncPeriod: 15m0s + conntrack: + max: 0 + maxPerCore: 32768 + min: 131072 + tcpCloseWaitTimeout: 1h0m0s + tcpEstablishedTimeout: 24h0m0s + enableProfiling: false + featureGates: "" + healthzBindAddress: 0.0.0.0:10256 + hostnameOverride: "" + iptables: + masqueradeAll: false + masqueradeBit: 14 + minSyncPeriod: 0s + syncPeriod: 30s + metricsBindAddress: 127.0.0.1:10249 + mode: "" + oomScoreAdj: -999 + portRange: "" + resourceContainer: /kube-proxy + udpTimeoutMilliseconds: 250ms + - path: /etc/kubernetes/openstack/openstack.config + filesystem: root + mode: 0644 + contents: + inline: |- + [Global] + auth-url = {{ .OpenstackAuthURL }} + username = {{ .OpenstackUsername }} + password = {{ .OpenstackPassword }} + domain-name = {{ .OpenstackDomain }} + region = {{ .OpenstackRegion }} + + [LoadBalancer] + lb-version=v2 + subnet-id = {{ .OpenstackLBSubnetID }} + floating-network-id = {{ .OpenstackLBFloatingNetworkID }} + create-monitor = yes + monitor-delay = 1m + monitor-timeout = 30s + monitor-max-retries = 3 + + [BlockStorage] + trust-device-path = no + + [Route] + router-id = {{ .OpenstackRouterID }} +` diff --git a/pkg/util/constants.go b/pkg/util/constants.go index 46181efd83..b3d176d8ee 100644 --- a/pkg/util/constants.go +++ b/pkg/util/constants.go @@ -3,4 +3,7 @@ package util const ( CA_ISSUER_KUBERNIKUS_IDENTIFIER_0 = "SAP Converged Cloud" CA_ISSUER_KUBERNIKUS_IDENTIFIER_1 = "Kubernikus" + + // This is the default Kubernetes version that clusters are created in + DEFAULT_KUBERNETES_VERSION = "1.8.5" ) diff --git a/pkg/util/helm/helm.go b/pkg/util/helm/helm.go index 8d15664599..d406ffc2bf 100644 --- a/pkg/util/helm/helm.go +++ b/pkg/util/helm/helm.go @@ -18,14 +18,15 @@ type OpenstackOptions struct { } type openstackValues struct { - AuthURL string `yaml:"authURL"` - Username string `yaml:"username"` - Password string `yaml:"password"` - DomainName string `yaml:"domainName"` - ProjectID string `yaml:"projectID"` - Region string `yaml:"region"` - LbSubnetID string `yaml:"lbSubnetID"` - RouterID string `yaml:"routerID"` + AuthURL string `yaml:"authURL"` + Username string `yaml:"username"` + Password string `yaml:"password"` + DomainName string `yaml:"domainName"` + ProjectID string `yaml:"projectID"` + Region string `yaml:"region"` + LbSubnetID string `yaml:"lbSubnetID"` + LbFloatingNetworkID string `yaml:"lbFloatingNetworkID"` + RouterID string `yaml:"routerID"` } type persistenceValues struct { @@ -41,6 +42,11 @@ type apiValues struct { WormholeHost string `yaml:"wormholeHost,omitempty"` } +type versionValues struct { + Kubernetes string `yaml:"kubernetes,omitempty"` + Kubernikus string `yaml:"kubernikus,omitempty"` +} + type kubernikusHelmValues struct { Openstack openstackValues `yaml:"openstack,omitempty"` Certs map[string]string `yaml:"certs,omitempty"` @@ -48,7 +54,7 @@ type kubernikusHelmValues struct { ServiceCIDR string `yaml:"serviceCIDR,omitempty"` AdvertiseAddress string `yaml:"advertiseAddress,omitempty"` BoostrapToken string `yaml:"bootstrapToken,omitempty"` - Version string `yaml:"version,omitempty"` + Version versionValues `yaml:"version,omitempty"` Etcd etcdValues `yaml:"etcd,omitempty"` Api apiValues `yaml:"api,omitempty"` } @@ -70,16 +76,20 @@ func KlusterToHelmValues(kluster *v1.Kluster, openstack *OpenstackOptions, certi ClusterCIDR: kluster.Spec.ClusterCIDR, ServiceCIDR: kluster.Spec.ServiceCIDR, AdvertiseAddress: kluster.Spec.AdvertiseAddress, - Version: kluster.Status.Version, + Version: versionValues{ + Kubernetes: kluster.Spec.Version, + Kubernikus: kluster.Status.Version, + }, Openstack: openstackValues{ - AuthURL: openstack.AuthURL, - Username: openstack.Username, - Password: openstack.Password, - DomainName: openstack.DomainName, - Region: openstack.Region, - ProjectID: kluster.Spec.Openstack.ProjectID, - LbSubnetID: kluster.Spec.Openstack.LBSubnetID, - RouterID: kluster.Spec.Openstack.RouterID, + AuthURL: openstack.AuthURL, + Username: openstack.Username, + Password: openstack.Password, + DomainName: openstack.DomainName, + Region: openstack.Region, + ProjectID: kluster.Spec.Openstack.ProjectID, + LbSubnetID: kluster.Spec.Openstack.LBSubnetID, + LbFloatingNetworkID: kluster.Spec.Openstack.LBFloatingNetworkID, + RouterID: kluster.Spec.Openstack.RouterID, }, Etcd: etcdValues{ Persistence: persistenceValues{ diff --git a/swagger.yml b/swagger.yml index f4a26a38ba..2b02a2468c 100644 --- a/swagger.yml +++ b/swagger.yml @@ -354,6 +354,10 @@ definitions: x-nullable: false type: string default: cluster.local + version: + pattern: '^[0-9]+\.[0-9]+\.[0-9]+$' + type: string + readOnly: true name: type: string readOnly: true @@ -370,6 +374,9 @@ definitions: lbSubnetID: x-go-name: LBSubnetID type: string + lbFloatingNetworkID: + x-go-name: LBFloatingNetworkID + type: string securityGroupID: type: string NodePool: