Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

✨ Add bootCommands to cloud-init file generation #11271

Open
wants to merge 18 commits into
base: main
Choose a base branch
from
Open

✨ Add bootCommands to cloud-init file generation #11271

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
31406da
Add bootCommands cloud-init file generation
davidumea Sep 26, 2024
97311d6
fixup: go lint
davidumea Oct 18, 2024
4367074
fixup: lint: added missing newline
davidumea Oct 18, 2024
0c23af8
fixup: add changes from verify-gen
davidumea Oct 18, 2024
504dfd5
fixup: filter tests
davidumea Oct 18, 2024
19d52df
fixup: clearer documentation for BootCommands, PreKubeadmCommands and…
davidumea Oct 18, 2024
51068a2
fixup: verify: reflect new documentation in the crds
davidumea Oct 18, 2024
8ed0171
fixup: restore v1alpha3,v1alpha4 and add conversions
davidumea Oct 22, 2024
332e78d
fixup: non-nil values for bootcommands in cloudinit tests
davidumea Oct 22, 2024
28edd6c
fixup: lint
davidumea Oct 22, 2024
8918ff3
fixup: verify crds
davidumea Nov 5, 2024
94835dc
fixup: use yaml names in go doc comments
davidumea Nov 25, 2024
84e06c6
fixup: removed extra linebreak
davidumea Nov 25, 2024
bd63a05
fixup: remove bootcommands from testing AdditionalFileEncodings
davidumea Nov 26, 2024
a4f3bdb
fixup: make sure bootcommands show up in generated output
davidumea Nov 26, 2024
4673fb0
fixup: add test for join cp commands
davidumea Nov 26, 2024
c059138
fixup: add test for join node commands
davidumea Nov 26, 2024
6fb468d
fixup: ensure commands show up in the right section
davidumea Dec 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions bootstrap/kubeadm/api/v1beta1/kubeadmconfig_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,11 +75,20 @@ type KubeadmConfigSpec struct {
// +optional
Mounts []MountPoints `json:"mounts,omitempty"`

// preKubeadmCommands specifies extra commands to run before kubeadm runs
// BootCommands specifies extra commands to run very early in the boot process via the cloud-init bootcmd
davidumea marked this conversation as resolved.
Show resolved Hide resolved
// module. This is typically run in the cloud-init.service systemd unit. This has no effect in Ignition.
// +optional
BootCommands []BootCommand `json:"bootCommands,omitempty"`

// PreKubeadmCommands specifies extra commands to run before kubeadm runs.
// With cloud-init, this is prepended to the runcmd module configuration, and is typically executed in
// the cloud-final.service systemd unit. In Ignition, this is prepended to /etc/kubeadm.sh.
// +optional
PreKubeadmCommands []string `json:"preKubeadmCommands,omitempty"`

// postKubeadmCommands specifies extra commands to run after kubeadm runs
// PostKubeadmCommands specifies extra commands to run after kubeadm runs.
// With cloud-init, this is appended to the runcmd module configuration, and is typically executed in
// the cloud-final.service systemd unit. In Ignition, this is appended to /etc/kubeadm.sh.
// +optional
PostKubeadmCommands []string `json:"postKubeadmCommands,omitempty"`

Expand Down Expand Up @@ -720,3 +729,6 @@ type Filesystem struct {

// MountPoints defines input for generated mounts in cloud-init.
type MountPoints []string

// BootCommand defines input for each bootcmd command in cloud-init.
type BootCommand []string
30 changes: 30 additions & 0 deletions bootstrap/kubeadm/api/v1beta1/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

29 changes: 22 additions & 7 deletions bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

29 changes: 22 additions & 7 deletions bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

29 changes: 29 additions & 0 deletions bootstrap/kubeadm/internal/cloudinit/boot_commands.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cloudinit

const (
bootCommandsTemplate = `{{ define "boot_commands" -}}
{{- if . }}
bootcmd:{{ range . }}
- {{ range . }}- {{ . }}
{{ end -}}
{{- end -}}
{{- end -}}
{{- end -}}
`
)
5 changes: 5 additions & 0 deletions bootstrap/kubeadm/internal/cloudinit/cloudinit.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ const (
// BaseUserData is shared across all the various types of files written to disk.
type BaseUserData struct {
Header string
BootCommands []bootstrapv1.BootCommand
PreKubeadmCommands []string
PostKubeadmCommands []string
AdditionalFiles []bootstrapv1.File
Expand Down Expand Up @@ -83,6 +84,10 @@ func generate(kind string, tpl string, data interface{}) ([]byte, error) {
return nil, errors.Wrap(err, "failed to parse files template")
}

if _, err := tm.Parse(bootCommandsTemplate); err != nil {
return nil, errors.Wrap(err, "failed to parse boot commands template")
}

if _, err := tm.Parse(commandsTemplate); err != nil {
return nil, errors.Wrap(err, "failed to parse commands template")
}
Expand Down
9 changes: 9 additions & 0 deletions bootstrap/kubeadm/internal/cloudinit/cloudinit_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ func TestNewInitControlPlaneAdditionalFileEncodings(t *testing.T) {
cpinput := &ControlPlaneInput{
BaseUserData: BaseUserData{
Header: "test",
BootCommands: nil,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we have a test for BootCommands not nil for init, join CP, join workers?

Copy link
Author

@davidumea davidumea Oct 22, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I added non-nil values for BootCommands in TestNewInitControlPlaneCommands and TestNewJoinControlPlaneAdditionalFileEncodings, did you want me to add more tests or is this enough? I couldn't find one for "join workers" daf4f97

Copy link
Member

@fabriziopandini fabriziopandini Nov 25, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

WRT to changes in TestNewInitControlPlaneCommands, TestNewJoinControlPlaneAdditionalFileEncodings I have dedicated comments.

We still need test coverage for join and join CP (if they are missing, kindly add them so we pay down some tech debt before adding more on top)

PreKubeadmCommands: nil,
PostKubeadmCommands: nil,
AdditionalFiles: []bootstrapv1.File{
Expand Down Expand Up @@ -94,6 +95,9 @@ func TestNewInitControlPlaneCommands(t *testing.T) {

cpinput := &ControlPlaneInput{
BaseUserData: BaseUserData{
BootCommands: []bootstrapv1.BootCommand{
{"echo", "hello"},
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In order to make this test to be useful to validate the code actually works/to prevent regressions, you should also modify test assertions down below that boot commands actually show up in the generated output (and possibly, ensuring that command show up in the right section of the generated file)

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wasn't sure how to ensure that the commands show up in the right section of the generated file, but I added a check that the bootcommands show up in the generated file and also that the bootcmd: string is there (which is omitted when bootCommands are unset)

a4f3bdb

What do you think?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would suggest to test if the out put has the expected bootcmd and runcmd block vs testing the output has single lines.

Something like

	expectedRunCmd := `runcmd:
  - "\"echo $(date) ': hello world!'\""
  - 'kubeadm init --config /run/kubeadm/kubeadm.yaml  && echo success > /run/cluster-api/bootstrap-success.complete'
  - "echo $(date) ': hello world!'"`

	g.Expect(out).To(ContainSubstring(expectedRunCmd))

Should work; if you want to make it fancy, you can use regex, but up to you (similar for bootCmd)
Might be, to improve readability, let's also replace hello world! with something more specific like "hello preKubeadmCommands!" and "hello postKubeadmCommands!"

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the suggestion!

6fb468d

},
Header: "test",
PreKubeadmCommands: []string{`"echo $(date) ': hello world!'"`},
PostKubeadmCommands: []string{"echo $(date) ': hello world!'"},
Expand Down Expand Up @@ -132,6 +136,7 @@ func TestNewInitControlPlaneDiskMounts(t *testing.T) {
cpinput := &ControlPlaneInput{
BaseUserData: BaseUserData{
Header: "test",
BootCommands: nil,
PreKubeadmCommands: nil,
PostKubeadmCommands: nil,
WriteFiles: nil,
Expand Down Expand Up @@ -194,6 +199,9 @@ func TestNewJoinControlPlaneAdditionalFileEncodings(t *testing.T) {

cpinput := &ControlPlaneJoinInput{
BaseUserData: BaseUserData{
BootCommands: []bootstrapv1.BootCommand{
davidumea marked this conversation as resolved.
Show resolved Hide resolved
{"echo", "hello"},
},
Header: "test",
PreKubeadmCommands: nil,
PostKubeadmCommands: nil,
Expand Down Expand Up @@ -247,6 +255,7 @@ func TestNewJoinControlPlaneExperimentalRetry(t *testing.T) {
cpinput := &ControlPlaneJoinInput{
BaseUserData: BaseUserData{
Header: "test",
BootCommands: nil,
PreKubeadmCommands: nil,
PostKubeadmCommands: nil,
UseExperimentalRetry: true,
Expand Down
1 change: 1 addition & 0 deletions bootstrap/kubeadm/internal/cloudinit/controlplane_init.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ const (
owner: root:root
permissions: '0640'
content: "This placeholder file is used to create the /run/cluster-api sub directory in a way that is compatible with both Linux and Windows (mkdir -p /run/cluster-api does not work with Windows)"
{{- template "boot_commands" .BootCommands }}
runcmd:
{{- template "commands" .PreKubeadmCommands }}
- 'kubeadm init --config /run/kubeadm/kubeadm.yaml {{.KubeadmVerbosity}} && {{ .SentinelFileCommand }}'
Expand Down
1 change: 1 addition & 0 deletions bootstrap/kubeadm/internal/cloudinit/controlplane_join.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ const (
owner: root:root
permissions: '0640'
content: "This placeholder file is used to create the /run/cluster-api sub directory in a way that is compatible with both Linux and Windows (mkdir -p /run/cluster-api does not work with Windows)"
{{- template "boot_commands" .BootCommands }}
runcmd:
{{- template "commands" .PreKubeadmCommands }}
- {{ .KubeadmCommand }} && {{ .SentinelFileCommand }}
Expand Down
1 change: 1 addition & 0 deletions bootstrap/kubeadm/internal/cloudinit/node.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ const (
owner: root:root
permissions: '0640'
content: "This placeholder file is used to create the /run/cluster-api sub directory in a way that is compatible with both Linux and Windows (mkdir -p /run/cluster-api does not work with Windows)"
{{- template "boot_commands" .BootCommands }}
runcmd:
{{- template "commands" .PreKubeadmCommands }}
- {{ .KubeadmCommand }} && {{ .SentinelFileCommand }}
Expand Down
3 changes: 3 additions & 0 deletions bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -531,6 +531,7 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex
BaseUserData: cloudinit.BaseUserData{
AdditionalFiles: files,
NTP: scope.Config.Spec.NTP,
BootCommands: scope.Config.Spec.BootCommands,
PreKubeadmCommands: scope.Config.Spec.PreKubeadmCommands,
PostKubeadmCommands: scope.Config.Spec.PostKubeadmCommands,
Users: users,
Expand Down Expand Up @@ -652,6 +653,7 @@ func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope)
BaseUserData: cloudinit.BaseUserData{
AdditionalFiles: files,
NTP: scope.Config.Spec.NTP,
BootCommands: scope.Config.Spec.BootCommands,
PreKubeadmCommands: scope.Config.Spec.PreKubeadmCommands,
PostKubeadmCommands: scope.Config.Spec.PostKubeadmCommands,
Users: users,
Expand Down Expand Up @@ -771,6 +773,7 @@ func (r *KubeadmConfigReconciler) joinControlplane(ctx context.Context, scope *S
BaseUserData: cloudinit.BaseUserData{
AdditionalFiles: files,
NTP: scope.Config.Spec.NTP,
BootCommands: scope.Config.Spec.BootCommands,
PreKubeadmCommands: scope.Config.Spec.PreKubeadmCommands,
PostKubeadmCommands: scope.Config.Spec.PostKubeadmCommands,
Users: users,
Expand Down
Loading