Downstream patches for 1.1.0-1 release branch

.github/workflows/release.yaml

@@ -0,0 +1,67 @@
+name: Release
+on:
+  push:
+    tags:
+      - v*-d2iq.*
+
+# Because variables are not exported, they are not visible by child processes, e.g. make
+env:
+  registry: ghcr.io
+  registry_username: ${{ github.actor }}
+  registry_password: ${{ secrets.GITHUB_TOKEN }}
+  # The repository follows the naming convention of other Cluster API providers.
+  repository: mesosphere/cluster-api-vcd-controller
+  version: ${{ github.ref_name }}
+
+jobs:
+  release:
+    name: Release
+    runs-on:
+      # For a list of pre-installed software, see https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2204-Readme.md
+      - ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version-file: go.mod
+          cache: true # cache go action in github actions cache store.
+      - name: Login to container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.registry }}
+          username: ${{ env.registry_username }}
+          password: ${{ env.registry_password }}
+      - name: Build multi-arch container image, and push to container registry
+        # Note: We set make variables using positional arguments (to the right of the make command), not environment
+        # variables (to the left of the make command). While make converts environment variables to make variables, the
+        # conversion is not straightforward. For example, conditional assignments are not overriden by environment
+        # variables.
+        run: |
+          make push-capvcd-image \
+            --makefile d2iq.Makefile \
+            REGISTRY=${{ env.registry }} \
+            CAPVCD_IMG=${{ env.repository }} \
+            VERSION=${{ env.version }}
+      - name: Generate GitHub release artifacts
+        # See note on make variables, above.
+        run: |
+          make release-manifests \
+            --makefile d2iq.Makefile
+            REGISTRY=${{ env.registry }} \
+            CAPVCD_IMG=${{ env.repository }} \
+            VERSION=${{ env.version }}
+      - name: Create (draft) GitHub release
+        run: |
+          gh release create ${{ env.version }} \
+            --title ${{ env.version }} \
+            --draft
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Upload GitHub release artifacts
+        run: |
+          gh release upload ${{ env.version }} \
+            metadata.yaml \
+            templates/infrastructure-components.yaml
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yaml

@@ -0,0 +1,25 @@
+name: Test
+on:
+  push:
+    branches:
+      - d2iq/*
+  pull_request:
+    branches:
+      - d2iq/*
+
+jobs:
+  test:
+    name: Test
+    runs-on: 
+      - self-hosted
+      - small
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version-file: go.mod
+          cache: true # cache go action in github actions cache store.
+      - name: make test
+        run: |
+          make test TEST_PACKAGES='$(shell go list ./... | grep -v tests/e2e)'

Makefile

@@ -37,6 +37,7 @@ GOLANGCI_LINT ?= bin/golangci-lint
 GOSEC ?= bin/gosec
 SHELLCHECK ?= bin/shellcheck
 
+TEST_PACKAGES := ./...
 
 .PHONY: all
 all: vendor lint dev
@@ -128,11 +129,12 @@ build: bin ## Build CAPVCD binary. To be used from within a Dockerfile
 
 .PHONY: test
 test: bin/testbin manifests generate ## Run tests.
-	test -f bin/testbin/setup-envtest.sh || curl -sSLo bin/testbin/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.8.3/hack/setup-envtest.sh
-	source bin/testbin/setup-envtest.sh
-	fetch_envtest_tools bin/testbin
-	setup_envtest_env bin/testbin
-	go test ./... -coverprofile cover.out
+	test -f bin/testbin/setup-envtest.sh || \
+	curl -sSLo bin/testbin/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.8.3/hack/setup-envtest.sh; \
+	source bin/testbin/setup-envtest.sh; \
+	fetch_envtest_tools bin/testbin; \
+	setup_envtest_env "$(shell pwd)/bin/testbin"; \
+	go test $(TEST_PACKAGES) -coverprofile cover.out
 
 .PHONY: manager
 manager: bin generate ## Build manager binary.

config/crd/bases/infrastructure.cluster.x-k8s.io_vcdmachines.yaml

@@ -208,8 +208,8 @@ spec:
                       description: The machine address.
                       type: string
                     type:
-                      description: Machine address type, one of Hostname, ExternalIP
-                        or InternalIP.
+                      description: Machine address type, one of Hostname, ExternalIP,
+                        InternalIP, ExternalDNS or InternalDNS.
                       type: string
                   required:
                   - address
@@ -388,8 +388,8 @@ spec:
                       description: The machine address.
                       type: string
                     type:
-                      description: Machine address type, one of Hostname, ExternalIP
-                        or InternalIP.
+                      description: Machine address type, one of Hostname, ExternalIP,
+                        InternalIP, ExternalDNS or InternalDNS.
                       type: string
                   required:
                   - address

controllers/cluster_scripts/cloud_init.tmpl

@@ -3,63 +3,6 @@ users:
   - name: root
     lock_passwd: false
 write_files:
-- path: /etc/cloud/cloud.cfg.d/cse.cfg
-  owner: root
-  content: |
-     ssh_deletekeys: false {{- if .NvidiaGPU }}
-- path: /etc/containerd/config.toml
-  owner: root
-  content: |
-    version = 2
-
-    [plugins]
-      [plugins."io.containerd.grpc.v1.cri"]
-        sandbox_image = "projects.registry.vmware.com/tkg/pause:3.4.1"
-
-        [plugins."io.containerd.grpc.v1.cri".containerd]
-          default_runtime_name = "nvidia"
-
-          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
-            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia]
-              privileged_without_host_devices = false
-              runtime_engine = ""
-              runtime_root = ""
-              runtime_type = "io.containerd.runc.v2"
-
-              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options]
-                BinaryName = "/usr/bin/nvidia-container-runtime"
-
-        [plugins."io.containerd.grpc.v1.cri".registry]
-          [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
-            [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
-              endpoint = ["https://registry-1.docker.io"] {{- end }}
-- path: /opt/vmware/cloud-director/metering.sh
-  owner: root
-  content: |
-     #!/usr/bin/env bash
-     vmtoolsd --cmd "info-set guestinfo.metering.vcd_site_id $VCD_SITE_ID"
-     vmtoolsd --cmd "info-set guestinfo.metering.cluster_id $CLUSTER_ID"
-     vmtoolsd --cmd "info-set guestinfo.metering.tkg_version $TKG_VERSION"
-     vmtoolsd --cmd "info-set guestinfo.metering.machine_type $MACHINE_TYPE"
-     vmtoolsd --cmd "info-set guestinfo.metering.mgmt $MGMT"
-- path: /etc/vcloud/metering
-  owner: root
-  content: |
-    VCD_SITE_ID={{ .VcdHostFormatted }}
-    CLUSTER_ID={{ .ClusterID }}
-    TKG_VERSION={{ .TKGVersion }}
-    MACHINE_TYPE={{- if or .ControlPlane .ResizedControlPlane -}} control_plane {{- else -}} worker {{- end }}
-    MGMT=true
-- path: /etc/systemd/system/metering.service
-  owner: root
-  content: |
-    [Service]
-    Type=simple
-    EnvironmentFile=/etc/vcloud/metering
-    ExecStart=/bin/bash /opt/vmware/cloud-director/metering.sh
-
-    [Install]
-    WantedBy=multi-user.target
 - path: /root/ {{- if .ControlPlane -}} control_plane {{- else -}} node {{- end -}} .sh
   owner: root
   content: |
@@ -74,13 +17,6 @@ write_files:
         ERROR_MESSAGE="$ERROR_MESSAGE $KUBEADM_FAILURE"
       fi
       vmtoolsd --cmd "info-set guestinfo.post_customization_script_execution_failure_reason $ERROR_MESSAGE"
-
-      CLOUD_INIT_OUTPUT=""
-      if [[ -f /var/log/cloud-init-output.log ]]
-      then
-        CLOUD_INIT_OUTPUT=$(</var/log/cloud-init-output.log)
-      fi
-      vmtoolsd --cmd "info-set guestinfo.post_customization_cloud_init_output $CLOUD_INIT_OUTPUT"
     }
     mkdir -p /var/log/capvcd/customization
     trap 'catch $? $LINENO' ERR EXIT
@@ -97,60 +33,21 @@ write_files:
     CSI_NODE_PATH=/root/csi-node.yaml {{- end }}
 
     vmtoolsd --cmd "info-set guestinfo.postcustomization.networkconfiguration.status in_progress"
-    echo 'net.ipv6.conf.all.disable_ipv6 = 1' >> /etc/sysctl.conf
-    echo 'net.ipv6.conf.default.disable_ipv6 = 1' >> /etc/sysctl.conf
-    echo 'net.ipv6.conf.lo.disable_ipv6 = 1' >> /etc/sysctl.conf
-    sudo sysctl -p
-    # also remove ipv6 localhost entry from /etc/hosts
-    sed -i 's/::1/127.0.0.1/g' /etc/hosts || true
+    hostname "{{ .MachineName }}"
+    echo "::1         ipv6-localhost ipv6-loopback" >/etc/hosts
+    echo "127.0.0.1   localhost" >>/etc/hosts
+    echo "{{ .MachineName }}" >/etc/hostname
+    echo "127.0.0.1" `hostname` >>/etc/hosts
     vmtoolsd --cmd "info-set guestinfo.postcustomization.networkconfiguration.status successful"
 
     vmtoolsd --cmd "info-set guestinfo.metering.status in_progress"
-    systemctl enable --now metering
-    vmtoolsd --cmd "info-set guestinfo.metering.status successful" {{- if or .HTTPProxy .HTTPSProxy }}
+    vmtoolsd --cmd "info-set guestinfo.metering.status successful"
 
     vmtoolsd --cmd "info-set guestinfo.postcustomization.proxy.setting.status in_progress"
-    export HTTP_PROXY="{{.HTTPProxy}}"
-    export HTTPS_PROXY="{{.HTTPSProxy}}"
-    export http_proxy="{{.HTTPProxy}}"
-    export https_proxy="{{.HTTPSProxy}}"
-    export NO_PROXY="{{.NoProxy}}"
-    export no_proxy="{{.NoProxy}}"
-    cat <<END > /etc/systemd/system/containerd.service.d/http-proxy.conf
-    [Service]
-    Environment="HTTP_PROXY={{.HTTPProxy}}"
-    Environment="HTTPS_PROXY={{.HTTPSProxy}}"
-    Environment="http_proxy={{.HTTPProxy}}"
-    Environment="https_proxy={{.HTTPSProxy}}"
-    Environment="no_proxy={{.NoProxy}}"
-    Environment="NO_PROXY={{.NoProxy}}"
-    END
-    systemctl daemon-reload
-    systemctl restart containerd
-    vmtoolsd --cmd "info-set guestinfo.postcustomization.proxy.setting.status successful" {{- end }} {{- if .NvidiaGPU }}
-
-    vmtoolsd --cmd "info-set guestinfo.postcustomization.nvidia.runtime.install.status in_progress"
-    distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
-    curl -s -L https://nvidia.github.io/libnvidia-container/gpgkey | sudo apt-key add -
-    curl -s -L https://nvidia.github.io/libnvidia-container/$distribution/libnvidia-container.list | sudo tee /etc/apt/sources.list.d/libnvidia-container.list
-
-    sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit
-
-    vmtoolsd --cmd "info-set guestinfo.postcustomization.nvidia.runtime.install.status successful" {{- end }}
+    vmtoolsd --cmd "info-set guestinfo.postcustomization.proxy.setting.status successful"
 
     vmtoolsd --cmd "info-set {{ if .ControlPlane -}} guestinfo.postcustomization.kubeinit.status {{- else -}} guestinfo.postcustomization.kubeadm.node.join.status {{- end }} in_progress"
-    for IMAGE in "coredns" "etcd" "kube-proxy" "kube-apiserver" "kube-controller-manager" "kube-scheduler"
-    do
-      IMAGE_REF=$(ctr -n=k8s.io image list | cut -d" " -f1 | grep $IMAGE)
-      REF_PATH=$(echo $IMAGE_REF | sed 's/:.*//')
-      NEW_TAG_VERSION=$(echo $IMAGE_REF | sed 's/.*://' | sed 's/_/-/')
-      ctr -n=k8s.io image tag $IMAGE_REF $REF_PATH:$NEW_TAG_VERSION
-    done
-    set +x
-    {
-      {{ .BootstrapRunCmd }}
-    } 2> /root/kubeadm.err
-    set -x
+    {{ .BootstrapRunCmd }}
     if [[ ! -f /run/cluster-api/bootstrap-success.complete ]]
     then
       echo "file /run/cluster-api/bootstrap-success.complete not found" &>> /var/log/capvcd/customization/error.log
@@ -161,11 +58,12 @@ write_files:
     echo "$(date) post customization script execution completed" &>> /var/log/capvcd/customization/status.log
     exit 0
 runcmd:
-- 'cloud-init clean'
-- '[ ! -f /opt/vmware/cloud-director/metering.sh ] && sudo reboot'
-- '[ ! -f /etc/cloud/cloud.cfg.d/cse.cfg ] && sudo reboot'
-- '[ ! -f /etc/vcloud/metering ] && sudo reboot'
+- 'sudo cloud-init clean --seed --logs'
+- 'sudo cat /dev/null > /var/log/cloud-init-output.log'
 {{ if .ControlPlane }}
+- '[ ! -f /run/kubeadm/konvoy-set-kube-proxy-configuration.sh] && sudo reboot'
+- '[ ! -f /run/konvoy/containerd-apply-patches.sh] && sudo reboot'
+- '[ ! -f /run/konvoy/restart-containerd-and-wait.sh] && sudo reboot'
 - '[ ! -f /root/control_plane.sh ] && sudo reboot'
 - '[ ! -f /run/kubeadm/kubeadm.yaml ] && sudo reboot'
 - bash /root/control_plane.sh
@@ -176,6 +74,9 @@ runcmd:
 {{ end }}
 timezone: UTC
 disable_root: false
+disable_vmware_customization: true
+network:
+  config: disabled
 preserve_hostname: false
 hostname: "{{ .MachineName }}"
-final_message: "The system is ready after $UPTIME seconds"
+final_message: "The system is ready after $UPTIME seconds"

d2iq.Dockerfile

@@ -0,0 +1,32 @@
+# syntax=docker/dockerfile:1
+
+# Build the manager binary
+FROM golang:1.19 as builder
+
+RUN apt-get update && \
+    apt-get -y install \
+        bash \
+        git  \
+        make
+
+ADD . /go/src/github.com/vmware/cluster-api-provider-cloud-director
+WORKDIR /go/src/github.com/vmware/cluster-api-provider-cloud-director
+
+ENV GOPATH /go
+ENV GOARCH $TARGETARCH
+ARG VERSION
+RUN make -f d2iq.Makefile build-within-docker VERSION=$VERSION && \
+    chmod +x /build/vcloud/cluster-api-provider-cloud-director
+
+########################################################
+
+FROM scratch
+
+WORKDIR /opt/vcloud/bin
+
+COPY --from=builder /build/vcloud/cluster-api-provider-cloud-director .
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+
+# nobody user ID
+USER 65534
+ENTRYPOINT ["/opt/vcloud/bin/cluster-api-provider-cloud-director"]

d2iq.Makefile

@@ -0,0 +1,33 @@
+include Makefile
+
+PLATFORMS := "linux/amd64,linux/arm64"
+
+# The multi-platform build must push its output to a registry, because the Docker Engine image store does not support
+# multi-platform images.  This will be addressed when Docker Engine supports the containerd image store. See
+# https://docs.docker.com/storage/containerd.
+.PHONY: push-capvcd-image
+push-capvcd-image: docker-buildx-builder generate fmt vet vendor
+	docker buildx build \
+		--builder capvcd \
+		--platform $(PLATFORMS) \
+		--output=type=registry \
+		--build-arg VERSION=$(VERSION) \
+		--tag $(REGISTRY)/$(CAPVCD_IMG):$(VERSION) \
+		--file d2iq.Dockerfile \
+		.
+
+.PHONY: docker-buildx-builder
+docker-buildx-builder:
+	docker buildx inspect --bootstrap capvcd &>/dev/null || docker buildx create --name capvcd
+
+# The upstream 'release-manifests' target does not correctly override the image.
+# We work around this by using `kustomize edit set image`.
+release-manifests: kustomize
+	mkdir -p $(MANIFEST_DIR)
+	cd config/manager && $(KUSTOMIZE) edit set image projects.registry.vmware.com/vmware-cloud-director/cluster-api-provider-cloud-director=$(REGISTRY)/$(CAPVCD_IMG):$(VERSION)
+	$(KUSTOMIZE) build config/default > $(MANIFEST_DIR)/infrastructure-components.yaml
+
+.PHONY: build-within-docker
+build-within-docker: vendor
+	mkdir -p /build/cluster-api-provider-cloud-director
+	CGO_ENABLED=0 go build -ldflags "-X github.com/vmware/$(CAPVCD_IMG)/version.Version=${VERSION}" -o /build/vcloud/cluster-api-provider-cloud-director main.go