From 26932449fe7eafcc61978c2ccda7754d18e0bbdf Mon Sep 17 00:00:00 2001 From: Matthew Booth Date: Thu, 8 Aug 2024 08:40:36 +0100 Subject: [PATCH] Install tools with go install * controller-gen * golangci-lint * kustomize * setup-envtest This patch follows the model used by kubebuilder scaffolding v4. The advantage of this model is that it independent of any go module. --- Makefile | 87 ++++-- hack/tools/Makefile | 19 -- hack/tools/ensure-golangci-lint.sh | 416 ----------------------------- hack/tools/go.mod | 15 -- hack/tools/go.sum | 101 ------- hack/tools/tools.go | 3 - 6 files changed, 70 insertions(+), 571 deletions(-) delete mode 100755 hack/tools/ensure-golangci-lint.sh diff --git a/Makefile b/Makefile index 341358ff2f..cffe50551c 100644 --- a/Makefile +++ b/Makefile @@ -47,17 +47,13 @@ GO_APIDIFF_BIN := go-apidiff GO_APIDIFF_PKG := github.com/joelanford/go-apidiff # Binaries. -CONTROLLER_GEN := $(TOOLS_BIN_DIR)/controller-gen CONVERSION_GEN := $(TOOLS_BIN_DIR)/conversion-gen ENVSUBST := $(TOOLS_BIN_DIR)/envsubst GINKGO := $(TOOLS_BIN_DIR)/ginkgo GOJQ := $(TOOLS_BIN_DIR)/gojq -GOLANGCI_LINT := $(TOOLS_BIN_DIR)/golangci-lint GOTESTSUM := $(TOOLS_BIN_DIR)/gotestsum -KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize MOCKGEN := $(TOOLS_BIN_DIR)/mockgen RELEASE_NOTES := $(TOOLS_BIN_DIR)/release-notes -SETUP_ENVTEST := $(TOOLS_BIN_DIR)/setup-envtest GEN_CRD_API_REFERENCE_DOCS := $(TOOLS_BIN_DIR)/gen-crd-api-reference-docs GO_APIDIFF := $(TOOLS_BIN_DIR)/$(GO_APIDIFF_BIN)-$(GO_APIDIFF_VER) @@ -141,9 +137,9 @@ ifdef KUBEBUILDER_ASSETS_DIR endif .PHONY: kubebuilder_assets -kubebuilder_assets: $(SETUP_ENVTEST) +kubebuilder_assets: envtest @echo Fetching assets for $(KUBEBUILDER_ENVTEST_KUBERNETES_VERSION) - $(eval KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use --use-env -p path $(setup_envtest_extra_args) $(KUBEBUILDER_ENVTEST_KUBERNETES_VERSION))) + $(eval KUBEBUILDER_ASSETS ?= $(shell $(ENVTEST) use --use-env -p path $(setup_envtest_extra_args) $(KUBEBUILDER_ENVTEST_KUBERNETES_VERSION))) .PHONY: test TEST_PATHS ?= ./... @@ -173,10 +169,10 @@ e2e-templates: $(addprefix $(E2E_NO_ARTIFACT_TEMPLATES_DIR)/, \ # Currently no templates that require CI artifacts # $(addprefix $(E2E_TEMPLATES_DIR)/, add-templates-here.yaml) \ -$(E2E_NO_ARTIFACT_TEMPLATES_DIR)/cluster-template.yaml: $(E2E_KUSTOMIZE_DIR)/with-tags $(KUSTOMIZE) FORCE +$(E2E_NO_ARTIFACT_TEMPLATES_DIR)/cluster-template.yaml: $(E2E_KUSTOMIZE_DIR)/with-tags kustomize FORCE $(KUSTOMIZE) build "$<" > "$@" -$(E2E_NO_ARTIFACT_TEMPLATES_DIR)/cluster-template-%.yaml: $(E2E_KUSTOMIZE_DIR)/% $(KUSTOMIZE) FORCE +$(E2E_NO_ARTIFACT_TEMPLATES_DIR)/cluster-template-%.yaml: $(E2E_KUSTOMIZE_DIR)/% kustomize FORCE $(KUSTOMIZE) build "$<" > "$@" e2e-prerequisites: e2e-templates e2e-image test-e2e-image-prerequisites ## Build all artifacts required by e2e tests @@ -248,14 +244,15 @@ $(GO_APIDIFF): # Build go-apidiff. ## -------------------------------------- .PHONY: lint -lint: $(GOLANGCI_LINT) ## Lint codebase +lint: golangci-lint ## Lint codebase $(GOLANGCI_LINT) run -v --fast=false .PHONY: lint-update -lint-update: $(GOLANGCI_LINT) ## Lint codebase +lint-update: golangci-lint ## Lint codebase $(GOLANGCI_LINT) run -v --fast=false --fix -lint-fast: $(GOLANGCI_LINT) ## Run only faster linters to detect possible issues +.PHONY: lint-fast +lint-fast: golangci-lint ## Run only faster linters to detect possible issues $(GOLANGCI_LINT) run -v --fast=true ## -------------------------------------- @@ -275,7 +272,7 @@ generate-go: $(MOCKGEN) go generate ./... .PHONY: generate-controller-gen -generate-controller-gen: $(CONTROLLER_GEN) +generate-controller-gen: controller-gen $(CONTROLLER_GEN) \ paths=./api/... \ object:headerFile=./hack/boilerplate/boilerplate.generatego.txt @@ -296,7 +293,7 @@ generate-conversion-gen: $(CONVERSION_GEN) --go-header-file=./hack/boilerplate/boilerplate.generatego.txt .PHONY: generate-manifests -generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc. +generate-manifests: controller-gen ## Generate manifests e.g. CRD, RBAC etc. $(CONTROLLER_GEN) \ paths=./api/... \ crd:crdVersions=v1 \ @@ -451,10 +448,10 @@ templates: templates/cluster-template.yaml \ templates/cluster-template-flatcar.yaml \ templates/cluster-template-flatcar-sysext.yaml -templates/cluster-template.yaml: kustomize/v1beta1/default $(KUSTOMIZE) FORCE +templates/cluster-template.yaml: kustomize/v1beta1/default kustomize FORCE $(KUSTOMIZE) build "$<" > "$@" -templates/cluster-template-%.yaml: kustomize/v1beta1/% $(KUSTOMIZE) FORCE +templates/cluster-template-%.yaml: kustomize/v1beta1/% kustomize FORCE $(KUSTOMIZE) build "$<" > "$@" .PHONY: release-templates @@ -477,14 +474,14 @@ $(RELEASE_DIR)/$(MANIFEST_FILE).yaml: NAMESPACE=$(NAMESPACE) .PHONY: compiled-manifest -compiled-manifest: $(RELEASE_DIR) $(KUSTOMIZE) +compiled-manifest: $(RELEASE_DIR) kustomize $(MAKE) image-patch-source-manifest $(MAKE) image-patch-pull-policy $(MAKE) image-patch-kustomization $(KUSTOMIZE) build $(IMAGE_PATCH_DIR)/$(PROVIDER) > $(RELEASE_DIR)/$(PROVIDER).yaml .PHONY: image-patch-source-manifest -image-patch-source-manifest: $(IMAGE_PATCH_DIR) $(KUSTOMIZE) +image-patch-source-manifest: $(IMAGE_PATCH_DIR) kustomize mkdir -p $(IMAGE_PATCH_DIR)/$(PROVIDER) $(KUSTOMIZE) build $(PROVIDER_CONFIG_DIR)/default > $(IMAGE_PATCH_DIR)/$(PROVIDER)/source-manifest.yaml @@ -551,3 +548,59 @@ compile-e2e: ## Test e2e compilation .PHONY: FORCE FORCE: + +##@ Dependencies + +## Location to install dependencies to +LOCALBIN ?= $(shell pwd)/bin +$(LOCALBIN): + mkdir -p $(LOCALBIN) + +## Tool Binaries +KUBECTL ?= kubectl +KUSTOMIZE ?= $(LOCALBIN)/kustomize +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen +ENVTEST ?= $(LOCALBIN)/setup-envtest +GOLANGCI_LINT = $(LOCALBIN)/golangci-lint + +## Tool Versions +KUSTOMIZE_VERSION ?= v5.4.2 +CONTROLLER_TOOLS_VERSION ?= v0.14.0 +ENVTEST_VERSION ?= release-0.18 +GOLANGCI_LINT_VERSION ?= v1.57.2 + +.PHONY: kustomize +kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. +$(KUSTOMIZE): $(LOCALBIN) + $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) + +.PHONY: controller-gen +controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. +$(CONTROLLER_GEN): $(LOCALBIN) + $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION)) + +.PHONY: envtest +envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. +$(ENVTEST): $(LOCALBIN) + $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION)) + +.PHONY: golangci-lint +golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. +$(GOLANGCI_LINT): $(LOCALBIN) + $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) + +# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist +# $1 - target path with name of binary +# $2 - package url which can be installed +# $3 - specific version of package +define go-install-tool +@[ -f "$(1)-$(3)" ] || { \ +set -e; \ +package=$(2)@$(3) ;\ +echo "Downloading $${package}" ;\ +rm -f $(1) || true ;\ +GOBIN=$(LOCALBIN) go install $${package} ;\ +mv $(1) $(1)-$(3) ;\ +} ;\ +ln -sf $(1)-$(3) $(1) +endef diff --git a/hack/tools/Makefile b/hack/tools/Makefile index bac2a946fa..e6a42fc08b 100644 --- a/hack/tools/Makefile +++ b/hack/tools/Makefile @@ -15,8 +15,6 @@ ROOT_DIR_RELATIVE := ../.. include $(ROOT_DIR_RELATIVE)/common.mk -GOLANGCI_LINT_VERSION ?= v1.57.2 - # GOTESTSUM version without the leading 'v' GOTESTSUM_VERSION ?= 1.11.0 @@ -62,11 +60,6 @@ $(SHARE_DIR): $(GTAR): @$(GTAR) --version > /dev/null || (echo Install GNU Tar with brew install gnu-tar && exit -1) - -CONTROLLER_GEN := $(BIN_DIR)/controller-gen -$(CONTROLLER_GEN): go.mod go.sum | $(BIN_DIR) # Build controller-gen from tools folder. - go build -tags=tools -o $@ sigs.k8s.io/controller-tools/cmd/controller-gen - CONVERSION_GEN := $(BIN_DIR)/conversion-gen $(CONVERSION_GEN): go.mod go.sum | $(BIN_DIR) go build -tags=tools -o $@ k8s.io/code-generator/cmd/conversion-gen @@ -75,10 +68,6 @@ ENVSUBST := $(BIN_DIR)/envsubst $(ENVSUBST): go.mod go.sum | $(BIN_DIR) # Build envsubst from tools folder. go build -tags=tools -o $@ github.com/a8m/envsubst/cmd/envsubst -SETUP_ENVTEST := $(BIN_DIR)/setup-envtest -$(SETUP_ENVTEST): go.mod go.sum | $(BIN_DIR) - go build -tags=tools -o $@ sigs.k8s.io/controller-runtime/tools/setup-envtest - GOTESTSUM := $(BIN_DIR)/gotestsum $(GOTESTSUM): | $(BIN_DIR) curl -L https://github.com/gotestyourself/gotestsum/releases/download/v$(GOTESTSUM_VERSION)/gotestsum_$(GOTESTSUM_VERSION)_linux_$(ARCH).tar.gz | \ @@ -106,14 +95,6 @@ GOJQ := $(BIN_DIR)/gojq $(GOJQ): go.mod go.sum | $(BIN_DIR) go build -tags=tools -o $@ github.com/itchyny/gojq/cmd/gojq -GOLANGCI_LINT := $(BIN_DIR)/golangci-lint -$(GOLANGCI_LINT): Makefile ensure-golangci-lint.sh | $(BIN_DIR) - ./ensure-golangci-lint.sh -b $(BIN_DIR) $(GOLANGCI_LINT_VERSION) - -KUSTOMIZE := $(BIN_DIR)/kustomize -$(KUSTOMIZE): go.mod go.sum | $(BIN_DIR) # Build kustomize from tools folder. - CGO_ENABLED=0 go build -tags=tools -o $@ sigs.k8s.io/kustomize/kustomize/v5 - MDBOOK_SHARE := $(SHARE_DIR)/mdbook$(MDBOOK_ARCHIVE_EXT) $(MDBOOK_SHARE): ../../versions.mk $(SHARE_DIR) curl -sL -o $(MDBOOK_SHARE) "https://github.com/rust-lang/mdBook/releases/download/$(MDBOOK_VERSION)/mdBook-$(MDBOOK_VERSION)-x86_64-$(RUST_TARGET)$(MDBOOK_ARCHIVE_EXT)" diff --git a/hack/tools/ensure-golangci-lint.sh b/hack/tools/ensure-golangci-lint.sh deleted file mode 100755 index cf05ba4fc5..0000000000 --- a/hack/tools/ensure-golangci-lint.sh +++ /dev/null @@ -1,416 +0,0 @@ -#!/bin/sh - -# Downloaded from -# https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh - -set -e - -usage() { - this=$1 - cat </dev/null -} -echoerr() { - echo "$@" 1>&2 -} -log_prefix() { - echo "$0" -} -_logp=6 -log_set_priority() { - _logp="$1" -} -log_priority() { - if test -z "$1"; then - echo "$_logp" - return - fi - [ "$1" -le "$_logp" ] -} -log_tag() { - case $1 in - 0) echo "emerg" ;; - 1) echo "alert" ;; - 2) echo "crit" ;; - 3) echo "err" ;; - 4) echo "warning" ;; - 5) echo "notice" ;; - 6) echo "info" ;; - 7) echo "debug" ;; - *) echo "$1" ;; - esac -} -log_debug() { - log_priority 7 || return 0 - echoerr "$(log_prefix)" "$(log_tag 7)" "$@" -} -log_info() { - log_priority 6 || return 0 - echoerr "$(log_prefix)" "$(log_tag 6)" "$@" -} -log_err() { - log_priority 3 || return 0 - echoerr "$(log_prefix)" "$(log_tag 3)" "$@" -} -log_crit() { - log_priority 2 || return 0 - echoerr "$(log_prefix)" "$(log_tag 2)" "$@" -} -uname_os() { - os=$(uname -s | tr '[:upper:]' '[:lower:]') - case "$os" in - msys*) os="windows" ;; - mingw*) os="windows" ;; - cygwin*) os="windows" ;; - win*) os="windows" ;; - esac - echo "$os" -} -uname_arch() { - arch=$(uname -m) - case $arch in - x86_64) arch="amd64" ;; - x86) arch="386" ;; - i686) arch="386" ;; - i386) arch="386" ;; - aarch64) arch="arm64" ;; - armv5*) arch="armv5" ;; - armv6*) arch="armv6" ;; - armv7*) arch="armv7" ;; - esac - echo ${arch} -} -uname_os_check() { - os=$(uname_os) - case "$os" in - darwin) return 0 ;; - dragonfly) return 0 ;; - freebsd) return 0 ;; - linux) return 0 ;; - android) return 0 ;; - nacl) return 0 ;; - netbsd) return 0 ;; - openbsd) return 0 ;; - plan9) return 0 ;; - solaris) return 0 ;; - windows) return 0 ;; - esac - log_crit "uname_os_check '$(uname -s)' got converted to '$os' which is not a GOOS value." - return 1 -} -uname_arch_check() { - arch=$(uname_arch) - case "$arch" in - 386) return 0 ;; - amd64) return 0 ;; - arm64) return 0 ;; - armv5) return 0 ;; - armv6) return 0 ;; - armv7) return 0 ;; - ppc64) return 0 ;; - ppc64le) return 0 ;; - mips) return 0 ;; - mipsle) return 0 ;; - mips64) return 0 ;; - mips64le) return 0 ;; - s390x) return 0 ;; - riscv64) return 0 ;; - amd64p32) return 0 ;; - esac - log_crit "uname_arch_check '$(uname -m)' got converted to '$arch' which is not a GOARCH value." - return 1 -} -untar() { - tarball=$1 - case "${tarball}" in - *.tar.gz | *.tgz) tar --no-same-owner -xzf "${tarball}" ;; - *.tar) tar --no-same-owner -xf "${tarball}" ;; - *.zip) unzip "${tarball}" ;; - *) - log_err "untar unknown archive format for ${tarball}" - return 1 - ;; - esac -} -http_download_curl() { - local_file=$1 - source_url=$2 - header=$3 - if [ -z "$header" ]; then - code=$(curl -w '%{http_code}' -sL -o "$local_file" "$source_url") - else - code=$(curl -w '%{http_code}' -sL -H "$header" -o "$local_file" "$source_url") - fi - if [ "$code" != "200" ]; then - log_debug "http_download_curl received HTTP status $code" - return 1 - fi - return 0 -} -http_download_wget() { - local_file=$1 - source_url=$2 - header=$3 - if [ -z "$header" ]; then - wget -q -O "$local_file" "$source_url" - else - wget -q --header "$header" -O "$local_file" "$source_url" - fi -} -http_download() { - log_debug "http_download $2" - if is_command curl; then - http_download_curl "$@" - return - elif is_command wget; then - http_download_wget "$@" - return - fi - log_crit "http_download unable to find wget or curl" - return 1 -} -http_copy() { - tmp=$(mktemp) - http_download "${tmp}" "$1" "$2" || return 1 - body=$(cat "$tmp") - rm -f "${tmp}" - echo "$body" -} -github_release() { - owner_repo=$1 - version=$2 - test -z "$version" && version="latest" - giturl="https://github.com/${owner_repo}/releases/${version}" - json=$(http_copy "$giturl" "Accept:application/json") - test -z "$json" && return 1 - version=$(echo "$json" | tr -s '\n' ' ' | sed 's/.*"tag_name":"//' | sed 's/".*//') - test -z "$version" && return 1 - echo "$version" -} -hash_sha256() { - TARGET=${1:-/dev/stdin} - if is_command gsha256sum; then - hash=$(gsha256sum "$TARGET") || return 1 - echo "$hash" | cut -d ' ' -f 1 - elif is_command sha256sum; then - hash=$(sha256sum "$TARGET") || return 1 - echo "$hash" | cut -d ' ' -f 1 - elif is_command shasum; then - hash=$(shasum -a 256 "$TARGET" 2>/dev/null) || return 1 - echo "$hash" | cut -d ' ' -f 1 - elif is_command openssl; then - hash=$(openssl -dst openssl dgst -sha256 "$TARGET") || return 1 - echo "$hash" | cut -d ' ' -f a - else - log_crit "hash_sha256 unable to find command to compute sha-256 hash" - return 1 - fi -} -hash_sha256_verify() { - TARGET=$1 - checksums=$2 - if [ -z "$checksums" ]; then - log_err "hash_sha256_verify checksum file not specified in arg2" - return 1 - fi - BASENAME=${TARGET##*/} - want=$(grep "${BASENAME}" "${checksums}" 2>/dev/null | tr '\t' ' ' | cut -d ' ' -f 1) - if [ -z "$want" ]; then - log_err "hash_sha256_verify unable to find checksum for '${TARGET}' in '${checksums}'" - return 1 - fi - got=$(hash_sha256 "$TARGET") - if [ "$want" != "$got" ]; then - log_err "hash_sha256_verify checksum for '$TARGET' did not verify ${want} vs $got" - return 1 - fi -} -cat /dev/null <