Feature: custom metrics response time (#29)

- feat: metrics-exposer prototype
- refactor: cleanup code
- fix: adjust response time conversion
- fix: adjustments

.gitignore

@@ -23,3 +23,4 @@ pkg/*/pod-autoscaler
 pkg/*/pod-replicas-updater
 pkg/*/containerscale-controller
 pkg/*/http-metrics
+pkg/*/metrics-exposer

go.mod

@@ -5,16 +5,21 @@ go 1.15
 require (
 	github.com/asecurityteam/rolling v2.0.4+incompatible
 	github.com/go-logr/logr v0.3.0 // indirect
+	github.com/go-openapi/spec v0.20.0
+	github.com/kubernetes-sigs/custom-metrics-apiserver v0.0.0-20201216091021-1b9fa998bbaa
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd
 	github.com/onsi/ginkgo v1.14.2
 	github.com/onsi/gomega v1.10.3
 	github.com/stretchr/testify v1.6.1
 	golang.org/x/tools v0.0.0-20200616195046-dc31b401abb5 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
-	k8s.io/api v0.19.4
-	k8s.io/apimachinery v0.19.4
-	k8s.io/client-go v0.19.4
-	k8s.io/code-generator v0.19.4
-	k8s.io/klog/v2 v2.2.0
+	k8s.io/api v0.20.0
+	k8s.io/apimachinery v0.20.0
+	k8s.io/apiserver v0.20.0
+	k8s.io/client-go v0.20.0
+	k8s.io/code-generator v0.20.0
+	k8s.io/component-base v0.20.0
+	k8s.io/klog/v2 v2.4.0
+	k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd
+	k8s.io/metrics v0.20.0
 	sigs.k8s.io/controller-runtime v0.6.4
 )

hack/tools.go

@@ -1,3 +1,6 @@
 package tools
 
-import _ "k8s.io/code-generator" // This package imports things required by build scripts, to force `go mod` to see them as dependencies
+import (
+	_ "k8s.io/code-generator" // This package imports things required by build scripts, to force `go mod` to see them as dependencies
+	_ "k8s.io/kube-openapi/cmd/openapi-gen"
+)

pkg/containerscale-controller/main.go

@@ -2,9 +2,10 @@ package main
 
 import (
 	"flag"
-	informers "github.com/lterrac/system-autoscaler/pkg/informers"
 	"time"
 
+	informers "github.com/lterrac/system-autoscaler/pkg/informers"
+
 	coreinformers "k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"

pkg/metrics-exposer/Dockerfile

@@ -0,0 +1,3 @@
+FROM gcr.io/distroless/static:nonroot
+COPY metrics-exposer /
+ENTRYPOINT ["/metrics-exposer", "--logtostderr=true"]

pkg/metrics-exposer/Makefile

@@ -0,0 +1,58 @@
+BUILD_SETTINGS = CGO_ENABLED=0 GOOS=linux GOARCH=amd64
+IMAGE = metrics-exposer
+IMAGE_VERSION = $(shell git tag --points-at HEAD | sed '/$(IMAGE)\/.*/!s/.*//' | sed 's/\//:/')
+REPO = systemautoscaler
+TEMP_DIR:=$(shell mktemp -d)
+ARCH?=amd64
+OUT_DIR?=./_output
+
+OPENAPI_PATH=../../vendor/k8s.io/kube-openapi
+
+VERSION?=latest
+
+.PHONY: all build coverage clean e2e fmt release test vet
+
+all: build test coverage clean
+
+build: generated/openapi/zz_generated.openapi.go fmt vet test
+	@sed -i.bak 's|REGISTRY|'${REPO}'|g' configure.yaml
+	@rm configure.yaml.bak
+	$(BUILD_SETTINGS) go build -trimpath -o "$(IMAGE)" ./main.go
+
+fmt:
+	@go fmt ./...
+
+test:
+	@go test -race $(shell go list ./... | grep -v e2e) --coverprofile=coverage.out
+
+e2e:
+	@go test -race $(shell go list ./... | grep e2e)
+
+coverage: test
+	@go tool cover -func=coverage.out
+
+release:
+	@if [ -n "$(IMAGE_VERSION)" ]; then \
+		echo "Building $(IMAGE_VERSION)" ;\
+		docker build -t $(REPO)/$(IMAGE_VERSION) . ;\
+		docker push $(REPO)/$(IMAGE_VERSION) ;\
+	else \
+		echo "$(IMAGE) unchanged: no version tag on HEAD commit" ;\
+	fi
+
+vet:
+	@go vet ./...
+
+clean:
+	@rm -rf ./$(IMAGE)
+	@go clean -cache
+	@rm -rf *.out
+
+generated/openapi/zz_generated.openapi.go:
+	@go run $(OPENAPI_PATH)/cmd/openapi-gen/openapi-gen.go --logtostderr \
+	    -i k8s.io/metrics/pkg/apis/custom_metrics,k8s.io/metrics/pkg/apis/custom_metrics/v1beta1,k8s.io/metrics/pkg/apis/custom_metrics/v1beta2,k8s.io/metrics/pkg/apis/external_metrics,k8s.io/metrics/pkg/apis/external_metrics/v1beta1,k8s.io/apimachinery/pkg/apis/meta/v1,k8s.io/apimachinery/pkg/api/resource,k8s.io/apimachinery/pkg/version,k8s.io/api/core/v1 \
+	    -h ../../hack/boilerplate.go.txt \
+	    -p ./pkg/generated/openapi \
+	    -O zz_generated.openapi \
+	    -o ./ \
+	    -r /dev/null

pkg/metrics-exposer/configure.yaml

@@ -0,0 +1,155 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: custom-metrics
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: custom-metrics:system:auth-delegator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+- kind: ServiceAccount
+  name: custom-metrics-apiserver
+  namespace: custom-metrics
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: custom-metrics-auth-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- kind: ServiceAccount
+  name: custom-metrics-apiserver
+  namespace: custom-metrics
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: custom-metrics-apiserver
+  name: custom-metrics-apiserver
+  namespace: custom-metrics
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: custom-metrics-apiserver
+  template:
+    metadata:
+      labels:
+        app: custom-metrics-apiserver
+      name: custom-metrics-apiserver
+    spec:
+      serviceAccountName: custom-metrics-apiserver
+      containers:
+      - name: custom-metrics-apiserver
+        image: lterrac/metrics-exposer:0.1.0
+        imagePullPolicy: Always
+        resources:
+          requests:
+            cpu: 200m
+            memory: 200Mi
+          limits:
+            cpu: 200m
+            memory: 200Mi
+        args:
+        - /metrics-exposer
+        - --secure-port=6443
+        - --logtostderr=true
+        - --v=10
+        ports:
+        - containerPort: 6443
+          name: https
+        - containerPort: 8080
+          name: http
+        volumeMounts:
+        - mountPath: /tmp
+          name: temp-vol
+      volumes:
+      - name: temp-vol
+        emptyDir: {}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: custom-metrics-resource-reader
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: custom-metrics-resource-reader
+subjects:
+- kind: ServiceAccount
+  name: custom-metrics-apiserver
+  namespace: custom-metrics
+---
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: custom-metrics-apiserver
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: custom-metrics-apiserver
+  namespace: custom-metrics
+spec:
+  ports:
+  - name: https
+    port: 443
+    targetPort: 6443
+  - name: http
+    port: 80
+    targetPort: 8080
+  selector:
+    app: custom-metrics-apiserver
+---
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  name: v1beta1.custom.metrics.k8s.io
+spec:
+  service:
+    name: custom-metrics-apiserver
+    namespace: custom-metrics
+  group: custom.metrics.k8s.io
+  version: v1beta1
+  insecureSkipTLSVerify: true
+  groupPriorityMinimum: 100
+  versionPriority: 100
+---
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  name: v1beta2.custom.metrics.k8s.io
+spec:
+  service:
+    name: custom-metrics-apiserver
+    namespace: custom-metrics
+  group: custom.metrics.k8s.io
+  version: v1beta2
+  insecureSkipTLSVerify: true
+  groupPriorityMinimum: 100
+  versionPriority: 200
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: custom-metrics-resource-reader
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  - pods
+  - services
+  verbs:
+  - get
+  - list

pkg/metrics-exposer/main.go

@@ -0,0 +1,96 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"time"
+
+	sainformers "github.com/lterrac/system-autoscaler/pkg/informers"
+	"github.com/lterrac/system-autoscaler/pkg/signals"
+
+	coreinformers "k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/component-base/logs"
+	"k8s.io/klog/v2"
+
+	"github.com/kubernetes-sigs/custom-metrics-apiserver/pkg/apiserver"
+	basecmd "github.com/kubernetes-sigs/custom-metrics-apiserver/pkg/cmd"
+	"github.com/kubernetes-sigs/custom-metrics-apiserver/pkg/provider"
+	generatedopenapi "github.com/lterrac/system-autoscaler/pkg/metrics-exposer/pkg/generated/openapi"
+	rtprovider "github.com/lterrac/system-autoscaler/pkg/metrics-exposer/pkg/provider"
+	openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+)
+
+var (
+	masterURL  string
+	kubeconfig string
+)
+
+// ResponseTimeMetricsAdapter contains a basic adapter used to serve custom metrics
+type ResponseTimeMetricsAdapter struct {
+	basecmd.AdapterBase
+	informers sainformers.Informers
+}
+
+func (a *ResponseTimeMetricsAdapter) makeProviderOrDie(informers sainformers.Informers) provider.CustomMetricsProvider {
+	client, err := a.DynamicClient()
+	if err != nil {
+		klog.Fatalf("unable to construct dynamic client: %v", err)
+	}
+
+	mapper, err := a.RESTMapper()
+	if err != nil {
+		klog.Fatalf("unable to construct discovery REST mapper: %v", err)
+	}
+
+	return rtprovider.NewResponseTimeMetricsProvider(client, mapper, informers)
+}
+
+func main() {
+	logs.InitLogs()
+	defer logs.FlushLogs()
+	stopCh := signals.SetupSignalHandler()
+
+	cmd := &ResponseTimeMetricsAdapter{}
+
+	cmd.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(apiserver.Scheme))
+	cmd.OpenAPIConfig.Info.Title = "response-time-metrics-adapter"
+	cmd.OpenAPIConfig.Info.Version = "0.1.0"
+
+	cmd.Flags().AddGoFlagSet(flag.CommandLine) // make sure we get the klog flags
+	cmd.Flags().Parse(os.Args)
+
+	cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig)
+	if err != nil {
+		klog.Fatalf("Error building kubeconfig: %s", err.Error())
+	}
+
+	kubernetesClient, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		klog.Fatalf("Error building example clientset: %s", err.Error())
+	}
+
+	coreInformerFactory := coreinformers.NewSharedInformerFactory(kubernetesClient, time.Second*30)
+
+	informers := sainformers.Informers{
+		Pod: coreInformerFactory.Core().V1().Pods(),
+	}
+
+	coreInformerFactory.Start(stopCh)
+
+	go informers.Pod.Informer().Run(stopCh)
+
+	if ok := cache.WaitForCacheSync(stopCh, informers.Pod.Informer().HasSynced); !ok {
+		klog.Fatalf("failed to wait for caches to sync")
+	}
+
+	responseTimeMetricsProvider := cmd.makeProviderOrDie(informers)
+	cmd.WithCustomMetrics(responseTimeMetricsProvider)
+
+	if err := cmd.Run(stopCh); err != nil {
+		klog.Fatalf("unable to run custom metrics adapter: %v", err)
+	}
+}