feat(agent): add pod mutation webhook to inject agent (#991)

* wip: pod mutation * More agent config * wip: working without server TLS * wip: working with TLS * Work with agent callback components * Wrap webhook to never deny admission * Add tests for pod mutation webhook, related image * Refactor agent proxy service, add test * Add size limit for emptyDir volume * Resource requests, more cleanup * sample app config makefile recipe * fixup! sample app config makefile recipe * Use slices.Contains * Switch to FNV-1 hash, add service tests --------- Co-authored-by: Andrew Azores <[email protected]> Co-authored-by: Andrew Azores <[email protected]>
cryostatio · Jan 14, 2025 · c891bbe · c891bbe
1 parent e101b6a
commit c891bbe
Show file tree

Hide file tree

Showing 26 changed files with 1,828 additions and 97 deletions.
diff --git a/Makefile b/Makefile
@@ -96,7 +96,11 @@ export STORAGE_IMG ?= $(STORAGE_NAMESPACE)/$(STORAGE_NAME):$(STORAGE_VERSION)
 AGENT_PROXY_NAMESPACE ?= registry.access.redhat.com/ubi8
 AGENT_PROXY_NAME ?= nginx-124
 AGENT_PROXY_VERSION ?= latest
-export AGENT_PROXY_IMG = $(AGENT_PROXY_NAMESPACE)/$(AGENT_PROXY_NAME):$(AGENT_PROXY_VERSION)
+export AGENT_PROXY_IMG ?= $(AGENT_PROXY_NAMESPACE)/$(AGENT_PROXY_NAME):$(AGENT_PROXY_VERSION)
+AGENT_INIT_NAMESPACE ?= $(DEFAULT_NAMESPACE)
+AGENT_INIT_NAME ?= cryostat-agent-init
+AGENT_INIT_VERSION ?= latest
+export AGENT_INIT_IMG ?= $(AGENT_INIT_NAMESPACE)/$(AGENT_INIT_NAME):$(AGENT_INIT_VERSION)
 
 CERT_MANAGER_VERSION ?= 1.12.14
 CERT_MANAGER_MANIFEST ?= \
@@ -396,25 +400,13 @@ ifneq ($(origin SAMPLE_APP_NAMESPACE), undefined)
 SAMPLE_APP_FLAGS += -n $(SAMPLE_APP_NAMESPACE)
 endif
 
-.PHONY: sample_app
-sample_app: undeploy_sample_app ## Deploy sample app.
-	$(CLUSTER_CLIENT) apply $(SAMPLE_APP_FLAGS) -f config/samples/sample-app.yaml
-
 .PHONY: undeploy_sample_app
 undeploy_sample_app: ## Undeploy sample app.
 	- $(CLUSTER_CLIENT) delete $(SAMPLE_APP_FLAGS) --ignore-not-found=$(ignore-not-found) -f config/samples/sample-app.yaml
 
-.PHONY: sample_app_agent
-sample_app_agent: undeploy_sample_app_agent ## Deploy sample app with Cryostat Agent.
-	@if [ -z "${AUTH_TOKEN}" ]; then \
-		if [ "${CLUSTER_CLIENT}" = "oc" ]; then\
-			AUTH_TOKEN=`oc whoami -t`; \
-		else \
-			echo "'AUTH_TOKEN' must be specified."; \
-			exit 1; \
-		fi; \
-	fi; \
-	$(CLUSTER_CLIENT) apply $(SAMPLE_APP_FLAGS) -f config/samples/sample-app-agent.yaml; \
+.PHONY: sample_app
+sample_app: undeploy_sample_app ## Deploy sample app.
+	$(CLUSTER_CLIENT) apply $(SAMPLE_APP_FLAGS) -f config/samples/sample-app.yaml
 
 .PHONY: undeploy_sample_app_agent_proxy
 undeploy_sample_app_agent_proxy: ## Undeploy sample app with Cryostat Agent configured for TLS client auth on nginx proxy.
@@ -437,6 +429,19 @@ sample_app_agent_proxy: undeploy_sample_app_agent_proxy ## Deploy sample app wit
 undeploy_sample_app_agent: ## Undeploy sample app with Cryostat Agent.
 	- $(CLUSTER_CLIENT) delete $(SAMPLE_APP_FLAGS) --ignore-not-found=$(ignore-not-found) -f config/samples/sample-app-agent.yaml
 
+.PHONY: sample_app_agent
+sample_app_agent: undeploy_sample_app_agent ## Deploy sample app with Cryostat Agent.
+	$(CLUSTER_CLIENT) apply $(SAMPLE_APP_FLAGS) -f config/samples/sample-app-agent.yaml
+
+.PHONY: undeploy_sample_app_agent_injected
+undeploy_sample_app_agent_injected: ## Undeploy sample app with Cryostat Agent deployed by Operator injection.
+	- $(CLUSTER_CLIENT) delete $(SAMPLE_APP_FLAGS) --ignore-not-found=$(ignore-not-found) -f config/samples/sample-app-agent-injected.yaml
+
+.PHONY: sample_app_agent_injected
+sample_app_agent_injected: undeploy_sample_app_agent_injected ## Deploy sample app with Cryostat Agent deployed by Operator injection.
+	$(CLUSTER_CLIENT) apply $(SAMPLE_APP_FLAGS) -f config/samples/sample-app-agent-injected.yaml
+	$(CLUSTER_CLIENT) patch --type=merge -p "{\"spec\":{\"template\":{\"metadata\":{\"labels\":{\"cryostat.io/namespace\":\"${DEPLOY_NAMESPACE}\"}}}}}" deployment/quarkus-cryostat-agent
+
 .PHONY: cert_manager
 cert_manager: remove_cert_manager ## Install cert manager.
 	$(CLUSTER_CLIENT) create --validate=false -f $(CERT_MANAGER_MANIFEST)

diff --git a/bundle/manifests/cryostat-operator.clusterserviceversion.yaml b/bundle/manifests/cryostat-operator.clusterserviceversion.yaml
@@ -30,7 +30,7 @@ metadata:
     capabilities: Seamless Upgrades
     categories: Monitoring, Developer Tools
     containerImage: quay.io/cryostat/cryostat-operator:4.0.0-dev
-    createdAt: "2025-01-07T19:12:00Z"
+    createdAt: "2025-01-13T22:30:07Z"
     description: JVM monitoring and profiling tool
     operatorframework.io/initialization-resource: |-
       {
@@ -1220,6 +1220,25 @@ spec:
       targetPort: 9443
       type: MutatingAdmissionWebhook
       webhookPath: /mutate-operator-cryostat-io-v1beta2-cryostat
+    - admissionReviewVersions:
+        - v1
+      containerPort: 443
+      deploymentName: cryostat-operator-controller
+      failurePolicy: Ignore
+      generateName: mpod.cryostat.io
+      rules:
+        - apiGroups:
+            - ""
+          apiVersions:
+            - v1
+          operations:
+            - CREATE
+          resources:
+            - pods
+      sideEffects: None
+      targetPort: 9443
+      type: MutatingAdmissionWebhook
+      webhookPath: /mutate--v1-pod
     - admissionReviewVersions:
         - v1
       containerPort: 443

diff --git a/config/samples/sample-app-agent-injected.yaml b/config/samples/sample-app-agent-injected.yaml
@@ -0,0 +1,93 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: quarkus-cryostat-agent
+  name: quarkus-cryostat-agent
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: quarkus-cryostat-agent
+  template:
+    metadata:
+      labels:
+        app: quarkus-cryostat-agent
+        cryostat.io/name: cryostat-sample
+        cryostat.io/namespace: cryostat-operator-system
+    spec:
+      serviceAccountName: quarkus-cryostat-agent-serviceaccount
+      containers:
+      - env:
+        - name: JAVA_OPTS_APPEND
+          value: |-
+            -Dquarkus.http.host=0.0.0.0
+            -Djava.util.logging.manager=org.jboss.logmanager.LogManager
+            -Dio.cryostat.agent.shaded.org.slf4j.simpleLogger.defaultLogLevel=debug
+        image: quay.io/redhat-java-monitoring/quarkus-cryostat-agent:latest
+        imagePullPolicy: Always
+        name: quarkus-cryostat-agent
+        ports:
+        - containerPort: 10010
+          protocol: TCP
+        resources:
+          limits:
+            cpu: 500m
+            memory: 256Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+      restartPolicy: Always
+      securityContext:
+        runAsNonRoot: true
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: quarkus-cryostat-agent
+  name: quarkus-cryostat-agent
+spec:
+  selector:
+    app: quarkus-cryostat-agent
+  ports:
+  - name: agent-http
+    port: 9977
+    protocol: TCP
+    targetPort: 9977
+  - name: app-http
+    port: 10010
+    protocol: TCP
+    targetPort: 10010
+---
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: quarkus-cryostat-agent-serviceaccount
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: quarkus-cryostat-agent-role
+rules:
+- apiGroups:
+  - ""
+  verbs:
+  - create
+  resources:
+  - pods/exec
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: quarkus-cryostat-agent-role-binding
+subjects:
+- kind: ServiceAccount
+  name: quarkus-cryostat-agent-serviceaccount
+roleRef:
+  kind: Role
+  name: quarkus-cryostat-agent-role
+  apiGroup: rbac.authorization.k8s.io
diff --git a/config/samples/sample-app-agent.yaml b/config/samples/sample-app-agent.yaml
@@ -60,12 +60,9 @@ spec:
         - containerPort: 9097
           protocol: TCP
         resources:
-          requests:
-            cpu: 200m
-            memory: 96Mi
           limits:
             cpu: 500m
-            memory: 192Mi
+            memory: 256Mi
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:

diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml
@@ -24,6 +24,25 @@ webhooks:
     resources:
     - cryostats
   sideEffects: None
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate--v1-pod
+  failurePolicy: Ignore
+  name: mpod.cryostat.io
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    resources:
+    - pods
+  sideEffects: None
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration

diff --git a/internal/controllers/common/common_utils.go b/internal/controllers/common/common_utils.go
@@ -17,9 +17,11 @@ package common
 import (
 	"crypto/sha256"
 	"fmt"
+	"hash/fnv"
 	"io/ioutil"
 	"math/rand"
 	"os"
+	"regexp"
 	"strings"
 	"time"
 
@@ -36,6 +38,7 @@ var log = logf.Log.WithName("common")
 // OSUtils is an abstraction on functionality that interacts with the operating system
 type OSUtils interface {
 	GetEnv(name string) string
+	GetEnvOrDefault(name string, defaultVal string) string
 	GetFileContents(path string) ([]byte, error)
 	GenPasswd(length int) string
 }
@@ -48,6 +51,16 @@ func (o *DefaultOSUtils) GetEnv(name string) string {
 	return os.Getenv(name)
 }
 
+// GetEnvOrDefault returns the value of the environment variable with the provided name.
+// If no such variable exists, the provided default value is returned.
+func (o *DefaultOSUtils) GetEnvOrDefault(name string, defaultVal string) string {
+	val := o.GetEnv(name)
+	if len(val) > 0 {
+		return val
+	}
+	return defaultVal
+}
+
 // GetFileContents reads and returns the entire file contents specified by the path
 func (o *DefaultOSUtils) GetFileContents(path string) ([]byte, error) {
 	return ioutil.ReadFile(path)
@@ -77,21 +90,44 @@ func ClusterUniqueNameWithPrefix(gvk *schema.GroupVersionKind, prefix string, na
 	return ClusterUniqueNameWithPrefixTargetNS(gvk, prefix, name, namespace, "")
 }
 
+// ClusterUniqueShortName returns a name for cluster-scoped objects that is
+// uniquely identified by a namespace and name. Appends the prefix to the
+// provided Kind. The total length should be at most 63 characters.
+func ClusterUniqueShortNameWithPrefix(gvk *schema.GroupVersionKind, prefix string, name string, namespace string) string {
+	return clusterUniqueName(gvk, prefix, name, namespace, "", true)
+}
+
 // ClusterUniqueNameWithPrefixTargetNS returns a name for cluster-scoped objects that is
 // uniquely identified by a namespace and name, and a target namespace.
 // Appends the prefix to the provided Kind.
 func ClusterUniqueNameWithPrefixTargetNS(gvk *schema.GroupVersionKind, prefix string, name string, namespace string,
 	targetNS string) string {
+	return clusterUniqueName(gvk, prefix, name, namespace, targetNS, false)
+}
+
+func clusterUniqueName(gvk *schema.GroupVersionKind, prefix string, name string, namespace string, targetNS string,
+	short bool) string {
 	prefixWithKind := strings.ToLower(gvk.Kind)
 	if len(prefix) > 0 {
 		prefixWithKind += "-" + prefix
 	}
+
 	toHash := namespace + "/" + name
 	if len(targetNS) > 0 {
 		toHash += "/" + targetNS
 	}
-	// Use the SHA256 checksum of the namespaced name as a suffix
-	suffix := fmt.Sprintf("%x", sha256.Sum256([]byte(toHash)))
+
+	var suffix string
+	if short {
+		// Use the 128-bit FNV-1 checksum of the namespaced name as a suffix.
+		// Suffix is 32 bytes
+		hash := fnv.New128()
+		hash.Write([]byte(toHash))
+		suffix = fmt.Sprintf("%x", hash.Sum([]byte{}))
+	} else {
+		// Use the SHA256 checksum of the namespaced name as a suffix
+		suffix = fmt.Sprintf("%x", sha256.Sum256([]byte(toHash)))
+	}
 	return prefixWithKind + "-" + suffix
 }
 
@@ -125,6 +161,19 @@ func LabelsForTargetNamespaceObject(cr *model.CryostatInstance) map[string]strin
 	}
 }
 
+// Matches image tags of the form "major.minor.patch"
+var develVerRegexp = regexp.MustCompile(`(?i)(:latest|SNAPSHOT|dev|BETA\d+)$`)
+
+// GetPullPolicy returns an image pull policy based on the image tag provided.
+func GetPullPolicy(imageTag string) corev1.PullPolicy {
+	// Use Always for tags that have a known development suffix
+	if develVerRegexp.MatchString(imageTag) {
+		return corev1.PullAlways
+	}
+	// Likely a release, use IfNotPresent
+	return corev1.PullIfNotPresent
+}
+
 // SeccompProfile returns a SeccompProfile for the restricted
 // Pod Security Standard that, on OpenShift, is backwards-compatible
 // with OpenShift < 4.11.

diff --git a/internal/controllers/common/naming.go b/internal/controllers/common/naming.go
@@ -0,0 +1,32 @@
+// Copyright The Cryostat Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"github.com/cryostatio/cryostat-operator/internal/controllers/model"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+func AgentHeadlessServiceName(gvk *schema.GroupVersionKind, cr *model.CryostatInstance) string {
+	return ClusterUniqueShortNameWithPrefix(gvk, "agent", cr.Name, cr.InstallNamespace)
+}
+
+func AgentProxyServiceName(cr *model.CryostatInstance) string {
+	return cr.Name + "-agent"
+}
+
+func AgentCertificateName(gvk *schema.GroupVersionKind, cr *model.CryostatInstance, targetNamespace string) string {
+	return ClusterUniqueNameWithPrefixTargetNS(gvk, "agent", cr.Name, cr.InstallNamespace, targetNamespace)
+}