diff --git a/.ci/pipeline_definitions b/.ci/pipeline_definitions
index 57a7c9393719..246b2711133c 100644
--- a/.ci/pipeline_definitions
+++ b/.ci/pipeline_definitions
@@ -103,38 +103,17 @@ vertical-pod-autoscaler:
         - linux/arm64
         dockerimages:
           vpa-recommender:
-            inputs:
-              repos:
-                source: ~ # default
-              steps:
-                build: ~
             image: europe-docker.pkg.dev/gardener-project/snapshots/gardener/autoscaler/vertical-pod-autoscaler/vpa-recommender
-            dockerfile: 'Dockerfile.recommender'
-            dir: 'vertical-pod-autoscaler'
+            dockerfile: 'Dockerfile'
+            dir: 'vertical-pod-autoscaler/pkg/recommender'
           vpa-updater:
-            inputs:
-              repos:
-                source: ~ # default
-              steps:
-                build: ~
             image: europe-docker.pkg.dev/gardener-project/snapshots/gardener/autoscaler/vertical-pod-autoscaler/vpa-updater
-            dockerfile: 'Dockerfile.updater'
-            dir: 'vertical-pod-autoscaler'
+            dockerfile: 'Dockerfile'
+            dir: 'vertical-pod-autoscaler/pkg/updater'
           vpa-admission-controller:
-            inputs:
-              repos:
-                source: ~ # default
-              steps:
-                build: ~
             image: europe-docker.pkg.dev/gardener-project/snapshots/gardener/autoscaler/vertical-pod-autoscaler/vpa-admission-controller
-            dockerfile: 'Dockerfile.admissioncontroller'
-            dir: 'vertical-pod-autoscaler'
-    steps:
-      test:
-        image: 'golang:1.20.5'
-      build:
-        image: 'golang:1.20.5'
-        output_dir: 'binary'
+            dockerfile: 'Dockerfile'
+            dir: 'vertical-pod-autoscaler/pkg/admission-controller'
   jobs:
     release:
       traits:
diff --git a/vertical-pod-autoscaler/Dockerfile.admissioncontroller b/vertical-pod-autoscaler/Dockerfile.admissioncontroller
deleted file mode 100644
index f953f7cb76ad..000000000000
--- a/vertical-pod-autoscaler/Dockerfile.admissioncontroller
+++ /dev/null
@@ -1,34 +0,0 @@
-# Copyright 2017 The Kubernetes Authors. All rights reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-FROM --platform=$BUILDPLATFORM golang:1.20.5 as builder
-
-ENV GOPATH /gopath/
-ENV PATH $GOPATH/bin:$PATH
-
-COPY . /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler
-WORKDIR /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler
-
-ARG TARGETOS TARGETARCH
-
-RUN CGO_ENABLED=0 GOARCH=$TARGETARCH GOOS=$TARGETOS go build -C pkg/admission-controller -mod vendor -o admission-controller-$TARGETARCH
-
-FROM gcr.io/distroless/static:latest
-MAINTAINER Krzysztof Grygiel "kgrygiel@google.com"
-
-ARG TARGETARCH
-
-COPY --from=builder /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/admission-controller-$TARGETARCH /admission-controller
-
-ENTRYPOINT ["/admission-controller"]
-CMD ["--v=4", "--stderrthreshold=info"]
diff --git a/vertical-pod-autoscaler/Dockerfile.recommender b/vertical-pod-autoscaler/Dockerfile.recommender
deleted file mode 100644
index 19b0ee4d0413..000000000000
--- a/vertical-pod-autoscaler/Dockerfile.recommender
+++ /dev/null
@@ -1,34 +0,0 @@
-# Copyright 2017 The Kubernetes Authors. All rights reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-FROM --platform=$BUILDPLATFORM golang:1.20.5 as builder
-
-ENV GOPATH /gopath/
-ENV PATH $GOPATH/bin:$PATH
-
-COPY . /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler
-WORKDIR /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler
-
-ARG TARGETOS TARGETARCH
-
-RUN CGO_ENABLED=0 GOARCH=$TARGETARCH GOOS=$TARGETOS go build -C pkg/recommender -mod vendor -o recommender-$TARGETARCH
-
-FROM gcr.io/distroless/static:latest
-MAINTAINER Krzysztof Grygiel "kgrygiel@google.com"
-
-ARG TARGETARCH
-
-COPY --from=builder /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/recommender-$TARGETARCH /recommender
-
-ENTRYPOINT ["/recommender"]
-CMD ["--v=4", "--stderrthreshold=info", "--prometheus-address=http://prometheus.monitoring.svc"]
diff --git a/vertical-pod-autoscaler/Dockerfile.updater b/vertical-pod-autoscaler/Dockerfile.updater
deleted file mode 100644
index 8bc4d42b913f..000000000000
--- a/vertical-pod-autoscaler/Dockerfile.updater
+++ /dev/null
@@ -1,34 +0,0 @@
-# Copyright 2017 The Kubernetes Authors. All rights reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-FROM --platform=$BUILDPLATFORM golang:1.20.5 as builder
-
-ENV GOPATH /gopath/
-ENV PATH $GOPATH/bin:$PATH
-
-COPY . /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler
-WORKDIR /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler
-
-ARG TARGETOS TARGETARCH
-
-RUN CGO_ENABLED=0 GOARCH=$TARGETARCH GOOS=$TARGETOS go build -C pkg/updater -mod vendor -o updater-$TARGETARCH
-
-FROM gcr.io/distroless/static:latest
-MAINTAINER Krzysztof Grygiel "kgrygiel@google.com"
-
-ARG TARGETARCH
-
-COPY --from=builder /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler/pkg/updater/updater-$TARGETARCH /updater
-
-ENTRYPOINT ["/updater"]
-CMD ["--v=4", "--stderrthreshold=info"]