diff --git a/internal/controller/component_customizer.go b/internal/controller/component_customizer.go
index b817aa88c..cad8a8cef 100644
--- a/internal/controller/component_customizer.go
+++ b/internal/controller/component_customizer.go
@@ -29,8 +29,10 @@ import (
 	"k8s.io/client-go/kubernetes/scheme"
 	configv1alpha1 "k8s.io/component-base/config/v1alpha1"
 	"k8s.io/utils/pointer"
-	operatorv1 "sigs.k8s.io/cluster-api-operator/api/v1alpha2"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/util"
+
+	operatorv1 "sigs.k8s.io/cluster-api-operator/api/v1alpha2"
 )
 
 const (
@@ -95,6 +97,11 @@ func customizeObjectsFn(provider operatorv1.GenericProvider) func(objs []unstruc
 
 // customizeDeployment customize provider deployment base on provider spec input.
 func customizeDeployment(pSpec operatorv1.ProviderSpec, d *appsv1.Deployment) error {
+	// Ensure that we customize a manager deployment. It must contain "cluster.x-k8s.io/provider" label.
+	if _, ok := d.GetLabels()[clusterv1.ProviderNameLabel]; !ok {
+		return nil
+	}
+
 	// Customize deployment spec first.
 	if pSpec.Deployment != nil {
 		customizeDeploymentSpec(pSpec, d)
diff --git a/internal/controller/component_customizer_test.go b/internal/controller/component_customizer_test.go
index eaee35010..79567d0df 100644
--- a/internal/controller/component_customizer_test.go
+++ b/internal/controller/component_customizer_test.go
@@ -29,6 +29,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	configv1alpha1 "k8s.io/component-base/config/v1alpha1"
 	"k8s.io/utils/pointer"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 
 	operatorv1 "sigs.k8s.io/cluster-api-operator/api/v1alpha2"
 )
@@ -40,6 +41,9 @@ func TestCustomizeDeployment(t *testing.T) {
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "manager",
 			Namespace: metav1.NamespaceSystem,
+			Labels: map[string]string{
+				clusterv1.ProviderNameLabel: "infra-provider",
+			},
 		},
 		Spec: appsv1.DeploymentSpec{
 			Template: corev1.PodTemplateSpec{
@@ -83,6 +87,7 @@ func TestCustomizeDeployment(t *testing.T) {
 		inputDeploymentSpec    *operatorv1.DeploymentSpec
 		inputManagerSpec       *operatorv1.ManagerSpec
 		expectedDeploymentSpec func(*appsv1.DeploymentSpec) (*appsv1.DeploymentSpec, bool)
+		noDeployementLabels    bool
 	}{
 		{
 			name:                "empty",
@@ -550,6 +555,16 @@ func TestCustomizeDeployment(t *testing.T) {
 				return expectedDS, reflect.DeepEqual(inputDS.Template.Spec.Containers[0], expectedDS.Template.Spec.Containers[0])
 			},
 		},
+		{
+			name: "no provider label on the deployment",
+			inputDeploymentSpec: &operatorv1.DeploymentSpec{
+				NodeSelector: map[string]string{"a": "b"},
+			},
+			expectedDeploymentSpec: func(inputDS *appsv1.DeploymentSpec) (*appsv1.DeploymentSpec, bool) {
+				return &managerDepl.Spec, true
+			},
+			noDeployementLabels: true,
+		},
 	}
 
 	for _, tc := range tests {