add white-list cidrs to internal ingress for prometheus (#300)

* add white-list cidrs to internal ingress for prometheus * Updating Nginx default version to 2.15.0 * Adding backslash to solve the issue when list of cidrs is passed and helm things is a new argument * Fix nit error message Co-authored-by: Gabe Jackson <[email protected]> * Changing errors.Error to errors.New as errors.Error not supported Co-authored-by: Stylianos Rigas <[email protected]> Co-authored-by: Gabe Jackson <[email protected]>
mattermost · Sep 7, 2020 · 6071b29 · 6071b29
1 parent 7016521
commit 6071b29
Show file tree

Hide file tree

Showing 7 changed files with 22 additions and 11 deletions.
diff --git a/README.md b/README.md
@@ -234,7 +234,7 @@ For more information on this change and reasoning for it, check out the [kops re
 This is related to the changes introduced in [PR-263](https://github.com/mattermost/mattermost-cloud/pull/263)
 
 Please follow the steps below for the reprovisioning of existing clusters:
-- Reprovision the cluster by running ```cloud cluster provision --cluster <cluster_id> --nginx-version 2.11.0```.
+- Reprovision the cluster by running ```cloud cluster provision --cluster <cluster_id> --nginx-version 2.15.0```.
 - Check that new nginx deployed both internal and public Load Balancers (nginx-ingress-nginx-controller-internal and nginx-ingress-nginx-controller).
 - Manually update Prometheus Route53 record to use the new private Load Balancer (nginx-ingress-nginx-controller-internal).
 - Manually update cluster installations Route53 records one by one to use the new public Load balancer (nginx-ingress-nginx-controller).

diff --git a/cmd/cloud/server.go b/cmd/cloud/server.go
@@ -46,6 +46,8 @@ func init() {
 	serverCmd.PersistentFlags().Bool("installation-supervisor", true, "Whether this server will run an installation supervisor or not.")
 	serverCmd.PersistentFlags().Bool("cluster-installation-supervisor", true, "Whether this server will run a cluster installation supervisor or not.")
 	serverCmd.PersistentFlags().String("state-store", "dev.cloud.mattermost.com", "The S3 bucket used to store cluster state.")
+	serverCmd.PersistentFlags().StringSlice("allow-list-cidr-range", []string{"0.0.0.0/0"}, "The list of CIDRs to allow communication with the private ingress.")
+
 	serverCmd.PersistentFlags().Int("poll", 30, "The interval in seconds to poll for background work.")
 	serverCmd.PersistentFlags().Int("cluster-resource-threshold", 80, "The percent threshold where new installations won't be scheduled on a multi-tenant cluster.")
 	serverCmd.PersistentFlags().Int("cluster-resource-threshold-scale-value", 0, "The number of worker nodes to scale up by when the threshold is passed. Set to 0 for no scaling. Scaling will never exceed the cluster max worker configuration value.")
@@ -76,6 +78,11 @@ var serverCmd = &cobra.Command{
 			logger.SetFormatter(&logrus.JSONFormatter{})
 		}
 
+		allowListCIDRRange, _ := command.Flags().GetStringSlice("allow-list-cidr-range")
+		if len(allowListCIDRRange) == 0 {
+			return errors.New("allow-list-cidr-range must have at least one value")
+		}
+
 		logger := logger.WithField("instance", instanceID)
 
 		sqlStore, err := sqlStore(command)
@@ -182,6 +189,7 @@ var serverCmd = &cobra.Command{
 			s3StateStore,
 			owner,
 			useExistingResources,
+			allowListCIDRRange,
 			resourceUtil,
 			logger,
 			sqlStore,

diff --git a/helm-charts/nginx_values.yaml b/helm-charts/nginx_values.yaml
@@ -10,15 +10,16 @@ controller:
       service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
       service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp
       service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "443"
-    
+
     internal:
       enabled: true
       annotations:
         service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
         service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp
         service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "443"
         service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0
-
+      externalTrafficPolicy: Local
+
     enableHttp: true
     enableHttps: true
     targetPorts:
@@ -62,7 +63,7 @@ controller:
          return 308 https://$host$request_uri;
       }
 
-  resources: 
+  resources:
    limits:
      cpu: 1000m
      memory: 500Mi

diff --git a/internal/api/request_test.go b/internal/api/request_test.go
@@ -24,7 +24,7 @@ func TestNewCreateClusterRequestFromReader(t *testing.T) {
 			NodeMinCount:           2,
 			NodeMaxCount:           2,
 			Zones:                  []string{"us-east-1a"},
-			DesiredUtilityVersions: map[string]string{"fluentbit": "2.8.7", "nginx": "2.11.0", "prometheus": "10.4.0", "teleport": "0.3.0"},
+			DesiredUtilityVersions: map[string]string{"fluentbit": "2.8.7", "nginx": "2.15.0", "prometheus": "10.4.0", "teleport": "0.3.0"},
 		}
 	}
 
@@ -79,7 +79,7 @@ func TestNewCreateClusterRequestFromReader(t *testing.T) {
 			Zones:              []string{"zone1", "zone2"},
 			DesiredUtilityVersions: map[string]string{
 				"fluentbit":  "2.8.7",
-				"nginx":      "2.11.0",
+				"nginx":      "2.15.0",
 				"prometheus": "10.4.0",
 				"teleport":   "0.3.0"},
 		}, clusterRequest)

diff --git a/internal/provisioner/kops_provisioner_cluster.go b/internal/provisioner/kops_provisioner_cluster.go
@@ -35,6 +35,7 @@ type KopsProvisioner struct {
 	s3StateStore            string
 	privateSubnetIds        string
 	publicSubnetIds         string
+	allowCIDRRangeList      []string
 	owner                   string
 	useExistingAWSResources bool
 	resourceUtil            *utils.ResourceUtil
@@ -44,14 +45,15 @@ type KopsProvisioner struct {
 
 // NewKopsProvisioner creates a new KopsProvisioner.
 // TODO(gsagula): Consider replacing all these paramaters with a struct for readability.
-func NewKopsProvisioner(s3StateStore, owner string, useExistingAWSResources bool,
+func NewKopsProvisioner(s3StateStore, owner string, useExistingAWSResources bool, allowCIDRRangeList []string,
 	resourceUtil *utils.ResourceUtil, logger log.FieldLogger, store model.InstallationDatabaseStoreInterface) *KopsProvisioner {
 
 	logger = logger.WithField("provisioner", "kops")
 
 	return &KopsProvisioner{
 		s3StateStore:            s3StateStore,
 		useExistingAWSResources: useExistingAWSResources,
+		allowCIDRRangeList:      allowCIDRRangeList,
 		logger:                  logger,
 		resourceUtil:            resourceUtil,
 		owner:                   owner,

diff --git a/internal/provisioner/prometheus.go b/internal/provisioner/prometheus.go
@@ -133,14 +133,16 @@ func (p *prometheus) NewHelmDeployment() *helmDeployment {
 	}
 	prometheusDNS := fmt.Sprintf("%s.prometheus.%s", p.cluster.ID, privateDomainName)
 
+	helmValueArguments := fmt.Sprintf("server.ingress.hosts={%s},server.ingress.annotations.nginx\\.ingress\\.kubernetes\\.io/whitelist-source-range=%s", prometheusDNS, strings.Join(p.provisioner.allowCIDRRangeList, "\\,"))
+
 	return &helmDeployment{
 		chartDeploymentName: "prometheus",
 		chartName:           "stable/prometheus",
 		kops:                p.kops,
 		kopsProvisioner:     p.provisioner,
 		logger:              p.logger,
 		namespace:           "prometheus",
-		setArgument:         fmt.Sprintf("server.ingress.hosts={%s}", prometheusDNS),
+		setArgument:         helmValueArguments,
 		valuesPath:          "helm-charts/prometheus_values.yaml",
 		desiredVersion:      p.desiredVersion,
 	}

diff --git a/model/cluster_utility.go b/model/cluster_utility.go
@@ -24,11 +24,9 @@ const (
 	// PrometheusDefaultVersion defines the default version for the Helm chart
 	PrometheusDefaultVersion = "10.4.0"
 	// NginxDefaultVersion defines the default version for the Helm chart
-	NginxDefaultVersion = "2.11.0"
+	NginxDefaultVersion = "2.15.0"
 	// FluentbitDefaultVersion defines the default version for the Helm chart
 	FluentbitDefaultVersion = "2.8.7"
-	// PublicNginxDefaultVersion defines the default version for the Helm chart
-	PublicNginxDefaultVersion = "2.11.0"
 	// TeleportDefaultVersion defines the default version for the Helm chart
 	TeleportDefaultVersion = "0.3.0"
 )