From 76ff2c9a997c196c7e1bbc7d193c9de2e060fbd0 Mon Sep 17 00:00:00 2001 From: Brandon Bjelland Date: Wed, 9 Aug 2023 22:19:21 -0700 Subject: [PATCH] everything is working consistently. shipping it --- install/kubernetes/aws/eks-cluster.yaml.tpl | 8 ++--- .../aws/karpenter-provisioner.yaml.tpl | 2 +- install/scripts/aws-down.sh | 4 +++ install/scripts/aws-up.sh | 29 +++++++++++++++++-- 4 files changed, 33 insertions(+), 10 deletions(-) diff --git a/install/kubernetes/aws/eks-cluster.yaml.tpl b/install/kubernetes/aws/eks-cluster.yaml.tpl index 9c5de2ec..f9bee6e5 100644 --- a/install/kubernetes/aws/eks-cluster.yaml.tpl +++ b/install/kubernetes/aws/eks-cluster.yaml.tpl @@ -10,10 +10,6 @@ metadata: environment: dev karpenter.sh/discovery: ${CLUSTER_NAME} -karpenter: - withSpotInterruptionQueue: true - version: "v0.29.0" - managedNodeGroups: - name: builder-ng privateNetworking: true @@ -41,7 +37,7 @@ addons: - name: coredns iamIdentityMappings: - - arn: "arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:role/KarpenterNodeRole-${CLUSTER_NAME}" + - arn: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/KarpenterNodeRole-${CLUSTER_NAME}" username: system:node:{{EC2PrivateDNSName}} groups: - system:bootstrappers @@ -56,7 +52,7 @@ iam: roleName: ${CLUSTER_NAME}-karpenter attachPolicyARNs: # this is used as spec.instanceProfile in the karpenter AWSNodeTemplate - - arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:policy/KarpenterControllerPolicy-${CLUSTER_NAME} + - arn:aws:iam::${AWS_ACCOUNT_ID}:policy/KarpenterControllerPolicy-${CLUSTER_NAME} roleOnly: true - metadata: name: ebs-csi-controller-sa diff --git a/install/kubernetes/aws/karpenter-provisioner.yaml.tpl b/install/kubernetes/aws/karpenter-provisioner.yaml.tpl index 608cef04..951e0aca 100644 --- a/install/kubernetes/aws/karpenter-provisioner.yaml.tpl +++ b/install/kubernetes/aws/karpenter-provisioner.yaml.tpl @@ -3,7 +3,7 @@ kind: AWSNodeTemplate metadata: name: default spec: - instanceProfile: eksctl-KarpenterNodeInstanceProfile-${CLUSTER_NAME} + instanceProfile: KarpenterNodeInstanceProfile-${CLUSTER_NAME} subnetSelector: karpenter.sh/discovery: ${CLUSTER_NAME} securityGroupSelector: diff --git a/install/scripts/aws-down.sh b/install/scripts/aws-down.sh index 2d31a3f5..53ca7934 100755 --- a/install/scripts/aws-down.sh +++ b/install/scripts/aws-down.sh @@ -25,6 +25,10 @@ export ARTIFACTS_BUCKET_NAME=${AWS_ACCOUNT_ID}-${CLUSTER_NAME}-artifacts envsubst <${kubernetes_dir}/aws/eks-cluster.yaml.tpl >${kubernetes_dir}/aws/eks-cluster.yaml eksctl delete cluster -f ${kubernetes_dir}/aws/eks-cluster.yaml || true +aws cloudformation delete-stack \ + --stack-name "Karpenter-${CLUSTER_NAME}" \ + --region ${REGION} || true + aws ecr delete-repository \ --repository-name ${ARTIFACTS_REPO_NAME} \ --region ${REGION} >/dev/null || true diff --git a/install/scripts/aws-up.sh b/install/scripts/aws-up.sh index c41d5f46..0fee3e8a 100755 --- a/install/scripts/aws-up.sh +++ b/install/scripts/aws-up.sh @@ -11,10 +11,12 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" kubernetes_dir=${script_dir}/../kubernetes EKSCTL_ENABLE_CREDENTIAL_CACHE=1 +karpenter_version=v0.29.2 export CLUSTER_NAME=substratus export REGION=us-west-2 export ARTIFACTS_REPO_NAME=${CLUSTER_NAME} export ARTIFACTS_BUCKET_NAME=${AWS_ACCOUNT_ID}-${CLUSTER_NAME}-artifacts +karpenter_iam_role_arn="arn:aws:iam::${AWS_ACCOUNT_ID}:role/${CLUSTER_NAME}-karpenter" tempout=$(mktemp) aws s3 mb s3://${ARTIFACTS_BUCKET_NAME} \ @@ -24,6 +26,14 @@ aws ecr create-repository \ --repository-name ${ARTIFACTS_REPO_NAME} \ --region ${REGION} >/dev/null || true +curl -fsSL https://raw.githubusercontent.com/aws/karpenter/"${karpenter_version}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml >$tempout +aws cloudformation deploy \ + --stack-name "Karpenter-${CLUSTER_NAME}" \ + --template-file "${tempout}" \ + --capabilities CAPABILITY_NAMED_IAM \ + --region ${REGION} \ + --parameter-overrides "ClusterName=${CLUSTER_NAME}" + envsubst <${kubernetes_dir}/aws/eks-cluster.yaml.tpl >${kubernetes_dir}/aws/eks-cluster.yaml eksctl create cluster -f ${kubernetes_dir}/aws/eks-cluster.yaml || eksctl upgrade cluster -f ${kubernetes_dir}/aws/eks-cluster.yaml @@ -31,9 +41,22 @@ eksctl create cluster -f ${kubernetes_dir}/aws/eks-cluster.yaml || aws iam create-service-linked-role \ --aws-service-name spot.amazonaws.com || true -aws eks update-kubeconfig \ - --region ${REGION} \ - --name ${CLUSTER_NAME} +# Logout of helm registry to perform an unauthenticated pull against the public ECR +helm registry logout public.ecr.aws || true +helm upgrade \ + --create-namespace \ + --install karpenter oci://public.ecr.aws/karpenter/karpenter \ + --version ${karpenter_version} \ + --namespace karpenter \ + --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"=${karpenter_iam_role_arn} \ + --set settings.aws.clusterName=${CLUSTER_NAME} \ + --set settings.aws.defaultInstanceProfile=KarpenterNodeInstanceProfile-${CLUSTER_NAME} \ + --set settings.aws.interruptionQueueName=${CLUSTER_NAME} \ + --set controller.resources.requests.cpu=1 \ + --set controller.resources.requests.memory=1Gi \ + --set controller.resources.limits.cpu=1 \ + --set controller.resources.limits.memory=1Gi \ + --wait envsubst <${kubernetes_dir}/aws/karpenter-provisioner.yaml.tpl >${kubernetes_dir}/aws/karpenter-provisioner.yaml kubectl apply -f ${kubernetes_dir}/aws/karpenter-provisioner.yaml