From d8e3d7337dae469b8e70f560de2c6e0be41195fa Mon Sep 17 00:00:00 2001
From: YuviPanda <yuvipanda@gmail.com>
Date: Thu, 19 Aug 2021 22:02:07 +0530
Subject: [PATCH] Grant S3 access to notebook & dask pods

eksctl
[supports](https://eksctl.io/usage/iamserviceaccounts/#usage-with-config-files)
creating kubernetes service acocunts bound with
[IRSA](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html).
We create one with S3 access, and bind it to our notebook and dask
pods. This should give them full s3 access.

Remove separate eksctl cluster jsonnet object, since it was
not doing anything useful.

Stolen from https://github.com/2i2c-org/pilot-hubs/pull/436

Fixes https://github.com/2i2c-org/pilot-hubs/issues/492
---
 config/hubs/carbonplan.cluster.yaml |  8 ++++++++
 eksctl/carbonplan.jsonnet           | 26 +++++++++++++++++++++++---
 eksctl/libsonnet/cluster.jsonnet    | 19 -------------------
 3 files changed, 31 insertions(+), 22 deletions(-)
 delete mode 100644 eksctl/libsonnet/cluster.jsonnet

diff --git a/config/hubs/carbonplan.cluster.yaml b/config/hubs/carbonplan.cluster.yaml
index b409a6799..ca84d5965 100644
--- a/config/hubs/carbonplan.cluster.yaml
+++ b/config/hubs/carbonplan.cluster.yaml
@@ -64,6 +64,7 @@ hubs:
                   name: Carbon Plan
                   url: https://carbonplan.org
           singleuser:
+            serviceAccountName: cloud-user-sa
             initContainers:
               # Need to explicitly fix ownership here, since EFS doesn't do anonuid
             - name: volume-mount-ownership-fix
@@ -168,6 +169,13 @@ hubs:
                 admin_users: *users
 
       dask-gateway:
+        backend:
+          scheduler:
+            extraPodConfig:
+              serviceAccountName: cloud-user-sa
+          worker:
+            extraPodConfig:
+              serviceAccountName: cloud-user-sa
         traefik:
           resources:
             requests:
diff --git a/eksctl/carbonplan.jsonnet b/eksctl/carbonplan.jsonnet
index a7c7c0243..cc5976662 100644
--- a/eksctl/carbonplan.jsonnet
+++ b/eksctl/carbonplan.jsonnet
@@ -1,5 +1,4 @@
 // Exports an eksctl config file for carbonplan cluster
-local cluster = import "./libsonnet/cluster.jsonnet";
 local ng = import "./libsonnet/nodegroup.jsonnet";
 
 // place all cluster nodes here
@@ -7,6 +6,11 @@ local clusterRegion = "us-west-2";
 local masterAzs = ["us-west-2a", "us-west-2b", "us-west-2c"];
 local nodeAz = "us-west-2a";
 
+// List of namespaces where we have hubs deployed
+// Each will get a ServiceAccount that will get credentials to talk
+// to AWS services, via https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html
+local namespaces = ['staging', 'prod'];
+
 // Node definitions for notebook nodes. Config here is merged
 // with our notebook node definition.
 // A `node.kubernetes.io/instance-type label is added, so pods
@@ -33,12 +37,28 @@ local daskNodes = [
     { instancesDistribution+: { instanceTypes: ["r5.8xlarge"] }},
 ];
 
-cluster {
+{
+    apiVersion: 'eksctl.io/v1alpha5',
+    kind: 'ClusterConfig',
     metadata+: {
         name: "carbonplanhub",
-        region: clusterRegion
+        region: clusterRegion,
+        version: '1.19'
     },
     availabilityZones: masterAzs,
+    iam: {
+        withOIDC: true,
+
+        serviceAccounts: [{
+            metadata: {
+                name: "cloud-user-sa",
+                namespace: namespace
+            },
+            attachPolicyARNs:[
+                "arn:aws:iam::aws:policy/AmazonS3FullAccess"
+            ],
+        } for namespace in namespaces],
+    },
     nodeGroups: [
         ng {
             name: 'core-a',
diff --git a/eksctl/libsonnet/cluster.jsonnet b/eksctl/libsonnet/cluster.jsonnet
deleted file mode 100644
index 7f2ad2e20..000000000
--- a/eksctl/libsonnet/cluster.jsonnet
+++ /dev/null
@@ -1,19 +0,0 @@
-// Exports a customizable eksctl cluster object
-// https://eksctl.io/usage/schema/ lists the config
-//
-// The default configuration is pretty bare, and only
-// sets the default k8s version. Everything else must
-// be merged in by the jsonnet file for each cluster
-{
-  apiVersion: 'eksctl.io/v1alpha5',
-  kind: 'ClusterConfig',
-  metadata: {
-    name: '',
-    region: '',
-    version: '1.19',
-  },
-  availabilityZones: [],
-  iam: {
-    withOIDC: true,
-  },
-}