Skip to content

Commit

Permalink
fix: update IAM policy for karpenter (#308)
Browse files Browse the repository at this point in the history 
* scope down IAM policy for karpenter within SparkEmrContainerRuntime
  • Loading branch information
@lmouhib
lmouhib authored Dec 20, 2023
1 parent 2e7e60c commit ee444b8
Show file tree
Hide file tree
Showing 4 changed files with 88 additions and 57 deletions.
121 changes: 85 additions & 36 deletions framework/src/processing/lib/spark-runtime/emr-containers/eks-karpenter-helpers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { SubnetType, ISubnet, SecurityGroup, Port, ISecurityGroup } from 'aws-cd
import { HelmChart, ICluster } from 'aws-cdk-lib/aws-eks';
import { IRule, Rule } from 'aws-cdk-lib/aws-events';
import { SqsQueue } from 'aws-cdk-lib/aws-events-targets';
import { CfnInstanceProfile, IRole, PolicyStatement, Effect, ServicePrincipal } from 'aws-cdk-lib/aws-iam';
import { IRole, PolicyStatement, Effect, ServicePrincipal } from 'aws-cdk-lib/aws-iam';
import { IQueue, Queue } from 'aws-cdk-lib/aws-sqs';
import { Construct } from 'constructs';
import { SparkEmrContainersRuntime } from './spark-emr-containers-runtime';
Expand Down Expand Up @@ -84,7 +84,6 @@ export function karpenterManifestSetup(cluster: ICluster, path: string, subnet:
export function karpenterSetup(cluster: ICluster,
clusterName: string,
scope: Construct,
instanceProfile: CfnInstanceProfile,
nodeRole: IRole,
karpenterRemovalPolicy: RemovalPolicy,
karpenterVersion?: KarpenterVersion,
Expand Down Expand Up @@ -122,13 +121,19 @@ export function karpenterSetup(cluster: ICluster,
targets: [new SqsQueue(karpenterInterruptionQueue)],
});

const karpenterControllerPolicyStatementSSM: PolicyStatement = new PolicyStatement({
const allowSSMReadActions: PolicyStatement = new PolicyStatement({
effect: Effect.ALLOW,
actions: ['ssm:GetParameter', 'pricing:GetProducts'],
resources: ['*'],
});

const karpenterControllerPolicyStatementEC2: PolicyStatement = new PolicyStatement({
const allowPricingReadActions: PolicyStatement = new PolicyStatement({
effect: Effect.ALLOW,
actions: ['pricing:GetProducts'],
resources: ['*'],
});

const allowRegionalReadActions: PolicyStatement = new PolicyStatement({
effect: Effect.ALLOW,
actions: [
'ec2:DescribeAvailabilityZones',
Expand Down Expand Up @@ -162,35 +167,22 @@ export function karpenterSetup(cluster: ICluster,
actions: ['ec2:RunInstances', 'ec2:CreateFleet'],
});

const allowScopedEC2LaunchTemplateActions: PolicyStatement = new PolicyStatement({
effect: Effect.ALLOW,
resources: [`arn:aws:ec2:${Stack.of(scope).region}:*:launch-template/*`],
actions: ['ec2:CreateLaunchTemplate'],
conditions: {
StringEquals: {
[`aws:RequestTag/kubernetes.io/cluster/${clusterName}`]: 'owned',
},
StringLike: {
'aws:RequestTag/karpenter.sh/provisioner-name': '*',
},
},
});

const allowScopedEC2InstanceActionsWithTags: PolicyStatement = new PolicyStatement({
effect: Effect.ALLOW,
resources: [
`arn:aws:ec2:${Stack.of(scope).region}:*:fleet/*`,
`arn:aws:ec2:${Stack.of(scope).region}:*:instance/*`,
`arn:aws:ec2:${Stack.of(scope).region}:*:volume/*`,
`arn:aws:ec2:${Stack.of(scope).region}:*:network-interface/*`,
`arn:aws:ec2:${Stack.of(scope).region}:*:launch-template/*`,
],
actions: ['ec2:RunInstances', 'ec2:CreateFleet'],
actions: ['ec2:RunInstances', 'ec2:CreateFleet', 'ec2:CreateLaunchTemplate'],
conditions: {
StringEquals: {
[`aws:RequestTag/kubernetes.io/cluster/${clusterName}`]: 'owned',
},
StringLike: {
'aws:RequestTag/karpenter.sh/provisioner-name': '*',
'aws:RequestTag/karpenter.sh/nodepool': '*',
},
},
});
Expand All @@ -212,26 +204,30 @@ export function karpenterSetup(cluster: ICluster,
'ec2:CreateAction': ['RunInstances', 'CreateFleet', 'CreateLaunchTemplate'],
},
StringLike: {
'aws:RequestTag/karpenter.sh/provisioner-name': '*',
'aws:RequestTag/karpenter.sh/nodepool': '*',
},
},
});

const allowMachineMigrationTagging: PolicyStatement = new PolicyStatement({
sid: 'AllowMachineMigrationTagging',
const allowScopedResourceTagging: PolicyStatement = new PolicyStatement({
sid: 'allowScopedResourceTagging',
effect: Effect.ALLOW,
resources: [`arn:aws:ec2:${Stack.of(scope).region}:*:instance/*`],
resources: [
`arn:aws:ec2:${Stack.of(scope).region}:*:instance/*`,
],
actions: ['ec2:CreateTags'],
conditions: {
'StringEquals': {
[`aws:ResourceTag/kubernetes.io/cluster/${clusterName}`]: 'owned',
'aws:RequestTag/karpenter.sh/managed-by': `${clusterName}`,
},
'StringLike': {
'aws:RequestTag/karpenter.sh/provisioner-name': '*',
'aws:ResourceTag/karpenter.sh/nodepool': '*',
},
'ForAllValues:StringEquals': {
'aws:TagKeys': ['karpenter.sh/provisioner-name', 'karpenter.sh/managed-by'],
'aws:TagKeys': [
'karpenter.sh/nodeclaim',
'Name',
],
},
},
});
Expand All @@ -249,12 +245,12 @@ export function karpenterSetup(cluster: ICluster,
[`aws:ResourceTag/kubernetes.io/cluster/${clusterName}`]: 'owned',
},
StringLike: {
'aws:ResourceTag/karpenter.sh/provisioner-name': '*',
'aws:ResourceTag/karpenter.sh/nodepool': '*',
},
},
});

const karpenterControllerPolicyStatementIAM: PolicyStatement = new PolicyStatement({
const allowPassingInstanceRole: PolicyStatement = new PolicyStatement({
effect: Effect.ALLOW,
actions: ['iam:PassRole'],
resources: [`${nodeRole.roleArn}`],
Expand Down Expand Up @@ -286,6 +282,57 @@ export function karpenterSetup(cluster: ICluster,
actions: ['iam:GetInstanceProfile'],
});

const allowScopedInstanceProfileCreationActions: PolicyStatement = new PolicyStatement({
sid: 'AllowScopedInstanceProfileCreationActions',
effect: Effect.ALLOW,
resources: ['*'],
actions: ['iam:CreateInstanceProfile'],
conditions: {
StringEquals: {
[`aws:RequestTag/kubernetes.io/cluster/${clusterName}`]: 'owned',
'aws:RequestTag/topology.kubernetes.io/region': `${Stack.of(scope).region}`,
},
StringLike: {
'aws:RequestTag/karpenter.k8s.aws/ec2nodeclass': '*',
},
},
});

const allowScopedInstanceProfileTagActions: PolicyStatement = new PolicyStatement({
sid: 'AllowScopedInstanceProfileTagActions',
effect: Effect.ALLOW,
resources: ['*'],
actions: ['iam:TagInstanceProfile'],
conditions: {
StringEquals: {
[`aws:ResourceTag/kubernetes.io/cluster/${clusterName}`]: 'owned',
'aws:ResourceTag/topology.kubernetes.io/region': `${Stack.of(scope).region}`,
[`aws:RequestTag/kubernetes.io/cluster/${clusterName}`]: 'owned',
'aws:RequestTag/topology.kubernetes.io/region': `${Stack.of(scope).region}`,
},
StringLike: {
'aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass': '*',
'aws:RequestTag/karpenter.k8s.aws/ec2nodeclass': '*',
},
},
});

const allowScopedInstanceProfileActions: PolicyStatement = new PolicyStatement({
sid: 'AllowScopedInstanceProfileActions',
effect: Effect.ALLOW,
resources: ['*'],
actions: ['iam:AddRoleToInstanceProfile', 'iam:RemoveRoleFromInstanceProfile', 'iam:DeleteInstanceProfile'],
conditions: {
StringEquals: {
[`aws:ResourceTag/kubernetes.io/cluster/${clusterName}`]: 'owned',
'aws:ResourceTag/topology.kubernetes.io/region': `${Stack.of(scope).region}`,
},
StringLike: {
'aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass': '*',
},
},
});


const karpenterNS = cluster.addManifest('karpenterNS', {
apiVersion: 'v1',
Expand All @@ -300,18 +347,21 @@ export function karpenterSetup(cluster: ICluster,

karpenterAccount.node.addDependency(karpenterNS);

karpenterAccount.addToPrincipalPolicy(karpenterControllerPolicyStatementSSM);
karpenterAccount.addToPrincipalPolicy(karpenterControllerPolicyStatementEC2);
karpenterAccount.addToPrincipalPolicy(karpenterControllerPolicyStatementIAM);
karpenterAccount.addToPrincipalPolicy(allowScopedEC2InstanceActions);
karpenterAccount.addToPrincipalPolicy(allowSSMReadActions);
karpenterAccount.addToPrincipalPolicy(allowScopedEC2InstanceActionsWithTags);
karpenterAccount.addToPrincipalPolicy(allowScopedEC2LaunchTemplateActions);
karpenterAccount.addToPrincipalPolicy(allowMachineMigrationTagging);
karpenterAccount.addToPrincipalPolicy(allowScopedEC2InstanceActions);
karpenterAccount.addToPrincipalPolicy(allowPricingReadActions);
karpenterAccount.addToPrincipalPolicy(allowPassingInstanceRole);
karpenterAccount.addToPrincipalPolicy(allowScopedInstanceProfileCreationActions);
karpenterAccount.addToPrincipalPolicy(allowScopedInstanceProfileTagActions);
karpenterAccount.addToPrincipalPolicy(allowScopedInstanceProfileActions);
karpenterAccount.addToPrincipalPolicy(allowScopedResourceTagging);
karpenterAccount.addToPrincipalPolicy(allowScopedResourceCreationTagging);
karpenterAccount.addToPrincipalPolicy(allowScopedDeletion);
karpenterAccount.addToPrincipalPolicy(allowInterruptionQueueActions);
karpenterAccount.addToPrincipalPolicy(allowAPIServerEndpointDiscovery);
karpenterAccount.addToPrincipalPolicy(allowInstanceProfileReadActions);
karpenterAccount.addToPrincipalPolicy(allowRegionalReadActions);

//Deploy Karpenter Chart
const karpenterChart = cluster.addHelmChart('KarpenterHelmChart', {
Expand All @@ -332,7 +382,6 @@ export function karpenterSetup(cluster: ICluster,
},
settings: {
aws: {
defaultInstanceProfile: instanceProfile.instanceProfileName,
clusterName: clusterName,
clusterEndpoint: cluster.clusterEndpoint,
interruptionQueueName: karpenterInterruptionQueue.queueName,
Expand Down
9 changes: 0 additions & 9 deletions framework/src/processing/lib/spark-runtime/emr-containers/spark-emr-containers-runtime.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ import {
import { CfnVirtualCluster } from 'aws-cdk-lib/aws-emrcontainers';
import { IRule } from 'aws-cdk-lib/aws-events';
import {
CfnInstanceProfile,
CfnServiceLinkedRole,
Effect,
FederatedPrincipal,
Expand Down Expand Up @@ -230,13 +229,6 @@ export class SparkEmrContainersRuntime extends TrackedConstruct {
this.ec2InstanceNodeGroupRole.addManagedPolicy(ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore'));
this.ec2InstanceNodeGroupRole.addManagedPolicy(ManagedPolicy.fromAwsManagedPolicyName('AmazonEKS_CNI_Policy'));

//Create instance profile to be used by Managed nodegroup and karpenter
const clusterInstanceProfile = new CfnInstanceProfile(scope, 'KarpenterInstanceProfile', {
roles: [this.ec2InstanceNodeGroupRole.roleName],
// instanceProfileName: `adsfNodeInstanceProfile-${clusterName ?? 'default'}`,
path: '/',
});

const karpenterVersion = props.karpenterVersion ?? DEFAULT_KARPENTER_VERSION;

let eksCluster: Cluster;
Expand Down Expand Up @@ -322,7 +314,6 @@ export class SparkEmrContainersRuntime extends TrackedConstruct {
eksCluster,
clusterName,
scope,
clusterInstanceProfile,
this.ec2InstanceNodeGroupRole,
this.removalPolicy,
karpenterVersion,
Expand Down
4 changes: 3 additions & 1 deletion framework/test/e2e/s3-data-copy.e2e.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,9 @@ beforeAll(async() => {
}, 900000);

test(' S3DataCopy is created', async() => {
expect(deployResult.Target).toContain('test-eu-west-1-145388625860-e3c8c022');
const regexPattern = /test-([a-z]+-\d+)-(\d+)-([a-z0-9]+)/;
expect(deployResult.Target).toMatch(regexPattern);

});

afterAll(async () => {
Expand Down
11 changes: 0 additions & 11 deletions framework/test/unit/processing/spark-runtime-containers.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -409,17 +409,6 @@ describe('With default configuration, the construct ', () => {
});
});

test('should create an instance profile for Karpenter', () => {
template.hasResourceProperties('AWS::IAM::InstanceProfile', {
Path: '/',
Roles: [
{
Ref: Match.stringLikeRegexp('.*Ec2InstanceNodeGroupRole.*'),
},
],
});
});

test('should create a KMS key for EKS secrets', () => {
template.hasResourceProperties('AWS::KMS::Key', {
Description: 'eks-secrets-key',
Expand Down

0 comments on commit ee444b8

Please sign in to comment.