From bf32e0d422dfcb3e49f78f1b8f01ec5cdc1a7d27 Mon Sep 17 00:00:00 2001 From: nemanja-kovacevic-thinkit Date: Fri, 24 May 2024 14:02:31 +0200 Subject: [PATCH] feat: added construct for application ingress --- .../annotation-util.ts | 0 lib/common/index.ts | 1 + lib/index.ts | 1 + lib/ingress/index.ts | 2 + lib/ingress/ingress-props.ts | 75 ++++ lib/ingress/ingress.ts | 165 +++++++ lib/web-service/index.ts | 1 - lib/web-service/web-service.ts | 4 +- .../__snapshots__/ingress.test.ts.snap | 158 +++++++ test/ingress/ingress.test.ts | 422 ++++++++++++++++++ 10 files changed, 826 insertions(+), 3 deletions(-) rename lib/{web-service => common}/annotation-util.ts (100%) create mode 100644 lib/ingress/index.ts create mode 100644 lib/ingress/ingress-props.ts create mode 100644 lib/ingress/ingress.ts create mode 100644 test/ingress/__snapshots__/ingress.test.ts.snap create mode 100644 test/ingress/ingress.test.ts diff --git a/lib/web-service/annotation-util.ts b/lib/common/annotation-util.ts similarity index 100% rename from lib/web-service/annotation-util.ts rename to lib/common/annotation-util.ts diff --git a/lib/common/index.ts b/lib/common/index.ts index 35a9741..3919136 100644 --- a/lib/common/index.ts +++ b/lib/common/index.ts @@ -5,3 +5,4 @@ export * from "./name-util"; export * from "./statefulset-util"; export * from "./value-util"; export * from "./workload-props"; +export * from "./annotation-util"; diff --git a/lib/index.ts b/lib/index.ts index fb8d5e0..a7065a1 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -10,3 +10,4 @@ export * from "./postgres"; export * from "./redis"; export * from "./talis-chart"; export * from "./web-service"; +export * from "./ingress"; diff --git a/lib/ingress/index.ts b/lib/ingress/index.ts new file mode 100644 index 0000000..de4f9dc --- /dev/null +++ b/lib/ingress/index.ts @@ -0,0 +1,2 @@ +export * from "./ingress-props"; +export * from "./ingress"; diff --git a/lib/ingress/ingress-props.ts b/lib/ingress/ingress-props.ts new file mode 100644 index 0000000..efbd540 --- /dev/null +++ b/lib/ingress/ingress-props.ts @@ -0,0 +1,75 @@ +export interface ServiceRouteProps { + /** + * Service name. + */ + readonly name: string; + + /** + * Service port. + */ + readonly port: number; + + /** + * Service namespace. + */ + readonly namespace: string; + + /** + * Weight of the service. Has to be an integer in between 0 and 100. + */ + readonly weight: number; +} + +export interface IngressProps { + /** + * Custom labels, they will be merged with the default app, role, and instance. + * @default { app: "", role: "server", instance: "" } + */ + readonly labels?: { [key: string]: string }; + + /** + * ARN of one or more certificate managed by AWS Certificate Manager + */ + readonly certificateArn?: string[]; + + /** + * Overrides for Ingress annotations. + * @see https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.3/guide/ingress/annotations/ + */ + readonly ingressAnnotations?: { [key: string]: string }; + + /** + * Hostname to add External DNS record for the ingress. + */ + readonly externalHostname?: string; + + /** + * Hostnames, they will be added as Ingress rules. + */ + readonly hostnames: string[]; + + /** + * Defines routing for each service. This will be used to create target groups for service weighting action. + */ + readonly serviceRouting: ServiceRouteProps[]; + + /** + * Ingress class name. + */ + readonly ingressClassName: string; + + /** + * Ingress class priority, between -1000 and 1000 + */ + readonly ingressClassPriority?: number; + + /** + * Application name. + */ + readonly app: string; + + /** + * Instance name. + */ + readonly instance: string; +} diff --git a/lib/ingress/ingress.ts b/lib/ingress/ingress.ts new file mode 100644 index 0000000..320dee6 --- /dev/null +++ b/lib/ingress/ingress.ts @@ -0,0 +1,165 @@ +import { Chart } from "cdk8s"; +import { Construct } from "constructs"; +import { + IngressRule, + IntOrString, + KubeIngress, + KubeService, +} from "../../imports/k8s"; +import { + convertToStringMap, + convertToJsonContent, + joinNameParts, + convertToStringList, +} from "../common"; +import { IngressProps } from "."; +import { ServiceSpecType } from "../k8s"; + +export class Ingress extends Construct { + constructor(scope: Construct, id: string, props: IngressProps) { + super(scope, id); + this.validateProps(props); + + const chart = Chart.of(this); + const environment = chart.labels.environment; + const region = chart.labels.region; + const service = joinNameParts([props.app, environment, region]); + + const labels: { [key: string]: string } = { + ...chart.labels, + ...props.labels, + app: props.app, + instance: props.instance, + role: "server", + service: service, + }; + + const externalDns: Record = {}; + const ingressRules: IngressRule[] = []; + const ingressListenPorts: Record[] = [ + { HTTP: 80 }, + { HTTPS: 443 }, + ]; + const ingressTlsAnnotations: Record = {}; + + if (props.externalHostname) { + externalDns["external-dns.alpha.kubernetes.io/hostname"] = + props.externalHostname; + } + + const targetGroups: Record[] = []; + for (const service of props.serviceRouting) { + const serviceName = `${service.name}-${service.namespace}`; + new KubeService(this, `${id}-${serviceName}-service`, { + metadata: { + name: serviceName, + labels: labels, + }, + spec: { + type: ServiceSpecType.EXTERNAL_NAME, + externalName: `${service.name}.${service.namespace}.svc.cluster.local`, + ports: [ + { + port: service.port, + targetPort: IntOrString.fromNumber(service.port), + }, + ], + }, + }); + + const targetGroup = { + serviceName: serviceName, + servicePort: service.port, + weight: service.weight, + }; + targetGroups.push(targetGroup); + } + + const ingressAnnotations: { [key: string]: string } = { + "alb.ingress.kubernetes.io/listen-ports": + convertToJsonContent(ingressListenPorts), + "alb.ingress.kubernetes.io/success-codes": "200,303", + "alb.ingress.kubernetes.io/target-type": "instance", + "alb.ingress.kubernetes.io/group.order": String( + props.ingressClassPriority ?? 0, + ), + "alb.ingress.kubernetes.io/tags": convertToStringMap({ + service: labels.service, + instance: id, + environment: environment, + }), + "alb.ingress.kubernetes.io/actions.service-weighting": + convertToJsonContent({ + type: "forward", + forwardConfig: { + targetGroups: targetGroups, + }, + }), + ...ingressTlsAnnotations, + ...props.ingressAnnotations, + ...externalDns, + }; + + if (props.certificateArn) { + ingressAnnotations["alb.ingress.kubernetes.io/certificate-arn"] = + convertToStringList(props.certificateArn); + } + + for (const hostname of props.hostnames) { + ingressRules.push({ + host: hostname, + http: { + paths: [ + { + pathType: "Prefix", + path: "/", + backend: { + service: { + name: "service-weighting", + port: { + name: "use-annotation", + }, + }, + }, + }, + ], + }, + }); + } + + new KubeIngress(this, id, { + metadata: { + annotations: ingressAnnotations, + labels: labels, + }, + spec: { + ingressClassName: props.ingressClassName, + rules: ingressRules, + }, + }); + } + + private validateProps(props: IngressProps): void { + if (props.ingressClassPriority) + if ( + props.ingressClassPriority < -1000 || + props.ingressClassPriority > 1000 + ) + throw new Error( + "Ingress class priority has to be between -1000 and 1000", + ); + + if (props.hostnames.length < 1) + throw new Error("At least one hostname has to be defined."); + + if (props.serviceRouting.length < 1) + throw new Error("At least one service route has to be defined."); + + const totalWeight = props.serviceRouting.reduce( + (sum, serviceRoute) => sum + serviceRoute.weight, + 0, + ); + if (totalWeight != 100) + throw new Error("Total service routing weigth must be 100."); + } +} diff --git a/lib/web-service/index.ts b/lib/web-service/index.ts index cd13db5..7ea493b 100644 --- a/lib/web-service/index.ts +++ b/lib/web-service/index.ts @@ -1,4 +1,3 @@ -export * from "./annotation-util"; export * from "./nginx-util"; export * from "./resque-web"; export * from "./web-service-props"; diff --git a/lib/web-service/web-service.ts b/lib/web-service/web-service.ts index e118b49..9d447ef 100644 --- a/lib/web-service/web-service.ts +++ b/lib/web-service/web-service.ts @@ -15,8 +15,6 @@ import { Volume, } from "../../imports/k8s"; import { - convertToJsonContent, - convertToStringMap, HorizontalPodAutoscalerProps, NginxContainerProps, PodDisruptionBudgetProps, @@ -27,6 +25,8 @@ import { makeLoadBalancerName, ensureArray, getValueFromIntOrPercent, + convertToJsonContent, + convertToStringMap, } from "../common"; import { supportsTls } from "./tls-util"; import { diff --git a/test/ingress/__snapshots__/ingress.test.ts.snap b/test/ingress/__snapshots__/ingress.test.ts.snap new file mode 100644 index 0000000..6a5c70e --- /dev/null +++ b/test/ingress/__snapshots__/ingress.test.ts.snap @@ -0,0 +1,158 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`Ingress > Props > All the props 1`] = ` +[ + { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "labels": { + "app": "my-app", + "environment": "test", + "instance": "my-instance", + "my-key": "my-value", + "region": "local", + "role": "server", + "service": "my-app-test-local", + }, + "name": "my-name-my-namespace", + "namespace": "test", + }, + "spec": { + "externalName": "my-name.my-namespace.svc.cluster.local", + "ports": [ + { + "port": 80, + "targetPort": 80, + }, + ], + "type": "ExternalName", + }, + }, + { + "apiVersion": "networking.k8s.io/v1", + "kind": "Ingress", + "metadata": { + "annotations": { + "alb.ingress.kubernetes.io/actions.service-weighting": "{"type":"forward","forwardConfig":{"targetGroups":[{"serviceName":"my-name-my-namespace","servicePort":80,"weight":100}]}}", + "alb.ingress.kubernetes.io/certificate-arn": "my-certificate", + "alb.ingress.kubernetes.io/group.order": "1000", + "alb.ingress.kubernetes.io/listen-ports": "[{"HTTP":80},{"HTTPS":443}]", + "alb.ingress.kubernetes.io/success-codes": "200,303", + "alb.ingress.kubernetes.io/tags": "service=my-app-test-local,instance=ingress-test,environment=test", + "alb.ingress.kubernetes.io/target-type": "instance", + "external-dns.alpha.kubernetes.io/hostname": "my-external-hostname", + "my-key": "my-value", + }, + "labels": { + "app": "my-app", + "environment": "test", + "instance": "my-instance", + "my-key": "my-value", + "region": "local", + "role": "server", + "service": "my-app-test-local", + }, + "name": "test-ingress-test-c813de7d", + "namespace": "test", + }, + "spec": { + "ingressClassName": "my-ingress-class", + "rules": [ + { + "host": "my-hostname", + "http": { + "paths": [ + { + "backend": { + "service": { + "name": "service-weighting", + "port": { + "name": "use-annotation", + }, + }, + }, + "path": "/", + "pathType": "Prefix", + }, + ], + }, + }, + ], + }, + }, +] +`; + +exports[`Ingress > Props > Minimal required props 1`] = ` +[ + { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "labels": { + "app": "my-app", + "instance": "my-instance", + "role": "server", + "service": "my-app", + }, + "name": "my-name-my-namespace", + }, + "spec": { + "externalName": "my-name.my-namespace.svc.cluster.local", + "ports": [ + { + "port": 80, + "targetPort": 80, + }, + ], + "type": "ExternalName", + }, + }, + { + "apiVersion": "networking.k8s.io/v1", + "kind": "Ingress", + "metadata": { + "annotations": { + "alb.ingress.kubernetes.io/actions.service-weighting": "{"type":"forward","forwardConfig":{"targetGroups":[{"serviceName":"my-name-my-namespace","servicePort":80,"weight":100}]}}", + "alb.ingress.kubernetes.io/group.order": "0", + "alb.ingress.kubernetes.io/listen-ports": "[{"HTTP":80},{"HTTPS":443}]", + "alb.ingress.kubernetes.io/success-codes": "200,303", + "alb.ingress.kubernetes.io/tags": "service=my-app,instance=ingress-test", + "alb.ingress.kubernetes.io/target-type": "instance", + }, + "labels": { + "app": "my-app", + "instance": "my-instance", + "role": "server", + "service": "my-app", + }, + "name": "test-ingress-test-c813de7d", + }, + "spec": { + "ingressClassName": "my-ingress-class", + "rules": [ + { + "host": "my-hostname", + "http": { + "paths": [ + { + "backend": { + "service": { + "name": "service-weighting", + "port": { + "name": "use-annotation", + }, + }, + }, + "path": "/", + "pathType": "Prefix", + }, + ], + }, + }, + ], + }, + }, +] +`; diff --git a/test/ingress/ingress.test.ts b/test/ingress/ingress.test.ts new file mode 100644 index 0000000..2a4ad59 --- /dev/null +++ b/test/ingress/ingress.test.ts @@ -0,0 +1,422 @@ +import { Chart, Testing } from "cdk8s"; +import { Ingress, IngressProps } from "../../lib"; + +const requiredProps: IngressProps = { + app: "my-app", + instance: "my-instance", + serviceRouting: [ + { + name: "my-name", + port: 80, + namespace: "my-namespace", + weight: 100, + }, + ], + ingressClassName: "my-ingress-class", + hostnames: ["my-hostname"], +}; + +function synthIngress(props: IngressProps, chart: Chart = Testing.chart()) { + new Ingress(chart, "ingress-test", props); + const results = Testing.synth(chart); + return results; +} + +describe("Ingress", () => { + describe("Props", () => { + test("Minimal required props", () => { + const results = synthIngress(requiredProps); + expect(results).toMatchSnapshot(); + }); + + test("All the props", () => { + const props: IngressProps = { + ...requiredProps, + labels: { "my-key": "my-value" }, + certificateArn: ["my-certificate"], + ingressAnnotations: { "my-key": "my-value" }, + externalHostname: "my-external-hostname", + ingressClassPriority: 1000, + }; + + const app = Testing.app(); + const chart = new Chart(app, "test", { + namespace: "test", + labels: { + app: "test-app", + environment: "test", + region: "local", + }, + }); + const results = synthIngress(props, chart); + expect(results).toMatchSnapshot(); + }); + }); + + describe("Props validation", () => { + test("throws error when at least one hostname is not set", () => { + const props: IngressProps = { + ...requiredProps, + hostnames: [], + }; + expect(() => synthIngress(props)).toThrowError(); + }); + + test("throws error when ingress class priority has invalid value", () => { + const propsForUnderLowerLimit: IngressProps = { + ...requiredProps, + ingressClassPriority: -1001, + }; + expect(() => synthIngress(propsForUnderLowerLimit)).toThrowError(); + + const propsForOverUpperLimit: IngressProps = { + ...requiredProps, + ingressClassPriority: 1001, + }; + expect(() => synthIngress(propsForOverUpperLimit)).toThrowError(); + }); + + test("throws error when weight sum is not 100", () => { + const props: IngressProps = { + ...requiredProps, + serviceRouting: [ + { + name: "test-name1", + port: 104, + namespace: "test-namespace1", + weight: 30, + }, + { + name: "test-name2", + port: 102, + namespace: "test-namespace2", + weight: 50, + }, + ], + }; + + expect(() => synthIngress(props)).toThrowError(); + }); + + test("throws error when service routing is not set", () => { + const props: IngressProps = { + ...requiredProps, + serviceRouting: [], + }; + + expect(() => synthIngress(props)).toThrowError(); + }); + }); + + describe("Services", () => { + test("expected length when more than one service exists", () => { + const props: IngressProps = { + ...requiredProps, + serviceRouting: [ + { + name: "test-name1", + port: 104, + namespace: "test-namespace1", + weight: 20, + }, + { + name: "test-name2", + port: 102, + namespace: "test-namespace2", + weight: 80, + }, + ], + }; + + const results = synthIngress(props); + const service = results.filter((obj) => obj.kind === "Service"); + expect(service.length).toBe(2); + }); + + test("all labels and namespace are set", () => { + const props: IngressProps = { + ...requiredProps, + labels: { "my-key": "my-value" }, + }; + + const app = Testing.app(); + const chart = new Chart(app, "test", { + namespace: "test", + labels: { + app: "test-app", // should be overriden with "my-app" from props + environment: "test", + region: "local", + }, + }); + const results = synthIngress(props, chart); + const service = results.find((obj) => obj.kind === "Service"); + expect(service).toHaveProperty("metadata.labels.my-key", "my-value"); + expect(service).toHaveProperty("metadata.labels.app", "my-app"); + expect(service).toHaveProperty("metadata.labels.instance", "my-instance"); + expect(service).toHaveProperty("metadata.labels.role", "server"); + expect(service).toHaveProperty("metadata.labels.region", "local"); + expect(service).toHaveProperty("metadata.labels.environment", "test"); + expect(service).toHaveProperty( + "metadata.labels.service", + "my-app-test-local", + ); + + expect(service).toHaveProperty("metadata.namespace", "test"); + }); + + test("all properties are set", () => { + const props: IngressProps = { + ...requiredProps, + serviceRouting: [ + { + name: "test-name", + port: 80, + namespace: "test-namespace", + weight: 100, + }, + ], + }; + + const results = synthIngress(props); + const service = results.find((obj) => obj.kind === "Service"); + + expect(service).toHaveProperty( + "metadata.name", + "test-name-test-namespace", + ); + expect(service).toHaveProperty( + "spec.externalName", + "test-name.test-namespace.svc.cluster.local", + ); + const ports = service["spec"]["ports"]; + expect(ports.length).toBe(1); + expect(ports[0]).toHaveProperty("port", 80); + expect(ports[0]).toHaveProperty("targetPort", 80); + }); + }); + + describe("Ingress", () => { + test("all labels and namespace are set", () => { + const props: IngressProps = { + ...requiredProps, + labels: { "my-key": "my-value" }, + }; + + const app = Testing.app(); + const chart = new Chart(app, "test", { + namespace: "test", + labels: { + app: "test-app", // should be overriden with "my-app" from props + environment: "test", + region: "local", + }, + }); + const results = synthIngress(props, chart); + const ingress = results.find((obj) => obj.kind === "Ingress"); + expect(ingress).toHaveProperty("metadata.labels.my-key", "my-value"); + expect(ingress).toHaveProperty("metadata.labels.app", "my-app"); + expect(ingress).toHaveProperty("metadata.labels.instance", "my-instance"); + expect(ingress).toHaveProperty("metadata.labels.role", "server"); + expect(ingress).toHaveProperty("metadata.labels.region", "local"); + expect(ingress).toHaveProperty("metadata.labels.environment", "test"); + expect(ingress).toHaveProperty( + "metadata.labels.service", + "my-app-test-local", + ); + + expect(ingress).toHaveProperty("metadata.namespace", "test"); + }); + + test("annotation exists when externalHostname is set", () => { + const props: IngressProps = { + ...requiredProps, + externalHostname: "test-external-hostname", + }; + + const results = synthIngress(props); + const ingress = results.find((obj) => obj.kind === "Ingress"); + const annotations = ingress["metadata"]["annotations"]; + expect(annotations).toHaveProperty( + "external-dns.alpha.kubernetes.io/hostname", + "test-external-hostname", + ); + }); + + test("annotation exists when certificateArn is set", () => { + const props: IngressProps = { + ...requiredProps, + certificateArn: ["test-certificate-arn"], + }; + + const results = synthIngress(props); + const ingress = results.find((obj) => obj.kind === "Ingress"); + const annotations = ingress["metadata"]["annotations"]; + expect(annotations).toHaveProperty( + "alb.ingress.kubernetes.io/certificate-arn", + "test-certificate-arn", + ); + }); + + test("annotation exists when forwarded through additional ingress annotations", () => { + const props: IngressProps = { + ...requiredProps, + ingressAnnotations: { + "test-annotation-key": "test-annotation-value", + "test-second-annotation-key": "test-second-annotation-value", + }, + }; + + const results = synthIngress(props); + const ingress = results.find((obj) => obj.kind === "Ingress"); + const annotations = ingress["metadata"]["annotations"]; + expect(annotations).toHaveProperty( + "test-annotation-key", + "test-annotation-value", + ); + expect(annotations).toHaveProperty( + "test-second-annotation-key", + "test-second-annotation-value", + ); + }); + + test("ingress has a annotation with proper tags", () => { + const props: IngressProps = { + ...requiredProps, + certificateArn: ["test-certificate-arn"], + }; + + const app = Testing.app(); + const chart = new Chart(app, "test", { + namespace: "test", + labels: { + environment: "test", + region: "local", + }, + }); + const results = synthIngress(props, chart); + const ingress = results.find((obj) => obj.kind === "Ingress"); + const annotations = ingress["metadata"]["annotations"]; + expect(annotations).toHaveProperty( + "alb.ingress.kubernetes.io/tags", + "service=my-app-test-local,instance=ingress-test,environment=test", + ); + }); + + test("ingress has a annotation with proper value for ingress class priority when it's defined", () => { + const props: IngressProps = { + ...requiredProps, + ingressClassPriority: 101, + }; + + const results = synthIngress(props); + const ingress = results.find((obj) => obj.kind === "Ingress"); + const annotations = ingress["metadata"]["annotations"]; + expect(annotations).toHaveProperty( + "alb.ingress.kubernetes.io/group.order", + "101", + ); + }); + + test("ingress has a annotation with proper value for ingress class priority when it's not defined", () => { + const props: IngressProps = { + ...requiredProps, + }; + + const results = synthIngress(props); + const ingress = results.find((obj) => obj.kind === "Ingress"); + const annotations = ingress["metadata"]["annotations"]; + expect(annotations).toHaveProperty( + "alb.ingress.kubernetes.io/group.order", + "0", + ); + }); + + test("rules exists when multiple hostnames are defined", () => { + const props: IngressProps = { + ...requiredProps, + hostnames: ["firstHostname", "secondHostname"], + }; + + const results = synthIngress(props); + const ingress = results.find((obj) => obj.kind === "Ingress"); + const rules = ingress["spec"]["rules"]; + expect(rules.length).toBe(2); + expect(rules[0]).toHaveProperty("host", "firstHostname"); + expect(rules[1]).toHaveProperty("host", "secondHostname"); + }); + + test("ingress class name has proper value", () => { + const props: IngressProps = { + ...requiredProps, + ingressClassName: "test-class-name", + }; + + const results = synthIngress(props); + const ingress = results.find((obj) => obj.kind === "Ingress"); + expect(ingress).toHaveProperty( + "spec.ingressClassName", + "test-class-name", + ); + }); + + test("service weighting annotation has a proper value when multiple service routes are set", () => { + const props: IngressProps = { + ...requiredProps, + serviceRouting: [ + { + name: "test-first-name", + port: 80, + namespace: "test-namespace", + weight: 80, + }, + { + name: "test-second-name", + port: 81, + namespace: "test-namespace", + weight: 20, + }, + ], + }; + + const results = synthIngress(props); + const ingress = results.find((obj) => obj.kind === "Ingress"); + const annotations = ingress["metadata"]["annotations"]; + const serviceWeighting = + '{"type":"forward","forwardConfig":{"targetGroups":[{"serviceName":"test-first-name-test-namespace","servicePort":80,"weight":80},{"serviceName":"test-second-name-test-namespace","servicePort":81,"weight":20}]}}'; + expect(annotations).toHaveProperty( + "alb.ingress.kubernetes.io/actions.service-weighting", + serviceWeighting, + ); + }); + + test("all properties are set", () => { + const props: IngressProps = { + ...requiredProps, + serviceRouting: [ + { + name: "test-name", + port: 80, + namespace: "test-namespace", + weight: 100, + }, + ], + }; + + const results = synthIngress(props); + const service = results.find((obj) => obj.kind === "Service"); + + expect(service).toHaveProperty( + "metadata.name", + "test-name-test-namespace", + ); + expect(service).toHaveProperty( + "spec.externalName", + "test-name.test-namespace.svc.cluster.local", + ); + const ports = service["spec"]["ports"]; + expect(ports.length).toBe(1); + expect(ports[0]).toHaveProperty("port", 80); + expect(ports[0]).toHaveProperty("targetPort", 80); + }); + }); +});