diff --git a/config/crd/bases/systemautoscaler.polimi.it_podscales.yaml b/config/crd/bases/systemautoscaler.polimi.it_podscales.yaml new file mode 100644 index 0000000..000d92c --- /dev/null +++ b/config/crd/bases/systemautoscaler.polimi.it_podscales.yaml @@ -0,0 +1,102 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.2 + creationTimestamp: null + name: podscales.systemautoscaler.polimi.it +spec: + group: systemautoscaler.polimi.it + names: + kind: PodScale + listKind: PodScaleList + plural: podscales + singular: podscale + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: PodScale defines the mapping between a `ServiceLevelAgreement` + and a `Pod` matching the selector. It also keeps track of the resource values + computed by `Recommender` and adjusted by `Contention Manager`. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: PodScaleSpec is the spec for a PodScale resource + properties: + container: + type: string + desired: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, quantity) pairs. + type: object + namespace: + type: string + pod: + type: string + service: + type: string + serviceLevelAgreement: + type: string + required: + - container + - namespace + - pod + - service + - serviceLevelAgreement + type: object + status: + description: PodScaleStatus contains the resources patched by the `Contention + Manager` according to the available node resources and other pods' SLA + properties: + actual: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, quantity) pairs. + type: object + capped: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, quantity) pairs. + type: object + type: object + required: + - spec + - status + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/bases/systemautoscaler.polimi.it_servicelevelagreements.yaml b/config/crd/bases/systemautoscaler.polimi.it_servicelevelagreements.yaml new file mode 100644 index 0000000..02cf997 --- /dev/null +++ b/config/crd/bases/systemautoscaler.polimi.it_servicelevelagreements.yaml @@ -0,0 +1,172 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.2 + creationTimestamp: null + name: servicelevelagreements.systemautoscaler.polimi.it +spec: + group: systemautoscaler.polimi.it + names: + kind: ServiceLevelAgreement + listKind: ServiceLevelAgreementList + plural: servicelevelagreements + singular: servicelevelagreement + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: ServiceLevelAgreement is a configuration for the autoscaling + system. It sets a requirement on the services that matches the selector. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ServiceLevelAgreementSpec defines the agreement specifying + the metric requirement to honor by System Autoscaler, a Selector used + to match a service with the Service Level Agreement and the default + resources assigned to pods in case the `requests` field is empty in + the `PodSpec`. + properties: + defaultResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Specify the default resources assigned to pods in case + `requests` field is empty in `PodSpec`. + type: object + integralGain: + default: 25 + description: Integral gain used by recommender. BC parameter + format: int32 + type: integer + maxReplicas: + default: 100 + description: The upper bound of replicas for the application. + format: int32 + type: integer + maxResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: The upper bound of resources to assign to containers. + type: object + metric: + description: Specify the metric on which the requirement is set. + properties: + responseTime: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + minReplicas: + default: 1 + description: The lower bound of replicas for the application. + format: int32 + type: integer + minResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: The lower bound of resources to assign to containers. + type: object + proportionalGain: + default: 50 + description: Proportional gain used by recommender. DC parameter + format: int32 + type: integer + service: + description: Identify the Service on which the agreement is defined + properties: + container: + description: The container to track inside the Pods. + type: string + selector: + description: Specify the selector to match Services and Service + Level Agreement + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If + the operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A + single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is "key", + the operator is "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + required: + - container + - selector + type: object + required: + - metric + - service + type: object + required: + - spec + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/deploy/metrics-exposer.yaml b/config/deploy/metrics-exposer.yaml new file mode 100644 index 0000000..ed39da1 --- /dev/null +++ b/config/deploy/metrics-exposer.yaml @@ -0,0 +1,161 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: custom-metrics +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: custom-metrics:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: custom-metrics-apiserver + namespace: custom-metrics +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: custom-metrics-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: custom-metrics-apiserver + namespace: custom-metrics +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: custom-metrics-apiserver + name: custom-metrics-apiserver + namespace: custom-metrics +spec: + replicas: 1 + selector: + matchLabels: + app: custom-metrics-apiserver + template: + metadata: + labels: + app: custom-metrics-apiserver + name: custom-metrics-apiserver + spec: + nodeSelector: + node-role.kubernetes.io/master: "true" + serviceAccountName: custom-metrics-apiserver + containers: + - name: custom-metrics-apiserver + image: systemautoscaler/metrics-exposer:dev + imagePullPolicy: Always + resources: + requests: + cpu: 200m + memory: 200Mi + limits: + cpu: 200m + memory: 200Mi + args: + - /metrics-exposer + - --secure-port=6443 + - --logtostderr=true + - --v=10 + ports: + - containerPort: 6443 + name: https + - containerPort: 8080 + name: http + volumeMounts: + - mountPath: /tmp + name: temp-vol + volumes: + - name: temp-vol + emptyDir: {} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: custom-metrics-resource-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: custom-metrics-resource-reader +subjects: +- kind: ServiceAccount + name: custom-metrics-apiserver + namespace: custom-metrics +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: custom-metrics-apiserver + namespace: custom-metrics +--- +apiVersion: v1 +kind: Service +metadata: + name: custom-metrics-apiserver + namespace: custom-metrics +spec: + ports: + - name: https + port: 443 + targetPort: 6443 + - name: http + port: 80 + targetPort: 8080 + selector: + app: custom-metrics-apiserver +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1beta1.custom.metrics.k8s.io +spec: + service: + name: custom-metrics-apiserver + namespace: custom-metrics + group: custom.metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: true + groupPriorityMinimum: 100 + versionPriority: 100 +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1beta2.custom.metrics.k8s.io +spec: + service: + name: custom-metrics-apiserver + namespace: custom-metrics + group: custom.metrics.k8s.io + version: v1beta2 + insecureSkipTLSVerify: true + groupPriorityMinimum: 100 + versionPriority: 200 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: custom-metrics-resource-reader +rules: + - apiGroups: [ "" ] + resources: [ "pods", "services", "nodes" ] + verbs: [ "update", "get", "watch", "list" ] + - apiGroups: [ "" ] + resources: [ "events" ] + verbs: [ "*" ] + - apiGroups: [ "systemautoscaler.polimi.it" ] + resources: [ "servicelevelagreements" ] + verbs: [ "get", "watch", "list" ] + - apiGroups: [ "systemautoscaler.polimi.it" ] + resources: [ "podscales" ] + verbs: [ "get", "watch", "list" ] diff --git a/config/deploy/monitoring.yaml b/config/deploy/monitoring.yaml index f3277dc..f23fc06 100644 --- a/config/deploy/monitoring.yaml +++ b/config/deploy/monitoring.yaml @@ -57,15 +57,5 @@ spec: serviceAccountName: sa-monitoring containers: - name: monitoring - image: systemautoscaler/system-autoscaler-monitoring:dev - imagePullPolicy: Always - - name: cpu-monitoring - image: systemautoscaler/cpu-monitoring:dev - imagePullPolicy: Always - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - value: "default" \ No newline at end of file + image: systemautoscaler/system-autoscaler-monitoring:distributed + imagePullPolicy: Always \ No newline at end of file diff --git a/config/deploy/pod-autoscaler.yaml b/config/deploy/pod-autoscaler.yaml new file mode 100644 index 0000000..0a3e780 --- /dev/null +++ b/config/deploy/pod-autoscaler.yaml @@ -0,0 +1,74 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pod-autoscaler + namespace: kube-system +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system-autoscaler:pod-autoscaler +rules: + - apiGroups: [""] + resources: ["pods", "services", "nodes"] + verbs: ["update", "get", "watch", "list"] + - apiGroups: [""] + resources: ["events"] + verbs: ["*"] + - apiGroups: ["systemautoscaler.polimi.it"] + resources: ["servicelevelagreements"] + verbs: ["get", "watch", "list"] + - apiGroups: ["systemautoscaler.polimi.it"] + resources: ["podscales"] + verbs: ["*"] + - apiGroups: ["custom.metrics.k8s.io"] + resources: ["pods/response_time"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system-autoscaler:pod-autoscaler +subjects: + - kind: ServiceAccount + name: pod-autoscaler + namespace: kube-system + apiGroup: "" +roleRef: + kind: ClusterRole + name: system-autoscaler:pod-autoscaler + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pod-autoscaler + namespace: kube-system +spec: + strategy: + type: Recreate + selector: + matchLabels: + app: pod-autoscaler + replicas: 1 + template: + metadata: + labels: + app: pod-autoscaler + spec: + nodeSelector: + node-role.kubernetes.io/master: "true" + automountServiceAccountToken: true + serviceAccountName: pod-autoscaler + containers: + - name: pod-autoscaler + image: systemautoscaler/pod-autoscaler:dev + imagePullPolicy: Always + resources: + limits: + cpu: 500m + memory: 500Mi + requests: + cpu: 500m + memory: 500Mi diff --git a/config/deploy/podscale-controller.yaml b/config/deploy/podscale-controller.yaml new file mode 100644 index 0000000..f77b10f --- /dev/null +++ b/config/deploy/podscale-controller.yaml @@ -0,0 +1,71 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: podscale-controller + namespace: kube-system +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system-autoscaler:podscale-controller +rules: + - apiGroups: [""] + resources: ["pods", "services", "nodes"] + verbs: ["update", "get", "watch", "list"] + - apiGroups: [""] + resources: ["events"] + verbs: ["*"] + - apiGroups: ["systemautoscaler.polimi.it"] + resources: ["servicelevelagreements"] + verbs: ["get", "watch", "list"] + - apiGroups: ["systemautoscaler.polimi.it"] + resources: ["podscales"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system-autoscaler:podscale-controller +subjects: + - kind: ServiceAccount + name: podscale-controller + namespace: kube-system + apiGroup: "" +roleRef: + kind: ClusterRole + name: system-autoscaler:podscale-controller + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podscale-controller + namespace: kube-system +spec: + strategy: + type: Recreate + selector: + matchLabels: + app: podscale-controller + replicas: 1 + template: + metadata: + labels: + app: podscale-controller + spec: + nodeSelector: + node-role.kubernetes.io/master: "true" + automountServiceAccountToken: true + serviceAccountName: podscale-controller + containers: + - name: podscale-controller + image: systemautoscaler/podscale-controller:dev + imagePullPolicy: Always + resources: + limits: + cpu: 200m + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi diff --git a/config/deploy/prime-numbers-function.yaml b/config/deploy/prime-numbers-function.yaml index ab61efd..c7e0a15 100644 --- a/config/deploy/prime-numbers-function.yaml +++ b/config/deploy/prime-numbers-function.yaml @@ -4,7 +4,7 @@ metadata: name: prime-numbers namespace: openfaas-fn spec: - image: systemautoscaler/prime-numbers:0.1.0 + image: systemautoscaler/prime-numbers:dev labels: com.openfaas.scale.factor: "20" com.openfaas.scale.max: "100" @@ -13,5 +13,51 @@ spec: edgeautoscaler.polimi.it/scheduler: edge-autoscaler name: prime-numbers readOnlyRootFilesystem: false + limits: + cpu: "200m" + memory: "1000Mi" requests: - memory: 1M + cpu: "200m" + memory: "1000Mi" +--- +apiVersion: v1 +kind: Service +metadata: + name: prime-numbers + labels: + name: prime-numbers + namespace: openfaas-fn +spec: + ports: + - port: 80 + targetPort: 8000 + selector: + edgeautoscaler.polimi.it/function-name: "prime-numbers" + edgeautoscaler.polimi.it/function-namespace: "openfaas-fn" + topologyKeys: + - "kubernetes.io/hostname" + - "*" + +--- +apiVersion: systemautoscaler.polimi.it/v1beta1 +kind: ServiceLevelAgreement +metadata: + name: prime-numbers + namespace: openfaas-fn +spec: + metric: + responseTime: 100m + defaultResources: + cpu: "300m" + memory: "768Mi" + minResources: + memory: 10Mi + cpu: 300m + maxResources: + memory: 2048Mi + cpu: 20000m + service: + container: prime-numbers + selector: + matchLabels: + name: prime-numbers \ No newline at end of file diff --git a/config/metric-db/002_init.sql b/config/metric-db/002_init.sql index f0596fd..54ca47f 100644 --- a/config/metric-db/002_init.sql +++ b/config/metric-db/002_init.sql @@ -63,3 +63,19 @@ CREATE TABLE IF NOT EXISTS proxy_metric SELECT create_hypertable('proxy_metric', 'timestamp', chunk_time_interval => INTERVAL '30 seconds'); SELECT add_dimension('proxy_metric', 'community', number_partitions => 4); SELECT add_dimension('proxy_metric', 'namespace', number_partitions => 4); + +CREATE TABLE IF NOT EXISTS pod_log +( + timestamp TIMESTAMP, + pod_name VARCHAR(50), + container_name VARCHAR(50), + pod_address VARCHAR(50), + node VARCHAR(50), + cpu DOUBLE PRECISION, + mem DOUBLE PRECISION, + response_time DOUBLE PRECISION, + PRIMARY KEY (timestamp, pod_name) + ); + +SELECT create_hypertable('pod_log', 'timestamp', chunk_time_interval => INTERVAL '30 seconds'); +SELECT add_dimension('pod_log', 'pod_name', number_partitions => 4); diff --git a/run_on_kind.md b/run_on_kind.md new file mode 100644 index 0000000..2b3451c --- /dev/null +++ b/run_on_kind.md @@ -0,0 +1,44 @@ +# Run NEPTUNE locally on KinD Cluster + +# Requirements +- Docker (tested with version 23.0.1, cgroup driver 'systemd' and version '2') +- Kind (tested with version 0.13.0, installed with 'go install sigs.k8s.io/kind@v0.13.0') +- Kubectl + +### Create KinD cluster + +Use a custom KinD configuration and image to enable in-place vertical scaling. + +```shell +kind create cluster --config config/cluster-conf/kind.conf --image systemautoscaler/kindest-node +``` + +### Install Custom Resources + +```shell +kubectl apply -f config/crd/bases +kubectl apply -f config/cluster-conf/openfaas-fn-namespace.yaml +kubectl apply -f config/openfaas/openfaas-function.yaml +``` + +### Install Permission and RBAC + +```shell +kubectl apply -f config/permissions +``` + +### Allow scheduling on kind-control-plane +```shell +kubectl taint node kind-control-plane node-role.kubernetes.io/master:NoSchedule- +kubectl label nodes kind-control-plane node-role.kubernetes.io/master=true --overwrite +``` + +### Install Controllers + +```shell +kubectl apply -f config/deploy +``` + +Everything should work except for delay-discovery. + +Enjoy! \ No newline at end of file