From f86b65e53b514bf6eebe901d29c084465152742f Mon Sep 17 00:00:00 2001 From: Simon Murray Date: Fri, 9 Aug 2024 14:00:55 +0100 Subject: [PATCH] Separate Openstack from Identity As previously commented, having openstack state in a controlled resource is a bad idea. This separates stuff out so that a shadow record is created to hold state, so the managed resource is not required to be updated. --- .../region.unikorn-cloud.org_identities.yaml | 34 --- ...unikorn-cloud.org_openstackidentities.yaml | 87 ++++++++ .../identity-controller/clusterrole.yaml | 11 +- .../identity-controller/deployment.yaml | 2 + .../region-controller/clusterrole.yaml | 7 + .../templates/region-controller/ingress.yaml | 5 + cmd/unikorn-region-controller/main.go | 7 - go.mod | 7 +- go.sum | 29 +-- pkg/apis/unikorn/v1alpha1/register.go | 1 + pkg/apis/unikorn/v1alpha1/types.go | 35 +++- .../unikorn/v1alpha1/zz_generated.deepcopy.go | 194 ++++++++++++------ pkg/client/client.go | 104 +++------- pkg/handler/handler.go | 63 ++++-- pkg/managers/identity/manager.go | 9 +- pkg/openapi/client.go | 155 ++++++++++++++ pkg/openapi/router.go | 57 +++++ pkg/openapi/schema.go | 180 ++++++++-------- pkg/openapi/server.spec.yaml | 15 ++ pkg/providers/openstack/provider.go | 180 +++++++++++----- .../managers/identity/provisioner.go | 33 +-- pkg/reaper/reaper.go | 151 -------------- pkg/server/options.go | 5 - pkg/server/server.go | 50 ++--- 24 files changed, 828 insertions(+), 593 deletions(-) create mode 100644 charts/region/crds/region.unikorn-cloud.org_openstackidentities.yaml delete mode 100644 pkg/reaper/reaper.go diff --git a/charts/region/crds/region.unikorn-cloud.org_identities.yaml b/charts/region/crds/region.unikorn-cloud.org_identities.yaml index 8dc394b..1fcae08 100644 --- a/charts/region/crds/region.unikorn-cloud.org_identities.yaml +++ b/charts/region/crds/region.unikorn-cloud.org_identities.yaml @@ -57,40 +57,6 @@ spec: spec: description: IdentitySpec stores any state necessary to manage identity. properties: - openstack: - description: OpenStack is populated when the provider type is set - to "openstack". - properties: - applicationCredentialID: - description: ApplicationCredentialID is the ID of the user's application - credential. - type: string - applicationCredentialSecret: - description: ApplicationCredentialSecret is the one-time secret - for the application credential. - type: string - cloud: - description: Cloud is the cloud name in the cloud config to use. - type: string - cloudConfig: - description: CloudConfig is a client compatible cloud configuration. - format: byte - type: string - password: - description: Password is the login for the user. - type: string - projectID: - description: ProjectID is the ID of the project created for the - identity. - type: string - serverGroupID: - description: ServerGroupID is the ID of the server group created - for the identity. - type: string - userID: - description: UserID is the ID of the user created for the identity. - type: string - type: object pause: description: Pause, if true, will inhibit reconciliation. type: boolean diff --git a/charts/region/crds/region.unikorn-cloud.org_openstackidentities.yaml b/charts/region/crds/region.unikorn-cloud.org_openstackidentities.yaml new file mode 100644 index 0000000..8f232cd --- /dev/null +++ b/charts/region/crds/region.unikorn-cloud.org_openstackidentities.yaml @@ -0,0 +1,87 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: openstackidentities.region.unikorn-cloud.org +spec: + group: region.unikorn-cloud.org + names: + categories: + - unikorn + kind: OpenstackIdentity + listKind: OpenstackIdentityList + plural: openstackidentities + singular: openstackidentity + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.provider + name: provider + type: string + - jsonPath: .metadata.creationTimestamp + name: age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: OpenstackIdentity has no controller, its a database record of + state. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + applicationCredentialID: + description: ApplicationCredentialID is the ID of the user's application + credential. + type: string + applicationCredentialSecret: + description: ApplicationCredentialSecret is the one-time secret for + the application credential. + type: string + cloud: + description: Cloud is the cloud name in the cloud config to use. + type: string + cloudConfig: + description: CloudConfig is a client compatible cloud configuration. + format: byte + type: string + password: + description: Password is the login for the user. + type: string + projectID: + description: ProjectID is the ID of the project created for the identity. + type: string + serverGroupID: + description: ServerGroupID is the ID of the server group created for + the identity. + type: string + userID: + description: UserID is the ID of the user created for the identity. + type: string + type: object + status: + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} diff --git a/charts/region/templates/identity-controller/clusterrole.yaml b/charts/region/templates/identity-controller/clusterrole.yaml index 916bee5..ab41885 100644 --- a/charts/region/templates/identity-controller/clusterrole.yaml +++ b/charts/region/templates/identity-controller/clusterrole.yaml @@ -21,6 +21,16 @@ rules: - identities/status verbs: - update +- apiGroups: + - region.unikorn-cloud.org + resources: + - openstackidentities + verbs: + - list + - watch + - create + - update + - delete - apiGroups: - "" resources: @@ -35,4 +45,3 @@ rules: verbs: - list - watch - diff --git a/charts/region/templates/identity-controller/deployment.yaml b/charts/region/templates/identity-controller/deployment.yaml index 505374a..3159847 100644 --- a/charts/region/templates/identity-controller/deployment.yaml +++ b/charts/region/templates/identity-controller/deployment.yaml @@ -17,6 +17,8 @@ spec: containers: - name: unikorn-identity-controller image: {{ include "unikorn.identityControllerImage" . }} + args: + {{- include "unikorn.otlp.flags" . | nindent 8 }} ports: - name: http containerPort: 6080 diff --git a/charts/region/templates/region-controller/clusterrole.yaml b/charts/region/templates/region-controller/clusterrole.yaml index 729b7f3..dbf86bc 100644 --- a/charts/region/templates/region-controller/clusterrole.yaml +++ b/charts/region/templates/region-controller/clusterrole.yaml @@ -22,6 +22,13 @@ rules: - watch - create - delete +- apiGroups: + - region.unikorn-cloud.org + resources: + - openstackidentities + verbs: + - list + - watch - apiGroups: - "" resources: diff --git a/charts/region/templates/region-controller/ingress.yaml b/charts/region/templates/region-controller/ingress.yaml index fb4f0de..16418c6 100644 --- a/charts/region/templates/region-controller/ingress.yaml +++ b/charts/region/templates/region-controller/ingress.yaml @@ -6,6 +6,11 @@ metadata: {{- include "unikorn.labels" . | nindent 4 }} annotations: {{- include "unikorn.ingress.clusterIssuer.annotations" . | nindent 4 }} + # mTLS is used to authenticate other Unikorn services + nginx.ingress.kubernetes.io/auth-tls-verify-client: optional + nginx.ingress.kubernetes.io/auth-tls-secret: cert-manager/unikorn-client-ca + nginx.ingress.kubernetes.io/auth-tls-verify-depth: "1" + nginx.ingress.kubernetes.io/auth-tls-pass-certificate-to-upstream: "true" {{- if (include "unikorn.ingress.externalDNS" .) }} external-dns.alpha.kubernetes.io/hostname: {{ include "unikorn.region.host" . }} {{- end }} diff --git a/cmd/unikorn-region-controller/main.go b/cmd/unikorn-region-controller/main.go index 3fb942e..e37ccad 100644 --- a/cmd/unikorn-region-controller/main.go +++ b/cmd/unikorn-region-controller/main.go @@ -32,7 +32,6 @@ import ( "github.com/unikorn-cloud/core/pkg/client" unikornv1 "github.com/unikorn-cloud/region/pkg/apis/unikorn/v1alpha1" "github.com/unikorn-cloud/region/pkg/constants" - "github.com/unikorn-cloud/region/pkg/reaper" "github.com/unikorn-cloud/region/pkg/server" "sigs.k8s.io/controller-runtime/pkg/log" @@ -78,12 +77,6 @@ func start() { return } - if err := reaper.New(client, s.Options.Namespace).Run(ctx); err != nil { - logger.Error(err, "failed to setup 'The Reaper'") - - return - } - // Register a signal handler to trigger a graceful shutdown. stop := make(chan os.Signal, 1) diff --git a/go.mod b/go.mod index 41eed32..f789201 100644 --- a/go.mod +++ b/go.mod @@ -10,10 +10,9 @@ require ( github.com/oapi-codegen/runtime v1.1.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 - github.com/unikorn-cloud/core v0.1.64 - github.com/unikorn-cloud/identity v0.2.29 + github.com/unikorn-cloud/core v0.1.66 + github.com/unikorn-cloud/identity v0.2.30 go.opentelemetry.io/otel v1.28.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/trace v1.28.0 k8s.io/api v0.30.2 @@ -32,7 +31,6 @@ require ( github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/go-jose/go-jose/v4 v4.0.2 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -69,6 +67,7 @@ require ( github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 // indirect go.opentelemetry.io/otel/metric v1.28.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect diff --git a/go.sum b/go.sum index eb4f2c2..ed75d2c 100644 --- a/go.sum +++ b/go.sum @@ -25,8 +25,6 @@ github.com/getkin/kin-openapi v0.126.0 h1:c2cSgLnAsS0xYfKsgt5oBV6MYRM/giU8/RtwUY github.com/getkin/kin-openapi v0.126.0/go.mod h1:7mONz8IwmSRg6RttPu6v8U/OJ+gr+J99qSFNjPGSQqw= github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= -github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k= -github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= github.com/go-jose/go-jose/v4 v4.0.2 h1:R3l3kkBds16bO7ZFAEEcofK0MkrAJt3jlJznWZG0nvk= github.com/go-jose/go-jose/v4 v4.0.2/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -133,17 +131,14 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= -github.com/unikorn-cloud/core v0.1.63 h1:Jl/xuoGRKESMXhS1+apcaS/1I776agTyT75BGz9AKBA= -github.com/unikorn-cloud/core v0.1.63/go.mod h1:JcUIQW3+oiZPUQmOlENw3OCi35IBxPKa+J4MbP3TO7k= -github.com/unikorn-cloud/core v0.1.64 h1:4GqACg1YOmUrDo5euXz0yi753Mzx1TBg7fioRKysb1g= -github.com/unikorn-cloud/core v0.1.64/go.mod h1:JcUIQW3+oiZPUQmOlENw3OCi35IBxPKa+J4MbP3TO7k= -github.com/unikorn-cloud/identity v0.2.29 h1:kKEJmh6tjjdvZWYdZhyRewG3aHf9wmWwG5C/kb+Rm9A= -github.com/unikorn-cloud/identity v0.2.29/go.mod h1:ujrL+6kRUrPIk4Z0Yc12A+FDy6L4b2Hgzz6oGZlKfGI= +github.com/unikorn-cloud/core v0.1.66 h1:5FbosJk2+XsOsvO+nxs7ei/q+5y+uu4PgKtn2W5MnnQ= +github.com/unikorn-cloud/core v0.1.66/go.mod h1:MhA9xeueMB7EJgdyF0rE7lFQSkuHEcoPen6iwt/QCBE= +github.com/unikorn-cloud/identity v0.2.30 h1:geai/ypq393HuVBAPfNk7Gopkig2WK3hNhsm2RuWcKw= +github.com/unikorn-cloud/identity v0.2.30/go.mod h1:GudlMY6lJrlTWlnvHQnpdohK4w79VmuEy2mgT9vkVUM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -174,7 +169,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20240707233637-46b078467d37 h1:uLDX+AfeFCct3a2C7uIWBKMJIR3CJMhcgfrUAqjRK6w= @@ -182,7 +176,6 @@ golang.org/x/exp v0.0.0-20240707233637-46b078467d37/go.mod h1:M4RDyNAINzryxdtnbR golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -190,8 +183,6 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= @@ -200,7 +191,6 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -210,16 +200,10 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -227,9 +211,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= @@ -239,7 +220,6 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -264,7 +244,6 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI= diff --git a/pkg/apis/unikorn/v1alpha1/register.go b/pkg/apis/unikorn/v1alpha1/register.go index 77de2bb..fde2a74 100644 --- a/pkg/apis/unikorn/v1alpha1/register.go +++ b/pkg/apis/unikorn/v1alpha1/register.go @@ -50,6 +50,7 @@ var ( func init() { SchemeBuilder.Register(&Region{}, &RegionList{}) SchemeBuilder.Register(&Identity{}, &IdentityList{}) + SchemeBuilder.Register(&OpenstackIdentity{}, &OpenstackIdentityList{}) SchemeBuilder.Register(&PhysicalNetwork{}, &PhysicalNetworkList{}) } diff --git a/pkg/apis/unikorn/v1alpha1/types.go b/pkg/apis/unikorn/v1alpha1/types.go index 6c97854..29d82d9 100644 --- a/pkg/apis/unikorn/v1alpha1/types.go +++ b/pkg/apis/unikorn/v1alpha1/types.go @@ -299,11 +299,35 @@ type IdentitySpec struct { Tags TagList `json:"tags,omitempty"` // Provider defines the provider type. Provider Provider `json:"provider"` - // OpenStack is populated when the provider type is set to "openstack". - OpenStack *IdentitySpecOpenStack `json:"openstack,omitempty"` } -type IdentitySpecOpenStack struct { +type IdentityStatus struct { + // Current service state of a cluster manager. + Conditions []unikornv1core.Condition `json:"conditions,omitempty"` +} + +// OpenstackIdentityList is a typed list of identities. +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type OpenstackIdentityList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []OpenstackIdentity `json:"items"` +} + +// OpenstackIdentity has no controller, its a database record of state. +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:resource:scope=Namespaced,categories=unikorn +// +kubebuilder:printcolumn:name="provider",type="string",JSONPath=".spec.provider" +// +kubebuilder:printcolumn:name="age",type="date",JSONPath=".metadata.creationTimestamp" +type OpenstackIdentity struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec OpenstackIdentitySpec `json:"spec"` + Status OpenstackIdentityStatus `json:"status,omitempty"` +} + +type OpenstackIdentitySpec struct { // CloudConfig is a client compatible cloud configuration. CloudConfig []byte `json:"cloudConfig,omitempty"` // Cloud is the cloud name in the cloud config to use. @@ -322,10 +346,7 @@ type IdentitySpecOpenStack struct { ServerGroupID *string `json:"serverGroupID,omitempty"` } -type IdentityStatus struct { - // Current service state of a cluster manager. - Conditions []unikornv1core.Condition `json:"conditions,omitempty"` -} +type OpenstackIdentityStatus struct{} // PhysicalNetworkList s a typed list of physical networks. // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/apis/unikorn/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/unikorn/v1alpha1/zz_generated.deepcopy.go index fdc3e78..4e23872 100644 --- a/pkg/apis/unikorn/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/unikorn/v1alpha1/zz_generated.deepcopy.go @@ -215,11 +215,6 @@ func (in *IdentitySpec) DeepCopyInto(out *IdentitySpec) { *out = make(TagList, len(*in)) copy(*out, *in) } - if in.OpenStack != nil { - in, out := &in.OpenStack, &out.OpenStack - *out = new(IdentitySpecOpenStack) - (*in).DeepCopyInto(*out) - } return } @@ -233,62 +228,6 @@ func (in *IdentitySpec) DeepCopy() *IdentitySpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *IdentitySpecOpenStack) DeepCopyInto(out *IdentitySpecOpenStack) { - *out = *in - if in.CloudConfig != nil { - in, out := &in.CloudConfig, &out.CloudConfig - *out = make([]byte, len(*in)) - copy(*out, *in) - } - if in.Cloud != nil { - in, out := &in.Cloud, &out.Cloud - *out = new(string) - **out = **in - } - if in.UserID != nil { - in, out := &in.UserID, &out.UserID - *out = new(string) - **out = **in - } - if in.Password != nil { - in, out := &in.Password, &out.Password - *out = new(string) - **out = **in - } - if in.ProjectID != nil { - in, out := &in.ProjectID, &out.ProjectID - *out = new(string) - **out = **in - } - if in.ApplicationCredentialID != nil { - in, out := &in.ApplicationCredentialID, &out.ApplicationCredentialID - *out = new(string) - **out = **in - } - if in.ApplicationCredentialSecret != nil { - in, out := &in.ApplicationCredentialSecret, &out.ApplicationCredentialSecret - *out = new(string) - **out = **in - } - if in.ServerGroupID != nil { - in, out := &in.ServerGroupID, &out.ServerGroupID - *out = new(string) - **out = **in - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IdentitySpecOpenStack. -func (in *IdentitySpecOpenStack) DeepCopy() *IdentitySpecOpenStack { - if in == nil { - return nil - } - out := new(IdentitySpecOpenStack) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IdentityStatus) DeepCopyInto(out *IdentityStatus) { *out = *in @@ -408,6 +347,139 @@ func (in *OpenstackFlavorsSpec) DeepCopy() *OpenstackFlavorsSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenstackIdentity) DeepCopyInto(out *OpenstackIdentity) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenstackIdentity. +func (in *OpenstackIdentity) DeepCopy() *OpenstackIdentity { + if in == nil { + return nil + } + out := new(OpenstackIdentity) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *OpenstackIdentity) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenstackIdentityList) DeepCopyInto(out *OpenstackIdentityList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]OpenstackIdentity, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenstackIdentityList. +func (in *OpenstackIdentityList) DeepCopy() *OpenstackIdentityList { + if in == nil { + return nil + } + out := new(OpenstackIdentityList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *OpenstackIdentityList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenstackIdentitySpec) DeepCopyInto(out *OpenstackIdentitySpec) { + *out = *in + if in.CloudConfig != nil { + in, out := &in.CloudConfig, &out.CloudConfig + *out = make([]byte, len(*in)) + copy(*out, *in) + } + if in.Cloud != nil { + in, out := &in.Cloud, &out.Cloud + *out = new(string) + **out = **in + } + if in.UserID != nil { + in, out := &in.UserID, &out.UserID + *out = new(string) + **out = **in + } + if in.Password != nil { + in, out := &in.Password, &out.Password + *out = new(string) + **out = **in + } + if in.ProjectID != nil { + in, out := &in.ProjectID, &out.ProjectID + *out = new(string) + **out = **in + } + if in.ApplicationCredentialID != nil { + in, out := &in.ApplicationCredentialID, &out.ApplicationCredentialID + *out = new(string) + **out = **in + } + if in.ApplicationCredentialSecret != nil { + in, out := &in.ApplicationCredentialSecret, &out.ApplicationCredentialSecret + *out = new(string) + **out = **in + } + if in.ServerGroupID != nil { + in, out := &in.ServerGroupID, &out.ServerGroupID + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenstackIdentitySpec. +func (in *OpenstackIdentitySpec) DeepCopy() *OpenstackIdentitySpec { + if in == nil { + return nil + } + out := new(OpenstackIdentitySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenstackIdentityStatus) DeepCopyInto(out *OpenstackIdentityStatus) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenstackIdentityStatus. +func (in *OpenstackIdentityStatus) DeepCopy() *OpenstackIdentityStatus { + if in == nil { + return nil + } + out := new(OpenstackIdentityStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OpenstackProviderNetworkSpec) DeepCopyInto(out *OpenstackProviderNetworkSpec) { *out = *in diff --git a/pkg/client/client.go b/pkg/client/client.go index 70cd864..48d2dd7 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -18,36 +18,24 @@ package client import ( "context" - "crypto/tls" - "crypto/x509" - "errors" - "fmt" "net/http" - "github.com/spf13/pflag" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/propagation" - "github.com/unikorn-cloud/core/pkg/authorization/accesstoken" + coreclient "github.com/unikorn-cloud/core/pkg/client" + "github.com/unikorn-cloud/identity/pkg/middleware/authorization" + "github.com/unikorn-cloud/identity/pkg/middleware/openapi/accesstoken" "github.com/unikorn-cloud/region/pkg/openapi" - corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" ) -var ( - // ErrFormatError is returned when a secret doesn't meet the specification. - ErrFormatError = errors.New("secret incorrectly formatted") -) +type Options = coreclient.HTTPOptions -type Options struct { - // host is the region host name. - host string - // caSecretNamespace tells us where to source the CA secret. - caSecretNamespace string - // caSecretName is the root CA secret of the region endpoint. - caSecretName string +// NewOptions must be used to create options for consistency. +func NewOptions() *Options { + return coreclient.NewHTTPOptions("region") } // Client wraps up the raw OpenAPI client with things to make it useable e.g. @@ -55,77 +43,28 @@ type Options struct { type Client struct { // client is a Kubenetes client. client client.Client - // namespace is the namespace the client is running in. - namespace string // options allows setting of option from the CLI options *Options -} - -// AddFlags adds the options to the CLI flags. -func (o *Options) AddFlags(f *pflag.FlagSet) { - f.StringVar(&o.host, "region-host", "", "Region endpoint URL.") - f.StringVar(&o.caSecretNamespace, "region-ca-secret-namespace", "", "Region endpoint CA certificate secret namespace.") - f.StringVar(&o.caSecretName, "region-ca-secret-name", "", "Region endpoint CA certificate secret.") + // clientOptions may be specified to inject client certificates etc. + clientOptions *coreclient.HTTPClientOptions } // New creates a new client. -func New(client client.Client, namespace string, options *Options) *Client { +func New(client client.Client, options *Options, clientOptions *coreclient.HTTPClientOptions) *Client { return &Client{ - client: client, - namespace: namespace, - options: options, - } -} - -// tlsClientConfig abstracts away private TLS CAs or self signed certificates. -func (c *Client) tlsClientConfig(ctx context.Context) (*tls.Config, error) { - if c.options.caSecretName == "" { - //nolint:nilnil - return nil, nil - } - - namespace := c.namespace - - if c.options.caSecretNamespace != "" { - namespace = c.options.caSecretNamespace - } - - secret := &corev1.Secret{} - - if err := c.client.Get(ctx, client.ObjectKey{Namespace: namespace, Name: c.options.caSecretName}, secret); err != nil { - return nil, err - } - - if secret.Type != corev1.SecretTypeTLS { - return nil, fmt.Errorf("%w: issuer CA not of type kubernetes.io/tls", ErrFormatError) + client: client, + options: options, + clientOptions: clientOptions, } - - cert, ok := secret.Data[corev1.TLSCertKey] - if !ok { - return nil, fmt.Errorf("%w: issuer CA missing tls.crt", ErrFormatError) - } - - certPool := x509.NewCertPool() - - if ok := certPool.AppendCertsFromPEM(cert); !ok { - return nil, fmt.Errorf("%w: failed to load region CA certificate", ErrFormatError) - } - - config := &tls.Config{ - RootCAs: certPool, - MinVersion: tls.VersionTLS13, - } - - return config, nil } -// httpClient returns a new http client that will transparently do oauth2 header +// HTTPClient returns a new http client that will transparently do oauth2 header // injection and refresh token updates. -func (c *Client) httpClient(ctx context.Context) (*http.Client, error) { +func (c *Client) HTTPClient(ctx context.Context) (*http.Client, error) { // Handle non-system CA certificates for the OIDC discovery protocol // and oauth2 token refresh. This will return nil if none is specified // and default to the system roots. - tlsClientConfig, err := c.tlsClientConfig(ctx) + tlsClientConfig, err := coreclient.TLSClientConfig(ctx, c.client, c.options, c.clientOptions) if err != nil { return nil, err } @@ -141,21 +80,26 @@ func (c *Client) httpClient(ctx context.Context) (*http.Client, error) { // accessTokenInjector implements OAuth2 bearer token authorization. func accessTokenInjector(ctx context.Context, req *http.Request) error { - req.Header.Set("Authorization", "bearer "+accesstoken.FromContext(ctx)) + accessToken, err := accesstoken.FromContext(ctx) + if err != nil { + return err + } + req.Header.Set("Authorization", "bearer "+accessToken) otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(req.Header)) + authorization.InjectClientCert(ctx, req.Header) return nil } // Client returns a new OpenAPI client that can be used to access the API. func (c *Client) Client(ctx context.Context) (*openapi.ClientWithResponses, error) { - httpClient, err := c.httpClient(ctx) + httpClient, err := c.HTTPClient(ctx) if err != nil { return nil, err } - client, err := openapi.NewClientWithResponses(c.options.host, openapi.WithHTTPClient(httpClient), openapi.WithRequestEditorFn(accessTokenInjector)) + client, err := openapi.NewClientWithResponses(c.options.Host(), openapi.WithHTTPClient(httpClient), openapi.WithRequestEditorFn(accessTokenInjector)) if err != nil { return nil, err } diff --git a/pkg/handler/handler.go b/pkg/handler/handler.go index 237776d..651bc42 100644 --- a/pkg/handler/handler.go +++ b/pkg/handler/handler.go @@ -27,12 +27,14 @@ import ( "slices" "time" + unikornv1core "github.com/unikorn-cloud/core/pkg/apis/unikorn/v1alpha1" coreconstants "github.com/unikorn-cloud/core/pkg/constants" coreapi "github.com/unikorn-cloud/core/pkg/openapi" "github.com/unikorn-cloud/core/pkg/server/conversion" "github.com/unikorn-cloud/core/pkg/server/errors" coreutil "github.com/unikorn-cloud/core/pkg/util" identityclient "github.com/unikorn-cloud/identity/pkg/client" + "github.com/unikorn-cloud/identity/pkg/middleware/authorization" identityapi "github.com/unikorn-cloud/identity/pkg/openapi" "github.com/unikorn-cloud/identity/pkg/rbac" unikornv1 "github.com/unikorn-cloud/region/pkg/apis/unikorn/v1alpha1" @@ -297,9 +299,15 @@ func convertTags(in unikornv1.TagList) openapi.TagList { return out } -func convertIdentity(in *unikornv1.Identity) *openapi.IdentityRead { +func (h *Handler) convertIdentity(ctx context.Context, in *unikornv1.Identity) *openapi.IdentityRead { + provisioningStatus := coreapi.ResourceProvisioningStatusUnknown + + if condition, err := in.StatusConditionRead(unikornv1core.ConditionAvailable); err == nil { + provisioningStatus = conversion.ConvertStatusCondition(condition) + } + out := &openapi.IdentityRead{ - Metadata: conversion.ProjectScopedResourceReadMetadata(in, coreapi.ResourceProvisioningStatusProvisioned), + Metadata: conversion.ProjectScopedResourceReadMetadata(in, provisioningStatus), Spec: openapi.IdentitySpec{ RegionId: in.Labels[constants.RegionLabel], }, @@ -313,32 +321,31 @@ func convertIdentity(in *unikornv1.Identity) *openapi.IdentityRead { case unikornv1.ProviderOpenstack: out.Spec.Type = openapi.Openstack - if in.Spec.OpenStack != nil { + var openstackIdentity unikornv1.OpenstackIdentity + + if err := h.client.Get(ctx, client.ObjectKey{Namespace: in.Namespace, Name: in.Name}, &openstackIdentity); err == nil { out.Spec.Openstack = &openapi.IdentitySpecOpenStack{ - Cloud: in.Spec.OpenStack.Cloud, - UserId: in.Spec.OpenStack.UserID, - ProjectId: in.Spec.OpenStack.ProjectID, + Cloud: openstackIdentity.Spec.Cloud, + UserId: openstackIdentity.Spec.UserID, + ProjectId: openstackIdentity.Spec.ProjectID, + ServerGroupId: openstackIdentity.Spec.ServerGroupID, } - if in.Spec.OpenStack.CloudConfig != nil { - cloudConfig := base64.URLEncoding.EncodeToString(in.Spec.OpenStack.CloudConfig) + if openstackIdentity.Spec.CloudConfig != nil { + cloudConfig := base64.URLEncoding.EncodeToString(openstackIdentity.Spec.CloudConfig) out.Spec.Openstack.CloudConfig = &cloudConfig } - - if in.Spec.OpenStack.ServerGroupID != nil { - out.Spec.Openstack.ServerGroupId = in.Spec.OpenStack.ServerGroupID - } } } return out } -func convertIdentityList(in unikornv1.IdentityList) openapi.IdentitiesRead { +func (h *Handler) convertIdentityList(ctx context.Context, in unikornv1.IdentityList) openapi.IdentitiesRead { out := make(openapi.IdentitiesRead, len(in.Items)) for i := range in.Items { - out[i] = *convertIdentity(&in.Items[i]) + out[i] = *h.convertIdentity(ctx, &in.Items[i]) } return out @@ -367,7 +374,7 @@ func (h *Handler) GetApiV1OrganizationsOrganizationIDIdentities(w http.ResponseW return cmp.Compare(a.Name, b.Name) }) - util.WriteJSONResponse(w, r, http.StatusOK, convertIdentityList(result)) + util.WriteJSONResponse(w, r, http.StatusOK, h.convertIdentityList(r.Context(), result)) } func generateTag(in openapi.Tag) unikornv1.Tag { @@ -418,8 +425,14 @@ func (h *Handler) PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitie return } + userinfo, err := authorization.UserinfoFromContext(r.Context()) + if err != nil { + errors.HandleError(w, r, errors.OAuth2ServerError("unable to get userinfo").WithError(err)) + return + } + identity := &unikornv1.Identity{ - ObjectMeta: conversion.NewObjectMetadata(&request.Metadata, h.namespace).WithOrganization(organizationID).WithProject(projectID).WithLabel(constants.RegionLabel, request.Spec.RegionId).Get(r.Context()), + ObjectMeta: conversion.NewObjectMetadata(&request.Metadata, h.namespace, userinfo.Sub).WithOrganization(organizationID).WithProject(projectID).WithLabel(constants.RegionLabel, request.Spec.RegionId).Get(), Spec: unikornv1.IdentitySpec{ Tags: generateTagList(request.Spec.Tags), Provider: region.Spec.Provider, @@ -431,8 +444,22 @@ func (h *Handler) PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitie return } - h.setCacheable(w) - util.WriteJSONResponse(w, r, http.StatusCreated, convertIdentity(identity)) + util.WriteJSONResponse(w, r, http.StatusCreated, h.convertIdentity(r.Context(), identity)) +} + +func (h *Handler) GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityID(w http.ResponseWriter, r *http.Request, organizationID openapi.OrganizationIDParameter, projectID openapi.ProjectIDParameter, identityID openapi.IdentityIDParameter) { + if err := rbac.AllowProjectScope(r.Context(), "identities", identityapi.Read, organizationID, projectID); err != nil { + errors.HandleError(w, r, err) + return + } + + identity, err := h.getIdentity(r.Context(), identityID) + if err != nil { + errors.HandleError(w, r, err) + return + } + + util.WriteJSONResponse(w, r, http.StatusOK, h.convertIdentity(r.Context(), identity)) } func (h *Handler) DeleteApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityID(w http.ResponseWriter, r *http.Request, organizationID openapi.OrganizationIDParameter, projectID openapi.ProjectIDParameter, identityID openapi.IdentityIDParameter) { diff --git a/pkg/managers/identity/manager.go b/pkg/managers/identity/manager.go index 04db442..b3e5bf2 100644 --- a/pkg/managers/identity/manager.go +++ b/pkg/managers/identity/manager.go @@ -43,9 +43,14 @@ func (*Factory) Metadata() (string, string, string) { return constants.Application, constants.Version, constants.Revision } +// Options returns any options to be added to the CLI flags and passed to the reconciler. +func (*Factory) Options() coremanager.ControllerOptions { + return nil +} + // Reconciler returns a new reconciler instance. -func (*Factory) Reconciler(options *options.Options, manager manager.Manager) reconcile.Reconciler { - return coremanager.NewReconciler(options, manager, identity.New) +func (*Factory) Reconciler(options *options.Options, controllerOptions coremanager.ControllerOptions, manager manager.Manager) reconcile.Reconciler { + return coremanager.NewReconciler(options, controllerOptions, manager, identity.New) } // RegisterWatches adds any watches that would trigger a reconcile. diff --git a/pkg/openapi/client.go b/pkg/openapi/client.go index 3fb0817..d0c6baa 100644 --- a/pkg/openapi/client.go +++ b/pkg/openapi/client.go @@ -101,6 +101,9 @@ type ClientInterface interface { // DeleteApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityID request DeleteApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityID(ctx context.Context, organizationID OrganizationIDParameter, projectID ProjectIDParameter, identityID IdentityIDParameter, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityID request + GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityID(ctx context.Context, organizationID OrganizationIDParameter, projectID ProjectIDParameter, identityID IdentityIDParameter, reqEditors ...RequestEditorFn) (*http.Response, error) + // PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworksWithBody request with any body PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworksWithBody(ctx context.Context, organizationID OrganizationIDParameter, projectID ProjectIDParameter, identityID IdentityIDParameter, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -167,6 +170,18 @@ func (c *Client) DeleteApiV1OrganizationsOrganizationIDProjectsProjectIDIdentiti return c.Client.Do(req) } +func (c *Client) GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityID(ctx context.Context, organizationID OrganizationIDParameter, projectID ProjectIDParameter, identityID IdentityIDParameter, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDRequest(c.Server, organizationID, projectID, identityID) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworksWithBody(ctx context.Context, organizationID OrganizationIDParameter, projectID ProjectIDParameter, identityID IdentityIDParameter, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewPostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworksRequestWithBody(c.Server, organizationID, projectID, identityID, contentType, body) if err != nil { @@ -375,6 +390,54 @@ func NewDeleteApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentit return req, nil } +// NewGetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDRequest generates requests for GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityID +func NewGetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDRequest(server string, organizationID OrganizationIDParameter, projectID ProjectIDParameter, identityID IdentityIDParameter) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "organizationID", runtime.ParamLocationPath, organizationID) + if err != nil { + return nil, err + } + + var pathParam1 string + + pathParam1, err = runtime.StyleParamWithLocation("simple", false, "projectID", runtime.ParamLocationPath, projectID) + if err != nil { + return nil, err + } + + var pathParam2 string + + pathParam2, err = runtime.StyleParamWithLocation("simple", false, "identityID", runtime.ParamLocationPath, identityID) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/api/v1/organizations/%s/projects/%s/identities/%s", pathParam0, pathParam1, pathParam2) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + // NewPostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworksRequest calls the generic PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworks builder with application/json body func NewPostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworksRequest(server string, organizationID OrganizationIDParameter, projectID ProjectIDParameter, identityID IdentityIDParameter, body PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworksJSONRequestBody) (*http.Request, error) { var bodyReader io.Reader @@ -647,6 +710,9 @@ type ClientWithResponsesInterface interface { // DeleteApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDWithResponse request DeleteApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDWithResponse(ctx context.Context, organizationID OrganizationIDParameter, projectID ProjectIDParameter, identityID IdentityIDParameter, reqEditors ...RequestEditorFn) (*DeleteApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDResponse, error) + // GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDWithResponse request + GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDWithResponse(ctx context.Context, organizationID OrganizationIDParameter, projectID ProjectIDParameter, identityID IdentityIDParameter, reqEditors ...RequestEditorFn) (*GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDResponse, error) + // PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworksWithBodyWithResponse request with any body PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworksWithBodyWithResponse(ctx context.Context, organizationID OrganizationIDParameter, projectID ProjectIDParameter, identityID IdentityIDParameter, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworksResponse, error) @@ -743,6 +809,32 @@ func (r DeleteApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentit return 0 } +type GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *IdentityResponse + JSON400 *externalRef0.BadRequestResponse + JSON401 *externalRef0.UnauthorizedResponse + JSON403 *externalRef0.ForbiddenResponse + JSON500 *externalRef0.InternalServerErrorResponse +} + +// Status returns HTTPResponse.Status +func (r GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworksResponse struct { Body []byte HTTPResponse *http.Response @@ -904,6 +996,15 @@ func (c *ClientWithResponses) DeleteApiV1OrganizationsOrganizationIDProjectsProj return ParseDeleteApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDResponse(rsp) } +// GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDWithResponse request returning *GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDResponse +func (c *ClientWithResponses) GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDWithResponse(ctx context.Context, organizationID OrganizationIDParameter, projectID ProjectIDParameter, identityID IdentityIDParameter, reqEditors ...RequestEditorFn) (*GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDResponse, error) { + rsp, err := c.GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityID(ctx, organizationID, projectID, identityID, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDResponse(rsp) +} + // PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworksWithBodyWithResponse request with arbitrary body returning *PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworksResponse func (c *ClientWithResponses) PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworksWithBodyWithResponse(ctx context.Context, organizationID OrganizationIDParameter, projectID ProjectIDParameter, identityID IdentityIDParameter, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworksResponse, error) { rsp, err := c.PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworksWithBody(ctx, organizationID, projectID, identityID, contentType, body, reqEditors...) @@ -1119,6 +1220,60 @@ func ParseDeleteApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdent return response, nil } +// ParseGetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDResponse parses an HTTP response from a GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDWithResponse call +func ParseGetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDResponse(rsp *http.Response) (*GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest IdentityResponse + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest externalRef0.BadRequestResponse + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401: + var dest externalRef0.UnauthorizedResponse + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON401 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 403: + var dest externalRef0.ForbiddenResponse + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON403 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest externalRef0.InternalServerErrorResponse + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + // ParsePostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworksResponse parses an HTTP response from a PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworksWithResponse call func ParsePostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworksResponse(rsp *http.Response) (*PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworksResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) diff --git a/pkg/openapi/router.go b/pkg/openapi/router.go index dd68832..1a03c51 100644 --- a/pkg/openapi/router.go +++ b/pkg/openapi/router.go @@ -24,6 +24,9 @@ type ServerInterface interface { // (DELETE /api/v1/organizations/{organizationID}/projects/{projectID}/identities/{identityID}) DeleteApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityID(w http.ResponseWriter, r *http.Request, organizationID OrganizationIDParameter, projectID ProjectIDParameter, identityID IdentityIDParameter) + // (GET /api/v1/organizations/{organizationID}/projects/{projectID}/identities/{identityID}) + GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityID(w http.ResponseWriter, r *http.Request, organizationID OrganizationIDParameter, projectID ProjectIDParameter, identityID IdentityIDParameter) + // (POST /api/v1/organizations/{organizationID}/projects/{projectID}/identities/{identityID}/physicalNetworks) PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworks(w http.ResponseWriter, r *http.Request, organizationID OrganizationIDParameter, projectID ProjectIDParameter, identityID IdentityIDParameter) @@ -59,6 +62,11 @@ func (_ Unimplemented) DeleteApiV1OrganizationsOrganizationIDProjectsProjectIDId w.WriteHeader(http.StatusNotImplemented) } +// (GET /api/v1/organizations/{organizationID}/projects/{projectID}/identities/{identityID}) +func (_ Unimplemented) GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityID(w http.ResponseWriter, r *http.Request, organizationID OrganizationIDParameter, projectID ProjectIDParameter, identityID IdentityIDParameter) { + w.WriteHeader(http.StatusNotImplemented) +} + // (POST /api/v1/organizations/{organizationID}/projects/{projectID}/identities/{identityID}/physicalNetworks) func (_ Unimplemented) PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworks(w http.ResponseWriter, r *http.Request, organizationID OrganizationIDParameter, projectID ProjectIDParameter, identityID IdentityIDParameter) { w.WriteHeader(http.StatusNotImplemented) @@ -204,6 +212,52 @@ func (siw *ServerInterfaceWrapper) DeleteApiV1OrganizationsOrganizationIDProject handler.ServeHTTP(w, r.WithContext(ctx)) } +// GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityID operation middleware +func (siw *ServerInterfaceWrapper) GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityID(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + var err error + + // ------------- Path parameter "organizationID" ------------- + var organizationID OrganizationIDParameter + + err = runtime.BindStyledParameterWithLocation("simple", false, "organizationID", runtime.ParamLocationPath, chi.URLParam(r, "organizationID"), &organizationID) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "organizationID", Err: err}) + return + } + + // ------------- Path parameter "projectID" ------------- + var projectID ProjectIDParameter + + err = runtime.BindStyledParameterWithLocation("simple", false, "projectID", runtime.ParamLocationPath, chi.URLParam(r, "projectID"), &projectID) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "projectID", Err: err}) + return + } + + // ------------- Path parameter "identityID" ------------- + var identityID IdentityIDParameter + + err = runtime.BindStyledParameterWithLocation("simple", false, "identityID", runtime.ParamLocationPath, chi.URLParam(r, "identityID"), &identityID) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "identityID", Err: err}) + return + } + + ctx = context.WithValue(ctx, Oauth2AuthenticationScopes, []string{}) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityID(w, r, organizationID, projectID, identityID) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r.WithContext(ctx)) +} + // PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworks operation middleware func (siw *ServerInterfaceWrapper) PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworks(w http.ResponseWriter, r *http.Request) { ctx := r.Context() @@ -511,6 +565,9 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl r.Group(func(r chi.Router) { r.Delete(options.BaseURL+"/api/v1/organizations/{organizationID}/projects/{projectID}/identities/{identityID}", wrapper.DeleteApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityID) }) + r.Group(func(r chi.Router) { + r.Get(options.BaseURL+"/api/v1/organizations/{organizationID}/projects/{projectID}/identities/{identityID}", wrapper.GetApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityID) + }) r.Group(func(r chi.Router) { r.Post(options.BaseURL+"/api/v1/organizations/{organizationID}/projects/{projectID}/identities/{identityID}/physicalNetworks", wrapper.PostApiV1OrganizationsOrganizationIDProjectsProjectIDIdentitiesIdentityIDPhysicalNetworks) }) diff --git a/pkg/openapi/schema.go b/pkg/openapi/schema.go index da86fa9..1eb12f9 100644 --- a/pkg/openapi/schema.go +++ b/pkg/openapi/schema.go @@ -19,96 +19,96 @@ import ( // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x8e3PbRpL4V5nCb6uyWz+A4kuUyH/2FDtxVHFsnS07dxv6XAOgQUwEzGBnBpQZlb77", - "1TzwBkjqkeT2zlVJWSTn0d3T7+6ZOydgacYoUCmc1Z2TYY5TkMD1JxIClUTuLl9eFd+rr0MQASeZJIw6", - "K+c6BlQMtH9EBPjIcR2ifs+wjB3XoTgFZ1Vb0nEdDv/MCYfQWUmeg+uIIIYUqy3+wiFyVs7/O6nAOzG/", - "ipOb3AdOQYJ4g1OoILu/dx3GN5iS37CCbS/UFxTVx6LLlwMAN1fcC7TcZWqGkJzQjQYn4+xXCORB+tlx", - "SO05AEe51O9CNw6bQxRTcJphh0+5WO53gPXeLAlCfstCAg0+fWd+UF8FjEqg+k+cZQkJ9AGe/CoULncO", - "fMFploD6MwWJQyxxD4+gLXCfCUD17zus7OnP964jMgjUKhb70Fk5wdnp4hymoRctse/NT2eht8Qz7J1O", - "Zmen0dn5fLrwHdeReCOc1S93xdJBkgsJ3COh4zpbnOTqy+VsMZmPp4EXLZfn3nwZBB72pxNv6fvLJY6C", - "KIRz5/6TotBxRC4Q+JkTCYa0bQJYUqOIcYRpKeejzsEqZo93ggQ4eQPylvGbP/Ywis09anbvHMqfQ+IW", - "TY6lNCrmIYvOyEqpyBgVhulxEEAmIXxnvxySWLNsjAXyASgqpiFMQ3RLkgT5gKI8iUiSqG/FjgYxZ5Tl", - "ItmN1vQ/WY5SvEMZSxIk9YqC5TwAvUDKKJGMIyIFEhLLXGgEFCUSUGCM1Bn4OLTcUAf2eK4AzhlXEke3", - "OCHhZ4uU45pfPjfRLlD2WbhDdopz9ImZvXqO6F192QgTRS0zCektNPQuYtxSyYwOGQhEmUQKW0zomuKS", - "jkZ+UEQgCYUmFHyRwGnJLuIx5PpF6UPF4LNoOT2bLLxJFAbe3D/zveV4Ad48gvHkdB5GQRhV8hMx5tx/", - "OppILTj7WTohQiIWGfKgYk7B0gbjKMFbxh+LaF1dBBz0wGuiEZosz8beeOKNJ9fj8Ur/9w9lrBRplvg8", - "WMzOxt58vDj15uEce8sQj72zxdl5GM3HQbgMK9JsRvNRTDZxCukIT8bj0WQzmow3fl27BFn+PU5JsnNW", - "ziWVkKD/AEbRVYIloXmKzieL8TX66/ubXYJv4G+Oq2YIZzV3nZCIG2c1HbvOJssN/rnCfuI6KaSM75zV", - "ZDl1nZSFkDgr54fJeKxUFtBQC8Wbj5cvLy8UMMXw2fT++KO0B7D/BO0gc2KM+yQMgT5NlstlBqQ4F8BR", - "wEFbHJwIFDItRzHeQlN+Mk62JIENiGeU8lssUAiUQIj8HcK5jBknwsq4jInQStEHFOBcmEEKqMbANZXs", - "BmgBNqGbJuAiYBkUpvXi6rJUHhp3pTnoNxXCa0ohACEw39VQRozqKRlnWxICR1mCZcR4qs/K2msCzyZg", - "EH6rePxXFtNRyODfcJDCKGCp4uimAE7H07k3PvVmk+vJfDWZ1AUQL+bRcrpYerMFjL35bDL1/PNw4p1O", - "w+UsPF0s/TO/EsCcKhI7Le/+AYJc+M5qCswWwfj0HHvn4GNvHp363nISzb1oEUX+8nx2tjwNzJQtEYRR", - "QjfvtWEzPrj5EsK68LMMqJA4uNFUSliu9gkhwnmibJT+5gWjEdmo71/FWbD7Vv0fX/7wLglm//5jG0R/", - "GSwVJc7mi3k4mfvR+RmcjiN8Nl3MzscKI8UheiyeLBdn53h6Ppku5suz0MfTuX86D5YLPF7MI+xUzr2G", - "6nw5Cf1o7I3xeOLNIQo8DMofDc/OokU4m0/n2h81kUyF2AMUSp3ncLhfr9ixIOrcunucYvnKql9Z1bDq", - "Q+OfQT6tAh5UuN6GUVO8gd/BZ5mOpzNvPPWm0+vJdDWeryazx/Khn0+n47m3nYymp6OFt8ly73R6Ojo/", - "HY1PvbMAwvnkdF7nDOt8hJxsVfTvlKMd63qo6Mm5MM6H9UF+mI7HzqdeX0SwSN5iDh+BKy7UEUsV1Dsr", - "x0Kmxm4JlzlOrLSo34ovFPM+QPPoYzmgcfQYJGMsEeagIxUsiZ8AuiUyNqa9aUOp8VvfA98C/045Dk/z", - "fIRe6LP52O/82PBCMmQ8iCDBJH0G7+aCopzClwwCFQHqYYgFQc45hE23BjdGSo6pIEClnYNpuKZqpMiD", - "ACBUXghGHCTfjdBlZFYi2n1RzkmABbgoSwAL5f5kjEtEJMJC5xSEyI1YUSa/ZzkNn0ZeyuTnSC0zQNta", - "dAZhFc2WgRp8IUI+A60/UKy4SjIUERpq8pitNK6dZMlXk/esJu+xGZphn6UnJVPYq9/NFJzp8xmv5qer", - "+ak6n24S+8suZZxREiBJgHszpBYMQCkt5GMVmhCKXiulnjGWjIozPjIxWZzxjXdrEi4PcTYiwDLnJlvV", - "IrMos4ZPcTUt9fdrfDtIH1dObYD2GzxRy+BAxWKfTYg4oGnUXsp9MKvZnNFzaPG+dYvY0QBm7UaMBYIv", - "mYo2RzWZEDVM2knDV0CBk8Aq+lQFnBtwO3aSKeSmI8MRGXBpU/EDq14gCVyAXdVUSBRkmIbqLxvE/nB9", - "fWWHBCyEEdLWVmhDbXjZDnyrSDBFitFIZOngIj83Nt2sC6GBVMHHCUgVN9sUpVrcJCovri4FYjIGRTys", - "FmcCinVNWG/2UpgCzVPlBHXTkHW++hwkylI6bodHciryTBk/UHMN933W/O+Wa+qcgOO2nQQJacY45iTZ", - "fc4p3mKSKOtSm1juWnyx4ZjK1q76u2LLuqEMGI0SEqjxKciYhZ/VrzhJ2G0H9BRCgotFqjTOJ7ddAOuV", - "ijZnfLRZfctpNrvvF8kSvcLIcXuKa1UB4hdn2JmqwGK+MjM9adbe6uDbQiF1EpddplcatT/5bp0LU13s", - "kMdo170zi4rgfvRJma48Al3RJ56FtmSDaAttOySk4oHZYafS8phzvKvSvn2AmF+6NK7byn2bKxEnwTtL", - "v5+KWTWzdDgb+l6NbNO4BMCu1Efp2vQHoOZjDmr1pDvp5xiscgI7HRFha0QhhEr1QYhSHMSE1vnEZywB", - "TBVMtdR0D0gcdAYyRS+uPqBIj6sX2BCMNiOk4z9E89QH7iLMg5hICJRt72Vrk9ruY2uzhOKzF1cfRG2y", - "CrI2wNVskw/vm41TllPNpZDFkALHCVKjlYfz6tv+1WxYu+/MN1luDrzKoO/f3YzSu5LebVuco+lRLm4x", - "HGafvfJZZuKPlEUraD0iuMnyn0xJobvbq6sPjUPvPeZigdfEVHmHQG4vdjzwJYj94PfLmdqu4Rh0Bc5W", - "WPZz6KurDwKVlrafu4b4RaN8iEvKks4e+vcSvki3HCTeRzOwzY92frF/jTMNYfpYs1qtl2wKYLNs3VGy", - "6SDXufjpZa9/0MoV72Gisr5RHC2q5h7NT82EX5enGr/3ADMIxO7x9spGv++V3xcWZksB8FDTVYDyaOPV", - "WOBB2LvoNiaJKcsZPxQFmJqzs+EXkgwRGhlTtqZqcxfdAgoZ/UYWJTFhEkeYhoiDzDlFRBZpOahSsAhd", - "x9hsoQKhNfV1yUxHunqWZCgECTwlFBRoQdwF3sROkiEV3Fnj2TzBRq78WMort/G9iWPr2ew9zVS1thpl", - "22uxNCK0V/5NK8t+mCTevLZ5LDP/mEj6Wo1ss4519EtcDrFORYEO1t9tge9krHx7bDxsPbDgGQoQakaJ", - "choMaG5TsujV3DgFpSkME7LcsEP5IdBVjX5/pV716PK9ijwXcwRUhY1hYzkUkaTfB6oltdorXtlmv6qL", - "DqlAy7hykQ3jCY04FpLnw26WCRFfcZZnfduYfDXaqN8P7SUP7VUUcdqbfBDAS00cPQ6T+z0MZRqm+uKz", - "WmVGR+GP18BFrKX3eqzW1ZOfrHqrVTooF40JVYeu6bTSaUMjUMPG6H+CImqRZb8+SfFm4NjVL39WdKg3", - "f/whq9mvbJmt14MyxTdEIkSU5UkSCLuoFhW6A4tsTeHNLWox1j0r7V/vgRaFvuPc8sLAPJs/alEbJN6A", - "ZzLIFUcEf+WhDNQs9zJTe3xPLfOI3T82p3So0/x5kDgfOzu3UwlYIjVVh7AmY2AUtJpt0wp1771ehXVr", - "SQrXwXTX786bAuweN/6B5ddj/XqtLnoc+qri/Br7kHw0bb49rcW6Q/HH3Ac9GCVqNNJdwa6iGQlwkuyM", - "k6m0biM9Z9FR7qgPa0poCF+g9D6UVlAehOZOLCVwteV//TL2lhfeP7D326e//n1VffI+jz7djd3F5L42", - "4m9//0uftA41yfcg+GM51GQU0U+5kLrOa3F/+eZ90aJqEvLJDiXsFrgu3qIgxhwHyuq4RRSPGEfxLouB", - "ChcJibnUjjtQm3fH1SQ1tMwe0VDvK1HKhESLWW1tRbME6EbGilop/vJaf3BWi5nrpIQWHyc9xKhXH/cE", - "U6s7ByfJ20jX347xClqh2F07SGgVPfu0cuOiSc0Ta3RT+5AwulHu7+F0b2vTrkr41FdhHohpO2XNPz2S", - "bUG+3+D26cO+BfpRr2P+jTiUN3qov3MIsCHvtnMof7KT+/QDOcgaR8vlkXLeldQ9EVn9CtZzyGe1Vb9o", - "Gt/3+1p5vM0AAky21w6xoGg3XV/GMNkMFSub2Fl0OaNbcW9vcxmpfQoPkZpqgrVmtQ1t5U90uFKYPiBG", - "AYmY5YlW7PWwQQco5p6Njutzm7xRPkeWkIAYlRgDV/7HmvZtqmynpx2PwmkRxojIGASg1Nqx2rYKovoV", - "mo+vL94gkMFoTXsKJO2ja1Otj50NjEMa1fz6dFF9jOo0ez86RKlN73ILVT6ZOTHss7zOI11k670fhwEu", - "ReHZ0lXl/sNoXtuNhgLib0SVNVRL1H3jKjfY5wcXfSl7HOGyK+VIF7fGcj1+Lm8HtD3pA6vHCjawfp8u", - "JSYJuri6rHQdBxyaNOwtN93ZnSB0X0m/UcCu/WQ1DdMftEON802q0NSspBN42nVMmQ4NqIQvcm/h/Lgr", - "pDXfv80tpmZeo+BVT3/TgLkox+m2Eh1W1Zv8KmbJ6Q1lt7TVPVX/qCOsEFo/m6aGfgZ7igkdTILcdU7Z", - "XOIzPWl9ZJAkhaadNBdnEpAmd2F0hrNyQizBU8MHsqU9VD9GSfacV481bg/pMcvuAwVGy8iofhhN1+qr", - "BD5QAgWk2/7gVUCKqSRBkdNqhdLb9Tr8/+v1qPZPb7jcl95pmTidCck4lOm3Ysvy3+LwugdS724/IH8G", - "094IYUA2Hxy27pHqWqdwH9/oy2e3MUN2XEO8++snjdbV49WE3eB4NTHUZ5VT8s/8cLtVykLd1XcQ8zwL", - "j8O8WPEA5riJt13+WLz7er0aJD9Cm13r1v5C8dgWogIo6/L/qlxp3f1uHOxGaXZNMd01rZ4aEwNOZGz7", - "Kk0Hpg8UIiJRxFmKsPqJhlh3Rq5pCYHBu+GRVzIg8aY3y4u5TyTHfIck3hhlpWDQabquPPY3110UzFIs", - "0d9h0Z8oVAeqfyrqjBJvDkeDGpBizU/9+B7qoZF4c7yvqOjXcRK1hg1yTuTuvRpn01e6jbfZUNyF420G", - "3Dj+ZaHeduD6gLlyj3W3cbPfWbN3wm7NYwW2PVb/8oKF0PnyA0+clRNLmYnVSVlWG+WU3DBOPV11HTG+", - "OTEgn2ynJ435KqwJWKbRUsgriB6xpp7XUM36J9OGTWjEutR5oQvCNm4NiQjYFvjOdDGwXBfnBPAtsTqE", - "yEStW8vIvjNT35tByhHQ97u1wXFWzng0GU10ijMDijPirJzZaDyaGSsYa/qe4IycbCeN7Ig4uWs+XnNf", - "u5vZReMnTPEGwqoGaIEWI4Quy3m1SF8Qukm01jRdgrj4xob8pohFAxitqdY/CUmJiugTLCTiOCS5KLLk", - "sAXTW4drd75RAvhGX4omFAmWmotEAuEtI6FAfr5R89e06YtbK69ovQHZ1+Autb9V3j0197D1bSvcfBVI", - "rcEK3tf3U16BvMjIx8nbOp3fNqhc0cppvdgxHY+HRLccd9JzZfvedebHTO15ZkNPnRye2nsxQk+eHZ7c", - "fRHg3nVOj0J2z926usbSbk+/rvrlk8l21x6uGnCRqiEnQ89E6aWOlCWb5BMnd+XTTP/nBOyZqO4enNrz", - "kJZycjLWZzZfaL8SYUThttbSQFsJq6ZkXzFxULRt5464KqBpyXrxLNVumPNrL1edtJ+tuu/oi8nR+mL3", - "VVscrS2eTcZP7qrn7O7LhEmP4/hSf9/osFHugXKpKyccC8ECogMPHYMT2eVSs9AT+PSy+f5eg9umh4+g", - "8+jUvyS3zcfzwzM7t5H/eKP21X78nvbj8Ky+9y+f0UVoqI+Tw/U6wwA9dbgRQlfd2hyH6oMO8O3V/urh", - "McbXtOzrQZSFrby25ZqPry/ejBB6wySYhXR3SMlNZZGkKFYTgfSTAlQmu3V1cRBlVa/ozkVY1Fr4NbRK", - "cPS9Td2VrSJfNSMjEOiAu9tI+i/JPkd5LW2aPru/UtmBqzbfPcKTGXj68VEOzdDLCF/9mufzawYLlO/s", - "bY0QIkIVM5ocCULXtbdBNsA2HGex1jf6NZAdSthGf8wwVwzG6GhNvyP67uIt3pVNqOapMmVmyNYqEyJM", - "X50KvwsPqEpVijyIERZr2tg0YQFOwK3idfPg2jdCuVOKiiHyE+YrraEonkuwDQDf4SAuUjWx0kBSIHZL", - "K3nrOmGuzpXaJwCqSwuuaX4oFrANG/UH6wRD+q6DsK2B9TxDdYdGJMSoN7ymIsa8vGEgY87yTYxuYyxh", - "CxylEMQK1VSRrLyYZq7EY2lnFYgMpj9eK71qClhlBfrBOQ7LJo9KcLSfzHiqcP6vTzRYgp3cFa8q35c3", - "rOnwje4kYbeieg0CrZ3Ohe61o1m7YBnrJVh7rUQ1Ha3pz/pm14uLq7eajcs7XJ374UqWIIlcRCQKOM4E", - "YrlE3ppioe14LnKcIA+RyFQX9XsLjNrO45yGLrrlOLgpJY8qjLQvov3TXKBbQEKSJNE3hRRSMaZhAsVb", - "QkaocIIEZbdRgm8OJAHL7HrvVffHCsU7e0rftc/oMcIy+CDs1+DpDxLUw25e9+30J0r34DXwF9aW2ZcA", - "yvhvn64XWtkHjZm1Hr9azQZCUwFVdrHUG88gCN9bdB7D/+3ngf9EJ/Ar+x7JvkO3QAruNfdNHsG89csj", - "x/Duc2jxS4PMoyo5zVciv7LuH8O69/f/HQAA//8tzMno1WQAAA==", + "H4sIAAAAAAAC/+x8+3PbOJL/v4Lid6tmt76irLct/bLnSWYyrs0kvsTJ3u0olwLJpogxCXABUI7G5f/9", + "Cg++SUmWPY+9TdVMxZLw6kZ349MP4N7xWZIyClQKZ3XvpJjjBCRw/YkEQCWRu6uX1/n36usAhM9JKgmj", + "zsq5iQDlDe0fIQE+dAYOUb+nWEbOwKE4AWdVGdIZOBz+mREOgbOSPIOBI/wIEqym+BOH0Fk5/++sXN6Z", + "+VWc3WYecAoSxBucQLmyh4eBw/gGU/ILVmvbu+pLiqpt0dXLngXXR9y7aLlLVQ8hOaEbvZyUs5/Blwf5", + "Z9shNWfPOoqhfhW+cdgc4phap2l2eJfz4X6FtT6YIUHIb1lAoCan78wP6iufUQlU/4nTNCa+3sCzn4Wi", + "5d6BLzhJY1B/JiBxgCXukBG0Be4xAaj6fUuUXf35YeCIFHw1iqU+cFaOfz5fXMAkcMMl9tzZfBq4SzzF", + "7nw8PZ+H5xezycJzBo7EG+GsfrrPh/bjTEjgLgmcgbPFcaa+XE4X49lo4rvhcnnhzpa+72JvMnaXnrdc", + "4tAPA7hwHj4pDh3H5JyAv3MiwbC2yQDLahQyjjAt9HzY2lgl7NFOEB/Hb0DeMX77225GPrlLzeytTfl9", + "WNzgybGcRnk/ZMkZWi0VKaPCCD32fUglBO/sl30aa4aNsEAeAEV5N4RpgO5IHCMPUJjFIYlj9a3YUT/i", + "jLJMxLvhmv43y1CCdyhlcYykHlGwjPugB0gYJZJxRKRAQmKZCU2A4kQMahlDtQceDqw0VBd7vFQA54wr", + "jaNbHJPgsyXKGZhfPtfJzkn2WLBDtotz9I6ZuTq26F112BATxS3TCekp9OoHiHHLJdM6YCAQZRIpajGh", + "a4oLPhr9QSGBOBCaUfBFAqeFuIhT2PWTsodKwKfhcnI+XrjjMPDdmXfuucvRAtxZCKPxfBaEfhCW+hMy", + "5jx8OppJjXV2i3RMhEQsNOxBeZ9cpA3FYYy3jJ9KaNVc+Bx0wxuiCRovz0fuaOyOxjej0Ur/9w91WCnW", + "LPGFv5iej9zZaDF3Z8EMu8sAj9zzxflFEM5GfrAMStZshrNhRDZRAskQj0ej4XgzHI82XtW6+Gn2PU5I", + "vHNWzhWVEKP/AkbRdYwloVmCLsaL0Q368/vbXYxv4S/OQPUQzmo2cAIibp3VZDRwNmlm6M8U9eOBk0DC", + "+M5ZjZeTgZOwAGJn5fwwHo2UyQIaaKV48/Hq5dWlWkzefDp5OH4r7Qbs30HbyOwY4x4JAqBP0+VimB4t", + "zgRw5HPQJw6OBQqY1qMIb6GuPyknWxLDBsQzavkdFigASiBA3g7hTEaME2F1XEZEaKPoAfJxJkwjtaha", + "wzWV7BZovmxCN/WFC5+lkB+tl9dXhfHQtCvLQb8pCV5TCj4IgfmuQjJiVHdJOduSADhKYyxDxhO9V/a8", + "JvBsCgbBt0rGf2YRHQYM/gP7CQx9liiJrivgZDSZuaO5Ox3fjGer8biqgHgxC5eTxdKdLmDkzqbjietd", + "BGN3PgmW02C+WHrnXqmAGVUsdhro/hGKnGNn1QWmC380v8DuBXjYnYVzz12Ow5kbLsLQW15Mz5dz33TZ", + "EkEYJXTzXh9sBoObLyGoKj9LgQqJ/VvNpZhlap4AQpzF6ozS37xgNCQb9f2rKPV336r/o6sf3sX+9D//", + "1lyit/SXihPns8UsGM+88OIc5qMQn08W04uRokhJiG6Lx8vF+QWeXIwni9nyPPDwZObNZ/5ygUeLWYid", + "EtzrVV0sx4EXjtwRHo3dGYS+i0Hh0eD8PFwE09lkpvGo8WRKwh5hUKoyh4P9dsW2BVGV1t1phuWrqH4V", + "VSOqj/V/euW0dHhQDr2NoCZ4A78CZpmMJlN3NHEnk5vxZDWarcbTU+XQyyaT0czdjoeT+XDhbtLMnU/m", + "w4v5cDR3z30IZuP5rCoZFnwEnGyV9+8UrR0LPZT35Fwa8GExyA+T0cj51IlFBAvlHebwEbiSQu2xlE69", + "s3LsylTbLeEyw7HVFvVb/oUS3kdYHr0tByyOboNkhCXCHLSngiXxYkB3REbmaK+fodTg1vfAt8C/U8Dh", + "achH6IE+m4/d4Me6F5IhgyD8GJPkGdDNJUUZhS8p+MoD1M0Q8/2McwjqsAbXWkqOqSBApe2DabCmqqXI", + "fB8gUCgEIw6S74boKjQjEQ1fFDjxsYABSmPAQsGflHGJiERY6JiCEJlRK8rk9yyjwdPYS5n8HKphenhb", + "8c4gKL3ZwlGDL0TIZ+D1B4qVVEmGQkIDzR4zlaa1FSz5euQ965F3aoSmH7N0hGTy8+pXOwrO9f6MVrP5", + "ajZX+9MOYn/ZJYwzSnwkCXB3itSAPiijhTysXBNC0Wtl1FPG4mG+x0cGJvM9vnXvTMDlMWAjBCwzbqJV", + "DTaLImr4FKhpub/f4ttGersyah20X+CJVgb7yhf7bFzEHkuj5lLwwYxmY0bPYcW7xs19R7Mwe25EWCD4", + "kipvc1jRCVGhpBk0fAUUOPGtoU+Uw7mBQeucZIq4ydBIRApc2lB8z6iXSAIXYEc1GRK1MkwD9Zd1Yn+4", + "ubm2TXwWwBDp01bog9rIsm34VrFggpSgkdDyYYC8zJzpZlwIzErV+jgBqfxmG6JUg5tA5eX1lUBMRqCY", + "h9XgTEA+rnHrzVyKUqBZokBQOwxZlavPfqxOSmfQkpGMiixVhx+ovkb6Pmv5HxRj6piAM2iCBAlJyjjm", + "JN59zijeYhKr06XSsZg1/2LDMZWNWfV3+ZTVg9JnNIyJr9onICMWfFa/4jhmd62lJxAQnA9ShnE+DZoJ", + "sE6taErGRxvVt5Jmo/teHizRIwydQUdyrUxA/OT0g6lyWcxTx0xHmLUzO/g2N0itwGVb6JVF7Q6+W3Bh", + "sost9hjrurdnnhHcTz4pwpVHkCu61DO3lqyXbKHPDgmJeGR02CmtPOYc78qwb9dCzC9tHlfPyn2TKxUn", + "/jvLvx/zXpVj6XA09L1q2eRxsQA7UhenK90fQZqHOajR43anv0dgjRPY7ogImyMKIFCmDwKUYD8itCon", + "HmMxYKrWVAlNdyyJg45AJujF9QcU6nbVBBuC4WaItP+HaJZ4wAcIcz8iEnx1tneKtQltd4m1GULJ2Yvr", + "D6LSWTlZG+Cqt4mHd/XGCcuollJII0iA4xip1grhvPq2ezTr1u7b802amQ0vI+j7Zzet9Kykc9qG5Gh+", + "FINbCvvFZ69+FpH4I3XRKlqHCm7S7EeTUmjP9ur6Q23TO7c5H+A1MVneviU3Bzt+8cUSu5ffrWdquhow", + "aCuczbDsl9BX1x8EKk7abunqkxdN8iEpKVI6e/jfyfg83HKQeR9Nw6Y82v75/BXJNIzpEs1ytE62qQWb", + "YatAyYaDBs7ljy878UEjVrxHiIr8Rr61qOx7tDzVA35tmar93rGY3kXsTj+vrPf7XuG+ID+21AIee3Tl", + "Szn58KoN8CjqB+guIrFJyxkcinxMzd5Z9wtJhggNzVG2pmryAboDFDD6jcxTYsIEjjANEAeZcYqIzMNy", + "UIZgEbqJsJlCOUJr6umUmfZ0dS/JUAASeEIoqKX5UXvxxneSDCnnzh6e9R2sxcqP5byCje+NH1uNZu8p", + "pqqU1aizveJLI0I79d+Usuxfk8Sb1zaOZfof40nfqJZN0bFAv6DlkOiUHGhR/d0W+E5GCttjg7B1w1xm", + "KECgBSXMqN9juU3KotNy4wSUpTBCyDIjDsUHX2c1uvFKNevRlnvleS5mCKhyG4PacCgkcTcGqgS1miNe", + "22K/sooOKUfLQLnQuvGEhhwLybN+mGVcxFecZWnXNCZejTbq90NzyUNz5Umc5iQfBPDCEoenUfKwR6BM", + "wVSXf1bJzGgv/HQLnPtaeq5Tra7u/GTTW47SIjkvTCgrdE2llQ4bGoXqP4z+CIaowZb99iTBm55tV7/8", + "Xt6hnvz0TVa9X9k0WyeCMsk3REJE1MkTxxC0Sc0zdAcG2ZrE2yDPxVh4Vpx/nRuaJ/qOg+X5AfNseNSS", + "1su8HmTSKxVHOH/FpvTkLPcKU7N9Ry7ziNk/1ru0uFP/uZc5H1szN0MJWCLVVbuwJmJgDLTqbcMKVfRe", + "zcIOKkGKgYPprhvOmwTsHhj/yPTrsbhem4sOQF9mnF9jD+KPpsy3o7RYVyj+LfNAN0axao10VfBA8Yz4", + "OI53BmQqq1sLz1lyFBz1YE0JDeALFOhDWQWFILR0YimBqyn/56eRu7x0/4HdXz79+a+r8pP7efjpfjRY", + "jB8qLf7y1z91aWtfkXwHgX8rmpqIIvoxE1LneS3tL9+8z0tUTUA+3qGY3QHXyVvkR5hjX506g9yLR4yj", + "aJdGQMUACYm51MAdqI2747KTalpEj2ig55UoYUKixbQytuJZDHQjI8WtBH95rT84q8V04CSE5h/HHcyo", + "Zh/3OFOrewfH8dtQ59+OQQUNV+y+6SQ0kp5dVrl20aSCxGrV1B7EjG4U/D0c7m1M2jYJn7oyzD0+bSut", + "+bt7so2V7z9wu+xh1wDdpFcp/0Ycihs9Fu8cWlgfum1tyu8Mcp++IQdF42i9PFLP25q6xyOrXsF6Dv0s", + "p+pWTYN9v6+kx5sCIMBEe20TuxQN0/VlDBPNUL6y8Z1FWzLaGffmNFehmidHiNRkE+xpVpnQZv5ESyqF", + "qQNiFJCIWBZrw151G7SDYu7ZaL8+s8EbhTnSmPjEmMQIuMIfa9o1qTo7XQ08ctAizCEiIxCAEnuOVaZV", + "K6peofn4+vINAukP17QjQdLcuibXusTZrLHPoppfn66qp5hOM/fJLkqle1taqMJkZsewx7KqjLSJrdZ+", + "HF5woQrPFq4q5u8n88ZO1OcQfyPKqKEaooqNy9hgFw7O61L2AOGiKuVIiFsRuQ6cy5sObUf4wNqxXAws", + "7tOpxDhGl9dXpa3jgAMThr3jpjq75YTuS+nXEtiVn6ylYfqDBtQ42ySKTC1KOoCnoWPCtGtAJXyRexPn", + "x10hrWD/prSYnHmFg9cd9U09x0XRTpeVaLeqWuRXCktGbym7o43qqepH7WEF0PjZFDV0C9hTjtDeIMh9", + "a5fNJT5Tk9bFBkkSqJ+T5uJMDNLELozNcFZOgCW4qnlPtLSD68cYyY796jiNm006juXBIxVG68iwuhl1", + "aPVVAx+pgQKSbbfzKiDBVBI/j2k1XOnteh38//V6WPmn013uCu80jjgdCUk5FOG3fMri33zz2htSrW4/", + "oH+G0k4PoUc3H+227tHqSqVwl9zoy2d3EUO2XU29u/MntdLV482EneB4M9FXZ5VR8s/scLlVwgJd1XeQ", + "8iwNjqM8H/EA5bhOtx3+WLq7ar1qLD/Cmt3o0v7c8NgSonxRFvL/rKC0rn43ALuWml1TTHf1U0+1iQDH", + "MrJ1laYC0wMKIZEo5CxBWP1EA6wrI9e0WIGhu4bISx2QeNMZ5cXcI5JjvkMSb4yxUmvQYbq2PnYX113m", + "wpIP0V1h0R0oVBuqf8rzjBJvDnuDeiH5mJ+66T1UQyPx5nisqPjXAonawvoZJ3L3XrWz4StdxlsvKG6v", + "420K3AD/IlFvK3A9wFzBY11tXK931uIdszvzWIEtj9W/vGABtL78wGNn5URSpmJ1VqTVhhklt4xTV2dd", + "h4xvzsySz7aTs1p/5db4LNVkKeLVik4YU/ermWb9kynDJjRkbe680Alh67cGRPhsC3xnqhhYppNzAviW", + "WBtCZKzGrURk35mu700jBQT0/W594DgrZzQcD8c6xJkCxSlxVs50OBpOzSkYaf6e4ZScbce16Ig4u68/", + "XvNQuZvZJuNHTPEGgjIHaBcthghdFf0qnr4gdBNrq2mqBHH+jXX5TRKL+jBcU21/YpIQ5dHHWEjEcUAy", + "kUfJYQumtg5X7nyjGPCtvhRNKBIsMReJBMJbRgKBvGyj+q9pHYvbU17xegOyq8BdarxV3D0197D1bStc", + "fxVIjcFy2df3U16BvEzJx/HbKp/f1rhc8sppvNgxGY36VLdod9ZxZfth4MyO6drxzIbuOj7ctfNihO48", + "Pdy5/SLAw8CZH0Xsnrt1VYulYU+3rfrpk4l2Vx6u6oFIZZOzvmei9FBH6pIN8omz++Jppn87BXsmrg8O", + "du14SEuBnJR1HZsvNK5EGFG4q5Q00EbAqq7Z10wcVG1buSOu89U0dD1/lmrXL/mVl6vOms9WPbTsxfho", + "e7H7ai2OthbPpuNn9+Vzdg9FwKQDOL7U39cqbBQ8UJC6BOFYCOYT7XhoH5zItpSagZ4gp1f19/dq0jY5", + "vAWtR6f+JaVtNpod7tm6jfzbH2r/xgAtX2q1pu3RUOxEPRh9tbp/ZIx2Glo43KvrtdNnBIS1w+LscHbW", + "qHtH1nWI0HU7E8uh/KDDOfYhh/KZOcbXtKjiQpQFjSyGtREfX1++GSL0hkkwA+laoEIhi5RYXppABNIP", + "SFAZ79blNVGUlpXBuwHConJhQ69WyZW+patr8ImQukdKwNfhlXbZ8L+k+ByFUZs8fXZ0Wlq766bcnYBb", + "ex76PAm+9r2D8dWePh+K7U1Hv7N3cwIICVXCaCJiCN1UXoLZANtwnEba3ui3X3YoZhv9McVcCRijwzX9", + "juibqnd4V5Qcm4fpFKggW2tMiDBVlJKVeLcMTIvMjxAWa1qbNGY+jmFQRmfM83rfCAWeFRcD5MXMU1ZD", + "cTyTYMs9vsN+lAfmImWBpEDsjpb61obcAx0Ztw8+lFdUBqbUJR/AludUnycUDOmbLcIWglajSuWNKRET", + "Y97wmooI8+I+iYw4yzYRuouwhC1wlIAfKVITxbLiGqJ5AAFL2ysnpBdLvVZ21aQri3qDR8MoKyYnAaXm", + "AylPVc7/82Ely7Cz+/wN7YfiPj3tv78fx+xOlG9/oLXTur6/drRo5yJjUYI9r5WqJsM1/bu+x/fi8vqt", + "FuPixl7rNQClSxCHA0Qk8jlOBWKZRO6aYqHP8UxkOEYuIqHJJevXNRi1deYZDQbojmP/ttA8qijSWER7", + "I5lAd4CEJHGs74UpoiJMA+USmJejjFLhGAnK7sIY3x7yKPJcSufDBqcqxTu7S9819+gUZel9/verq/yH", + "8S3aL+U/Ubt7L/2/sGeZffeh8Pb32Xqhjb1f61mp6Kxk6CAw+W51LhZ24xkU4XtLziny33wM+ncEgV/F", + "90jx7bvzk0uvuV10gvBWrwodI7vPYcWvDDEnRYTqb4J+Fd3fRnQfHv43AAD//6Uk9AnDZgAA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/pkg/openapi/server.spec.yaml b/pkg/openapi/server.spec.yaml index 340c3fd..5fab2d1 100644 --- a/pkg/openapi/server.spec.yaml +++ b/pkg/openapi/server.spec.yaml @@ -145,6 +145,21 @@ paths: - $ref: '#/components/parameters/organizationIDParameter' - $ref: '#/components/parameters/projectIDParameter' - $ref: '#/components/parameters/identityIDParameter' + get: + description: Get a single identity. + security: + - oauth2Authentication: [] + responses: + '200': + $ref: '#/components/responses/identityResponse' + '400': + $ref: 'https://raw.githubusercontent.com/unikorn-cloud/core/main/pkg/openapi/common.spec.yaml#/components/responses/badRequestResponse' + '401': + $ref: 'https://raw.githubusercontent.com/unikorn-cloud/core/main/pkg/openapi/common.spec.yaml#/components/responses/unauthorizedResponse' + '403': + $ref: 'https://raw.githubusercontent.com/unikorn-cloud/core/main/pkg/openapi/common.spec.yaml#/components/responses/forbiddenResponse' + '500': + $ref: 'https://raw.githubusercontent.com/unikorn-cloud/core/main/pkg/openapi/common.spec.yaml#/components/responses/internalServerErrorResponse' delete: description: Delete an identity and any resources associated with it. security: diff --git a/pkg/providers/openstack/provider.go b/pkg/providers/openstack/provider.go index f75e7b9..dd88e70 100644 --- a/pkg/providers/openstack/provider.go +++ b/pkg/providers/openstack/provider.go @@ -30,16 +30,20 @@ import ( coreconstants "github.com/unikorn-cloud/core/pkg/constants" "github.com/unikorn-cloud/core/pkg/server/conversion" + "github.com/unikorn-cloud/identity/pkg/middleware/authorization" unikornv1 "github.com/unikorn-cloud/region/pkg/apis/unikorn/v1alpha1" "github.com/unikorn-cloud/region/pkg/constants" "github.com/unikorn-cloud/region/pkg/openapi" "github.com/unikorn-cloud/region/pkg/providers" corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/yaml" ) @@ -368,7 +372,7 @@ const ( ) // projectTags defines how to tag projects. -func projectTags(identity *unikornv1.Identity) []string { +func projectTags(identity *unikornv1.OpenstackIdentity) []string { tags := []string{ OrganizationTag + "=" + identity.Labels[coreconstants.OrganizationLabel], ProjectTag + "=" + identity.Labels[coreconstants.ProjectLabel], @@ -377,15 +381,15 @@ func projectTags(identity *unikornv1.Identity) []string { return tags } -func identityResourceName(identity *unikornv1.Identity) string { +func identityResourceName(identity *unikornv1.OpenstackIdentity) string { return "unikorn-identity-" + identity.Name } // provisionUser creates a new user in the managed domain with a random password. // There is a 1:1 mapping of user to project, and the project name is unique in the // domain, so just reuse this, we can clean them up at the same time. -func (p *Provider) provisionUser(ctx context.Context, identityService *IdentityClient, identity *unikornv1.Identity) error { - if identity.Spec.OpenStack.UserID != nil { +func (p *Provider) provisionUser(ctx context.Context, identityService *IdentityClient, identity *unikornv1.OpenstackIdentity) error { + if identity.Spec.UserID != nil { return nil } @@ -397,8 +401,8 @@ func (p *Provider) provisionUser(ctx context.Context, identityService *IdentityC return err } - identity.Spec.OpenStack.UserID = &user.ID - identity.Spec.OpenStack.Password = &password + identity.Spec.UserID = &user.ID + identity.Spec.Password = &password return nil } @@ -406,8 +410,8 @@ func (p *Provider) provisionUser(ctx context.Context, identityService *IdentityC // provisionProject creates a project per-cluster. Cluster API provider Openstack is // somewhat broken in that networks can alias and cause all kinds of disasters, so it's // safest to have one cluster in one project so it has its own namespace. -func (p *Provider) provisionProject(ctx context.Context, identityService *IdentityClient, identity *unikornv1.Identity) error { - if identity.Spec.OpenStack.ProjectID != nil { +func (p *Provider) provisionProject(ctx context.Context, identityService *IdentityClient, identity *unikornv1.OpenstackIdentity) error { + if identity.Spec.ProjectID != nil { return nil } @@ -418,7 +422,7 @@ func (p *Provider) provisionProject(ctx context.Context, identityService *Identi return err } - identity.Spec.OpenStack.ProjectID = &project.ID + identity.Spec.ProjectID = &project.ID return nil } @@ -453,7 +457,7 @@ func (p *Provider) getRequiredRoles() []string { // provisionProjectRoles creates a binding between our service account and the project // with the required roles to provision an application credential that will allow cluster // creation, deletion and life-cycle management. -func (p *Provider) provisionProjectRoles(ctx context.Context, identityService *IdentityClient, identity *unikornv1.Identity) error { +func (p *Provider) provisionProjectRoles(ctx context.Context, identityService *IdentityClient, identity *unikornv1.OpenstackIdentity) error { allRoles, err := identityService.ListRoles(ctx) if err != nil { return err @@ -465,7 +469,7 @@ func (p *Provider) provisionProjectRoles(ctx context.Context, identityService *I return err } - if err := identityService.CreateRoleAssignment(ctx, *identity.Spec.OpenStack.UserID, *identity.Spec.OpenStack.ProjectID, roleID); err != nil { + if err := identityService.CreateRoleAssignment(ctx, *identity.Spec.UserID, *identity.Spec.ProjectID, roleID); err != nil { return err } } @@ -473,13 +477,13 @@ func (p *Provider) provisionProjectRoles(ctx context.Context, identityService *I return nil } -func (p *Provider) provisionApplicationCredential(ctx context.Context, identity *unikornv1.Identity) error { - if identity.Spec.OpenStack.ApplicationCredentialID != nil { +func (p *Provider) provisionApplicationCredential(ctx context.Context, identity *unikornv1.OpenstackIdentity) error { + if identity.Spec.ApplicationCredentialID != nil { return nil } // Rescope to the user/project... - providerClient := NewPasswordProvider(p.region.Spec.Openstack.Endpoint, *identity.Spec.OpenStack.UserID, *identity.Spec.OpenStack.Password, *identity.Spec.OpenStack.ProjectID) + providerClient := NewPasswordProvider(p.region.Spec.Openstack.Endpoint, *identity.Spec.UserID, *identity.Spec.Password, *identity.Spec.ProjectID) identityService, err := NewIdentityClient(ctx, providerClient) if err != nil { @@ -488,19 +492,19 @@ func (p *Provider) provisionApplicationCredential(ctx context.Context, identity name := identityResourceName(identity) - appcred, err := identityService.CreateApplicationCredential(ctx, *identity.Spec.OpenStack.UserID, name, "IaaS lifecycle management", p.getRequiredRoles()) + appcred, err := identityService.CreateApplicationCredential(ctx, *identity.Spec.UserID, name, "IaaS lifecycle management", p.getRequiredRoles()) if err != nil { return err } - identity.Spec.OpenStack.ApplicationCredentialID = &appcred.ID - identity.Spec.OpenStack.ApplicationCredentialSecret = &appcred.Secret + identity.Spec.ApplicationCredentialID = &appcred.ID + identity.Spec.ApplicationCredentialSecret = &appcred.Secret return nil } -func (p *Provider) createClientConfig(identity *unikornv1.Identity) error { - if identity.Spec.OpenStack.Cloud != nil { +func (p *Provider) createClientConfig(identity *unikornv1.OpenstackIdentity) error { + if identity.Spec.Cloud != nil { return nil } @@ -512,8 +516,8 @@ func (p *Provider) createClientConfig(identity *unikornv1.Identity) error { AuthType: clientconfig.AuthV3ApplicationCredential, AuthInfo: &clientconfig.AuthInfo{ AuthURL: p.region.Spec.Openstack.Endpoint, - ApplicationCredentialID: *identity.Spec.OpenStack.ApplicationCredentialID, - ApplicationCredentialSecret: *identity.Spec.OpenStack.ApplicationCredentialSecret, + ApplicationCredentialID: *identity.Spec.ApplicationCredentialID, + ApplicationCredentialSecret: *identity.Spec.ApplicationCredentialSecret, }, }, }, @@ -524,8 +528,8 @@ func (p *Provider) createClientConfig(identity *unikornv1.Identity) error { return err } - identity.Spec.OpenStack.Cloud = &cloud - identity.Spec.OpenStack.CloudConfig = clientConfigYAML + identity.Spec.Cloud = &cloud + identity.Spec.CloudConfig = clientConfigYAML return nil } @@ -553,13 +557,13 @@ func convertTagList(in *openapi.TagList) unikornv1.TagList { return out } -func (p *Provider) createIdentityServerGroup(ctx context.Context, identity *unikornv1.Identity) error { - if identity.Spec.OpenStack.ServerGroupID != nil { +func (p *Provider) createIdentityServerGroup(ctx context.Context, identity *unikornv1.OpenstackIdentity) error { + if identity.Spec.ServerGroupID != nil { return nil } // Rescope to the user/project... - providerClient := NewPasswordProvider(p.region.Spec.Openstack.Endpoint, *identity.Spec.OpenStack.UserID, *identity.Spec.OpenStack.Password, *identity.Spec.OpenStack.ProjectID) + providerClient := NewPasswordProvider(p.region.Spec.Openstack.Endpoint, *identity.Spec.UserID, *identity.Spec.Password, *identity.Spec.ProjectID) computeService, err := NewComputeClient(ctx, providerClient, p.region.Spec.Openstack.Compute) if err != nil { @@ -573,11 +577,51 @@ func (p *Provider) createIdentityServerGroup(ctx context.Context, identity *unik return err } - identity.Spec.OpenStack.ServerGroupID = &result.ID + identity.Spec.ServerGroupID = &result.ID return nil } +func (p *Provider) GetOpenstackIdentity(ctx context.Context, identity *unikornv1.Identity) (*unikornv1.OpenstackIdentity, error) { + var result unikornv1.OpenstackIdentity + + if err := p.client.Get(ctx, client.ObjectKey{Namespace: identity.Namespace, Name: identity.Name}, &result); err != nil { + return nil, err + } + + return &result, nil +} + +func (p *Provider) GetOrCreateOpenstackIdentity(ctx context.Context, identity *unikornv1.Identity) (*unikornv1.OpenstackIdentity, bool, error) { + create := false + + openstackIdentity, err := p.GetOpenstackIdentity(ctx, identity) + if err != nil { + if !kerrors.IsNotFound(err) { + return nil, false, err + } + + openstackIdentity = &unikornv1.OpenstackIdentity{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: identity.Namespace, + Name: identity.Name, + Labels: map[string]string{ + constants.IdentityLabel: identity.Name, + }, + Annotations: identity.Annotations, + }, + } + + for k, v := range identity.Labels { + openstackIdentity.Labels[k] = v + } + + create = true + } + + return openstackIdentity, create, nil +} + // CreateIdentity creates a new identity for cloud infrastructure. func (p *Provider) CreateIdentity(ctx context.Context, identity *unikornv1.Identity) error { identityService, err := p.identity(ctx) @@ -585,41 +629,60 @@ func (p *Provider) CreateIdentity(ctx context.Context, identity *unikornv1.Ident return err } - if identity.Spec.OpenStack == nil { - identity.Spec.OpenStack = &unikornv1.IdentitySpecOpenStack{} + openstackIdentity, create, err := p.GetOrCreateOpenstackIdentity(ctx, identity) + if err != nil { + return err + } + + record := func() { + log := log.FromContext(ctx) + + if create { + if err := p.client.Create(ctx, openstackIdentity); err != nil { + log.Error(err, "failed to create openstack identity") + } + + return + } + + if err := p.client.Update(ctx, openstackIdentity); err != nil { + log.Error(err, "failed to update openstack identity") + } } + defer record() + // Every cluster has its own project to mitigate "nuances" in CAPO i.e. it's // totally broken when it comes to network aliasing. - if err := p.provisionProject(ctx, identityService, identity); err != nil { + if err := p.provisionProject(ctx, identityService, openstackIdentity); err != nil { return err } // You MUST provision a new user, if we rotate a password, any application credentials // hanging off it will stop working, i.e. doing that to the unikorn management user // will be pretty catastrophic for all clusters in the region. - if err := p.provisionUser(ctx, identityService, identity); err != nil { + if err := p.provisionUser(ctx, identityService, openstackIdentity); err != nil { return err } // Give the user only what permissions they need to provision a cluster and // manage it during its lifetime. - if err := p.provisionProjectRoles(ctx, identityService, identity); err != nil { + if err := p.provisionProjectRoles(ctx, identityService, openstackIdentity); err != nil { return err } // Always use application credentials, they are scoped to a single project and // cannot be used to break from that jail. - if err := p.provisionApplicationCredential(ctx, identity); err != nil { + if err := p.provisionApplicationCredential(ctx, openstackIdentity); err != nil { return err } - if err := p.createClientConfig(identity); err != nil { + if err := p.createClientConfig(openstackIdentity); err != nil { return err } // Add in any optional configuration. - if err := p.createIdentityServerGroup(ctx, identity); err != nil { + if err := p.createIdentityServerGroup(ctx, openstackIdentity); err != nil { return err } @@ -628,17 +691,26 @@ func (p *Provider) CreateIdentity(ctx context.Context, identity *unikornv1.Ident // DeleteIdentity cleans up an identity for cloud infrastructure. func (p *Provider) DeleteIdentity(ctx context.Context, identity *unikornv1.Identity) error { - // Rescope to the user/project... - if identity.Spec.OpenStack.UserID != nil && identity.Spec.OpenStack.ProjectID != nil { - providerClient := NewPasswordProvider(p.region.Spec.Openstack.Endpoint, *identity.Spec.OpenStack.UserID, *identity.Spec.OpenStack.Password, *identity.Spec.OpenStack.ProjectID) + openstackIdentity, err := p.GetOpenstackIdentity(ctx, identity) + if err != nil { + if !kerrors.IsNotFound(err) { + return err + } + + return nil + } + + if openstackIdentity.Spec.UserID != nil && openstackIdentity.Spec.ProjectID != nil { + // Rescope to the user/project... + providerClient := NewPasswordProvider(p.region.Spec.Openstack.Endpoint, *openstackIdentity.Spec.UserID, *openstackIdentity.Spec.Password, *openstackIdentity.Spec.ProjectID) computeService, err := NewComputeClient(ctx, providerClient, p.region.Spec.Openstack.Compute) if err != nil { return err } - if identity.Spec.OpenStack.ServerGroupID != nil { - if err := computeService.DeleteServerGroup(ctx, *identity.Spec.OpenStack.ServerGroupID); err != nil { + if openstackIdentity.Spec.ServerGroupID != nil { + if err := computeService.DeleteServerGroup(ctx, *openstackIdentity.Spec.ServerGroupID); err != nil { return err } } @@ -649,41 +721,55 @@ func (p *Provider) DeleteIdentity(ctx context.Context, identity *unikornv1.Ident return err } - if identity.Spec.OpenStack.UserID != nil { - if err := identityService.DeleteUser(ctx, *identity.Spec.OpenStack.UserID); err != nil { + if openstackIdentity.Spec.UserID != nil { + if err := identityService.DeleteUser(ctx, *openstackIdentity.Spec.UserID); err != nil { return err } } - if identity.Spec.OpenStack.ProjectID != nil { - if err := identityService.DeleteProject(ctx, *identity.Spec.OpenStack.ProjectID); err != nil { + if openstackIdentity.Spec.ProjectID != nil { + if err := identityService.DeleteProject(ctx, *openstackIdentity.Spec.ProjectID); err != nil { return err } } + if err := p.client.Delete(ctx, openstackIdentity); err != nil { + return err + } + return nil } // CreatePhysicalNetwork creates a physical network for an identity. func (p *Provider) CreatePhysicalNetwork(ctx context.Context, identity *unikornv1.Identity, request *openapi.PhysicalNetworkWrite) (*unikornv1.PhysicalNetwork, error) { + openstackIdentity, err := p.GetOpenstackIdentity(ctx, identity) + if err != nil { + return nil, err + } + networkService, err := p.network(ctx) if err != nil { return nil, err } - vlanID, providerNetwork, err := networkService.CreateVLANProviderNetwork(ctx, "cluster-provider-network", *identity.Spec.OpenStack.ProjectID) + vlanID, providerNetwork, err := networkService.CreateVLANProviderNetwork(ctx, "cluster-provider-network", *openstackIdentity.Spec.ProjectID) + if err != nil { + return nil, err + } + + userinfo, err := authorization.UserinfoFromContext(ctx) if err != nil { return nil, err } - objectMeta := conversion.NewObjectMetadata(&request.Metadata, p.region.Namespace) + objectMeta := conversion.NewObjectMetadata(&request.Metadata, p.region.Namespace, userinfo.Sub) objectMeta = objectMeta.WithOrganization(identity.Labels[coreconstants.OrganizationLabel]) objectMeta = objectMeta.WithProject(identity.Labels[coreconstants.ProjectLabel]) objectMeta = objectMeta.WithLabel(constants.RegionLabel, p.region.Name) objectMeta = objectMeta.WithLabel(constants.IdentityLabel, identity.Name) physicalNetwork := &unikornv1.PhysicalNetwork{ - ObjectMeta: objectMeta.Get(ctx), + ObjectMeta: objectMeta.Get(), Spec: unikornv1.PhysicalNetworkSpec{ Tags: convertTagList(request.Spec.Tags), ProviderNetwork: &unikornv1.OpenstackProviderNetworkSpec{ diff --git a/pkg/provisioners/managers/identity/provisioner.go b/pkg/provisioners/managers/identity/provisioner.go index 4fe759a..980134e 100644 --- a/pkg/provisioners/managers/identity/provisioner.go +++ b/pkg/provisioners/managers/identity/provisioner.go @@ -21,13 +21,11 @@ import ( unikornv1core "github.com/unikorn-cloud/core/pkg/apis/unikorn/v1alpha1" coreclient "github.com/unikorn-cloud/core/pkg/client" + coremanager "github.com/unikorn-cloud/core/pkg/manager" "github.com/unikorn-cloud/core/pkg/provisioners" unikornv1 "github.com/unikorn-cloud/region/pkg/apis/unikorn/v1alpha1" "github.com/unikorn-cloud/region/pkg/constants" "github.com/unikorn-cloud/region/pkg/handler/region" - - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" ) // Provisioner encapsulates control plane provisioning. @@ -39,7 +37,7 @@ type Provisioner struct { } // New returns a new initialized provisioner object. -func New() provisioners.ManagerProvisioner { +func New(_ coremanager.ControllerOptions) provisioners.ManagerProvisioner { return &Provisioner{ identity: &unikornv1.Identity{}, } @@ -64,32 +62,7 @@ func (p *Provisioner) Provision(ctx context.Context) error { return err } - object := p.identity.DeepCopy() - - // Always try to update the resource as that carries state that allows us to - // be idempotent. - // TODO: most all of this mess goes away if we create a separate CR that can - // be updated by the controller independently of the identity. - update := func() { - log := log.FromContext(ctx) - - // This unfortunately will trigger another reconcile, but experience has told us - // that carrying infromation in the status is a bad idea, first as some backup - // solutions won't restore the status, and second we cannot re-geenrate things - // like passwords and secrets that are only available once. - if err := cli.Patch(ctx, object, client.MergeFrom(p.identity)); err != nil { - log.Error(err, "failed to update resource") - } - - // Update the object that the core controller refers to so that the resource - // version is up to date when it updates the status. This doesn't always work - // either! - p.identity = object - } - - defer update() - - if err := provider.CreateIdentity(ctx, object); err != nil { + if err := provider.CreateIdentity(ctx, p.identity); err != nil { return err } diff --git a/pkg/reaper/reaper.go b/pkg/reaper/reaper.go deleted file mode 100644 index d8d20d6..0000000 --- a/pkg/reaper/reaper.go +++ /dev/null @@ -1,151 +0,0 @@ -/* -Copyright 2024 the Unikorn Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package reaper - -import ( - "context" - "errors" - "fmt" - - coreconstants "github.com/unikorn-cloud/core/pkg/constants" - unikornv1 "github.com/unikorn-cloud/region/pkg/apis/unikorn/v1alpha1" - "github.com/unikorn-cloud/region/pkg/constants" - "github.com/unikorn-cloud/region/pkg/handler/region" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/watch" - - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" -) - -var ( - ErrInvalidClient = errors.New("client invalid") - - ErrDataMissing = errors.New("data missing") - - ErrUnexpectedType = errors.New("unexpected type") - - ErrCleanup = errors.New("cleanup failed") -) - -// Seasons don't fear the reaper, nor do wind or the sun or the rain. -// Peforms asynchronous cleanup tasks. -type Reaper struct { - client client.Client - namespace string -} - -func New(client client.Client, namespace string) *Reaper { - return &Reaper{ - client: client, - namespace: namespace, - } -} - -func (r *Reaper) Run(ctx context.Context) error { - log := log.FromContext(ctx) - - watchingClient, ok := r.client.(client.WithWatch) - if !ok { - return fmt.Errorf("%w: client does not implement watches", ErrInvalidClient) - } - - // TODO: This is insecure, enything can raise an event and have identities deleted. - // We should syncronously trigger the cleanup from various component deprovisioners - // but then you are faced with the problem that there is no access token to allow - // API access. - options := &client.ListOptions{ - FieldSelector: fields.SelectorFromSet(fields.Set{ - "reason": coreconstants.IdentityCleanupReadyEventReason, - }), - } - - // Please note when using Kubernetes watches of events that you may see - // some historical events for things in the last hour. - go func() { - var zeroEvent watch.Event - - for { - var events corev1.EventList - - watcher, err := watchingClient.Watch(ctx, &events, options) - if err != nil { - log.Error(err, "failed to setup watch") - continue - } - - eventStream := watcher.ResultChan() - - for { - event := <-eventStream - - // Zero value returned, closed channel, this happens, start it back - // up again... - if event == zeroEvent { - break - } - - log.V(1).Info("witnessed an event", "event", event) - - if event.Type != watch.Added { - continue - } - - realEvent, ok := event.Object.(*corev1.Event) - if !ok { - log.Error(ErrUnexpectedType, "unable to decode event") - } - - if err := r.handleEvent(ctx, realEvent); err != nil { - log.Error(err, "event handling failed") - } - } - } - }() - - return nil -} - -func (r *Reaper) handleEvent(ctx context.Context, event *corev1.Event) error { - log := log.FromContext(ctx) - - log.Info("processing cluster deletion event", "event", event) - - identityID, ok := event.Annotations[coreconstants.CloudIdentityAnnotation] - if !ok { - return fmt.Errorf("%w: identity annotation not present", ErrDataMissing) - } - - identity := &unikornv1.Identity{} - - if err := r.client.Get(ctx, client.ObjectKey{Namespace: r.namespace, Name: identityID}, identity); err != nil { - return err - } - - provider, err := region.NewClient(r.client, r.namespace).Provider(ctx, identity.Labels[constants.RegionLabel]) - if err != nil { - return fmt.Errorf("%w: failed to create provider client", err) - } - - if err := provider.DeleteIdentity(ctx, identity); err != nil { - return fmt.Errorf("%w: failed to delete identity", err) - } - - return nil -} diff --git a/pkg/server/options.go b/pkg/server/options.go index 2e745b7..16e58dd 100644 --- a/pkg/server/options.go +++ b/pkg/server/options.go @@ -47,10 +47,6 @@ type Options struct { // turn creates a project. WriteTimeout time.Duration - // OTLPEndpoint defines whether to ship spans to an OTLP consumer or - // not, and where to send them to. - OTLPEndpoint string - // RequestTimeout places a hard limit on all requests lengths. RequestTimeout time.Duration } @@ -63,5 +59,4 @@ func (o *Options) AddFlags(f *pflag.FlagSet) { f.DurationVar(&o.ReadHeaderTimeout, "server-read-header-timeout", time.Second, "How long to wait for the client to send headers.") f.DurationVar(&o.WriteTimeout, "server-write-timeout", 10*time.Second, "How long to wait for the API to respond to the client.") f.DurationVar(&o.RequestTimeout, "server-request-timeout", 30*time.Second, "How long to wait of a request to be serviced.") - f.StringVar(&o.OTLPEndpoint, "otlp-endpoint", "", "An optional OTLP endpoint to ship spans to.") } diff --git a/pkg/server/server.go b/pkg/server/server.go index 6d2fff5..3fbff0a 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -26,17 +26,16 @@ import ( chi "github.com/go-chi/chi/v5" "github.com/spf13/pflag" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" - "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/trace" + coreclient "github.com/unikorn-cloud/core/pkg/client" + "github.com/unikorn-cloud/core/pkg/manager/otel" coreapi "github.com/unikorn-cloud/core/pkg/openapi" - "github.com/unikorn-cloud/core/pkg/server/middleware/audit" "github.com/unikorn-cloud/core/pkg/server/middleware/cors" "github.com/unikorn-cloud/core/pkg/server/middleware/opentelemetry" "github.com/unikorn-cloud/core/pkg/server/middleware/timeout" identityclient "github.com/unikorn-cloud/identity/pkg/client" + "github.com/unikorn-cloud/identity/pkg/middleware/audit" openapimiddleware "github.com/unikorn-cloud/identity/pkg/middleware/openapi" openapimiddlewareremote "github.com/unikorn-cloud/identity/pkg/middleware/openapi/remote" "github.com/unikorn-cloud/region/pkg/constants" @@ -58,20 +57,32 @@ type Server struct { // HandlerOptions sets options for the HTTP handler. HandlerOptions handler.Options + // ClientOptions are for generic TLS client options e.g. certificates. + ClientOptions coreclient.HTTPClientOptions + // IdentityOptions allow configuration of the authorization middleware. - IdentityOptions identityclient.Options + IdentityOptions *identityclient.Options // CORSOptions are for remote resource sharing. CORSOptions cors.Options + + // OTelOptions are for tracing. + OTelOptions otel.Options } func (s *Server) AddFlags(goflags *flag.FlagSet, flags *pflag.FlagSet) { + if s.IdentityOptions == nil { + s.IdentityOptions = identityclient.NewOptions() + } + s.ZapOptions.BindFlags(goflags) s.Options.AddFlags(flags) s.HandlerOptions.AddFlags(flags) + s.ClientOptions.AddFlags(flags) s.IdentityOptions.AddFlags(flags) s.CORSOptions.AddFlags(flags) + s.OTelOptions.AddFlags(flags) } func (s *Server) SetupLogging() { @@ -82,30 +93,7 @@ func (s *Server) SetupLogging() { // logs by default, and optionally ship the spans to an OTLP listener. // TODO: move config into an otel specific options struct. func (s *Server) SetupOpenTelemetry(ctx context.Context) error { - otel.SetLogger(log.Log) - - otel.SetTextMapPropagator(propagation.TraceContext{}) - - opts := []trace.TracerProviderOption{ - trace.WithSpanProcessor(&opentelemetry.LoggingSpanProcessor{}), - } - - if s.Options.OTLPEndpoint != "" { - exporter, err := otlptracehttp.New(ctx, - otlptracehttp.WithEndpoint(s.Options.OTLPEndpoint), - otlptracehttp.WithInsecure(), - ) - - if err != nil { - return err - } - - opts = append(opts, trace.WithBatcher(exporter)) - } - - otel.SetTracerProvider(trace.NewTracerProvider(opts...)) - - return nil + return s.OTelOptions.Setup(ctx, trace.WithSpanProcessor(&opentelemetry.LoggingSpanProcessor{})) } func (s *Server) GetServer(client client.Client) (*http.Server, error) { @@ -143,7 +131,7 @@ func (s *Server) GetServer(client client.Client) (*http.Server, error) { router.NotFound(http.HandlerFunc(handler.NotFound)) router.MethodNotAllowed(http.HandlerFunc(handler.MethodNotAllowed)) - authorizer := openapimiddlewareremote.NewAuthorizer(client, s.Options.Namespace, &s.IdentityOptions) + authorizer := openapimiddlewareremote.NewAuthorizer(client, s.IdentityOptions, &s.ClientOptions) // Middleware specified here is applied to all requests post-routing. // NOTE: these are applied in reverse order!! @@ -156,7 +144,7 @@ func (s *Server) GetServer(client client.Client) (*http.Server, error) { }, } - identity := identityclient.New(client, s.Options.Namespace, &s.IdentityOptions) + identity := identityclient.New(client, s.IdentityOptions, &s.ClientOptions) handlerInterface, err := handler.New(client, s.Options.Namespace, &s.HandlerOptions, identity) if err != nil {