From 8efdd18f1a1c5c4cff41769730b9958df055090b Mon Sep 17 00:00:00 2001 From: Simon Murray Date: Fri, 26 Jul 2024 10:17:38 +0100 Subject: [PATCH] Rework Flavor Configuraiton (#40) Make flavors behave the same as external networks and images by having an idiomatic selector. Separate out the metadata to keep selection and mutation distinct. Allow flavors to be mutated to skirt around naughty operators who lie. --- charts/region/Chart.yaml | 4 +- .../region.unikorn-cloud.org_regions.yaml | 69 +++---- charts/region/values.schema.json | 36 ++-- charts/region/values.yaml | 26 +-- pkg/apis/unikorn/v1alpha1/types.go | 48 ++--- .../unikorn/v1alpha1/zz_generated.deepcopy.go | 113 ++++++----- pkg/handler/handler.go | 4 + pkg/openapi/schema.go | 175 +++++++++--------- pkg/openapi/server.spec.yaml | 3 + pkg/openapi/types.go | 3 + pkg/providers/openstack/compute.go | 65 ++++--- pkg/providers/openstack/provider.go | 8 +- 12 files changed, 281 insertions(+), 273 deletions(-) diff --git a/charts/region/Chart.yaml b/charts/region/Chart.yaml index 656d5b1..be2aa1a 100644 --- a/charts/region/Chart.yaml +++ b/charts/region/Chart.yaml @@ -4,8 +4,8 @@ description: A Helm chart for deploying Unikorn's Region Controller type: application -version: v0.1.26 -appVersion: v0.1.26 +version: v0.1.27 +appVersion: v0.1.27 icon: https://raw.githubusercontent.com/unikorn-cloud/assets/main/images/logos/dark-on-light/icon.png diff --git a/charts/region/crds/region.unikorn-cloud.org_regions.yaml b/charts/region/crds/region.unikorn-cloud.org_regions.yaml index 233d359..a951219 100644 --- a/charts/region/crds/region.unikorn-cloud.org_regions.yaml +++ b/charts/region/crds/region.unikorn-cloud.org_regions.yaml @@ -69,25 +69,12 @@ spec: Flavors defines how flavors are filtered and reported to clients. If not defined, then all flavors are exported. properties: - exclude: - description: Exclude inhibits the export of flavors from - the region service. - items: - properties: - id: - description: ID flavor ID is the immutable Openstack - identifier for the flavor. - type: string - required: - - id - type: object - type: array - include: + metadata: description: |- - Include allows or augments flavors that can be exported by the region - service as defined by the "selectionPolicy" property. This explcitly - allows a flavor to be used, and or allows metadata to be mapped to the - flavor e.g. CPU/GPU information that isn't supported by OpenStack. + Metadata allows flavors to be explicitly augmented with additional metadata. + This acknowledges the fact that OpenStack is inadequate acting as a source + of truth for machine topology, and needs external input to describe things + like add on peripherals. items: properties: baremetal: @@ -99,6 +86,12 @@ spec: cpu: description: CPU defines additional CPU metadata. properties: + count: + description: |- + Count allows you to override the number of CPUs. Usually this wouldn't + be necessary, but alas some operators may not set this correctly for baremetal + flavors to make horizon display overcommit correctly... + type: integer family: description: |- Family is a free-form string that can communicate the CPU family to clients @@ -143,28 +136,36 @@ spec: - vendor type: object id: - description: |- - ID is the immutable Openstack identifier for the flavor. - While most flavor metadata (CPUs/Memory) should be immutable, the name is - not, and may change due to sales and marketing people. + description: ID is the immutable Openstack identifier + for the flavor. type: string + memory: + anyOf: + - type: integer + - type: string + description: Memory allows the memory amount to + be overridden. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true required: - id type: object type: array - selectionPolicy: + selector: description: |- - SelectionPolicy defines the default set of flavors to export. "All" exports - all flavors, the "include" property defines additional metadata to - merge with matching flavors and the "exclude" inhibits export. "None" is a - more secure policy that only exports those flavors defined in the "include" - property, the "exclude" property is ignored as it's redundant. - enum: - - All - - None - type: string - required: - - selectionPolicy + Selector allows flavors to be manually selected for inclusion. The selected + set is a boolean intersection of all defined filters in the selector. + Note that there are some internal rules that will fiter out flavors such as + if the flavor does not have enough resource to function correctly. + properties: + ids: + description: |- + IDs is an explicit list of allowed flavors IDs. If not specified, + then all flavors are considered. + items: + type: string + type: array + type: object type: object serverGroupPolicy: description: |- diff --git a/charts/region/values.schema.json b/charts/region/values.schema.json index 03bd173..7d6cab1 100644 --- a/charts/region/values.schema.json +++ b/charts/region/values.schema.json @@ -217,25 +217,19 @@ "properties": { "flavors": { "type": "object", - "required": [ - "selectionPolicy" - ], "properties": { - "exclude": { - "type": "array", - "items": { - "type": "object", - "required": [ - "id" - ], - "properties": { - "id": { + "selector": { + "type": "object", + "properties": { + "ids": { + "type": "array", + "items": { "type": "string" } } } }, - "include": { + "metadata": { "type": "array", "items": { "type": "object", @@ -248,12 +242,19 @@ }, "cpu": { "properties": { + "count": { + "type": "integer" + }, "family": { "type": "string" } }, "type": "object" }, + "memory": { + "type": "string", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$" + }, "gpu": { "type": "object", "required": [ @@ -267,14 +268,7 @@ "type": "integer" }, "memory": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "string" - } - ], + "type": "string", "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$" }, "model": { diff --git a/charts/region/values.yaml b/charts/region/values.yaml index 79c048a..9d9277d 100644 --- a/charts/region/values.yaml +++ b/charts/region/values.yaml @@ -47,32 +47,22 @@ organization: unikorn-cloud # serverGroupPolicy: soft-anti-affinity # # Flavor selection and configuration. # flavors: -# # The selection policy can be "All" or "None" -# selectionPolicy: All -# # Include specific flavors when the policy is "None". In all cases -# # allows additional metadata to be exposed by the API that Openstack -# # cannot act as a source of truth for. -# include: +# # Include specific flavors. +# selector: +# ids: +# - 60ab8c22-ac61-467d-8680-03d0ecca23c9 +# # Mutate flavors or provide extra information. +# metadata: # - id: 60ab8c22-ac61-467d-8680-03d0ecca23c9 # cpu: +# count: 8 # family: Intel Xeon Platinum 8160T (Skylake) +# memory: 256Gi # gpu: # vendor: NVIDIA # model: H100 # memory: 192Gi # count: 2 -# # Eclude specific flavors when the policy is "All". -# exclude: -# - id: d04d82d9-faa6-4b3b-9247-28b8a158b7ed -# # Flavors containing any of the specified extra specs will be discarded. -# flavorExtraSpecsExclude: -# - resources:CUSTOM_BAREMETAL -# # Define properties on flavors and how to extract the number of GPUs from them. -# gpuDescriptors: -# - property: resources:PGPU -# expression: ^(\d+)$ -# - property: resources:VGPU -# expression: ^(\d+)$ # # Image service configuration. # image: # # Image selection, the result is a boolean intersection of chosen options. diff --git a/pkg/apis/unikorn/v1alpha1/types.go b/pkg/apis/unikorn/v1alpha1/types.go index bb26717..a99914f 100644 --- a/pkg/apis/unikorn/v1alpha1/types.go +++ b/pkg/apis/unikorn/v1alpha1/types.go @@ -114,25 +114,26 @@ const ( ) type OpenstackFlavorsSpec struct { - // SelectionPolicy defines the default set of flavors to export. "All" exports - // all flavors, the "include" property defines additional metadata to - // merge with matching flavors and the "exclude" inhibits export. "None" is a - // more secure policy that only exports those flavors defined in the "include" - // property, the "exclude" property is ignored as it's redundant. - SelectionPolicy OpenstackFlavorSelectionPolicy `json:"selectionPolicy"` - // Include allows or augments flavors that can be exported by the region - // service as defined by the "selectionPolicy" property. This explcitly - // allows a flavor to be used, and or allows metadata to be mapped to the - // flavor e.g. CPU/GPU information that isn't supported by OpenStack. - Include []OpenstackFlavorInclude `json:"include,omitempty"` - // Exclude inhibits the export of flavors from the region service. - Exclude []OpenstackFlavorExclude `json:"exclude,omitempty"` -} - -type OpenstackFlavorInclude struct { + // Selector allows flavors to be manually selected for inclusion. The selected + // set is a boolean intersection of all defined filters in the selector. + // Note that there are some internal rules that will fiter out flavors such as + // if the flavor does not have enough resource to function correctly. + Selector *FlavorSelector `json:"selector,omitempty"` + // Metadata allows flavors to be explicitly augmented with additional metadata. + // This acknowledges the fact that OpenStack is inadequate acting as a source + // of truth for machine topology, and needs external input to describe things + // like add on peripherals. + Metadata []FlavorMetadata `json:"metadata,omitempty"` +} + +type FlavorSelector struct { + // IDs is an explicit list of allowed flavors IDs. If not specified, + // then all flavors are considered. + IDs []string `json:"ids,omitempty"` +} + +type FlavorMetadata struct { // ID is the immutable Openstack identifier for the flavor. - // While most flavor metadata (CPUs/Memory) should be immutable, the name is - // not, and may change due to sales and marketing people. ID string `json:"id"` // Baremetal indicates that this is a baremetal flavor, as opposed to a // virtualized one in case this affects image selection or even how instances @@ -140,17 +141,18 @@ type OpenstackFlavorInclude struct { Baremetal bool `json:"baremetal,omitempty"` // CPU defines additional CPU metadata. CPU *CPUSpec `json:"cpu,omitempty"` + // Memory allows the memory amount to be overridden. + Memory *resource.Quantity `json:"memory,omitempty"` // GPU defines additional GPU metadata. When provided it will enable selection // of images based on GPU vendor and model. GPU *GPUSpec `json:"gpu,omitempty"` } -type OpenstackFlavorExclude struct { - // ID flavor ID is the immutable Openstack identifier for the flavor. - ID string `json:"id"` -} - type CPUSpec struct { + // Count allows you to override the number of CPUs. Usually this wouldn't + // be necessary, but alas some operators may not set this correctly for baremetal + // flavors to make horizon display overcommit correctly... + Count *int `json:"count,omitempty"` // Family is a free-form string that can communicate the CPU family to clients // e.g. "Xeon Platinum 8160T (Skylake)", and allows users to make scheduling // decisions based on CPU architecture and performance etc. diff --git a/pkg/apis/unikorn/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/unikorn/v1alpha1/zz_generated.deepcopy.go index 1b2e301..f5bf1ba 100644 --- a/pkg/apis/unikorn/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/unikorn/v1alpha1/zz_generated.deepcopy.go @@ -29,6 +29,11 @@ import ( // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CPUSpec) DeepCopyInto(out *CPUSpec) { *out = *in + if in.Count != nil { + in, out := &in.Count, &out.Count + *out = new(int) + **out = **in + } if in.Family != nil { in, out := &in.Family, &out.Family *out = new(string) @@ -68,6 +73,58 @@ func (in *ExternalNetworks) DeepCopy() *ExternalNetworks { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FlavorMetadata) DeepCopyInto(out *FlavorMetadata) { + *out = *in + if in.CPU != nil { + in, out := &in.CPU, &out.CPU + *out = new(CPUSpec) + (*in).DeepCopyInto(*out) + } + if in.Memory != nil { + in, out := &in.Memory, &out.Memory + x := (*in).DeepCopy() + *out = &x + } + if in.GPU != nil { + in, out := &in.GPU, &out.GPU + *out = new(GPUSpec) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FlavorMetadata. +func (in *FlavorMetadata) DeepCopy() *FlavorMetadata { + if in == nil { + return nil + } + out := new(FlavorMetadata) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FlavorSelector) DeepCopyInto(out *FlavorSelector) { + *out = *in + if in.IDs != nil { + in, out := &in.IDs, &out.IDs + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FlavorSelector. +func (in *FlavorSelector) DeepCopy() *FlavorSelector { + if in == nil { + return nil + } + out := new(FlavorSelector) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GPUSpec) DeepCopyInto(out *GPUSpec) { *out = *in @@ -277,62 +334,20 @@ func (in *NetworkSelector) DeepCopy() *NetworkSelector { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OpenstackFlavorExclude) DeepCopyInto(out *OpenstackFlavorExclude) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenstackFlavorExclude. -func (in *OpenstackFlavorExclude) DeepCopy() *OpenstackFlavorExclude { - if in == nil { - return nil - } - out := new(OpenstackFlavorExclude) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OpenstackFlavorInclude) DeepCopyInto(out *OpenstackFlavorInclude) { +func (in *OpenstackFlavorsSpec) DeepCopyInto(out *OpenstackFlavorsSpec) { *out = *in - if in.CPU != nil { - in, out := &in.CPU, &out.CPU - *out = new(CPUSpec) - (*in).DeepCopyInto(*out) - } - if in.GPU != nil { - in, out := &in.GPU, &out.GPU - *out = new(GPUSpec) + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = new(FlavorSelector) (*in).DeepCopyInto(*out) } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenstackFlavorInclude. -func (in *OpenstackFlavorInclude) DeepCopy() *OpenstackFlavorInclude { - if in == nil { - return nil - } - out := new(OpenstackFlavorInclude) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OpenstackFlavorsSpec) DeepCopyInto(out *OpenstackFlavorsSpec) { - *out = *in - if in.Include != nil { - in, out := &in.Include, &out.Include - *out = make([]OpenstackFlavorInclude, len(*in)) + if in.Metadata != nil { + in, out := &in.Metadata, &out.Metadata + *out = make([]FlavorMetadata, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.Exclude != nil { - in, out := &in.Exclude, &out.Exclude - *out = make([]OpenstackFlavorExclude, len(*in)) - copy(*out, *in) - } return } diff --git a/pkg/handler/handler.go b/pkg/handler/handler.go index bc117de..b9aae92 100644 --- a/pkg/handler/handler.go +++ b/pkg/handler/handler.go @@ -123,6 +123,10 @@ func convertFlavor(in providers.Flavor) openapi.Flavor { }, } + if in.Baremetal { + out.Spec.Baremetal = coreutil.ToPointer(true) + } + if in.GPU != nil { out.Spec.Gpu = &openapi.GpuSpec{ Vendor: convertGpuVendor(in.GPU.Vendor), diff --git a/pkg/openapi/schema.go b/pkg/openapi/schema.go index 6b92250..6f11f65 100644 --- a/pkg/openapi/schema.go +++ b/pkg/openapi/schema.go @@ -19,93 +19,94 @@ import ( // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+w8a3PbyJF/ZQqXqk3qCIpvifyS09obrypeW2fLzl2WPtcA0yBmBcwgMwPKXJX++9U8", - "AAIEQFKUvJvcuZIti8A8unv63T2490KeZpwBU9Jb3HsZFjgFBcL8ogSYompz9fK6eK4fE5ChoJminHkL", - "7yYGVAx0f0QURN/reVS/z7CKvZ7HcAreorKk1/ME/COnAoi3UCKHnifDGFKst/iDgMhbeP92tgXvzL6V", - "Z7d5AIKBAvkGp7CF7OGh53Gxwoz+ijVse6G+ZKg6Fl297AC4vuJeoNUm0zOkEpStDDiZ4L9AqA7Sz41D", - "es8OOMqlvgrdBKwOUUzDaYcdPuViua8A64NdEqT6nhMKNT59Z1/oRyFnCpj5E2dZQkNzgGe/SI3LvQdf", - "cJoloP9MQWGCFW7hEbQGEXAJqPq851HiLbzwfDq7gBHxozkO/Ml0TPw5HmN/OhyfT6Pzi8loFjTZ3je/", - "H3qezCDUOzpKPWJFhVfSW/x8XywdJrlUIHxKvJ63xkmuH87Hs+FkMAr9aD6/8CfzMPRxMBr68yCYz3EU", - "RgQuvIdPmprHHUiBwN8EVWCPYZdY7lhQxAXCrNQJ/QYTaMGIN5KGOHkD6o6L23/egysA9ZmFtHGAmYCI", - "fvEW3nDQN/87u/jdDmmHqseeFSrmIYdk3+kEmXEmrYjhMIRMAXnnHnbpB7tsjCUKABgqpiHMCLqjSYIC", - "QFGeRDRJ9FO5YWEsOOO5TDb9JftvnqMUb1DGkwQps6LkuQjBLJByRhUXiCqJpMIqlwYBTYkENBh9fTIB", - "Jo6fqsAez1cgBBdaZtkaJ5R8dkh5Pfvmcx3tAuWAkw1yU7yjT8zu1XJE76rLRphqatlJyGxhoO8hLhyV", - "7GjCQSLGFdLYYsqWDJd0tBKIIgoJkYZQ8EWBYCW7yFPI9fO9k6pxNB+dD2f+MCKhPwnOA38+mIE/iWAw", - "nE5IFJJoK1UR597Dp6OJtANnO0snVCrEI0seVMwpWNpiHCV4zcWpiFYVTijADLyhBqHh/HzgD4b+YHgz", - "GCzM//9eKJw5vghn4/OBPxnMpv6ETLA/J3jgn8/OL0g0GYRkTrakWfUn/Ziu4hTSPh4OBv3hqj8crIKq", - "zgmz/C84pcnGW3hXTEGC/gs4Q9cJVpTlKboYzgY36I/vbzcJvoU/eT09Q3qLSc8jVN56i9Gg562y3OKf", - "a+yHPS+FlIuNtxjORz0v5QQSb+H9OBwMtMoCRoxQvPl49fLqUgNTDB+PHo4/SncA+0/QDbInxkVACQH2", - "NFkul+mQ4lyCQKEAY7NwIhHhRo5ivIa6/GSCrmkCK5DPKOV3WCICjAJBwQbhXMVcUOlkXMVUGqUYAApx", - "Lu0gDVRt4JIpfgusAJuyVR1wGfIMCuN8eX1VKg+Du9Yc7LstwkvGIAQpsdhUUEacmSmZ4GtKQKAswSri", - "IjVn5Sw+hWcTMCDfax7/hcesTzj8Bw5T6Ic81RxdF8DRYDTxB1N/PLwZThbDYVUA8WwSzUezuT+ewcCf", - "jIcjP7ggQ386IvMxmc7mwXnF4udMk9jbiSUeIciFp66nwHgWDqYX2L+AAPuTaBr482E08aNZFAXzi/H5", - "fBraKWsqKWeUrd4bw2Y9fvsQSFX4eQZMKhzeGiolPNf7EIhwnmgbZZ684CyiK/38VZyFm+/1f/HVj++S", - "cPyff90FMZiHc02J88lsQoaTILo4h+kgwuej2fhioDHSHGLG4uF8dn6BRxfD0WwyPycBHk2C6SScz/Bg", - "Nomwtw0lDFQX8yEJooE/wIOhP4Eo9DFoV4ucn0czMp6MJsajtXHTFrFHKJQqz2GyX6+4sSCr3Lo5TbF8", - "Y9VvrGpZ9bERVCefbkMmVLjellFTvIKv4LOMBqOxPxj5o9HNcLQYTBbD8al8GOSj0WDir4f90bQ/81dZ", - "7k9H0/7FtD+Y+uchkMlwOqlyhnM+iKBr0Pa5HO0510NHT96ldT6cD/LjaDDwPrX6IpJH6g4L+AhCc6GJ", - "WLYpBG/hOcj02DUVKseJkxb9rnigmfcRmsccywGNY8YgFWOFsAATqWBFgwTQHVWxNe11G8qs3/oexBrE", - "D9pxeJrnI81Cn+3PdufHhReKI+tBhAmm6TN4N5cM5Qy+ZBDqCNAMQzwMcyGA1N0aXBupBGaSAlNuDmZk", - "yfRImYchANFeCEYClNj00VVkV6LGfdHOSYgl9FCWAJba/cm4UIgqhKXJSkiZW7FiXP2F54w8jbyMq8+R", - "XqaDtpXoDMg2mi0DNfhCpXoGWn9gWHOV4iiijBjy2K0Mro10yzeT95VMXls66OQETrdL05KxKczZV7MU", - "5+b4BovJdDGZ6uNrZtS/bFIuOKMhUhSEP0Z6wRC0TkMB1pELZei11vkZ50n/tJRcfuvf2XzMYw7mKU6m", - "I+x+Xe8GmZPImQvNfoUn6hcc6ijssw0OO3SM3ks7DnY1ly16Dv3dtm4RNVrAnMWIsUTwJdNxZr/C7rKC", - "yW668BUwEDR0Kj7VoeYKeg0LyTVyo7497AyEcin/jlUvkQIhwa1qKzEaMsyI/suFrz/e3Fy7ISEn0EfG", - "zkpjoi2buoFvNQlGSPMQjRwdeijIrTW36wKxkGr4BAWlI2aXnNSL2xTl5fWVRFzFoImH9eJcQrGuDejt", - "XhpTYHmq3Z9mArLKV5/DRNtIr9fgkZzJPNNmD/Rcy32fDf/3yjVNNsDr7boHCtKMCyxosvmcM7zGNNF2", - "pTKx3LV4sBKYqZ1dzbNiy6qJDDmLEhrq8SmomJPP+i1OEn7XAD0FQnGxyDaB86m3W2hrlYpdzvjoKgKO", - "01xlICjSJGaFvtdrKeJtixc/e91u1BYsHmgD05Jgba1Cvi0UUiNl2WR6rSzb0+7OrbBVzAZ5rOLcO7Oo", - "PO5Hn5aJyiPQlW3iWWhL3om2NGZBQSofmRfWIDiYsBB4s034tgFi3zRpXDWD+zbXIk7Dd45+PxWzKhbn", - "cB70vR65S+MSALdSG6Ur0x+BWiVt3DJJgMkOpujF9QcUmXHV8hmC/qqPTGyGWJ4GIHoIizCmCkKVC2hl", - "PJt2bmM8u4TmhBfXH2Rlsg6AViD0bJurbpuNU54zw0eQxZCCwAnSo7V78er79tVcyLnvVFZZbo9km93e", - "v7sdZXalrdvunK2hR7m4w7D7gPdKUJklP1JanCi0CMkqy3+y6f7mbq+uP9QOvfWYiwVeU1vD7QJ5d7Hj", - "gS9BbAe/XRL0djXT3SIStvqxn0NfXX+QqLSF7dzVxS8G5UNcUpZb9tC/lfBFKuQg8T7agbv86OYX+1c4", - "0xKmjTW3q7WSTQNsl626Mi5V0/Muf3rZasF38rh7mKisPRRHi7Zzj+anejKuyVO19y3AdAKxOd2iuMj0", - "vfbMSGFYNACPNS4FKCebl9oCj8K+h+5imtiSmfUUUYiZPTsXICHFEWWR8YRhyfTmPXQHiHD2nSrKVdIm", - "dTAjSIDKBUNUFSkz2KZHEbqJsd1ChypLFphylgkzzSzFEQEFIqUMNGhh3ATeRjeKIx1+UQbNE6zlsY+l", - "vHbs3ttIs5pp3tNWVWmaQVSiSiCLKGuVf9tmsh8mhVevXY7Jzj8m1r3RI3dZx7niJS6HWGdLgQbWP6xB", - "bFSsvW9sfWAzsOAZBkAMo0Q5Czs0ty0ntGpunILWFJYJeW7ZofwRmopDu79SrUg0+V7HhrMJAqYDO1Jb", - "DkU0afeBKgmn3RWvXdvftp8O6VAoxAqICRs10JRFAksl8m43qyh67K7/QYIotWN02uo7POC2qqK1jw1s", - "C1Jb3FOpdZjo9nS9WcQwZq9TdaWZ/GSFuV2lgXJR6t922NreJZNps2LQbUL+GdTHDln2a4EUrzqOXb/5", - "vaIus/nph6xnv3KFq1a/x5azEI0Q1fYiSYA0US1qXgcWWdtSVq+objinqrRarQdalM6Oc6YLs/BsXqRD", - "rZN4Hf5EJ1ccEbKVh9JRBdzLTLvjW6qDR+z+sT6lQZ36607ifGzsXCfT37Sno6eawBOHsXZpjArXszWH", - "RHWfu1rX7HkBFqC5XHv5mG3anXBb0tzjfD+yoHmsN27URYsbvq3hvsYBJB9t42xLu6/p+ftrHoAZjBI9", - "Gpk+256mGQ1xkmysa6i1bi3t5dDRTmQAS0YZgS9Q+gxaK2i7b7gTKwVCb/k/Pw/8+aX/d+z/+umPf15s", - "f/mf+5/uB73Z8KEy4k9//kObtHY1ubcg+NdyqM3UoZ9yqUzl1OH+8s37ounTJrqTDUr4HQhTDkVhjAUO", - "tdXpFbE34gLFmywGJntIKiyUcbeBuXw23k7SQ8ucDyNmX4VSLhWajStra5olwFYq1tRK8ZfX5oe3mI17", - "XkpZ8XPYQoxqPW9PCLS493CSvI1MyeoYr2AngLrfde13yohtWrl2UaTiq9X6kwNIOFtpp/WwE7WzaVMl", - "fGqr2XZEoo1K4O8ef+5AfrLBbVunnQJVAnwnDyV9ihptiyG6ul5PECZEgNQelB737J6T2/4IjLu858ah", - "/85O9LMd+EEOPFr8j1QnTYWwJ1yr3tR6DjWwL4T6VCYNugTfvn36iZ8i4Xbvk8+5Mr2B2BXTroPVtTjg", - "uXL0bUf2OTIa3RDeuNW7Qq7v5DabpJeoel/bnFGbp1V0FOxxtcp+giOdqAq3tHhSYjdkaglQHQsXJ+g8", - "C9MxniTo8vpqy+YCMLHpuTthO2obYc6+Ymyt9Fh55aIdbn4Ylw3nq1SjabjAJHaMc5Jy43wyBV/U3pLn", - "cZcMK97lLovYameFgtctTScdmqIcZxoCjONebczaMkvObhm/YzstLdWfxocnsPPalqPbGewp2rMzzL5v", - "nLK9eGUbhdrIoGgKdRVpLzskoGx0bMXdW3gEK/D18I4sWgvVj9FvLefVooh3h7Ro5N4jBcbISL96GHXj", - "+k0CHymBEtJ1e3gkIcVM0bDImuwEa+vlkvz7ctmv/NMakLUlEHask4m1MwFlgqfYsvy3OLzmgVQ7kg/I", - "n8X04aHFNHXI5qMDoz1SXenubOMbc2HoLubIjauJd3tevdZPeLyacBscrya6OmRyRv+RH26USTkx/VgH", - "Mc8zchzmxYoHMMd1vN3yx+Ld1qVTI/kR2uzGtGMXisdmk0qgXIrkl1y6jmWbB6iV7JYMs03d6ukxMeBE", - "xa4jzvbOBcAgogpFgqcI61eMYNPTtmQlBBbv/pJ5LTKg8Ko1fMMioEpgsUEKr6yy0jCYRFBTHtvboi4L", - "ZimWaK+8t6ei9IGaV0X9SeHV4UDAAFKs+akd30O9FToiPdpX1PRrOIlGw4a5oGrzXo9zCRLTgFlvBW3C", - "8TYDYX32soDreicDwEK7x6ZPtN6patg74Xf2grlrbDRvXnACjYcfROItvFipTC7OysJNP2f0lgvmm2pc", - "n4vVmQX5bD06q83XEYmOBPV2GnkN0Qlrmnk11Wxe2QZayiLepM4LUyh0ZRpCZcjXIDa2us1zU/6RINbU", - "6RCqEr1uJef3zk59bwdpR8DcyTUGx1t4g/6wPzRJtAwYzqi38Mb9QX9srWBs6HuGM3q2HtYCY3l2X/+8", - "yUPlPl0TjZ8wwysg2yqTA1r2Eboq5yEZ8zwxSUJJ2SoxWtN2j+Hiifscgi2TsBD6S2b0T0JTqiQKEiwV", - "EpjQXBZ5WFiD7bnClXu6KAF8ay6yUoYkT+3lD4nwmlMiUZCv9Pwlq/vizsprWq9AtbUmK+NvlfcF7d1Z", - "c0MG178bo9fgBe+bOwWvQF1m9OPwbZXOb2tU3tLK2/nKwmgw6BLdctxZyzXbh543OWZqy6cRzNTh4amt", - "Le1m8vjw5OYt7oeeNz0K2T33oaoay7g97brq5082n1r5tFGHi7Qdctb1ISGz1JGy5PI78uy+/HjP/zsB", - "eyaq9w5ObfnUknZyMt5mNl8YvxJhxOCuUjRnO7mmumRfc3lQtF1Hh7wuoNmR9eLDRZtuzq982+hs98NG", - "Dw19MTxaX2y+aYujtcWzyfjZ/faDZw9lwqTFcXxpntd6OLR7oF3qrROOpeQhNYGHicGpanKpXegJfHpV", - "/0JbjdtGh4+g8aGgf0lumwwmh2c2bpD+9kbtm/34mvbj8Ky2LyQ+o4tQUx+7ZbZOB0I26oOaAa53n5kG", - "jvKHCfDddeztx6K4WLKycwQxTnby2o5rPr6+fNNH6A1XYBcy/QclN5VFkqJcSSUy18CZSjbL7ZUvlG37", - "FTc9hGWltdtAqwXH3Lgz3bo68tUzMgqhCbibzYz/kuxzlNeyS9Nn91e2duB6l+9O8GQ6Pvh3kkPTdZv9", - "m1/zfH5NZ4HyneviJxBRppnR5kgQuql8z2EFfCVwFht9Y77gsEEJX5mfGRaawTjrL9kP1FyJvcObss3R", - "fl5Kmxm6dsqEStu5pcPvwgPapiplHsYIyyWrbZrwECfQ28br9iNZ30ntTmkqEhQkPNBaQ1M8V4BAhRok", - "HMZFqibWGkhJxO/YVt6aTljP5Erd5e1tM3vPfriiWECCsY7Vj4xJjkwPvHTNZ9U8w/ZuhUyoVW94yWSM", - "Rdl5rmLB81WM7mKsYA0CpRDGGtVUk6y8sGQvM2PlZhWIdKY/Xmu9agtYZQX60TkOxyYnJTh2v2PwVOH8", - "P59ocAQ7uy++u/tQ3o1l3Xdxk4Tfye09frT0Gldxl55h7YJlnJfg7LUW1bS/ZH8zN35eXF6/NWxc3u1p", - "3OzVsgRJ1ENUoVDgTCKeK+QvGZbGjucyxwnyEY1sddHclOfM9bbmjPTQncDhbSl5TGNkfBHjn+YS3QGS", - "iiaJuUGikYoxIwkU33+xQoUTJBm/ixJ8eyAJWGbXWy8pnyoU79wp/bB7RqcIS+dHPL8FT7+RoB5285pf", - "136idHdeD37hbJkdsI3/9ul6aZR9WJvpJL34yKQjHhBbAdV2sdQbzyAIf3HonML/u590/R2dwG/seyT7", - "dt0zKLjX3mg4gXmr1xOO4d3n0OJXFpmTKjn1L/t9Y93fhnUfHv43AAD//x5uZRf3YgAA", + "H4sIAAAAAAAC/+w8a3PbyJF/ZQqXqk3qCIovUSK/5LTejVcVr62zZecuS59rgGkQswJmkJkBZUal/341", + "D4B4khQtZ5M7V7JlEZhHd0+/uwcPXsjTjDNgSnrLBy/DAqegQJhflABTVG2vf7gpnuvHBGQoaKYoZ97S", + "u40BFQPdHxEFMfQGHtXvM6xib+AxnIK3rCzpDTwBf8upAOItlchh4MkwhhTrLX4nIPKW3r+d7cA7s2/l", + "2V0egGCgQL7GKewge3wceFysMaN/xxq2vVBfMVQdi65/6AG4vuJeoNU20zOkEpStDTiZ4L9CqA7Sz41D", + "es8eOMqlvgrdBKwPUUzDaYcdPuViua8A66NdEqT6nhMKNT59a1/oRyFnCpj5E2dZQkNzgGe/So3Lgwef", + "cZoloP9MQWGCFe7gEbQBEXAJqPp84FHiLb3w4nx+CRPiRwsc+LPzKfEXeIr98/H04jy6uJxN5kGb7X3z", + "+3HgyQxCvaOj1BNWVHgtveUvD8XSYZJLBcKnxBt4G5zk+uFiOh/PRpPQjxaLS3+2CEMfB5OxvwiCxQJH", + "YUTg0nv8qKl53IEUCPxFUAX2GJrEcseCIi4QZqVOGLaYQAtGvJU0xMlrUPdc3P3zHlwBqM8spK0DzARE", + "9LO39Majofnf2eVvdkgNqh57VqiYhxySQ6cTZMaZtCKGwxAyBeSte9inH+yyMZYoAGComIYwI+ieJgkK", + "AEV5EtEk0U/lloWx4IznMtkOV+y/eY5SvEUZTxKkzIqS5yIEs0DKGVVcIKokkgqrXBoENCUS0GAM9ckE", + "mDh+qgJ7PF+BEFxomWUbnFDyySHlDeybT3W0C5QDTrbITfGOPjG7V8cRva0uG2GqqWUnIbOFgX6AuHBU", + "sqMJB4kYV0hjiylbMVzS0UogiigkRBpCwWcFgpXsIk8h1y8PTqqm0WJyMZ7744iE/iy4CPzFaA7+LILR", + "+HxGopBEO6mKOPcePx5NpAac3SydUKkQjyx5UDGnYGmLcZTgDRenIlpVOKEAM/CWGoTGi4uRPxr7o/Ht", + "aLQ0//9roXAW+DKcTy9G/mw0P/dnZIb9BcEj/2J+cUmi2SgkC7IjzXo4G8Z0HaeQDvF4NBqO18PxaB1U", + "dU6Y5X/CKU223tK7ZgoS9F/AGbpJsKIsT9HleD66Rb9/d7dN8B38wRvoGdJbzgYeofLOW05GA2+d5Rb/", + "XGM/HngppFxsveV4MRl4KSeQeEvvp/FopFUWMGKE4vWH6x+urzQwxfDp5PH4o3QHsP8E3SB7YlwElBBg", + "XybL5TI9UpxLECgUYGwWTiQi3MhRjDdQl59M0A1NYA3yGaX8HktEgFEgKNginKuYCyqdjKuYSqMUA0Ah", + "zqUdpIGqDVwxxe+AFWBTtq4DLkOeQWGcr26uS+VhcNeag323Q3jFGIQgJRbbCsqIMzMlE3xDCQiUJVhF", + "XKTmrJzFp/BsAgbke83jv/KYDQmH/8BhCsOQp5qj6wI4GU1m/ujcn45vx7PleFwVQDyfRYvJfOFP5zDy", + "Z9PxxA8uydg/n5DFlJzPF8FFxeLnTJPYa8QSTxDkwlPXU2A6D0fnl9i/hAD7s+g88BfjaOZH8ygKFpfT", + "i8V5aKdsqKScUbZ+Zwyb9fjtQyBV4ecZMKlweGeolPBc70MgwnmibZR58oKziK7185dxFm6/1//F1z+9", + "TcLpf/65CWKwCBeaEhez+YyMZ0F0eQHnowhfTObTy5HGSHOIGYvHi/nFJZ5cjifz2eKCBHgyC85n4WKO", + "R/NZhL1dKGGgulyMSRCN/BEejf0ZRKGPQbta5OIimpPpbDIzHq2Nm3aIPUGhVHkOk/16xY0FWeXW7WmK", + "5RurfmNVy6pPjaB6+XQXMqHC9baMmuI1fAWfZTKaTP3RxJ9MbseT5Wi2HE9P5cMgn0xGM38zHk7Oh3N/", + "neX++eR8eHk+HJ37FyGQ2fh8VuUM53wQQTeg7XM52nOuh46evCvrfDgf5KfJaOR97PRFJI/UPRbwAYTm", + "QhOx7FII3tJzkOmxGypUjhMnLfpd8UAz7xM0jzmWAxrHjEEqxgphASZSwYoGCaB7qmJr2us2lFm/9R2I", + "DYgftePwZZ6PNAt9sj+7nR8XXiiOrAcRJpimz+DdXDGUM/icQagjQDMM8TDMhQBSd2twbaQSmEkKTLk5", + "mJEV0yNlHoYARHshGAlQYjtE15FdiRr3RTsnIZYwQFkCWGr3J+NCIaoQliYrIWVuxYpx9SeeM/Jl5GVc", + "fYr0Mj20rURnQHbRbBmowWcq1TPQ+j3DmqsURxFlxJDHbmVwbaVbvpm8r2TyutJBJydw+l2ajoxNYc6+", + "mqW4MMc3Ws7Ol7NzfXztjPrnbcoFZzREioLwp0gvGILWaSjAOnKhDL3SOj/jPBmelpLL7/x7m495ysF8", + "iZPpCLtf17tB5iRy5kKzv8MX6hcc6ijskw0Oe3SM3ks7DnY1ly16Dv3dtW4RNVrAnMWIsUTwOdNx5rDC", + "7rKCSTNd+BIYCBo6FZ/qUHMNg5aF5Bq5ydAedgZCuZR/z6pXSIGQ4Fa1lRgNGWZE/+XC159ub2/ckJAT", + "GCJjZ6Ux0ZZN3cA3mgQTpHmIRo4OAxTk1prbdYFYSDV8goLSEbNLTurFbYry6uZaIq5i0MTDenEuoVjX", + "BvR2L40psDzV7k87AVnlq09hom2kN2jxSM5knmmzB3qu5b5Phv8H5ZomG+ANmu6BgjTjAguabD/lDG8w", + "TbRdqUwsdy0erAVmqrGreVZsWTWRIWdRQkM9PgUVc/JJv8VJwu9boKdAKC4W2SVwPg6ahbZOqWhyxgdX", + "EXCc5ioDQZEmMSsMvUFHEW9XvPjF63ejdmDxQBuYjgRrZxXyTaGQWinLNtNrZdmddnduha1itshjFefe", + "mUXlcT/6tExUHoGu7BLPQlvyXrSlMQsKUvnEvLAGwcGEhcDbXcK3CxD7pk3jqhnct7kWcRq+dfT7uZhV", + "sTiH86Dv9MgmjUsA3EpdlK5MfwJqARagV0/ak/4Sg1NO4KYjKl11iADRqg8ISnEYU1blk4DzBDDTMFWS", + "0h0gCTC5xxS9uHmPIjOuWpxDMFwPkYn8EMvTAMQAYRHGVEGocgGdbG2T2l1sbZfQfPbi5r2sTNbh1RqE", + "nm0z4V2zccpzZrgUshhSEDhBerR2Xl5+372aC2j3nfk6y+2B73Ln+3e3o8yutHPbBucYepSLOwz72Wev", + "fJY5+CNl0Qlahwius/xnW0xo7/by5n3t0DuPuVjgFbUV4j6Qm4sdD3wJYjf43XKmt6s5Bm2Bc7WV/Rz6", + "8ua9RKWl7eauPn4xKB/ikrKYs4f+nYQvEi0HiffBDmzyo5tf7F/hTEuYLtbcrdZJNg2wXbbqKLlE0MC7", + "+vmHTv+gkSXew0RlZaM4WrSbezQ/1VN9bZ6qve8ApheI7en2ysW977TfRwqzpQF4qukqQDnZeNUWeBL2", + "A3Qf08QW5KwfikLM7Nm58AspjiiLrClbMb35AN0DIpx9p4pimLQpI8wIEqBywRBVRUIOdslXhG5jbLfQ", + "gdCKBaZYZoJYM0txRECBSCkDDVoYt4G3sZPiSAd3znjWT7CWJT+W8tptfGfj2Goee0/TVqUlR9v2SpiM", + "KOuUf9vEsh8mhdevXAbLzj8mkr7VI5us4xz9EpdDrLOjQAvrHzcgtirWvj22HrYZWPAMAyCGUaKchT2a", + "2xYrOjU3TkFrCsuEPLfsUP4ITT2j21+p1jvafK8jz/kMAdNhI6kthyKadPtAlXRWc8Ub11S469ZDOtCy", + "rlzkwnjKIoGlEnm/m1WUVJrrv5cgSu0YnbZ6gwfcVlW09rGBbXDqiqoqlRQTO5+uN4sIyex1qq40k79Y", + "Ye5WaaFcNBLs+ndtZ5TJ41kx6Dch/wzqo0GW/VogxeueY9dvfquYzmx++iHr2S9dWazT77HFMkQjRLW9", + "SBIgbVSLitqBRTa2UDYoaifOqSqtVueBFoW545zpwiw8mxfpUOslXo8/0csVR4Rs5aH01Bj3MlNzfEft", + "8YjdP9SntKhTf91LnA+tnZsJAKyQnmoCTxvnWxWuZ7tkQNXnrlZNB5XUwsDDbNvthNuC6R7n+4nl0mO9", + "caMuOtzwXYX4FQ4g+WDbcjuaiU1H4Z/zAMxglOjRyHTxDjTNaIiTZGtdQ611a0k1h452IgNYMcoIfIbS", + "Z9BaQdt9w51YKRB6y//5ZeQvrvy/Yv/vH3//x+Xul/9p+PFhNJiPHysj/vDH33VJa18LfQeCfy6H2jwg", + "+jmXytRlHe4/vH5XtJTaNHqyRQm/B2GKrSiMscChtjqDIvZGXKB4m8XA5ABJhYUy7jYwly3Hu0l6aJnz", + "YcTsq1DKpULzaWVtTbME2FrFmlop/vzK/PCW8+nASykrfo47iFGtFu4JgZYPHk6SN5EpiB3jFTQCqIem", + "a98oUnZp5do1lIqvVut+DiDhbK2d1sNOVGPTtkr42FUR7olEW3XG3zz+bEB+ssHtWqebAlUCfCcPJX2K", + "CnCHIbq+2cwQJkSA1B6UHvfsnpPb/giM+7zn1qH/xk70sx34QQ48WvyPVCdthbAnXKveA3sONbAvhPpY", + "Jg36BN++/fITP0XC7d4nn3Nleguxa6ZdB6trccBz5ejbjexzZDT6Ibx1q/eFXN/JXTZJL1H1vnY5oy5P", + "q+hX2ONqld0KRzpRFW7p8KREM2TqCFAdCxcn6DwLU2JKEnR1c71jcwGY2PTcvbD9uq0wZ1+pt1bYrLxy", + "0Q43P4zLhvN1qtE0XGASO8Y5SblxPpmCz2pvQfW4K4wV77LJIraWWqHgTUdLS4+mKMeZdgPjuFfbvnbM", + "krM7xu9Zo2Gm+tP48AQar22xu5vBvkR79obZD61Ttte6bBtSFxkUTaGuIu1VigSUjY6tuHtLj2AFvh7e", + "k0XroPox+q3jvDoUcXNIh0YePFFgjIwMq4dRN67fJPCJEigh3XSHRxJSzBQNi6xJI1jbrFbk31erYeWf", + "zoCsK4HQsE4m1s4ElAmeYsvy3+Lw2gdS7Xc+IH8W08fHDtPUI5tPDoz2SHWld7SLb8x1pPuYIzeuJt7d", + "efVat+LxasJtcLya6Ou/yRn9W364DSflxHR7HcQ8z8hxmBcrHsAc1/F2yx+Ld1cPUI3kR2izW9PsXSge", + "11pSAOVSJL/m0vVD2zxArWS3Ypht61ZPj4kBJyp2/Xa2My8ABhFVKBI8RVi/YgSbjrkVKyGweA9XzOuQ", + "AYXXneEbFgFVAostUnhtlZWGwSSC2vLY3XR1VTBLsUR35b07FaUP1Lwq6k8Krw8HAgaQYs2P3fge6q3Q", + "EenRvqKmX8tJNBo2zAVV23d6nEuQmPbOeqNpG443GQjrs5cFXNeZGQAW2j02Xaj1PljD3gm/t9fXXduk", + "efOCE2g9fC8Sb+nFSmVyeVYWboY5o3dcMN9U44ZcrM8syGebyVltvo5IdCSot9PIa4hOWNPMq6lm88q2", + "51IW8TZ1XphCoSvTECpDvgGxtdVtnpvyjwSxoU6HUJXodSs5v7d26js7SDsC5savMTje0hsNx8OxSaJl", + "wHBGvaU3HY6GU2sFY0PfM5zRs824FhjLs4f6x1MeK7f12mj8jBleA9lVmRzQcojQdTkPyZjniUkSSsrW", + "idGatnsMF0/cxxZsmYSFMFwxo38SmlIlUZBgqZDAhOayyMPCBmzPFa7cAkYJ4DtzTZYyJHlqr5ZIhDec", + "EomCfK3nr1jdF3dWXtN6Daqr8VkZf6u8jWhv5pr7N7j+VRq9Bi9439xYeAnqKqMfxm+qdH5To/KOVl7j", + "Gw6T0ahPdMtxZx2XeB8H3uyYqR0fXjBTx4endjbMm8nTw5Pbd8QfB975UcjuuW1V1VjG7enWVb98tPnU", + "yoeTelyk3ZCzvs8UmaWOlCWX35FnD+Wngf7fCdgzUX1wcGrHh5y0k5PxLrP5wviVCCMG95WiOWvkmuqS", + "fcPlQdF2HR3ypoCmIevFZ5G2/Zxf+XLSWfOzSY8tfTE+Wl9sv2mLo7XFs8n42cPuc2qPZcKkw3H8wTyv", + "9XBo90C71DsnHEvJQ2oCDxODU9XmUrvQF/Dpdf37bzVumxw+gtZniP4luW02mh2e2bqf+o83at/sx9e0", + "H4dndX1/8RldhJr6aJbZeh0I2aoPaga4aT4zDRzlDxPgu8veu09RcbFiZecIYpw08tqOaz68uno9ROg1", + "V2AXMv0HJTeVRZKiXEklMpfMmUq2q92FMpTt+hW3A4RlpbXbQKsFx9znM926OvLVMzIKoQm4282M/5Ls", + "c5TX0qTps/srOztw0+S7EzyZns8JnuTQ9N2V/+bXPJ9f01ugfOu6+AlElGlmtDkShG4rX4tYA18LnMVG", + "35jvQ2xRwtfmZ4aFZjDOhiv2IzV32u7xtmxztB+v0maGbpwyodJ2bunwu/CAdqlKmYcxwnLFapsmPMQJ", + "DHbxuv0E13dSu1OaigQFCQ+01tAUzxUgUKEGCYdxkaqJtQZSEvF7tpO3thM2MLlSdzV818w+sJ/FKBaQ", + "YKxj9RNmkiPTAy9d81k1z7C7WyETatUbXjEZY1F2nqtY8Hwdo/sYK9iAQCmEsUY11SQrLyzZq9JYuVkF", + "Ir3pj1dar9oCVlmBfnKOw7HJSQmO5lcSvlQ4/88nGhzBzh6Kr/o+ljdvWf9N3yTh93L3lQC08loXfVee", + "Ye2CZZyX4Oy1FtV0uGJ/MTd+XlzdvDFsXN7tad0b1rIESTRAVKFQ4EwinivkrxiWxo7nMscJ8hGNbHXR", + "3MPnzPW25owM0L3A4V0peUxjZHwR45/mEt0DkoomiblBopGKMSMJFF+XsUKFEyQZv48SfHcgCVhm1zuv", + "QJ8qFG/dKf3YPKNThKX3E6Hfgqd/kKAedvPa3+7+QunuvR78wtkyd0O8jP/26XpplH1Ym+kkvfiEpSMe", + "EFsB1Xax1BvPIAh/cuicwv/ND8b+hk7gN/Y9kn377hkU3GtvNJzAvNXrCcfw7nNo8WuLzEmVnPp3A7+x", + "7j+GdR8f/zcAAP//DBP4glVjAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/pkg/openapi/server.spec.yaml b/pkg/openapi/server.spec.yaml index 46c238f..455cc40 100644 --- a/pkg/openapi/server.spec.yaml +++ b/pkg/openapi/server.spec.yaml @@ -353,6 +353,9 @@ components: - memory - disk properties: + baremetal: + description: Whether the flavor is for a dedicated machine. + type: boolean cpus: description: The number of CPUs. type: integer diff --git a/pkg/openapi/types.go b/pkg/openapi/types.go index 01497a9..b493eb6 100644 --- a/pkg/openapi/types.go +++ b/pkg/openapi/types.go @@ -54,6 +54,9 @@ type Flavor struct { // FlavorSpec A flavor. type FlavorSpec struct { + // Baremetal Whether the flavor is for a dedicated machine. + Baremetal *bool `json:"baremetal,omitempty"` + // CpuFamily A free form CPU family description e.g. model number, architecture. CpuFamily *string `json:"cpuFamily,omitempty"` diff --git a/pkg/providers/openstack/compute.go b/pkg/providers/openstack/compute.go index 0ee6771..4848613 100644 --- a/pkg/providers/openstack/compute.go +++ b/pkg/providers/openstack/compute.go @@ -84,6 +84,32 @@ func (c *ComputeClient) KeyPairs(ctx context.Context) ([]keypairs.KeyPair, error return keypairs.ExtractKeyPairs(page) } +// mutateFlavors allows nova's view of fact to be altered... +func (c *ComputeClient) mutateFlavors(f []flavors.Flavor) { + if c.options == nil || c.options.Flavors == nil { + return + } + + for _, metadata := range c.options.Flavors.Metadata { + index := slices.IndexFunc(f, func(flavor flavors.Flavor) bool { + return flavor.ID == metadata.ID + }) + + if index < 0 { + continue + } + + if metadata.CPU != nil && metadata.CPU.Count != nil { + f[index].VCPUs = *metadata.CPU.Count + } + + if metadata.Memory != nil { + // Convert from bytes to MiB + f[index].RAM = int(metadata.Memory.Value() >> 20) + } + } +} + // Flavors returns a list of flavors. // //nolint:cyclop @@ -107,21 +133,8 @@ func (c *ComputeClient) Flavors(ctx context.Context) ([]flavors.Flavor, error) { return nil, err } - // ************************************************************************* - // HACK HACK HACK - // ************************************************************************* - for i := range result { - f := &result[i] - - if f.ID == "c9b3b8c6-7268-4ed3-98d3-76743e3436cf" { - f.VCPUs = 128 - f.RAM = 2 * 1024 * 1024 - } - - } - // ************************************************************************* - // HACK HACK HACK - // ************************************************************************* + // Mutate any flavors first, as this may alter their selection criteria. + c.mutateFlavors(result) result = slices.DeleteFunc(result, func(flavor flavors.Flavor) bool { // We are admin, so see all the things, throw out private flavors. @@ -137,28 +150,12 @@ func (c *ComputeClient) Flavors(ctx context.Context) ([]flavors.Flavor, error) { } // Don't remove the flavor if it's implicitly selected by a lack of configuration. - if c.options == nil || c.options.Flavors == nil { + if c.options == nil || c.options.Flavors == nil || c.options.Flavors.Selector == nil { return false } - // If the selection policy is "allow all", then only reject if the flavor ID - // is in the exclusion list. If the section policy is "reject all", then only - // allow if the flavor ID is in the inclusion list. - switch c.options.Flavors.SelectionPolicy { - case unikornv1.OpenstackFlavorSelectionPolicySelectAll: - ok := slices.ContainsFunc(c.options.Flavors.Exclude, func(exclusion unikornv1.OpenstackFlavorExclude) bool { - return flavor.ID == exclusion.ID - }) - - if ok { - return true - } - case unikornv1.OpenstackFlavorSelectionPolicySelectNone: - ok := slices.ContainsFunc(c.options.Flavors.Include, func(inclusion unikornv1.OpenstackFlavorInclude) bool { - return flavor.ID == inclusion.ID - }) - - if !ok { + if len(c.options.Flavors.Selector.IDs) > 0 { + if !slices.Contains(c.options.Flavors.Selector.IDs, flavor.ID) { return true } } diff --git a/pkg/providers/openstack/provider.go b/pkg/providers/openstack/provider.go index 88ec509..5e6adec 100644 --- a/pkg/providers/openstack/provider.go +++ b/pkg/providers/openstack/provider.go @@ -254,14 +254,12 @@ func (p *Provider) Flavors(ctx context.Context) (providers.FlavorList, error) { // Apply any extra metadata to the flavor. if p.region.Spec.Openstack.Compute != nil && p.region.Spec.Openstack.Compute.Flavors != nil { - inclusions := p.region.Spec.Openstack.Compute.Flavors.Include - - i := slices.IndexFunc(inclusions, func(inclusion unikornv1.OpenstackFlavorInclude) bool { - return flavor.ID == inclusion.ID + i := slices.IndexFunc(p.region.Spec.Openstack.Compute.Flavors.Metadata, func(metadata unikornv1.FlavorMetadata) bool { + return flavor.ID == metadata.ID }) if i >= 0 { - metadata := &inclusions[i] + metadata := &p.region.Spec.Openstack.Compute.Flavors.Metadata[i] f.Baremetal = metadata.Baremetal