From bade36e2c71ae27f43678a152efa64e8dd7899ba Mon Sep 17 00:00:00 2001 From: Simon Murray Date: Fri, 18 Oct 2024 14:12:44 +0100 Subject: [PATCH] GPU Count Ambiguity We used to report logical GPUs based on what was needed for cluster autoscaling, but users may be confued if UX elements talk about logical things rather than phyiscal, so support both. And also update compilers and other jazz. --- Makefile | 2 +- charts/region/Chart.yaml | 4 +- .../region.unikorn-cloud.org_regions.yaml | 15 +- charts/region/values.schema.json | 8 +- charts/region/values.yaml | 3 +- go.mod | 61 ++--- go.sum | 60 +++++ pkg/apis/unikorn/v1alpha1/types.go | 8 +- pkg/handler/handler.go | 15 +- pkg/openapi/schema.go | 209 +++++++++--------- pkg/openapi/server.spec.yaml | 19 +- pkg/openapi/types.go | 7 +- pkg/providers/helpers.go | 2 +- pkg/providers/openstack/blockstorage.go | 7 +- pkg/providers/openstack/compute.go | 9 +- pkg/providers/openstack/provider.go | 9 +- pkg/providers/types.go | 8 +- 17 files changed, 270 insertions(+), 176 deletions(-) diff --git a/Makefile b/Makefile index 87cb06f..e518fdf 100644 --- a/Makefile +++ b/Makefile @@ -61,7 +61,7 @@ GOBIN := $(if $(shell go env GOBIN),$(shell go env GOBIN),$(GOPATH)/bin) FLAGS=-trimpath -ldflags '-X $(MODULE)/pkg/constants.Version=$(VERSION) -X $(MODULE)/pkg/constants.Revision=$(REVISION)' # Defines the linter version. -LINT_VERSION=v1.59.1 +LINT_VERSION=v1.61.0 # Defines the version of the CRD generation tools to use. CONTROLLER_TOOLS_VERSION=v0.16.1 diff --git a/charts/region/Chart.yaml b/charts/region/Chart.yaml index cce1aa8..bd32443 100644 --- a/charts/region/Chart.yaml +++ b/charts/region/Chart.yaml @@ -4,8 +4,8 @@ description: A Helm chart for deploying Unikorn's Region Controller type: application -version: v0.1.43 -appVersion: v0.1.43 +version: v0.1.44 +appVersion: v0.1.44 icon: https://raw.githubusercontent.com/unikorn-cloud/assets/main/images/logos/dark-on-light/icon.png diff --git a/charts/region/crds/region.unikorn-cloud.org_regions.yaml b/charts/region/crds/region.unikorn-cloud.org_regions.yaml index 5605d18..89d9026 100644 --- a/charts/region/crds/region.unikorn-cloud.org_regions.yaml +++ b/charts/region/crds/region.unikorn-cloud.org_regions.yaml @@ -104,9 +104,10 @@ spec: GPU defines additional GPU metadata. When provided it will enable selection of images based on GPU vendor and model. properties: - count: - description: Count is the number of logical - GPUs in the flavor. + logicalCount: + description: |- + LogicalCount is the number of logical GPUs e.g. an AMD MI250 is 2 MI200s. + This is primarily for scheduling e.g. autoscaling. type: integer memory: anyOf: @@ -121,6 +122,11 @@ spec: Model is a free-form model name that corresponds to the supported models property included on images, and must be an exact match e.g. H100. type: string + physicalCount: + description: |- + PhysicalCount is the number of physical cards in the flavor. + This is primarily for end users, so it's not confusing. + type: integer vendor: description: |- Vendor is the GPU vendor, used for coarse grained flavor and image @@ -130,9 +136,10 @@ spec: - AMD type: string required: - - count + - logicalCount - memory - model + - physicalCount - vendor type: object id: diff --git a/charts/region/values.schema.json b/charts/region/values.schema.json index 7d6cab1..e7b8246 100644 --- a/charts/region/values.schema.json +++ b/charts/region/values.schema.json @@ -258,13 +258,17 @@ "gpu": { "type": "object", "required": [ - "count", + "physicalCount", + "logicalCount", "memory", "model", "vendor" ], "properties": { - "count": { + "physicalCount": { + "type": "integer" + }, + "logicalCount": { "type": "integer" }, "memory": { diff --git a/charts/region/values.yaml b/charts/region/values.yaml index f31a74c..918afcc 100644 --- a/charts/region/values.yaml +++ b/charts/region/values.yaml @@ -62,7 +62,8 @@ organization: unikorn-cloud # vendor: NVIDIA # model: H100 # memory: 192Gi -# count: 2 +# physicalCount: 2 +# logicalCount: 2 # # Image service configuration. # image: # # Image selection, the result is a boolean intersection of chosen options. diff --git a/go.mod b/go.mod index c210490..2f0cc20 100644 --- a/go.mod +++ b/go.mod @@ -1,20 +1,20 @@ module github.com/unikorn-cloud/region -go 1.22.5 +go 1.23.2 require ( - github.com/getkin/kin-openapi v0.127.0 + github.com/getkin/kin-openapi v0.128.0 github.com/go-chi/chi/v5 v5.1.0 - github.com/gophercloud/gophercloud/v2 v2.1.0 + github.com/gophercloud/gophercloud/v2 v2.1.1 github.com/gophercloud/utils v0.0.0-20231010081019-80377eca5d56 github.com/oapi-codegen/runtime v1.1.1 github.com/spf13/pflag v1.0.5 - github.com/unikorn-cloud/core v0.1.72 - github.com/unikorn-cloud/identity v0.2.40 - go.opentelemetry.io/otel v1.29.0 - go.opentelemetry.io/otel/sdk v1.29.0 - go.opentelemetry.io/otel/trace v1.29.0 - golang.org/x/crypto v0.26.0 + github.com/unikorn-cloud/core v0.1.76 + github.com/unikorn-cloud/identity v0.2.42 + go.opentelemetry.io/otel v1.31.0 + go.opentelemetry.io/otel/sdk v1.31.0 + go.opentelemetry.io/otel/trace v1.31.0 + golang.org/x/crypto v0.28.0 k8s.io/api v0.31.1 k8s.io/apimachinery v0.31.1 sigs.k8s.io/controller-runtime v0.19.0 @@ -46,7 +46,7 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/gophercloud/gophercloud v1.14.0 // indirect + github.com/gophercloud/gophercloud v1.14.1 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect @@ -54,8 +54,9 @@ require ( github.com/invopop/yaml v0.3.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/masterminds/semver v1.5.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -63,36 +64,36 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.20.2 // indirect + github.com/prometheus/client_golang v1.20.5 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/common v0.60.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/x448/float16 v0.8.4 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.29.0 // indirect - go.opentelemetry.io/otel/metric v1.29.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect - golang.org/x/net v0.28.0 // indirect - golang.org/x/oauth2 v0.22.0 // indirect - golang.org/x/sys v0.24.0 // indirect - golang.org/x/term v0.23.0 // indirect - golang.org/x/text v0.17.0 // indirect - golang.org/x/time v0.6.0 // indirect + golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/oauth2 v0.23.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/term v0.25.0 // indirect + golang.org/x/text v0.19.0 // indirect + golang.org/x/time v0.7.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 // indirect - google.golang.org/grpc v1.65.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect + google.golang.org/grpc v1.67.1 // indirect + google.golang.org/protobuf v1.35.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/client-go v0.31.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20240826222958-65a50c78dec5 // indirect - k8s.io/utils v0.0.0-20240821151609-f90d01438635 // indirect - sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + k8s.io/kube-openapi v0.0.0-20241009091222-67ed5848f094 // indirect + k8s.io/utils v0.0.0-20240921022957-49e7df575cb6 // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) diff --git a/go.sum b/go.sum index c6d011a..6f4932d 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,8 @@ github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/getkin/kin-openapi v0.127.0 h1:Mghqi3Dhryf3F8vR370nN67pAERW+3a95vomb3MAREY= github.com/getkin/kin-openapi v0.127.0/go.mod h1:OZrfXzUfGrNbsKj+xmFBx6E5c6yH3At/tAKSc2UszXM= +github.com/getkin/kin-openapi v0.128.0 h1:jqq3D9vC9pPq1dGcOCv7yOp1DaEe7c/T1vzcLbITSp4= +github.com/getkin/kin-openapi v0.128.0/go.mod h1:OZrfXzUfGrNbsKj+xmFBx6E5c6yH3At/tAKSc2UszXM= github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E= @@ -68,8 +70,12 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/gophercloud/gophercloud v1.3.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= github.com/gophercloud/gophercloud v1.14.0 h1:Bt9zQDhPrbd4qX7EILGmy+i7GP35cc+AAL2+wIJpUE8= github.com/gophercloud/gophercloud v1.14.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= +github.com/gophercloud/gophercloud v1.14.1 h1:DTCNaTVGl8/cFu58O1JwWgis9gtISAFONqpMKNg/Vpw= +github.com/gophercloud/gophercloud v1.14.1/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= github.com/gophercloud/gophercloud/v2 v2.1.0 h1:91p6c+uMckXyx39nSIYjDirDBnPVFQq0q1njLNPX+NY= github.com/gophercloud/gophercloud/v2 v2.1.0/go.mod h1:f2hMRC7Kakbv5vM7wSGHrIPZh6JZR60GVHryJlF/K44= +github.com/gophercloud/gophercloud/v2 v2.1.1 h1:KUeVTUoq6um/CijR+hl1JRZ35SXRY62LiYUbgp4qqKw= +github.com/gophercloud/gophercloud/v2 v2.1.1/go.mod h1:f2hMRC7Kakbv5vM7wSGHrIPZh6JZR60GVHryJlF/K44= github.com/gophercloud/utils v0.0.0-20231010081019-80377eca5d56 h1:sH7xkTfYzxIEgzq1tDHIMKRh1vThOEOGNsettdEeLbE= github.com/gophercloud/utils v0.0.0-20231010081019-80377eca5d56/go.mod h1:VSalo4adEk+3sNkmVJLnhHoOyOYYS8sTWLG4mv5BKto= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= @@ -91,6 +97,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -99,6 +107,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/masterminds/semver v1.5.0 h1:hTxJTTY7tjvnWMrl08O6u3G6BLlKVwxSz01lVac9P8U= +github.com/masterminds/semver v1.5.0/go.mod h1:s7KNT9fnd7edGzwwP7RBX4H0v/CYd5qdOLfkL1V75yg= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -125,10 +135,14 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg= github.com/prometheus/client_golang v1.20.2/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/common v0.60.0 h1:+V9PAREWNvJMAuJ1x1BaWl9dewMW4YrHZQbx0sJNllA= +github.com/prometheus/common v0.60.0/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= @@ -144,8 +158,12 @@ github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65E github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/unikorn-cloud/core v0.1.72 h1:MvxIZOXeacolvVvS06stO+S4zxplxSz+LUL+8sz5dYc= github.com/unikorn-cloud/core v0.1.72/go.mod h1:7ilhT3GIRtBHl7/D+9wh4RB5gjC/5/ts/MT4WdpIT9M= +github.com/unikorn-cloud/core v0.1.76 h1:h9TsNTYimmu7N23RB3J7PLKp+ekJF7vM9AF1yMuVgIo= +github.com/unikorn-cloud/core v0.1.76/go.mod h1:S9AF4PwTQljImb9w0P2jKjzRe8fLM+rx+ZbxrAHw/yE= github.com/unikorn-cloud/identity v0.2.40 h1:Y2H2cCUumDOiaCsJP5VjgVezgpB+EchZogK5Qg3G408= github.com/unikorn-cloud/identity v0.2.40/go.mod h1:DbS+R9Om75HBQPMG02Iu3j2tjeE552KwAEcdAl5kPJE= +github.com/unikorn-cloud/identity v0.2.42 h1:9amEcydDq23RZYO4rTtxOhVgw/BH1mdXQgq0fWT+RM0= +github.com/unikorn-cloud/identity v0.2.42/go.mod h1:JMbS6iTYzt0OVt5AkqZys3WVnpLabGvUl8kGWcxzFZI= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -153,16 +171,28 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0 h1:dIIDULZJpgdiHz5tXrTgKIMLkus6jEFa7x5SOKcyR7E= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0/go.mod h1:jlRVBe7+Z1wyxFSUs48L6OBQZ5JwH2Hg/Vbl+t9rAgI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 h1:K0XaT3DwHAcV4nKLzcQvwAgSyisUghWoY20I7huthMk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0/go.mod h1:B5Ki776z/MBnVha1Nzwp5arlzBbE3+1jk+pGmaP5HME= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.29.0 h1:JAv0Jwtl01UFiyWZEMiJZBiTlv5A50zNs8lsthXqIio= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.29.0/go.mod h1:QNKLmUEAq2QUbPQUfvw4fmv0bgbK7UlOSFCnXyfvSNc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0 h1:lUsI2TYsQw2r1IASwoROaCnjdj2cvC2+Jbxvk6nHnWU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0/go.mod h1:2HpZxxQurfGxJlJDblybejHB6RX6pmExPNe517hREw4= go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo= go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok= +go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= +go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -180,8 +210,12 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= +golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -194,8 +228,12 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -211,10 +249,14 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -222,8 +264,12 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -239,12 +285,20 @@ gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 h1:YcyjlL1PRr2Q17/I0dPk2JmYS5CDXfcdb2Z3YRioEbw= google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo= +google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1:fVoAXEKA4+yufmbdVYv+SE73+cPZbbbe8paLsHfkK+U= +google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4= google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 h1:2035KHhUv+EpyB+hWgJnaWKJOdX1E95w2S8Rr4uWKTs= google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -269,12 +323,18 @@ k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240826222958-65a50c78dec5 h1:6OWzFh8WiQWeeE7apc3kRV3z0CzprqBxVjntsPA0ed4= k8s.io/kube-openapi v0.0.0-20240826222958-65a50c78dec5/go.mod h1:i67DWA0Mm5+JPl+R2ku1eehbRGBDthd8+S2jS9nKLQk= +k8s.io/kube-openapi v0.0.0-20241009091222-67ed5848f094 h1:MErs8YA0abvOqJ8gIupA1Tz6PKXYUw34XsGlA7uSL1k= +k8s.io/kube-openapi v0.0.0-20241009091222-67ed5848f094/go.mod h1:7ioBJr1A6igWjsR2fxq2EZ0mlMwYLejazSIc2bzMp2U= k8s.io/utils v0.0.0-20240821151609-f90d01438635 h1:2wThSvJoW/Ncn9TmQEYXRnevZXi2duqHWf5OX9S3zjI= k8s.io/utils v0.0.0-20240821151609-f90d01438635/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240921022957-49e7df575cb6 h1:MDF6h2H/h4tbzmtIKTuctcwZmY0tY9mD9fNT47QO6HI= +k8s.io/utils v0.0.0-20240921022957-49e7df575cb6/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q= sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= diff --git a/pkg/apis/unikorn/v1alpha1/types.go b/pkg/apis/unikorn/v1alpha1/types.go index 15bd35f..d68d40a 100644 --- a/pkg/apis/unikorn/v1alpha1/types.go +++ b/pkg/apis/unikorn/v1alpha1/types.go @@ -174,10 +174,14 @@ type GPUSpec struct { // Model is a free-form model name that corresponds to the supported models // property included on images, and must be an exact match e.g. H100. Model string `json:"model"` + // PhysicalCount is the number of physical cards in the flavor. + // This is primarily for end users, so it's not confusing. + PhysicalCount int `json:"physicalCount"` + // LogicalCount is the number of logical GPUs e.g. an AMD MI250 is 2 MI200s. + // This is primarily for scheduling e.g. autoscaling. + LogicalCount int `json:"logicalCount"` // Memory is the amount of memory each logical GPU has access to. Memory *resource.Quantity `json:"memory"` - // Count is the number of logical GPUs in the flavor. - Count int `json:"count"` } type RegionOpenstackImageSpec struct { diff --git a/pkg/handler/handler.go b/pkg/handler/handler.go index 9a19c71..7ef42b9 100644 --- a/pkg/handler/handler.go +++ b/pkg/handler/handler.go @@ -34,7 +34,6 @@ import ( "github.com/unikorn-cloud/core/pkg/server/conversion" "github.com/unikorn-cloud/core/pkg/server/errors" "github.com/unikorn-cloud/core/pkg/server/util" - coreutil "github.com/unikorn-cloud/core/pkg/util" identityclient "github.com/unikorn-cloud/identity/pkg/client" "github.com/unikorn-cloud/identity/pkg/middleware/authorization" identityapi "github.com/unikorn-cloud/identity/pkg/openapi" @@ -47,6 +46,7 @@ import ( kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -156,15 +156,16 @@ func convertFlavor(in providers.Flavor) openapi.Flavor { } if in.Baremetal { - out.Spec.Baremetal = coreutil.ToPointer(true) + out.Spec.Baremetal = ptr.To(true) } if in.GPU != nil { out.Spec.Gpu = &openapi.GpuSpec{ - Vendor: convertGpuVendor(in.GPU.Vendor), - Model: in.GPU.Model, - Memory: int(in.GPU.Memory.Value()) >> 30, - Count: in.GPU.Count, + Vendor: convertGpuVendor(in.GPU.Vendor), + Model: in.GPU.Model, + Memory: int(in.GPU.Memory.Value()) >> 30, + PhysicalCount: in.GPU.PhysicalCount, + LogicalCount: in.GPU.LogicalCount, } } @@ -240,7 +241,7 @@ func convertImage(in providers.Image) openapi.Image { } if in.KubernetesVersion != "" { - out.Spec.SoftwareVersions.Kubernetes = coreutil.ToPointer(in.KubernetesVersion) + out.Spec.SoftwareVersions.Kubernetes = ptr.To(in.KubernetesVersion) } if in.GPU != nil { diff --git a/pkg/openapi/schema.go b/pkg/openapi/schema.go index 7525377..587ca40 100644 --- a/pkg/openapi/schema.go +++ b/pkg/openapi/schema.go @@ -19,110 +19,111 @@ import ( // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x9e3PbuPXoV8HwdmbbuaKsty390+vN7mY9m03cvHpvV7kZiDwUsSYBFgDlqB5/99/g", - "wZcISrLibNLW0+7EIvE6B+ccnBcO77yApRmjQKXwFndehjlOQQLXv0gIVBK5vfrhuniuHocgAk4ySRj1", - "Ft7bGFDR0P4REeB9r+cR9T7DMvZ6HsUpeIvakF7P4/DPnHAIvYXkOfQ8EcSQYjXFnzhE3sL7X2fV8s7M", - "W3F2k6+AU5AgXuIUqpXd3/c8xteYkn9htba9q76kqN4WXf3QseDmiHsXLbeZ6iEkJ3Stl5PFW0ECnLwE", - "ecv4zUE8Fu0RNR0O47M1wxdBa8bZ7xDIw+s37ZBaXNeCi6G+yEI5rA/tvFqnaXYYu8VwX2Ct92ZIEPJ7", - "FhJo8Ntr80I9ChiVQPWfOMsSEmhCPPtdKFjuPPiE0ywB9WcKEodYYgetow3wFROA6s9bLOnr3/c9T2QQ", - "qFEs9KG38ILz6ewCRqEfzfHKn0zHoT/HY+xPh+PzaXR+MRnNVl7Pk3gtvMVvd8XQQZILCdwnodfzNjjJ", - "1cP5eDacDEaBH83nF/5kHgQ+Xo2G/ny1ms9xFEQhXHj3HxSGjkNyAcDfOZFgULuLAItqFDGOMC3lVb+1", - "sW2m/WM3o5jctxKgtSkh1aQkgG+0lP7Nu+jr/3kfFJ9CRD55C284H/WHs4v+oD84G02+2s7soPLYDWrJ", - "wb7CwD9zJrE4aTuiBG8YNwgIWK76XPQ8oih7ji+C2fh84E8Gs6k/CSfYn4d44J/Pzi/CaDIIwnmoYD4W", - "ZLPKN2q39gIqGRIgkWnet8JLZIwKIwtwEEAmIXxtH3YJMjNajAVaAVBUdEOYhuiWJAlaAYryJCJJop6K", - "LQ1izijLRbLtL+n/YzlK8RZlLEmQ1CMKlvMA9AApo0QyjogUSEgsc6E3SIGdgFqG3pgVDu2u1Bd7/O4A", - "54wrQUQ3OCHhRwuU1zNvPjbBLkBesXCLbBfv6O0xczl25nV92AgThS3TCekp9Op7iHGLJdM6ZCAQZRIp", - "aDGhS4pLPBqxgiICSSg0ouCTBE5LdhCnoOu3O0O3wTiaj86HM38YhYE/WZ2v/PlgBv4kgsFwOgmjIIwq", - "sRIx5t1/OBpJO+t0U3JChEQsMuhBRZ+CZQ3ElvFOBLQuRQMOuuFbogEazs8H/mDoD4ZvB4OF/v8/vIew", - "dImadX/Sj8k6TiHt4+Fg0B+u+8PBelUXukGW/4RTkmy9hXdFJSTo/wKj6DrBktA8RRfD2eAt+vObm22C", - "b+AvXk/1EN5i0vNCIm68xWjQ89ZZbuDX8mfY81JIGd96i+F81PNSFkLiLbyfh4OBEslAQ80UL99f/XB1", - "qRZTNB+P7o/fykLy7d1B28jsGOMrEoZAP4+Xy2E6uDgXwFHAQR/EOBEoZJqPYryBJv9knGxIAmsQj8jl", - "t1igECiBEK22COcyZpwIy+MyJkILxRWgAOfCNFKLajRcUslugBbLJnTdXLgIWAaFxnF5fVUKDw27khz0", - "uwrgJaUQgBCYb2sgI0Z1l4yzDQmBoyzBMmI81Xtl1RgCj8ZgEH6vaPx3FtN+yOD/4CCFfsBSRdFNBhwN", - "RhN/MPXHw7fDyWI4rDMgnk2i+Wg298czGPiT8XDkry7CoT8dhfNxOJ3NV+erigFzqlDs7RhvD2DkwqRQ", - "XWA8CwbTC+xfwAr7k2i68ufDaOJHsyhazS/G5/NpYLpsiCCMErp+ow82Y5qYhxDWmZ9lQIXEwY3GUsJy", - "NU8IEc4TdUbpJ88YjchaPX8eZ8H2e/VffPXz6yQY/+2X3SWu5sFcYeJ8MpuEw8kqujiH6SDC56PZ+GKg", - "IBIi/gW2L1NcyO6eZhjdHQ/ns/MLPLoYjmaT+Xm4wqPJajoJ5jM8mE0i7FVmkF7oxXwYrqKBP8CDoT+B", - "KPAxKM09PD+PZuF4Mppozd3YrhWsD5AxdTLE4X5RY9uCqBPw9jRZ80S9T9TbSb0PNR47SbeyFlGhoBva", - "TfEavoBmMxqMxv5g5I9Gb4ejxWCyGI5PJc1VPhoNJv5m2B9N+zN/neX+dDTtX0z7g6l/HkA4GU4ndWKx", - "KkrIyQbUKV629qyCog3OS6OiWE3l59FgoGxPh8YiWCRvMYf3wBVharum8oh4C8+uTLXdEC5znFgGUu+K", - "B4qeHyCM9LYcEEK6DZIxlghz0PYMlmSVALolMjYKQPOkpUa7faPN7h+VevF5+pGx3z+an24VyRohkiGj", - "ZwQJJukj6ECXFOUUPmUQKDtRN0MsCHLOIWwqP7jRUnJMBQEqbR9MwyVVLUUeBACh0lUw4iD5to+uIjMS", - "0UqOUmECLKCHsgSwUEpSxrhERCIstENGiNywFWXyJ5bT8PPQS5n8GKlhOnBbs+EgrGze0pyDT0TIR8D1", - "O4oVVUmGIkJDjR4zlYa15Wl6OgW/0Cm412vWOCKt+aoXcjGawEUwCfzpxfTCn6wGI38+H8z88XwA48l0", - "NlxFY2WrJVjDOhyMJvf7nHB/6MHWIq4u1czpbNvp/WRgPBHno9oMbfo6ZDnskqloOIVPkZzfkFdYgIbR", - "NLNO8CBRh22/0oi/mLJ5rplssJhMF5OpYrJ2rPTTNmWcURIgSYD7Y6QGDECpRWiFBYSIUPRCqY0ZY0m/", - "YNQj40YFo974t8bx+xD2iQDLnBuv+S5ZlUGdz6FVi/395Gkb6e3KqXUU/Qs+U4/BQQBCfDSuqg5dRs2l", - "DBQzmvVdP4ae6Bq38GGZhVnNNMYCwaeMcAj7tRNK1CDZDV48BwqcBFaVTEEIvIZeSxNnCrhR31BEBlza", - "SGnHqJdIAhdgRzWBeLUyTEP1l3Wm/fz27bVtErAQ+kjr80KbAoaWbcNXCgUjpAiNRBYPPbTKjdVgxoXQ", - "rFStjxOQmG+LUIka3DDz5fWVQEzGoJCH1eBMQDGucS+auRSkQPNUHQLtcEidrj4a8eD1WjSSU5FnSr0G", - "1ddQ30dN/71yTO2bVKZ60wyRkGaMY06S7cec4g0midJfax3LWYsHa46p3JlVPyumrKviAaNRQgLVPgUZ", - "s/CjeouThN22lp5CSHAxSOVO/tDbzbNwcsUuZby3QVdLaTb4uiqctnqEvtdz5HBU8eHfvG5zrVoWWyld", - "wRHucSahvCoEUiuA0iZ6JVHdQUBrvpgklhZ6jHTd27NI2NgPPinDJkeAK1zsWUhL1gm20GeHhFQ8MErl", - "VVIec463VfjJtRDzpo3j+lm5b3LF4iR4bfH3a9GrdiwdjspYdaCJ43IBdiQXpk33vylloRs2o0y0IbSa", - "joscaJ6ugKv9aUYwS1zZlRAqYQ3ceHDdQ9k1OCnSRVVmVQeAfUGE3EdVdcCPp6M6Njtp6E2hqx9JRyvM", - "QW1l0u709xjsSVCiiRQ6XwihOmcgRCkOYkLrTLliLAFM1Zpq8UjHkjjosFOKnl2/Q5FuV082QdBf95F2", - "59kd7yHMg5hICJQi5ZQhJp65n2ieXb8TbioxQVBXb5yqjVe9IYshBY4TpFordfL59+7RrJdy356us9xw", - "VxU23T+7aaVnJc5pd4hW46Mc3ELYTb7iMNk+lGBdtLrO8l9NHLk92/Prd41Nd25zMcAhTtsd7PjFl0t0", - "L9/NZ2q6hhZ2olh7fv1OoFKtcVNXF71okA9RSRnH34N/J+IL7/lB5L03DXfp0fYv5q9RZrdkrUZzok0t", - "2Axb10qtd7/nXf76g1MZ24kG7iGiMqhdbC2q+h5NT834TZumGu8di+lcxPZ05cD6i94oJTssdAS1gIfq", - "CcVSTtYUGgM8CPoeuo1JYnIxjNKPAkzN3llbF0mGCI3MUbakavIeugUUMvqdLNQHYeIAmIaIg8w5RUQW", - "URaoImoIvY2xmUJZnUu60nkS2q2ge0mGQpDAU0JBLS2I24s3hqpkSFnS9vBs7mDDm3Ys5pWO/sY4Depu", - "sj2JxbUUU3W21xwXiFAn/5v8zP1rknj9woYlTP9j3BZvVctd0rFWVQnLIdKpMNCC+scN8K2MlSGFjTmj", - "GxY0QwFCTShRToMOyW2C0k7JjVMo1FHdTJND+SPQcWu3vlKPa7fpXpn5swkCqmz0sDEcikji1oFqbuDd", - "Ea9t4nuVUY6UVWtUucj6TAiNOBaS591qlrHHn3OWZ65pTPgRrdX7Q3PJg3OZML3TOPyxVMbevPkZ3cAW", - "rYECb05Rk5OtwYuA/+7A7wTwUsxHp6Hpfg+1mhRjl6Vdi+Jrf8rp4r2wmvVcp4p03fmz5Xo1SgvkItWt", - "utJjcne1A9hwa/dJ9y1IuR207BdWKV53bLt687XsfD356Zusej+3KRlO9cwkaiASIaKOtSSBsA1qkc1x", - "YJCNSdLoFXF7q/uVh6tzQ4ukkON0/uL0ejRl14LWibwOtaeTKo6wLMtN6chv2UtMu+0deS9HzP6+2aWF", - "nebrTuS8b82866fAEqmu2j427ggjoFVv67Oomwb1jJ1ezQPS8zDdum0Fk6yzx0Z4YKrOsUaDFhcuayHb", - "TC7DkIMQTrq5ut5MEDYNnAxRG+CQDV0f6yEGT22JDgiq/KoXeAXJe3O1x3ELSWft/5KvQDdGiWqN9E2g", - "ntp1EuAk2RodXJ0bDVex3RClra9gSQkN4ROUypmSa0rB0vyFpQSupvz/vw38+aX/D+z/68Of/7qofvkf", - "+x/uBr3Z8L7W4i9//ZMLvV336RwA/lI2Nd5t9GsupM5qsrD/8PJNcW3DBIeSLUrYLXCdqoSCGHMcqHOz", - "Vzg5EOMo3mYxUNFDQmIutV0D1MaAcNVJNS2dazTU80qUMiHRbFwbW+EsAbqWscJWij+90D+8xWzc81JC", - "i59DBzLq6Qx7bM3FnYeT5FWkY8HH6DU7lurdrg21k0XhOlcad2trimrjhtEKEkbXyjo47CTembQt1D64", - "8qk6TP5WwstXN/QdKz9Zbegay42JOiK+E4e8bLupKkeLquLoP9oA34GisEJFYYcXiSxdQtq8L42K2la3", - "+Og/wqQvgShR09vdLTvHETTTxLbz8nwZwDvMTbWsJaesKIcqbr13xDE5y6XbqmwOY9p1jSLyFQV5eBTT", - "rmuUIq3KGVUoTdr3Ly5fNkeo4gltrHfZru2yAF/XhHUte7+0OoLk9liyX0NOPaJ4+Ryjt4uVj0DoYfd7", - "O6HuSBXUmVraVkUPn45HqyZHqjptZWWPz65esOIxVJRqKrd2UssBXOwusxYlPDJUXZJLiw7MSfBTLSuu", - "I9uwSJyzIOtjTt8FN351JJn14oo2g7UT7XanuYrUPIU7gZq4tjUcahPahB/RJkdzwYBRQCJmeaJ16Pqx", - "q71ZpoyB9jDnNoygDNQsIQEx2mcMXBmrS+qaVJkpvrZSCwtXGH1dxiAApdZkqE2rVlSvUKBFPMigv6SO", - "UP0uiexi7UPn/nVxr3n7+ZL/FC3VzH2yYlrr3qYWqgx4s2N4xfI6jbSBrad8Hl5wyQqPFjgp5+8G862d", - "qEuh/E5U8Ss1RN2RUinJHzrV1b1ekzIZ9UiBXiM5hxznu95Ph6/ZysuCDKyJrZNakgRdXl9VMpUDDk1A", - "8Jabm6Ct03tfJl8jb632ykoapn9o3wXO16kC05zQOLUuo5RpPxKV8EnuzZc7rrBPzc2ySy0mVa6GwWtH", - "WnPHsVS209mk2gdXvz1UEUtObyi7pTtJ0/Wf2h0Xws5rk8voJrDPOao7PeZ3rV02NURMKroLDZKk0DyP", - "zb39BKRxdBuZ4S28EEvwVfOOuJ0D68cIScd+OU793SaO47/3QIbRPNKvb0ZTU3/iwAdyoIB04/YTCkgx", - "lSQoAiA7XsvNchn+7+WyX/vH6Zl0xQJ2jjjtNs84lLGaYsry32Lz2htSvzZ7gP8MpE7VsIM3H+wh3MPV", - "tVteLrrRtS9uY4ZsuwZ7uyP5jRsrx4sJO8HxYqIrlTWn5J/54SzrlIU6mf8g5HkWHgd5MeIByHETbjv8", - "sXA7k3HrKD9Cmr3Vd4YLwWOTWYtFWZX/d6VK62u1RsFuJAktKabb5qmn2sSAExnb6xTm4sUKKEREooiz", - "FGH1ioZYX4hY0nIFBu6GRl7xgMRrp1mP+YpIjvkWSbw2wkqtQUdEHD4tZ9rEZUEsxRBu35E7JqM2VL8q", - "Ml4kXh+2OvVCijE/uOE9FImSeH28rqjw11IStYQNck7k9o1qZyMF+vZO8x5Rex2vMuBG8S9TxuzFmxVg", - "rtRjfcmoec1Jk3fCbk2tNHsrRr95xkJoPXzHE2/hxVJmYnFW5mD0c0puGKe+zv/pM74+M0s+24zOGv2V", - "WROwTIOlgFcrOmFM3a8hmvUr4w0kNGJt7DzTqUnWbg2JCNgG+Nbk07FcZ3II4BtiZQiRiRq3Fvx6bbq+", - "MY2UIqDLS+kDx1t4g/6wPyw88zgj3sIb9wf9sTkFY43fM5yRs82w4YURZ3fN0qj3tTowbTB+xRSvIax8", - "6HbRoo/QVdmvZukLQteJlpomXx0XT6zJbzIeaAD9JdXyJyEpURZ9goVEHIckF0VAEjZgsrxxreQUSgDf", - "6JpMhCLBUlOhQCC8YSQUaJWvVf8lberi9pRXuF6DdN1rk1rfKuvcmDJQuowDbtacVWOwgvb1vdvnIC8z", - "8n74qo7nVw0sV7jydgoGjgaDLtYt2505Kkbd97zJMV0dVf501+Hhrs77kLrz+HDndkGy+543PQrYPUU7", - "6hJLqz1uWfXbBxNYrJVF7lCRqiZnXUWI9VBH8lLhKaKd/jXDUQ6/WR+h67YvjUP1Qx/ItsZHdcuH8SUt", - "kzYQZeGOHWrZ8P2Ly5d9hF4yCWYgHTgv2bN0apQljAXStUWoTLbL6n4fyqpEwG0PYVFL/tarVbunr1fq", - "fF51UqkeGYFAH5DtLEEnP77QHV2e7gez3vXulpzCgJ11FZ7Y8JtkQ+PTF2d3Zd3q/7pz7pGw3jvY1VFl", - "XNkaGXNpr8+0eYcwonBbi8zTHb9xk8uvmTjM5nbLr4vV7By5Rc3ubTfl18p6n+3W9L5vSY3h0cf29kla", - "HC0tHo3Hz+6qbxbcl35Lh/32g37eSBRRWrqybCtbGAvBAqLtf+0KI7JNpWagz6DTq+ZHFhrUNjq8Ba3S", - "0/+W1DYZTA73bFUb++MPtf9iO6lYav0ewsPVstP4YPAkdb9lHe00beFwL9cnbR5RIWwcFk9G3MONuH9L", - "8jlKR3XlST6qdlpJO6el+kC9teMrKCepr111Lp/k6dfRYluC6eyu9R2po1TdIwn7ERXaXdK+dn7/6r9T", - "4/0GFNenk2y/O9KovVm8JWIvxzyW3nsCuwye5PmTfrx96GTdHzr8Ytq1yad21mJQLC8Ug/q2DIVui1It", - "o1KgsqfYWFvC2vA1n2q5JTK2lnDDgxOTEJbUlG+0CdEm/wBwEJuyjn2ELtdrDmt7402gGNMwKaqmZ9gI", - "nOI7MgGjkrMkAd5f0mtTEp2WEsxWgQ0wpUynHwONGA+MkLIQ9Yz8s+Bd6quYOAj0unGiRFsuysIt338n", - "iksphNE+Qs/McwW2EqW15W4INrkHOuYucp1G0EOCISK/E0tKUiUvMZXFDVAFhEC6WkstFq3WwnIqRc+U", - "U6ehyX/Rw4pO8XhpV+yX5WjahXF3S098GcH5N0Ncp0jHnZLET0LxP9Rp0POyXLpqtMiW5DmGcq/zR6fc", - "B5p8zQ8s3j8p0t+YideZ+P/a1uMKISIUdOlwlof6Mkz1MY81sDXHWawVcf35ji1K2Fr/zDBXZMRof0l/", - "JPqgu8XbshKI+QKZOi3Jxp5RRJjSAJJVIY0qBVDkQYywWNLGpOoASqBX5cGY76h9JxA3F9tCtErYSsl7", - "hfFcgr1Y86M6Z20KVKxUcykQu6WVct+OqvQ0x9mK2lVZqp65VFQMYE+X+nfoBEO6mpWw1Q3q+TtVlTSR", - "EKP34yUVMeZlDSkZc5avY3QbYwkb4CiFIFagpgplZelBU2FaHaC6VwHI/jQGkxhe3ux48MFnyeSkQ223", - "Av3nMud/fOaARdjZXfEN6fuyYHG3O/gySditqIqro6XXqo+89DRpFyRjzWdryCpWTftL+nddu+/Z5fUr", - "TcZllb5WuWXFS5BEPUQkCjjOBGK5RP6SYq0/o1zkOEE+IpHJ2tflyxm15V9yGvbQLcfBTcl5VEGkjXQd", - "cMoFugUkJEkSXQtOAWV0zeLjP4apcIIEZbdRgm8OWc9F1qqzcvSpTPHa7tKPu3t0CrN0fuf1KRr6zWiC", - "7S/FfyZ3dxb6fWbPMlvruQzo7pP12l4rT0E7dHV3tpYLDdayUudiKTcegRF+suCcQv+7X/39ikrgE/ke", - "Sb5dpbgK6jVFv04g3noFr2No9zGk+JUB5qSgf/Ozjk+k+8eQ7v39/wQAAP//Ghx7uouGAAA=", + "H4sIAAAAAAAC/+x9e3PbuPXoV8HwdmbbuaSsty390+vN7mY9m03cvHpvV7kZiDyUsCYBFgDlqB5/99/g", + "wZcISrLibNLW0+7EIvE6B+ccnBcO77yQpRmjQKXw5ndehjlOQQLXv0gEVBK5vfrhuniuHkcgQk4ySRj1", + "5t7bNaCiof0jJsB7nu8R9T7Dcu35HsUpePPakJ7vcfhnTjhE3lzyHHxPhGtIsZriTxxib+79r7NqeWfm", + "rTi7yZfAKUgQL3EK1cru732P8RWm5F9YrW3vqi8pqrdFVz90LLg54t5Fy22megjJCV3p5WTrrSAhTl6C", + "vGX85iAei/aImg6H8dma4YugNePsdwjl4fWbdkgtrmvBxVBfZKEcVod2Xq3TNDuM3WK4L7DWezMkCPk9", + "iwg0+O21eaEehYxKoPpPnGUJCTUhnv0uFCx3HnzCaZaA+jMFiSMssYPW0Qb4kglA9ectlgz073vfExmE", + "ahQLfeTNvfB8Mr2AYRTEM7wMxpNRFMzwCAeTweh8Ep9fjIfTped7Eq+EN//trhg6THIhgQck8nxvg5Nc", + "PZyNpoNxfxgG8Wx2EYxnYRjg5XAQzJbL2QzHYRzBhXf/QWHoOCQXAPydEwkGtbsIsKhGMeMI01Je9Vob", + "22baP3YziskDKwFamxJRTUoC+EZL6d+8i57+n/dB8SnE5JM39wazYW8wvej1e/2z4fir7cwOKo/doJYc", + "7CkM/DNnEouTtiNO8IZxg4CQ5arPhe8RRdkzfBFOR+f9YNyfToJxNMbBLML94Hx6fhHF434YzSIF87Eg", + "m1W+Ubu1F1DJkACJTPOeFV4iY1QYWYDDEDIJ0Wv7sEuQmdHWWKAlAEVFN4RphG5JkqAloDhPYpIk6qnY", + "0nDNGWW5SLa9Bf1/LEcp3qKMJQmSekTBch6CHiBllEjGEZECCYllLvQGKbATUMvQG7PEkd2V+mKP3x3g", + "nHEliOgGJyT6aIHyfPPmYxPsAuQli7bIdvGO3h4zl2NnXteHjTFR2DKdkJ5Cr95HjFssmdYRA4Eok0hB", + "iwldUFzi0YgVFBNIIqERBZ8kcFqygzgFXb/dGboNR/FseD6YBoM4CoPx8nwZzPpTCMYx9AeTcRSHUVyJ", + "lZgx7/7D0UjaWaebkhMiJGKxQQ8q+hQsayC2jHcioHUpGnLQDd8SDdBgdt4P+oOgP3jb78/1///hPYSl", + "S9SseuPemqzWKaQ9POj3e4NVb9BfLetCN8zyn3BKkq03966ohAT9X2AUXSdYEpqn6GIw7b9Ff35zs03w", + "DfzF81UP4c3HvhcRcePNh33fW2W5GithKyXanhkxNPS9FFLGt958Ova9lEWQ6EmEJDSU6Ner4aTvVYeS", + "7TbwvQ3QSLPN5a8/qLUWw4yG98fvdCEY926wbWQ2lPEliSKgn8fq5TAdTJ4L4CjkoM9pnAgUMc1ma7yB", + "JntlnGxIAisQjygEbrFAEVACEVpuEc7lmnEirAiQayK0zFwCCnEuTCO1qEbDBZXsBmixbEJXzYWLkGVQ", + "KCSX11elbNGwK8FCv6sAXlAKIQiB+bYGMmJUd8k425AIOMoSLGPGU71XVssh8Gj8B9H3igV+Z2vaixj8", + "Hxym0AtZqgi+yZ/D/nAc9CfBaPB2MJ4PBnX+xNNxPBtOZ8FoCv1gPBoMg+VFNAgmw2g2iibT2fJ8WfFn", + "ThWKvR3b7gF8XlgcqguMpmF/coGDC1jiYBxPlsFsEI+DeBrHy9nF6Hw2CU2XDRGEUUJXb/S5ZywX8xCi", + "umxgGVAhcXijsZSwXM0TQYzzRB1h+skzRmOyUs+fr7Nw+736b3318+skHP3tl90lLmfhTGHifDwdR4Px", + "Mr44h0k/xufD6eiiryASYv0LbF+muBDtvmYY3R0PZtPzCzy8GAyn49l5tMTD8XIyDmdT3J+OY+xVVpJe", + "6MVsEC3jftDH/UEwhjgMMCjFPjo/j6fRaDwca8XemLYVrA+QMXUyxNF+UWPbgqgT8PY0WfNEvU/U20m9", + "D7UtO0m3MiZRob8b2k3xCr6A4jPsD0dBfxgMh28Hw3l/PB+MTiXNZT4c9sfBZtAbTnrTYJXlwWQ46V1M", + "ev1JcB5CNB5MxnVisRpMxMkG1Cletvas5qLt0ctBX2ksP9t/hv2+Mk1LdeXl+6sfri71sCyWt5jDe+CK", + "MLXZUzlMvLlnV6babgiXOU4sA6l3xQNFzw8QRnpbDggh3QbJNZYIc9DmDpZkmQC6JXJtFIDmSUuN8vtG", + "W+U/KvXi8/QjY95/ND/dKpK1USRDRs8IE0zSR9CBLinKKXzKIFRmpG6GWBjmnEPUVH5wo6XkmAoCVNo+", + "mEYLqlqKPAwBIqWrYMRB8m0PXcVmJKKVHKXChFiAj7IEsFBKUsa4REQiLLS/RojcsBVl8ieW0+jz0EuZ", + "/BirYTpwWzPxIKpM4tLag09EyEfA9TuKFVVJhmJCI40eM5WGteWIejoFv9ApuNep1jgirXWrF3IxHMNF", + "OA6DycXkIhgv+8NgNutPg9GsD6PxZDpYxiPP9zYJ1rAO+sPx/T4f3R96sLWIq0s1c/ridno/GRhPxPmo", + "NkObvg5ZDrtkKho+41Mk5zfkNBagYTTNrI88TNRh26s04i+mbJ5rJuvPx5P5eKKYrB1K/bRNGWeUhEgS", + "4MEIqQFDUGoRWmIBESIUvVBqY8ZY0isY9ciwUsGoN8Gt8Qs/hH1iwDLnxqm+S1ZlzOdzaNVifz952kZ6", + "u3JqHUX/gs/UY3AYghAfjauqQ5dRcykDxYxmXduPoSe6xi18WGZhVjNdY4HgU0Y4RL3aCSVqkOzGNp4D", + "BU5Cq0qmIARegd/SxJkCbtgzFJEBlzaQ2jHqJZLABdhRTZxerQzTSP1lnWk/v317bZuELIIe0vq80KaA", + "oWXb8JVCwRApQiOxxYOPlrmxGsy4EJmVqvVxAhLzbRFJUYMbZr68vhKIyTUo5GE1OBNQjGvci2YuBSnQ", + "PFWHQDtaUqerj0Y8eH6LRnIq8kyp16D6Gur7qOnfL8fUvkllqjfNEAlpxjjmJNl+zCneYJIo/bXWsZy1", + "eLDimMqdWfWzYsq6Kh4yGickVO1TkGsWfVRvcZKw29bSU4gILgap3Mkf/N00DCdX7FLGexuTtZRmY7PL", + "wmmrR+h5viPFowof/+Z1m2vVsthS6QqOaJAzR+VVIZBa8ZU20SuJ6o4RWvPF5Li00GOk696eRT7HfvBJ", + "GVU5AlzhYs9CWrJOsIU+OySk4oFBLK+S8phzvK2iU66FmDdtHNfPyn2TKxYn4WuLv1+LXrVj6XBUxqoD", + "TRyXC7AjuTBtuv9NKQvdsBllog2h1XRc5EDzdAlc7U8zwFniyq6EUAkr4MaD6x7KrsFJkS6qMqs6AOwL", + "IuQ+qqoDfjwd1bHZSUNvCl39SDpaYg5qK5N2p7+vwZ4EJZpIofNFEKlzBiKU4nBNaJ0pl4wlgKlaUy1c", + "6VgSBx12StGz63co1u3quSgIeqse0u48u+M+wjxcEwmhUqScMsSEO/cTzbPrd8JNJSZG6uqNU7Xxqjdk", + "a0iB4wSp1kqdfP69ezTrpdy3p6ssN9xVhU33z25a6VmJc9odotX4KAe3EHaTrzhMtg8lWBetrrL8VxNg", + "bs/2/PpdY9Od21wMcIjTdgc7fvHlEt3Ld/OZmq6hhbUZrhl1d222bVEj2OfX7wQqtRyEBRIAVBGB1v7e", + "uMmvi6A0Tg6RUZkBsGeDnDuzkx+wP7m0CeGfQ8wj8ZcKUvfCCgf+wf17bxrusoTtX0BYY47m2v3mXrmY", + "pprECajClJmtri/buIOv8yVcauJOnHIPeZfh9oLoUNX3aEpvRpba1N5471hM5yK2p6st1pP1Rqn/UaG9", + "qAU8VIMplnKyDtMY4EHQ++h2TRKTJWLMERRiavbOWuFIMkRobA7ZBVWT++gWUMTod7JQbISJUGAaIQ4y", + "5xQRWcR/oIr1IfR2jc0Uyh5e0KXO4NAOD91LMhSBBJ4SCmpp4bq9eGNCS4aUjW+P9eYONvx8x2JeWQ9v", + "jDuj7sDbkxFdy41VWkfNpYIIdQoek1i6f00Sr17YgInpf4xD5a1quUs61t4rYTlEOhUGWlD/uAG+lWtl", + "4mFjaOmGBc1QgEgTSpzT0H2m2HC5U+vBKRSKsm5WnBrmR6gj6m5Nqh5xb9P9EguYjhHQkEUQNYZDMUnc", + "2lnNQb074rXN2K9S4ZGyt42SGVtvDqExx0LyvFsBNJ6C55zlmWsaExhFK/X+0Fzy4FwmgcBptv5Yqolv", + "3vyMbmCLVkCBN6eoycnW4EUqwu7A7wTwUszHp6Hpfg+1mtxolw+gll+gPT2ni/fCntdznSrSdefPluvV", + "KC2QiyS86i6SSTrWrmnDrd0n3bcg5XbQsl9YpXjVse3qzdfyQOjJT99k1fu5TRZxqmcmhQSRGBF1rCUJ", + "RG1QizyTA4NsTPqIX2QUWN2vPFydG1qkqxxnjRSn16PpwBa0TuR1qD2dVHGEzVtuSkfmzV5i2m3vyMg5", + "Yvb3zS4t7DRfdyLnfWvmXQ8Klkh11Za7cZQYAa16W29K3TSo5xL5Nd+M72G6ddsKJo1oj43wwCSiY40G", + "LS5c1kK2GV9GEQchnHRzdb0ZI2waOBmiNsAh674+1kMMntoSHRBUmV8v8BKS9+ZOkuP6lL5u8Eu+BN0Y", + "Jao10leYfLXrynRMtkYHV+dGw4ltN0Rp60tYUEIj+ASlcqbkmlKwNH9hKYGrKf//b/1gdhn8Awf/+vDn", + "v86rX8HH3oe7vj8d3Nda/OWvf3Kht+sioAPAX8qmxu+Ofs2F1PlWFvYfXr4p7puYsFWyRQm7Ba6TqFC4", + "xhyH6tz0C/cLYhytt9kaqPCRkJhLbdcAtdEpXHVSTUu3H430vBKlTEg0HdXGVjhLgK7kWmErxZ9e6B/e", + "fDryvZTQ4ufAgYx6osUeW3N+5+EkeRXrKPUxes2OpXq3a0Pt5He4zpXGpeCaotq4GrWEhNGVsg4Ou693", + "Jm0LtQ+uTK8Ok7+VivPVDX3Hyk9WG7rGcmOijojvxCH/324SzdGiqjj6jzbAd6AorFBR2OFFik2XkDbv", + "S6OittUtPvqPMOlLIErU+Lu7Zec4gmaa2Hbe+i9Di4e5qZZP5ZQV5VDFdf2OCCtnuXRblc1hTLuuUUS+", + "pCAPj2LadY1SJHw54x2lSfv+xeXL5ghVpKON9S7btV3P4OuasK5l75dWR5DcHkv2a8ipRxQvn2P0drHy", + "EQg97H5vp/odqYI6k17bqujh0/Fo1eRIVaetrOzx2dUrbTyGilJN5dZOatmJ891l1uKXRwbRS3Jp0YE5", + "CX6q5et15EEWKX0WZH3M6Uvsxq+OJLNeXNFmsHYK4O40V7Gap3AnUBNxt4ZDbUKbiiTa5GiuPjAKSKxZ", + "nmgdun7sam+Wqb+gPcy5DSMoAzVLSEiM9rkGrozVBXVNqsyUQFuphYUrjL4u1yAApdZkqE2rVlQvraBF", + "PMiwt6COJIJdEtnF2ofO/eviXvP28yX/KVqqmftkxbTWvU0tVBnwZsfwkuV1GmkDW09GPbzgkhUeLXBS", + "zt8N5ls7UZdC+Z2o4ldqiLojpVKSP3Sqq3u9JmWa7JECvUZyDjnOd72fDl+zlZcFGVgTW6fbJAm6vL6q", + "ZCoHHJmA4C03d1Rbp/e+HMNGRl3tlZU0TP/Qvgucr1IFpjmhcWpdRinTfiQq4ZPcm8l3XEWimptll1pM", + "El8Ng9eOhOuOY6lsp/NctQ+ufq+pIpac3lB2S3fSues/tTsugp3XJsvSTWCfc1R3eszvWrtsip+YJHkX", + "GiRJoXkem4oCCUjj6DYyw5t7EZYQqOYdcTsH1o8Rko79cpz6u00cx7//QIbRPNKrb0ZTU3/iwAdyoIB0", + "4/YTCkgxlSQsAiA7XsvNYhH978WiV/vH6Zl0xQJ2jjjtNs84lLGaYsry32Lz2htSv9B7gP8MpE7VsIM3", + "H+wh3MPVtftnLrrRVTlu1wzZdg32dkfyG3dpjhcTdoLjxURXkm1OyT/zw/nfKYv0NYODkOdZdBzkxYgH", + "IMdNuO3wx8LtTBOuo/wIafZW32YuBI9Nsy0WZVX+35UqrS/8GgW7kSS0oJhum6eearMGnMi1vehhroQs", + "gUJMJIo5SxFWr2iE9VWNBS1XYOBuaOQVD0i8cpr1mC+J5JhvkcQrI6zUGnRExOHTcqZNXBbEUgzh9h25", + "YzJqQ/WrIuNF4tVhq1MvpBjzgxveQ5EoiVfH64oKfy0lUUvYMOdEbt+odjZSoO8VNW84tdfxKgNuFP8y", + "ZcxeCVoC5ko91tefmhewNHkn7NYUebP3dfSbZyyC1sN3PPHm3lrKTMzPyhyMXk7JDeM00Pk/PcZXZ2bJ", + "Z5vhWaO/MmtClmmwFPBqRSeMqfs1RLN+ZbyBhMasjZ1nOjXJ2q0RESHbAN+afDqW60wOAXxDrAwhMlHj", + "1oJfr03XN6aRUgQUuZgDx5t7/d6gNyg88zgj3twb9fq9kTkF1xq/ZzgjZ5tBwwsjzu6aNV3vaxVq2mD8", + "iileQVT50O2iRQ+hq7JfzdIXhK4SLTVNJj0unliT32Q80BB6C6rlT0JSoiz6BAuJOI5ILoqAJGzA5J/j", + "WjEslAC+0dWiCEWCpaZ2gkB4w0gk0DJfqf4L2tTF7SmvcL0C6bpxJ7W+VVbgMQWqdIEJ3CyWq8ZgBe3r", + "G8HPQV5m5P3gVR3PrxpYrnDl7VQ6HPb7Xaxbtjtz1LK6973xMV0d5Ql118Hhrs6bmrrz6HDndqm0e9+b", + "HAXsnnIidYml1R63rPrtgwks1uo5d6hIVZOzrurJeqgjeanwFNFO/5rhKIffrIfQdduXxqH6oQ9kW32k", + "un/E+IKWSRuIsmjHDrVs+P7F5cseQi+ZBDOQDpyX7Fk6NcraywLpqidUJttFdfMQZVUi4NZHWNSSv/Vq", + "1e7pi586n1edVKpHRiDUB2Q7S9DJjy90R5en+8Gsd727JacwYGfFhyc2/CbZ0Pj0xdldWXD7v+6ceySs", + "+we7OsqjK1sjYy7t9Zk27xBGFG5rkXm64zducvk1E4fZ3G75dbGanSO3KDa+7ab8Wj3ys91i5PctqTE4", + "+tjePkmLo6XFo/H42V31sYX70m/psN9+0M8biSJKS1eWbWULYyFYSLT9r11hRLap1Az0GXR61fw6RIPa", + "hoe3oFUz+9+S2sb98eGerTpof/yh9l9sJxVLrd9DeLhadhof9J+k7reso52mLRzu5foWzyMqhI3D4smI", + "e7gR929JPkfpqK48yUfVTitp57RUH6i3dny+5ST1tasC55M8/TpabEswnd21PoB1lKp7JGE/okK7S9rX", + "zg93/XdqvN+A4vp0ku13Rxq1N1tvidjLMY+l957ALv0nef6kH28fOln3Fxq/mHZt8qmdtRgUywvFoIEt", + "Q6HbolTLqBSo9BUba0tYG77mIzK3RK6tJdzw4KxJBAtqCkvahGiTfwA4XJuCkz2ELlcrDit7402gNaZR", + "UtRzz7AROMUXbkJGJWdJAry3oNemWDstJZitTxtiSplOPwYaMx4aIWUh8o38s+Bd6quYOAz1unGiRFsu", + "ysIt338niksphNEeQs/McwW2EqW15W4INrkHOuYucp1G4CPBEJHfiQUlqZKXmMriBqgCQiBdraUWi1Zr", + "YTmVwjeF3mlk8l/0sKJTPF7aFQdlOZp2yd7d0hNfRnD+zRDXKdJxp1jyk1D8D3Ua+F6WS1eNFtmSPMdQ", + "7nX+6JT7QJOv+WXI+ydF+hsz8ToT/1/belwRxISCLmrO8khfhqk+M7ICtuI4W2tFXH9YZFsW68swV2TE", + "aG9BfyT6oLvF27ISiPk2mjotycaeUUSY0gCSVSGNKgVQ5OEaYbGgjUnVAZSAX+XBmC+8fScQNxfbIrRM", + "2FLJe4XxXIK9WPOjOmdtCtRaqeZSIHZLK+W+HVXxNcfZWt9VWSrfXCoqBrCnS/0LeYIhXc1K2OoG9fyd", + "qkqaSIjR+/GCijXmZQ0pueYsX63R7RpL2ABHKYRrBWqqUFZVQdS1r9UBqnsVgOxPYzCJ4eXNjgcffJZM", + "TjrUdmvjfy5z/sdnDliEnd0VH7++L0spd7uDL5OE3Yqq7DtaeK3KzQtPk3ZBMtZ8toasYtW0t6B/17X7", + "nl1ev9JkXFbpaxWCVrwESewjIlHIcSYQyyUKFhRr/RnlIscJChCJTda+LqzOqC3/ktPIR7cchzcl51EF", + "kTbSdcApF+gWkJAkSXQtOAWU0TWLzxIZpsIJEpTdxgm+OWQ9F1mrzprWpzLFa7tLP+7u0SnM0vmB2qdo", + "6DejCbY/cf+Z3N1ZgviZPctsFeoyoLtP1mt7rTwF7dDV3dlaLjRYy0qdi6XceARG+MmCcwr9736u+Csq", + "gU/keyT5dpXiKqjXFP06gXjrFbyOod3HkOJXBpiTgv7ND04+ke4fQ7r39/8TAAD///f7c51EhwAA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/pkg/openapi/server.spec.yaml b/pkg/openapi/server.spec.yaml index b7e4357..8a52fa4 100644 --- a/pkg/openapi/server.spec.yaml +++ b/pkg/openapi/server.spec.yaml @@ -482,7 +482,8 @@ components: - vendor - model - memory - - count + - physicalCount + - logicalCount properties: vendor: $ref: '#/components/schemas/gpuVendor' @@ -492,8 +493,11 @@ components: memory: description: GPU memory in GiB. type: integer - count: - description: The number of GPUs available. + physicalCount: + description: The physical number of GPUs (cards) available. + type: integer + logicalCount: + description: The logical number of GPUs available as seen in the OS. type: integer flavorSpec: description: A flavor. @@ -861,10 +865,11 @@ components: disk: 20 memory: 32 gpu: - vendor: NVIDIA - model: H100 - memory: 192 - count: 1 + vendor: AMD + model: Instinct MI250 + memory: 64 + physicalCount: 1 + logicalCount: 2 identityResponse: description: An identity response. content: diff --git a/pkg/openapi/types.go b/pkg/openapi/types.go index 3581c31..6f7cbc7 100644 --- a/pkg/openapi/types.go +++ b/pkg/openapi/types.go @@ -96,8 +96,8 @@ type GpuModelList = []GpuModel // GpuSpec GPU specification. type GpuSpec struct { - // Count The number of GPUs available. - Count int `json:"count"` + // LogicalCount The logical number of GPUs available as seen in the OS. + LogicalCount int `json:"logicalCount"` // Memory GPU memory in GiB. Memory int `json:"memory"` @@ -105,6 +105,9 @@ type GpuSpec struct { // Model A GPU model. Model string `json:"model"` + // PhysicalCount The physical number of GPUs (cards) available. + PhysicalCount int `json:"physicalCount"` + // Vendor The GPU vendor. Vendor GpuVendor `json:"vendor"` } diff --git a/pkg/providers/helpers.go b/pkg/providers/helpers.go index cec1354..7c12cec 100644 --- a/pkg/providers/helpers.go +++ b/pkg/providers/helpers.go @@ -26,7 +26,7 @@ import ( func (f Flavor) GPUCount() int { if f.GPU != nil { - return f.GPU.Count + return f.GPU.LogicalCount } return 0 diff --git a/pkg/providers/openstack/blockstorage.go b/pkg/providers/openstack/blockstorage.go index a10342b..63c2735 100644 --- a/pkg/providers/openstack/blockstorage.go +++ b/pkg/providers/openstack/blockstorage.go @@ -27,8 +27,9 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/trace" - "github.com/unikorn-cloud/core/pkg/util" "github.com/unikorn-cloud/region/pkg/constants" + + "k8s.io/utils/ptr" ) // BlockStorageClient wraps the generic client because gophercloud is unsafe. @@ -94,8 +95,8 @@ func (c *BlockStorageClient) UpdateQuotas(ctx context.Context, projectID string) defer span.End() opts := "asets.UpdateOpts{ - Volumes: util.ToPointer(-1), - Gigabytes: util.ToPointer(-1), + Volumes: ptr.To(-1), + Gigabytes: ptr.To(-1), } return quotasets.Update(ctx, c.client, projectID, opts).Err diff --git a/pkg/providers/openstack/compute.go b/pkg/providers/openstack/compute.go index 073aebb..59cebd6 100644 --- a/pkg/providers/openstack/compute.go +++ b/pkg/providers/openstack/compute.go @@ -33,10 +33,11 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/trace" - "github.com/unikorn-cloud/core/pkg/util" "github.com/unikorn-cloud/core/pkg/util/cache" unikornv1 "github.com/unikorn-cloud/region/pkg/apis/unikorn/v1alpha1" "github.com/unikorn-cloud/region/pkg/constants" + + "k8s.io/utils/ptr" ) // ComputeClient wraps the generic client because gophercloud is unsafe. @@ -267,9 +268,9 @@ func (c *ComputeClient) UpdateQuotas(ctx context.Context, projectID string) erro opts := "asets.UpdateOpts{ // TODO: instances, cores and ram need to be driven by client input. - Instances: util.ToPointer(-1), - Cores: util.ToPointer(-1), - RAM: util.ToPointer(-1), + Instances: ptr.To(-1), + Cores: ptr.To(-1), + RAM: ptr.To(-1), } return quotasets.Update(ctx, c.client, projectID, opts).Err diff --git a/pkg/providers/openstack/provider.go b/pkg/providers/openstack/provider.go index b3e8ed4..20aca23 100644 --- a/pkg/providers/openstack/provider.go +++ b/pkg/providers/openstack/provider.go @@ -320,10 +320,11 @@ func (p *Provider) Flavors(ctx context.Context) (providers.FlavorList, error) { f.GPU = &providers.GPU{ // TODO: while these align, you should really put a // proper conversion in here. - Vendor: providers.GPUVendor(metadata.GPU.Vendor), - Model: metadata.GPU.Model, - Memory: metadata.GPU.Memory, - Count: metadata.GPU.Count, + Vendor: providers.GPUVendor(metadata.GPU.Vendor), + Model: metadata.GPU.Model, + Memory: metadata.GPU.Memory, + PhysicalCount: metadata.GPU.PhysicalCount, + LogicalCount: metadata.GPU.LogicalCount, } } } diff --git a/pkg/providers/types.go b/pkg/providers/types.go index fd8e31f..cb0b95a 100644 --- a/pkg/providers/types.go +++ b/pkg/providers/types.go @@ -59,8 +59,12 @@ type GPU struct { Model string // Memory is the amount of memory each GPU has. Memory *resource.Quantity - // Count is the number of GPUs in the flavor. - Count int + // PhysicalCount is the number of physical cards in the flavor. + // This is primarily for end users, so it's not confusing. + PhysicalCount int + // LogicalCount is the number of logical GPUs e.g. an AMD MI250 is 2 MI200s. + // This is primarily for scheduling e.g. autoscaling. + LogicalCount int } // FlavorList allows us to attach sort functions and the like.