From 632c86f3bb7f298bfb27c822219504f9235449ca Mon Sep 17 00:00:00 2001 From: catttam Date: Thu, 16 Nov 2023 18:05:07 +0100 Subject: [PATCH 1/8] Added parameter enable_sgx to FDL documentation --- docs/fdl.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/fdl.md b/docs/fdl.md index d56b5114..a6811a8e 100644 --- a/docs/fdl.md +++ b/docs/fdl.md @@ -80,6 +80,7 @@ storage_providers: | `memory`
*string* | Memory limit for the service following the [kubernetes format](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory). Optional (default: 256Mi) | | `cpu`
*string* | CPU limit for the service following the [kubernetes format](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu). Optional (default: 0.2) | | `enable_gpu`
*bool* | Parameter to enable the use of GPU for the service. Requires a device plugin deployed on the cluster (More info: [Kubernetes device plugins](https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/#using-device-plugins)). Optional (default: false) | +| `enable_sgx`
*bool* | Parameter to enable the use of SGX plugin on the cluster containers. (More info: [SGX plugin documentation](https://sconedocs.github.io/helm_sgxdevplugin/)). Optional (default: false) | | `image_prefetch`
*bool* | Parameter to enable the use of image caching. Optional (default: false) | | `total_memory`
*string* | Limit for the memory used by all the service's jobs running simultaneously. Apache YuniKorn scheduler is required to work. Same format as Memory, but internally translated to MB (integer). Optional (default: "") | | `total_cpu`
*string* | Limit for the virtual CPUs used by all the service's jobs running simultaneously. Apache YuniKorn scheduler is required to work. Same format as CPU, but internally translated to millicores (integer). Optional (default: "") | From 45c0e7cd50348af61100aad00e4f3fb99950f6f7 Mon Sep 17 00:00:00 2001 From: catttam Date: Thu, 23 Nov 2023 17:34:49 +0100 Subject: [PATCH 2/8] Added VO to service definition --- pkg/backends/k8s.go | 8 ++++++++ pkg/backends/knative.go | 8 ++++++++ pkg/handlers/create.go | 21 +++++++++++++++++++++ pkg/types/service.go | 4 ++++ pkg/utils/auth/oidc.go | 17 +++++++++++++++-- 5 files changed, 56 insertions(+), 2 deletions(-) diff --git a/pkg/backends/k8s.go b/pkg/backends/k8s.go index d583e0da..bd10c4d6 100644 --- a/pkg/backends/k8s.go +++ b/pkg/backends/k8s.go @@ -85,6 +85,14 @@ func (k *KubeBackend) CreateService(service types.Service) error { return err } + if service.VO != "" { + for _, vo := range k.config.OIDCGroups { + if vo == service.VO { + service.Labels["vo"] = service.VO + } + } + } + // Create podSpec from the service podSpec, err := service.ToPodSpec(k.config) if err != nil { diff --git a/pkg/backends/knative.go b/pkg/backends/knative.go index 3a5800fe..4170bce1 100644 --- a/pkg/backends/knative.go +++ b/pkg/backends/knative.go @@ -102,6 +102,14 @@ func (kn *KnativeBackend) CreateService(service types.Service) error { return err } + if service.VO != "" { + for _, vo := range kn.config.OIDCGroups { + if vo == service.VO { + service.Labels["vo"] = service.VO + } + } + } + // Create the Knative service definition knSvc, err := kn.createKNServiceDefinition(&service) if err != nil { diff --git a/pkg/handlers/create.go b/pkg/handlers/create.go index 780ec664..c908077f 100644 --- a/pkg/handlers/create.go +++ b/pkg/handlers/create.go @@ -31,6 +31,7 @@ import ( "github.com/grycap/cdmi-client-go" "github.com/grycap/oscar/v2/pkg/types" "github.com/grycap/oscar/v2/pkg/utils" + "github.com/grycap/oscar/v2/pkg/utils/auth" k8sErrors "k8s.io/apimachinery/pkg/api/errors" ) @@ -46,6 +47,22 @@ var errInput = errors.New("unrecognized input (valid inputs are MinIO and dCache func MakeCreateHandler(cfg *types.Config, back types.ServerlessBackend) gin.HandlerFunc { return func(c *gin.Context) { var service types.Service + oidcManager, _ := auth.NewOIDCManager(cfg.OIDCIssuer, cfg.OIDCSubject, cfg.OIDCGroups) + + authHeader := c.GetHeader("Authorization") + rawToken := strings.TrimPrefix(authHeader, "Bearer ") + hasVO, err2 := oidcManager.UserHasVO(rawToken, service.VO) + + if err2 != nil { + c.String(http.StatusInternalServerError, err2.Error()) + return + } + + if !hasVO { + c.String(http.StatusBadRequest, fmt.Sprintf("This user isn't enrrolled on the vo: %v", service.VO)) + return + } + if err := c.ShouldBindJSON(&service); err != nil { c.String(http.StatusBadRequest, fmt.Sprintf("The service specification is not valid: %v", err)) return @@ -120,6 +137,10 @@ func checkValues(service *types.Service, cfg *types.Config) { service.Labels[types.YunikornApplicationIDLabel] = service.Name service.Labels[types.YunikornQueueLabel] = fmt.Sprintf("%s.%s.%s", types.YunikornRootQueue, types.YunikornOscarQueue, service.Name) + if service.VO != "" { + service.Labels["vo"] = service.VO + } + // Create default annotations map if service.Annotations == nil { service.Annotations = make(map[string]string) diff --git a/pkg/types/service.go b/pkg/types/service.go index 13f83180..45182cb1 100644 --- a/pkg/types/service.go +++ b/pkg/types/service.go @@ -224,6 +224,10 @@ type Service struct { // Optional Annotations map[string]string `json:"annotations"` + // Parameter to specify the VO from the user creating the service + // Optional + VO string `json:"vo"` + // Labels user-defined Kubernetes labels to be set in job's definition // https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ // Optional diff --git a/pkg/utils/auth/oidc.go b/pkg/utils/auth/oidc.go index a1c7bb46..b1baa65c 100644 --- a/pkg/utils/auth/oidc.go +++ b/pkg/utils/auth/oidc.go @@ -45,7 +45,7 @@ type userInfo struct { } // newOIDCManager returns a new oidcManager or error if the oidc.Provider can't be created -func newOIDCManager(issuer string, subject string, groups []string) (*oidcManager, error) { +func NewOIDCManager(issuer string, subject string, groups []string) (*oidcManager, error) { provider, err := oidc.NewProvider(context.TODO(), issuer) if err != nil { return nil, err @@ -66,7 +66,7 @@ func newOIDCManager(issuer string, subject string, groups []string) (*oidcManage // getIODCMiddleware returns the Gin's handler middleware to validate OIDC-based auth func getOIDCMiddleware(issuer string, subject string, groups []string) gin.HandlerFunc { - oidcManager, err := newOIDCManager(issuer, subject, groups) + oidcManager, err := NewOIDCManager(issuer, subject, groups) if err != nil { return func(c *gin.Context) { c.AbortWithStatus(http.StatusUnauthorized) @@ -140,6 +140,19 @@ func getGroups(urns []string) []string { return groups } +func (om *oidcManager) UserHasVO(rawToken string, vo string) (bool, error) { + ui, err := om.getUserInfo(rawToken) + if err != nil { + return false, err + } + for _, gr := range ui.groups { + if vo == gr { + return true, nil + } + } + return false, nil +} + // isAuthorised checks if a token is authorised to access the API func (om *oidcManager) isAuthorised(rawToken string) bool { // Check if the token is valid From 40540873c0895ee12a51be48c30bd6dca4ec95c3 Mon Sep 17 00:00:00 2001 From: catttam Date: Fri, 24 Nov 2023 10:09:41 +0100 Subject: [PATCH 3/8] Changes in VO check --- pkg/handlers/create.go | 33 ++++++++++++++++++--------------- pkg/types/service_test.go | 1 + 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/pkg/handlers/create.go b/pkg/handlers/create.go index c908077f..bf6e15bd 100644 --- a/pkg/handlers/create.go +++ b/pkg/handlers/create.go @@ -47,21 +47,6 @@ var errInput = errors.New("unrecognized input (valid inputs are MinIO and dCache func MakeCreateHandler(cfg *types.Config, back types.ServerlessBackend) gin.HandlerFunc { return func(c *gin.Context) { var service types.Service - oidcManager, _ := auth.NewOIDCManager(cfg.OIDCIssuer, cfg.OIDCSubject, cfg.OIDCGroups) - - authHeader := c.GetHeader("Authorization") - rawToken := strings.TrimPrefix(authHeader, "Bearer ") - hasVO, err2 := oidcManager.UserHasVO(rawToken, service.VO) - - if err2 != nil { - c.String(http.StatusInternalServerError, err2.Error()) - return - } - - if !hasVO { - c.String(http.StatusBadRequest, fmt.Sprintf("This user isn't enrrolled on the vo: %v", service.VO)) - return - } if err := c.ShouldBindJSON(&service); err != nil { c.String(http.StatusBadRequest, fmt.Sprintf("The service specification is not valid: %v", err)) @@ -71,6 +56,24 @@ func MakeCreateHandler(cfg *types.Config, back types.ServerlessBackend) gin.Hand // Check service values and set defaults checkValues(&service, cfg) + if service.VO != "" { + oidcManager, _ := auth.NewOIDCManager(cfg.OIDCIssuer, cfg.OIDCSubject, cfg.OIDCGroups) + + authHeader := c.GetHeader("Authorization") + rawToken := strings.TrimPrefix(authHeader, "Bearer ") + hasVO, err2 := oidcManager.UserHasVO(rawToken, service.VO) + + if err2 != nil { + c.String(http.StatusInternalServerError, err2.Error()) + return + } + + if !hasVO { + c.String(http.StatusBadRequest, fmt.Sprintf("This user isn't enrrolled on the vo: %v", service.VO)) + return + } + } + // Create the service if err := back.CreateService(service); err != nil { // Check if error is caused because the service name provided already exists diff --git a/pkg/types/service_test.go b/pkg/types/service_test.go index 20b5242a..18f1b900 100644 --- a/pkg/types/service_test.go +++ b/pkg/types/service_test.go @@ -216,6 +216,7 @@ total_memory: "" total_cpu: "" enable_gpu: false enable_sgx: false +vo: "" image_prefetch: false synchronous: min_scale: 0 From f6839a93774a0a33e7bf7a9bebae97aaf15aa993 Mon Sep 17 00:00:00 2001 From: catttam Date: Fri, 24 Nov 2023 11:16:47 +0100 Subject: [PATCH 4/8] Changes in knative definition --- pkg/backends/knative.go | 16 ++++++++-------- pkg/types/service_test.go | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/backends/knative.go b/pkg/backends/knative.go index 4170bce1..0b938d85 100644 --- a/pkg/backends/knative.go +++ b/pkg/backends/knative.go @@ -102,14 +102,6 @@ func (kn *KnativeBackend) CreateService(service types.Service) error { return err } - if service.VO != "" { - for _, vo := range kn.config.OIDCGroups { - if vo == service.VO { - service.Labels["vo"] = service.VO - } - } - } - // Create the Knative service definition knSvc, err := kn.createKNServiceDefinition(&service) if err != nil { @@ -286,6 +278,14 @@ func (kn *KnativeBackend) createKNServiceDefinition(service *types.Service) (*kn // https://knative.dev/docs/serving/services/private-services/ service.Labels[types.KnativeVisibilityLabel] = types.KnativeClusterLocalValue + if service.VO != "" { + for _, vo := range kn.config.OIDCGroups { + if vo == service.VO { + service.Labels["vo"] = service.VO + } + } + } + podSpec, err := service.ToPodSpec(kn.config) if err != nil { return nil, err diff --git a/pkg/types/service_test.go b/pkg/types/service_test.go index 18f1b900..a8a08562 100644 --- a/pkg/types/service_test.go +++ b/pkg/types/service_test.go @@ -216,7 +216,6 @@ total_memory: "" total_cpu: "" enable_gpu: false enable_sgx: false -vo: "" image_prefetch: false synchronous: min_scale: 0 @@ -252,6 +251,7 @@ environment: TEST_VAR: testvalue annotations: testannotation: testannotationvalue +vo: "" labels: testlabel: testlabelvalue storage_providers: From b7a41066cef40c136247cd07ce40e0e11100798e Mon Sep 17 00:00:00 2001 From: catttam Date: Fri, 24 Nov 2023 12:36:51 +0100 Subject: [PATCH 5/8] Added logs to debug --- pkg/backends/k8s.go | 4 ++++ pkg/backends/knative.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/pkg/backends/k8s.go b/pkg/backends/k8s.go index bd10c4d6..17c9ef07 100644 --- a/pkg/backends/k8s.go +++ b/pkg/backends/k8s.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "log" + "os" "github.com/goccy/go-yaml" "github.com/grycap/oscar/v2/pkg/imagepuller" @@ -30,6 +31,8 @@ import ( "k8s.io/client-go/kubernetes" ) +var K8sLoggerInfo = log.New(os.Stdout, "[BACKEND-INFO] ", log.Flags()) + // KubeBackend struct to represent a Kubernetes client to store services as podTemplates type KubeBackend struct { kubeClientset kubernetes.Interface @@ -77,6 +80,7 @@ func (k *KubeBackend) ListServices() ([]*types.Service, error) { // CreateService creates a new service as a k8s podTemplate func (k *KubeBackend) CreateService(service types.Service) error { + K8sLoggerInfo.Println("Creating service with K8s backend") // Validate the input variables of the service service = utils.ValidateService(service) // Create the configMap with FDL and user-script diff --git a/pkg/backends/knative.go b/pkg/backends/knative.go index 0b938d85..f090e0a0 100644 --- a/pkg/backends/knative.go +++ b/pkg/backends/knative.go @@ -21,6 +21,7 @@ import ( "fmt" "log" "net/http" + "os" "strconv" "github.com/grycap/oscar/v2/pkg/imagepuller" @@ -33,6 +34,8 @@ import ( knclientset "knative.dev/serving/pkg/client/clientset/versioned" ) +var KnativeLoggerInfo = log.New(os.Stdout, "[BACKEND-INFO] ", log.Flags()) + // KnativeBackend struct to represent a Knative client type KnativeBackend struct { kubeClientset kubernetes.Interface @@ -94,6 +97,7 @@ func (kn *KnativeBackend) ListServices() ([]*types.Service, error) { // CreateService creates a new service as a Knative service func (kn *KnativeBackend) CreateService(service types.Service) error { + KnativeLoggerInfo.Println("Creating service with Knative backend") // Validate the input variables of the service service = utils.ValidateService(service) // Create the configMap with FDL and user-script From a7cbedd8e31afe6f7dd29f866ee920ca6bc414c8 Mon Sep 17 00:00:00 2001 From: catttam Date: Tue, 28 Nov 2023 16:09:53 +0100 Subject: [PATCH 6/8] Added VO info to Knative pods --- pkg/backends/knative.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/backends/knative.go b/pkg/backends/knative.go index f090e0a0..a4f6ae4d 100644 --- a/pkg/backends/knative.go +++ b/pkg/backends/knative.go @@ -282,6 +282,7 @@ func (kn *KnativeBackend) createKNServiceDefinition(service *types.Service) (*kn // https://knative.dev/docs/serving/services/private-services/ service.Labels[types.KnativeVisibilityLabel] = types.KnativeClusterLocalValue + // Add to the service labels the user VO for accounting on k8s pods if service.VO != "" { for _, vo := range kn.config.OIDCGroups { if vo == service.VO { @@ -324,6 +325,11 @@ func (kn *KnativeBackend) createKNServiceDefinition(service *types.Service) (*kn }, } + // Add to the service labels the user VO for accounting on knative pods + if service.Labels["vo"] != "" { + knSvc.Spec.ConfigurationSpec.Template.ObjectMeta.Labels["vo"] = service.Labels["vo"] + } + if service.EnableSGX { knSvc.Spec.ConfigurationSpec.Template.ObjectMeta.Annotations["kubernetes.podspec-securitycontext"] = "enabled" knSvc.Spec.ConfigurationSpec.Template.ObjectMeta.Annotations["kubernetes.containerspec-addcapabilities"] = "enabled" From 1e017002b11083611aae96d5bee822d0d200442f Mon Sep 17 00:00:00 2001 From: catttam Date: Tue, 28 Nov 2023 16:44:02 +0100 Subject: [PATCH 7/8] Fixed Knative nil pointer error on labels --- pkg/backends/knative.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/backends/knative.go b/pkg/backends/knative.go index a4f6ae4d..eb39823f 100644 --- a/pkg/backends/knative.go +++ b/pkg/backends/knative.go @@ -315,6 +315,8 @@ func (kn *KnativeBackend) createKNServiceDefinition(service *types.Service) (*kn types.KnativeMinScaleAnnotation: strconv.Itoa(service.Synchronous.MinScale), types.KnativeMaxScaleAnnotation: strconv.Itoa(service.Synchronous.MaxScale), }, + //Empty labels map to avoid nil pointer errors + Labels: map[string]string{}, }, Spec: knv1.RevisionSpec{ ContainerConcurrency: &containerConcurrency, From 6c291ec9451310216db28b1ed826b92814c3eb57 Mon Sep 17 00:00:00 2001 From: catttam Date: Wed, 29 Nov 2023 09:11:27 +0100 Subject: [PATCH 8/8] Deleted debugging logs --- pkg/backends/k8s.go | 4 ---- pkg/backends/knative.go | 4 ---- 2 files changed, 8 deletions(-) diff --git a/pkg/backends/k8s.go b/pkg/backends/k8s.go index 17c9ef07..bd10c4d6 100644 --- a/pkg/backends/k8s.go +++ b/pkg/backends/k8s.go @@ -20,7 +20,6 @@ import ( "context" "fmt" "log" - "os" "github.com/goccy/go-yaml" "github.com/grycap/oscar/v2/pkg/imagepuller" @@ -31,8 +30,6 @@ import ( "k8s.io/client-go/kubernetes" ) -var K8sLoggerInfo = log.New(os.Stdout, "[BACKEND-INFO] ", log.Flags()) - // KubeBackend struct to represent a Kubernetes client to store services as podTemplates type KubeBackend struct { kubeClientset kubernetes.Interface @@ -80,7 +77,6 @@ func (k *KubeBackend) ListServices() ([]*types.Service, error) { // CreateService creates a new service as a k8s podTemplate func (k *KubeBackend) CreateService(service types.Service) error { - K8sLoggerInfo.Println("Creating service with K8s backend") // Validate the input variables of the service service = utils.ValidateService(service) // Create the configMap with FDL and user-script diff --git a/pkg/backends/knative.go b/pkg/backends/knative.go index eb39823f..3629ce57 100644 --- a/pkg/backends/knative.go +++ b/pkg/backends/knative.go @@ -21,7 +21,6 @@ import ( "fmt" "log" "net/http" - "os" "strconv" "github.com/grycap/oscar/v2/pkg/imagepuller" @@ -34,8 +33,6 @@ import ( knclientset "knative.dev/serving/pkg/client/clientset/versioned" ) -var KnativeLoggerInfo = log.New(os.Stdout, "[BACKEND-INFO] ", log.Flags()) - // KnativeBackend struct to represent a Knative client type KnativeBackend struct { kubeClientset kubernetes.Interface @@ -97,7 +94,6 @@ func (kn *KnativeBackend) ListServices() ([]*types.Service, error) { // CreateService creates a new service as a Knative service func (kn *KnativeBackend) CreateService(service types.Service) error { - KnativeLoggerInfo.Println("Creating service with Knative backend") // Validate the input variables of the service service = utils.ValidateService(service) // Create the configMap with FDL and user-script