From e18f10714b2d072b34d825014c99498b94ff81fd Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Fri, 9 Aug 2024 12:43:53 +0530 Subject: [PATCH 1/2] feat: add taskmaster env in FOCA config --- deployment/config.yaml | 19 ++++++- tesk/custom_config.py | 73 ++++++++++++++++++++++++- tesk/utils.py | 117 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 207 insertions(+), 2 deletions(-) diff --git a/deployment/config.yaml b/deployment/config.yaml index 0006f11..f250c41 100644 --- a/deployment/config.yaml +++ b/deployment/config.yaml @@ -72,6 +72,23 @@ custom: tesResources_backend_parameters: - VmSize - ParamToRecogniseDataComingFromConfig + taskmaster_env_properties: + imageName: docker.io/elixircloud/tesk-core-taskmaster + imageVersion: v0.10.2 + filerImageName: docker.io/elixircloud/tesk-core-filer + filerImageVersion: v0.10.2 + ftp: + # Name of the secret with FTP account credentials + secretName: account-secret + # If FTP account enabled (based on non-emptiness of secretName) + enabled: true + # If verbose (debug) mode of taskmaster is on (passes additional flag to taskmaster and sets image pull policy to Always) + debug: false + # Environment variables, that will be passed to taskmaster + environment: + key: value + # Service Account name for taskmaster + serviceAccountName: default # Logging configuration # Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.LogConfig @@ -101,4 +118,4 @@ exceptions: exceptions: tesk.exceptions.exceptions storeLogs: - execution_trace: True + execution_trace: True \ No newline at end of file diff --git a/tesk/custom_config.py b/tesk/custom_config.py index ae1cbaf..84b8a7a 100644 --- a/tesk/custom_config.py +++ b/tesk/custom_config.py @@ -1,8 +1,78 @@ """Custom configuration model for the FOCA app.""" -from pydantic import BaseModel +from typing import Dict, Optional + +from pydantic import BaseModel, Field from tesk.api.ga4gh.tes.models import Service +from tesk.constants import TeskConstants + + +class FtpConfig(BaseModel): + """Ftp configuration model for the TESK.""" + + secretName: Optional[str] = Field( + default=None, description="Name of the secret with FTP account credentials" + ) + enabled: bool = Field( + default=False, + description="If FTP account enabled (based on non-emptiness of secretName)", + ) + + +class ExecutorSecret(BaseModel): + """Executor secret configuration.""" + + name: Optional[str] = Field( + default=None, + description=( + "Name of a secret that will be mounted as volume to each executor. The same" + " name will be used for the secret and the volume" + ), + ) + mountPath: Optional[str] = Field( + default=None, + alias="mountPath", + description="The path where the secret will be mounted to executors", + ) + enabled: bool = Field( + default=False, description="Indicates whether the secret is enabled" + ) + + +class TaskmasterEnvProperties(BaseModel): + """Taskmaster environment properties model for the TESK.""" + + imageName: str = Field( + default=TeskConstants.taskmaster_image_name, + description="Taskmaster image name", + ) + imageVersion: str = Field( + default=TeskConstants.taskmaster_image_version, + description="Taskmaster image version", + ) + filerImageName: str = Field( + default=TeskConstants.filer_image_name, description="Filer image name" + ) + filerImageVersion: str = Field( + default=TeskConstants.filer_image_version, description="Filer image version" + ) + ftp: FtpConfig = Field(default=None, description="Test FTP account settings") + debug: bool = Field( + default=False, + description="If verbose (debug) mode of taskmaster is on (passes additional " + "flag to taskmaster and sets image pull policy to Always)", + ) + environment: Optional[Dict[str, str]] = Field( + default=None, + description="Environment variables, that will be passed to taskmaster", + ) + serviceAccountName: str = Field( + default="default", description="Service Account name for taskmaster" + ) + executorSecret: Optional[ExecutorSecret] = Field( + default=None, description="Executor secret configuration" + ) class CustomConfig(BaseModel): @@ -10,3 +80,4 @@ class CustomConfig(BaseModel): # Define custom configuration fields here service_info: Service + taskmaster_env_properties: TaskmasterEnvProperties diff --git a/tesk/utils.py b/tesk/utils.py index 214ffd6..b0fe052 100644 --- a/tesk/utils.py +++ b/tesk/utils.py @@ -3,6 +3,27 @@ import os from pathlib import Path +from foca import Foca +from kubernetes.client.models import ( + V1Container, + V1EnvVar, + V1EnvVarSource, + V1Job, + V1JobSpec, + V1ObjectMeta, + V1PodSpec, + V1PodTemplateSpec, + V1SecretKeySelector, + V1VolumeMount, +) + +from tesk.api.kubernetes.constants import Constants, K8sConstants +from tesk.custom_config import ( + CustomConfig, + TaskmasterEnvProperties, +) +from tesk.exceptions import ConfigNotFoundError + def get_config_path() -> Path: """Get the configuration path. @@ -15,3 +36,99 @@ def get_config_path() -> Path: return Path(config_path_env).resolve() else: return (Path(__file__).parents[1] / "deployment" / "config.yaml").resolve() + + +def get_custom_config() -> CustomConfig: + """Get the custom configuration. + + Returns: + The custom configuration. + """ + conf = Foca(config_file=get_config_path()).conf + try: + return CustomConfig(**conf.custom) + except AttributeError: + raise ConfigNotFoundError( + "Custom configuration not found in config file." + ) from None + + +def get_taskmaster_template() -> V1Job: + """Get the taskmaster template from the custom configuration. + + Returns: + The taskmaster template. + """ + job = V1Job( + api_version="batch/v1", + kind="Job", + metadata=V1ObjectMeta(name="taskmaster", labels={"app": "taskmaster"}), + spec=V1JobSpec( + template=V1PodTemplateSpec( + metadata=V1ObjectMeta(name="taskmaster"), + spec=V1PodSpec( + service_account_name="default", + containers=[ + V1Container( + name="taskmaster", + image="docker.io/elixircloud/tesk-core-taskmaster:v0.10.2", + args=["-f", f"/jsoninput/{Constants.taskmaster_input}.gz"], + env=[ + V1EnvVar( + name=Constants.ftp_secret_username_env, + value_from=V1EnvVarSource( + secret_key_ref=V1SecretKeySelector( + name="ftp-secret", + key="username", + optional=True, + ) + ), + ), + V1EnvVar( + name=Constants.ftp_secret_password_env, + value_from=V1EnvVarSource( + secret_key_ref=V1SecretKeySelector( + name="ftp-secret", + key="password", + optional=True, + ) + ), + ), + ], + volume_mounts=[ + V1VolumeMount( + name="podinfo", + mount_path="/podinfo", + read_only=True, + ), + V1VolumeMount( + name="jsoninput", + mount_path="/jsoninput", + read_only=True, + ), + ], + ) + ], + volumes=[], + restart_policy=K8sConstants.job_restart_policy, + ), + ) + ), + ) + return job + + +def get_taskmaster_env_property() -> TaskmasterEnvProperties: + """Get the taskmaster env property from the custom configuration. + + Returns: + The taskmaster env property. + """ + custom_conf = get_custom_config() + try: + return custom_conf.taskmaster_env_properties + except AttributeError: + raise ConfigNotFoundError( + "Custom configuration doesn't seem to have taskmaster_env_properties in " + "config file." + ) from None From 7c20576f8e51964b818855b1e793d09e490076c2 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Fri, 9 Aug 2024 15:29:08 +0530 Subject: [PATCH 2/2] feat: harcoded values --- tesk/utils.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tesk/utils.py b/tesk/utils.py index b0fe052..39cf7a4 100644 --- a/tesk/utils.py +++ b/tesk/utils.py @@ -18,6 +18,7 @@ ) from tesk.api.kubernetes.constants import Constants, K8sConstants +from tesk.constants import TeskConstants from tesk.custom_config import ( CustomConfig, TaskmasterEnvProperties, @@ -62,16 +63,19 @@ def get_taskmaster_template() -> V1Job: job = V1Job( api_version="batch/v1", kind="Job", - metadata=V1ObjectMeta(name="taskmaster", labels={"app": "taskmaster"}), + metadata=V1ObjectMeta( + name=Constants.label_jobtype_value_taskm, + labels={"app": Constants.label_jobtype_value_taskm}, + ), spec=V1JobSpec( template=V1PodTemplateSpec( - metadata=V1ObjectMeta(name="taskmaster"), + metadata=V1ObjectMeta(name=Constants.label_jobtype_value_taskm), spec=V1PodSpec( service_account_name="default", containers=[ V1Container( - name="taskmaster", - image="docker.io/elixircloud/tesk-core-taskmaster:v0.10.2", + name=Constants.label_jobtype_value_taskm, + image=f"{TeskConstants.taskmaster_image_name}:{TeskConstants.taskmaster_image_version}", args=["-f", f"/jsoninput/{Constants.taskmaster_input}.gz"], env=[ V1EnvVar( @@ -131,4 +135,5 @@ def get_taskmaster_env_property() -> TaskmasterEnvProperties: raise ConfigNotFoundError( "Custom configuration doesn't seem to have taskmaster_env_properties in " "config file." + f"Custom config:\n{custom_conf}" ) from None