diff --git a/applications/cm-service/README.md b/applications/cm-service/README.md index c3ed233f19..5a148d79a0 100644 --- a/applications/cm-service/README.md +++ b/applications/cm-service/README.md @@ -42,6 +42,10 @@ Campaign Management for Rubin Data Release Production | worker.butler.dbUrl | string | `nil` | URL of a Butler Registry Database | | worker.butler.dbUsername | string | `nil` | Name of a user to use with the Butler Registry Database | | worker.butler.mountPath | string | `nil` | If specified, location for htcondor config file to be injected into worker containers | +| worker.htcondor.allocateNodes.contentsCondorInfo | string | `nil` | If specified, contents of the condor-info.py file used by allocate_nodes | +| worker.htcondor.allocateNodes.contentsExecConfig | string | `nil` | If specified, contents of the execConfig.py file used by allocate_nodes | +| worker.htcondor.allocateNodes.contentsSlurmConfig | string | `nil` | If specified, contents of the slurmConfig.py file used by allocate_nodes | +| worker.htcondor.allocateNodes.mountPath | string | `nil` | If specified, location for ctrl_exec config files for use with allocate_nodes | | worker.htcondor.config.contents | string | `nil` | If specified, contents of htcondor config file to be injected into worker containers | | worker.htcondor.config.mountPath | string | `nil` | If specified, location for htcondor config file to be injected into worker containers | | worker.htcondor.fsRemoteDir.storage | string | `"1Gi"` | Minimum storage requested in the condor remote area PVC | diff --git a/applications/cm-service/templates/configmap.yaml b/applications/cm-service/templates/configmap.yaml index e345349928..d68a806ec3 100644 --- a/applications/cm-service/templates/configmap.yaml +++ b/applications/cm-service/templates/configmap.yaml @@ -26,3 +26,4 @@ data: HTCONDOR__condor_submit_bin: /opt/htcondor/bin/condor_submit HTCONDOR__condor_q_bin: /opt/htcondor/bin/condor_q HTCONDOR__alias_path: /sdf/group/rubin/{{ .Values.config.outputVolume.subPath }} + CTRL_PLATFORM_S3DF_DIR: /home/lsstsvc1/.config/ctrl_exec diff --git a/applications/cm-service/templates/worker-configmap.yaml b/applications/cm-service/templates/worker-configmap.yaml index 21667b7574..dda3ff58d9 100644 --- a/applications/cm-service/templates/worker-configmap.yaml +++ b/applications/cm-service/templates/worker-configmap.yaml @@ -15,3 +15,11 @@ data: username: {{ .dbUsername }} password: ${env:DAF_BUTLER_PASSWORD} {{- end }} + {{- with .Values.worker.htcondor.allocateNodes }} + contentsExecConfig: | + {{- .contentsExecConfig | nindent 4 }} + contentsSlurmConfig: | + {{- .contentsSlurmConfig | nindent 4 }} + contentsCondorInfo: | + {{- .contentsCondorInfo | nindent 4 }} + {{- end }} diff --git a/applications/cm-service/templates/worker-deployment.yaml b/applications/cm-service/templates/worker-deployment.yaml index edaedee1df..787002d178 100644 --- a/applications/cm-service/templates/worker-deployment.yaml +++ b/applications/cm-service/templates/worker-deployment.yaml @@ -77,9 +77,12 @@ spec: {{- with .Values.worker.htcondor.fsRemoteDir }} - mountPath: {{ .mountPath }} name: "condor-remote-volume" - {{- with .subPath }} - subPath: {{ . }} - {{- end }} + subPath: {{ .subPath }} + {{- end }} + {{- with .Values.worker.htcondor.allocateNodes }} + - mountPath: {{ .mountPath }} + name: "ctrl-exec-remote-volume" + readOnly: true {{- end }} volumes: - name: output-volume @@ -94,6 +97,16 @@ spec: items: - key: config path: condor-config + - name: ctrl-exec-config-volume + configMap: + name: {{ $.Chart.Name }}-config-worker + items: + - key: contentsExecConfig + path: execConfig.py + - key: contentsSlurmConfig + path: slurmConfig.py + - key: contentsCondorInfo + path: condor-info.py - name: condor-remote-volume persistentVolumeClaim: claimName: {{ $.Chart.Name }}-htcondor diff --git a/applications/cm-service/values-usdf-cm-dev.yaml b/applications/cm-service/values-usdf-cm-dev.yaml index 7d961ae67e..0ae2a3f603 100644 --- a/applications/cm-service/values-usdf-cm-dev.yaml +++ b/applications/cm-service/values-usdf-cm-dev.yaml @@ -36,3 +36,18 @@ worker: storageClassName: "sdf-group-rubin" subPath: "services/htcondor/shared" mountPath: "/sdf/group/rubin/services/htcondor/shared" + allocateNodes: + mountPath: "/home/lsstsvc1/.config/ctrl_exec/etc/config" + contentsExecConfig: | + config.platform.nodeSetRequired = False + config.platform.localScratch = "$USER_SCRATCH/condor_scratch" + config.platform.fileSystemDomain = "slac.stanford.edu" + config.platform.scheduler = "slurm" + contentsSlurmConfig: | + config.platform.queue = "$QUEUE" + config.platform.scratchDirectory = "$USER_SCRATCH/condor_scratch" + config.platform.loginHostName = "sdfiana012.sdf.slac.stanford.edu" + config.platform.utilityPath = "/usr/sbin/" + contentsCondorInfo: | + config.platform["s3df"].user.name="lsstsvc1" + config.platform["s3df"].user.home="/sdf/home/l/lsstsvc1" diff --git a/applications/cm-service/values.yaml b/applications/cm-service/values.yaml index 6bbb216ec2..6b5868e851 100644 --- a/applications/cm-service/values.yaml +++ b/applications/cm-service/values.yaml @@ -171,6 +171,19 @@ worker: # -- If specified, sub-path within bound PV to be mounted as condor remote area subPath: null + allocateNodes: + # -- If specified, location for ctrl_exec config files for use with allocate_nodes + mountPath: null + + # -- If specified, contents of the execConfig.py file used by allocate_nodes + contentsExecConfig: null + + # -- If specified, contents of the slurmConfig.py file used by allocate_nodes + contentsSlurmConfig: null + + # -- If specified, contents of the condor-info.py file used by allocate_nodes + contentsCondorInfo: null + # The following will be set by parameters injected by Argo CD and should not # be set in the individual environment values files. global: