Add SLURM interface (#96)

* Add validation for `use_slurm` flag in input.go

* Add job file creation and sbatch function

This commit adds the functionality to create a job file and run sbatch. It includes the implementation of the `CreateJobHeader`, `CreateJobBody`, `PrepareJobFile`, and `Sbatch` functions. Tests for these functions have also been added.

* Refactor job file creation and remove unnecessary test cases

* Add support for Slurm job submission

This commit adds support for submitting jobs using Slurm. It checks if there is a `job.sh` file in the run directory and if so, it uses `sbatch` to submit the job. Otherwise, it continues to use the existing command to run the job. Additionally, a new method `PrepareJobFile` is added to create the `job.sh` file.

* upgrade trunk

* Update execution mode key in input validation

* update dev container setup

* update main.go

* Refactor Job struct and add support for detecting submitted jobs

* Add FindNewestLogFile function to utils.go and corresponding unit test

This commit adds a new function called FindNewestLogFile to the utils.go file. This function finds the newest log file in a given directory by comparing the modification times of all the files in the directory. It also includes a corresponding unit test in the utils_test.go file to verify the functionality of the new function.

* Add logging for job status and elapsed time

.devcontainer/Dockerfile

@@ -4,23 +4,68 @@ FROM golang:1.21 AS golang
 #========================================================================================================#
 FROM xenonmiddleware/slurm:20
 
+#==============================================================================================
+# Define ARGs
+ARG HADDOCK_VERSION=v3.0.0-beta.5
 ARG USERNAME=dev
 ARG USER_UID=1000
 ARG USER_GID=$USER_UID
 
+#==============================================================================================
+# Install system dependencies
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends \
+  build-essential=12.1ubuntu2 \
+  sudo=1.8.16-0ubuntu1.10 \
+  wget=1.17.1-1ubuntu1.5 \
+  git=1:2.7.4-0ubuntu1.10 \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+#==============================================================================================
+# Configure User
 RUN groupadd --gid $USER_GID $USERNAME \
   && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \
-  && apt-get update \
-  && apt-get install -y --no-install-recommends sudo=1.8.16-0ubuntu1.10 \
-  && apt-get clean \
-  && rm -rf /var/lib/apt/lists/* \
   && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
   && chmod 0440 /etc/sudoers.d/$USERNAME
 
+#==============================================================================================
+# Install miniconda
+ENV CONDA_DIR /opt/conda
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh  \
+  && /bin/bash ~/miniconda.sh -b -p /opt/conda
+
+ENV PATH=$CONDA_DIR/bin:$PATH
+RUN conda install python=3.9
+
+#==============================================================================================
+# Install HADDOCK3
+WORKDIR /opt
+
+RUN git clone --recursive https://github.com/haddocking/haddock3.git
+WORKDIR /opt/haddock3
+RUN git checkout ${HADDOCK_VERSION}
+
+WORKDIR /opt/haddock3/src/fcc/src
+RUN make
+
+WORKDIR /opt/haddock3
+RUN pip install --no-cache-dir -r requirements.txt \
+  && python setup.py develop
+
+WORKDIR /opt/haddock3/bin
+COPY cns /opt/haddock3/bin/cns
+
+#==============================================================================================
+# Copy Go
+COPY --from=golang /usr/local/go/ /usr/local/go/
+
+#========================================================================================================#
+# Configure container startup
 WORKDIR /app
 COPY start.sh /app/start.sh
 RUN chmod +x /app/start.sh
 
-COPY --from=golang /usr/local/go/ /usr/local/go/
-
+USER $USERNAME
+WORKDIR $HOME
 #========================================================================================================#

.devcontainer/devcontainer.json

@@ -1,16 +1,24 @@
 {
   "build": {
-    "dockerfile": "Dockerfile"
+    "dockerfile": "Dockerfile",
   },
   "features": {
     "ghcr.io/devcontainers/features/go:1": {},
-    "ghcr.io/devcontainers/features/git:1": {}
+    "ghcr.io/devcontainers/features/git:1": {},
   },
-  "postCreateCommand": "sudo /app/start.sh",
+  "postCreateCommand": "sudo bash /app/start.sh",
   "customizations": {
     "vscode": {
-      "extensions": ["golang.go", "GitHub.copilot"]
-    }
+      "extensions": ["golang.go", "GitHub.copilot"],
+      "settings": {
+        "terminal.integrated.defaultProfile.linux": "bash",
+        "terminal.integrated.profiles.linux": {
+          "zsh": {
+            "path": "/bin/bash",
+          },
+        },
+      },
+    },
   },
-  "containerUser": "dev"
+  "containerUser": "dev",
 }

.gitignore

@@ -2,7 +2,7 @@
 #***********************************************************************
 # VSCode things
 .vscode
-
+.devcontainer/cns
 
 #***********************************************************************
 # Benchmark things

.trunk/.gitignore

@@ -2,6 +2,8 @@
 *logs
 *actions
 *notifications
+*tools
 plugins
 user_trunk.yaml
 user.yaml
+tmp

.trunk/configs/.hadolint.yaml

@@ -0,0 +1,4 @@
+# Following source doesn't work in most setups
+ignored:
+  - SC1090
+  - SC1091

.trunk/configs/.yamllint.yaml

@@ -0,0 +1,10 @@
+rules:
+  quoted-strings:
+    required: only-when-needed
+    extra-allowed: ["{|}"]
+  empty-values:
+    forbid-in-block-mappings: true
+    forbid-in-flow-mappings: true
+  key-duplicates: {}
+  octal-values:
+    forbid-implicit-octal: true

.trunk/trunk.yaml

@@ -1,27 +1,38 @@
+# This file controls the behavior of Trunk: https://docs.trunk.io/cli
+# To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml
 version: 0.1
 cli:
-  version: 1.5.1
+  version: 1.19.0
+# Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins)
 plugins:
   sources:
     - id: trunk
-      ref: v0.0.12
+      ref: v1.4.2
       uri: https://github.com/trunk-io/plugins
-lint:
-  enabled:
-    - git-diff-check
-    - [email protected]
-    - [email protected]
-    - [email protected]
-    - [email protected]
-    - [email protected]
-    - [email protected]
-  disabled:
-    - shellcheck
-    - actionlint
+# Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes)
 runtimes:
   enabled:
-    - go@1.19.5
+    - go@1.21.0
     - [email protected]
+    - [email protected]
+# This is the section where you manage your linters. (https://docs.trunk.io/check/configuration)
+lint:
+  enabled:
+    - [email protected]
+    - [email protected]
+    - git-diff-check
+    - [email protected]
+    - [email protected]
+    - [email protected]
+    - [email protected]
+    - [email protected]
+    - [email protected]
+    - [email protected]
+    - [email protected]
+    - [email protected]
+    - [email protected]
+    - [email protected]
+    - [email protected]
 actions:
   enabled:
     - trunk-announce

example/example_haddock30.yml

@@ -1,11 +1,12 @@
 general:
-  executable: /home/rodrigo/repos/haddock-runner/example/haddock3.sh
-  max_concurrent: 4
-  haddock_dir: /home/rodrigo/repos/haddock3
+  executable: /workspaces/haddock-runner/example/haddock3.sh
+  max_concurrent: 1
+  haddock_dir: /opt/haddock3
   receptor_suffix: _r_u
   ligand_suffix: _l_u
-  input_list: /home/rodrigo/repos/haddock-runner/example/input_list.txt
-  work_dir: /home/rodrigo/repos/haddock-runner/bm-goes-here
+  input_list: /workspaces/haddock-runner/example/input_list.txt
+  work_dir: /workspaces/haddock-runner/bm-goes-here
+  use_slurm: true
 
 scenarios:
   - name: true-interface
@@ -31,7 +32,7 @@ scenarios:
           unambig_fname: _unambig.tbl
           ligand_top_fname: _ligand.top
           ligand_param_fname: _ligand.param
-        emref:
+        emref: ~
         caprieval:
           reference_fname: _ref.pdb
 

example/haddock3.sh

@@ -1,12 +1,8 @@
 #!/bin/bash
 #===============================================================================
-HADDOCK3_DIR="$HOME/repos/haddock3"
 
-### Activate the virtual environment
-## if your haddock3 installation uses venv
-source "$HADDOCK3_DIR/venv/bin/activate" || exit
-## if your haddock3  installation uses conda
-# conda activate haddock3
+source /opt/conda/etc/profile.d/conda.sh
+conda activate env
 
 haddock3 "$@"
 #===============================================================================

input/input.go

@@ -34,6 +34,7 @@ type GeneralStruct struct {
 	InputList         string `yaml:"input_list"`
 	WorkDir           string `yaml:"work_dir"`
 	MaxConcurrent     int    `yaml:"max_concurrent"`
+	UseSlurm          bool   `yaml:"use_slurm"`
 }
 
 // Scenario is the scenario structure
@@ -212,6 +213,27 @@ func ValidateRunCNSParams(known map[string]interface{}, params map[string]interf
 	return nil
 }
 
+// ValidateExecutionModes checks if the execution modes are valid
+func (inp *Input) ValidateExecutionModes() error {
+
+	if inp.General.UseSlurm {
+		// Check if the executable is HADDOCK3
+		if utils.IsHaddock24(inp.General.HaddockDir) {
+			err := errors.New("cannot use `use_slurm` with HADDOCK2")
+			return err
+		} else if utils.IsHaddock3(inp.General.HaddockDir) {
+			// We need to check if the Scenarios are using the correct execution modes
+			for _, scenario := range inp.Scenarios {
+				if scenario.Parameters.General["mode"] != "local" {
+					err := errors.New("cannot use `use_slurm` with `mode: " + scenario.Parameters.General["mode"].(string) + "`")
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
 // LoadInput loads the input file
 func LoadInput(filename string) (*Input, error) {
 

input/input_test.go

@@ -532,6 +532,113 @@ func TestValidateRunCNSParams(t *testing.T) {
 
 }
 
+func TestValidateExecutionModes(t *testing.T) {
+
+	// Setup a haddock3 folder structure, it must have this subdirectories:
+	wd, _ := os.Getwd()
+	haddock3Dir := filepath.Join(wd, "TestValidateExecutionModes")
+	_ = os.MkdirAll(haddock3Dir, 0755)
+	defer os.RemoveAll("TestValidateExecutionModes")
+
+	// Add the subdirectories
+	_ = os.MkdirAll(filepath.Join(haddock3Dir, "src/haddock/modules"), 0755)
+
+	// Add an empty defaults.yaml file
+	defaultsF := filepath.Join(haddock3Dir, "src/haddock/modules/defaults.yaml")
+	err := os.WriteFile(defaultsF, []byte(""), 0755)
+	if err != nil {
+		t.Errorf("Failed to write defaults.yaml: %s", err)
+	}
+
+	// Setup a haddock2 folder structure, it must have this subdirectories
+	// protocols/run.cns-conf
+	haddock2Dir := filepath.Join(wd, "TestValidateExecutionModes2")
+	_ = os.MkdirAll(haddock2Dir, 0755)
+	defer os.RemoveAll("TestValidateExecutionModes2")
+
+	// Add the subdirectories
+	_ = os.MkdirAll(filepath.Join(haddock2Dir, "protocols"), 0755)
+
+	// Add an empty run.cns-conf file
+	runCnsF := filepath.Join(haddock2Dir, "protocols/run.cns-conf")
+	err = os.WriteFile(runCnsF, []byte(""), 0755)
+	if err != nil {
+		t.Errorf("Failed to write run.cns-conf: %s", err)
+	}
+
+	type fields struct {
+		General   GeneralStruct
+		Scenarios []Scenario
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		wantErr bool
+	}{
+		{
+			name: "valid",
+			fields: fields{
+				General: GeneralStruct{
+					UseSlurm:   true,
+					HaddockDir: haddock3Dir,
+				},
+				Scenarios: []Scenario{
+					{
+						Name: "true-interface",
+						Parameters: ParametersStruct{
+							General: map[string]any{
+								"mode": "local",
+							},
+						},
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "invalid-haddock2",
+			fields: fields{
+				General: GeneralStruct{
+					UseSlurm:   true,
+					HaddockDir: haddock2Dir,
+				},
+				Scenarios: []Scenario{},
+			},
+			wantErr: true,
+		},
+		{
+			name: "invalid-haddock3",
+			fields: fields{
+				General: GeneralStruct{
+					UseSlurm:   true,
+					HaddockDir: haddock3Dir,
+				},
+				Scenarios: []Scenario{
+					{
+						Name: "true-interface",
+						Parameters: ParametersStruct{
+							General: map[string]any{
+								"mode": "anything",
+							},
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		inp := &Input{
+			General:   tt.fields.General,
+			Scenarios: tt.fields.Scenarios,
+		}
+		if err := inp.ValidateExecutionModes(); (err != nil) != tt.wantErr {
+			t.Errorf("Input.ValidateExecutionModes() error = %v, wantErr %v", err, tt.wantErr)
+		}
+	}
+
+}
+
 func TestLoadHaddock3DefaultParams(t *testing.T) {
 
 	// Create a folder structure and fill it with dummy files