From b550eaa55d5b3badf16f7cf3e3de3287e975ce6d Mon Sep 17 00:00:00 2001 From: Rodrigo Vargas Honorato Date: Thu, 25 Jan 2024 14:14:41 +0100 Subject: [PATCH] Add SLURM interface (#96) * Add validation for `use_slurm` flag in input.go * Add job file creation and sbatch function This commit adds the functionality to create a job file and run sbatch. It includes the implementation of the `CreateJobHeader`, `CreateJobBody`, `PrepareJobFile`, and `Sbatch` functions. Tests for these functions have also been added. * Refactor job file creation and remove unnecessary test cases * Add support for Slurm job submission This commit adds support for submitting jobs using Slurm. It checks if there is a `job.sh` file in the run directory and if so, it uses `sbatch` to submit the job. Otherwise, it continues to use the existing command to run the job. Additionally, a new method `PrepareJobFile` is added to create the `job.sh` file. * upgrade trunk * Update execution mode key in input validation * update dev container setup * update main.go * Refactor Job struct and add support for detecting submitted jobs * Add FindNewestLogFile function to utils.go and corresponding unit test This commit adds a new function called FindNewestLogFile to the utils.go file. This function finds the newest log file in a given directory by comparing the modification times of all the files in the directory. It also includes a corresponding unit test in the utils_test.go file to verify the functionality of the new function. * Add logging for job status and elapsed time --- .devcontainer/Dockerfile | 57 +++++++++++++++-- .devcontainer/devcontainer.json | 20 ++++-- .gitignore | 2 +- .trunk/.gitignore | 2 + .trunk/configs/.hadolint.yaml | 4 ++ .trunk/configs/.yamllint.yaml | 10 +++ .trunk/trunk.yaml | 41 +++++++----- example/example_haddock30.yml | 13 ++-- example/haddock3.sh | 8 +-- input/input.go | 22 +++++++ input/input_test.go | 107 ++++++++++++++++++++++++++++++++ main.go | 30 +++++++-- runner/jobs.go | 52 +++++++++++++++- runner/jobs_test.go | 99 ++++++++++++++++++++++++++++- runner/status/status.go | 1 + utils/utils.go | 47 ++++++++++++++ utils/utils_test.go | 57 +++++++++++++++++ 17 files changed, 526 insertions(+), 46 deletions(-) create mode 100644 .trunk/configs/.hadolint.yaml create mode 100644 .trunk/configs/.yamllint.yaml diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index cf825ca..365d66d 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -4,23 +4,68 @@ FROM golang:1.21 AS golang #========================================================================================================# FROM xenonmiddleware/slurm:20 +#============================================================================================== +# Define ARGs +ARG HADDOCK_VERSION=v3.0.0-beta.5 ARG USERNAME=dev ARG USER_UID=1000 ARG USER_GID=$USER_UID +#============================================================================================== +# Install system dependencies +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential=12.1ubuntu2 \ + sudo=1.8.16-0ubuntu1.10 \ + wget=1.17.1-1ubuntu1.5 \ + git=1:2.7.4-0ubuntu1.10 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +#============================================================================================== +# Configure User RUN groupadd --gid $USER_GID $USERNAME \ && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \ - && apt-get update \ - && apt-get install -y --no-install-recommends sudo=1.8.16-0ubuntu1.10 \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ && chmod 0440 /etc/sudoers.d/$USERNAME +#============================================================================================== +# Install miniconda +ENV CONDA_DIR /opt/conda +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh \ + && /bin/bash ~/miniconda.sh -b -p /opt/conda + +ENV PATH=$CONDA_DIR/bin:$PATH +RUN conda install python=3.9 + +#============================================================================================== +# Install HADDOCK3 +WORKDIR /opt + +RUN git clone --recursive https://github.com/haddocking/haddock3.git +WORKDIR /opt/haddock3 +RUN git checkout ${HADDOCK_VERSION} + +WORKDIR /opt/haddock3/src/fcc/src +RUN make + +WORKDIR /opt/haddock3 +RUN pip install --no-cache-dir -r requirements.txt \ + && python setup.py develop + +WORKDIR /opt/haddock3/bin +COPY cns /opt/haddock3/bin/cns + +#============================================================================================== +# Copy Go +COPY --from=golang /usr/local/go/ /usr/local/go/ + +#========================================================================================================# +# Configure container startup WORKDIR /app COPY start.sh /app/start.sh RUN chmod +x /app/start.sh -COPY --from=golang /usr/local/go/ /usr/local/go/ - +USER $USERNAME +WORKDIR $HOME #========================================================================================================# diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 93c86b9..1db92e2 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,16 +1,24 @@ { "build": { - "dockerfile": "Dockerfile" + "dockerfile": "Dockerfile", }, "features": { "ghcr.io/devcontainers/features/go:1": {}, - "ghcr.io/devcontainers/features/git:1": {} + "ghcr.io/devcontainers/features/git:1": {}, }, - "postCreateCommand": "sudo /app/start.sh", + "postCreateCommand": "sudo bash /app/start.sh", "customizations": { "vscode": { - "extensions": ["golang.go", "GitHub.copilot"] - } + "extensions": ["golang.go", "GitHub.copilot"], + "settings": { + "terminal.integrated.defaultProfile.linux": "bash", + "terminal.integrated.profiles.linux": { + "zsh": { + "path": "/bin/bash", + }, + }, + }, + }, }, - "containerUser": "dev" + "containerUser": "dev", } diff --git a/.gitignore b/.gitignore index c52e7a2..a0bd9bb 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ #*********************************************************************** # VSCode things .vscode - +.devcontainer/cns #*********************************************************************** # Benchmark things diff --git a/.trunk/.gitignore b/.trunk/.gitignore index cf2f254..15966d0 100644 --- a/.trunk/.gitignore +++ b/.trunk/.gitignore @@ -2,6 +2,8 @@ *logs *actions *notifications +*tools plugins user_trunk.yaml user.yaml +tmp diff --git a/.trunk/configs/.hadolint.yaml b/.trunk/configs/.hadolint.yaml new file mode 100644 index 0000000..98bf0cd --- /dev/null +++ b/.trunk/configs/.hadolint.yaml @@ -0,0 +1,4 @@ +# Following source doesn't work in most setups +ignored: + - SC1090 + - SC1091 diff --git a/.trunk/configs/.yamllint.yaml b/.trunk/configs/.yamllint.yaml new file mode 100644 index 0000000..4d44466 --- /dev/null +++ b/.trunk/configs/.yamllint.yaml @@ -0,0 +1,10 @@ +rules: + quoted-strings: + required: only-when-needed + extra-allowed: ["{|}"] + empty-values: + forbid-in-block-mappings: true + forbid-in-flow-mappings: true + key-duplicates: {} + octal-values: + forbid-implicit-octal: true diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml index 5e86c20..b48f1eb 100644 --- a/.trunk/trunk.yaml +++ b/.trunk/trunk.yaml @@ -1,27 +1,38 @@ +# This file controls the behavior of Trunk: https://docs.trunk.io/cli +# To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml version: 0.1 cli: - version: 1.5.1 + version: 1.19.0 +# Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins) plugins: sources: - id: trunk - ref: v0.0.12 + ref: v1.4.2 uri: https://github.com/trunk-io/plugins -lint: - enabled: - - git-diff-check - - gitleaks@8.16.0 - - gofmt@1.19.3 - - golangci-lint@1.51.2 - - markdownlint@0.33.0 - - prettier@2.8.4 - - shfmt@3.5.0 - disabled: - - shellcheck - - actionlint +# Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes) runtimes: enabled: - - go@1.19.5 + - go@1.21.0 - node@18.12.1 + - python@3.10.8 +# This is the section where you manage your linters. (https://docs.trunk.io/check/configuration) +lint: + enabled: + - actionlint@1.6.26 + - checkov@3.1.69 + - git-diff-check + - gofmt@1.20.4 + - golangci-lint@1.55.2 + - hadolint@2.12.0 + - markdownlint@0.38.0 + - osv-scanner@1.6.1 + - prettier@3.2.4 + - shellcheck@0.9.0 + - shfmt@3.6.0 + - terrascan@1.18.11 + - trivy@0.48.3 + - trufflehog@3.63.11 + - yamllint@1.33.0 actions: enabled: - trunk-announce diff --git a/example/example_haddock30.yml b/example/example_haddock30.yml index c807002..858f75c 100644 --- a/example/example_haddock30.yml +++ b/example/example_haddock30.yml @@ -1,11 +1,12 @@ general: - executable: /home/rodrigo/repos/haddock-runner/example/haddock3.sh - max_concurrent: 4 - haddock_dir: /home/rodrigo/repos/haddock3 + executable: /workspaces/haddock-runner/example/haddock3.sh + max_concurrent: 1 + haddock_dir: /opt/haddock3 receptor_suffix: _r_u ligand_suffix: _l_u - input_list: /home/rodrigo/repos/haddock-runner/example/input_list.txt - work_dir: /home/rodrigo/repos/haddock-runner/bm-goes-here + input_list: /workspaces/haddock-runner/example/input_list.txt + work_dir: /workspaces/haddock-runner/bm-goes-here + use_slurm: true scenarios: - name: true-interface @@ -31,7 +32,7 @@ scenarios: unambig_fname: _unambig.tbl ligand_top_fname: _ligand.top ligand_param_fname: _ligand.param - emref: + emref: ~ caprieval: reference_fname: _ref.pdb diff --git a/example/haddock3.sh b/example/haddock3.sh index e1cd74e..db533f3 100755 --- a/example/haddock3.sh +++ b/example/haddock3.sh @@ -1,12 +1,8 @@ #!/bin/bash #=============================================================================== -HADDOCK3_DIR="$HOME/repos/haddock3" -### Activate the virtual environment -## if your haddock3 installation uses venv -source "$HADDOCK3_DIR/venv/bin/activate" || exit -## if your haddock3 installation uses conda -# conda activate haddock3 +source /opt/conda/etc/profile.d/conda.sh +conda activate env haddock3 "$@" #=============================================================================== diff --git a/input/input.go b/input/input.go index 70a88c4..aee36e5 100644 --- a/input/input.go +++ b/input/input.go @@ -34,6 +34,7 @@ type GeneralStruct struct { InputList string `yaml:"input_list"` WorkDir string `yaml:"work_dir"` MaxConcurrent int `yaml:"max_concurrent"` + UseSlurm bool `yaml:"use_slurm"` } // Scenario is the scenario structure @@ -212,6 +213,27 @@ func ValidateRunCNSParams(known map[string]interface{}, params map[string]interf return nil } +// ValidateExecutionModes checks if the execution modes are valid +func (inp *Input) ValidateExecutionModes() error { + + if inp.General.UseSlurm { + // Check if the executable is HADDOCK3 + if utils.IsHaddock24(inp.General.HaddockDir) { + err := errors.New("cannot use `use_slurm` with HADDOCK2") + return err + } else if utils.IsHaddock3(inp.General.HaddockDir) { + // We need to check if the Scenarios are using the correct execution modes + for _, scenario := range inp.Scenarios { + if scenario.Parameters.General["mode"] != "local" { + err := errors.New("cannot use `use_slurm` with `mode: " + scenario.Parameters.General["mode"].(string) + "`") + return err + } + } + } + } + return nil +} + // LoadInput loads the input file func LoadInput(filename string) (*Input, error) { diff --git a/input/input_test.go b/input/input_test.go index d617378..4a3044f 100644 --- a/input/input_test.go +++ b/input/input_test.go @@ -532,6 +532,113 @@ func TestValidateRunCNSParams(t *testing.T) { } +func TestValidateExecutionModes(t *testing.T) { + + // Setup a haddock3 folder structure, it must have this subdirectories: + wd, _ := os.Getwd() + haddock3Dir := filepath.Join(wd, "TestValidateExecutionModes") + _ = os.MkdirAll(haddock3Dir, 0755) + defer os.RemoveAll("TestValidateExecutionModes") + + // Add the subdirectories + _ = os.MkdirAll(filepath.Join(haddock3Dir, "src/haddock/modules"), 0755) + + // Add an empty defaults.yaml file + defaultsF := filepath.Join(haddock3Dir, "src/haddock/modules/defaults.yaml") + err := os.WriteFile(defaultsF, []byte(""), 0755) + if err != nil { + t.Errorf("Failed to write defaults.yaml: %s", err) + } + + // Setup a haddock2 folder structure, it must have this subdirectories + // protocols/run.cns-conf + haddock2Dir := filepath.Join(wd, "TestValidateExecutionModes2") + _ = os.MkdirAll(haddock2Dir, 0755) + defer os.RemoveAll("TestValidateExecutionModes2") + + // Add the subdirectories + _ = os.MkdirAll(filepath.Join(haddock2Dir, "protocols"), 0755) + + // Add an empty run.cns-conf file + runCnsF := filepath.Join(haddock2Dir, "protocols/run.cns-conf") + err = os.WriteFile(runCnsF, []byte(""), 0755) + if err != nil { + t.Errorf("Failed to write run.cns-conf: %s", err) + } + + type fields struct { + General GeneralStruct + Scenarios []Scenario + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + { + name: "valid", + fields: fields{ + General: GeneralStruct{ + UseSlurm: true, + HaddockDir: haddock3Dir, + }, + Scenarios: []Scenario{ + { + Name: "true-interface", + Parameters: ParametersStruct{ + General: map[string]any{ + "mode": "local", + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "invalid-haddock2", + fields: fields{ + General: GeneralStruct{ + UseSlurm: true, + HaddockDir: haddock2Dir, + }, + Scenarios: []Scenario{}, + }, + wantErr: true, + }, + { + name: "invalid-haddock3", + fields: fields{ + General: GeneralStruct{ + UseSlurm: true, + HaddockDir: haddock3Dir, + }, + Scenarios: []Scenario{ + { + Name: "true-interface", + Parameters: ParametersStruct{ + General: map[string]any{ + "mode": "anything", + }, + }, + }, + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + inp := &Input{ + General: tt.fields.General, + Scenarios: tt.fields.Scenarios, + } + if err := inp.ValidateExecutionModes(); (err != nil) != tt.wantErr { + t.Errorf("Input.ValidateExecutionModes() error = %v, wantErr %v", err, tt.wantErr) + } + } + +} + func TestLoadHaddock3DefaultParams(t *testing.T) { // Create a folder structure and fill it with dummy files diff --git a/main.go b/main.go index 71d06fb..c6333d6 100644 --- a/main.go +++ b/main.go @@ -82,6 +82,11 @@ func main() { glog.Exit("ERROR: " + errPatt.Error()) } + errExecutionModes := inp.ValidateExecutionModes() + if errExecutionModes != nil { + glog.Exit("ERROR: " + errExecutionModes.Error()) + } + // haddockVersion := inp.General.HaddockVersion var haddockVersion int if utils.IsHaddock24(inp.General.HaddockDir) { @@ -211,8 +216,18 @@ func main() { // -------------------------------------------- fallthrough + case job.Status == status.SUBMITTED: + glog.Info(job.ID + " - " + job.Status + " - waiting") + default: now := time.Now() + if inp.General.UseSlurm { + err := job.PrepareJobFile(inp.General.HaddockExecutable) + if err != nil { + glog.Exit("Failed to prepare job file: " + err.Error()) + } + } + _, runErr := job.Run(haddockVersion, inp.General.HaddockExecutable) if runErr != nil { glog.Exit("Failed to run HADDOCK: " + runErr.Error()) @@ -223,17 +238,24 @@ func main() { glog.Exit("Failed to get job status: " + err.Error()) } elapsed := time.Since(now) - glog.Info(job.ID + " - " + job.Status + " in " + fmt.Sprintf("%.2f", elapsed.Seconds()) + " seconds") + if job.Status != status.QUEUED { + glog.Info(job.ID + " - " + job.Status + " in " + fmt.Sprintf("%.2f", elapsed.Seconds()) + " seconds") + } else { + glog.Info(job.ID + " - " + job.Status) + } } done <- true }(job, i) } - // Wait until all the jobs are done. + // Wait until all the jobs are submitted. <-waitForAllJobs glog.Info("############################################") - - glog.Info("haddock-runner finished successfully") + if inp.General.UseSlurm { + glog.Info("haddock-runner finished successfully (things might still be running on the cluster)") + } else { + glog.Info("haddock-runner finished successfully") + } } diff --git a/runner/jobs.go b/runner/jobs.go index 97ea21f..1d90c95 100644 --- a/runner/jobs.go +++ b/runner/jobs.go @@ -131,7 +131,15 @@ func (j Job) RunHaddock3(cmd string) (string, error) { // Run HADDOCK3 runWD := filepath.Join(j.Path) - cmd = cmd + " run.toml" + + // Check if there is a `job.sh` file in the run directory + jobF := filepath.Join(runWD, "job.sh") + _, err := os.Stat(jobF) + if err == nil { + cmd = "sbatch " + jobF + } else { + cmd = cmd + " run.toml" + } logF, err := Run(cmd, runWD) if err != nil { err := errors.New("Error running HADDOCK: " + err.Error()) @@ -171,6 +179,35 @@ func (j Job) Run(version int, cmd string) (string, error) { } +// PrepareJobFile prepares the job file, returns the path to the job file +func (j *Job) PrepareJobFile(executable string) error { + + var header string + var body string + + // Create the JobFile + header = utils.CreateJobHeader() + + // Create the JobBody + body = utils.CreateJobBody(executable, j.Path) + + // Create the JobFile + jobFile := filepath.Join(j.Path, "job.sh") + f, err := os.Create(jobFile) + if err != nil { + err := errors.New("Error creating job file: " + err.Error()) + return err + } + + // Write the JobFile + f.WriteString(header + body) + + _ = f.Close() + + return nil + +} + func (j *Job) GetStatus(version int) error { var logF string @@ -223,6 +260,19 @@ func (j *Job) GetStatus(version int) error { } } + // Before saying that the job is incomplete, check if its running on slurm + newestFile := utils.FindNewestLogFile(j.Path) + + found, _ := utils.SearchInLog(newestFile, "Submitted batch job") + // if err != nil { + // return err + // } + + if found { + j.Status = status.SUBMITTED + return nil + } + j.Status = status.INCOMPLETE return nil diff --git a/runner/jobs_test.go b/runner/jobs_test.go index e98fa92..d90ee17 100644 --- a/runner/jobs_test.go +++ b/runner/jobs_test.go @@ -43,6 +43,12 @@ func TestRunHaddock3(t *testing.T) { _ = os.MkdirAll("_run-test", 0755) defer os.RemoveAll("_run-test") + // Create a directory that contains a job.sh file + _ = os.MkdirAll("_run-test-with-job-file", 0755) + jobF := filepath.Join("_run-test-with-job-file", "job.sh") + _ = os.WriteFile(jobF, []byte(""), 0644) + defer os.RemoveAll("_run-test-with-job-file") + // Create a Job j := Job{ ID: "test", @@ -71,6 +77,13 @@ func TestRunHaddock3(t *testing.T) { t.Errorf("Error running haddock: %v", err) } + // Fail by running a command in a directory that contains a job.sh file + j.Path = "_run-test-with-job-file" + _, err = j.RunHaddock3(cmd) + if err == nil { + t.Errorf("Error running haddock: %v", err) + } + } func TestJob_SetupHaddock24(t *testing.T) { @@ -216,7 +229,9 @@ func TestJobRun(t *testing.T) { { name: "pass v3", fields: fields{ - j: Job{}, + j: Job{ + Path: temptestP, + }, }, args: args{ version: 3, @@ -309,6 +324,18 @@ func TestJobGetStatus(t *testing.T) { logF = filepath.Join(v3PositiveTempD, "run1", "log") _ = os.WriteFile(logF, []byte("This HADDOCK3 run took"), 0644) + // Setup a submitted test for v3 + v3SubmittedTempD, _ := os.MkdirTemp("", "v3-submitted") + defer os.RemoveAll(v3SubmittedTempD) + err = os.MkdirAll(filepath.Join(v3SubmittedTempD, "run1"), 0755) + if err != nil { + t.Errorf("Error creating test directory: %v", err) + } + logF = filepath.Join(v3SubmittedTempD, "run1", "log") + _ = os.WriteFile(logF, []byte(""), 0644) + subLogF := filepath.Join(v3SubmittedTempD, "something.txt") + _ = os.WriteFile(subLogF, []byte("Submitted batch job"), 0644) + // Setup the incomplete scenario incompleteTempD, _ := os.MkdirTemp("", "incomplete") defer os.RemoveAll(incompleteTempD) @@ -403,6 +430,18 @@ func TestJobGetStatus(t *testing.T) { }, wantErr: false, }, + { + name: "pass with submitted v3", + fields: fields{ + j: Job{ + Path: v3SubmittedTempD, + }, + }, + args: args{ + version: 3, + }, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -417,6 +456,64 @@ func TestJobGetStatus(t *testing.T) { } } +func TestJobPrepareJobFile(t *testing.T) { + + // Create a valid Path + err := os.MkdirAll("_test_prepare_job_file", 0755) + if err != nil { + t.Errorf("Failed to create folder: %s", err) + } + defer os.RemoveAll("_test_prepare_job_file") + + type fields struct { + j Job + } + type args struct { + executable string + } + + tests := []struct { + name string + args args + fields fields + wantErr bool + }{ + { + name: "pass by creating a job file", + args: args{ + executable: "echo test", + }, + fields: fields{ + j: Job{ + Path: "_test_prepare_job_file", + }, + }, + }, + { + name: "fail by passing a non-existing path", + args: args{ + executable: "echo test", + }, + fields: fields{ + j: Job{ + Path: "does-not-exist", + }, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.fields.j.PrepareJobFile(tt.args.executable) + if (err != nil) != tt.wantErr { + t.Errorf("PrepareJobFile() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + // setupHaddock24ForTest is a helper function that creates a directory structure for testing // // Note: This could be a Mock- feel free to change it (: diff --git a/runner/status/status.go b/runner/status/status.go index 87fa0c9..789bcc7 100644 --- a/runner/status/status.go +++ b/runner/status/status.go @@ -6,4 +6,5 @@ const ( QUEUED = "QUEUED" INCOMPLETE = "INCOMPLETE" UNKNOWN = "UNKNOWN" + SUBMITTED = "SUBMITTED" ) diff --git a/utils/utils.go b/utils/utils.go index a59d081..fdf1e05 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -237,3 +237,50 @@ func SearchInLog(filePath, searchString string) (bool, error) { return false, nil } + +func CreateJobHeader() string { + + header := "#!/bin/bash\n" + header += "#SBATCH --job-name=haddock\n" + header += "#SBATCH --output=haddock-%j.out\n" + header += "#SBATCH --error=haddock-%j.err\n" + header += "#SBATCH --time=7-00:00:00\n" + header += "#SBATCH --nodes=1\n" + header += "#SBATCH --ntasks-per-node=1\n" + header += "#SBATCH --cpus-per-task=1\n" + header += "\n" + + return header +} + +func CreateJobBody(cmd, path string) string { + + body := "cd " + path + "\n" + body += cmd + " run.toml\n" + + return body +} + +func FindNewestLogFile(path string) string { + + files, _ := filepath.Glob(filepath.Join(path, "*.txt")) + // if err != nil { + // return "" + // } + + // Find what is the newest file + var newestFile string + var newestTime int64 + + for _, f := range files { + fi, _ := os.Stat(f) + // if err != nil { + // return "" + // } + if fi.ModTime().Unix() > newestTime { + newestTime = fi.ModTime().Unix() + newestFile = f + } + } + return newestFile +} diff --git a/utils/utils_test.go b/utils/utils_test.go index 81c00d4..b2c048b 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -6,6 +6,7 @@ import ( "reflect" "regexp" "testing" + "time" ) func TestCopyFile(t *testing.T) { @@ -401,3 +402,59 @@ func TestSearchInLog(t *testing.T) { } } + +func TestCreateJobHeader(t *testing.T) { + // TODO: Improve this test, now its just checking if the function runs without errors + // it should check if the output is correct + output := CreateJobHeader() + + if output == "" { + t.Errorf("Failed to create job header") + } + +} + +func TestCreateJobBody(t *testing.T) { + // TODO: Improve this test, now its just checking if the function runs without errors + // it should check if the output is correct + output := CreateJobBody("", "") + + if output == "" { + t.Errorf("Failed to create job body") + } +} + +func TestFindNewestLogFile(t *testing.T) { + + // Write two files, return the newest one + err := os.WriteFile("file1.txt", []byte("file1"), 0644) + if err != nil { + t.Errorf("Failed to write file: %s", err) + } + defer os.Remove("file1.txt") + + err = os.WriteFile("file2.txt", []byte("file2"), 0644) + if err != nil { + t.Errorf("Failed to write file: %s", err) + } + defer os.Remove("file2.txt") + + // Set the modification time of file1.txt to be older than file2.txt + err = os.Chtimes("file1.txt", time.Now().Add(-1*time.Hour), time.Now().Add(-1*time.Hour)) + if err != nil { + t.Errorf("Failed to change file modification time: %s", err) + } + + // Check if the newest file is returned + newestFile := FindNewestLogFile(".") + if newestFile != "file2.txt" { + t.Errorf("Failed to find newest file") + } + + // Fail with a folder that does not exist + newestFile = FindNewestLogFile("does-not-exist") + if newestFile != "" { + t.Errorf("Failed to detect wrong folder") + } + +}