diff --git a/src/utils/operandi_utils/oton/constants.py b/src/utils/operandi_utils/oton/constants.py index 8075002..180fc94 100644 --- a/src/utils/operandi_utils/oton/constants.py +++ b/src/utils/operandi_utils/oton/constants.py @@ -32,17 +32,4 @@ PARAMS_KEY_CPUS_PER_FORK: str = 'params.cpus_per_fork' PARAMS_KEY_RAM_PER_FORK: str = 'params.ram_per_fork' -REPR_INPUT_FILE_GRP: str = f"""{PARAMS_KEY_INPUT_FILE_GRP} = "null\"""" -REPR_METS_PATH: str = f"""{PARAMS_KEY_METS_PATH} = "null\"""" -REPR_METS_SOCKET_PATH: str = f"""{PARAMS_KEY_METS_SOCKET_PATH} = "null\"""" -REPR_WORKSPACE_DIR: str = f"""{PARAMS_KEY_WORKSPACE_DIR} = "null\"""" -REPR_ENV_WRAPPER_CMD_CORE: str = f"""{PARAMS_KEY_ENV_WRAPPER_CMD_CORE} = "null\"""" -REPR_PAGES: str = f"""{PARAMS_KEY_PAGES} = "null\"""" -REPR_CPUS: str = f"""{PARAMS_KEY_CPUS} = "null\"""" -REPR_RAM: str = f"""{PARAMS_KEY_RAM} = "null\"""" -REPR_FORKS: str = f"""{PARAMS_KEY_FORKS} = {PARAMS_KEY_CPUS}""" -REPR_FORKS_NULL: str = f"""{PARAMS_KEY_FORKS} = "4\"""" -REPR_CPUS_PER_FORK: str = f"""{PARAMS_KEY_CPUS_PER_FORK} = ({PARAMS_KEY_CPUS}.toInteger() / {PARAMS_KEY_FORKS}.toInteger()).intValue()""" -REPR_RAM_PER_FORK: str = f"""{PARAMS_KEY_RAM_PER_FORK} = sprintf("%dGB", ({PARAMS_KEY_RAM}.toInteger() / {PARAMS_KEY_FORKS}.toInteger()).intValue())""" - WORKFLOW_COMMENT = f"// This workflow was automatically generated by the v{OPERANDI_VERSION} operandi_utils.oton module" diff --git a/src/utils/operandi_utils/oton/nf_file_executable.py b/src/utils/operandi_utils/oton/nf_file_executable.py index 61cf247..57d8d31 100644 --- a/src/utils/operandi_utils/oton/nf_file_executable.py +++ b/src/utils/operandi_utils/oton/nf_file_executable.py @@ -11,20 +11,12 @@ PARAMS_KEY_ENV_WRAPPER_CMD_CORE, PARAMS_KEY_ENV_WRAPPER_CMD_STEP, PARAMS_KEY_FORKS, + PARAMS_KEY_PAGES, + PARAMS_KEY_CPUS, PARAMS_KEY_CPUS_PER_FORK, + PARAMS_KEY_RAM, PARAMS_KEY_RAM_PER_FORK, - REPR_ENV_WRAPPER_CMD_CORE, - REPR_INPUT_FILE_GRP, - REPR_METS_PATH, - REPR_METS_SOCKET_PATH, - REPR_WORKSPACE_DIR, - REPR_PAGES, - REPR_CPUS, - REPR_RAM, - REPR_FORKS, - REPR_FORKS_NULL, - REPR_CPUS_PER_FORK, - REPR_RAM_PER_FORK, + PARAMS_KEY_METS_SOCKET_PATH, SPACES, WORKFLOW_COMMENT ) @@ -38,7 +30,7 @@ def __init__(self): self.logger.setLevel(getLevelName(OTON_LOG_LEVEL)) self.supported_environments = ["local", "docker", "apptainer"] - self.nf_lines_parameters: List[str] = [] + self.nf_lines_parameters = {} self.nf_process_split_range = None self.nf_process_merging_mets = None self.nf_blocks_process: List[NextflowBlockProcess] = [] @@ -48,28 +40,28 @@ def build_parameters(self, environment: str, with_mets_server: bool): if environment not in self.supported_environments: raise ValueError(f"Invalid environment value: {environment}. Must be one of: {self.supported_environments}") - self.nf_lines_parameters.append('nextflow.enable.dsl = 2') - self.nf_lines_parameters.append('') - self.nf_lines_parameters.append(REPR_INPUT_FILE_GRP) - self.nf_lines_parameters.append(REPR_METS_PATH) - self.nf_lines_parameters.append(REPR_WORKSPACE_DIR) - self.nf_lines_parameters.append(REPR_PAGES) + self.nf_lines_parameters[PARAMS_KEY_INPUT_FILE_GRP] = '"null"' + self.nf_lines_parameters[PARAMS_KEY_METS_PATH] = '"null"' + self.nf_lines_parameters[PARAMS_KEY_WORKSPACE_DIR] = '"null"' + self.nf_lines_parameters[PARAMS_KEY_PAGES] = '"null"' if with_mets_server: - self.nf_lines_parameters.append(REPR_METS_SOCKET_PATH) + self.nf_lines_parameters[PARAMS_KEY_METS_SOCKET_PATH] = '"null"' if environment == "local": - self.nf_lines_parameters.append(REPR_FORKS_NULL) + self.nf_lines_parameters[PARAMS_KEY_FORKS] = '"4"' if environment == "docker": - self.nf_lines_parameters.append(REPR_FORKS_NULL) - self.nf_lines_parameters.append(REPR_ENV_WRAPPER_CMD_CORE) + self.nf_lines_parameters[PARAMS_KEY_FORKS] = '"4"' + self.nf_lines_parameters[PARAMS_KEY_ENV_WRAPPER_CMD_CORE] = '"null"' if environment == "apptainer": - self.nf_lines_parameters.append(REPR_CPUS) - self.nf_lines_parameters.append(REPR_RAM) - self.nf_lines_parameters.append(REPR_FORKS) - self.nf_lines_parameters.append(REPR_CPUS_PER_FORK) - self.nf_lines_parameters.append(REPR_RAM_PER_FORK) - self.nf_lines_parameters.append(REPR_ENV_WRAPPER_CMD_CORE) + self.nf_lines_parameters[PARAMS_KEY_CPUS] = '"null"' + self.nf_lines_parameters[PARAMS_KEY_RAM] = '"null"' + self.nf_lines_parameters[PARAMS_KEY_FORKS] = f'{PARAMS_KEY_CPUS}' + self.nf_lines_parameters[PARAMS_KEY_CPUS_PER_FORK] = \ + f'({PARAMS_KEY_CPUS}.toInteger() / {PARAMS_KEY_FORKS}.toInteger()).intValue()' + self.nf_lines_parameters[PARAMS_KEY_RAM_PER_FORK] = \ + f'sprintf("%dGB", ({PARAMS_KEY_RAM}.toInteger() / {PARAMS_KEY_FORKS}.toInteger()).intValue())' + self.nf_lines_parameters[PARAMS_KEY_ENV_WRAPPER_CMD_CORE] = '"null"' # TODO: Refactor later def build_split_page_ranges_process(self, environment: str, with_mets_server: bool) -> NextflowBlockProcess: @@ -180,35 +172,22 @@ def build_nextflow_processes( nf_process_block.add_parameter_output(parameter=CONST_METS_PATH, parameter_type='val') nf_process_block.add_parameter_output(parameter=CONST_PAGE_RANGE, parameter_type='val') nf_process_block.add_parameter_output(parameter=CONST_WORKSPACE_DIR, parameter_type='val') - self.nf_lines_parameters.append(f'{PARAMS_KEY_ENV_WRAPPER_CMD_STEP}{index} = "null"') + self.nf_lines_parameters[f'{PARAMS_KEY_ENV_WRAPPER_CMD_STEP}{index}'] = '"null"' self.nf_blocks_process.append(nf_process_block) index += 1 - def __assign_first_file_grps_param(self): - first_file_grps = self.nf_blocks_process[0].processor_call_arguments.input_file_grps - index = 0 - for parameter in self.nf_lines_parameters: - if PARAMS_KEY_INPUT_FILE_GRP in parameter: - self.nf_lines_parameters[index] = parameter.replace("null", first_file_grps) - break - index += 1 - def build_log_info_prints(self) -> str: log_info = f'log.info """\\\n' log_info += f"{SPACES}OPERANDI HPC - Nextflow Workflow\n" log_info += f"{SPACES}===================================================\n" - for param in self.nf_lines_parameters: - if not param or "params." not in param: - continue - param_key = param[param.find(".") + 1:param.find("=") - 1] - log_info += f"{SPACES}{param_key}: " - log_info += f'${BS[0]}{param[0:param.find("=") - 1]}{BS[1]}\n' + for key, value in self.nf_lines_parameters.items(): + log_info += f"{SPACES}{key[len('params.'):]}: ${BS[0]}{key}{BS[1]}\n" log_info += f'{SPACES}""".stripIndent()\n' return log_info - # TODO: Refactor later def build_main_workflow(self, with_mets_server: bool): - self.__assign_first_file_grps_param() + first_file_grps = self.nf_blocks_process[0].processor_call_arguments.input_file_grps + self.nf_lines_parameters[PARAMS_KEY_INPUT_FILE_GRP] = f'"{first_file_grps}"' nf_workflow_block = NextflowBlockWorkflow( workflow_name="main", nf_processes=self.nf_blocks_process, @@ -223,8 +202,10 @@ def produce_nextflow_file(self, output_path: str, environment: str, with_mets_se # Write Nextflow line tokens to an output file with open(output_path, mode='w', encoding='utf-8') as nextflow_file: nextflow_file.write(f"{WORKFLOW_COMMENT}\n") - for nextflow_line in self.nf_lines_parameters: - nextflow_file.write(f'{nextflow_line}\n') + nextflow_file.write("nextflow.enable.dsl = 2\n") + nextflow_file.write("\n") + for key, value in self.nf_lines_parameters.items(): + nextflow_file.write(f'{key} = {value}\n') nextflow_file.write("\n") nextflow_file.write(self.build_log_info_prints()) nextflow_file.write("\n") diff --git a/tests/assets/oton/constants.py b/tests/assets/oton/constants.py index b8f3e05..6e540e8 100644 --- a/tests/assets/oton/constants.py +++ b/tests/assets/oton/constants.py @@ -105,33 +105,32 @@ } """ -PARAMETERS_COMMON = [ - 'nextflow.enable.dsl = 2', - 'params.mets_path = "null"', - 'params.workspace_dir = "null"', - 'params.pages = "null"', -] +PARAMETERS_COMMON = { + 'params.mets_path': '"null"', + 'params.workspace_dir': '"null"', + 'params.pages': '"null"', +} -PARAMETERS_LOCAL = [ - 'params.forks = "4"' -] +PARAMETERS_LOCAL = { + 'params.forks': '"4"', +} -PARAMETERS_DOCKER = [ - 'params.forks = "4"', - 'params.env_wrapper_cmd_core = "null"', - 'params.env_wrapper_cmd_step0 = "null"', - 'params.env_wrapper_cmd_step1 = "null"', - 'params.env_wrapper_cmd_step2 = "null"', -] +PARAMETERS_DOCKER = { + 'params.forks': '"4"', + 'params.env_wrapper_cmd_core': '"null"', + 'params.env_wrapper_cmd_step0': '"null"', + 'params.env_wrapper_cmd_step1': '"null"', + 'params.env_wrapper_cmd_step2': '"null"', +} -PARAMETERS_APPTAINER = [ - 'params.cpus = "null"', - 'params.ram = "null"', - 'params.forks = params.cpus', - 'params.cpus_per_fork = (params.cpus.toInteger() / params.forks.toInteger()).intValue()', - 'params.ram_per_fork = sprintf("%dGB", (params.ram.toInteger() / params.forks.toInteger()).intValue())', - 'params.env_wrapper_cmd_core = "null"', - 'params.env_wrapper_cmd_step0 = "null"', - 'params.env_wrapper_cmd_step1 = "null"', - 'params.env_wrapper_cmd_step2 = "null"', -] +PARAMETERS_APPTAINER = { + 'params.cpus': '"null"', + 'params.ram': '"null"', + 'params.forks': 'params.cpus', + 'params.cpus_per_fork': '(params.cpus.toInteger() / params.forks.toInteger()).intValue()', + 'params.ram_per_fork': 'sprintf("%dGB", (params.ram.toInteger() / params.forks.toInteger()).intValue())', + 'params.env_wrapper_cmd_core': '"null"', + 'params.env_wrapper_cmd_step0': '"null"', + 'params.env_wrapper_cmd_step1': '"null"', + 'params.env_wrapper_cmd_step2': '"null"', +} diff --git a/tests/tests_utils/test_2_oton/assert_utils.py b/tests/tests_utils/test_2_oton/assert_utils.py index ed197be..11d334b 100644 --- a/tests/tests_utils/test_2_oton/assert_utils.py +++ b/tests/tests_utils/test_2_oton/assert_utils.py @@ -13,7 +13,7 @@ def assert_common_features( for parameter in PARAMETERS_COMMON: assert parameter in parameters if with_mets_server: - assert 'params.mets_socket_path = "null"' in parameters, f"params.mets_socket_path is missing in {parameters}" + assert parameters['params.mets_socket_path'] == '"null"', f"params.mets_socket_path is missing in {parameters}" blocks_process = nextflow_file_class.nf_blocks_process assert len(blocks_process) == num_blocks_process for block in blocks_process: diff --git a/tests/tests_utils/test_2_oton/test_3_converter_1_local.py b/tests/tests_utils/test_2_oton/test_3_converter_1_local.py index e113bca..1232a75 100644 --- a/tests/tests_utils/test_2_oton/test_3_converter_1_local.py +++ b/tests/tests_utils/test_2_oton/test_3_converter_1_local.py @@ -9,20 +9,20 @@ def test_convert_wf1_with_env_local(oton_converter): nextflow_file_class = oton_converter.convert_oton(IN_TXT_WF1, OUT_NF_WF1_LOCAL, "local", False) - assert 'params.input_file_group = "OCR-D-IMG"' in nextflow_file_class.nf_lines_parameters + assert nextflow_file_class.nf_lines_parameters['params.input_file_group'] == '"OCR-D-IMG"' assert_common_features(nextflow_file_class, 8, 1, False) assert_compare_workflow_blocks(OUT_NF_WF1_LOCAL, EXPECTED_WF1) def test_convert_wf1_with_env_local_with_mets_server(oton_converter): nextflow_file_class = oton_converter.convert_oton(IN_TXT_WF1, OUT_NF_WF1_LOCAL_WITH_MS, "local", True) - assert 'params.input_file_group = "OCR-D-IMG"' in nextflow_file_class.nf_lines_parameters + assert nextflow_file_class.nf_lines_parameters['params.input_file_group'] == '"OCR-D-IMG"' assert_common_features(nextflow_file_class, 8, 1, True) assert_compare_workflow_blocks(OUT_NF_WF1_LOCAL_WITH_MS, EXPECTED_WF1_WITH_MS) def test_convert_wf2_with_env_local(oton_converter): nextflow_file_class = oton_converter.convert_oton(IN_TXT_WF2, OUT_NF_WF2_LOCAL, "local", False) - assert 'params.input_file_group = "OCR-D-IMG"' in nextflow_file_class.nf_lines_parameters + assert nextflow_file_class.nf_lines_parameters['params.input_file_group'] == '"OCR-D-IMG"' assert_common_features(nextflow_file_class, 7, 1, False) assert_common_features_local(nextflow_file_class) assert_compare_workflow_blocks(OUT_NF_WF2_LOCAL, EXPECTED_WF2) @@ -30,7 +30,7 @@ def test_convert_wf2_with_env_local(oton_converter): def test_convert_wf3_with_env_local(oton_converter): nextflow_file_class = oton_converter.convert_oton(IN_TXT_WF3, OUT_NF_WF3_LOCAL, "local", False) - assert 'params.input_file_group = "OCR-D-GT-SEG-BLOCK,OCR-D-OCR"' in nextflow_file_class.nf_lines_parameters + assert nextflow_file_class.nf_lines_parameters['params.input_file_group'] == '"OCR-D-GT-SEG-BLOCK,OCR-D-OCR"' assert_common_features(nextflow_file_class, 3, 1, False) assert_common_features_local(nextflow_file_class) assert_compare_workflow_blocks(OUT_NF_WF3_LOCAL, EXPECTED_WF3) @@ -38,7 +38,7 @@ def test_convert_wf3_with_env_local(oton_converter): def test_convert_wf4_with_env_local(oton_converter): nextflow_file_class = oton_converter.convert_oton(IN_TXT_WF4, OUT_NF_WF4_LOCAL, "local", False) - assert 'params.input_file_group = "OCR-D-IMG"' in nextflow_file_class.nf_lines_parameters + assert nextflow_file_class.nf_lines_parameters['params.input_file_group'] == '"OCR-D-IMG"' assert_common_features(nextflow_file_class, 13, 1, False) assert_common_features_local(nextflow_file_class) assert_compare_workflow_blocks(OUT_NF_WF4_LOCAL, EXPECTED_WF4)