Skip to content

Commit

Permalink
used dict for nf_params
Browse files Browse the repository at this point in the history
  • Loading branch information
@MehmedGIT
MehmedGIT committed Dec 2, 2024
1 parent 5177e12 commit 172474d
Show file tree
Hide file tree
Showing 5 changed files with 62 additions and 95 deletions.
13 changes: 0 additions & 13 deletions src/utils/operandi_utils/oton/constants.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,17 +32,4 @@
PARAMS_KEY_CPUS_PER_FORK: str = 'params.cpus_per_fork'
PARAMS_KEY_RAM_PER_FORK: str = 'params.ram_per_fork'

REPR_INPUT_FILE_GRP: str = f"""{PARAMS_KEY_INPUT_FILE_GRP} = "null\""""
REPR_METS_PATH: str = f"""{PARAMS_KEY_METS_PATH} = "null\""""
REPR_METS_SOCKET_PATH: str = f"""{PARAMS_KEY_METS_SOCKET_PATH} = "null\""""
REPR_WORKSPACE_DIR: str = f"""{PARAMS_KEY_WORKSPACE_DIR} = "null\""""
REPR_ENV_WRAPPER_CMD_CORE: str = f"""{PARAMS_KEY_ENV_WRAPPER_CMD_CORE} = "null\""""
REPR_PAGES: str = f"""{PARAMS_KEY_PAGES} = "null\""""
REPR_CPUS: str = f"""{PARAMS_KEY_CPUS} = "null\""""
REPR_RAM: str = f"""{PARAMS_KEY_RAM} = "null\""""
REPR_FORKS: str = f"""{PARAMS_KEY_FORKS} = {PARAMS_KEY_CPUS}"""
REPR_FORKS_NULL: str = f"""{PARAMS_KEY_FORKS} = "4\""""
REPR_CPUS_PER_FORK: str = f"""{PARAMS_KEY_CPUS_PER_FORK} = ({PARAMS_KEY_CPUS}.toInteger() / {PARAMS_KEY_FORKS}.toInteger()).intValue()"""
REPR_RAM_PER_FORK: str = f"""{PARAMS_KEY_RAM_PER_FORK} = sprintf("%dGB", ({PARAMS_KEY_RAM}.toInteger() / {PARAMS_KEY_FORKS}.toInteger()).intValue())"""

WORKFLOW_COMMENT = f"// This workflow was automatically generated by the v{OPERANDI_VERSION} operandi_utils.oton module"
79 changes: 30 additions & 49 deletions src/utils/operandi_utils/oton/nf_file_executable.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,20 +11,12 @@
PARAMS_KEY_ENV_WRAPPER_CMD_CORE,
PARAMS_KEY_ENV_WRAPPER_CMD_STEP,
PARAMS_KEY_FORKS,
PARAMS_KEY_PAGES,
PARAMS_KEY_CPUS,
PARAMS_KEY_CPUS_PER_FORK,
PARAMS_KEY_RAM,
PARAMS_KEY_RAM_PER_FORK,
REPR_ENV_WRAPPER_CMD_CORE,
REPR_INPUT_FILE_GRP,
REPR_METS_PATH,
REPR_METS_SOCKET_PATH,
REPR_WORKSPACE_DIR,
REPR_PAGES,
REPR_CPUS,
REPR_RAM,
REPR_FORKS,
REPR_FORKS_NULL,
REPR_CPUS_PER_FORK,
REPR_RAM_PER_FORK,
PARAMS_KEY_METS_SOCKET_PATH,
SPACES,
WORKFLOW_COMMENT
)
Expand All @@ -38,7 +30,7 @@ def __init__(self):
self.logger.setLevel(getLevelName(OTON_LOG_LEVEL))

self.supported_environments = ["local", "docker", "apptainer"]
self.nf_lines_parameters: List[str] = []
self.nf_lines_parameters = {}
self.nf_process_split_range = None
self.nf_process_merging_mets = None
self.nf_blocks_process: List[NextflowBlockProcess] = []
Expand All @@ -48,28 +40,28 @@ def build_parameters(self, environment: str, with_mets_server: bool):
if environment not in self.supported_environments:
raise ValueError(f"Invalid environment value: {environment}. Must be one of: {self.supported_environments}")

self.nf_lines_parameters.append('nextflow.enable.dsl = 2')
self.nf_lines_parameters.append('')
self.nf_lines_parameters.append(REPR_INPUT_FILE_GRP)
self.nf_lines_parameters.append(REPR_METS_PATH)
self.nf_lines_parameters.append(REPR_WORKSPACE_DIR)
self.nf_lines_parameters.append(REPR_PAGES)
self.nf_lines_parameters[PARAMS_KEY_INPUT_FILE_GRP] = '"null"'
self.nf_lines_parameters[PARAMS_KEY_METS_PATH] = '"null"'
self.nf_lines_parameters[PARAMS_KEY_WORKSPACE_DIR] = '"null"'
self.nf_lines_parameters[PARAMS_KEY_PAGES] = '"null"'

if with_mets_server:
self.nf_lines_parameters.append(REPR_METS_SOCKET_PATH)
self.nf_lines_parameters[PARAMS_KEY_METS_SOCKET_PATH] = '"null"'

if environment == "local":
self.nf_lines_parameters.append(REPR_FORKS_NULL)
self.nf_lines_parameters[PARAMS_KEY_FORKS] = '"4"'
if environment == "docker":
self.nf_lines_parameters.append(REPR_FORKS_NULL)
self.nf_lines_parameters.append(REPR_ENV_WRAPPER_CMD_CORE)
self.nf_lines_parameters[PARAMS_KEY_FORKS] = '"4"'
self.nf_lines_parameters[PARAMS_KEY_ENV_WRAPPER_CMD_CORE] = '"null"'
if environment == "apptainer":
self.nf_lines_parameters.append(REPR_CPUS)
self.nf_lines_parameters.append(REPR_RAM)
self.nf_lines_parameters.append(REPR_FORKS)
self.nf_lines_parameters.append(REPR_CPUS_PER_FORK)
self.nf_lines_parameters.append(REPR_RAM_PER_FORK)
self.nf_lines_parameters.append(REPR_ENV_WRAPPER_CMD_CORE)
self.nf_lines_parameters[PARAMS_KEY_CPUS] = '"null"'
self.nf_lines_parameters[PARAMS_KEY_RAM] = '"null"'
self.nf_lines_parameters[PARAMS_KEY_FORKS] = f'{PARAMS_KEY_CPUS}'
self.nf_lines_parameters[PARAMS_KEY_CPUS_PER_FORK] = \
f'({PARAMS_KEY_CPUS}.toInteger() / {PARAMS_KEY_FORKS}.toInteger()).intValue()'
self.nf_lines_parameters[PARAMS_KEY_RAM_PER_FORK] = \
f'sprintf("%dGB", ({PARAMS_KEY_RAM}.toInteger() / {PARAMS_KEY_FORKS}.toInteger()).intValue())'
self.nf_lines_parameters[PARAMS_KEY_ENV_WRAPPER_CMD_CORE] = '"null"'

# TODO: Refactor later
def build_split_page_ranges_process(self, environment: str, with_mets_server: bool) -> NextflowBlockProcess:
Expand Down Expand Up @@ -180,35 +172,22 @@ def build_nextflow_processes(
nf_process_block.add_parameter_output(parameter=CONST_METS_PATH, parameter_type='val')
nf_process_block.add_parameter_output(parameter=CONST_PAGE_RANGE, parameter_type='val')
nf_process_block.add_parameter_output(parameter=CONST_WORKSPACE_DIR, parameter_type='val')
self.nf_lines_parameters.append(f'{PARAMS_KEY_ENV_WRAPPER_CMD_STEP}{index} = "null"')
self.nf_lines_parameters[f'{PARAMS_KEY_ENV_WRAPPER_CMD_STEP}{index}'] = '"null"'
self.nf_blocks_process.append(nf_process_block)
index += 1

def __assign_first_file_grps_param(self):
first_file_grps = self.nf_blocks_process[0].processor_call_arguments.input_file_grps
index = 0
for parameter in self.nf_lines_parameters:
if PARAMS_KEY_INPUT_FILE_GRP in parameter:
self.nf_lines_parameters[index] = parameter.replace("null", first_file_grps)
break
index += 1

def build_log_info_prints(self) -> str:
log_info = f'log.info """\\\n'
log_info += f"{SPACES}OPERANDI HPC - Nextflow Workflow\n"
log_info += f"{SPACES}===================================================\n"
for param in self.nf_lines_parameters:
if not param or "params." not in param:
continue
param_key = param[param.find(".") + 1:param.find("=") - 1]
log_info += f"{SPACES}{param_key}: "
log_info += f'${BS[0]}{param[0:param.find("=") - 1]}{BS[1]}\n'
for key, value in self.nf_lines_parameters.items():
log_info += f"{SPACES}{key[len('params.'):]}: ${BS[0]}{key}{BS[1]}\n"
log_info += f'{SPACES}""".stripIndent()\n'
return log_info

# TODO: Refactor later
def build_main_workflow(self, with_mets_server: bool):
self.__assign_first_file_grps_param()
first_file_grps = self.nf_blocks_process[0].processor_call_arguments.input_file_grps
self.nf_lines_parameters[PARAMS_KEY_INPUT_FILE_GRP] = f'"{first_file_grps}"'
nf_workflow_block = NextflowBlockWorkflow(
workflow_name="main",
nf_processes=self.nf_blocks_process,
Expand All @@ -223,8 +202,10 @@ def produce_nextflow_file(self, output_path: str, environment: str, with_mets_se
# Write Nextflow line tokens to an output file
with open(output_path, mode='w', encoding='utf-8') as nextflow_file:
nextflow_file.write(f"{WORKFLOW_COMMENT}\n")
for nextflow_line in self.nf_lines_parameters:
nextflow_file.write(f'{nextflow_line}\n')
nextflow_file.write("nextflow.enable.dsl = 2\n")
nextflow_file.write("\n")
for key, value in self.nf_lines_parameters.items():
nextflow_file.write(f'{key} = {value}\n')
nextflow_file.write("\n")
nextflow_file.write(self.build_log_info_prints())
nextflow_file.write("\n")
Expand Down
53 changes: 26 additions & 27 deletions tests/assets/oton/constants.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,33 +105,32 @@
}
"""

PARAMETERS_COMMON = [
'nextflow.enable.dsl = 2',
'params.mets_path = "null"',
'params.workspace_dir = "null"',
'params.pages = "null"',
]
PARAMETERS_COMMON = {
'params.mets_path': '"null"',
'params.workspace_dir': '"null"',
'params.pages': '"null"',
}

PARAMETERS_LOCAL = [
'params.forks = "4"'
]
PARAMETERS_LOCAL = {
'params.forks': '"4"',
}

PARAMETERS_DOCKER = [
'params.forks = "4"',
'params.env_wrapper_cmd_core = "null"',
'params.env_wrapper_cmd_step0 = "null"',
'params.env_wrapper_cmd_step1 = "null"',
'params.env_wrapper_cmd_step2 = "null"',
]
PARAMETERS_DOCKER = {
'params.forks': '"4"',
'params.env_wrapper_cmd_core': '"null"',
'params.env_wrapper_cmd_step0': '"null"',
'params.env_wrapper_cmd_step1': '"null"',
'params.env_wrapper_cmd_step2': '"null"',
}

PARAMETERS_APPTAINER = [
'params.cpus = "null"',
'params.ram = "null"',
'params.forks = params.cpus',
'params.cpus_per_fork = (params.cpus.toInteger() / params.forks.toInteger()).intValue()',
'params.ram_per_fork = sprintf("%dGB", (params.ram.toInteger() / params.forks.toInteger()).intValue())',
'params.env_wrapper_cmd_core = "null"',
'params.env_wrapper_cmd_step0 = "null"',
'params.env_wrapper_cmd_step1 = "null"',
'params.env_wrapper_cmd_step2 = "null"',
]
PARAMETERS_APPTAINER = {
'params.cpus': '"null"',
'params.ram': '"null"',
'params.forks': 'params.cpus',
'params.cpus_per_fork': '(params.cpus.toInteger() / params.forks.toInteger()).intValue()',
'params.ram_per_fork': 'sprintf("%dGB", (params.ram.toInteger() / params.forks.toInteger()).intValue())',
'params.env_wrapper_cmd_core': '"null"',
'params.env_wrapper_cmd_step0': '"null"',
'params.env_wrapper_cmd_step1': '"null"',
'params.env_wrapper_cmd_step2': '"null"',
}
2 changes: 1 addition & 1 deletion tests/tests_utils/test_2_oton/assert_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ def assert_common_features(
for parameter in PARAMETERS_COMMON:
assert parameter in parameters
if with_mets_server:
assert 'params.mets_socket_path = "null"' in parameters, f"params.mets_socket_path is missing in {parameters}"
assert parameters['params.mets_socket_path'] == '"null"', f"params.mets_socket_path is missing in {parameters}"
blocks_process = nextflow_file_class.nf_blocks_process
assert len(blocks_process) == num_blocks_process
for block in blocks_process:
Expand Down
10 changes: 5 additions & 5 deletions tests/tests_utils/test_2_oton/test_3_converter_1_local.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,36 +9,36 @@

def test_convert_wf1_with_env_local(oton_converter):
nextflow_file_class = oton_converter.convert_oton(IN_TXT_WF1, OUT_NF_WF1_LOCAL, "local", False)
assert 'params.input_file_group = "OCR-D-IMG"' in nextflow_file_class.nf_lines_parameters
assert nextflow_file_class.nf_lines_parameters['params.input_file_group'] == '"OCR-D-IMG"'
assert_common_features(nextflow_file_class, 8, 1, False)
assert_compare_workflow_blocks(OUT_NF_WF1_LOCAL, EXPECTED_WF1)

def test_convert_wf1_with_env_local_with_mets_server(oton_converter):
nextflow_file_class = oton_converter.convert_oton(IN_TXT_WF1, OUT_NF_WF1_LOCAL_WITH_MS, "local", True)
assert 'params.input_file_group = "OCR-D-IMG"' in nextflow_file_class.nf_lines_parameters
assert nextflow_file_class.nf_lines_parameters['params.input_file_group'] == '"OCR-D-IMG"'
assert_common_features(nextflow_file_class, 8, 1, True)
assert_compare_workflow_blocks(OUT_NF_WF1_LOCAL_WITH_MS, EXPECTED_WF1_WITH_MS)


def test_convert_wf2_with_env_local(oton_converter):
nextflow_file_class = oton_converter.convert_oton(IN_TXT_WF2, OUT_NF_WF2_LOCAL, "local", False)
assert 'params.input_file_group = "OCR-D-IMG"' in nextflow_file_class.nf_lines_parameters
assert nextflow_file_class.nf_lines_parameters['params.input_file_group'] == '"OCR-D-IMG"'
assert_common_features(nextflow_file_class, 7, 1, False)
assert_common_features_local(nextflow_file_class)
assert_compare_workflow_blocks(OUT_NF_WF2_LOCAL, EXPECTED_WF2)


def test_convert_wf3_with_env_local(oton_converter):
nextflow_file_class = oton_converter.convert_oton(IN_TXT_WF3, OUT_NF_WF3_LOCAL, "local", False)
assert 'params.input_file_group = "OCR-D-GT-SEG-BLOCK,OCR-D-OCR"' in nextflow_file_class.nf_lines_parameters
assert nextflow_file_class.nf_lines_parameters['params.input_file_group'] == '"OCR-D-GT-SEG-BLOCK,OCR-D-OCR"'
assert_common_features(nextflow_file_class, 3, 1, False)
assert_common_features_local(nextflow_file_class)
assert_compare_workflow_blocks(OUT_NF_WF3_LOCAL, EXPECTED_WF3)


def test_convert_wf4_with_env_local(oton_converter):
nextflow_file_class = oton_converter.convert_oton(IN_TXT_WF4, OUT_NF_WF4_LOCAL, "local", False)
assert 'params.input_file_group = "OCR-D-IMG"' in nextflow_file_class.nf_lines_parameters
assert nextflow_file_class.nf_lines_parameters['params.input_file_group'] == '"OCR-D-IMG"'
assert_common_features(nextflow_file_class, 13, 1, False)
assert_common_features_local(nextflow_file_class)
assert_compare_workflow_blocks(OUT_NF_WF4_LOCAL, EXPECTED_WF4)

0 comments on commit 172474d

Please sign in to comment.