From 87420071bcd1b4f5a496ef8d001bc96529b5862c Mon Sep 17 00:00:00 2001 From: David Sutherland Date: Sat, 30 Mar 2024 00:24:22 +1300 Subject: [PATCH 1/2] fix workflow-state on alternate run-dir --- cylc/flow/id_cli.py | 16 +++++++++++++--- cylc/flow/pathutil.py | 14 ++++++++++---- cylc/flow/scripts/workflow_state.py | 8 +++----- cylc/flow/workflow_files.py | 17 +++++++++++++---- cylc/flow/xtriggers/workflow_state.py | 12 ++++++------ 5 files changed, 45 insertions(+), 22 deletions(-) diff --git a/cylc/flow/id_cli.py b/cylc/flow/id_cli.py index b6d2a9c4150..862786fc1a5 100644 --- a/cylc/flow/id_cli.py +++ b/cylc/flow/id_cli.py @@ -198,6 +198,7 @@ async def parse_ids_async( constraint: str = 'tasks', max_workflows: Optional[int] = None, max_tasks: Optional[int] = None, + cylc_run_dir: Optional[str] = None, ) -> Tuple[Dict[str, List[Tokens]], Any]: """Parse IDs from the command line. @@ -232,6 +233,8 @@ async def parse_ids_async( max_tasks: Specify the maximum number of tasks permitted to be specified in the ids. + cylc_run_dir: + Infer from a non-stardard run directory. Returns: With src=True": @@ -294,7 +297,11 @@ async def parse_ids_async( # infer the run number if not specified the ID (and if possible) if infer_latest_runs: - _infer_latest_runs(tokens_list, src_path=src_path) + _infer_latest_runs( + tokens_list, + src_path=src_path, + cylc_run_dir=cylc_run_dir + ) _validate_number( *tokens_list, @@ -409,13 +416,16 @@ def _validate_workflow_ids(*tokens_list, src_path): detect_both_flow_and_suite(src_path) -def _infer_latest_runs(tokens_list, src_path): +def _infer_latest_runs(tokens_list, src_path, cylc_run_dir): for ind, tokens in enumerate(tokens_list): if ind == 0 and src_path: # source workflow passed in as a path continue tokens_list[ind] = tokens.duplicate( - workflow=infer_latest_run_from_id(tokens['workflow']) + workflow=infer_latest_run_from_id( + tokens['workflow'], + cylc_run_dir=cylc_run_dir, + ) ) pass diff --git a/cylc/flow/pathutil.py b/cylc/flow/pathutil.py index 0760383ceb4..65ef023305e 100644 --- a/cylc/flow/pathutil.py +++ b/cylc/flow/pathutil.py @@ -71,20 +71,26 @@ def get_remote_workflow_run_job_dir( return get_remote_workflow_run_dir(workflow_id, 'log', 'job', *args) -def get_cylc_run_dir() -> str: +def get_cylc_run_dir(cylc_run_dir: Optional[str] = None) -> str: """Return the cylc-run dir path with vars/user expanded.""" - return expand_path(_CYLC_RUN_DIR) + if cylc_run_dir is None: + return expand_path(_CYLC_RUN_DIR) + return expand_path(cylc_run_dir) def get_workflow_run_dir( - workflow_id: Union[Path, str], *args: Union[Path, str] + workflow_id: Union[Path, str], + *args: Union[Path, str], + cylc_run_dir: Optional[str] = None, ) -> str: """Return local workflow run directory, joining any extra args, and expanding vars and user. Does not check that the directory exists. """ - return expand_path(_CYLC_RUN_DIR, workflow_id, *args) + if cylc_run_dir is None: + return expand_path(_CYLC_RUN_DIR, workflow_id, *args) + return expand_path(cylc_run_dir, workflow_id, *args) def get_workflow_run_job_dir(workflow, *args): diff --git a/cylc/flow/scripts/workflow_state.py b/cylc/flow/scripts/workflow_state.py index 170b63062d1..904e972219e 100755 --- a/cylc/flow/scripts/workflow_state.py +++ b/cylc/flow/scripts/workflow_state.py @@ -68,7 +68,7 @@ from cylc.flow.task_state import TASK_STATUSES_ORDERED from cylc.flow.terminal import cli_function from cylc.flow.cycling.util import add_offset -from cylc.flow.pathutil import expand_path, get_cylc_run_dir +from cylc.flow.pathutil import get_cylc_run_dir from metomi.isodatetime.parsers import TimePointParser @@ -199,6 +199,7 @@ def main(parser: COP, options: 'Values', workflow_id: str) -> None: workflow_id, *_ = parse_id( workflow_id, constraint='workflows', + cylc_run_dir=options.run_dir, ) if options.use_task_point and options.cycle: @@ -232,10 +233,7 @@ def main(parser: COP, options: 'Values', workflow_id: str) -> None: raise InputError(f"invalid status '{options.status}'") # this only runs locally - if options.run_dir: - run_dir = expand_path(options.run_dir) - else: - run_dir = get_cylc_run_dir() + run_dir = get_cylc_run_dir(cylc_run_dir=options.run_dir) pollargs = { 'workflow_id': workflow_id, diff --git a/cylc/flow/workflow_files.py b/cylc/flow/workflow_files.py index 7d4a67756ad..84501ed0dac 100644 --- a/cylc/flow/workflow_files.py +++ b/cylc/flow/workflow_files.py @@ -862,9 +862,17 @@ def check_reserved_dir_names(name: Union[Path, str]) -> None: raise WorkflowFilesError(err_msg.format('run')) -def infer_latest_run_from_id(workflow_id: str) -> str: - run_dir = Path(get_workflow_run_dir(workflow_id)) - _, id_ = infer_latest_run(run_dir) +def infer_latest_run_from_id( + workflow_id: str, + cylc_run_dir: Optional[str] = None, +) -> str: + run_dir = Path( + get_workflow_run_dir( + workflow_id, + cylc_run_dir=cylc_run_dir, + ) + ) + _, id_ = infer_latest_run(run_dir, cylc_run_dir=cylc_run_dir) return id_ @@ -872,6 +880,7 @@ def infer_latest_run( path: Path, implicit_runN: bool = True, warn_runN: bool = True, + cylc_run_dir: Optional[str] = None, ) -> Tuple[Path, str]: """Infer the numbered run dir if the workflow has a runN symlink. @@ -890,7 +899,7 @@ def infer_latest_run( - WorkflowFilesError if the runN symlink is not valid. - InputError if the path does not exist. """ - cylc_run_dir = get_cylc_run_dir() + cylc_run_dir = get_cylc_run_dir(cylc_run_dir=cylc_run_dir) try: id_ = str(path.relative_to(cylc_run_dir)) except ValueError: diff --git a/cylc/flow/xtriggers/workflow_state.py b/cylc/flow/xtriggers/workflow_state.py index f20cd214067..13463426270 100644 --- a/cylc/flow/xtriggers/workflow_state.py +++ b/cylc/flow/xtriggers/workflow_state.py @@ -22,7 +22,7 @@ from cylc.flow.cycling.util import add_offset from cylc.flow.dbstatecheck import CylcWorkflowDBChecker -from cylc.flow.pathutil import expand_path, get_cylc_run_dir +from cylc.flow.pathutil import get_cylc_run_dir from cylc.flow.workflow_files import infer_latest_run @@ -78,13 +78,13 @@ def workflow_state( to this xtrigger. """ - if cylc_run_dir: - cylc_run_dir = expand_path(cylc_run_dir) - else: - cylc_run_dir = get_cylc_run_dir() + cylc_run_dir = get_cylc_run_dir(cylc_run_dir=cylc_run_dir) if offset is not None: point = str(add_offset(point, offset)) - _, workflow = infer_latest_run(Path(cylc_run_dir, workflow)) + _, workflow = infer_latest_run( + Path(cylc_run_dir, workflow), + cylc_run_dir=cylc_run_dir, + ) try: checker = CylcWorkflowDBChecker(cylc_run_dir, workflow) except (OSError, sqlite3.Error): From db0b0692674bf102854d4777cc2cbf4d3ef9a572 Mon Sep 17 00:00:00 2001 From: David Sutherland Date: Sat, 30 Mar 2024 00:59:58 +1300 Subject: [PATCH 2/2] test added --- .../workflow-state/08-other-run-dir.t | 31 +++++++++++++++ .../workflow-state/other-run-dir/flow.cylc | 39 +++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100755 tests/functional/workflow-state/08-other-run-dir.t create mode 100644 tests/functional/workflow-state/other-run-dir/flow.cylc diff --git a/tests/functional/workflow-state/08-other-run-dir.t b/tests/functional/workflow-state/08-other-run-dir.t new file mode 100755 index 00000000000..9dd91666e64 --- /dev/null +++ b/tests/functional/workflow-state/08-other-run-dir.t @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#------------------------------------------------------------------------------- +# Test cylc workflow-state "template" option +. "$(dirname "$0")/test_header" +#------------------------------------------------------------------------------- +set_test_number 1 +#------------------------------------------------------------------------------- +install_workflow "${TEST_NAME_BASE}" other-run-dir +#------------------------------------------------------------------------------- +TEST_NAME="${TEST_NAME_BASE}-run" +workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" +#------------------------------------------------------------------------------- +rm -rfv ~/cylc-run-other-test +purge +#------------------------------------------------------------------------------- +exit 0 diff --git a/tests/functional/workflow-state/other-run-dir/flow.cylc b/tests/functional/workflow-state/other-run-dir/flow.cylc new file mode 100644 index 00000000000..9126b01743b --- /dev/null +++ b/tests/functional/workflow-state/other-run-dir/flow.cylc @@ -0,0 +1,39 @@ +[meta] + title = "Checks task state of alternate run dir" +[scheduler] + UTC mode = True + [[events]] + inactivity timeout = PT1M + abort on inactivity timeout = True +[scheduling] + initial cycle point = 20240101T00 + [[xtriggers]] + up_dir_foo = workflow_state(\ + workflow=other-run-dir, \ + task=foo, \ + point=20240101T0000Z, \ + cylc_run_dir=~/cylc-run-other-test \ + ):PT10S + [[graph]] + R1 = """ +foo +@up_dir_foo => bar +""" +[runtime] + [[foo]] + script = """ +mkdir -pv ~/cylc-run-other-test +cd ~/cylc-run-other-test +echo 'Current directory:' +pwd +echo 'Creating link to original run dir:' +ln -sfv ${CYLC_RUN_DIR}/${CYLC_WORKFLOW_NAME} other-run-dir +""" + [[bar]] + script = """ +cylc workflow-state --debug other-run-dir \ + --run-dir=~/cylc-run-other-test \ + --task=foo \ + --point=20240101T0000Z +rm -rfv ~/cylc-run-other-test +"""