From 8f975c40d835db3a313ca1f898cf27dc13f9ac7e Mon Sep 17 00:00:00 2001 From: Mark Dawson Date: Tue, 19 Dec 2023 10:57:16 +0000 Subject: [PATCH 1/3] Add chained offset logic for FCP (#5885) --- changes.d/5885.fix.md | 1 + cylc/flow/cycling/iso8601.py | 15 ++++++++++++--- tests/unit/test_config.py | 17 ++++++++++++----- 3 files changed, 25 insertions(+), 8 deletions(-) create mode 100644 changes.d/5885.fix.md diff --git a/changes.d/5885.fix.md b/changes.d/5885.fix.md new file mode 100644 index 00000000000..b9071bae612 --- /dev/null +++ b/changes.d/5885.fix.md @@ -0,0 +1 @@ +Fixed bug in using a final cycle point with chained offsets e.g. 'final cycle point = +PT6H+PT1S'. \ No newline at end of file diff --git a/cylc/flow/cycling/iso8601.py b/cylc/flow/cycling/iso8601.py index ed5b4f21979..3c8bd3955fa 100644 --- a/cylc/flow/cycling/iso8601.py +++ b/cylc/flow/cycling/iso8601.py @@ -895,14 +895,23 @@ def get_dump_format(): def get_point_relative(offset_string, base_point): """Create a point from offset_string applied to base_point.""" try: - interval = ISO8601Interval(str(interval_parse(offset_string))) + operator = '+' + base_point_relative = base_point + for part in re.split(r'(\+|-)', offset_string): + if part == '+' or part == '-': + operator = part + elif part != '': + interval = interval_parse(part) + if operator == '-': + interval *= -1 + base_point_relative += ISO8601Interval(str(interval)) + return base_point_relative except IsodatetimeError: + # It's a truncated time point rather than an interval return ISO8601Point(str( WorkflowSpecifics.abbrev_util.parse_timepoint( offset_string, context_point=point_parse(base_point.value)) )) - else: - return base_point + interval def interval_parse(interval_string): diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 736bd34ed9d..b3936f7c328 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -532,6 +532,17 @@ def test_process_startcp( None, id="Relative fcp" ), + pytest.param( + ISO8601_CYCLING_TYPE, + { + 'initial cycle point': '2017-02-11', + 'final cycle point': '+P4D+PT3H-PT2H', + }, + None, + '20170215T0100+0530', + None, + id="Relative fcp chained" + ), pytest.param( ISO8601_CYCLING_TYPE, { @@ -1429,11 +1440,7 @@ def test_zero_interval( ('1988-02-29', '+P1M+P1Y', '1989-03-29'), ('1910-08-14', '+P2D-PT6H', '1910-08-15T18:00'), ('1850-04-10', '+P1M-P1D+PT1H', '1850-05-09T01:00'), - pytest.param( - '1066-10-14', '+PT1H+PT1M', '1066-10-14T01:01', - marks=pytest.mark.xfail - # https://github.com/cylc/cylc-flow/issues/5047 - ), + ('1066-10-14', '+PT1H+PT1M', '1066-10-14T01:01'), ] ) def test_chain_expr( From 3320c050a350a3f0c2622405a2739d28ff827eb9 Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Tue, 19 Dec 2023 12:28:47 +0000 Subject: [PATCH 2/3] dump: restrict window to n=0 (#5600) This restores the old `cylc dump` behaviour of displaying only the pool contents. --- changes.d/5600.break.md | 3 ++ cylc/flow/scripts/dump.py | 40 +++++++++----- tests/functional/runahead/06-release-update.t | 1 - tests/integration/scripts/test_dump.py | 52 +++++++++++++++++++ 4 files changed, 81 insertions(+), 15 deletions(-) create mode 100644 changes.d/5600.break.md create mode 100644 tests/integration/scripts/test_dump.py diff --git a/changes.d/5600.break.md b/changes.d/5600.break.md new file mode 100644 index 00000000000..fd11f650ff5 --- /dev/null +++ b/changes.d/5600.break.md @@ -0,0 +1,3 @@ +The `cylc dump` command now only shows active tasks (e.g. running & queued +tasks). This restores its behaviour of only showing the tasks which currently +exist in the pool as it did in Cylc 7 and earlier versions of Cylc 8. diff --git a/cylc/flow/scripts/dump.py b/cylc/flow/scripts/dump.py index c62bf6719aa..d3ffda59fce 100755 --- a/cylc/flow/scripts/dump.py +++ b/cylc/flow/scripts/dump.py @@ -20,6 +20,9 @@ Print information about a running workflow. +This command can provide information about active tasks, e.g. running or queued +tasks. For more detailed view of the workflow see `cylc tui` or `cylc gui`. + For command line monitoring: * `cylc tui` * `watch cylc dump WORKFLOW_ID` works for small simple workflows @@ -28,20 +31,21 @@ its prerequisites and outputs, see 'cylc show'. Examples: - # Display the state of all running tasks, sorted by cycle point: + # Display the state of all active tasks, sorted by cycle point: $ cylc dump --tasks --sort WORKFLOW_ID | grep running - # Display the state of all tasks in a particular cycle point: + # Display the state of all active in a particular cycle point: $ cylc dump -t WORKFLOW_ID | grep 2010082406 """ -from graphene.utils.str_converters import to_snake_case +import asyncio import json -import sys from typing import TYPE_CHECKING +from graphene.utils.str_converters import to_snake_case + from cylc.flow.exceptions import CylcError -from cylc.flow.id_cli import parse_id +from cylc.flow.id_cli import parse_id_async from cylc.flow.option_parsers import ( WORKFLOW_ID_ARG_DOC, CylcOptionParser as COP, @@ -59,6 +63,7 @@ name cyclePoint state + graphDepth isHeld isQueued isRunahead @@ -179,7 +184,11 @@ def get_option_parser(): @cli_function(get_option_parser) def main(_, options: 'Values', workflow_id: str) -> None: - workflow_id, *_ = parse_id( + asyncio.run(dump(workflow_id, options)) + + +async def dump(workflow_id, options, write=print): + workflow_id, *_ = await parse_id_async( workflow_id, constraint='workflows', ) @@ -195,6 +204,9 @@ def main(_, options: 'Values', workflow_id: str) -> None: else: sort_args = {'keys': ['name', 'cyclePoint']} + # retrict to the n=0 window + graph_depth = 0 + if options.disp_form == "raw": query = f''' {TASK_SUMMARY_FRAGMENT} @@ -203,10 +215,10 @@ def main(_, options: 'Values', workflow_id: str) -> None: query ($wFlows: [ID]!, $sortBy: SortArgs) {{ workflows (ids: $wFlows, stripNull: false) {{ ...wFlow - taskProxies (sort: $sortBy) {{ + taskProxies (sort: $sortBy, graphDepth: {graph_depth}) {{ ...tProxy }} - familyProxies (sort: $sortBy) {{ + familyProxies (sort: $sortBy, graphDepth: {graph_depth}) {{ ...fProxy }} }} @@ -224,7 +236,7 @@ def main(_, options: 'Values', workflow_id: str) -> None: {TASK_SUMMARY_FRAGMENT} query ($wFlows: [ID]!, $sortBy: SortArgs) {{ workflows (ids: $wFlows, stripNull: false) {{ - taskProxies (sort: $sortBy) {{ + taskProxies (sort: $sortBy, graphDepth: {graph_depth}) {{ ...tProxy }} }} @@ -235,15 +247,15 @@ def main(_, options: 'Values', workflow_id: str) -> None: 'variables': {'wFlows': [workflow_id], 'sortBy': sort_args} } - workflows = pclient('graphql', query_kwargs) + workflows = await pclient.async_request('graphql', query_kwargs) try: for summary in workflows['workflows']: if options.disp_form == "raw": if options.pretty: - sys.stdout.write(json.dumps(summary, indent=4) + '\n') + write(json.dumps(summary, indent=4)) else: - print(summary) + write(summary) else: if options.disp_form != "tasks": node_urls = { @@ -261,7 +273,7 @@ def main(_, options: 'Values', workflow_id: str) -> None: del summary['families'] del summary['meta'] for key, value in sorted(summary.items()): - print( + write( f'{to_snake_case(key).replace("_", " ")}={value}') else: for item in summary['taskProxies']: @@ -282,7 +294,7 @@ def main(_, options: 'Values', workflow_id: str) -> None: else 'not-runahead') if options.show_flows: values.append(item['flowNums']) - print(', '.join(values)) + write(', '.join(values)) except Exception as exc: raise CylcError( json.dumps(workflows, indent=4) + '\n' + str(exc) + '\n') diff --git a/tests/functional/runahead/06-release-update.t b/tests/functional/runahead/06-release-update.t index d52b12272a6..c4ab28530e3 100644 --- a/tests/functional/runahead/06-release-update.t +++ b/tests/functional/runahead/06-release-update.t @@ -35,7 +35,6 @@ sleep 10 # (gratuitous use of --flows for test coverage) cylc dump --flows -t "${WORKFLOW_NAME}" | awk '{print $1 $2 $3 $7}' >'log' cmp_ok 'log' - <<__END__ -bar,$NEXT1,waiting,[1] foo,$NEXT1,waiting,[1] __END__ diff --git a/tests/integration/scripts/test_dump.py b/tests/integration/scripts/test_dump.py new file mode 100644 index 00000000000..681167a564f --- /dev/null +++ b/tests/integration/scripts/test_dump.py @@ -0,0 +1,52 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Test the "cylc dump" command.""" + +from cylc.flow.option_parsers import ( + Options, +) +from cylc.flow.scripts.dump import ( + dump, + get_option_parser, +) + + +DumpOptions = Options(get_option_parser()) + + +async def test_dump_tasks(flow, scheduler, start): + """It should show n=0 tasks. + + See: https://github.com/cylc/cylc-flow/pull/5600 + """ + id_ = flow({ + 'scheduler': { + 'allow implicit tasks': 'true', + }, + 'scheduling': { + 'graph': { + 'R1': 'a => b => c', + }, + }, + }) + schd = scheduler(id_) + async with start(schd): + # schd.release_queued_tasks() + await schd.update_data_structure() + ret = [] + await dump(id_, DumpOptions(disp_form='tasks'), write=ret.append) + assert ret == ['a, 1, waiting, not-held, queued, not-runahead'] From 031fc0560e930fcd63d5e6a543396d668468e7f1 Mon Sep 17 00:00:00 2001 From: Tim Pillinger <26465611+wxtim@users.noreply.github.com> Date: Wed, 20 Dec 2023 10:03:13 +0000 Subject: [PATCH 3/3] Feat.lint obsolete vars (#5879) * lint.warn about deprecated templating * Update changes.d/5879.feat.md Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com> * Update cylc/flow/scripts/lint.py Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com> * Clarify templated vars which have to be set in metadata sections. Fix an rst bug. * fix broken code --------- Co-authored-by: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com> --- changes.d/5879.feat.md | 1 + cylc/flow/scripts/lint.py | 85 ++++++++++++++++++++++++++++++--- tests/unit/scripts/test_lint.py | 2 +- 3 files changed, 81 insertions(+), 7 deletions(-) create mode 100644 changes.d/5879.feat.md diff --git a/changes.d/5879.feat.md b/changes.d/5879.feat.md new file mode 100644 index 00000000000..be4c7e14e94 --- /dev/null +++ b/changes.d/5879.feat.md @@ -0,0 +1 @@ +`cylc lint` now warns of use of old templated items such as `%(suite)s` diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py index cc06512988c..ef7635f1f0d 100755 --- a/cylc/flow/scripts/lint.py +++ b/cylc/flow/scripts/lint.py @@ -67,7 +67,7 @@ TOMLDecodeError, ) from typing import ( - TYPE_CHECKING, Any, Callable, Dict, Iterator, List, Union) + TYPE_CHECKING, Any, Callable, Dict, Iterator, List, Optional, Union) from cylc.flow import LOG from cylc.flow.exceptions import CylcError @@ -106,6 +106,31 @@ } +DEPRECATED_STRING_TEMPLATES = { + 'suite': 'workflow', + 'suite_uuid': 'uuid', + 'batch_sys_name': 'job_runner_name', + 'batch_sys_job_id': 'job_id', + 'user@host': 'platform_name', + 'task_url': '``URL`` (if set in :cylc:conf:`[meta]URL`)', + 'workflow_url': ( + '``workflow_URL`` (if set in ' + ':cylc:conf:`[runtime][][meta]URL`)'), +} + + +LIST_ITEM = ' * ' + + +deprecated_string_templates = { + key: ( + re.compile(r'%\(' + key + r'\)s'), + value + ) + for key, value in DEPRECATED_STRING_TEMPLATES.items() +} + + def check_jinja2_no_shebang( line: str, file: Path, @@ -233,6 +258,36 @@ def check_for_obsolete_environment_variables(line: str) -> List[str]: return [i for i in OBSOLETE_ENV_VARS if i in line] +def check_for_deprecated_task_event_template_vars( + line: str +) -> Optional[Dict[str, str]]: + """Look for string variables which are no longer supported + + Examples: + >>> this = check_for_deprecated_task_event_template_vars + + >>> this('hello = "My name is %(suite)s"') + {'list': ' * %(suite)s ⇒ %(workflow)s'} + + >>> expect = {'list': ( + ... ' * %(suite)s ⇒ %(workflow)s * %(task_url)s' + ... ' - get ``URL`` (if set in :cylc:conf:`[meta]URL`)')} + >>> this('hello = "My name is %(suite)s, %(task_url)s"') == expect + True + """ + result = [] + for key, (regex, replacement) in deprecated_string_templates.items(): + search_outcome = regex.findall(line) + if search_outcome and ' ' in replacement: + result.append(f'%({key})s - get {replacement}') + elif search_outcome: + result.append(f'%({key})s ⇒ %({replacement})s') + + if result: + return {'list': LIST_ITEM + LIST_ITEM.join(result)} + return None + + INDENTATION = re.compile(r'^(\s*)(.*)') @@ -584,6 +639,23 @@ def check_indentation(line: str) -> bool: ), FUNCTION: re.compile(r'rose +date').findall, }, + 'U015': { + 'short': ( + 'Deprecated template variables.'), + 'rst': ( + 'The following template variables, mostly used in event handlers,' + 'are deprecated, and should be replaced:' + + ''.join([ + f'\n * ``{old}`` ⇒ {new}' + for old, new in DEPRECATED_STRING_TEMPLATES.items() + ]) + ), + 'url': ( + 'https://cylc.github.io/cylc-doc/stable/html/user-guide/' + 'writing-workflows/runtime.html#task-event-template-variables' + ), + FUNCTION: check_for_deprecated_task_event_template_vars, + } } RULESETS = ['728', 'style', 'all'] EXTRA_TOML_VALIDATION = { @@ -1077,7 +1149,7 @@ def get_cylc_files( 'section heading': '\n{title}\n{underline}\n', 'issue heading': { 'text': '\n{check}:\n {summary}\n {url}\n\n', - 'rst': '\n`{check} <{url}>`_\n{underline}\n{summary}\n\n', + 'rst': '\n{url}_\n{underline}\n{summary}\n\n', }, 'auto gen message': ( 'U998 and U999 represent automatically generated' @@ -1122,16 +1194,17 @@ def get_reference(linter, output_type): output += '\n' output += '\n* ' + summary else: + check = get_index_str(meta, index) template = issue_heading_template url = get_url(meta) + if output_type == 'rst': + url = f'`{check} <{url}>`' if url else f'{check}' msg = template.format( title=index, - check=get_index_str(meta, index), + check=check, summary=summary, url=url, - underline=( - len(get_index_str(meta, index)) + len(url) + 6 - ) * '^' + underline=(len(url) + 1) * '^' ) output += msg output += '\n' diff --git a/tests/unit/scripts/test_lint.py b/tests/unit/scripts/test_lint.py index 6ee1018cf96..349854be312 100644 --- a/tests/unit/scripts/test_lint.py +++ b/tests/unit/scripts/test_lint.py @@ -119,7 +119,7 @@ submission failed handler = giaSEHFUIHJ failed handler = woo execution timeout handler = sdfghjkl - expired handler = dafuhj + expired handler = %(suite_uuid)s %(user@host)s late handler = dafuhj submitted handler = dafuhj started handler = dafuhj