Merge branch 'master' into feat.lint.no_cylc_CYLC_VERSION=template

changes.d/5600.break.md

@@ -0,0 +1,3 @@
+The `cylc dump` command now only shows active tasks (e.g. running & queued
+tasks). This restores its behaviour of only showing the tasks which currently
+exist in the pool as it did in Cylc 7 and earlier versions of Cylc 8.

changes.d/5879.feat.md

@@ -0,0 +1 @@
+`cylc lint` now warns of use of old templated items such as `%(suite)s`

changes.d/5885.fix.md

@@ -0,0 +1 @@
+Fixed bug in using a final cycle point with chained offsets e.g. 'final cycle point = +PT6H+PT1S'.

cylc/flow/cycling/iso8601.py

@@ -895,14 +895,23 @@ def get_dump_format():
 def get_point_relative(offset_string, base_point):
     """Create a point from offset_string applied to base_point."""
     try:
-        interval = ISO8601Interval(str(interval_parse(offset_string)))
+        operator = '+'
+        base_point_relative = base_point
+        for part in re.split(r'(\+|-)', offset_string):
+            if part == '+' or part == '-':
+                operator = part
+            elif part != '':
+                interval = interval_parse(part)
+                if operator == '-':
+                    interval *= -1
+                base_point_relative += ISO8601Interval(str(interval))
+        return base_point_relative
     except IsodatetimeError:
+        # It's a truncated time point rather than an interval
         return ISO8601Point(str(
             WorkflowSpecifics.abbrev_util.parse_timepoint(
                 offset_string, context_point=point_parse(base_point.value))
         ))
-    else:
-        return base_point + interval
 
 
 def interval_parse(interval_string):

cylc/flow/scripts/dump.py

@@ -20,6 +20,9 @@
 
 Print information about a running workflow.
 
+This command can provide information about active tasks, e.g. running or queued
+tasks. For more detailed view of the workflow see `cylc tui` or `cylc gui`.
+
 For command line monitoring:
 * `cylc tui`
 * `watch cylc dump WORKFLOW_ID` works for small simple workflows
@@ -28,20 +31,21 @@
 its prerequisites and outputs, see 'cylc show'.
 
 Examples:
-  # Display the state of all running tasks, sorted by cycle point:
+  # Display the state of all active tasks, sorted by cycle point:
   $ cylc dump --tasks --sort WORKFLOW_ID | grep running
 
-  # Display the state of all tasks in a particular cycle point:
+  # Display the state of all active in a particular cycle point:
   $ cylc dump -t WORKFLOW_ID | grep 2010082406
 """
 
-from graphene.utils.str_converters import to_snake_case
+import asyncio
 import json
-import sys
 from typing import TYPE_CHECKING
 
+from graphene.utils.str_converters import to_snake_case
+
 from cylc.flow.exceptions import CylcError
-from cylc.flow.id_cli import parse_id
+from cylc.flow.id_cli import parse_id_async
 from cylc.flow.option_parsers import (
     WORKFLOW_ID_ARG_DOC,
     CylcOptionParser as COP,
@@ -59,6 +63,7 @@
   name
   cyclePoint
   state
+  graphDepth
   isHeld
   isQueued
   isRunahead
@@ -179,7 +184,11 @@ def get_option_parser():
 
 @cli_function(get_option_parser)
 def main(_, options: 'Values', workflow_id: str) -> None:
-    workflow_id, *_ = parse_id(
+    asyncio.run(dump(workflow_id, options))
+
+
+async def dump(workflow_id, options, write=print):
+    workflow_id, *_ = await parse_id_async(
         workflow_id,
         constraint='workflows',
     )
@@ -195,6 +204,9 @@ def main(_, options: 'Values', workflow_id: str) -> None:
     else:
         sort_args = {'keys': ['name', 'cyclePoint']}
 
+    # retrict to the n=0 window
+    graph_depth = 0
+
     if options.disp_form == "raw":
         query = f'''
             {TASK_SUMMARY_FRAGMENT}
@@ -203,10 +215,10 @@ def main(_, options: 'Values', workflow_id: str) -> None:
             query ($wFlows: [ID]!, $sortBy: SortArgs) {{
               workflows (ids: $wFlows, stripNull: false) {{
                 ...wFlow
-                taskProxies (sort: $sortBy) {{
+                taskProxies (sort: $sortBy, graphDepth: {graph_depth}) {{
                   ...tProxy
                 }}
-                familyProxies (sort: $sortBy) {{
+                familyProxies (sort: $sortBy, graphDepth: {graph_depth}) {{
                   ...fProxy
                 }}
               }}
@@ -224,7 +236,7 @@ def main(_, options: 'Values', workflow_id: str) -> None:
             {TASK_SUMMARY_FRAGMENT}
             query ($wFlows: [ID]!, $sortBy: SortArgs) {{
               workflows (ids: $wFlows, stripNull: false) {{
-                taskProxies (sort: $sortBy) {{
+                taskProxies (sort: $sortBy, graphDepth: {graph_depth}) {{
                   ...tProxy
                 }}
               }}
@@ -235,15 +247,15 @@ def main(_, options: 'Values', workflow_id: str) -> None:
         'variables': {'wFlows': [workflow_id], 'sortBy': sort_args}
     }
 
-    workflows = pclient('graphql', query_kwargs)
+    workflows = await pclient.async_request('graphql', query_kwargs)
 
     try:
         for summary in workflows['workflows']:
             if options.disp_form == "raw":
                 if options.pretty:
-                    sys.stdout.write(json.dumps(summary, indent=4) + '\n')
+                    write(json.dumps(summary, indent=4))
                 else:
-                    print(summary)
+                    write(summary)
             else:
                 if options.disp_form != "tasks":
                     node_urls = {
@@ -261,7 +273,7 @@ def main(_, options: 'Values', workflow_id: str) -> None:
                     del summary['families']
                     del summary['meta']
                     for key, value in sorted(summary.items()):
-                        print(
+                        write(
                             f'{to_snake_case(key).replace("_", " ")}={value}')
                 else:
                     for item in summary['taskProxies']:
@@ -282,7 +294,7 @@ def main(_, options: 'Values', workflow_id: str) -> None:
                                       else 'not-runahead')
                         if options.show_flows:
                             values.append(item['flowNums'])
-                        print(', '.join(values))
+                        write(', '.join(values))
     except Exception as exc:
         raise CylcError(
             json.dumps(workflows, indent=4) + '\n' + str(exc) + '\n')

cylc/flow/scripts/lint.py

@@ -106,6 +106,31 @@
 }
 
 
+DEPRECATED_STRING_TEMPLATES = {
+    'suite': 'workflow',
+    'suite_uuid': 'uuid',
+    'batch_sys_name': 'job_runner_name',
+    'batch_sys_job_id': 'job_id',
+    'user@host': 'platform_name',
+    'task_url': '``URL`` (if set in :cylc:conf:`[meta]URL`)',
+    'workflow_url': (
+        '``workflow_URL`` (if set in '
+        ':cylc:conf:`[runtime][<namespace>][meta]URL`)'),
+}
+
+
+LIST_ITEM = '    * '
+
+
+deprecated_string_templates = {
+    key: (
+        re.compile(r'%\(' + key + r'\)s'),
+        value
+    )
+    for key, value in DEPRECATED_STRING_TEMPLATES.items()
+}
+
+
 def check_jinja2_no_shebang(
     line: str,
     file: Path,
@@ -233,6 +258,36 @@ def check_for_obsolete_environment_variables(line: str) -> List[str]:
     return [i for i in OBSOLETE_ENV_VARS if i in line]
 
 
+def check_for_deprecated_task_event_template_vars(
+    line: str
+) -> Optional[Dict[str, str]]:
+    """Look for string variables which are no longer supported
+
+    Examples:
+        >>> this = check_for_deprecated_task_event_template_vars
+
+        >>> this('hello = "My name is %(suite)s"')
+        {'list': '    * %(suite)s ⇒ %(workflow)s'}
+
+        >>> expect = {'list': (
+        ...     '    * %(suite)s ⇒ %(workflow)s    * %(task_url)s'
+        ... ' - get ``URL`` (if set in :cylc:conf:`[meta]URL`)')}
+        >>> this('hello = "My name is %(suite)s, %(task_url)s"') == expect
+        True
+    """
+    result = []
+    for key, (regex, replacement) in deprecated_string_templates.items():
+        search_outcome = regex.findall(line)
+        if search_outcome and ' ' in replacement:
+            result.append(f'%({key})s - get {replacement}')
+        elif search_outcome:
+            result.append(f'%({key})s ⇒ %({replacement})s')
+
+    if result:
+        return {'list': LIST_ITEM + LIST_ITEM.join(result)}
+    return None
+
+
 INDENTATION = re.compile(r'^(\s*)(.*)')
 
 
@@ -606,6 +661,23 @@ def list_wrapper(line: str, check: Callable) -> Optional[Dict[str, str]]:
         FUNCTION: re.compile(r'rose +date').findall,
     },
     'U015': {
+        'short': (
+            'Deprecated template variables.'),
+        'rst': (
+            'The following template variables, mostly used in event handlers,'
+            'are deprecated, and should be replaced:'
+            + ''.join([
+                f'\n * ``{old}`` ⇒ {new}'
+                for old, new in DEPRECATED_STRING_TEMPLATES.items()
+            ])
+        ),
+        'url': (
+            'https://cylc.github.io/cylc-doc/stable/html/user-guide/'
+            'writing-workflows/runtime.html#task-event-template-variables'
+        ),
+        FUNCTION: check_for_deprecated_task_event_template_vars,
+    },
+    'U016': {
         'short': 'Deprecated template vars: {vars}',
         'rst': (
             'agoigfva[or]'
@@ -1109,7 +1181,7 @@ def get_cylc_files(
     'section heading': '\n{title}\n{underline}\n',
     'issue heading': {
         'text': '\n{check}:\n    {summary}\n    {url}\n\n',
-        'rst': '\n`{check} <{url}>`_\n{underline}\n{summary}\n\n',
+        'rst': '\n{url}_\n{underline}\n{summary}\n\n',
     },
     'auto gen message': (
         'U998 and U999 represent automatically generated'
@@ -1154,16 +1226,17 @@ def get_reference(linter, output_type):
                 output += '\n'
             output += '\n* ' + summary
         else:
+            check = get_index_str(meta, index)
             template = issue_heading_template
             url = get_url(meta)
+            if output_type == 'rst':
+                url = f'`{check} <{url}>`' if url else f'{check}'
             msg = template.format(
                 title=index,
-                check=get_index_str(meta, index),
+                check=check,
                 summary=summary,
                 url=url,
-                underline=(
-                    len(get_index_str(meta, index)) + len(url) + 6
-                ) * '^'
+                underline=(len(url) + 1) * '^'
             )
             output += msg
     output += '\n'

tests/functional/runahead/06-release-update.t

@@ -35,7 +35,6 @@ sleep 10
 # (gratuitous use of --flows for test coverage)
 cylc dump --flows -t "${WORKFLOW_NAME}" | awk '{print $1 $2 $3 $7}' >'log'
 cmp_ok 'log' - <<__END__
-bar,$NEXT1,waiting,[1]
 foo,$NEXT1,waiting,[1]
 __END__
 

tests/integration/scripts/test_dump.py

@@ -0,0 +1,52 @@
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Test the "cylc dump" command."""
+
+from cylc.flow.option_parsers import (
+    Options,
+)
+from cylc.flow.scripts.dump import (
+    dump,
+    get_option_parser,
+)
+
+
+DumpOptions = Options(get_option_parser())
+
+
+async def test_dump_tasks(flow, scheduler, start):
+    """It should show n=0 tasks.
+
+    See: https://github.com/cylc/cylc-flow/pull/5600
+    """
+    id_ = flow({
+        'scheduler': {
+            'allow implicit tasks': 'true',
+        },
+        'scheduling': {
+            'graph': {
+                'R1': 'a => b => c',
+            },
+        },
+    })
+    schd = scheduler(id_)
+    async with start(schd):
+        # schd.release_queued_tasks()
+        await schd.update_data_structure()
+        ret = []
+        await dump(id_, DumpOptions(disp_form='tasks'), write=ret.append)
+        assert ret == ['a, 1, waiting, not-held, queued, not-runahead']

tests/unit/scripts/test_lint.py

@@ -125,7 +125,7 @@
             submission failed handler = giaSEHFUIHJ
             failed handler = woo
             execution timeout handler = sdfghjkl
-            expired handler = dafuhj
+            expired handler = %(suite_uuid)s %(user@host)s
             late handler = dafuhj
             submitted handler = dafuhj
             started handler = dafuhj

tests/unit/test_config.py

@@ -536,6 +536,17 @@ def test_process_startcp(
             None,
             id="Relative fcp"
         ),
+        pytest.param(
+            ISO8601_CYCLING_TYPE,
+            {
+                'initial cycle point': '2017-02-11',
+                'final cycle point': '+P4D+PT3H-PT2H',
+            },
+            None,
+            '20170215T0100+0530',
+            None,
+            id="Relative fcp chained"
+        ),
         pytest.param(
             ISO8601_CYCLING_TYPE,
             {
@@ -1504,11 +1515,7 @@ def test_zero_interval(
         ('1988-02-29', '+P1M+P1Y', '1989-03-29'),
         ('1910-08-14', '+P2D-PT6H', '1910-08-15T18:00'),
         ('1850-04-10', '+P1M-P1D+PT1H', '1850-05-09T01:00'),
-        pytest.param(
-            '1066-10-14', '+PT1H+PT1M', '1066-10-14T01:01',
-            marks=pytest.mark.xfail
-            # https://github.com/cylc/cylc-flow/issues/5047
-        ),
+        ('1066-10-14', '+PT1H+PT1M', '1066-10-14T01:01'),
     ]
 )
 def test_chain_expr(