From a967c006993adb277c92c5dcbcf60cc037e456c0 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Wed, 19 Jun 2024 11:05:31 +0100
Subject: [PATCH 01/28] Master Points to 8.4.0 (#6159)

---
 cylc/flow/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cylc/flow/__init__.py b/cylc/flow/__init__.py
index 40718ebdf7e..34897137506 100644
--- a/cylc/flow/__init__.py
+++ b/cylc/flow/__init__.py
@@ -53,7 +53,7 @@ def environ_init():
 
 environ_init()
 
-__version__ = '8.3.1.dev'
+__version__ = '8.4.0.dev'
 
 
 def iter_entry_points(entry_point_name):

From c3d5f1d158e06609d39c5a05b6cfa7952f4c34da Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Wed, 19 Jun 2024 16:34:27 +0100
Subject: [PATCH 02/28] lint: address RUF015

---
 cylc/flow/broadcast_report.py | 2 +-
 cylc/flow/id_cli.py           | 4 ++--
 cylc/flow/scripts/show.py     | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cylc/flow/broadcast_report.py b/cylc/flow/broadcast_report.py
index 72fedb4cbef..cb44f2212f3 100644
--- a/cylc/flow/broadcast_report.py
+++ b/cylc/flow/broadcast_report.py
@@ -72,7 +72,7 @@ def get_broadcast_change_iter(modified_settings, is_cancel=False):
         value = setting
         keys_str = ""
         while isinstance(value, dict):
-            key, value = list(value.items())[0]
+            key, value = next(iter(value.items()))
             if isinstance(value, dict):
                 keys_str += "[" + key + "]"
             else:
diff --git a/cylc/flow/id_cli.py b/cylc/flow/id_cli.py
index 9c7493fd612..cad615885e4 100644
--- a/cylc/flow/id_cli.py
+++ b/cylc/flow/id_cli.py
@@ -347,7 +347,7 @@ async def parse_ids_async(
     if src:
         if not flow_file_path:
             # get the workflow file path from the run dir
-            flow_file_path = get_flow_file(list(workflows)[0])
+            flow_file_path = get_flow_file(next(iter(workflows)))
         return workflows, flow_file_path
     return workflows, multi_mode
 
@@ -375,7 +375,7 @@ async def parse_id_async(
             'max_tasks': 1,
         },
     )
-    workflow_id = list(workflows)[0]
+    workflow_id = next(iter(workflows))
     tokens_list = workflows[workflow_id]
     tokens: Optional[Tokens]
     if tokens_list:
diff --git a/cylc/flow/scripts/show.py b/cylc/flow/scripts/show.py
index b146f339e10..2fdfaf7fde4 100755
--- a/cylc/flow/scripts/show.py
+++ b/cylc/flow/scripts/show.py
@@ -490,7 +490,7 @@ def main(_, options: 'Values', *ids) -> None:
         constraint='mixed',
         max_workflows=1,
     )
-    workflow_id = list(workflow_args)[0]
+    workflow_id = next(iter(workflow_args))
     tokens_list = workflow_args[workflow_id]
 
     if tokens_list and options.task_defs:

From fcabe74e83de14d2b9fa09c6279696216a1892a0 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Wed, 19 Jun 2024 17:11:42 +0100
Subject: [PATCH 03/28] lint: address B904

---
 cylc/flow/clean.py                        |  9 ++--
 cylc/flow/command_validation.py           |  2 +-
 cylc/flow/commands.py                     |  4 +-
 cylc/flow/config.py                       | 55 +++++++++++++----------
 cylc/flow/cycling/integer.py              |  2 +-
 cylc/flow/cycling/iso8601.py              |  6 +--
 cylc/flow/dbstatecheck.py                 |  6 +--
 cylc/flow/graph_parser.py                 |  8 ++--
 cylc/flow/host_select.py                  |  2 +-
 cylc/flow/id.py                           |  2 +-
 cylc/flow/id_cli.py                       |  4 +-
 cylc/flow/install.py                      |  4 +-
 cylc/flow/install_plugins/log_vc_info.py  |  2 +-
 cylc/flow/loggingutil.py                  |  2 +-
 cylc/flow/main_loop/__init__.py           |  4 +-
 cylc/flow/main_loop/health_check.py       |  2 +-
 cylc/flow/network/__init__.py             | 15 ++++---
 cylc/flow/network/client.py               |  2 +-
 cylc/flow/network/resolvers.py            |  2 +-
 cylc/flow/network/schema.py               |  2 +-
 cylc/flow/network/ssh_client.py           |  2 +-
 cylc/flow/param_expand.py                 | 14 +++---
 cylc/flow/parsec/config.py                |  6 ++-
 cylc/flow/parsec/empysupport.py           |  2 +-
 cylc/flow/parsec/fileparse.py             | 12 +++--
 cylc/flow/parsec/jinja2support.py         |  6 +--
 cylc/flow/parsec/upgrade.py               |  2 +-
 cylc/flow/parsec/validate.py              | 42 +++++++++++------
 cylc/flow/pathutil.py                     |  6 +--
 cylc/flow/prerequisite.py                 |  3 +-
 cylc/flow/scripts/broadcast.py            |  2 +-
 cylc/flow/scripts/cat_log.py              |  2 +-
 cylc/flow/scripts/clean.py                |  2 +-
 cylc/flow/scripts/cylc.py                 |  2 +-
 cylc/flow/scripts/dump.py                 |  3 +-
 cylc/flow/scripts/ext_trigger.py          |  2 +-
 cylc/flow/scripts/get_workflow_contact.py |  4 +-
 cylc/flow/scripts/lint.py                 |  2 +-
 cylc/flow/scripts/report_timings.py       |  4 +-
 cylc/flow/scripts/validate.py             | 21 +++++----
 cylc/flow/scripts/workflow_state.py       |  5 ++-
 cylc/flow/task_id.py                      |  3 +-
 cylc/flow/time_parser.py                  |  2 +-
 cylc/flow/tui/data.py                     |  4 +-
 cylc/flow/util.py                         |  4 +-
 cylc/flow/workflow_db_mgr.py              |  4 +-
 cylc/flow/workflow_files.py               |  8 ++--
 cylc/flow/xtrigger_mgr.py                 | 12 ++---
 cylc/flow/xtriggers/wall_clock.py         |  4 +-
 49 files changed, 186 insertions(+), 134 deletions(-)

diff --git a/cylc/flow/clean.py b/cylc/flow/clean.py
index aec2c4efeb4..3378cfd1591 100644
--- a/cylc/flow/clean.py
+++ b/cylc/flow/clean.py
@@ -129,7 +129,7 @@ def _clean_check(opts: 'Values', id_: str, run_dir: Path) -> None:
     except ContactFileExists as exc:
         raise ServiceFileError(
             f"Cannot clean running workflow {id_}.\n\n{exc}"
-        )
+        ) from None
 
 
 def init_clean(id_: str, opts: 'Values') -> None:
@@ -173,7 +173,7 @@ def init_clean(id_: str, opts: 'Values') -> None:
             try:
                 platform_names = get_platforms_from_db(local_run_dir)
             except ServiceFileError as exc:
-                raise ServiceFileError(f"Cannot clean {id_} - {exc}")
+                raise ServiceFileError(f"Cannot clean {id_} - {exc}") from None
             except sqlite3.OperationalError as exc:
                 # something went wrong with the query
                 # e.g. the table/field we need isn't there
@@ -184,7 +184,7 @@ def init_clean(id_: str, opts: 'Values') -> None:
                     ' with to remove it.'
                     '\nOtherwise please delete the database file.'
                 )
-                raise ServiceFileError(f"Cannot clean {id_} - {exc}")
+                raise ServiceFileError(f"Cannot clean {id_} - {exc}") from exc
 
         if platform_names and platform_names != {'localhost'}:
             remote_clean(
@@ -359,7 +359,8 @@ def remote_clean(
     except PlatformLookupError as exc:
         raise PlatformLookupError(
             f"Cannot clean {id_} on remote platforms as the workflow database "
-            f"is out of date/inconsistent with the global config - {exc}")
+            f"is out of date/inconsistent with the global config - {exc}"
+        ) from None
 
     queue: Deque[RemoteCleanQueueTuple] = deque()
     remote_clean_cmd = partial(
diff --git a/cylc/flow/command_validation.py b/cylc/flow/command_validation.py
index 1c57d452c43..ea67f00206e 100644
--- a/cylc/flow/command_validation.py
+++ b/cylc/flow/command_validation.py
@@ -70,7 +70,7 @@ def flow_opts(flows: List[str], flow_wait: bool) -> None:
             try:
                 int(val)
             except ValueError:
-                raise InputError(ERR_OPT_FLOW_VAL.format(val))
+                raise InputError(ERR_OPT_FLOW_VAL.format(val)) from None
 
     if flow_wait and flows[0] in [FLOW_NEW, FLOW_NONE]:
         raise InputError(ERR_OPT_FLOW_WAIT)
diff --git a/cylc/flow/commands.py b/cylc/flow/commands.py
index a4ea43df5cf..2abf7f1c773 100644
--- a/cylc/flow/commands.py
+++ b/cylc/flow/commands.py
@@ -208,7 +208,7 @@ async def stop(
         try:
             mode = StopMode(mode)
         except ValueError:
-            raise CommandFailedError(f"Invalid stop mode: '{mode}'")
+            raise CommandFailedError(f"Invalid stop mode: '{mode}'") from None
         schd._set_stop(mode)
         if mode is StopMode.REQUEST_KILL:
             schd.time_next_kill = time()
@@ -304,7 +304,7 @@ async def set_verbosity(schd: 'Scheduler', level: Union[int, str]):
         lvl = int(level)
         LOG.setLevel(lvl)
     except (TypeError, ValueError) as exc:
-        raise CommandFailedError(exc)
+        raise CommandFailedError(exc) from None
     cylc.flow.flags.verbosity = log_level_to_verbosity(lvl)
 
 
diff --git a/cylc/flow/config.py b/cylc/flow/config.py
index 5b7738f3e6c..fdcc10adb05 100644
--- a/cylc/flow/config.py
+++ b/cylc/flow/config.py
@@ -199,11 +199,11 @@ def interpolate_template(tmpl, params_dict):
     try:
         return tmpl % params_dict
     except KeyError:
-        raise ParamExpandError('bad parameter')
+        raise ParamExpandError('bad parameter') from None
     except TypeError:
-        raise ParamExpandError('wrong data type for parameter')
+        raise ParamExpandError('wrong data type for parameter') from None
     except ValueError:
-        raise ParamExpandError('bad template syntax')
+        raise ParamExpandError('bad template syntax') from None
 
 
 class WorkflowConfig:
@@ -480,8 +480,8 @@ def __init__(
                             get_interval(offset_string).standardise())
                     except IntervalParsingError:
                         raise WorkflowConfigError(
-                            "Illegal %s spec: %s" % (
-                                s_type, offset_string))
+                            "Illegal %s spec: %s" % (s_type, offset_string)
+                        ) from None
                     extn = "(" + offset_string + ")"
 
                 # Replace family names with members.
@@ -709,7 +709,7 @@ def process_initial_cycle_point(self) -> None:
                 try:
                     icp = ingest_time(orig_icp, get_current_time_string())
                 except IsodatetimeError as exc:
-                    raise WorkflowConfigError(str(exc))
+                    raise WorkflowConfigError(str(exc)) from None
         if orig_icp != icp:
             # now/next()/previous() was used, need to store
             # evaluated point in DB
@@ -761,7 +761,7 @@ def process_start_cycle_point(self) -> None:
                     for taskid in self.options.starttask
                 ]
             except ValueError as exc:
-                raise InputError(str(exc))
+                raise InputError(str(exc)) from None
             self.start_point = min(
                 get_point(cycle).standardise()
                 for cycle in cycle_points if cycle
@@ -1114,7 +1114,7 @@ def _check_completion_expression(self, task_name: str, expr: str) -> None:
                     f'\n  {expr}'
                     '\nThe "finished" output cannot be used in completion'
                     ' expressions, use "succeeded or failed".'
-                )
+                ) from None
 
             for alt_qualifier, qualifier in ALT_QUALIFIERS.items():
                 _alt_compvar = trigger_to_completion_variable(alt_qualifier)
@@ -1125,21 +1125,21 @@ def _check_completion_expression(self, task_name: str, expr: str) -> None:
                         f'\n  {expr}'
                         f'\nUse "{_compvar}" not "{_alt_compvar}" '
                         'in completion expressions.'
-                    )
+                    ) from None
 
             raise WorkflowConfigError(
                 # NOTE: str(exc) == "name 'x' is not defined" tested in
                 # tests/integration/test_optional_outputs.py
                 f'Error in [runtime][{task_name}]completion:'
                 f'\n{error}'
-            )
+            ) from None
         except Exception as exc:  # includes InvalidCompletionExpression
             # expression contains non-whitelisted syntax or any other error in
             # the expression e.g. SyntaxError
             raise WorkflowConfigError(
                 f'Error in [runtime][{task_name}]completion:'
                 f'\n{str(exc)}'
-            )
+            ) from None
 
         # ensure consistency between the graph and the completion expression
         for compvar in (
@@ -1415,11 +1415,12 @@ def compute_family_tree(self):
                     c3_single.mro(name))
             except RecursionError:
                 raise WorkflowConfigError(
-                    "circular [runtime] inheritance?")
+                    "circular [runtime] inheritance?"
+                ) from None
             except Exception as exc:
                 # catch inheritance errors
                 # TODO - specialise MRO exceptions
-                raise WorkflowConfigError(str(exc))
+                raise WorkflowConfigError(str(exc)) from None
 
         for name in self.cfg['runtime']:
             ancestors = self.runtime['linearized ancestors'][name]
@@ -1758,7 +1759,7 @@ def _check_task_event_handlers(self):
                                     f' {taskdef.name}:'
                                     f' {handler_template}:'
                                     f' {repr(exc)}'
-                                )
+                                ) from None
 
     def _check_special_tasks(self):
         """Check declared special tasks are valid, and detect special
@@ -1865,7 +1866,9 @@ def generate_triggers(self, lexpression, left_nodes, right, seq,
         try:
             expr_list = listify(lexpression)
         except SyntaxError:
-            raise WorkflowConfigError('Error in expression "%s"' % lexpression)
+            raise WorkflowConfigError(
+                'Error in expression "%s"' % lexpression
+            ) from None
 
         triggers = {}
         xtrig_labels = set()
@@ -1942,7 +1945,9 @@ def generate_triggers(self, lexpression, left_nodes, right, seq,
                 xtrig = xtrigs[label]
             except KeyError:
                 if label != 'wall_clock':
-                    raise WorkflowConfigError(f"xtrigger not defined: {label}")
+                    raise WorkflowConfigError(
+                        f"xtrigger not defined: {label}"
+                    ) from None
                 else:
                     # Allow "@wall_clock" in graph as implicit zero-offset.
                     xtrig = SubFuncContext('wall_clock', 'wall_clock', [], {})
@@ -2276,7 +2281,7 @@ def load_graph(self):
                 msg += ' (final cycle point=%s)' % fcp
                 if isinstance(exc, CylcError):
                     msg += ' %s' % exc.args[0]
-                raise WorkflowConfigError(msg)
+                raise WorkflowConfigError(msg) from None
             self.sequences.append(seq)
             parser = GraphParser(
                 family_map,
@@ -2431,7 +2436,7 @@ def get_taskdef(
             except TaskDefError as exc:
                 if orig_expr:
                     LOG.error(orig_expr)
-                raise WorkflowConfigError(str(exc))
+                raise WorkflowConfigError(str(exc)) from None
             else:
                 # Record custom message outputs from [runtime].
                 for output, message in (
@@ -2443,14 +2448,14 @@ def get_taskdef(
                             f'Invalid task output "'
                             f'[runtime][{name}][outputs]'
                             f'{output} = {message}" - {msg}'
-                        )
+                        ) from None
                     valid, msg = TaskMessageValidator.validate(message)
                     if not valid:
                         raise WorkflowConfigError(
                             f'Invalid task message "'
                             f'[runtime][{name}][outputs]'
                             f'{output} = {message}" - {msg}'
-                        )
+                        ) from None
                     self.taskdefs[name].add_output(output, message)
 
         return self.taskdefs[name]
@@ -2462,7 +2467,7 @@ def _get_taskdef(self, name: str) -> TaskDef:
         try:
             rtcfg = self.cfg['runtime'][name]
         except KeyError:
-            raise WorkflowConfigError("Task not defined: %s" % name)
+            raise WorkflowConfigError("Task not defined: %s" % name) from None
         # We may want to put in some handling for cases of changing the
         # initial cycle via restart (accidentally or otherwise).
 
@@ -2554,7 +2559,9 @@ def process_metadata_urls(self):
                     'workflow': self.workflow,
                 }
             except (KeyError, ValueError):
-                raise InputError(f'Invalid template [meta]URL: {url}')
+                raise InputError(
+                    f'Invalid template [meta]URL: {url}'
+                ) from None
             else:
                 LOG.warning(
                     'Detected deprecated template variables in [meta]URL.'
@@ -2590,7 +2597,9 @@ def process_metadata_urls(self):
                         'task': name,
                     }
                 except (KeyError, ValueError):
-                    raise InputError(f'Invalid template [meta]URL: {url}')
+                    raise InputError(
+                        f'Invalid template [meta]URL: {url}'
+                    ) from None
                 else:
                     LOG.warning(
                         'Detected deprecated template variables in'
diff --git a/cylc/flow/cycling/integer.py b/cylc/flow/cycling/integer.py
index 749c651fc08..20cc3765670 100644
--- a/cylc/flow/cycling/integer.py
+++ b/cylc/flow/cycling/integer.py
@@ -150,7 +150,7 @@ def standardise(self):
         try:
             self.value = str(int(self))
         except (TypeError, ValueError) as exc:
-            raise PointParsingError(type(self), self.value, exc)
+            raise PointParsingError(type(self), self.value, exc) from None
         return self
 
     def __int__(self):
diff --git a/cylc/flow/cycling/iso8601.py b/cylc/flow/cycling/iso8601.py
index a66ce3f5ba0..2ab311df425 100644
--- a/cylc/flow/cycling/iso8601.py
+++ b/cylc/flow/cycling/iso8601.py
@@ -102,7 +102,7 @@ def standardise(self):
                     WorkflowSpecifics.NUM_EXPANDED_YEAR_DIGITS)
             else:
                 message = str(exc)
-            raise PointParsingError(type(self), self.value, message)
+            raise PointParsingError(type(self), self.value, message) from None
         return self
 
     def sub(self, other):
@@ -176,7 +176,7 @@ def standardise(self):
         try:
             self.value = str(interval_parse(self.value))
         except IsodatetimeError:
-            raise IntervalParsingError(type(self), self.value)
+            raise IntervalParsingError(type(self), self.value) from None
         return self
 
     def add(self, other):
@@ -782,7 +782,7 @@ def prev_next(
             raise WorkflowConfigError(
                 f'Invalid offset: {my_time}:'
                 f' Offset lists are semicolon separated, try {suggest}'
-            )
+            ) from None
 
         timepoints.append(parsed_point + now)
 
diff --git a/cylc/flow/dbstatecheck.py b/cylc/flow/dbstatecheck.py
index b38c394ccdb..1890c697c71 100644
--- a/cylc/flow/dbstatecheck.py
+++ b/cylc/flow/dbstatecheck.py
@@ -78,7 +78,7 @@ def __init__(self, rund, workflow, db_path=None):
         try:
             self.db_point_fmt = self._get_db_point_format()
             self.c7_back_compat_mode = False
-        except sqlite3.OperationalError as exc:
+        except sqlite3.OperationalError:
             # BACK COMPAT: Cylc 7 DB (see method below).
             try:
                 self.db_point_fmt = self._get_db_point_format_compat()
@@ -86,7 +86,7 @@ def __init__(self, rund, workflow, db_path=None):
             except sqlite3.OperationalError:
                 with suppress(Exception):
                     self.conn.close()
-                raise exc  # original error
+                raise
 
     def __enter__(self):
         return self
@@ -137,7 +137,7 @@ def adjust_point_to_db(self, cycle, offset):
             raise InputError(
                 f'Cycle point "{cycle}" is not compatible'
                 f' with DB point format "{self.db_point_fmt}"'
-            )
+            ) from None
         return cycle
 
     @staticmethod
diff --git a/cylc/flow/graph_parser.py b/cylc/flow/graph_parser.py
index efbce16fb36..3fbfd8c1754 100644
--- a/cylc/flow/graph_parser.py
+++ b/cylc/flow/graph_parser.py
@@ -346,7 +346,7 @@ def parse_graph(self, graph_string: str) -> None:
                         raise GraphParseError(
                             f"Dangling {seq}:"
                             f"{this_line}"
-                        )
+                        ) from None
             part_lines.append(this_line)
 
             # Check that a continuation sequence doesn't end this line and
@@ -638,7 +638,8 @@ def _proc_dep_pair(
                     except KeyError:
                         # "FAM:bad => foo" in LHS (includes "FAM => bar" too).
                         raise GraphParseError(
-                            f"Illegal family trigger in {expr}")
+                            f"Illegal family trigger in {expr}"
+                        ) from None
                 else:
                     # Not a family.
                     if trig in self.__class__.fam_to_mem_trigger_map:
@@ -911,7 +912,8 @@ def _compute_triggers(
                 except KeyError:
                     # Illegal family trigger on RHS of a pair.
                     raise GraphParseError(
-                        f"Illegal family trigger: {name}:{output}")
+                        f"Illegal family trigger: {name}:{output}"
+                    ) from None
             else:
                 fam = False
                 if not output:
diff --git a/cylc/flow/host_select.py b/cylc/flow/host_select.py
index 0eb34d088ca..69e32c68a71 100644
--- a/cylc/flow/host_select.py
+++ b/cylc/flow/host_select.py
@@ -373,7 +373,7 @@ def _filter_by_ranking(hosts, rankings, results, data=None):
                     f'\n    Expression: {item}'
                     f'\n    Configuration: {GLBL_CFG_STR}'
                     f'\n    Error: {exc}'
-                )
+                ) from None
             if isinstance(result, bool):
                 host_rankings[item] = result
                 data[host][item] = result
diff --git a/cylc/flow/id.py b/cylc/flow/id.py
index 58fff7fa7bc..f2c8b05b4a1 100644
--- a/cylc/flow/id.py
+++ b/cylc/flow/id.py
@@ -128,7 +128,7 @@ def __getitem__(self, key):
             return dict.__getitem__(self, key)
         except KeyError:
             if key not in self._KEYS:
-                raise ValueError(f'Invalid token: {key}')
+                raise ValueError(f'Invalid token: {key}') from None
             return None
 
     def __str__(self):
diff --git a/cylc/flow/id_cli.py b/cylc/flow/id_cli.py
index cad615885e4..35afbf80d5a 100644
--- a/cylc/flow/id_cli.py
+++ b/cylc/flow/id_cli.py
@@ -167,9 +167,9 @@ def _parse_cli(*ids: str) -> List[Tokens]:
                     # this ID is invalid with or without the trailing slash
                     tokens = cli_tokenise(id_[:-1])
                 except ValueError:
-                    raise InputError(f'Invalid ID: {id_}')
+                    raise InputError(f'Invalid ID: {id_}') from None
             else:
-                raise InputError(f'Invalid ID: {id_}')
+                raise InputError(f'Invalid ID: {id_}') from None
         is_partial = tokens.get('workflow') and not tokens.get('cycle')
         is_relative = not tokens.get('workflow')
 
diff --git a/cylc/flow/install.py b/cylc/flow/install.py
index 27810f72e97..2e94943d9d6 100644
--- a/cylc/flow/install.py
+++ b/cylc/flow/install.py
@@ -328,7 +328,7 @@ def install_workflow(
         # This occurs when the file exists but is _not_ a directory.
         raise WorkflowFilesError(
             f"Cannot install as there is an existing file at {rundir}."
-        )
+        ) from None
     if relink:
         link_runN(rundir)
     rsync_cmd = get_rsync_rund_cmd(source, rundir)
@@ -529,7 +529,7 @@ def parse_cli_sym_dirs(symlink_dirs: str) -> Dict[str, Dict[str, Any]]:
                 'There is an error in --symlink-dirs option:'
                 f' {pair}. Try entering option in the form '
                 '--symlink-dirs=\'log=$DIR, share=$DIR2, ...\''
-            )
+            ) from None
         if key not in possible_symlink_dirs:
             dirs = ', '.join(possible_symlink_dirs)
             raise InputError(
diff --git a/cylc/flow/install_plugins/log_vc_info.py b/cylc/flow/install_plugins/log_vc_info.py
index 29d861f7654..41479d6f1ed 100644
--- a/cylc/flow/install_plugins/log_vc_info.py
+++ b/cylc/flow/install_plugins/log_vc_info.py
@@ -253,7 +253,7 @@ def _run_cmd(
     except FileNotFoundError as exc:
         # This will only be raised if the VCS command is not installed,
         # otherwise Popen() will succeed with a non-zero return code
-        raise VCSNotInstalledError(vcs, exc)
+        raise VCSNotInstalledError(vcs, exc) from None
     if stdout == PIPE:
         out, err = pipe_poller(proc, proc.stdout, proc.stderr)
     else:
diff --git a/cylc/flow/loggingutil.py b/cylc/flow/loggingutil.py
index 3d77bdcb037..a6f1fbee558 100644
--- a/cylc/flow/loggingutil.py
+++ b/cylc/flow/loggingutil.py
@@ -206,7 +206,7 @@ def should_rollover(self, record: logging.LogRecord) -> bool:
             self.stream.seek(0, 2)
         except ValueError as exc:
             # intended to catch - ValueError: I/O operation on closed file
-            raise SystemExit(exc)
+            raise SystemExit(exc) from None
         return self.stream.tell() + len(msg.encode('utf8')) >= max_bytes
 
     @property
diff --git a/cylc/flow/main_loop/__init__.py b/cylc/flow/main_loop/__init__.py
index 2350153842c..e9f9f35f5da 100644
--- a/cylc/flow/main_loop/__init__.py
+++ b/cylc/flow/main_loop/__init__.py
@@ -329,14 +329,14 @@ def load(config, additional_plugins=None):
                 f'No main-loop plugin: "{plugin_name}"\n'
                 + '    Available plugins:\n'
                 + indent('\n'.join(sorted(entry_points)), '        ')
-            )
+            ) from None
         # load plugin
         try:
             module = entry_point.load()
         except Exception as exc:
             raise PluginError(
                 'cylc.main_loop', entry_point.name, exc
-            )
+            ) from None
         # load coroutines
         log = []
         for coro_name, coro in getmembers(module):
diff --git a/cylc/flow/main_loop/health_check.py b/cylc/flow/main_loop/health_check.py
index b488c878c0e..a470fd3cb88 100644
--- a/cylc/flow/main_loop/health_check.py
+++ b/cylc/flow/main_loop/health_check.py
@@ -56,4 +56,4 @@ def _check_contact_file(scheduler):
         raise CylcError(
             '%s: contact file corrupted/modified and may be left'
             % workflow_files.get_contact_file_path(scheduler.workflow)
-        )
+        ) from None
diff --git a/cylc/flow/network/__init__.py b/cylc/flow/network/__init__.py
index 916b129e244..315a57a23ba 100644
--- a/cylc/flow/network/__init__.py
+++ b/cylc/flow/network/__init__.py
@@ -78,7 +78,7 @@ def get_location(workflow: str) -> Tuple[str, int, int]:
         contact = load_contact_file(workflow)
     except (IOError, ValueError, ServiceFileError):
         # Contact file does not exist or corrupted, workflow should be dead
-        raise WorkflowStopped(workflow)
+        raise WorkflowStopped(workflow) from None
 
     host = contact[ContactFileFields.HOST]
     host = get_fqdn_by_host(host)
@@ -179,13 +179,13 @@ def _socket_bind(self, min_port, max_port, srv_prv_key_loc=None):
                 f"Failed to find server's public "
                 f"key in "
                 f"{srv_prv_key_info.full_key_path}."
-            )
+            ) from None
         except OSError:
             raise ServiceFileError(
                 f"IO error opening server's private "
                 f"key from "
                 f"{srv_prv_key_info.full_key_path}."
-            )
+            ) from None
         if server_private_key is None:  # this can't be caught by exception
             raise ServiceFileError(
                 f"Failed to find server's private "
@@ -204,7 +204,9 @@ def _socket_bind(self, min_port, max_port, srv_prv_key_loc=None):
                 self.port = self.socket.bind_to_random_port(
                     'tcp://*', min_port, max_port)
         except (zmq.error.ZMQError, zmq.error.ZMQBindError) as exc:
-            raise CylcError(f'could not start Cylc ZMQ server: {exc}')
+            raise CylcError(
+                f'could not start Cylc ZMQ server: {exc}'
+            ) from None
 
     # Keeping srv_public_key_loc as optional arg so as to not break interface
     def _socket_connect(self, host, port, srv_public_key_loc=None):
@@ -237,7 +239,7 @@ def _socket_connect(self, host, port, srv_public_key_loc=None):
             client_public_key, client_priv_key = zmq.auth.load_certificate(
                 client_priv_key_info.full_key_path)
         except (OSError, ValueError):
-            raise ClientError(error_msg)
+            raise ClientError(error_msg) from None
         if client_priv_key is None:  # this can't be caught by exception
             raise ClientError(error_msg)
         self.socket.curve_publickey = client_public_key
@@ -256,7 +258,8 @@ def _socket_connect(self, host, port, srv_public_key_loc=None):
             self.socket.curve_serverkey = server_public_key
         except (OSError, ValueError):  # ValueError raised w/ no public key
             raise ClientError(
-                "Failed to load the workflow's public key, so cannot connect.")
+                "Failed to load the workflow's public key, so cannot connect."
+            ) from None
 
         self.socket.connect(f'tcp://{host}:{port}')
 
diff --git a/cylc/flow/network/client.py b/cylc/flow/network/client.py
index e7e26954d56..099ef8bc0ff 100644
--- a/cylc/flow/network/client.py
+++ b/cylc/flow/network/client.py
@@ -326,7 +326,7 @@ async def async_request(
             raise ClientError(
                 error.get('message'),  # type: ignore
                 error.get('traceback'),  # type: ignore
-            )
+            ) from None
 
     def get_header(self) -> dict:
         """Return "header" data to attach to each request for traceability.
diff --git a/cylc/flow/network/resolvers.py b/cylc/flow/network/resolvers.py
index 11eafd8bea5..fc9b67eeef5 100644
--- a/cylc/flow/network/resolvers.py
+++ b/cylc/flow/network/resolvers.py
@@ -750,7 +750,7 @@ async def _mutation_mapper(
         try:
             meth = COMMANDS[command]
         except KeyError:
-            raise ValueError(f"Command '{command}' not found")
+            raise ValueError(f"Command '{command}' not found") from None
 
         try:
             # Initiate the command. Validation may be performed at this point,
diff --git a/cylc/flow/network/schema.py b/cylc/flow/network/schema.py
index 70e40232c1d..b962b582b70 100644
--- a/cylc/flow/network/schema.py
+++ b/cylc/flow/network/schema.py
@@ -271,7 +271,7 @@ def field_name_from_type(
     try:
         return NODE_MAP[named_type.name]
     except KeyError:
-        raise ValueError(f"'{named_type.name}' is not a node type")
+        raise ValueError(f"'{named_type.name}' is not a node type") from None
 
 
 def get_resolvers(info: 'ResolveInfo') -> 'BaseResolvers':
diff --git a/cylc/flow/network/ssh_client.py b/cylc/flow/network/ssh_client.py
index d1dd4fb6da4..d2ed0dd33e9 100644
--- a/cylc/flow/network/ssh_client.py
+++ b/cylc/flow/network/ssh_client.py
@@ -90,7 +90,7 @@ async def async_request(
                 f"Command exceeded the timeout {timeout}s. "
                 "This could be due to network problems. "
                 "Check the workflow log."
-            )
+            ) from None
 
     def prepare_command(
         self, command: str, args: Optional[dict], timeout: Union[float, str]
diff --git a/cylc/flow/param_expand.py b/cylc/flow/param_expand.py
index 0707a46e1a3..6ab104404e4 100644
--- a/cylc/flow/param_expand.py
+++ b/cylc/flow/param_expand.py
@@ -195,8 +195,9 @@ def _expand_name(self, results, tmpl, params, spec_vals=None):
             try:
                 results.append((tmpl % current_values, current_values))
             except KeyError as exc:
-                raise ParamExpandError('parameter %s is not '
-                                       'defined.' % str(exc.args[0]))
+                raise ParamExpandError(
+                    'parameter %s is not ' 'defined.' % str(exc.args[0])
+                ) from None
         else:
             for param_val in params[0][1]:
                 spec_vals[params[0][0]] = param_val
@@ -306,8 +307,8 @@ def expand_parent_params(self, parent, param_values, origin):
                     used[item] = param_values[item]
                 except KeyError:
                     raise ParamExpandError(
-                        "parameter '%s' undefined in '%s'" % (
-                            item, origin))
+                        "parameter '%s' undefined in '%s'" % (item, origin)
+                    ) from None
 
         # For each parameter substitute the param_tmpl_cfg.
         tmpl = tmpl.format(**self.param_tmpl_cfg)
@@ -425,8 +426,9 @@ def _expand_graph(self, line, all_params,
                 try:
                     repl = tmpl % param_values
                 except KeyError as exc:
-                    raise ParamExpandError('parameter %s is not '
-                                           'defined.' % str(exc.args[0]))
+                    raise ParamExpandError(
+                        'parameter %s is not ' 'defined.' % str(exc.args[0])
+                    ) from None
                 line = line.replace('<' + p_group + '>', repl)
             if line:
                 line_set.add(line)
diff --git a/cylc/flow/parsec/config.py b/cylc/flow/parsec/config.py
index 19f937d8e5b..29944c03b30 100644
--- a/cylc/flow/parsec/config.py
+++ b/cylc/flow/parsec/config.py
@@ -150,10 +150,12 @@ def get(self, keys: Optional[Iterable[str]] = None, sparse: bool = False):
                         # setting not present in __MANY__ section:
                         key in self.spec.get(*parents)
                     ):
-                        raise ItemNotFoundError(itemstr(parents, key))
+                        raise ItemNotFoundError(
+                            itemstr(parents, key)
+                        ) from None
                     raise InvalidConfigError(
                         itemstr(parents, key), self.spec.name
-                    )
+                    ) from None
                 else:
                     parents.append(key)
 
diff --git a/cylc/flow/parsec/empysupport.py b/cylc/flow/parsec/empysupport.py
index b4164894e0f..e3dc5e28df4 100644
--- a/cylc/flow/parsec/empysupport.py
+++ b/cylc/flow/parsec/empysupport.py
@@ -66,7 +66,7 @@ def empyprocess(
         raise EmPyError(
             str(exc),
             lines={'<template>': flines[max(lineno - 4, 0): lineno]},
-        )
+        ) from None
     finally:
         interpreter.shutdown()
         xworkflow = xtempl.getvalue()
diff --git a/cylc/flow/parsec/fileparse.py b/cylc/flow/parsec/fileparse.py
index 57b3449561a..f4de85aee3c 100644
--- a/cylc/flow/parsec/fileparse.py
+++ b/cylc/flow/parsec/fileparse.py
@@ -493,8 +493,10 @@ def read_and_proc(
             try:
                 from cylc.flow.parsec.empysupport import empyprocess
             except ImportError:
-                raise ParsecError('EmPy Python package must be installed '
-                                  'to process file: ' + fpath)
+                raise ParsecError(
+                    'EmPy Python package must be installed '
+                    'to process file: ' + fpath
+                ) from None
             flines = empyprocess(
                 fpath, flines, fdir, template_vars
             )
@@ -513,8 +515,10 @@ def read_and_proc(
             try:
                 from cylc.flow.parsec.jinja2support import jinja2process
             except ImportError:
-                raise ParsecError('Jinja2 Python package must be installed '
-                                  'to process file: ' + fpath)
+                raise ParsecError(
+                    'Jinja2 Python package must be installed '
+                    'to process file: ' + fpath
+                ) from None
             flines = jinja2process(
                 fpath, flines, fdir, template_vars
             )
diff --git a/cylc/flow/parsec/jinja2support.py b/cylc/flow/parsec/jinja2support.py
index 34f92c11519..74335358421 100644
--- a/cylc/flow/parsec/jinja2support.py
+++ b/cylc/flow/parsec/jinja2support.py
@@ -75,7 +75,7 @@ def load(
         try:
             mdict = __import__(name, fromlist=['*']).__dict__
         except ImportError:
-            raise TemplateNotFound(name)
+            raise TemplateNotFound(name) from None
 
         # inject module dict into the context of an empty template
         def root_render_func(context, *args, **kwargs):
@@ -297,12 +297,12 @@ def jinja2process(
             exc,
             lines=get_error_lines(fpath, flines),
             filename=filename
-        )
+        ) from None
     except Exception as exc:
         raise Jinja2Error(
             exc,
             lines=get_error_lines(fpath, flines),
-        )
+        ) from None
 
     # Ignore blank lines (lone Jinja2 statements leave blank lines behind)
     return [line for line in lines if line.strip()]
diff --git a/cylc/flow/parsec/upgrade.py b/cylc/flow/parsec/upgrade.py
index 0f75732f6d5..3ef53b006f6 100644
--- a/cylc/flow/parsec/upgrade.py
+++ b/cylc/flow/parsec/upgrade.py
@@ -110,7 +110,7 @@ def get_item(self, keys):
                 raise UpgradeError(
                     f'{self.show_keys(keys[:-1], True)}'
                     f' ("{keys[-2]}" should be a [section] not a setting)'
-                )
+                ) from None
         return item
 
     def put_item(self, keys, val):
diff --git a/cylc/flow/parsec/validate.py b/cylc/flow/parsec/validate.py
index 18c19596a63..12e2163a894 100644
--- a/cylc/flow/parsec/validate.py
+++ b/cylc/flow/parsec/validate.py
@@ -280,7 +280,7 @@ def coerce_float(cls, value, keys):
         try:
             return float(value)
         except ValueError as exc:
-            raise IllegalValueError('float', keys, value, exc=exc)
+            raise IllegalValueError('float', keys, value, exc=exc) from None
 
     @classmethod
     def coerce_float_list(cls, value, keys):
@@ -309,7 +309,7 @@ def coerce_int(cls, value, keys):
         try:
             return int(value)
         except ValueError as exc:
-            raise IllegalValueError('int', keys, value, exc=exc)
+            raise IllegalValueError('int', keys, value, exc=exc) from None
 
     @classmethod
     def coerce_int_list(cls, value, keys):
@@ -480,13 +480,17 @@ def expand_list(cls, values, keys, type_):
                 try:
                     lvalues.append(type_(item))
                 except ValueError as exc:
-                    raise IllegalValueError('list', keys, item, exc=exc)
+                    raise IllegalValueError(
+                        'list', keys, item, exc=exc
+                    ) from None
             else:
                 # mult * val
                 try:
                     lvalues += int(mult) * [type_(val)]
                 except ValueError as exc:
-                    raise IllegalValueError('list', keys, item, exc=exc)
+                    raise IllegalValueError(
+                        'list', keys, item, exc=exc
+                    ) from None
         return lvalues
 
     @classmethod
@@ -832,7 +836,9 @@ def coerce_cycle_point(cls, value, keys):
                     parser.parse(value)
                 return value
             except IsodatetimeError as exc:
-                raise IllegalValueError('cycle point', keys, value, exc=exc)
+                raise IllegalValueError(
+                    'cycle point', keys, value, exc=exc
+                ) from None
         try:
             TimePointParser().parse(value)
         except IsodatetimeError as exc:
@@ -841,7 +847,9 @@ def coerce_cycle_point(cls, value, keys):
                 details = {'msg': "Invalid cycle point"}
             else:
                 details = {'exc': exc}
-            raise IllegalValueError('cycle point', keys, value, **details)
+            raise IllegalValueError(
+                'cycle point', keys, value, **details
+            ) from None
         return value
 
     @classmethod
@@ -883,7 +891,7 @@ def coerce_cycle_point_format(cls, value, keys):
             except IsodatetimeError as exc:
                 raise IllegalValueError(
                     'cycle point format', keys, value, exc=exc
-                )
+                ) from None
             return value
         if 'X' in value:
             for i in range(1, 101):
@@ -898,7 +906,9 @@ def coerce_cycle_point_format(cls, value, keys):
         try:
             dumper.dump(test_timepoint, value)
         except IsodatetimeError as exc:
-            raise IllegalValueError('cycle point format', keys, value, exc=exc)
+            raise IllegalValueError(
+                'cycle point format', keys, value, exc=exc
+            ) from None
         return value
 
     @classmethod
@@ -932,7 +942,7 @@ def coerce_cycle_point_time_zone(cls, value, keys):
         except ValueError as exc:  # not IsodatetimeError as too specific
             raise IllegalValueError(
                 'cycle point time zone format', keys, value, exc=exc
-            )
+            ) from None
         return value
 
     @classmethod
@@ -951,7 +961,9 @@ def coerce_interval(cls, value, keys):
         try:
             interval = DurationParser().parse(value)
         except IsodatetimeError as exc:
-            raise IllegalValueError("ISO 8601 interval", keys, value, exc=exc)
+            raise IllegalValueError(
+                "ISO 8601 interval", keys, value, exc=exc
+            ) from None
         return DurationFloat(interval.get_seconds())
 
     @classmethod
@@ -1012,8 +1024,11 @@ def coerce_parameter_list(cls, value, keys):
                 except ValueError:
                     if can_only_be == int:
                         raise IllegalValueError(
-                            'parameter', keys, value,
-                            'mixing int range and str')
+                            'parameter',
+                            keys,
+                            value,
+                            'mixing int range and str',
+                        ) from None
                     can_only_be = str
                 items.append(item)
             else:
@@ -1248,7 +1263,8 @@ def coerce_interval(cls, value, keys):
                 interval = DurationParser().parse(str(DurationFloat(value)))
             except IsodatetimeError as exc:
                 raise IllegalValueError(
-                    "ISO 8601 interval", keys, value, exc=exc)
+                    "ISO 8601 interval", keys, value, exc=exc
+                ) from None
         return DurationFloat(interval.get_seconds())
 
 
diff --git a/cylc/flow/pathutil.py b/cylc/flow/pathutil.py
index aa097a2e680..da3d6c93cde 100644
--- a/cylc/flow/pathutil.py
+++ b/cylc/flow/pathutil.py
@@ -272,7 +272,7 @@ def make_symlink_dir(path: Union[Path, str], target: Union[Path, str]) -> bool:
         except OSError as exc:
             raise WorkflowFilesError(
                 f"Failed to remove broken symlink {path}\n{exc}"
-            )
+            ) from None
     try:
         target.mkdir(parents=True, exist_ok=False)
     except FileExistsError:
@@ -280,7 +280,7 @@ def make_symlink_dir(path: Union[Path, str], target: Union[Path, str]) -> bool:
             f"Symlink dir target already exists: ({path} ->) {target}\n"
             "Tip: in future, use 'cylc clean' instead of manually deleting "
             "workflow run dirs."
-        )
+        ) from None
 
     # This is needed in case share and share/cycle have the same symlink dir:
     if path.exists():
@@ -291,7 +291,7 @@ def make_symlink_dir(path: Union[Path, str], target: Union[Path, str]) -> bool:
         path.symlink_to(target)
         return True
     except OSError as exc:
-        raise WorkflowFilesError(f"Error when symlinking\n{exc}")
+        raise WorkflowFilesError(f"Error when symlinking\n{exc}") from None
 
 
 def remove_dir_and_target(path: Union[Path, str]) -> None:
diff --git a/cylc/flow/prerequisite.py b/cylc/flow/prerequisite.py
index b388934e2de..449a5009656 100644
--- a/cylc/flow/prerequisite.py
+++ b/cylc/flow/prerequisite.py
@@ -195,7 +195,8 @@ def _conditional_is_satisfied(self):
                 err_msg += (
                     " (could be unmatched parentheses in the graph string?)")
             raise TriggerExpressionError(
-                '"%s":\n%s' % (self.get_raw_conditional_expression(), err_msg))
+                '"%s":\n%s' % (self.get_raw_conditional_expression(), err_msg)
+            ) from None
         return res
 
     def satisfy_me(self, outputs: Iterable['Tokens']) -> 'Set[Tokens]':
diff --git a/cylc/flow/scripts/broadcast.py b/cylc/flow/scripts/broadcast.py
index c7e5d2a4f3b..8cfda85bdd8 100755
--- a/cylc/flow/scripts/broadcast.py
+++ b/cylc/flow/scripts/broadcast.py
@@ -360,7 +360,7 @@ async def run(options: 'Values', workflow_id):
                 # TODO validate showtask?
                 raise InputError(
                     'TASK_ID_GLOB must be in the format: cycle/task'
-                )
+                ) from None
         result = await pclient.async_request('graphql', query_kwargs)
         for wflow in result['workflows']:
             settings = wflow['broadcasts']
diff --git a/cylc/flow/scripts/cat_log.py b/cylc/flow/scripts/cat_log.py
index 688f51ddd35..e53524cc648 100755
--- a/cylc/flow/scripts/cat_log.py
+++ b/cylc/flow/scripts/cat_log.py
@@ -495,7 +495,7 @@ def _main(
                     raise InputError(
                         f"--rotation {rotation_number} invalid "
                         f"(max value is {len(logs) - 1})"
-                    )
+                    ) from None
             else:
                 raise InputError('Log file not found.')
         else:
diff --git a/cylc/flow/scripts/clean.py b/cylc/flow/scripts/clean.py
index 2166a98f055..fffd27752d7 100644
--- a/cylc/flow/scripts/clean.py
+++ b/cylc/flow/scripts/clean.py
@@ -157,7 +157,7 @@ def parse_timeout(opts: 'Values') -> None:
                 raise InputError(
                     f"Invalid timeout: {opts.remote_timeout}. Must be "
                     "an ISO 8601 duration or number of seconds."
-                )
+                ) from None
         opts.remote_timeout = str(timeout)
 
 
diff --git a/cylc/flow/scripts/cylc.py b/cylc/flow/scripts/cylc.py
index 4ca411f8e9b..209512264a9 100644
--- a/cylc/flow/scripts/cylc.py
+++ b/cylc/flow/scripts/cylc.py
@@ -612,7 +612,7 @@ def pycoverage(cmd_args):  # pragma: no cover
             'Could not initiate coverage, likely because Cylc was not '
             'installed in editable mode.'
             '\n\n*****************************\n\n'
-        )
+        ) from None
 
     # start the coverage running
     cov.start()
diff --git a/cylc/flow/scripts/dump.py b/cylc/flow/scripts/dump.py
index f891c943d77..f47ee8d2df8 100755
--- a/cylc/flow/scripts/dump.py
+++ b/cylc/flow/scripts/dump.py
@@ -296,4 +296,5 @@ async def dump(workflow_id, options, write=print):
                         write(', '.join(values))
     except Exception as exc:
         raise CylcError(
-            json.dumps(workflows, indent=4) + '\n' + str(exc) + '\n')
+            json.dumps(workflows, indent=4) + '\n' + str(exc) + '\n'
+        ) from None
diff --git a/cylc/flow/scripts/ext_trigger.py b/cylc/flow/scripts/ext_trigger.py
index c76bbb86712..775d6eaa0da 100755
--- a/cylc/flow/scripts/ext_trigger.py
+++ b/cylc/flow/scripts/ext_trigger.py
@@ -145,7 +145,7 @@ def main(
             LOG.exception(exc)
             LOG.info(MSG_SEND_FAILED, i_try + 1, max_n_tries)
             if i_try == max_n_tries - 1:  # final attempt
-                raise CylcError('send failed')
+                raise CylcError('send failed') from None
             LOG.info(MSG_SEND_RETRY, retry_intvl_secs, options.comms_timeout)
             sleep(retry_intvl_secs)
         else:
diff --git a/cylc/flow/scripts/get_workflow_contact.py b/cylc/flow/scripts/get_workflow_contact.py
index 9903aec6abb..66194cce732 100755
--- a/cylc/flow/scripts/get_workflow_contact.py
+++ b/cylc/flow/scripts/get_workflow_contact.py
@@ -49,10 +49,10 @@ def main(parser: COP, options: 'Values', workflow_id: str) -> None:
     )
     try:
         data = load_contact_file(workflow_id)
-    except ServiceFileError:
+    except ServiceFileError as exc:
         raise CylcError(
             f"{workflow_id}: cannot get contact info, workflow not running?"
-        )
+        ) from exc
     else:
         for key, value in sorted(data.items()):
             print("%s=%s" % (key, value))
diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py
index d045ab70838..c452d3be8f5 100755
--- a/cylc/flow/scripts/lint.py
+++ b/cylc/flow/scripts/lint.py
@@ -805,7 +805,7 @@ def get_pyproject_toml(dir_: Path) -> Dict[str, Any]:
         try:
             loadeddata = toml_loads(tomlfile.read_text())
         except TOMLDecodeError as exc:
-            raise CylcError(f'pyproject.toml did not load: {exc}')
+            raise CylcError(f'pyproject.toml did not load: {exc}') from None
 
         _tool, _cylc, _lint = LINT_TABLE
         try:
diff --git a/cylc/flow/scripts/report_timings.py b/cylc/flow/scripts/report_timings.py
index b6d796ad732..362f40953c2 100755
--- a/cylc/flow/scripts/report_timings.py
+++ b/cylc/flow/scripts/report_timings.py
@@ -257,7 +257,7 @@ def _check_imports(self):
             raise CylcError(
                 'Cannot import pandas - summary unavailable.'
                 ' try: pip install cylc-flow[report-timings]'
-            )
+            ) from None
         else:
             del pandas
 
@@ -402,5 +402,5 @@ def _check_imports(self):
         except ImportError:
             raise CylcError(
                 'Cannot import matplotlib - HTML summary unavailable.'
-            )
+            ) from None
         super(HTMLTimingSummary, self)._check_imports()
diff --git a/cylc/flow/scripts/validate.py b/cylc/flow/scripts/validate.py
index a224302c1b0..f3a77087069 100755
--- a/cylc/flow/scripts/validate.py
+++ b/cylc/flow/scripts/validate.py
@@ -181,7 +181,8 @@ async def run(
             continue
         except Exception as exc:
             raise WorkflowConfigError(
-                'failed to instantiate task %s: %s' % (name, exc))
+                'failed to instantiate task %s: %s' % (name, exc)
+            ) from None
 
         # force trigger evaluation now
         try:
@@ -189,17 +190,21 @@ async def run(
         except TriggerExpressionError as exc:
             err = str(exc)
             if '@' in err:
-                print(f"ERROR, {name}: xtriggers can't be in conditional"
-                      f" expressions: {err}",
-                      file=sys.stderr)
+                print(
+                    f"ERROR, {name}: xtriggers can't be in conditional"
+                    f" expressions: {err}",
+                    file=sys.stderr,
+                )
             else:
-                print('ERROR, %s: bad trigger: %s' % (name, err),
-                      file=sys.stderr)
-            raise WorkflowConfigError("ERROR: bad trigger")
+                print(
+                    'ERROR, %s: bad trigger: %s' % (name, err), file=sys.stderr
+                )
+            raise WorkflowConfigError("ERROR: bad trigger") from None
         except Exception as exc:
             print(str(exc), file=sys.stderr)
             raise WorkflowConfigError(
-                '%s: failed to evaluate triggers.' % name)
+                '%s: failed to evaluate triggers.' % name
+            ) from None
         if cylc.flow.flags.verbosity > 0:
             print('  + %s ok' % itask.identity)
 
diff --git a/cylc/flow/scripts/workflow_state.py b/cylc/flow/scripts/workflow_state.py
index f6350110223..cea5ab906ab 100755
--- a/cylc/flow/scripts/workflow_state.py
+++ b/cylc/flow/scripts/workflow_state.py
@@ -179,7 +179,7 @@ def __init__(
         try:
             tokens = Tokens(self.id_)
         except ValueError as exc:
-            raise InputError(exc)
+            raise InputError(exc) from None
 
         self.workflow_id_raw = tokens.workflow_id
         self.selector = (
@@ -404,7 +404,8 @@ def main(parser: COP, options: 'Values', *ids: str) -> None:
                 options.depr_point = os.environ["CYLC_TASK_CYCLE_POINT"]
             except KeyError:
                 raise InputError(
-                    "--task-point: $CYLC_TASK_CYCLE_POINT is not defined")
+                    "--task-point: $CYLC_TASK_CYCLE_POINT is not defined"
+                ) from None
 
         if options.depr_point is not None:
             id_ += f"//{options.depr_point}"
diff --git a/cylc/flow/task_id.py b/cylc/flow/task_id.py
index dbe003222b6..50075f92b9b 100644
--- a/cylc/flow/task_id.py
+++ b/cylc/flow/task_id.py
@@ -92,7 +92,8 @@ def get_standardised_point_string(cls, point_string):
         except PointParsingError as exc:
             # (This is only needed to raise a clearer error message).
             raise ValueError(
-                "Invalid cycle point: %s (%s)" % (point_string, exc))
+                "Invalid cycle point: %s (%s)" % (point_string, exc)
+            ) from None
         return point_string
 
     @classmethod
diff --git a/cylc/flow/time_parser.py b/cylc/flow/time_parser.py
index 932a62677aa..c3d242d6883 100644
--- a/cylc/flow/time_parser.py
+++ b/cylc/flow/time_parser.py
@@ -255,7 +255,7 @@ def parse_recurrence(
             except CylcMissingContextPointError:
                 raise CylcMissingFinalCyclePointError(
                     "This workflow requires a final cycle point."
-                )
+                ) from None
 
             exclusion_points = []
             # Convert the exclusion strings to ISO8601 points
diff --git a/cylc/flow/tui/data.py b/cylc/flow/tui/data.py
index 90344e2462b..948d1906fe9 100644
--- a/cylc/flow/tui/data.py
+++ b/cylc/flow/tui/data.py
@@ -408,11 +408,11 @@ def online_mutate(mutation, selection):
     except WorkflowStopped:
         raise Exception(
             f'Cannot peform command {mutation} on a stopped workflow'
-        )
+        ) from None
     except (ClientError, ClientTimeout) as exc:
         raise Exception(
             f'Error connecting to workflow: {exc}'
-        )
+        ) from None
 
     request_string = generate_mutation(mutation, variables)
     client(
diff --git a/cylc/flow/util.py b/cylc/flow/util.py
index 74a11d0505a..5b7d5df06f8 100644
--- a/cylc/flow/util.py
+++ b/cylc/flow/util.py
@@ -281,7 +281,7 @@ def _eval(expr, **variables):
                 error_class,
                 f'{exc.msg}: {exc.text}',
                 {'expr': expr}
-            )
+            ) from None
 
         # check against whitelisted types
         try:
@@ -302,7 +302,7 @@ def _eval(expr, **variables):
                     'error_node': error_node,
                     'error_type': error_node.__class__.__name__,
                 },
-            )
+            ) from None
 
         # run the expresion
         # Note: this may raise runtime errors
diff --git a/cylc/flow/workflow_db_mgr.py b/cylc/flow/workflow_db_mgr.py
index 8e57d19292b..c3f3ea9a263 100644
--- a/cylc/flow/workflow_db_mgr.py
+++ b/cylc/flow/workflow_db_mgr.py
@@ -701,7 +701,9 @@ def _get_last_run_version(cls, pri_dao: CylcWorkflowDAO) -> Version:
                 [cls.KEY_CYLC_VERSION]
             ).fetchone()[0]
         except (TypeError, OperationalError):
-            raise ServiceFileError(f"{INCOMPAT_MSG}, or is corrupted.")
+            raise ServiceFileError(
+                f"{INCOMPAT_MSG}, or is corrupted."
+            ) from None
         return parse_version(last_run_ver)
 
     @classmethod
diff --git a/cylc/flow/workflow_files.py b/cylc/flow/workflow_files.py
index 023d481324c..ab390ec10aa 100644
--- a/cylc/flow/workflow_files.py
+++ b/cylc/flow/workflow_files.py
@@ -426,7 +426,7 @@ def _is_process_running(
         raise CylcError(
             f'Attempt to determine whether workflow is running on {host}'
             ' timed out after 10 seconds.'
-        )
+        ) from None
 
     if proc.returncode == 2:
         # the psutil call failed to gather metrics on the process
@@ -505,7 +505,7 @@ def detect_old_contact_file(
         # this can happen if contact file is from an outdated version of Cylc
         raise ServiceFileError(
             f'Found contact file with incomplete data:\n{exc}.'
-        )
+        ) from None
 
     # check if the workflow process is running ...
     # NOTE: can raise CylcError
@@ -631,7 +631,7 @@ def load_contact_file(id_: str, run_dir=None) -> Dict[str, str]:
         with open(path) as f:
             file_content = f.read()
     except IOError:
-        raise ServiceFileError("Couldn't load contact file")
+        raise ServiceFileError("Couldn't load contact file") from None
     data: Dict[str, str] = {}
     for line in file_content.splitlines():
         key, value = [item.strip() for item in line.split("=", 1)]
@@ -914,7 +914,7 @@ def infer_latest_run(
     try:
         id_ = str(path.relative_to(cylc_run_dir))
     except ValueError:
-        raise ValueError(f"{path} is not in the cylc-run directory")
+        raise ValueError(f"{path} is not in the cylc-run directory") from None
 
     if not path.exists():
         raise InputError(
diff --git a/cylc/flow/xtrigger_mgr.py b/cylc/flow/xtrigger_mgr.py
index 4f33ec7f3cc..60bd9b87504 100644
--- a/cylc/flow/xtrigger_mgr.py
+++ b/cylc/flow/xtrigger_mgr.py
@@ -250,12 +250,12 @@ def _validate(
         try:
             func = get_xtrig_func(fctx.mod_name, fctx.func_name, fdir)
         except (ImportError, AttributeError) as exc:
-            raise XtriggerConfigError(label, sig_str, exc)
+            raise XtriggerConfigError(label, sig_str, exc) from None
         try:
             sig = signature(func)
         except TypeError as exc:
             # not callable
-            raise XtriggerConfigError(label, sig_str, exc)
+            raise XtriggerConfigError(label, sig_str, exc) from None
 
         sig = cls._handle_sequential_kwarg(label, fctx, sig)
 
@@ -269,7 +269,7 @@ def _validate(
                     label, fctx, err
                 )
             else:
-                raise err
+                raise err from None
 
         # Specific xtrigger.validate(), if available.
         # Note arg string templating has not been done at this point.
@@ -293,7 +293,7 @@ def _validate(
                     raise XtriggerConfigError(
                         label, sig_str,
                         f"Illegal template in xtrigger: {match}",
-                    )
+                    ) from None
 
         # check for deprecated template variables
         deprecated_variables = template_vars & {
@@ -364,7 +364,7 @@ def _try_xtrig_validate_func(
         except Exception as exc:  # Note: catch all errors
             if not isinstance(exc, WorkflowConfigError):
                 LOG.exception(exc)
-            raise XtriggerConfigError(label, signature_str, exc)
+            raise XtriggerConfigError(label, signature_str, exc) from None
 
     # BACK COMPAT: workflow_state_backcompat
     # from: 8.0.0
@@ -390,7 +390,7 @@ def _try_workflow_state_backcompat(
             bound_args = sig.bind(*fctx.func_args, **fctx.func_kwargs)
         except TypeError:
             # failed signature check for backcompat function
-            raise err  # original signature check error
+            raise
 
         old_sig_str = fctx.get_signature()
         upg_sig_str = "workflow_state({})".format(
diff --git a/cylc/flow/xtriggers/wall_clock.py b/cylc/flow/xtriggers/wall_clock.py
index f000319d1ff..cea8d0b3026 100644
--- a/cylc/flow/xtriggers/wall_clock.py
+++ b/cylc/flow/xtriggers/wall_clock.py
@@ -75,4 +75,6 @@ def validate(args: Dict[str, Any]):
     try:
         interval_parse(args["offset"])
     except (ValueError, AttributeError):
-        raise WorkflowConfigError(f"Invalid offset: {args['offset']}")
+        raise WorkflowConfigError(
+            f"Invalid offset: {args['offset']}"
+        ) from None

From 2f43b4e1a5ea4483a960b7059509d8d06a4142f9 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Wed, 19 Jun 2024 17:17:45 +0100
Subject: [PATCH 04/28] lint: address PERF101

---
 cylc/flow/network/multi.py | 2 +-
 cylc/flow/scheduler.py     | 2 +-
 cylc/flow/task_pool.py     | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cylc/flow/network/multi.py b/cylc/flow/network/multi.py
index 2b9ea418976..9c190f68799 100644
--- a/cylc/flow/network/multi.py
+++ b/cylc/flow/network/multi.py
@@ -235,7 +235,7 @@ def _report(
     """
     try:
         ret: List[Tuple[Optional[str], Optional[str], bool]] = []
-        for _mutation_name, mutation_response in response.items():
+        for mutation_response in response.values():
             # extract the result of each mutation result in the response
             success, msg = mutation_response['result'][0]['response']
             out = None
diff --git a/cylc/flow/scheduler.py b/cylc/flow/scheduler.py
index ff593648e7b..48fe6df62d1 100644
--- a/cylc/flow/scheduler.py
+++ b/cylc/flow/scheduler.py
@@ -917,7 +917,7 @@ def process_queued_task_messages(self) -> None:
         # Remaining unprocessed messages have no corresponding task proxy.
         # For example, if I manually set a running task to succeeded, the
         # proxy can be removed, but the orphaned job still sends messages.
-        for _id, tms in messages.items():
+        for tms in messages.values():
             warn = "Undeliverable task messages received and ignored:"
             for _, msg in tms:
                 warn += f'\n  {msg.job_id}: {msg.severity} - "{msg.message}"'
diff --git a/cylc/flow/task_pool.py b/cylc/flow/task_pool.py
index 21893103f86..53f6f354eb5 100644
--- a/cylc/flow/task_pool.py
+++ b/cylc/flow/task_pool.py
@@ -870,8 +870,8 @@ def get_tasks(self) -> List[TaskProxy]:
         if self.active_tasks_changed:
             self.active_tasks_changed = False
             self._active_tasks_list = []
-            for _, itask_id_map in self.active_tasks.items():
-                for __, itask in itask_id_map.items():
+            for itask_id_map in self.active_tasks.values():
+                for itask in itask_id_map.values():
                     self._active_tasks_list.append(itask)
         return self._active_tasks_list
 

From b188a44045faeb64508e8c1398c91692f57d2b95 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Wed, 19 Jun 2024 17:23:35 +0100
Subject: [PATCH 05/28] lint: address PERF402

---
 cylc/flow/resources.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/cylc/flow/resources.py b/cylc/flow/resources.py
index d81b7f7211b..956ec27e8aa 100644
--- a/cylc/flow/resources.py
+++ b/cylc/flow/resources.py
@@ -18,7 +18,7 @@
 
 from contextlib import suppress
 from pathlib import Path
-from random import shuffle
+from random import choice
 import shutil
 import sys
 from typing import Optional
@@ -205,12 +205,9 @@ def get_api_key() -> str:
     load over a larger number of keys to prevent hitting the cap with group
     sessions.
     """
-    keys = []
     with open((TUTORIAL_DIR / 'api-keys'), 'r') as api_keys:
-        for api_key in api_keys:
-            keys.append(api_key)
-    shuffle(keys)
-    return keys[0].strip()
+        return choice(list(api_keys)).strip()  # nosec
+        # (the randomness of this choice is not a security concern)
 
 
 def set_api_key(tgt):

From e28e0c0a753f4d9fa11fa18fb60cff318eea9799 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Wed, 10 Jul 2024 09:56:16 +0100
Subject: [PATCH 06/28] Lint.job runner specific timeout (#6137)

* Add S014: Use Execution Time Limit in preference to directives.

* response to review

* remove wallclock limit from the directives example for SGE

* Apply suggestions from code review
---
 changes.d/6137.feat.md                       |  1 +
 cylc/flow/job_runner_handlers/loadleveler.py |  5 +-
 cylc/flow/job_runner_handlers/lsf.py         |  8 ++-
 cylc/flow/job_runner_handlers/moab.py        |  6 ++-
 cylc/flow/job_runner_handlers/pbs.py         | 10 ++--
 cylc/flow/job_runner_handlers/sge.py         |  6 +--
 cylc/flow/job_runner_handlers/slurm.py       |  6 ++-
 cylc/flow/scripts/lint.py                    | 55 ++++++++++++++++++++
 tests/unit/scripts/test_lint.py              |  2 +
 9 files changed, 85 insertions(+), 14 deletions(-)
 create mode 100644 changes.d/6137.feat.md

diff --git a/changes.d/6137.feat.md b/changes.d/6137.feat.md
new file mode 100644
index 00000000000..d947999de99
--- /dev/null
+++ b/changes.d/6137.feat.md
@@ -0,0 +1 @@
+New Cylc lint rule: S014: Don't use job runner specific execution time limit directives, use execution time limit.
\ No newline at end of file
diff --git a/cylc/flow/job_runner_handlers/loadleveler.py b/cylc/flow/job_runner_handlers/loadleveler.py
index ee8203b2b47..d097ff85faf 100644
--- a/cylc/flow/job_runner_handlers/loadleveler.py
+++ b/cylc/flow/job_runner_handlers/loadleveler.py
@@ -83,6 +83,7 @@ class LoadlevelerHandler():
         re.compile("^llsubmit: Processed command file through Submit Filter:")]
     SUBMIT_CMD_TMPL = "llsubmit '%(job)s'"
     VACATION_SIGNAL = "USR1"
+    TIME_LIMIT_DIRECTIVE = "wall_clock_limit"
 
     def format_directives(self, job_conf):
         """Format the job directives for a job file."""
@@ -96,8 +97,8 @@ def format_directives(self, job_conf):
         directives["output"] = job_file_path + ".out"
         directives["error"] = job_file_path + ".err"
         if (job_conf["execution_time_limit"] and
-                directives.get("wall_clock_limit") is None):
-            directives["wall_clock_limit"] = "%d,%d" % (
+                directives.get(self.TIME_LIMIT_DIRECTIVE) is None):
+            directives[self.TIME_LIMIT_DIRECTIVE] = "%d,%d" % (
                 job_conf["execution_time_limit"] + 60,
                 job_conf["execution_time_limit"])
         for key, value in list(job_conf["directives"].items()):
diff --git a/cylc/flow/job_runner_handlers/lsf.py b/cylc/flow/job_runner_handlers/lsf.py
index a465c9b7924..534d1205a9d 100644
--- a/cylc/flow/job_runner_handlers/lsf.py
+++ b/cylc/flow/job_runner_handlers/lsf.py
@@ -70,6 +70,7 @@ class LSFHandler():
     POLL_CMD = "bjobs"
     REC_ID_FROM_SUBMIT_OUT = re.compile(r"^Job <(?P<id>\d+)>")
     SUBMIT_CMD_TMPL = "bsub"
+    TIME_LIMIT_DIRECTIVE = "-W"
 
     @classmethod
     def format_directives(cls, job_conf):
@@ -82,8 +83,11 @@ def format_directives(cls, job_conf):
         )
         directives["-o"] = job_file_path + ".out"
         directives["-e"] = job_file_path + ".err"
-        if job_conf["execution_time_limit"] and directives.get("-W") is None:
-            directives["-W"] = str(math.ceil(
+        if (
+            job_conf["execution_time_limit"]
+            and directives.get(cls.TIME_LIMIT_DIRECTIVE) is None
+        ):
+            directives[cls.TIME_LIMIT_DIRECTIVE] = str(math.ceil(
                 job_conf["execution_time_limit"] / 60))
         for key, value in list(job_conf["directives"].items()):
             directives[key] = value
diff --git a/cylc/flow/job_runner_handlers/moab.py b/cylc/flow/job_runner_handlers/moab.py
index 839d246ccbc..ec068d48420 100644
--- a/cylc/flow/job_runner_handlers/moab.py
+++ b/cylc/flow/job_runner_handlers/moab.py
@@ -78,6 +78,7 @@ class MoabHandler:
     POLL_CMD = "checkjob"
     REC_ID_FROM_SUBMIT_OUT = re.compile(r"""\A\s*(?P<id>\S+)\s*\Z""")
     SUBMIT_CMD_TMPL = "msub '%(job)s'"
+    TIME_LIMIT_DIRECTIVE = "-l walltime"
 
     def format_directives(self, job_conf):
         """Format the job directives for a job file."""
@@ -91,8 +92,9 @@ def format_directives(self, job_conf):
         directives["-o"] = job_file_path + ".out"
         directives["-e"] = job_file_path + ".err"
         if (job_conf["execution_time_limit"] and
-                directives.get("-l walltime") is None):
-            directives["-l walltime"] = "%d" % job_conf["execution_time_limit"]
+                directives.get(self.TIME_LIMIT_DIRECTIVE) is None):
+            directives[self.TIME_LIMIT_DIRECTIVE] = "%d" % job_conf[
+                "execution_time_limit"]
         # restartable?
         directives.update(job_conf["directives"])
         lines = []
diff --git a/cylc/flow/job_runner_handlers/pbs.py b/cylc/flow/job_runner_handlers/pbs.py
index aa264311fc4..ac0d6c47a00 100644
--- a/cylc/flow/job_runner_handlers/pbs.py
+++ b/cylc/flow/job_runner_handlers/pbs.py
@@ -84,6 +84,7 @@ class PBSHandler:
     POLL_CANT_CONNECT_ERR = "Connection refused"
     REC_ID_FROM_SUBMIT_OUT = re.compile(r"^\s*(?P<id>\d+)", re.M)
     SUBMIT_CMD_TMPL = "qsub '%(job)s'"
+    TIME_LIMIT_DIRECTIVE = "-l walltime"
 
     def format_directives(self, job_conf):
         """Format the job directives for a job file."""
@@ -105,9 +106,12 @@ def format_directives(self, job_conf):
 
         directives["-o"] = job_file_path + ".out"
         directives["-e"] = job_file_path + ".err"
-        if (job_conf["execution_time_limit"] and
-                directives.get("-l walltime") is None):
-            directives["-l walltime"] = "%d" % job_conf["execution_time_limit"]
+        if (
+            job_conf["execution_time_limit"]
+            and directives.get(self.TIME_LIMIT_DIRECTIVE) is None
+        ):
+            directives[self.TIME_LIMIT_DIRECTIVE] = "%d" % job_conf[
+                "execution_time_limit"]
         for key, value in list(job_conf["directives"].items()):
             directives[key] = value
         lines = []
diff --git a/cylc/flow/job_runner_handlers/sge.py b/cylc/flow/job_runner_handlers/sge.py
index 33f7d5a26d7..c7c50956fb9 100644
--- a/cylc/flow/job_runner_handlers/sge.py
+++ b/cylc/flow/job_runner_handlers/sge.py
@@ -37,7 +37,6 @@
                -cwd =
                -q = foo
                -l h_data = 1024M
-               -l h_rt = 24:00:00
 
 These are written to the top of the job script like this:
 
@@ -76,6 +75,7 @@ class SGEHandler:
     POLL_CMD = "qstat"
     REC_ID_FROM_SUBMIT_OUT = re.compile(r"\D+(?P<id>\d+)\D+")
     SUBMIT_CMD_TMPL = "qsub '%(job)s'"
+    TIME_LIMIT_DIRECTIVE = "-l h_rt"
 
     def format_directives(self, job_conf):
         """Format the job directives for a job file."""
@@ -88,8 +88,8 @@ def format_directives(self, job_conf):
         directives['-o'] = job_file_path + ".out"
         directives['-e'] = job_file_path + ".err"
         if (job_conf["execution_time_limit"] and
-                directives.get("-l h_rt") is None):
-            directives["-l h_rt"] = "%d:%02d:%02d" % (
+                directives.get(self.TIME_LIMIT_DIRECTIVE) is None):
+            directives[self.TIME_LIMIT_DIRECTIVE] = "%d:%02d:%02d" % (
                 job_conf["execution_time_limit"] / 3600,
                 (job_conf["execution_time_limit"] / 60) % 60,
                 job_conf["execution_time_limit"] % 60)
diff --git a/cylc/flow/job_runner_handlers/slurm.py b/cylc/flow/job_runner_handlers/slurm.py
index 4ec6be20471..33df2ebc926 100644
--- a/cylc/flow/job_runner_handlers/slurm.py
+++ b/cylc/flow/job_runner_handlers/slurm.py
@@ -135,6 +135,8 @@ class SLURMHandler():
     #  Separator between het job directive sections
     SEP_HETJOB = "#SBATCH hetjob"
 
+    TIME_LIMIT_DIRECTIVE = "--time"
+
     @classmethod
     def filter_poll_many_output(cls, out):
         """Return list of job IDs extracted from job poll stdout.
@@ -161,8 +163,8 @@ def format_directives(cls, job_conf):
         directives['--output'] = job_file_path.replace('%', '%%') + ".out"
         directives['--error'] = job_file_path.replace('%', '%%') + ".err"
         if (job_conf["execution_time_limit"] and
-                directives.get("--time") is None):
-            directives["--time"] = "%d:%02d" % (
+                directives.get(cls.TIME_LIMIT_DIRECTIVE) is None):
+            directives[cls.TIME_LIMIT_DIRECTIVE] = "%d:%02d" % (
                 job_conf["execution_time_limit"] / 60,
                 job_conf["execution_time_limit"] % 60)
         for key, value in list(job_conf['directives'].items()):
diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py
index d045ab70838..1561ad743f3 100755
--- a/cylc/flow/scripts/lint.py
+++ b/cylc/flow/scripts/lint.py
@@ -50,6 +50,7 @@
    max-line-length = 130        # Max line length for linting
 """
 import functools
+import pkgutil
 from pathlib import Path
 import re
 import sys
@@ -78,6 +79,8 @@
 from cylc.flow import LOG
 from cylc.flow.exceptions import CylcError
 import cylc.flow.flags
+from cylc.flow import job_runner_handlers
+from cylc.flow.job_runner_mgr import JobRunnerManager
 from cylc.flow.loggingutil import set_timestamps
 from cylc.flow.option_parsers import (
     CylcOptionParser as COP,
@@ -162,6 +165,34 @@
 }
 
 
+def get_wallclock_directives():
+    """Get a set of directives equivalent to execution time limit"""
+    job_runner_manager = JobRunnerManager()
+    directives = {}
+    for module in pkgutil.iter_modules(job_runner_handlers.__path__):
+        directive = getattr(
+            job_runner_manager._get_sys(module.name),
+            'TIME_LIMIT_DIRECTIVE',
+            None
+        )
+        if directive:
+            directives[module.name] = directive
+    return directives
+
+
+WALLCLOCK_DIRECTIVES = get_wallclock_directives()
+
+
+def check_wallclock_directives(line: str) -> Union[Dict[str, str], bool]:
+    """Check for job runner specific directives
+    equivelent to exection time limit.
+    """
+    for directive in set(WALLCLOCK_DIRECTIVES.values()):
+        if line.strip().startswith(directive):
+            return {'directive': line.strip()}
+    return False
+
+
 def check_jinja2_no_shebang(
     line: str,
     file: Path,
@@ -533,6 +564,30 @@ def list_wrapper(line: str, check: Callable) -> Optional[Dict[str, str]]:
     'S013': {
         'short': 'Items should be indented in 4 space blocks.',
         FUNCTION: check_indentation
+    },
+    'S014': {
+        'short': (
+            'Use ``[runtime][TASK]execution time limit``'
+            ' rather than job runner directive: ``{directive}``.'
+        ),
+        'rst': (
+            'Using ``[runtime][TASK]execution time limit`` is'
+            ' recommended in preference to using job runner'
+            ' directives because it allows Cylc to retain awareness'
+            ' of whether the job should have finished, even if contact'
+            ' with the target job runner\'s platform has been lost.'
+            ' \n\nThe following directives are considered equivelent to'
+            ' execution time limit:\n * '
+        )
+        + '\n * '.join((
+            f'``{directive}`` ({job_runner})'
+            for job_runner, directive in WALLCLOCK_DIRECTIVES.items()
+        )) + (
+            '\n\n.. note:: Using ``execution time limit`` which'
+            ' is automatically translated to the job runner\'s timeout'
+            ' directive can make your workflow more portable.'
+        ),
+        FUNCTION: check_wallclock_directives,
     }
 }
 # Subset of deprecations which are tricky (impossible?) to scrape from the
diff --git a/tests/unit/scripts/test_lint.py b/tests/unit/scripts/test_lint.py
index 0f5f02648bd..527a33bdfb5 100644
--- a/tests/unit/scripts/test_lint.py
+++ b/tests/unit/scripts/test_lint.py
@@ -178,6 +178,8 @@
 something\t
     [[bar]]
         platform = $(some-script foo)
+            [[[directives]]]
+                -l walltime = 666
     [[baz]]
         platform = `no backticks`
 """ + (

From 5605265c97177fc19f975a4e8e9122d58c609972 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Wed, 10 Jul 2024 17:32:30 +0100
Subject: [PATCH 07/28] Correct changelog [skip ci]

---
 CHANGES.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 061d935a010..6ff82d9c920 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -15,8 +15,6 @@ $ towncrier create <PR-number>.<break|feat|fix>.md --content "Short description"
 
 ### 🔧 Fixes
 
-[#6178](https://github.com/cylc/cylc-flow/pull/6178) - Fix an issue where Tui could hang when closing.
-
 [#6186](https://github.com/cylc/cylc-flow/pull/6186) - Fixed bug where using flow numbers with `cylc set` would not work correctly.
 
 [#6200](https://github.com/cylc/cylc-flow/pull/6200) - Fixed bug where a stalled paused workflow would be incorrectly reported as running, not paused
@@ -33,6 +31,8 @@ $ towncrier create <PR-number>.<break|feat|fix>.md --content "Short description"
 
 [#6176](https://github.com/cylc/cylc-flow/pull/6176) - Fix bug where jobs which fail to submit are not shown in GUI/TUI if submission retries are set.
 
+[#6178](https://github.com/cylc/cylc-flow/pull/6178) - Fix an issue where Tui could hang when closing.
+
 ## __cylc-8.3.0 (Released 2024-06-18)__
 
 ### ⚠ Breaking Changes

From 497a74e769293f20ebd40619aa7818d1df14d6f3 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Tue, 30 Jan 2024 12:02:39 +0000
Subject: [PATCH 08/28] example: extending workflow

---
 .../etc/examples/extending-workflow/.validate | 80 +++++++++++++++++++
 .../etc/examples/extending-workflow/index.rst | 77 ++++++++++++++++++
 .../extending-workflow/simple/flow.cylc       | 24 ++++++
 3 files changed, 181 insertions(+)
 create mode 100755 cylc/flow/etc/examples/extending-workflow/.validate
 create mode 100644 cylc/flow/etc/examples/extending-workflow/index.rst
 create mode 100644 cylc/flow/etc/examples/extending-workflow/simple/flow.cylc

diff --git a/cylc/flow/etc/examples/extending-workflow/.validate b/cylc/flow/etc/examples/extending-workflow/.validate
new file mode 100755
index 00000000000..43c810372ce
--- /dev/null
+++ b/cylc/flow/etc/examples/extending-workflow/.validate
@@ -0,0 +1,80 @@
+#!/bin/bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -eux
+
+test_simple () {
+    local ID
+    ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)"
+    
+    # lint
+    cylc lint ./simple
+    
+    # copy into a temp directory
+    local SRC_DIR
+    SRC_DIR="$(mktemp -d)"
+    cp simple/flow.cylc "$SRC_DIR"
+    
+    # speed things up with simulation mode
+    cat >>"${SRC_DIR}/flow.cylc" <<__HERE__
+    [runtime]
+        [[root]]
+            [[[simulation]]]
+                default run length = PT0S
+__HERE__
+    
+    # start the workflow
+    cylc vip \
+        --check-circular \
+        --no-run-name \
+        --no-detach \
+        --workflow-name "$ID" \
+        --mode=simulation \
+        "$SRC_DIR"
+    
+    # it should have reached the 2002 cycle
+    grep '2002/a' "${HOME}/cylc-run/${ID}/log/scheduler/log"
+    if grep '2003/a' "${HOME}/cylc-run/${ID}/log/scheduler/log"; then
+        exit 1
+    fi
+    
+    # edit the "stop after cycle point"
+    sed -i \
+        's/stop after cycle point.*/stop after cycle point = 2004/' \
+        "${SRC_DIR}/flow.cylc"
+    
+    # continue the run
+    cylc vr \
+        --no-detach \
+        --mode=simulation \
+        --yes \
+        "$ID"
+    
+    # it should have reached the 2004 cycle
+    grep '2004/a' "${HOME}/cylc-run/${ID}/log/scheduler/log"
+    if grep '2005/a' "${HOME}/cylc-run/${ID}/log/scheduler/log"; then
+        exit 1
+    fi
+    
+    # clean up
+    cylc clean "$ID"
+    
+    rm -r "${SRC_DIR}"
+}
+
+
+test_simple
diff --git a/cylc/flow/etc/examples/extending-workflow/index.rst b/cylc/flow/etc/examples/extending-workflow/index.rst
new file mode 100644
index 00000000000..c5bc023dc11
--- /dev/null
+++ b/cylc/flow/etc/examples/extending-workflow/index.rst
@@ -0,0 +1,77 @@
+Extending Workflow
+==================
+
+.. cylc-scope:: flow.cylc[scheduling]
+
+Sometimes we may run a workflow to :term:`completion <workflow completion>`,
+but subsequently wish to run it for a few more cycles.
+
+With Cylc 7 this was often done by changing the `final cycle point` and
+restarting the workflow. This approach worked, but was a little awkward.
+It's possible with Cylc 8, but we would recommend moving away from this
+pattern instead.
+
+The recommended approach to this problem (Cylc 6+) is to use the
+`stop after cycle point` rather than the `final cycle point`.
+
+The `stop after cycle point` tells Cylc to **stop** after the workflow passes
+the specified point, whereas the `final cycle point` tells Cylc that the
+workflow **finishes** at the specified point.
+
+When a workflow **finishes**, it is a little awkward to restart as you have to
+tell Cylc which tasks to continue on from. The `stop after cycle point`
+solution avoids this issue.
+
+
+Example
+-------
+
+.. admonition:: Get a copy of this example
+   :class: hint
+
+   .. code-block:: console
+
+      $ cylc get-resources examples/extending-workflow/simple
+
+This workflow will stop at the end of the ``2002`` cycle:
+
+.. literalinclude:: simple/flow.cylc
+   :language: cylc
+
+After it has run and shut down, change the `stop after cycle point` to
+the desired value and restart it. E.g:
+
+.. code-block:: bash
+
+   # install and run the workflow:
+   cylc vip
+
+   # then later edit "stop after cycle point" to "2004"
+
+   # then reinstall and restart the workflow:
+   cylc vr
+
+The workflow will continue from where it left off and run until the end of the
+``2004`` cycle. Because the workflow never hit the `final cycle point` it
+never "finished" so no special steps are required to restart the workflow.
+
+You can also set the `stop after cycle point` when you start the workflow:
+
+.. code-block:: bash
+
+   cylc play --stop-cycle-point=2020 myworkflow
+
+Or change it at any point whilst the workflow is running:
+
+.. code-block:: bash
+
+   cylc stop myworkflow//2030  # change the stop after cycle point to 2030
+
+.. note::
+
+   If you set the `stop after cycle point` on the command line, this value will
+   take precedence over the one in the workflow configuration. Use
+   ``cylc play --stop-cycle-point=reload`` to restart the workflow using the
+   `stop after cycle point` configured in the workflow configuration.
+
+.. cylc-scope::
diff --git a/cylc/flow/etc/examples/extending-workflow/simple/flow.cylc b/cylc/flow/etc/examples/extending-workflow/simple/flow.cylc
new file mode 100644
index 00000000000..2de2f28f2ea
--- /dev/null
+++ b/cylc/flow/etc/examples/extending-workflow/simple/flow.cylc
@@ -0,0 +1,24 @@
+[meta]
+    title = "Basic extendable workflow"
+    description = """
+        Use the "stop after cycle point" rather than the "final cycle point"
+        to allow this workflow to be easily extended at a later date.
+    """
+
+[scheduler]
+    # use the year for the cycle point
+    # (strip off the month, day, hour and minute)
+    cycle point format = CCYY
+
+[scheduling]
+    initial cycle point = 2000
+    stop after cycle point = 2002  # stop after two years of simulated time
+    [[graph]]
+        P1Y = """
+            z[-P1Y] => a
+            a => z
+        """
+
+[runtime]
+    [[a]]
+    [[z]]

From f625c524b46e42058d466654f4efcc888f669ec4 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Tue, 16 Jul 2024 13:19:52 +0100
Subject: [PATCH 09/28] doc: add link to task comms section

---
 cylc/flow/cfgspec/globalcfg.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py
index db8c7bdd5ff..d6182f8f7ea 100644
--- a/cylc/flow/cfgspec/globalcfg.py
+++ b/cylc/flow/cfgspec/globalcfg.py
@@ -1311,7 +1311,7 @@ def default_for(
                 The means by which task progress messages are reported back to
                 the running workflow.
 
-                Options:
+                ..rubric:: Options:
 
                 zmq
                    Direct client-server TCP communication via network ports
@@ -1320,6 +1320,8 @@ def default_for(
                 ssh
                    Use non-interactive ssh for task communications
 
+                For more information, see :ref:`TaskComms`.
+
                 .. versionchanged:: 8.0.0
 
                    {REPLACES}``global.rc[hosts][<host>]task communication

From 3ff5b4e5731276dbffccb798b48cf12d41b5b270 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Tue, 25 Jun 2024 10:34:45 +0100
Subject: [PATCH 10/28] Allow the Symlinking for log/job

response to review: ensure cylc clean works.
---
 changes.d/feat.6168.md                      |  1 +
 cylc/flow/cfgspec/globalcfg.py              | 65 +++++----------------
 cylc/flow/pathutil.py                       | 10 +++-
 cylc/flow/workflow_files.py                 |  7 ++-
 tests/functional/cylc-clean/00-basic.t      |  6 +-
 tests/functional/cylc-clean/01-remote.t     |  1 +
 tests/functional/cylc-install/01-symlinks.t | 11 +++-
 tests/functional/remote/04-symlink-dirs.t   | 20 ++++++-
 tests/unit/test_pathutil.py                 |  2 +
 9 files changed, 68 insertions(+), 55 deletions(-)
 create mode 100644 changes.d/feat.6168.md

diff --git a/changes.d/feat.6168.md b/changes.d/feat.6168.md
new file mode 100644
index 00000000000..84ccc73b236
--- /dev/null
+++ b/changes.d/feat.6168.md
@@ -0,0 +1 @@
+Allow symlinking log/job separately from log
diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py
index d6182f8f7ea..94b0e347962 100644
--- a/cylc/flow/cfgspec/globalcfg.py
+++ b/cylc/flow/cfgspec/globalcfg.py
@@ -30,6 +30,7 @@
 from cylc.flow.exceptions import GlobalConfigError
 from cylc.flow.hostuserutil import get_user_home
 from cylc.flow.network.client_factory import CommsMeth
+from cylc.flow.pathutil import SYMLINKABLE_LOCATIONS
 from cylc.flow.parsec.config import (
     ConfigNode as Conf,
     ParsecConfig,
@@ -1148,55 +1149,21 @@ def default_for(
 
                     .. versionadded:: 8.0.0
                 """)
-                Conf('log', VDR.V_STRING, None, desc="""
-                    Alternative location for the log dir.
-
-                    If specified the workflow log directory will be created in
-                    ``<this-path>/cylc-run/<workflow-id>/log`` and a
-                    symbolic link will be created from
-                    ``$HOME/cylc-run/<workflow-id>/log``. If not specified
-                    the workflow log directory will be created in
-                    ``$HOME/cylc-run/<workflow-id>/log``.
-
-                    .. versionadded:: 8.0.0
-                """)
-                Conf('share', VDR.V_STRING, None, desc="""
-                    Alternative location for the share dir.
-
-                    If specified the workflow share directory will be
-                    created in ``<this-path>/cylc-run/<workflow-id>/share``
-                    and a symbolic link will be created from
-                    ``<$HOME/cylc-run/<workflow-id>/share``. If not specified
-                    the workflow share directory will be created in
-                    ``$HOME/cylc-run/<workflow-id>/share``.
-
-                    .. versionadded:: 8.0.0
-                """)
-                Conf('share/cycle', VDR.V_STRING, None, desc="""
-                    Alternative directory for the share/cycle dir.
-
-                    If specified the workflow share/cycle directory
-                    will be created in
-                    ``<this-path>/cylc-run/<workflow-id>/share/cycle``
-                    and a symbolic link will be created from
-                    ``$HOME/cylc-run/<workflow-id>/share/cycle``. If not
-                    specified the workflow share/cycle directory will be
-                    created in ``$HOME/cylc-run/<workflow-id>/share/cycle``.
-
-                    .. versionadded:: 8.0.0
-                """)
-                Conf('work', VDR.V_STRING, None, desc="""
-                    Alternative directory for the work dir.
-
-                    If specified the workflow work directory will be created in
-                    ``<this-path>/cylc-run/<workflow-id>/work`` and a
-                    symbolic link will be created from
-                    ``$HOME/cylc-run/<workflow-id>/work``. If not specified
-                    the workflow work directory will be created in
-                    ``$HOME/cylc-run/<workflow-id>/work``.
-
-                    .. versionadded:: 8.0.0
-                """)
+                for folder, versionadded in SYMLINKABLE_LOCATIONS.items():
+                    Conf(folder, VDR.V_STRING, None, desc=f"""
+                        Alternative location for the log dir.
+
+                        If specified the workflow {folder} directory will
+                        be created in
+                        ``<this-path>/cylc-run/<workflow-id>/{folder}``
+                        and a symbolic link will be created from
+                        ``$HOME/cylc-run/<workflow-id>/{folder}``.
+                        If not specified the workflow log directory will
+                        be created in
+                        ``$HOME/cylc-run/<workflow-id>/{folder}``.
+
+                        .. versionadded:: {versionadded}
+                    """)
     with Conf('platforms', desc='''
         Platforms allow you to define compute resources available at your
         site.
diff --git a/cylc/flow/pathutil.py b/cylc/flow/pathutil.py
index da3d6c93cde..a3c7f9d2b73 100644
--- a/cylc/flow/pathutil.py
+++ b/cylc/flow/pathutil.py
@@ -45,6 +45,14 @@
 """Matches relative paths that are explicit (starts with ./)"""
 
 SHELL_ENV_VARS = re.compile(r'\$[^$/]*')
+SYMLINKABLE_LOCATIONS: Dict[str, str] = {
+    # Location: Version Added
+    'log': '8.0.0',
+    'log/job': '8.4.0',
+    'share': '8.0.0',
+    'share/cycle': '8.0.0',
+    'work': '8.0.0'
+}
 
 
 def expand_path(*args: Union[Path, str]) -> str:
@@ -234,7 +242,7 @@ def get_dirs_to_symlink(
     if base_dir:
         dirs_to_symlink['run'] = os.path.join(
             base_dir, 'cylc-run', workflow_id)
-    for dir_ in ['log', 'share', 'share/cycle', 'work']:
+    for dir_ in SYMLINKABLE_LOCATIONS.keys():
         link = symlink_conf[install_target].get(dir_, None)
         if (not link) or link == base_dir:
             continue
diff --git a/cylc/flow/workflow_files.py b/cylc/flow/workflow_files.py
index ab390ec10aa..0aac6bc174a 100644
--- a/cylc/flow/workflow_files.py
+++ b/cylc/flow/workflow_files.py
@@ -203,6 +203,11 @@ class LogDir:
         DB = 'db'
         """The public database"""
 
+        JOB = 'job'
+        """The job log directory."""
+
+    LOG_JOB_DIR = os.path.join(LogDir.DIRNAME, LogDir.JOB)
+
     SHARE_DIR = 'share'
     """Workflow share directory."""
 
@@ -259,7 +264,7 @@ class Install:
     """Reserved filenames that cannot be used as run names."""
 
     SYMLINK_DIRS = frozenset([
-        SHARE_CYCLE_DIR, SHARE_DIR, LogDir.DIRNAME, WORK_DIR, ''
+        SHARE_CYCLE_DIR, SHARE_DIR, LogDir.DIRNAME, LOG_JOB_DIR, WORK_DIR, ''
     ])
     """The paths of the symlink dirs that may be set in
     global.cylc[install][symlink dirs], relative to the run dir
diff --git a/tests/functional/cylc-clean/00-basic.t b/tests/functional/cylc-clean/00-basic.t
index 1f4cec84984..7c880217aae 100644
--- a/tests/functional/cylc-clean/00-basic.t
+++ b/tests/functional/cylc-clean/00-basic.t
@@ -33,6 +33,7 @@ create_test_global_config "" "
         [[[localhost]]]
             run = ${TEST_DIR}/${SYM_NAME}/run
             log = ${TEST_DIR}/${SYM_NAME}/log
+            log/job = ${TEST_DIR}/${SYM_NAME}/log
             share = ${TEST_DIR}/${SYM_NAME}/share
             share/cycle = ${TEST_DIR}/${SYM_NAME}/cycle
             work = ${TEST_DIR}/${SYM_NAME}/work
@@ -68,8 +69,9 @@ ${TEST_DIR}/${SYM_NAME}/log/cylc-run/${CYLC_TEST_REG_BASE}
 |   \`-- cylc-clean
 |       \`-- ${TEST_NAME_BASE}
 |           \`-- log
-|               \`-- install
-|                   \`-- ${INSTALL_LOG_FILE}
+|               |-- install
+|               |   \`-- ${INSTALL_LOG_FILE}
+|               \`-- job
 \`-- leave-me-alone
 ${TEST_DIR}/${SYM_NAME}/run/cylc-run/${CYLC_TEST_REG_BASE}
 \`-- ${FUNCTIONAL_DIR}
diff --git a/tests/functional/cylc-clean/01-remote.t b/tests/functional/cylc-clean/01-remote.t
index 67148478eb0..db9669d91ec 100644
--- a/tests/functional/cylc-clean/01-remote.t
+++ b/tests/functional/cylc-clean/01-remote.t
@@ -37,6 +37,7 @@ create_test_global_config "" "
         [[[${CYLC_TEST_INSTALL_TARGET}]]]
             run = ${TEST_DIR}/${SYM_NAME}/run
             log = ${TEST_DIR}/${SYM_NAME}/other
+            log/job = ${TEST_DIR}/${SYM_NAME}/job
             share = ${TEST_DIR}/${SYM_NAME}/other
             share/cycle = ${TEST_DIR}/${SYM_NAME}/cycle
             work = ${TEST_DIR}/${SYM_NAME}/other
diff --git a/tests/functional/cylc-install/01-symlinks.t b/tests/functional/cylc-install/01-symlinks.t
index 8fbcc66c1ca..2f55b01d8b0 100644
--- a/tests/functional/cylc-install/01-symlinks.t
+++ b/tests/functional/cylc-install/01-symlinks.t
@@ -24,7 +24,7 @@ if [[ -z ${TMPDIR:-} || -z ${USER:-} || $TMPDIR/$USER == "$HOME" ]]; then
     skip_all '"TMPDIR" or "USER" not defined or "TMPDIR"/"USER" is "HOME"'
 fi
 
-set_test_number 30
+set_test_number 32
 
 create_test_global_config "" "
 [install]
@@ -33,6 +33,7 @@ create_test_global_config "" "
         run = \$TMPDIR/\$USER/test_cylc_symlink/cylctb_tmp_run_dir
         share = \$TMPDIR/\$USER/test_cylc_symlink/
         log = \$TMPDIR/\$USER/test_cylc_symlink/
+        log/job = \$TMPDIR/\$USER/test_cylc_symlink/job_log_dir
         share/cycle = \$TMPDIR/\$USER/test_cylc_symlink/cylctb_tmp_share_dir
         work = \$TMPDIR/\$USER/test_cylc_symlink/
 "
@@ -53,6 +54,10 @@ TEST_SYM="${TEST_NAME_BASE}-share-cycle-glblcfg"
 run_ok "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/share/cycle")" \
     = "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_share_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/share/cycle"
 
+TEST_SYM="${TEST_NAME_BASE}-log-job-glblcfg"
+run_ok "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/log/job")" \
+    = "$TMPDIR/${USER}/test_cylc_symlink/job_log_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/log/job"
+
 for DIR in 'work' 'share' 'log'; do
     TEST_SYM="${TEST_NAME_BASE}-${DIR}-glbcfg"
     run_ok "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/${DIR}")" \
@@ -137,6 +142,10 @@ TEST_SYM="${TEST_NAME}-share-cycle"
 run_fail "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/share/cycle")" \
     = "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_share_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/share/cycle"
 
+TEST_SYM="${TEST_NAME}-log-job"
+run_fail "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/log/job")" \
+    = "$TMPDIR/${USER}/test_cylc_symlink/job_log_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/log/job"
+
 for DIR in 'work' 'share' 'log'; do
     TEST_SYM="${TEST_NAME}-${DIR}"
     run_fail "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/${DIR}")" \
diff --git a/tests/functional/remote/04-symlink-dirs.t b/tests/functional/remote/04-symlink-dirs.t
index 28ab6049694..1c19f90c510 100644
--- a/tests/functional/remote/04-symlink-dirs.t
+++ b/tests/functional/remote/04-symlink-dirs.t
@@ -24,7 +24,7 @@ if [[ -z ${TMPDIR:-} || -z ${USER:-} || $TMPDIR/$USER == "$HOME" ]]; then
     skip_all '"TMPDIR" or "USER" not defined or "TMPDIR"/"USER" is "HOME"'
 fi
 
-set_test_number 12
+set_test_number 14
 
 create_test_global_config "" "
 [install]
@@ -33,12 +33,14 @@ create_test_global_config "" "
             run = \$TMPDIR/\$USER/cylctb_tmp_run_dir
             share = \$TMPDIR/\$USER
             log = \$TMPDIR/\$USER
+            log/job = \$TMPDIR/\$USER/cylctb_tmp_log_job_dir
             share/cycle = \$TMPDIR/\$USER/cylctb_tmp_share_dir
             work = \$TMPDIR/\$USER
         [[[$CYLC_TEST_INSTALL_TARGET]]]
             run = \$TMPDIR/\$USER/test_cylc_symlink/ctb_tmp_run_dir
             share = \$TMPDIR/\$USER/test_cylc_symlink/
             log = \$TMPDIR/\$USER/test_cylc_symlink/
+            log/job = \$TMPDIR/\$USER/cylctb_tmp_log_job_dir
             share/cycle = \$TMPDIR/\$USER/test_cylc_symlink/ctb_tmp_share_dir
             work = \$TMPDIR/\$USER/test_cylc_symlink/
 "
@@ -66,6 +68,14 @@ else
     fail "$TEST_SYM.localhost"
 fi
 
+TEST_SYM="${TEST_NAME_BASE}-log/job-symlink-exists-ok"
+if [[ $(readlink "$HOME/cylc-run/${WORKFLOW_NAME}/log/job") == \
+"$TMPDIR/$USER/cylctb_tmp_log_job_dir/cylc-run/${WORKFLOW_NAME}/log/job" ]]; then
+    ok "$TEST_SYM.localhost"
+else
+    fail "$TEST_SYM.localhost"
+fi
+
 for DIR in 'work' 'share' 'log'; do
     TEST_SYM="${TEST_NAME_BASE}-${DIR}-symlink-exists-ok"
     if [[ $(readlink "$HOME/cylc-run/${WORKFLOW_NAME}/${DIR}") == \
@@ -94,6 +104,14 @@ else
     fail "${TEST_NAME_BASE}-share/cycle-symlink-exists-ok.remotehost"
 fi
 
+# shellcheck disable=SC2016
+LINK="$(${SSH} "${CYLC_TEST_HOST}" 'readlink "$HOME/cylc-run/'"$WORKFLOW_NAME"/log/job'"')"
+if [[ "$LINK" == *"/test_cylc_symlink/cylctb_tmp_log_job_dir/cylc-run/${WORKFLOW_NAME}/log/job" ]]; then
+    ok "${TEST_NAME_BASE}-log/job-symlink-exists-ok.remotehost"
+else
+    fail "${TEST_NAME_BASE}-log/job-symlink-exists-ok.remotehost"
+fi
+
 for DIR in 'work' 'share' 'log'; do
 # shellcheck disable=SC2016
     LINK="$(${SSH} "${CYLC_TEST_HOST}" 'readlink "$HOME/cylc-run/'"$WORKFLOW_NAME"/$DIR'"')"
diff --git a/tests/unit/test_pathutil.py b/tests/unit/test_pathutil.py
index 6ed9e9ec120..24cc3999400 100644
--- a/tests/unit/test_pathutil.py
+++ b/tests/unit/test_pathutil.py
@@ -203,6 +203,7 @@ def test_make_workflow_run_tree(
                     run = $DEE
                     work = $DAH
                     log = $DUH
+                    log/job =$RAY
                     share = $DOH
                     share/cycle = $DAH
             ''',
@@ -210,6 +211,7 @@ def test_make_workflow_run_tree(
                 'run': '$DEE/cylc-run/morpheus',
                 'work': '$DAH/cylc-run/morpheus/work',
                 'log': '$DUH/cylc-run/morpheus/log',
+                'log/job': '$RAY/cylc-run/morpheus/log/job',
                 'share': '$DOH/cylc-run/morpheus/share',
                 'share/cycle': '$DAH/cylc-run/morpheus/share/cycle'
             },

From 44e846a8113b883fcb7212aaea4eff486b3621c5 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Thu, 27 Jun 2024 11:36:21 +0100
Subject: [PATCH 11/28] Provided a single source of truth for symlinkable dirs

Ensure that test actually creates symlinks to be cleaned.
---
 cylc/flow/pathutil.py                     |  8 ++++----
 cylc/flow/workflow_files.py               |  5 ++---
 tests/functional/cylc-clean/00-basic.t    | 14 ++++++++++++--
 tests/functional/cylc-clean/01-remote.t   | 16 +++++++++++-----
 tests/functional/remote/04-symlink-dirs.t |  2 +-
 5 files changed, 30 insertions(+), 15 deletions(-)

diff --git a/cylc/flow/pathutil.py b/cylc/flow/pathutil.py
index a3c7f9d2b73..5311c0ea6ac 100644
--- a/cylc/flow/pathutil.py
+++ b/cylc/flow/pathutil.py
@@ -47,11 +47,11 @@
 SHELL_ENV_VARS = re.compile(r'\$[^$/]*')
 SYMLINKABLE_LOCATIONS: Dict[str, str] = {
     # Location: Version Added
-    'log': '8.0.0',
-    'log/job': '8.4.0',
     'share': '8.0.0',
     'share/cycle': '8.0.0',
-    'work': '8.0.0'
+    'log': '8.0.0',
+    'log/job': '8.4.0',
+    'work': '8.0.0',
 }
 
 
@@ -242,7 +242,7 @@ def get_dirs_to_symlink(
     if base_dir:
         dirs_to_symlink['run'] = os.path.join(
             base_dir, 'cylc-run', workflow_id)
-    for dir_ in SYMLINKABLE_LOCATIONS.keys():
+    for dir_ in SYMLINKABLE_LOCATIONS:
         link = symlink_conf[install_target].get(dir_, None)
         if (not link) or link == base_dir:
             continue
diff --git a/cylc/flow/workflow_files.py b/cylc/flow/workflow_files.py
index 0aac6bc174a..95626125f89 100644
--- a/cylc/flow/workflow_files.py
+++ b/cylc/flow/workflow_files.py
@@ -59,6 +59,7 @@
     is_remote_host,
 )
 from cylc.flow.pathutil import (
+    SYMLINKABLE_LOCATIONS,
     expand_path,
     get_cylc_run_dir,
     get_workflow_run_dir,
@@ -263,9 +264,7 @@ class Install:
     RESERVED_NAMES = frozenset([FLOW_FILE, SUITE_RC, *RESERVED_DIRNAMES])
     """Reserved filenames that cannot be used as run names."""
 
-    SYMLINK_DIRS = frozenset([
-        SHARE_CYCLE_DIR, SHARE_DIR, LogDir.DIRNAME, LOG_JOB_DIR, WORK_DIR, ''
-    ])
+    SYMLINK_DIRS = frozenset(list(SYMLINKABLE_LOCATIONS) + [''])
     """The paths of the symlink dirs that may be set in
     global.cylc[install][symlink dirs], relative to the run dir
     ('' represents the run dir)."""
diff --git a/tests/functional/cylc-clean/00-basic.t b/tests/functional/cylc-clean/00-basic.t
index 7c880217aae..8fffd672e26 100644
--- a/tests/functional/cylc-clean/00-basic.t
+++ b/tests/functional/cylc-clean/00-basic.t
@@ -33,7 +33,7 @@ create_test_global_config "" "
         [[[localhost]]]
             run = ${TEST_DIR}/${SYM_NAME}/run
             log = ${TEST_DIR}/${SYM_NAME}/log
-            log/job = ${TEST_DIR}/${SYM_NAME}/log
+            log/job = ${TEST_DIR}/${SYM_NAME}/job
             share = ${TEST_DIR}/${SYM_NAME}/share
             share/cycle = ${TEST_DIR}/${SYM_NAME}/cycle
             work = ${TEST_DIR}/${SYM_NAME}/work
@@ -56,6 +56,10 @@ cmp_ok "${TEST_NAME}.stdout" <<< "${TEST_DIR}/${SYM_NAME}/run/cylc-run/${WORKFLO
 INSTALL_LOG_FILE=$(ls "${TEST_DIR}/${SYM_NAME}/log/cylc-run/${WORKFLOW_NAME}/log/install")
 TEST_NAME="test-dir-tree-pre-clean"
 run_ok "${TEST_NAME}" tree --noreport --charset=ascii "${TEST_DIR}/${SYM_NAME}/"*"/cylc-run/${CYLC_TEST_REG_BASE}"
+# cat ${TEST_NAME}.stdout >&2
+# echo "${CYLC_TEST_REG_BASE}" >&2
+# echo ${TEST_DIR}/${SYM_NAME} >&2
+# sleep 600
 # Note: backticks need to be escaped in the heredoc
 cmp_ok "${TEST_NAME}.stdout" << __TREE__
 ${TEST_DIR}/${SYM_NAME}/cycle/cylc-run/${CYLC_TEST_REG_BASE}
@@ -64,6 +68,12 @@ ${TEST_DIR}/${SYM_NAME}/cycle/cylc-run/${CYLC_TEST_REG_BASE}
         \`-- ${TEST_NAME_BASE}
             \`-- share
                 \`-- cycle
+${TEST_DIR}/${SYM_NAME}/job/cylc-run/${CYLC_TEST_REG_BASE}
+\`-- ${FUNCTIONAL_DIR}
+    \`-- cylc-clean
+        \`-- ${TEST_NAME_BASE}
+            \`-- log
+                \`-- job
 ${TEST_DIR}/${SYM_NAME}/log/cylc-run/${CYLC_TEST_REG_BASE}
 |-- ${FUNCTIONAL_DIR}
 |   \`-- cylc-clean
@@ -71,7 +81,7 @@ ${TEST_DIR}/${SYM_NAME}/log/cylc-run/${CYLC_TEST_REG_BASE}
 |           \`-- log
 |               |-- install
 |               |   \`-- ${INSTALL_LOG_FILE}
-|               \`-- job
+|               \`-- job -> ${TEST_DIR}/${SYM_NAME}/job/cylc-run/${WORKFLOW_NAME}/log/job
 \`-- leave-me-alone
 ${TEST_DIR}/${SYM_NAME}/run/cylc-run/${CYLC_TEST_REG_BASE}
 \`-- ${FUNCTIONAL_DIR}
diff --git a/tests/functional/cylc-clean/01-remote.t b/tests/functional/cylc-clean/01-remote.t
index db9669d91ec..330fc6c69ff 100644
--- a/tests/functional/cylc-clean/01-remote.t
+++ b/tests/functional/cylc-clean/01-remote.t
@@ -82,16 +82,22 @@ ${TEST_DIR}/${SYM_NAME}/cycle/cylc-run/${CYLC_TEST_REG_BASE}
 |           \`-- share
 |               \`-- cycle
 \`-- leave-me-alone
+${TEST_DIR}/${SYM_NAME}/job/cylc-run/${CYLC_TEST_REG_BASE}
+\`-- ${FUNCTIONAL_DIR}
+    \`-- cylc-clean
+        \`-- ${TEST_NAME_BASE}
+            \`-- log
+                \`-- job
+                    \`-- 1
+                        \`-- santa
+                            |-- 01
+                            \`-- NN -> 01
 ${TEST_DIR}/${SYM_NAME}/other/cylc-run/${CYLC_TEST_REG_BASE}
 \`-- ${FUNCTIONAL_DIR}
     \`-- cylc-clean
         \`-- ${TEST_NAME_BASE}
             |-- log
-            |   \`-- job
-            |       \`-- 1
-            |           \`-- santa
-            |               |-- 01
-            |               \`-- NN -> 01
+            |   \`-- job -> ${TEST_DIR}/${SYM_NAME}/job/cylc-run/${WORKFLOW_NAME}/log/job
             |-- share
             |   \`-- cycle -> ${TEST_DIR}/${SYM_NAME}/cycle/cylc-run/${WORKFLOW_NAME}/share/cycle
             \`-- work
diff --git a/tests/functional/remote/04-symlink-dirs.t b/tests/functional/remote/04-symlink-dirs.t
index 1c19f90c510..720c2f0fa34 100644
--- a/tests/functional/remote/04-symlink-dirs.t
+++ b/tests/functional/remote/04-symlink-dirs.t
@@ -106,7 +106,7 @@ fi
 
 # shellcheck disable=SC2016
 LINK="$(${SSH} "${CYLC_TEST_HOST}" 'readlink "$HOME/cylc-run/'"$WORKFLOW_NAME"/log/job'"')"
-if [[ "$LINK" == *"/test_cylc_symlink/cylctb_tmp_log_job_dir/cylc-run/${WORKFLOW_NAME}/log/job" ]]; then
+if [[ "$LINK" == *"/cylctb_tmp_log_job_dir/cylc-run/${WORKFLOW_NAME}/log/job" ]]; then
     ok "${TEST_NAME_BASE}-log/job-symlink-exists-ok.remotehost"
 else
     fail "${TEST_NAME_BASE}-log/job-symlink-exists-ok.remotehost"

From 152e794343193fe114de5699c06a58df40dc77a9 Mon Sep 17 00:00:00 2001
From: Hilary James Oliver <hilary.j.oliver@gmail.com>
Date: Thu, 1 Aug 2024 11:47:11 +1200
Subject: [PATCH 12/28] Update tests/functional/cylc-clean/00-basic.t [skip ci]

Co-authored-by: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
---
 tests/functional/cylc-clean/00-basic.t | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tests/functional/cylc-clean/00-basic.t b/tests/functional/cylc-clean/00-basic.t
index 8fffd672e26..72106b87198 100644
--- a/tests/functional/cylc-clean/00-basic.t
+++ b/tests/functional/cylc-clean/00-basic.t
@@ -56,10 +56,6 @@ cmp_ok "${TEST_NAME}.stdout" <<< "${TEST_DIR}/${SYM_NAME}/run/cylc-run/${WORKFLO
 INSTALL_LOG_FILE=$(ls "${TEST_DIR}/${SYM_NAME}/log/cylc-run/${WORKFLOW_NAME}/log/install")
 TEST_NAME="test-dir-tree-pre-clean"
 run_ok "${TEST_NAME}" tree --noreport --charset=ascii "${TEST_DIR}/${SYM_NAME}/"*"/cylc-run/${CYLC_TEST_REG_BASE}"
-# cat ${TEST_NAME}.stdout >&2
-# echo "${CYLC_TEST_REG_BASE}" >&2
-# echo ${TEST_DIR}/${SYM_NAME} >&2
-# sleep 600
 # Note: backticks need to be escaped in the heredoc
 cmp_ok "${TEST_NAME}.stdout" << __TREE__
 ${TEST_DIR}/${SYM_NAME}/cycle/cylc-run/${CYLC_TEST_REG_BASE}

From 7dc77e9d068fa9ab670f9b1abed2ad2fde42cbda Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Thu, 1 Aug 2024 16:24:40 +0100
Subject: [PATCH 13/28] Fix changelog fragment name

---
 changes.d/{feat.6168.md => 6168.feat.md} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename changes.d/{feat.6168.md => 6168.feat.md} (100%)

diff --git a/changes.d/feat.6168.md b/changes.d/6168.feat.md
similarity index 100%
rename from changes.d/feat.6168.md
rename to changes.d/6168.feat.md

From 6976854e1e643f3863c393ab21af864153e9fe30 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Tue, 6 Aug 2024 14:36:19 +0100
Subject: [PATCH 14/28] Remove cached prerequisite satisfaction

This manually cached property has the possibility to drift out of sync with individual `Prerequisite` satisfaction states, especially with `cylc remove` work in progress
---
 cylc/flow/task_state.py | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/cylc/flow/task_state.py b/cylc/flow/task_state.py
index ebb3dbc985b..c26b84c0414 100644
--- a/cylc/flow/task_state.py
+++ b/cylc/flow/task_state.py
@@ -195,9 +195,6 @@ class TaskState:
             Time string of latest update time.
         .xtriggers (dict):
             xtriggers as {trigger (str): satisfied (boolean), ...}.
-        ._is_satisfied (boolean):
-            Are prerequisites satisfied?
-            Reset None to force re-evaluation when a prereq gets satisfied.
         ._suicide_is_satisfied (boolean):
             Are prerequisites to trigger suicide satisfied?
             Reset None to force re-evaluation when a prereq gets satisfied.
@@ -217,7 +214,6 @@ class TaskState:
         "suicide_prerequisites",
         "time_updated",
         "xtriggers",
-        "_is_satisfied",
         "_suicide_is_satisfied",
     ]
 
@@ -229,7 +225,6 @@ def __init__(self, tdef, point, status, is_held):
         self.is_updated = False
         self.time_updated = None
 
-        self._is_satisfied = None
         self._suicide_is_satisfied = None
 
         # Prerequisites.
@@ -324,7 +319,6 @@ def satisfy_me(
             if yep:
                 valid = valid.union(yep)
                 continue
-            self._is_satisfied = None
             self._suicide_is_satisfied = None
         return valid
 
@@ -340,14 +334,10 @@ def set_all_satisfied(self):
         """Set all my prerequisites satisfied."""
         for p in self.prerequisites:
             p.set_satisfied()
-        self._is_satisfied = True
 
     def prerequisites_all_satisfied(self):
         """Return True if (non-suicide) prerequisites are fully satisfied."""
-        if self._is_satisfied is None:
-            self._is_satisfied = all(
-                preq.is_satisfied() for preq in self.prerequisites)
-        return self._is_satisfied
+        return all(preq.is_satisfied() for preq in self.prerequisites)
 
     def prerequisites_are_not_all_satisfied(self):
         """Return True if (any) prerequisites are not fully satisfied."""
@@ -380,7 +370,6 @@ def set_prerequisites_all_satisfied(self):
         """Set prerequisites to all satisfied."""
         for prereq in self.prerequisites:
             prereq.set_satisfied()
-        self._is_satisfied = None
 
     def get_resolved_dependencies(self):
         """Return a list of dependencies which have been met for this task.
@@ -467,7 +456,6 @@ def _add_prerequisites(self, point, tdef):
         """Add task prerequisites."""
         # Triggers for sequence_i only used if my cycle point is a
         # valid member of sequence_i's sequence of cycle points.
-        self._is_satisfied = None
         self._suicide_is_satisfied = None
 
         # Use dicts to avoid generating duplicate prerequisites from sequences

From f658e9351f77b2fee39ea08cdce3e0c309f0ee13 Mon Sep 17 00:00:00 2001
From: Paul Earnshaw <pdearnshaw+github@gmail.com>
Date: Tue, 6 Aug 2024 15:40:02 +0100
Subject: [PATCH 15/28] Made jinja2 subsitution regex less greedy

---
 cylc/flow/etc/syntax/cylc.lang | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cylc/flow/etc/syntax/cylc.lang b/cylc/flow/etc/syntax/cylc.lang
index 0270c7f3cfc..c3f43da2c95 100644
--- a/cylc/flow/etc/syntax/cylc.lang
+++ b/cylc/flow/etc/syntax/cylc.lang
@@ -219,7 +219,7 @@
       <end>#\}</end>
     </context>
     <context id="jinja2-substitution" style-ref="template">
-      <match>\{\{.*\}\}</match>
+      <match>\{\{.*?\}\}</match>
     </context>
     <context id="jinja2-block" style-ref="template" end-at-line-end="false">
       <start>\{%</start>

From 7d282bce2d61b8595e7e08569739a0b282df3f7d Mon Sep 17 00:00:00 2001
From: Paul Earnshaw <pdearnshaw+github@gmail.com>
Date: Tue, 6 Aug 2024 15:46:42 +0100
Subject: [PATCH 16/28] Added name to CONTRIBUTING.md

---
 CONTRIBUTING.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 30e81f26f93..adc342dcab5 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -95,6 +95,7 @@ requests_).
  - Mark Dawson
  - Diquan Jabbour
  - Shixian Sheng
+ - Paul Earnshaw
 <!-- end-shortlog -->
 
 (All contributors are identifiable with email addresses in the git version

From 2c2839e79a117913e7c9853166511f0d9187d877 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Tue, 6 Aug 2024 15:54:51 +0100
Subject: [PATCH 17/28] Remove cached suicide prerequisite satisfaction

---
 cylc/flow/task_state.py | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/cylc/flow/task_state.py b/cylc/flow/task_state.py
index c26b84c0414..06ee8ae6e97 100644
--- a/cylc/flow/task_state.py
+++ b/cylc/flow/task_state.py
@@ -195,9 +195,6 @@ class TaskState:
             Time string of latest update time.
         .xtriggers (dict):
             xtriggers as {trigger (str): satisfied (boolean), ...}.
-        ._suicide_is_satisfied (boolean):
-            Are prerequisites to trigger suicide satisfied?
-            Reset None to force re-evaluation when a prereq gets satisfied.
     """
 
     # Memory optimization - constrain possible attributes to this list.
@@ -214,7 +211,6 @@ class TaskState:
         "suicide_prerequisites",
         "time_updated",
         "xtriggers",
-        "_suicide_is_satisfied",
     ]
 
     def __init__(self, tdef, point, status, is_held):
@@ -225,8 +221,6 @@ def __init__(self, tdef, point, status, is_held):
         self.is_updated = False
         self.time_updated = None
 
-        self._suicide_is_satisfied = None
-
         # Prerequisites.
         self.prerequisites: List[Prerequisite] = []
         self.suicide_prerequisites: List[Prerequisite] = []
@@ -319,7 +313,6 @@ def satisfy_me(
             if yep:
                 valid = valid.union(yep)
                 continue
-            self._suicide_is_satisfied = None
         return valid
 
     def xtriggers_all_satisfied(self):
@@ -346,10 +339,7 @@ def prerequisites_are_not_all_satisfied(self):
 
     def suicide_prerequisites_all_satisfied(self):
         """Return True if all suicide prerequisites are satisfied."""
-        if self._suicide_is_satisfied is None:
-            self._suicide_is_satisfied = all(
-                preq.is_satisfied() for preq in self.suicide_prerequisites)
-        return self._suicide_is_satisfied
+        return all(preq.is_satisfied() for preq in self.suicide_prerequisites)
 
     def prerequisites_get_target_points(self):
         """Return a list of cycle points targeted by each prerequisite."""
@@ -456,7 +446,6 @@ def _add_prerequisites(self, point, tdef):
         """Add task prerequisites."""
         # Triggers for sequence_i only used if my cycle point is a
         # valid member of sequence_i's sequence of cycle points.
-        self._suicide_is_satisfied = None
 
         # Use dicts to avoid generating duplicate prerequisites from sequences
         # with coincident cycle points.

From 80fa008e18a6feb8ecb21123bde16c048fd1b599 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Tue, 6 Aug 2024 16:08:28 +0100
Subject: [PATCH 18/28] Tidy

---
 cylc/flow/task_pool.py  |  2 +-
 cylc/flow/task_state.py | 21 +++++++++------------
 2 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/cylc/flow/task_pool.py b/cylc/flow/task_pool.py
index 78a9d0c6f9f..72e8065d4ca 100644
--- a/cylc/flow/task_pool.py
+++ b/cylc/flow/task_pool.py
@@ -1990,7 +1990,7 @@ def _set_prereqs_itask(
 
         """
         if prereqs == ["all"]:
-            itask.state.set_all_satisfied()
+            itask.state.set_prerequisites_all_satisfied()
         else:
             # Attempt to set the given presrequisites.
             # Log any that aren't valid for the task.
diff --git a/cylc/flow/task_state.py b/cylc/flow/task_state.py
index 06ee8ae6e97..a4adb53d374 100644
--- a/cylc/flow/task_state.py
+++ b/cylc/flow/task_state.py
@@ -309,10 +309,9 @@ def satisfy_me(
         """
         valid: Set[Tokens] = set()
         for prereq in (*self.prerequisites, *self.suicide_prerequisites):
-            yep = prereq.satisfy_me(outputs)
-            if yep:
-                valid = valid.union(yep)
-                continue
+            valid.update(
+                prereq.satisfy_me(outputs)
+            )
         return valid
 
     def xtriggers_all_satisfied(self):
@@ -323,11 +322,6 @@ def external_triggers_all_satisfied(self):
         """Return True if all external triggers are satisfied."""
         return all(self.external_triggers.values())
 
-    def set_all_satisfied(self):
-        """Set all my prerequisites satisfied."""
-        for p in self.prerequisites:
-            p.set_satisfied()
-
     def prerequisites_all_satisfied(self):
         """Return True if (non-suicide) prerequisites are fully satisfied."""
         return all(preq.is_satisfied() for preq in self.prerequisites)
@@ -349,9 +343,12 @@ def prerequisites_get_target_points(self):
             for point in prerequisite.get_target_points()
         }
 
-    def prerequisites_eval_all(self):
-        """Set all prerequisites to satisfied."""
-        # (Validation: will abort on illegal trigger expressions.)
+    def prerequisites_eval_all(self) -> None:
+        """Evaluate satisifaction of all prerequisites and
+        suicide prerequisites.
+
+        Provides validation - will abort on illegal trigger expressions.
+        """
         for preqs in [self.prerequisites, self.suicide_prerequisites]:
             for preq in preqs:
                 preq.is_satisfied()

From be934fd8d67611f2673b4dad5103534652dca096 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Thu, 8 Aug 2024 12:24:46 +0100
Subject: [PATCH 19/28] Fix small bug in logging force-satisfied prereqs

---
 cylc/flow/prerequisite.py       |  2 +-
 tests/unit/test_prerequisite.py | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/cylc/flow/prerequisite.py b/cylc/flow/prerequisite.py
index 449a5009656..915397ee224 100644
--- a/cylc/flow/prerequisite.py
+++ b/cylc/flow/prerequisite.py
@@ -295,4 +295,4 @@ def get_resolved_dependencies(self):
         """
         return [f'{point}/{name}' for
                 (point, name, _), satisfied in self.satisfied.items() if
-                satisfied == self.DEP_STATE_SATISFIED]
+                satisfied]
diff --git a/tests/unit/test_prerequisite.py b/tests/unit/test_prerequisite.py
index 410176e508c..1514ad07f66 100644
--- a/tests/unit/test_prerequisite.py
+++ b/tests/unit/test_prerequisite.py
@@ -16,6 +16,7 @@
 
 import pytest
 
+from cylc.flow.cycling.integer import IntegerPoint
 from cylc.flow.cycling.loader import ISO8601_CYCLING_TYPE, get_point
 from cylc.flow.prerequisite import Prerequisite
 from cylc.flow.id import Tokens
@@ -102,3 +103,16 @@ def test_get_target_points(prereq):
         get_point('2000'),
         get_point('2001'),
     }
+
+
+def test_get_resolved_dependencies():
+    prereq = Prerequisite(IntegerPoint('2'))
+    prereq.satisfied[('1', 'a', 'x')] = 'satisfied naturally'
+    prereq.satisfied[('1', 'b', 'x')] = False
+    prereq.satisfied[('1', 'c', 'x')] = 'satisfied from database'
+    prereq.satisfied[('1', 'd', 'x')] = 'force satisfied'
+    assert prereq.get_resolved_dependencies() == [
+        '1/a',
+        '1/c',
+        '1/d',
+    ]

From 8c337c83daa9739d4220d48e5beb702855755232 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Thu, 8 Aug 2024 12:19:21 +0100
Subject: [PATCH 20/28] Rework `Prerequisite` class to ensure caching of
 satisfaction state is handled automatically

---
 cylc/flow/data_store_mgr.py              |  17 +-
 cylc/flow/prerequisite.py                | 222 ++++++++++++++---------
 cylc/flow/task_pool.py                   |  21 ++-
 cylc/flow/task_proxy.py                  |  35 ++--
 cylc/flow/task_state.py                  |  40 ++--
 cylc/flow/task_trigger.py                |  52 +++---
 cylc/flow/taskdef.py                     |  14 +-
 cylc/flow/workflow_db_mgr.py             |   2 +-
 tests/integration/scripts/test_set.py    |   8 +-
 tests/integration/test_data_store_mgr.py |  38 ++--
 tests/integration/test_task_pool.py      |  18 +-
 tests/unit/test_prerequisite.py          |  97 ++++++----
 tests/unit/test_task_state.py            |   3 +-
 13 files changed, 342 insertions(+), 225 deletions(-)

diff --git a/cylc/flow/data_store_mgr.py b/cylc/flow/data_store_mgr.py
index b98a055f882..459abf77e56 100644
--- a/cylc/flow/data_store_mgr.py
+++ b/cylc/flow/data_store_mgr.py
@@ -116,6 +116,7 @@
 if TYPE_CHECKING:
     from cylc.flow.cycling import PointBase
     from cylc.flow.flow_mgr import FlowNums
+    from cylc.flow.prerequisite import Prerequisite
 
 EDGES = 'edges'
 FAMILIES = 'families'
@@ -1444,7 +1445,7 @@ def apply_task_proxy_db_history(self):
             prereq_ids.add(f'{relative_id}/{flow_nums_str}')
 
         # Batch load prerequisites of tasks according to flow.
-        prereqs_map = {}
+        prereqs_map: Dict[str, dict] = {}
         for (
                 cycle, name, prereq_name,
                 prereq_cycle, prereq_output, satisfied
@@ -1458,16 +1459,14 @@ def apply_task_proxy_db_history(self):
             ] = satisfied if satisfied != '0' else False
 
         for ikey, prereqs in prereqs_map.items():
+            itask_prereq: Prerequisite
             for itask_prereq in (
-                    self.db_load_task_proxies[ikey][0].state.prerequisites
+                self.db_load_task_proxies[ikey][0].state.prerequisites
             ):
-                for key in itask_prereq.satisfied.keys():
-                    try:
-                        itask_prereq.satisfied[key] = prereqs[key]
-                    except KeyError:
-                        # This prereq is not in the DB: new dependencies
-                        # added to an already-spawned task before restart.
-                        itask_prereq.satisfied[key] = False
+                for key in itask_prereq:
+                    itask_prereq[key] = prereqs.get(key, False)
+                    # (False if prereq is not in the DB: new dependencies
+                    # added to an already-spawned task before restart.)
 
         # Extract info from itasks to data-store.
         for task_info in self.db_load_task_proxies.values():
diff --git a/cylc/flow/prerequisite.py b/cylc/flow/prerequisite.py
index 915397ee224..ad608af78a9 100644
--- a/cylc/flow/prerequisite.py
+++ b/cylc/flow/prerequisite.py
@@ -18,20 +18,66 @@
 
 import math
 import re
-from typing import Iterable, Set, TYPE_CHECKING
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    ItemsView,
+    Iterable,
+    Iterator,
+    List,
+    NamedTuple,
+    Optional,
+    Set,
+    Tuple,
+    Union,
+)
+
+# BACK COMPAT: typing_extensions.Literal
+# FROM: Python 3.7
+# TO: Python 3.8
+from typing_extensions import Literal
 
 from cylc.flow.cycling.loader import get_point
+from cylc.flow.data_messages_pb2 import PbCondition, PbPrerequisite
 from cylc.flow.exceptions import TriggerExpressionError
-from cylc.flow.data_messages_pb2 import (
-    PbPrerequisite,
-    PbCondition,
-)
 from cylc.flow.id import quick_relative_detokenise
 
+
 if TYPE_CHECKING:
+    from cylc.flow.cycling import PointBase
     from cylc.flow.id import Tokens
 
 
+AnyPrereqMessage = Tuple[Union['PointBase', str, int], str, str]
+
+
+class PrereqMessage(NamedTuple):
+    """A message pertaining to a Prerequisite."""
+    point: str
+    task: str
+    output: str
+
+    def get_id(self) -> str:
+        """Get the relative ID of the task in this prereq message."""
+        return quick_relative_detokenise(self.point, self.task)
+
+    @staticmethod
+    def coerce(tuple_: AnyPrereqMessage) -> 'PrereqMessage':
+        """Coerce a tuple to a PrereqMessage."""
+        if isinstance(tuple_, PrereqMessage):
+            return tuple_
+        point, task, output = tuple_
+        return PrereqMessage(point=str(point), task=task, output=output)
+
+
+SatisfiedState = Literal[
+    'satisfied naturally',
+    'satisfied from database',
+    'force satisfied',
+    False
+]
+
+
 class Prerequisite:
     """The concrete result of an abstract logical trigger expression.
 
@@ -46,70 +92,83 @@ class Prerequisite:
 
     # Memory optimization - constrain possible attributes to this list.
     __slots__ = (
-        "satisfied",
+        "_satisfied",
         "_all_satisfied",
         "conditional_expression",
         "point",
     )
 
     # Extracts T from "foo.T succeeded" etc.
-    SATISFIED_TEMPLATE = 'bool(self.satisfied[("%s", "%s", "%s")])'
+    SATISFIED_TEMPLATE = 'bool(self._satisfied[("%s", "%s", "%s")])'
     MESSAGE_TEMPLATE = r'%s/%s %s'
 
-    DEP_STATE_SATISFIED = 'satisfied naturally'
-    DEP_STATE_OVERRIDDEN = 'force satisfied'
-    DEP_STATE_UNSATISFIED = False
-
-    def __init__(self, point):
+    def __init__(self, point: 'PointBase'):
         # The cycle point to which this prerequisite belongs.
         # cylc.flow.cycling.PointBase
         self.point = point
 
         # Dictionary of messages pertaining to this prerequisite.
         # {('point string', 'task name', 'output'): DEP_STATE_X, ...}
-        self.satisfied = {}
+        self._satisfied: Dict[PrereqMessage, SatisfiedState] = {}
 
         # Expression present only when conditions are used.
         # '1/foo failed & 1/bar succeeded'
-        self.conditional_expression = None
+        self.conditional_expression: Optional[str] = None
 
         # The cached state of this prerequisite:
         # * `None` (no cached state)
         # * `True` (prerequisite satisfied)
         # * `False` (prerequisite unsatisfied).
-        self._all_satisfied = None
+        self._all_satisfied: Optional[bool] = None
 
-    def instantaneous_hash(self):
+    def instantaneous_hash(self) -> int:
         """Generate a hash of this prerequisite in its current state.
 
-        Note not using `__hash__` because Prerequisite objects are mutable.
+        NOTE: Is not affected by any change in satisfaction state.
+
+        (Not defining `self.__hash__()` because Prerequisite objects
+        are mutable.)
         """
         return hash((
             self.point,
             self.conditional_expression,
-            tuple(self.satisfied.keys()),
+            tuple(self._satisfied.keys()),
         ))
 
-    def add(self, name, point, output, pre_initial=False):
+    def __getitem__(self, key: AnyPrereqMessage) -> SatisfiedState:
+        """Return the satisfaction state of a dependency.
+
+        Args:
+            key: Tuple of (point, name, output) for a task.
+        """
+        return self._satisfied[PrereqMessage.coerce(key)]
+
+    def __setitem__(
+        self,
+        key: AnyPrereqMessage,
+        value: Union[SatisfiedState, bool] = False,
+    ) -> None:
         """Register an output with this prerequisite.
 
         Args:
-            name (str): The name of the task to which the output pertains.
-            point (str/cylc.flow.cycling.PointBase): The cycle point at which
-                this dependent output should appear.
-            output (str): String representing the output e.g. "succeeded".
-            pre_initial (bool): this is a pre-initial dependency.
+            key: Tuple of (point, name, output) for a prerequisite task.
+            value: Dependency satisfaction state (for pre-initial dependencies
+                this should be True).
 
         """
-        message = (str(point), name, output)
+        key = PrereqMessage.coerce(key)
+        if value is True:
+            value = 'satisfied naturally'
+        self._satisfied[key] = value
+        if not (self._all_satisfied and value):
+            # Force later recalculation of cached satisfaction state:
+            self._all_satisfied = None
 
-        # Add a new prerequisite as satisfied if pre-initial, else unsatisfied.
-        if pre_initial:
-            self.satisfied[message] = self.DEP_STATE_SATISFIED
-        else:
-            self.satisfied[message] = self.DEP_STATE_UNSATISFIED
-        if self._all_satisfied is not None:
-            self._all_satisfied = False
+    def __iter__(self) -> Iterator[PrereqMessage]:
+        return iter(self._satisfied)
+
+    def items(self) -> ItemsView[PrereqMessage, SatisfiedState]:
+        return self._satisfied.items()
 
     def get_raw_conditional_expression(self):
         """Return a representation of this prereq as a string.
@@ -120,7 +179,7 @@ def get_raw_conditional_expression(self):
         expr = self.conditional_expression
         if not expr:
             return None
-        for message in self.satisfied:
+        for message in self._satisfied:
             expr = expr.replace(self.SATISFIED_TEMPLATE % message,
                                 self.MESSAGE_TEMPLATE % message)
         return expr
@@ -134,21 +193,19 @@ def set_condition(self, expr):
             # is a substring of another: foo | xfoo => bar.
             # Add 'foo' to the 'satisfied' dict before 'xfoo'.
             >>> preq = Prerequisite(1)
-            >>> preq.satisfied = {
-            ...    ('1', 'foo', 'succeeded'): False,
-            ...    ('1', 'xfoo', 'succeeded'): False
-            ... }
+            >>> preq[(1, 'foo', 'succeeded')] = False
+            >>> preq[(1, 'xfoo', 'succeeded')] = False
             >>> preq.set_condition("1/foo succeeded|1/xfoo succeeded")
             >>> expr = preq.conditional_expression
             >>> expr.split('|')  # doctest: +NORMALIZE_WHITESPACE
-            ['bool(self.satisfied[("1", "foo", "succeeded")])',
-            'bool(self.satisfied[("1", "xfoo", "succeeded")])']
+            ['bool(self._satisfied[("1", "foo", "succeeded")])',
+            'bool(self._satisfied[("1", "xfoo", "succeeded")])']
 
         """
         self._all_satisfied = None
         if '|' in expr:
             # Make a Python expression so we can eval() the logic.
-            for message in self.satisfied:
+            for message in self._satisfied:
                 # Use '\b' in case one task name is a substring of another
                 # and escape special chars ('.', timezone '+') in task IDs.
                 expr = re.sub(
@@ -166,25 +223,23 @@ def is_satisfied(self):
 
         """
         if self._all_satisfied is not None:
+            # Cached value.
             return self._all_satisfied
-        else:
-            # No cached value.
-            if self.satisfied == {}:
-                # No prerequisites left after pre-initial simplification.
-                return True
-            if self.conditional_expression:
-                # Trigger expression with at least one '|': use eval.
-                self._all_satisfied = self._conditional_is_satisfied()
-            else:
-                self._all_satisfied = all(self.satisfied.values())
-            return self._all_satisfied
+        if self._satisfied == {}:
+            # No prerequisites left after pre-initial simplification.
+            return True
+        self._all_satisfied = self._eval_satisfied()
+        return self._all_satisfied
 
-    def _conditional_is_satisfied(self):
+    def _eval_satisfied(self) -> bool:
         """Evaluate the prerequisite's condition expression.
 
         Does not cache the result.
 
         """
+        if not self.conditional_expression:
+            return all(self._satisfied.values())
+
         try:
             res = eval(self.conditional_expression)  # nosec
             # * the expression is constructed internally
@@ -208,20 +263,18 @@ def satisfy_me(self, outputs: Iterable['Tokens']) -> 'Set[Tokens]':
         """
         valid = set()
         for output in outputs:
-            prereq = (output['cycle'], output['task'], output['task_sel'])
-            if prereq not in self.satisfied:
+            prereq = PrereqMessage(
+                output['cycle'], output['task'], output['task_sel']
+            )
+            if prereq not in self._satisfied:
                 continue
             valid.add(output)
-            self.satisfied[prereq] = self.DEP_STATE_SATISFIED
-            if self.conditional_expression is None:
-                self._all_satisfied = all(self.satisfied.values())
-            else:
-                self._all_satisfied = self._conditional_is_satisfied()
+            self[prereq] = 'satisfied naturally'
         return valid
 
-    def api_dump(self):
+    def api_dump(self) -> Optional[PbPrerequisite]:
         """Return list of populated Protobuf data objects."""
-        if not self.satisfied:
+        if not self._satisfied:
             return None
         if self.conditional_expression:
             expr = (
@@ -230,26 +283,22 @@ def api_dump(self):
         else:
             expr = ' & '.join(
                 self.MESSAGE_TEMPLATE % s_msg
-                for s_msg in self.satisfied
+                for s_msg in self._satisfied
             )
         conds = []
-        num_length = math.ceil(len(self.satisfied) / 10)
-        for ind, message_tuple in enumerate(sorted(self.satisfied)):
-            point, name = message_tuple[0:2]
-            t_id = quick_relative_detokenise(point, name)
+        num_length = math.ceil(len(self._satisfied) / 10)
+        for ind, message_tuple in enumerate(sorted(self._satisfied)):
+            t_id = message_tuple.get_id()
             char = 'c%.{0}d'.format(num_length) % ind
             c_msg = self.MESSAGE_TEMPLATE % message_tuple
-            c_val = self.satisfied[message_tuple]
-            c_bool = bool(c_val)
-            if c_bool is False:
-                c_val = "unsatisfied"
+            c_val = self._satisfied[message_tuple]
             conds.append(
                 PbCondition(
                     task_proxy=t_id,
                     expr_alias=char,
-                    req_state=message_tuple[2],
-                    satisfied=c_bool,
-                    message=c_val,
+                    req_state=message_tuple.output,
+                    satisfied=bool(c_val),
+                    message=(c_val or 'unsatisfied'),
                 )
             )
             expr = expr.replace(c_msg, char)
@@ -260,24 +309,23 @@ def api_dump(self):
             cycle_points=sorted(self.iter_target_point_strings()),
         )
 
-    def set_satisfied(self):
+    def set_satisfied(self) -> None:
         """Force this prerequisite into the satisfied state.
 
         State can be overridden by calling `self.satisfy_me`.
 
         """
-        for message in self.satisfied:
-            if not self.satisfied[message]:
-                self.satisfied[message] = self.DEP_STATE_OVERRIDDEN
-        if self.conditional_expression is None:
-            self._all_satisfied = True
+        for message in self._satisfied:
+            if not self._satisfied[message]:
+                self._satisfied[message] = 'force satisfied'
+        if self.conditional_expression:
+            self._all_satisfied = self._eval_satisfied()
         else:
-            self._all_satisfied = self._conditional_is_satisfied()
+            self._all_satisfied = True
 
     def iter_target_point_strings(self):
         yield from {
-            message[0]
-            for message in self.satisfied
+            message.point for message in self._satisfied
         }
 
     def get_target_points(self):
@@ -287,12 +335,14 @@ def get_target_points(self):
             get_point(p) for p in self.iter_target_point_strings()
         ]
 
-    def get_resolved_dependencies(self):
+    def get_resolved_dependencies(self) -> List[str]:
         """Return a list of satisfied dependencies.
 
         E.G: ['1/foo', '2/bar']
 
         """
-        return [f'{point}/{name}' for
-                (point, name, _), satisfied in self.satisfied.items() if
-                satisfied]
+        return [
+            msg.get_id()
+            for msg, satisfied in self._satisfied.items()
+            if satisfied
+        ]
diff --git a/cylc/flow/task_pool.py b/cylc/flow/task_pool.py
index 72e8065d4ca..2f953c9812d 100644
--- a/cylc/flow/task_pool.py
+++ b/cylc/flow/task_pool.py
@@ -89,11 +89,12 @@
     from cylc.flow.config import WorkflowConfig
     from cylc.flow.cycling import IntervalBase, PointBase
     from cylc.flow.data_store_mgr import DataStoreMgr
-    from cylc.flow.taskdef import TaskDef
+    from cylc.flow.flow_mgr import FlowMgr, FlowNums
+    from cylc.flow.prerequisite import SatisfiedState
     from cylc.flow.task_events_mgr import TaskEventsManager
-    from cylc.flow.xtrigger_mgr import XtriggerManager
+    from cylc.flow.taskdef import TaskDef
     from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager
-    from cylc.flow.flow_mgr import FlowMgr, FlowNums
+    from cylc.flow.xtrigger_mgr import XtriggerManager
 
 
 Pool = Dict['PointBase', Dict[str, TaskProxy]]
@@ -439,7 +440,7 @@ def check_task_output(
         task: str,
         output_msg: str,
         flow_nums: 'FlowNums',
-    ) -> Union[str, bool]:
+    ) -> 'SatisfiedState':
         """Returns truthy if the specified output is satisfied in the DB."""
         for task_outputs, task_flow_nums in (
             self.workflow_db_mgr.pri_dao.select_task_outputs(task, cycle)
@@ -563,16 +564,16 @@ def load_db_task_pool_for_restart(self, row_idx, row):
                 ] = satisfied if satisfied != '0' else False
 
             for itask_prereq in itask.state.prerequisites:
-                for key in itask_prereq.satisfied.keys():
-                    try:
-                        itask_prereq.satisfied[key] = sat[key]
-                    except KeyError:
+                for key in itask_prereq:
+                    if key in sat:
+                        itask_prereq[key] = sat[key]
+                    else:
                         # This prereq is not in the DB: new dependencies
                         # added to an already-spawned task before restart.
                         # Look through task outputs to see if is has been
                         # satisfied
                         prereq_cycle, prereq_task, prereq_output_msg = key
-                        itask_prereq.satisfied[key] = (
+                        itask_prereq[key] = (
                             self.check_task_output(
                                 prereq_cycle,
                                 prereq_task,
@@ -1995,7 +1996,7 @@ def _set_prereqs_itask(
             # Attempt to set the given presrequisites.
             # Log any that aren't valid for the task.
             presus = self._standardise_prereqs(prereqs)
-            unmatched = itask.satisfy_me(list(presus.keys()))
+            unmatched = itask.satisfy_me(presus.keys())
             for task_msg in unmatched:
                 LOG.warning(
                     f"{itask.identity} does not depend on"
diff --git a/cylc/flow/task_proxy.py b/cylc/flow/task_proxy.py
index 4e7b60d6e0a..27332ac9316 100644
--- a/cylc/flow/task_proxy.py
+++ b/cylc/flow/task_proxy.py
@@ -21,14 +21,15 @@
 from fnmatch import fnmatchcase
 from time import time
 from typing import (
+    TYPE_CHECKING,
     Any,
     Callable,
     Counter as TypingCounter,
     Dict,
+    Iterable,
     List,
     Optional,
     Set,
-    TYPE_CHECKING,
     Tuple,
 )
 
@@ -53,10 +54,12 @@
 
 if TYPE_CHECKING:
     from cylc.flow.cycling import PointBase
+    from cylc.flow.flow_mgr import FlowNums
+    from cylc.flow.id import Tokens
+    from cylc.flow.prerequisite import PrereqMessage, SatisfiedState
     from cylc.flow.simulation import ModeSettings
     from cylc.flow.task_action_timer import TaskActionTimer
     from cylc.flow.taskdef import TaskDef
-    from cylc.flow.id import Tokens
 
 
 class TaskProxy:
@@ -322,7 +325,11 @@ def __str__(self) -> str:
             f"{id_}{stringify_flow_nums(self.flow_nums)}:{self.state}"
         )
 
-    def copy_to_reload_successor(self, reload_successor, check_output):
+    def copy_to_reload_successor(
+        self,
+        reload_successor: 'TaskProxy',
+        check_output: Callable[[str, str, str, 'FlowNums'], 'SatisfiedState'],
+    ):
         """Copy attributes to successor on reload of this task proxy."""
         self.reload_successor = reload_successor
         reload_successor.submit_num = self.submit_num
@@ -346,10 +353,10 @@ def copy_to_reload_successor(self, reload_successor, check_output):
         # pre-reload state of prerequisites that still exist post-reload.
 
         # Get all prereq states, e.g. {('1', 'c', 'succeeded'): False, ...}
-        pre_reload = {
+        pre_reload: Dict[PrereqMessage, SatisfiedState] = {
             k: v
             for pre in self.state.prerequisites
-            for (k, v) in pre.satisfied.items()
+            for (k, v) in pre.items()
         }
         # Use them to update the new prerequisites.
         # - unchanged prerequisites will keep their pre-reload state.
@@ -357,16 +364,12 @@ def copy_to_reload_successor(self, reload_successor, check_output):
         # - added prerequisites will be recorded as unsatisfied
         #   NOTE: even if the corresponding output was completed pre-reload!
         for pre in reload_successor.state.prerequisites:
-            for k in pre.satisfied.keys():
-                try:
-                    pre.satisfied[k] = pre_reload[k]
-                except KeyError:
-                    # Look through task outputs to see if is has been
-                    # satisfied
-                    pre.satisfied[k] = check_output(
-                        *k,
-                        self.flow_nums,
-                    )
+            for k in pre:
+                pre[k] = pre_reload.get(
+                    k,
+                    # Else look thru task outputs to see if it's been satisfied
+                    check_output(*k, self.flow_nums)
+                )
 
         reload_successor.state.xtriggers.update({
             # copy across any special "_cylc" xtriggers which were added
@@ -544,7 +547,7 @@ def state_reset(
         return False
 
     def satisfy_me(
-        self, task_messages: 'List[Tokens]'
+        self, task_messages: 'Iterable[Tokens]'
     ) -> 'Set[Tokens]':
         """Try to satisfy my prerequisites with given output messages.
 
diff --git a/cylc/flow/task_state.py b/cylc/flow/task_state.py
index a4adb53d374..a6a6bc1258f 100644
--- a/cylc/flow/task_state.py
+++ b/cylc/flow/task_state.py
@@ -17,13 +17,24 @@
 """Task state related logic."""
 
 
-from typing import List, Iterable, Set, TYPE_CHECKING
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    Iterable,
+    List,
+    Set,
+)
+
 from cylc.flow.prerequisite import Prerequisite
 from cylc.flow.task_outputs import TaskOutputs
 from cylc.flow.wallclock import get_current_time_string
 
+
 if TYPE_CHECKING:
+    from cylc.flow.cycling import PointBase
     from cylc.flow.id import Tokens
+    from cylc.flow.prerequisite import PrereqMessage
+    from cylc.flow.taskdef import TaskDef
 
 
 # Task status names and meanings.
@@ -439,15 +450,15 @@ def is_gte(self, status):
         return (TASK_STATUSES_ORDERED.index(self.status) >=
                 TASK_STATUSES_ORDERED.index(status))
 
-    def _add_prerequisites(self, point, tdef):
+    def _add_prerequisites(self, point: 'PointBase', tdef: 'TaskDef'):
         """Add task prerequisites."""
         # Triggers for sequence_i only used if my cycle point is a
         # valid member of sequence_i's sequence of cycle points.
 
         # Use dicts to avoid generating duplicate prerequisites from sequences
         # with coincident cycle points.
-        prerequisites = {}
-        suicide_prerequisites = {}
+        prerequisites: Dict[int, Prerequisite] = {}
+        suicide_prerequisites: Dict[int, Prerequisite] = {}
 
         for sequence, dependencies in tdef.dependencies.items():
             if not sequence.is_valid(point):
@@ -470,8 +481,9 @@ def _add_prerequisites(self, point, tdef):
             if adjusted:
                 p_prev = max(adjusted)
                 cpre = Prerequisite(point)
-                cpre.add(tdef.name, p_prev, TASK_STATUS_SUCCEEDED,
-                         p_prev < tdef.start_point)
+                cpre[(p_prev, tdef.name, TASK_STATUS_SUCCEEDED)] = (
+                    p_prev < tdef.start_point
+                )
                 cpre.set_condition(tdef.name)
                 prerequisites[cpre.instantaneous_hash()] = cpre
 
@@ -497,13 +509,9 @@ def _add_xtriggers(self, point, tdef):
             for xtrig_label in xtrig_labels:
                 self.add_xtrigger(xtrig_label)
 
-    def get_unsatisfied_prerequisites(self):
-        unsat = []
-        for prereq in self.prerequisites:
-            if prereq.is_satisfied():
-                continue
-            for key, val in prereq.satisfied.items():
-                if val:
-                    continue
-                unsat.append(key)
-        return unsat
+    def get_unsatisfied_prerequisites(self) -> List['PrereqMessage']:
+        return [
+            key
+            for prereq in self.prerequisites if not prereq.is_satisfied()
+            for key, satisfied in prereq.items() if not satisfied
+        ]
diff --git a/cylc/flow/task_trigger.py b/cylc/flow/task_trigger.py
index d56c2a593b4..820b5eb8a18 100644
--- a/cylc/flow/task_trigger.py
+++ b/cylc/flow/task_trigger.py
@@ -14,11 +14,22 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from typing import TYPE_CHECKING, Tuple
+
 from cylc.flow.cycling.loader import (
-    get_point, get_point_relative, get_interval)
+    get_interval,
+    get_point,
+    get_point_relative,
+)
 from cylc.flow.prerequisite import Prerequisite
 from cylc.flow.task_qualifiers import ALT_QUALIFIERS
 
+
+if TYPE_CHECKING:
+    from cylc.flow.cycling import PointBase
+    from cylc.flow.taskdef import TaskDef
+
+
 # Task trigger names (e.g. foo:fail => bar).
 # Can use "foo:fail => bar" or "foo:failed => bar", etc.
 
@@ -176,23 +187,21 @@ class Dependency:
 
     def __init__(self, exp, task_triggers, suicide):
         self._exp = exp
-        self.task_triggers = tuple(task_triggers)  # More memory efficient.
+        self.task_triggers: Tuple[
+            TaskTrigger
+        ] = tuple(task_triggers)  # More memory efficient.
         self.suicide = suicide
 
-    def get_prerequisite(self, point, tdef):
+    def get_prerequisite(
+        self, point: 'PointBase', tdef: 'TaskDef'
+    ) -> Prerequisite:
         """Generate a Prerequisite object from this dependency.
 
         Args:
-            point (cylc.flow.cycling.PointBase): The cycle point at which to
-                generate the Prerequisite for.
-            tdef (cylc.flow.taskdef.TaskDef): The TaskDef of the dependent
-                task.
-
-        Returns:
-            cylc.flow.prerequisite.Prerequisite
+            point: The cycle point at which to generate the Prerequisite for.
+            tdef: The TaskDef of the dependent task.
 
         """
-        # Create Prerequisite.
         cpre = Prerequisite(point)
 
         # Loop over TaskTrigger instances.
@@ -213,21 +222,22 @@ def get_prerequisite(self, point, tdef):
                              tdef.max_future_prereq_offset)):
                         tdef.max_future_prereq_offset = (
                             prereq_offset)
-                cpre.add(
-                    task_trigger.task_name,
+                cpre[(
                     task_trigger.get_point(point),
-                    task_trigger.output,
-                    (
-                        (prereq_offset_point < tdef.start_point) &
-                        (point >= tdef.start_point)
-                    )
+                    task_trigger.task_name,
+                    task_trigger.output
+                )] = (
+                    (prereq_offset_point < tdef.start_point) &
+                    (point >= tdef.start_point)
                 )
             else:
                 # Trigger is within the same cycle point.
                 # Register task message with Prerequisite object.
-                cpre.add(task_trigger.task_name,
-                         task_trigger.get_point(point),
-                         task_trigger.output)
+                cpre[(
+                    task_trigger.get_point(point),
+                    task_trigger.task_name,
+                    task_trigger.output,
+                )] = False
         cpre.set_condition(self.get_expression(point))
         return cpre
 
diff --git a/cylc/flow/taskdef.py b/cylc/flow/taskdef.py
index 448844c8cc3..762324c9d75 100644
--- a/cylc/flow/taskdef.py
+++ b/cylc/flow/taskdef.py
@@ -17,20 +17,22 @@
 """Task definition."""
 
 from collections import deque
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Dict, List
 
 import cylc.flow.flags
 from cylc.flow.exceptions import TaskDefError
 from cylc.flow.task_id import TaskID
 from cylc.flow.task_outputs import (
+    SORT_ORDERS,
+    TASK_OUTPUT_FAILED,
     TASK_OUTPUT_SUBMITTED,
     TASK_OUTPUT_SUCCEEDED,
-    TASK_OUTPUT_FAILED,
-    SORT_ORDERS
 )
 
+
 if TYPE_CHECKING:
-    from cylc.flow.cycling import PointBase
+    from cylc.flow.cycling import PointBase, SequenceBase
+    from cylc.flow.task_trigger import Dependency
 
 
 def generate_graph_children(tdef, point):
@@ -145,7 +147,7 @@ def __init__(self, name, rtcfg, run_mode, start_point, initial_point):
         self.start_point = start_point
         self.initial_point = initial_point
 
-        self.sequences = []
+        self.sequences: List[SequenceBase] = []
         self.used_in_offset_trigger = False
 
         # some defaults
@@ -155,7 +157,7 @@ def __init__(self, name, rtcfg, run_mode, start_point, initial_point):
 
         self.expiration_offset = None
         self.namespace_hierarchy = []
-        self.dependencies = {}
+        self.dependencies: Dict[SequenceBase, List[Dependency]] = {}
         self.outputs = {}  # {output: (message, is_required)}
         self.graph_children = {}
         self.graph_parents = {}
diff --git a/cylc/flow/workflow_db_mgr.py b/cylc/flow/workflow_db_mgr.py
index c3f3ea9a263..d45108ca587 100644
--- a/cylc/flow/workflow_db_mgr.py
+++ b/cylc/flow/workflow_db_mgr.py
@@ -479,7 +479,7 @@ def put_task_pool(self, pool: 'TaskPool') -> None:
         for itask in pool.get_tasks():
             for prereq in itask.state.prerequisites:
                 for (p_cycle, p_name, p_output), satisfied_state in (
-                    prereq.satisfied.items()
+                    prereq.items()
                 ):
                     self.put_insert_task_prerequisites(itask, {
                         "flow_nums": serialise_set(itask.flow_nums),
diff --git a/tests/integration/scripts/test_set.py b/tests/integration/scripts/test_set.py
index 22cd44bbbce..af368aab0d5 100644
--- a/tests/integration/scripts/test_set.py
+++ b/tests/integration/scripts/test_set.py
@@ -20,8 +20,10 @@
 """
 
 from cylc.flow.cycling.integer import IntegerPoint
+from cylc.flow.data_messages_pb2 import PbTaskProxy
 from cylc.flow.data_store_mgr import TASK_PROXIES
-from cylc.flow.task_state import TASK_STATUS_WAITING, TASK_STATUS_SUCCEEDED
+from cylc.flow.scheduler import Scheduler
+from cylc.flow.task_state import TASK_STATUS_SUCCEEDED, TASK_STATUS_WAITING
 
 
 async def test_set_parentless_spawning(
@@ -106,11 +108,11 @@ async def test_data_store(
             'a': {'outputs': {'x': 'xyz'}},
         },
     })
-    schd = scheduler(id_)
+    schd: Scheduler = scheduler(id_)
     async with start(schd):
         await schd.update_data_structure()
         data = schd.data_store_mgr.data[schd.tokens.id]
-        task_a = data[TASK_PROXIES][
+        task_a: PbTaskProxy = data[TASK_PROXIES][
             schd.pool.get_task(IntegerPoint('1'), 'a').tokens.id
         ]
 
diff --git a/tests/integration/test_data_store_mgr.py b/tests/integration/test_data_store_mgr.py
index 906b1ac052d..e9c4e7616b6 100644
--- a/tests/integration/test_data_store_mgr.py
+++ b/tests/integration/test_data_store_mgr.py
@@ -14,18 +14,21 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from typing import Iterable, List, cast
+
 import pytest
-from typing import TYPE_CHECKING
 
+from cylc.flow.data_messages_pb2 import PbPrerequisite, PbTaskProxy
 from cylc.flow.data_store_mgr import (
     EDGES,
     FAMILY_PROXIES,
     JOBS,
-    TASKS,
     TASK_PROXIES,
-    WORKFLOW
+    TASKS,
+    WORKFLOW,
 )
 from cylc.flow.id import Tokens
+from cylc.flow.scheduler import Scheduler
 from cylc.flow.task_state import (
     TASK_STATUS_FAILED,
     TASK_STATUS_SUCCEEDED,
@@ -33,9 +36,6 @@
 )
 from cylc.flow.wallclock import get_current_time_string
 
-if TYPE_CHECKING:
-    from cylc.flow.scheduler import Scheduler
-
 
 # NOTE: These tests mutate the data store, so running them in isolation may
 # see failures when they actually pass if you run the whole file
@@ -88,6 +88,18 @@ def ext_id(schd):
     return f'~{schd.owner}/{schd.workflow}//{int_id(None)}'
 
 
+def get_pb_prereqs(schd: Scheduler) -> List[PbPrerequisite]:
+    """Get all protobuf prerequisites from the data store task proxies."""
+    return [
+        p
+        for t in cast(
+            'Iterable[PbTaskProxy]',
+            schd.data_store_mgr.updated[TASK_PROXIES].values()
+        )
+        for p in t.prerequisites
+    ]
+
+
 @pytest.fixture(scope='module')
 async def harness(mod_flow, mod_scheduler, mod_start):
     flow_def = {
@@ -296,6 +308,7 @@ async def test_update_data_structure(harness):
 
 def test_delta_task_prerequisite(harness):
     """Test delta_task_prerequisites."""
+    schd: Scheduler
     schd, data = harness
     schd.pool.set_prereqs_and_outputs(
         [t.identity for t in schd.pool.get_tasks()],
@@ -303,20 +316,15 @@ def test_delta_task_prerequisite(harness):
         [],
         "all"
     )
-    assert all({
-        p.satisfied
-        for t in schd.data_store_mgr.updated[TASK_PROXIES].values()
-        for p in t.prerequisites})
+    assert all(p.satisfied for p in get_pb_prereqs(schd))
     for itask in schd.pool.get_tasks():
         # set prereqs as not-satisfied
         for prereq in itask.state.prerequisites:
             prereq._all_satisfied = False
-            prereq.satisfied = {key: False for key in prereq.satisfied}
+            for key in prereq:
+                prereq._satisfied[key] = False
         schd.data_store_mgr.delta_task_prerequisite(itask)
-    assert not any({
-        p.satisfied
-        for t in schd.data_store_mgr.updated[TASK_PROXIES].values()
-        for p in t.prerequisites})
+    assert not any(p.satisfied for p in get_pb_prereqs(schd))
 
 
 async def test_absolute_graph_edges(flow, scheduler, start):
diff --git a/tests/integration/test_task_pool.py b/tests/integration/test_task_pool.py
index beba9075bd3..a38f33704ac 100644
--- a/tests/integration/test_task_pool.py
+++ b/tests/integration/test_task_pool.py
@@ -33,6 +33,7 @@
 from cylc.flow import commands
 from cylc.flow.cycling.integer import IntegerPoint
 from cylc.flow.cycling.iso8601 import ISO8601Point
+from cylc.flow.data_messages_pb2 import PbPrerequisite
 from cylc.flow.data_store_mgr import TASK_PROXIES
 from cylc.flow.task_events_mgr import TaskEventsManager
 from cylc.flow.task_outputs import (
@@ -699,7 +700,7 @@ async def test_restart_prereqs(
         }
     }
     id_ = flow(conf)
-    schd = scheduler(id_, run_mode='simulation', paused_start=False)
+    schd: Scheduler = scheduler(id_, paused_start=False)
 
     async with start(schd):
         # Release tasks 1/a and 1/b
@@ -737,7 +738,7 @@ async def test_restart_prereqs(
         ][0]
         assert sorted(
             (
-                p.satisfied
+                p._satisfied
                 for p in task_z.state.prerequisites
             ),
             key=lambda d: tuple(d.keys())[0],
@@ -822,7 +823,7 @@ async def test_reload_prereqs(
         }
     }
     id_ = flow(conf)
-    schd = scheduler(id_, run_mode='simulation', paused_start=False)
+    schd: Scheduler = scheduler(id_, paused_start=False)
 
     async with start(schd):
         # Release tasks 1/a and 1/b
@@ -849,7 +850,7 @@ async def test_reload_prereqs(
         ][0]
         assert sorted(
             (
-                p.satisfied
+                p._satisfied
                 for p in task_z.state.prerequisites
             ),
             key=lambda d: tuple(d.keys())[0],
@@ -857,6 +858,7 @@ async def test_reload_prereqs(
 
 
 async def _test_restart_prereqs_sat():
+    schd: Scheduler
     # YIELD: the workflow has now started...
     schd = yield
     await schd.update_data_structure()
@@ -891,7 +893,7 @@ async def _test_restart_prereqs_sat():
     assert sorted(
         (*key, satisfied)
         for prereq in task_c.state.prerequisites
-        for key, satisfied in prereq.satisfied.items()
+        for key, satisfied in prereq.items()
     ) == [
         ('1', 'a', 'succeeded', 'satisfied naturally'),
         ('1', 'b', 'succeeded', 'satisfied from database')
@@ -902,7 +904,7 @@ async def _test_restart_prereqs_sat():
     tasks = (
         schd.data_store_mgr.data[schd.data_store_mgr.workflow_id][TASK_PROXIES]
     )
-    task_c_prereqs = tasks[
+    task_c_prereqs: List[PbPrerequisite] = tasks[
         schd.data_store_mgr.id_.duplicate(cycle='1', task='c').id
     ].prerequisites
     assert sorted(
@@ -1414,7 +1416,7 @@ def set_prereqs(prereqs):
     async with start(schd) as log:
         # Invalid: task name wildcard:
         set_prereqs(["2040/*"])
-        assert log_filter(log, contains='Invalid prerequisite task name' )
+        assert log_filter(log, contains='Invalid prerequisite task name')
 
         # Invalid: cycle point wildcard.
         set_prereqs(["*/foo"])
@@ -2085,7 +2087,7 @@ async def test_set_future_flow(flow, scheduler, start, log_filter):
         # set b:succeeded in flow 2 and check downstream spawning
         schd.pool.set_prereqs_and_outputs(['1/b'], prereqs=[], outputs=[], flow=[2])
         assert schd.pool.get_task(IntegerPoint("1"), "c1") is None, '1/c1 (flow 2) should not be spawned after 1/b:succeeded'
-        assert schd.pool.get_task(IntegerPoint("1"), "c2") is not None, '1/c2 (flow 2) should be spawned after 1/b:succeeded' 
+        assert schd.pool.get_task(IntegerPoint("1"), "c2") is not None, '1/c2 (flow 2) should be spawned after 1/b:succeeded'
 
 
 async def test_trigger_queue(one, run, db_select, complete):
diff --git a/tests/unit/test_prerequisite.py b/tests/unit/test_prerequisite.py
index 1514ad07f66..105e5e85401 100644
--- a/tests/unit/test_prerequisite.py
+++ b/tests/unit/test_prerequisite.py
@@ -26,35 +26,15 @@
 def prereq(set_cycling_type):
     set_cycling_type(ISO8601_CYCLING_TYPE, "Z")
     prereq = Prerequisite(get_point('2000'))
-    prereq.add(
-        'a',
-        '1999',
-        'succeeded',
-        True
-    )
-    prereq.add(
-        'b',
-        '2000',
-        'succeeded',
-        False
-    )
-    prereq.add(
-        'c',
-        '2000',
-        'succeeded',
-        False
-    )
-    prereq.add(
-        'd',
-        '2001',
-        'custom',
-        False
-    )
+    prereq[(1999, 'a', 'succeeded')] = True
+    prereq[(2000, 'b', 'succeeded')] = False
+    prereq[(2000, 'c', 'succeeded')] = False
+    prereq[(2001, 'd', 'custom')] = False
     return prereq
 
 
-def test_satisfied(prereq):
-    assert prereq.satisfied == {
+def test_satisfied(prereq: Prerequisite):
+    assert prereq._satisfied == {
         # the pre-initial dependency should be marked as satisfied
         ('1999', 'a', 'succeeded'): 'satisfied naturally',
         # all others should not
@@ -62,12 +42,18 @@ def test_satisfied(prereq):
         ('2000', 'c', 'succeeded'): False,
         ('2001', 'd', 'custom'): False,
     }
+    # No cached satisfaction state yet:
+    assert prereq._all_satisfied is None
+    # Calling self.is_satisfied() should cache the result:
+    assert not prereq.is_satisfied()
+    assert prereq._all_satisfied is False
+
     # mark two prerequisites as satisfied
     prereq.satisfy_me([
         Tokens('2000/b:succeeded', relative=True),
         Tokens('2000/c:succeeded', relative=True),
     ])
-    assert prereq.satisfied == {
+    assert prereq._satisfied == {
         # the pre-initial dependency should be marked as satisfied
         ('1999', 'a', 'succeeded'): 'satisfied naturally',
         # the two newly-satisfied dependency should be satisfied
@@ -76,9 +62,13 @@ def test_satisfied(prereq):
         # the remaining dependency should not
         ('2001', 'd', 'custom'): False,
     }
+    # Should have reset cached satisfaction state:
+    assert prereq._all_satisfied is None
+    assert not prereq.is_satisfied()
+
     # mark all prereqs as satisfied
     prereq.set_satisfied()
-    assert prereq.satisfied == {
+    assert prereq._satisfied == {
         # the pre-initial dependency should be marked as satisfied
         ('1999', 'a', 'succeeded'): 'satisfied naturally',
         # the two newly-satisfied dependency should be satisfied
@@ -87,6 +77,49 @@ def test_satisfied(prereq):
         # the remaining dependency should be marked as forse-satisfied
         ('2001', 'd', 'custom'): 'force satisfied',
     }
+    # Should have set cached satisfaction state as must be true now:
+    assert prereq._all_satisfied is True
+    assert prereq.is_satisfied()
+
+
+def test_getitem_setitem(prereq: Prerequisite):
+    msg = ('2000', 'b', 'succeeded')
+    # __getitem__:
+    assert prereq[msg] is False
+
+    # __setitem__:
+    prereq[msg] = True
+    assert prereq[msg] == 'satisfied naturally'
+    prereq[msg] = 'force satisfied'
+    assert prereq[msg] == 'force satisfied'
+    # coercion of cycle point
+    assert prereq[(2000, 'b', 'succeeded')] == 'force satisfied'
+    assert prereq[(get_point('2000'), 'b', 'succeeded')] == 'force satisfied'
+
+
+def test_iter(prereq: Prerequisite):
+    assert list(prereq) == [
+        ('1999', 'a', 'succeeded'),
+        ('2000', 'b', 'succeeded'),
+        ('2000', 'c', 'succeeded'),
+        ('2001', 'd', 'custom'),
+    ]
+    assert [p.task for p in prereq] == ['a', 'b', 'c', 'd']
+
+
+def test_items(prereq: Prerequisite):
+    assert list(prereq.items()) == [
+        (('1999', 'a', 'succeeded'), 'satisfied naturally'),
+        (('2000', 'b', 'succeeded'), False),
+        (('2000', 'c', 'succeeded'), False),
+        (('2001', 'd', 'custom'), False),
+    ]
+
+
+def test_set_condition(prereq: Prerequisite):
+    assert not prereq.is_satisfied()
+    prereq.set_condition('1999/a succeeded | 2000/b succeeded')
+    assert prereq.is_satisfied()
 
 
 def test_iter_target_point_strings(prereq):
@@ -107,10 +140,10 @@ def test_get_target_points(prereq):
 
 def test_get_resolved_dependencies():
     prereq = Prerequisite(IntegerPoint('2'))
-    prereq.satisfied[('1', 'a', 'x')] = 'satisfied naturally'
-    prereq.satisfied[('1', 'b', 'x')] = False
-    prereq.satisfied[('1', 'c', 'x')] = 'satisfied from database'
-    prereq.satisfied[('1', 'd', 'x')] = 'force satisfied'
+    prereq[('1', 'a', 'x')] = True
+    prereq[('1', 'b', 'x')] = False
+    prereq[('1', 'c', 'x')] = 'satisfied from database'
+    prereq[('1', 'd', 'x')] = 'force satisfied'
     assert prereq.get_resolved_dependencies() == [
         '1/a',
         '1/c',
diff --git a/tests/unit/test_task_state.py b/tests/unit/test_task_state.py
index e655c74b7bb..4b5d796a043 100644
--- a/tests/unit/test_task_state.py
+++ b/tests/unit/test_task_state.py
@@ -100,7 +100,7 @@ def test_task_prereq_duplicates(set_cycling_type):
 
     tstate = TaskState(tdef, IntegerPoint("1"), TASK_STATUS_WAITING, False)
 
-    prereqs = [p.satisfied for p in tstate.prerequisites]
+    prereqs = [p._satisfied for p in tstate.prerequisites]
 
     assert prereqs == [{("1", "a", "succeeded"): False}]
 
@@ -118,4 +118,3 @@ def test_task_state_order():
     assert tstate.is_gte(TASK_STATUS_SUBMITTED)
     assert not tstate.is_gt(TASK_STATUS_RUNNING)
     assert not tstate.is_gte(TASK_STATUS_RUNNING)
-

From 8a4366850b467efb4d5529ca9072a6cb645a25f7 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Thu, 8 Aug 2024 12:29:04 +0100
Subject: [PATCH 21/28] Fix potential type error

---
 cylc/flow/task_pool.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/cylc/flow/task_pool.py b/cylc/flow/task_pool.py
index 2f953c9812d..aeee7505e66 100644
--- a/cylc/flow/task_pool.py
+++ b/cylc/flow/task_pool.py
@@ -1209,16 +1209,14 @@ def log_unsatisfied_prereqs(self) -> bool:
             task_point = itask.point
             if self.stop_point and task_point > self.stop_point:
                 continue
-            for point, task, msg in (
-                itask.state.get_unsatisfied_prerequisites()
-            ):
-                if get_point(point) > self.stop_point:
+            for pr in itask.state.get_unsatisfied_prerequisites():
+                if self.stop_point and get_point(pr.point) > self.stop_point:
                     continue
                 if itask.identity not in unsat:
                     unsat[itask.identity] = []
                 unsat[itask.identity].append(
-                    f"{point}/{task}:"
-                    f"{self.config.get_taskdef(task).get_output(msg)}"
+                    f"{pr.get_id()}:"
+                    f"{self.config.get_taskdef(pr.task).get_output(pr.output)}"
                 )
         if unsat:
             LOG.warning(

From 34c9fa5a57adae74dbdcd60d3416f3f32988db81 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Thu, 1 Aug 2024 13:20:15 +0100
Subject: [PATCH 22/28] examples: extending workflow

* Fix incorrect heading level
* Add a note about scheduling tasks to run at the
  "stop after cycle point".
---
 .../etc/examples/extending-workflow/index.rst | 32 +++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/cylc/flow/etc/examples/extending-workflow/index.rst b/cylc/flow/etc/examples/extending-workflow/index.rst
index c5bc023dc11..036438ff466 100644
--- a/cylc/flow/etc/examples/extending-workflow/index.rst
+++ b/cylc/flow/etc/examples/extending-workflow/index.rst
@@ -1,5 +1,5 @@
 Extending Workflow
-==================
+------------------
 
 .. cylc-scope:: flow.cylc[scheduling]
 
@@ -24,7 +24,7 @@ solution avoids this issue.
 
 
 Example
--------
+^^^^^^^
 
 .. admonition:: Get a copy of this example
    :class: hint
@@ -74,4 +74,32 @@ Or change it at any point whilst the workflow is running:
    ``cylc play --stop-cycle-point=reload`` to restart the workflow using the
    `stop after cycle point` configured in the workflow configuration.
 
+
+Running Tasks At The `stop after cycle point`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you have tasks that you want to run before the workflow shuts down at the
+`stop after cycle point`, use the recurrence ``R1/<cycle-point>`` to schedudule
+them, e.g:
+
+.. code-block:: cylc
+
+   #!Jinja2
+
+   {% set stop_cycle = '3000' %}
+
+   [scheduling]
+      initial cycle point = 2000
+      stop after cycle point = {{ stop_cycle }}
+      [[graph]]
+         R1/{{ stop_cycle }} = """
+            # these tasks will run *before* the workflow shuts down
+            z => run_me => and_me
+         """
+
+When the workflow is subsequently restarted with a later
+`stop after cycle point`, these tasks will be re-scheduled at the new
+stop point.
+
+
 .. cylc-scope::

From aa4835621c5ae305a27f18ff3bc95166f86a7882 Mon Sep 17 00:00:00 2001
From: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
Date: Tue, 13 Aug 2024 13:51:35 +0100
Subject: [PATCH 23/28] schema: minor docs changes

* Consistent string quotation.
* Consistent termination punctuation.
* Correct some ReST literals (double backquotes).
* Correct one incorrectly closed markdown literal.
---
 cylc/flow/network/schema.py | 139 +++++++++++++++++++-----------------
 1 file changed, 72 insertions(+), 67 deletions(-)

diff --git a/cylc/flow/network/schema.py b/cylc/flow/network/schema.py
index b962b582b70..8e20225d4b6 100644
--- a/cylc/flow/network/schema.py
+++ b/cylc/flow/network/schema.py
@@ -577,9 +577,10 @@ def resolve_mapping_to_list(root, info, **args):
 # Types:
 class NodeMeta(ObjectType):
     class Meta:
-        description = """
-Meta data fields,
-including custom fields in a generic user-defined dump"""
+        description = sstrip("""
+            Meta data fields, including custom fields in a generic user-defined
+            dump.
+        """)
     title = String(default_value=None)
     description = String(default_value=None)
     URL = String(default_value=None)
@@ -588,7 +589,7 @@ class Meta:
 
 class TimeZone(ObjectType):
     class Meta:
-        description = """Time zone info."""
+        description = 'Time zone info.'
     hours = Int()
     minutes = Int()
     string_basic = String()
@@ -597,7 +598,7 @@ class Meta:
 
 class Workflow(ObjectType):
     class Meta:
-        description = """Global workflow info."""
+        description = 'Global workflow info.'
     id = ID()  # noqa: A003 (required for definition)
     name = String(
         description='The workflow ID with the ~user/ prefix removed.',
@@ -730,7 +731,7 @@ class Meta:
     latest_state_tasks = GenericScalar(
         states=graphene.List(
             String,
-            description="List of task states to show",
+            description="List of task states to show.",
             default_value=TASK_STATUSES_ORDERED),
         resolver=resolve_state_tasks,
         description='The latest tasks to have entered each task state.',
@@ -769,8 +770,9 @@ class Meta:
         ids=graphene.List(
             ID,
             description=sstrip('''
-                Node IDs, cycle point and/or-just family/task namespace:
-                    ["1234/foo", "1234/FAM", "*/FAM"]
+                Node IDs, cycle point and/or-just family/task namespace.
+
+                E.g: `["1234/foo", "1234/FAM", "*/FAM"]`
             '''),
             default_value=[]
         ),
@@ -858,7 +860,7 @@ class Meta:
     )
     task_proxy = Field(
         lambda: TaskProxy,
-        description="The TaskProxy of the task which submitted this job",
+        description="The TaskProxy of the task which submitted this job.",
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
         delta_type=DELTA_TYPE_DEFAULT,
@@ -919,7 +921,7 @@ class Meta:
     )
     runtime = Field(
         Runtime,
-        description="The task's `[runtime`] section.",
+        description="The task's `[runtime]` section.",
     )
     mean_elapsed_time = Float(
         description="The task's average runtime."
@@ -955,7 +957,7 @@ class Meta:
 
 class PollTask(ObjectType):
     class Meta:
-        description = """Polling task edge"""
+        description = 'Polling task edge.'
     local_proxy = ID()
     workflow = String()
     remote_proxy = ID()
@@ -965,11 +967,11 @@ class Meta:
 
 class Condition(ObjectType):
     class Meta:
-        description = """Prerequisite conditions."""
+        description = 'Prerequisite conditions.'
     task_id = String()
     task_proxy = Field(
         lambda: TaskProxy,
-        description="""Associated Task Proxy""",
+        description='Associated Task Proxy.',
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
         delta_type=DELTA_TYPE_DEFAULT,
@@ -985,18 +987,19 @@ def resolve_task_id(root, info, **args):
 
 class Prerequisite(ObjectType):
     class Meta:
-        description = """Task prerequisite."""
+        description = 'Task prerequisite.'
     expression = String()
     conditions = graphene.List(
         Condition,
-        description="""Condition monomers of a task prerequisites.""")
+        description='Condition monomers of a task prerequisites.'
+    )
     cycle_points = graphene.List(String)
     satisfied = Boolean()
 
 
 class Output(ObjectType):
     class Meta:
-        description = """Task output"""
+        description = 'Task output.'
     label = String()
     message = String()
     satisfied = Boolean()
@@ -1004,16 +1007,16 @@ class Meta:
 
 
 class OutputLabel(String):
-    """Task output, e.g. "succeeded"."""
+    """Task output, e.g. `succeeded`."""
 
 
 class PrerequisiteString(String):
-    """A task prerequisite, e.g. "2040/foo:succeeded"."""
+    """A task prerequisite, e.g. `2040/foo:succeeded`."""
 
 
 class XTrigger(ObjectType):
     class Meta:
-        description = """Task trigger"""
+        description = 'Task trigger.'
     id = String()  # noqa: A003 (required for definition)
     label = String()
     message = String()
@@ -1023,7 +1026,7 @@ class Meta:
 
 class TaskProxy(ObjectType):
     class Meta:
-        description = """Task cycle instance."""
+        description = 'Task cycle instance.'
     id = ID()  # noqa: A003 (required for schema definition)
     task = Field(
         Task,
@@ -1073,7 +1076,7 @@ class Meta:
         '''),
     )
     depth = Int(
-        description='The family inheritance depth',
+        description='The family inheritance depth.',
     )
     graph_depth = Int(
         description=sstrip('''
@@ -1158,7 +1161,7 @@ class Meta:
 
 class Family(ObjectType):
     class Meta:
-        description = """Task definition, static fields"""
+        description = 'Task definition, static fields.'
     id = ID()  # noqa: A003 (required for schema definition)
     name = String()
     meta = Field(NodeMeta)
@@ -1166,7 +1169,7 @@ class Meta:
     depth = Int()
     proxies = graphene.List(
         lambda: FamilyProxy,
-        description="""Associated cycle point proxies""",
+        description='Associated cycle point proxies.',
         args=PROXY_ARGS,
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
@@ -1174,7 +1177,7 @@ class Meta:
         resolver=get_nodes_by_ids)
     parents = graphene.List(
         lambda: Family,
-        description="""Family definition parent.""",
+        description='Family definition parent.',
         args=DEF_ARGS,
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
@@ -1182,7 +1185,7 @@ class Meta:
         resolver=get_nodes_by_ids)
     child_tasks = graphene.List(
         Task,
-        description="""Descendant definition tasks.""",
+        description='Descendant definition tasks.',
         args=DEF_ARGS,
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
@@ -1190,7 +1193,7 @@ class Meta:
         resolver=get_nodes_by_ids)
     child_families = graphene.List(
         lambda: Family,
-        description="""Descendant desc families.""",
+        description='Descendant desc families.',
         args=DEF_ARGS,
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
@@ -1198,7 +1201,7 @@ class Meta:
         resolver=get_nodes_by_ids)
     first_parent = Field(
         lambda: Family,
-        description="""Family first parent.""",
+        description='Family first parent.',
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
         delta_type=DELTA_TYPE_DEFAULT,
@@ -1207,14 +1210,14 @@ class Meta:
 
 class FamilyProxy(ObjectType):
     class Meta:
-        description = """Family composite."""
+        description = 'Family composite.'
     id = ID()  # noqa: A003 (required for schema definition)
     cycle_point = String()
     # name & namespace for filtering/sorting
     name = String()
     family = Field(
         Family,
-        description="""Family definition""",
+        description='Family definition.',
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
         delta_type=DELTA_TYPE_DEFAULT,
@@ -1238,7 +1241,7 @@ class Meta:
     )
     child_tasks = graphene.List(
         TaskProxy,
-        description="""Descendant task proxies.""",
+        description='Descendant task proxies.',
         args=PROXY_ARGS,
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
@@ -1246,7 +1249,7 @@ class Meta:
         resolver=get_nodes_by_ids)
     child_families = graphene.List(
         lambda: FamilyProxy,
-        description="""Descendant family proxies.""",
+        description='Descendant family proxies.',
         args=PROXY_ARGS,
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
@@ -1254,7 +1257,7 @@ class Meta:
         resolver=get_nodes_by_ids)
     first_parent = Field(
         lambda: FamilyProxy,
-        description="""Task first parent.""",
+        description='Task first parent.',
         args=PROXY_ARGS,
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
@@ -1262,7 +1265,7 @@ class Meta:
         resolver=get_node_by_id)
     ancestors = graphene.List(
         lambda: FamilyProxy,
-        description="""First parent ancestors.""",
+        description='First parent ancestors.',
         args=PROXY_ARGS,
         strip_null=STRIP_NULL_DEFAULT,
         delta_store=DELTA_STORE_DEFAULT,
@@ -1283,7 +1286,7 @@ def resolve_type(cls, instance, info):
 
 class Edge(ObjectType):
     class Meta:
-        description = """Dependency edge task/family proxies"""
+        description = 'Dependency edge task/family proxies.'
     id = ID()  # noqa: A003 (required for schema definition)
     source = ID()
     source_node = Field(
@@ -1305,7 +1308,7 @@ class Meta:
 
 class Edges(ObjectType):
     class Meta:
-        description = """Dependency edge"""
+        description = 'Dependency edge.'
     edges = graphene.List(
         Edge,
         args=EDGE_ARGS,
@@ -1320,19 +1323,19 @@ class Meta:
 
 class NodesEdges(ObjectType):
     class Meta:
-        description = """Related Nodes & Edges."""
+        description = 'Related Nodes & Edges.'
     nodes = graphene.List(
         TaskProxy,
-        description="""Task nodes from and including root.""")
+        description='Task nodes from and including root.')
     edges = graphene.List(
         Edge,
-        description="""Edges associated with the nodes.""")
+        description='Edges associated with the nodes.')
 
 
 # Query declaration
 class Queries(ObjectType):
     class Meta:
-        description = """Multi-Workflow root level queries."""
+        description = 'Multi-Workflow root level queries.'
     workflows = graphene.List(
         Workflow,
         description=Workflow._meta.description,
@@ -1420,7 +1423,7 @@ class Meta:
 # Generic containers
 class GenericResponse(ObjectType):
     class Meta:
-        description = """Container for command queued response"""
+        description = 'Container for command queued response.'
 
     result = GenericScalar()
 
@@ -1455,6 +1458,7 @@ async def mutator(
             method). If None, uses mutation class name converted to snake_case.
         workflows: List of workflow IDs.
         exworkflows: List of workflow IDs.
+
     """
     if command is None:
         command = to_snake_case(info.field_name)
@@ -1567,13 +1571,13 @@ class TaskState(InputObjectType):
 
     status = TaskStatus()
     is_held = Boolean(description=sstrip('''
-        If a task is held no new job submissions will be made
+        If a task is held no new job submissions will be made.
     '''))
     is_queued = Boolean(description=sstrip('''
-        Task is queued for job submission
+        Task is queued for job submission.
     '''))
     is_runahead = Boolean(description=sstrip('''
-        Task is runahead limited
+        Task is runahead limited.
     '''))
 
 
@@ -1792,7 +1796,7 @@ class Arguments:
         event_time = String(default_value=None)
         messages = graphene.List(
             graphene.List(String),
-            description="""List in the form `[[severity, message], ...]`.""",
+            description='List in the form `[[severity, message], ...]`.',
             default_value=None
         )
 
@@ -1820,6 +1824,7 @@ class Resume(Mutation):
     class Meta:
         description = sstrip('''
             Resume a paused workflow.
+
             See also the opposite command `pause`.
 
             Valid for: paused workflows.
@@ -1952,8 +1957,8 @@ class Meta:
             trigger event. When an incoming message satisfies
             a task's external trigger the message ID is broadcast
             to all downstream tasks in the cycle point as
-            ``$CYLC_EXT_TRIGGER_ID``.  Tasks can use
-            ``$CYLC_EXT_TRIGGER_ID``, for example,  to
+            `$CYLC_EXT_TRIGGER_ID`.  Tasks can use
+            `$CYLC_EXT_TRIGGER_ID`, for example,  to
             identify a new data file that the external
             triggering system is responding to.
 
@@ -2113,9 +2118,9 @@ class Meta:
             Setting outputs contributes to the task's completion, sets the
             corresponding prerequisites of child tasks, and sets any implied
             outputs:
-             - ``started`` implies ``submitted``.
-             - ``succeeded`` and ``failed`` imply ``started``.
-             - custom outputs and ``expired`` do not imply any other outputs.
+             - `started` implies `submitted`.
+             - `succeeded` and `failed` imply `started`.
+             - custom outputs and `expired` do not imply any other outputs.
 
             Valid for: paused, running, stopping workflows.
         """)
@@ -2224,7 +2229,7 @@ def delta_subs(root, info: 'ResolveInfo', **args) -> AsyncGenerator[Any, None]:
 
 class Pruned(ObjectType):
     class Meta:
-        description = """WFS Nodes/Edges that have been removed."""
+        description = 'WFS Nodes/Edges that have been removed.'
     workflow = String()
     families = graphene.List(String, default_value=[])
     family_proxies = graphene.List(String, default_value=[])
@@ -2246,7 +2251,7 @@ class Delta(Interface):
 
     families = graphene.List(
         Family,
-        description="""Family definitions.""",
+        description='Family definitions.',
         args=DEF_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2255,7 +2260,7 @@ class Delta(Interface):
     )
     family_proxies = graphene.List(
         FamilyProxy,
-        description="""Family cycle instances.""",
+        description='Family cycle instances.',
         args=PROXY_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2264,7 +2269,7 @@ class Delta(Interface):
     )
     jobs = graphene.List(
         Job,
-        description="""Jobs.""",
+        description='Jobs.',
         args=JOB_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2273,7 +2278,7 @@ class Delta(Interface):
     )
     tasks = graphene.List(
         Task,
-        description="""Task definitions.""",
+        description='Task definitions.',
         args=DEF_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2282,7 +2287,7 @@ class Delta(Interface):
     )
     task_proxies = graphene.List(
         TaskProxy,
-        description="""Task cycle instances.""",
+        description='Task cycle instances.',
         args=PROXY_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2291,7 +2296,7 @@ class Delta(Interface):
     )
     edges = graphene.List(
         Edge,
-        description="""Graph edges""",
+        description='Graph edges',
         args=EDGE_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2310,18 +2315,18 @@ class Delta(Interface):
 
 class Added(ObjectType):
     class Meta:
-        description = """Added node/edge deltas."""
+        description = 'Added node/edge deltas.'
         interfaces = (Delta,)
 
 
 class Updated(ObjectType):
     class Meta:
-        description = """Updated node/edge deltas."""
+        description = 'Updated node/edge deltas.'
         interfaces = (Delta,)
 
     families = graphene.List(
         Family,
-        description="""Family definitions.""",
+        description='Family definitions.',
         args=DEF_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2330,7 +2335,7 @@ class Meta:
     )
     family_proxies = graphene.List(
         FamilyProxy,
-        description="""Family cycle instances.""",
+        description='Family cycle instances.',
         args=PROXY_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2339,7 +2344,7 @@ class Meta:
     )
     jobs = graphene.List(
         Job,
-        description="""Jobs.""",
+        description='Jobs.',
         args=JOB_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2348,7 +2353,7 @@ class Meta:
     )
     tasks = graphene.List(
         Task,
-        description="""Task definitions.""",
+        description='Task definitions.',
         args=DEF_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2357,7 +2362,7 @@ class Meta:
     )
     task_proxies = graphene.List(
         TaskProxy,
-        description="""Task cycle instances.""",
+        description='Task cycle instances.',
         args=PROXY_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2366,7 +2371,7 @@ class Meta:
     )
     edges = graphene.List(
         Edge,
-        description="""Graph edges""",
+        description='Graph edges.',
         args=EDGE_ARGS,
         strip_null=Boolean(),
         delta_store=Boolean(default_value=True),
@@ -2385,7 +2390,7 @@ class Meta:
 
 class Deltas(ObjectType):
     class Meta:
-        description = """Grouped deltas of the WFS publish"""
+        description = 'Grouped deltas of the WFS publish.'
     id = ID()  # noqa: A003 (required for schema definition)
     shutdown = Boolean(default_value=False)
     added = Field(
@@ -2408,13 +2413,13 @@ class Meta:
 class Subscriptions(ObjectType):
     """Defines the subscriptions available in the schema."""
     class Meta:
-        description = """Multi-Workflow root level subscriptions."""
+        description = 'Multi-Workflow root level subscriptions.'
 
     deltas = Field(
         Deltas,
         description=Deltas._meta.description,
         workflows=graphene.List(
-            ID, description="List of full ID, i.e. `~user/workflow_id`"
+            ID, description="List of full ID, i.e. `~user/workflow_id`."
         ),
         strip_null=Boolean(default_value=False),
         initial_burst=Boolean(default_value=True),

From 0cf10674c78499379a834ced9ed6feda4cba7495 Mon Sep 17 00:00:00 2001
From: Hilary James Oliver <hilary.j.oliver@gmail.com>
Date: Fri, 16 Aug 2024 10:46:44 +1200
Subject: [PATCH 24/28] Update cylc/flow/network/schema.py

Co-authored-by: Oliver Sanders <oliver.sanders@metoffice.gov.uk>
---
 cylc/flow/network/schema.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/cylc/flow/network/schema.py b/cylc/flow/network/schema.py
index 8e20225d4b6..788ec539dcb 100644
--- a/cylc/flow/network/schema.py
+++ b/cylc/flow/network/schema.py
@@ -2229,7 +2229,10 @@ def delta_subs(root, info: 'ResolveInfo', **args) -> AsyncGenerator[Any, None]:
 
 class Pruned(ObjectType):
     class Meta:
-        description = 'WFS Nodes/Edges that have been removed.'
+        description = sstrip('''
+            Objects (e.g. workflows, tasks, jobs) which have been removed from
+            the store.       
+        ''')   
     workflow = String()
     families = graphene.List(String, default_value=[])
     family_proxies = graphene.List(String, default_value=[])

From 4b58a6a7f53f69789466ac88b57832b1433b1679 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Fri, 16 Aug 2024 11:36:43 +0100
Subject: [PATCH 25/28] fixeth lint failure (#6306)

---
 cylc/flow/network/schema.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cylc/flow/network/schema.py b/cylc/flow/network/schema.py
index 788ec539dcb..355f12c4981 100644
--- a/cylc/flow/network/schema.py
+++ b/cylc/flow/network/schema.py
@@ -2231,8 +2231,8 @@ class Pruned(ObjectType):
     class Meta:
         description = sstrip('''
             Objects (e.g. workflows, tasks, jobs) which have been removed from
-            the store.       
-        ''')   
+            the store.
+        ''')
     workflow = String()
     families = graphene.List(String, default_value=[])
     family_proxies = graphene.List(String, default_value=[])

From dab785b395541bd66c8bcced5507d750d8f9fbd2 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Tue, 20 Aug 2024 16:34:28 +0100
Subject: [PATCH 26/28] flake8-bugbear: opt-in to a couple of opinionated
 warnings

---
 cylc/flow/config.py                 |  2 +-
 cylc/flow/scripts/lint.py           | 50 +++++++++++++++++------------
 cylc/flow/scripts/report_timings.py |  6 ++--
 tests/unit/scripts/test_lint.py     |  3 +-
 tox.ini                             |  6 ++++
 5 files changed, 41 insertions(+), 26 deletions(-)

diff --git a/cylc/flow/config.py b/cylc/flow/config.py
index cb987ac6d4c..b91e21afc7e 100644
--- a/cylc/flow/config.py
+++ b/cylc/flow/config.py
@@ -2463,7 +2463,7 @@ def get_taskdef(
                             'Duplicate task message in'
                             f' "[runtime][{name}][outputs]'
                             f'{output} = {message}" - messages must be unique'
-                        )
+                        ) from None
                     valid, msg = TaskOutputValidator.validate(output)
                     if not valid:
                         raise WorkflowConfigError(
diff --git a/cylc/flow/scripts/lint.py b/cylc/flow/scripts/lint.py
index c8aa7279590..3389143976d 100755
--- a/cylc/flow/scripts/lint.py
+++ b/cylc/flow/scripts/lint.py
@@ -51,53 +51,62 @@
 """
 import functools
 import pkgutil
-from pathlib import Path
 import re
-import sys
 import shutil
+import sys
+from collections import Counter
+from pathlib import Path
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Counter as CounterType,
+    Dict,
+    Iterator,
+    List,
+    Optional,
+    Union,
+)
+
 try:
     # BACK COMPAT: tomli
     #   Support for Python versions before tomllib was added to the
     #   standard library.
     # FROM: Python 3.7
     # TO: Python: 3.10
-    from tomli import (
-        loads as toml_loads,
-        TOMLDecodeError,
-    )
+    from tomli import TOMLDecodeError, loads as toml_loads
 except ImportError:
     from tomllib import (  # type: ignore[no-redef]
         loads as toml_loads,
         TOMLDecodeError,
     )
-from typing import (
-    TYPE_CHECKING, Any, Callable, Dict, Iterator, List, Optional, Union
-)
 
 from ansimarkup import parse as cparse
 
-from cylc.flow import LOG
-from cylc.flow.exceptions import CylcError
 import cylc.flow.flags
-from cylc.flow import job_runner_handlers
+from cylc.flow import LOG, job_runner_handlers
+from cylc.flow.cfgspec.workflow import SPEC, upg
+from cylc.flow.exceptions import CylcError
+from cylc.flow.id_cli import parse_id
 from cylc.flow.job_runner_mgr import JobRunnerManager
 from cylc.flow.loggingutil import set_timestamps
 from cylc.flow.option_parsers import (
+    WORKFLOW_ID_OR_PATH_ARG_DOC,
     CylcOptionParser as COP,
-    WORKFLOW_ID_OR_PATH_ARG_DOC
 )
-from cylc.flow.cfgspec.workflow import upg, SPEC
-from cylc.flow.id_cli import parse_id
 from cylc.flow.parsec.config import ParsecConfig
 from cylc.flow.scripts.cylc import DEAD_ENDS
 from cylc.flow.terminal import cli_function
 
+
 if TYPE_CHECKING:
+    from optparse import Values
+
     # BACK COMPAT: typing_extensions.Literal
     # FROM: Python 3.7
     # TO: Python 3.8
     from typing_extensions import Literal
-    from optparse import Values
+
 
 LINT_TABLE = ['tool', 'cylc', 'lint']
 LINT_SECTION = '.'.join(LINT_TABLE)
@@ -1114,7 +1123,7 @@ def check_cylc_file(
     file: Path,
     file_rel: Path,
     checks: Dict[str, dict],
-    counter: Dict[str, int],
+    counter: CounterType[str],
     modify: bool = False,
 ):
     """Check A Cylc File for Cylc 7 Config"""
@@ -1172,7 +1181,7 @@ def lint(
     file_rel: Path,
     lines: Iterator[str],
     checks: Dict[str, dict],
-    counter: Dict[str, int],
+    counter: CounterType[str],
     modify: bool = False,
     write: Callable = print
 ) -> Iterator[str]:
@@ -1186,7 +1195,7 @@ def lint(
             Iterator which produces one line of text at a time
             e.g. open(file) or iter(['foo\n', 'bar\n', 'baz\n'].
         counter:
-            Dictionary for counting lint hits per category.
+            Counter for counting lint hits per category.
         modify:
             If True, this generator will yield the file one line at a time
             with comments inserted to help users fix their lint.
@@ -1238,7 +1247,6 @@ def lint(
                     msg = check_meta['short'].format(**check)
                 else:
                     msg = check_meta['short']
-                counter.setdefault(check_meta['purpose'], 0)
                 counter[check_meta['purpose']] += 1
                 if modify:
                     # insert a command to help the user
@@ -1473,7 +1481,7 @@ def main(parser: COP, options: 'Values', target=None) -> None:
     )
 
     # Check each file matching a pattern:
-    counter: Dict[str, int] = {}
+    counter: CounterType[str] = Counter()
     for file in get_cylc_files(target, mergedopts[EXCLUDE]):
         LOG.debug(f'Checking {file}')
         check_cylc_file(
diff --git a/cylc/flow/scripts/report_timings.py b/cylc/flow/scripts/report_timings.py
index 362f40953c2..21a4df06edc 100755
--- a/cylc/flow/scripts/report_timings.py
+++ b/cylc/flow/scripts/report_timings.py
@@ -48,13 +48,12 @@
 the database query to obtain the timing information may take some time.
 """
 
-import collections
 import contextlib
 import io as StringIO
 import sys
+from collections import Counter
 from typing import TYPE_CHECKING
 
-
 from cylc.flow import LOG
 from cylc.flow.exceptions import CylcError
 from cylc.flow.id_cli import parse_id
@@ -66,6 +65,7 @@
 from cylc.flow.rundb import CylcWorkflowDAO
 from cylc.flow.terminal import cli_function
 
+
 if TYPE_CHECKING:
     from optparse import Values
 
@@ -278,7 +278,7 @@ def _reshape_timings(timings):
             # there are duplicate entries in the index (see #2509).  The
             # best way around this seems to be to add an intermediate index
             # level (in this case a retry counter) to de-duplicate indices.
-            counts = collections.defaultdict(int)
+            counts = Counter()
             retry = []
             for t in timings.index:
                 counts[t] += 1
diff --git a/tests/unit/scripts/test_lint.py b/tests/unit/scripts/test_lint.py
index 527a33bdfb5..20ff738a1ac 100644
--- a/tests/unit/scripts/test_lint.py
+++ b/tests/unit/scripts/test_lint.py
@@ -16,6 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """Tests `cylc lint` CLI Utility."""
 
+from collections import Counter
 import logging
 from pathlib import Path
 from pprint import pformat
@@ -190,7 +191,7 @@
 
 def lint_text(text, checks, ignores=None, modify=False):
     checks = parse_checks(checks, ignores)
-    counter = {}
+    counter = Counter()
     messages = []
     outlines = [
         line
diff --git a/tox.ini b/tox.ini
index d9954dbb7e8..5e87dc7443c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,4 +1,10 @@
 [flake8]
+extend-select=
+    ; Use `raise from` inside `except` for error handling
+    ; https://docs.python.org/3/tutorial/errors.html#exception-chaining
+    B904
+    ; Use Counter() instead of defaultdict(int) to avoid excessive memory use:
+    B910
 ignore=
     ; module level import not at top of file
     E402,

From 55e8b581b0746a3d52a66785b5a6373af2b96a99 Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Tue, 20 Aug 2024 18:20:59 +0100
Subject: [PATCH 27/28] Partially revert
 c24ff05546e049a365d48e1e244bef75c02012eb

These raises were incorrectly altered
---
 cylc/flow/config.py       | 4 ++--
 cylc/flow/dbstatecheck.py | 4 ++--
 cylc/flow/xtrigger_mgr.py | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/cylc/flow/config.py b/cylc/flow/config.py
index b91e21afc7e..df27078ee44 100644
--- a/cylc/flow/config.py
+++ b/cylc/flow/config.py
@@ -2470,14 +2470,14 @@ def get_taskdef(
                             f'Invalid task output "'
                             f'[runtime][{name}][outputs]'
                             f'{output} = {message}" - {msg}'
-                        ) from None
+                        )
                     valid, msg = TaskMessageValidator.validate(message)
                     if not valid:
                         raise WorkflowConfigError(
                             f'Invalid task message "'
                             f'[runtime][{name}][outputs]'
                             f'{output} = {message}" - {msg}'
-                        ) from None
+                        )
                     self.taskdefs[name].add_output(output, message)
 
         return self.taskdefs[name]
diff --git a/cylc/flow/dbstatecheck.py b/cylc/flow/dbstatecheck.py
index 1890c697c71..88dd5de3be1 100644
--- a/cylc/flow/dbstatecheck.py
+++ b/cylc/flow/dbstatecheck.py
@@ -78,7 +78,7 @@ def __init__(self, rund, workflow, db_path=None):
         try:
             self.db_point_fmt = self._get_db_point_format()
             self.c7_back_compat_mode = False
-        except sqlite3.OperationalError:
+        except sqlite3.OperationalError as exc:
             # BACK COMPAT: Cylc 7 DB (see method below).
             try:
                 self.db_point_fmt = self._get_db_point_format_compat()
@@ -86,7 +86,7 @@ def __init__(self, rund, workflow, db_path=None):
             except sqlite3.OperationalError:
                 with suppress(Exception):
                     self.conn.close()
-                raise
+                raise exc from None  # original error
 
     def __enter__(self):
         return self
diff --git a/cylc/flow/xtrigger_mgr.py b/cylc/flow/xtrigger_mgr.py
index 525a19cf2d3..ff7c8743846 100644
--- a/cylc/flow/xtrigger_mgr.py
+++ b/cylc/flow/xtrigger_mgr.py
@@ -390,7 +390,7 @@ def _try_workflow_state_backcompat(
             bound_args = sig.bind(*fctx.func_args, **fctx.func_kwargs)
         except TypeError:
             # failed signature check for backcompat function
-            raise
+            raise err from None  # original signature check error
 
         old_sig_str = fctx.get_signature()
         upg_sig_str = "workflow_state({})".format(

From a91e0ef0355a882de71d5c7a0f012e0b56c7310b Mon Sep 17 00:00:00 2001
From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com>
Date: Tue, 20 Aug 2024 18:23:45 +0100
Subject: [PATCH 28/28] Exception chaining: do not swallow potentially useful
 context of caught exceptions

---
 cylc/flow/main_loop/health_check.py |  4 ++--
 cylc/flow/network/__init__.py       | 25 ++++++++++++++-----------
 cylc/flow/param_expand.py           |  4 ++--
 cylc/flow/scripts/cylc.py           |  4 ++--
 cylc/flow/workflow_db_mgr.py        |  4 ++--
 cylc/flow/workflow_files.py         |  4 ++--
 6 files changed, 24 insertions(+), 21 deletions(-)

diff --git a/cylc/flow/main_loop/health_check.py b/cylc/flow/main_loop/health_check.py
index a470fd3cb88..6e42724ba91 100644
--- a/cylc/flow/main_loop/health_check.py
+++ b/cylc/flow/main_loop/health_check.py
@@ -52,8 +52,8 @@ def _check_contact_file(scheduler):
             scheduler.workflow)
         if contact_data != scheduler.contact_data:
             raise CylcError('contact file modified')
-    except (AssertionError, IOError, ValueError, ServiceFileError):
+    except (AssertionError, IOError, ValueError, ServiceFileError) as exc:
         raise CylcError(
             '%s: contact file corrupted/modified and may be left'
             % workflow_files.get_contact_file_path(scheduler.workflow)
-        ) from None
+        ) from exc
diff --git a/cylc/flow/network/__init__.py b/cylc/flow/network/__init__.py
index 315a57a23ba..42b79475ca5 100644
--- a/cylc/flow/network/__init__.py
+++ b/cylc/flow/network/__init__.py
@@ -176,16 +176,14 @@ def _socket_bind(self, min_port, max_port, srv_prv_key_loc=None):
                 srv_prv_key_info.full_key_path)
         except ValueError:
             raise ServiceFileError(
-                f"Failed to find server's public "
-                f"key in "
+                "Failed to find server's public key in "
                 f"{srv_prv_key_info.full_key_path}."
             ) from None
-        except OSError:
+        except OSError as exc:
             raise ServiceFileError(
-                f"IO error opening server's private "
-                f"key from "
+                "IO error opening server's private key from "
                 f"{srv_prv_key_info.full_key_path}."
-            ) from None
+            ) from exc
         if server_private_key is None:  # this can't be caught by exception
             raise ServiceFileError(
                 f"Failed to find server's private "
@@ -238,8 +236,10 @@ def _socket_connect(self, host, port, srv_public_key_loc=None):
         try:
             client_public_key, client_priv_key = zmq.auth.load_certificate(
                 client_priv_key_info.full_key_path)
-        except (OSError, ValueError):
+        except ValueError:
             raise ClientError(error_msg) from None
+        except OSError as exc:
+            raise ClientError(error_msg) from exc
         if client_priv_key is None:  # this can't be caught by exception
             raise ClientError(error_msg)
         self.socket.curve_publickey = client_public_key
@@ -247,6 +247,9 @@ def _socket_connect(self, host, port, srv_public_key_loc=None):
 
         # A client can only connect to the server if it knows its public key,
         # so we grab this from the location it was created on the filesystem:
+        error_msg = (
+            "Failed to load the workflow's public key, so cannot connect."
+        )
         try:
             # 'load_certificate' will try to load both public & private keys
             # from a provided file but will return None, not throw an error,
@@ -256,10 +259,10 @@ def _socket_connect(self, host, port, srv_public_key_loc=None):
             server_public_key = zmq.auth.load_certificate(
                 srv_pub_key_info.full_key_path)[0]
             self.socket.curve_serverkey = server_public_key
-        except (OSError, ValueError):  # ValueError raised w/ no public key
-            raise ClientError(
-                "Failed to load the workflow's public key, so cannot connect."
-            ) from None
+        except ValueError:  # ValueError raised w/ no public key
+            raise ClientError(error_msg) from None
+        except OSError as exc:
+            raise ClientError(error_msg) from exc
 
         self.socket.connect(f'tcp://{host}:{port}')
 
diff --git a/cylc/flow/param_expand.py b/cylc/flow/param_expand.py
index 6ab104404e4..22cf1aa8665 100644
--- a/cylc/flow/param_expand.py
+++ b/cylc/flow/param_expand.py
@@ -196,7 +196,7 @@ def _expand_name(self, results, tmpl, params, spec_vals=None):
                 results.append((tmpl % current_values, current_values))
             except KeyError as exc:
                 raise ParamExpandError(
-                    'parameter %s is not ' 'defined.' % str(exc.args[0])
+                    'parameter %s is not defined.' % str(exc.args[0])
                 ) from None
         else:
             for param_val in params[0][1]:
@@ -427,7 +427,7 @@ def _expand_graph(self, line, all_params,
                     repl = tmpl % param_values
                 except KeyError as exc:
                     raise ParamExpandError(
-                        'parameter %s is not ' 'defined.' % str(exc.args[0])
+                        'parameter %s is not defined.' % str(exc.args[0])
                     ) from None
                 line = line.replace('<' + p_group + '>', repl)
             if line:
diff --git a/cylc/flow/scripts/cylc.py b/cylc/flow/scripts/cylc.py
index 209512264a9..b0a707c5428 100644
--- a/cylc/flow/scripts/cylc.py
+++ b/cylc/flow/scripts/cylc.py
@@ -605,14 +605,14 @@ def pycoverage(cmd_args):  # pragma: no cover
             data_file=str(cylc_wc / '.coverage'),
             source=[str(cylc_wc / 'cylc')]
         )
-    except coverage.misc.CoverageException:
+    except coverage.misc.CoverageException as exc:
         raise Exception(
             # make sure this exception is visible in the traceback
             '\n\n*****************************\n\n'
             'Could not initiate coverage, likely because Cylc was not '
             'installed in editable mode.'
             '\n\n*****************************\n\n'
-        ) from None
+        ) from exc
 
     # start the coverage running
     cov.start()
diff --git a/cylc/flow/workflow_db_mgr.py b/cylc/flow/workflow_db_mgr.py
index d45108ca587..bb7a287e944 100644
--- a/cylc/flow/workflow_db_mgr.py
+++ b/cylc/flow/workflow_db_mgr.py
@@ -700,10 +700,10 @@ def _get_last_run_version(cls, pri_dao: CylcWorkflowDAO) -> Version:
                 ''',  # nosec (table name is a code constant)
                 [cls.KEY_CYLC_VERSION]
             ).fetchone()[0]
-        except (TypeError, OperationalError):
+        except (TypeError, OperationalError) as exc:
             raise ServiceFileError(
                 f"{INCOMPAT_MSG}, or is corrupted."
-            ) from None
+            ) from exc
         return parse_version(last_run_ver)
 
     @classmethod
diff --git a/cylc/flow/workflow_files.py b/cylc/flow/workflow_files.py
index 95626125f89..1550a390f37 100644
--- a/cylc/flow/workflow_files.py
+++ b/cylc/flow/workflow_files.py
@@ -634,8 +634,8 @@ def load_contact_file(id_: str, run_dir=None) -> Dict[str, str]:
     try:
         with open(path) as f:
             file_content = f.read()
-    except IOError:
-        raise ServiceFileError("Couldn't load contact file") from None
+    except IOError as exc:
+        raise ServiceFileError("Couldn't load contact file") from exc
     data: Dict[str, str] = {}
     for line in file_content.splitlines():
         key, value = [item.strip() for item in line.split("=", 1)]