From c68b22372a2e35a0874b39fbe1b7157f9a2553ef Mon Sep 17 00:00:00 2001 From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com> Date: Tue, 10 Sep 2024 18:15:56 +0100 Subject: [PATCH 1/5] Tidy & add unit test --- cylc/flow/clean.py | 5 +- cylc/flow/pathutil.py | 3 +- tests/unit/filetree.py | 222 ++++++++++++++++++++++++------------ tests/unit/test_clean.py | 41 ++++--- tests/unit/test_pathutil.py | 16 +++ 5 files changed, 195 insertions(+), 92 deletions(-) diff --git a/cylc/flow/clean.py b/cylc/flow/clean.py index b38f01b12fc..f73b0364766 100644 --- a/cylc/flow/clean.py +++ b/cylc/flow/clean.py @@ -259,7 +259,7 @@ def glob_in_run_dir( """Execute a (recursive) glob search in the given run directory. Returns list of any absolute paths that match the pattern. However: - * Does not follow symlinks (apart from the spcedified symlink dirs). + * Does not follow symlinks (apart from the specified symlink dirs). * Also does not return matching subpaths of matching directories (because that would be redundant). @@ -281,6 +281,9 @@ def glob_in_run_dir( results: List[Path] = [] subpath_excludes: Set[Path] = set() for path in matches: + # Iterate down through ancestors (starting at the run dir) to + # weed out redundant subpaths of matched directories and subpaths of + # non-standard symlinks for rel_ancestor in reversed(path.relative_to(run_dir).parents): ancestor = run_dir / rel_ancestor if ancestor in subpath_excludes: diff --git a/cylc/flow/pathutil.py b/cylc/flow/pathutil.py index aa097a2e680..99f4c4e518f 100644 --- a/cylc/flow/pathutil.py +++ b/cylc/flow/pathutil.py @@ -481,7 +481,8 @@ def parse_rm_dirs(rm_dirs: Iterable[str]) -> Set[str]: def is_relative_to(path1: Union[Path, str], path2: Union[Path, str]) -> bool: - """Return whether or not path1 is relative to path2. + """Return whether or not path1 is relative to path2 (including if they are + the same path). Normalizes both paths to avoid trickery with paths containing `..` somewhere in them. diff --git a/tests/unit/filetree.py b/tests/unit/filetree.py index bef09e4e556..d1830b51cd3 100644 --- a/tests/unit/filetree.py +++ b/tests/unit/filetree.py @@ -27,16 +27,21 @@ 'file.txt': None } }, - # Symlinks are represented by pathlib.Path, with the target represented - # by the relative path from the tmp_path directory: - 'symlink': Path('dir/another-dir') + # Symlinks are represented by the Symlink class, with the target + # represented by the relative path from the tmp_path directory: + 'symlink': Symlink('dir/another-dir') } """ -from pathlib import Path +from pathlib import Path, PosixPath from typing import Any, Dict, List +class Symlink(PosixPath): + """A class to represent a symlink target.""" + ... + + def create_filetree( filetree: Dict[str, Any], location: Path, root: Path ) -> None: @@ -53,10 +58,9 @@ def create_filetree( if isinstance(entry, dict): path.mkdir(exist_ok=True) create_filetree(entry, path, root) - elif isinstance(entry, Path): + elif isinstance(entry, Symlink): path.symlink_to(root / entry) else: - path.touch() @@ -79,89 +83,155 @@ def get_filetree_as_list( FILETREE_1 = { - 'cylc-run': {'foo': {'bar': { - '.service': {'db': None}, - 'flow.cylc': None, - 'log': Path('sym/cylc-run/foo/bar/log'), - 'mirkwood': Path('you-shall-not-pass/mirkwood'), - 'rincewind.txt': Path('you-shall-not-pass/rincewind.txt') - }}}, - 'sym': {'cylc-run': {'foo': {'bar': { - 'log': { - 'darmok': Path('you-shall-not-pass/darmok'), - 'temba.txt': Path('you-shall-not-pass/temba.txt'), - 'bib': { - 'fortuna.txt': None - } - } - }}}}, + 'cylc-run': { + 'foo': { + 'bar': { + '.service': { + 'db': None, + }, + 'flow.cylc': None, + 'log': Symlink('sym/cylc-run/foo/bar/log'), + 'mirkwood': Symlink('you-shall-not-pass/mirkwood'), + 'rincewind.txt': Symlink('you-shall-not-pass/rincewind.txt'), + }, + }, + }, + 'sym': { + 'cylc-run': { + 'foo': { + 'bar': { + 'log': { + 'darmok': Symlink('you-shall-not-pass/darmok'), + 'temba.txt': Symlink('you-shall-not-pass/temba.txt'), + 'bib': { + 'fortuna.txt': None, + }, + }, + }, + }, + }, + }, 'you-shall-not-pass': { # Nothing in here should get deleted 'darmok': { - 'jalad.txt': None + 'jalad.txt': None, }, 'mirkwood': { - 'spiders.txt': None + 'spiders.txt': None, }, 'rincewind.txt': None, - 'temba.txt': None - } + 'temba.txt': None, + }, } FILETREE_2 = { - 'cylc-run': {'foo': {'bar': Path('sym-run/cylc-run/foo/bar')}}, - 'sym-run': {'cylc-run': {'foo': {'bar': { - '.service': {'db': None}, - 'flow.cylc': None, - 'share': Path('sym-share/cylc-run/foo/bar/share') - }}}}, - 'sym-share': {'cylc-run': {'foo': {'bar': { - 'share': { - 'cycle': Path('sym-cycle/cylc-run/foo/bar/share/cycle') - } - }}}}, - 'sym-cycle': {'cylc-run': {'foo': {'bar': { - 'share': { - 'cycle': { - 'macklunkey.txt': None - } - } - }}}}, - 'you-shall-not-pass': {} + 'cylc-run': {'foo': {'bar': Symlink('sym-run/cylc-run/foo/bar')}}, + 'sym-run': { + 'cylc-run': { + 'foo': { + 'bar': { + '.service': { + 'db': None, + }, + 'flow.cylc': None, + 'share': Symlink('sym-share/cylc-run/foo/bar/share'), + }, + }, + }, + }, + 'sym-share': { + 'cylc-run': { + 'foo': { + 'bar': { + 'share': { + 'cycle': Symlink( + 'sym-cycle/cylc-run/foo/bar/share/cycle' + ), + }, + }, + }, + }, + }, + 'sym-cycle': { + 'cylc-run': { + 'foo': { + 'bar': { + 'share': { + 'cycle': { + 'macklunkey.txt': None, + }, + }, + }, + }, + }, + }, + 'you-shall-not-pass': {}, } FILETREE_3 = { - 'cylc-run': {'foo': {'bar': Path('sym-run/cylc-run/foo/bar')}}, - 'sym-run': {'cylc-run': {'foo': {'bar': { - '.service': {'db': None}, - 'flow.cylc': None, - 'share': { - 'cycle': Path('sym-cycle/cylc-run/foo/bar/share/cycle') - } - }}}}, - 'sym-cycle': {'cylc-run': {'foo': {'bar': { - 'share': { - 'cycle': { - 'sokath.txt': None - } - } - }}}}, - 'you-shall-not-pass': {} + 'cylc-run': { + 'foo': { + 'bar': Symlink('sym-run/cylc-run/foo/bar'), + }, + }, + 'sym-run': { + 'cylc-run': { + 'foo': { + 'bar': { + '.service': { + 'db': None, + }, + 'flow.cylc': None, + 'share': { + 'cycle': Symlink( + 'sym-cycle/cylc-run/foo/bar/share/cycle' + ), + }, + }, + }, + }, + }, + 'sym-cycle': { + 'cylc-run': { + 'foo': { + 'bar': { + 'share': { + 'cycle': { + 'sokath.txt': None, + }, + }, + }, + }, + }, + }, + 'you-shall-not-pass': {}, } FILETREE_4 = { - 'cylc-run': {'foo': {'bar': { - '.service': {'db': None}, - 'flow.cylc': None, - 'share': { - 'cycle': Path('sym-cycle/cylc-run/foo/bar/share/cycle') - } - }}}, - 'sym-cycle': {'cylc-run': {'foo': {'bar': { - 'share': { - 'cycle': { - 'kiazi.txt': None - } - } - }}}}, - 'you-shall-not-pass': {} + 'cylc-run': { + 'foo': { + 'bar': { + '.service': { + 'db': None, + }, + 'flow.cylc': None, + 'share': { + 'cycle': Symlink('sym-cycle/cylc-run/foo/bar/share/cycle'), + }, + }, + }, + }, + 'sym-cycle': { + 'cylc-run': { + 'foo': { + 'bar': { + 'share': { + 'cycle': { + 'kiazi.txt': None, + }, + }, + }, + }, + }, + }, + 'you-shall-not-pass': {}, } diff --git a/tests/unit/test_clean.py b/tests/unit/test_clean.py index 2a67daf4bf4..5227e9d3876 100644 --- a/tests/unit/test_clean.py +++ b/tests/unit/test_clean.py @@ -34,8 +34,7 @@ import pytest -from cylc.flow import CYLC_LOG -from cylc.flow import clean as cylc_clean +from cylc.flow import CYLC_LOG, clean as cylc_clean from cylc.flow.clean import ( _clean_using_glob, _remote_clean_cmd, @@ -64,10 +63,12 @@ FILETREE_2, FILETREE_3, FILETREE_4, + Symlink, create_filetree, get_filetree_as_list, ) + NonCallableFixture = Any @@ -536,7 +537,7 @@ def _filetree_for_testing_cylc_clean( 'cylc-run': {'foo': {'bar': { '.service': {'db': None}, 'flow.cylc': None, - 'rincewind.txt': Path('whatever') + 'rincewind.txt': Symlink('whatever') }}}, 'sym': {'cylc-run': {'foo': {'bar': {}}}} } @@ -548,12 +549,12 @@ def _filetree_for_testing_cylc_clean( 'cylc-run': {'foo': {'bar': { '.service': {'db': None}, 'flow.cylc': None, - 'log': Path('whatever'), - 'mirkwood': Path('whatever') + 'log': Symlink('whatever'), + 'mirkwood': Symlink('whatever') }}}, 'sym': {'cylc-run': {'foo': {'bar': { 'log': { - 'darmok': Path('whatever'), + 'darmok': Symlink('whatever'), 'bib': {} } }}}} @@ -612,7 +613,7 @@ def test__clean_using_glob( 'cylc-run': {'foo': {'bar': { '.service': {'db': None}, 'flow.cylc': None, - 'rincewind.txt': Path('whatever') + 'rincewind.txt': Symlink('whatever') }}}, 'sym': {'cylc-run': {}} }, @@ -625,12 +626,12 @@ def test__clean_using_glob( 'cylc-run': {'foo': {'bar': { '.service': {'db': None}, 'flow.cylc': None, - 'log': Path('whatever'), - 'mirkwood': Path('whatever') + 'log': Symlink('whatever'), + 'mirkwood': Symlink('whatever') }}}, 'sym': {'cylc-run': {'foo': {'bar': { 'log': { - 'darmok': Path('whatever'), + 'darmok': Symlink('whatever'), 'bib': {} } }}}} @@ -641,11 +642,13 @@ def test__clean_using_glob( {'**/cycle'}, FILETREE_2, { - 'cylc-run': {'foo': {'bar': Path('sym-run/cylc-run/foo/bar')}}, + 'cylc-run': {'foo': { + 'bar': Symlink('sym-run/cylc-run/foo/bar') + }}, 'sym-run': {'cylc-run': {'foo': {'bar': { '.service': {'db': None}, 'flow.cylc': None, - 'share': Path('sym-share/cylc-run/foo/bar/share') + 'share': Symlink('sym-share/cylc-run/foo/bar/share') }}}}, 'sym-share': {'cylc-run': {'foo': {'bar': { 'share': {} @@ -658,7 +661,9 @@ def test__clean_using_glob( {'share'}, FILETREE_2, { - 'cylc-run': {'foo': {'bar': Path('sym-run/cylc-run/foo/bar')}}, + 'cylc-run': {'foo': { + 'bar': Symlink('sym-run/cylc-run/foo/bar') + }}, 'sym-run': {'cylc-run': {'foo': {'bar': { '.service': {'db': None}, 'flow.cylc': None, @@ -689,7 +694,9 @@ def test__clean_using_glob( {'*'}, FILETREE_2, { - 'cylc-run': {'foo': {'bar': Path('sym-run/cylc-run/foo/bar')}}, + 'cylc-run': {'foo': { + 'bar': Symlink('sym-run/cylc-run/foo/bar') + }}, 'sym-run': {'cylc-run': {'foo': {'bar': { '.service': {'db': None}, }}}}, @@ -1088,6 +1095,12 @@ def test_clean_top_level(tmp_run_dir: Callable): 'cylc-run/foo/bar/share/cycle'], id="filetree2 **" ), + pytest.param( + 'share', + FILETREE_2, + ['cylc-run/foo/bar/share'], + id="filetree2 share" + ), pytest.param( '**', FILETREE_3, diff --git a/tests/unit/test_pathutil.py b/tests/unit/test_pathutil.py index 6ed9e9ec120..41a47a71409 100644 --- a/tests/unit/test_pathutil.py +++ b/tests/unit/test_pathutil.py @@ -40,6 +40,7 @@ get_workflow_run_share_dir, get_workflow_run_work_dir, get_workflow_test_log_path, + is_relative_to, make_localhost_symlinks, make_workflow_run_tree, parse_rm_dirs, @@ -576,3 +577,18 @@ def test_get_workflow_name_from_id( result = get_workflow_name_from_id(id_) assert result == name + + +@pytest.mark.parametrize('abs_path', [True, False]) +@pytest.mark.parametrize('path1, path2, expected', [ + param('/foo/bar/baz', '/foo/bar', True, id="child"), + param('/foo/bar', '/foo/bar/baz', False, id="parent"), + param('/foo/bar', '/foo/bar', True, id="same-path"), + param('/cat/dog', '/hat/bog', False, id="different"), + param('/a/b/c/../x/y', '/a/b/x', True, id="trickery"), +]) +def test_is_relative_to(abs_path, path1, path2, expected): + if not abs_path: # absolute & relative versions of same test + path1.lstrip('/') + path2.lstrip('/') + assert is_relative_to(path1, path2) == expected From a637aa334ac47aa005f16c20eae76fd27386c2ce Mon Sep 17 00:00:00 2001 From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com> Date: Tue, 10 Sep 2024 18:43:51 +0100 Subject: [PATCH 2/5] `cylc clean`: fix bug where `--rm share` would not clean `share/cycle` symlink first --- changes.d/6364.fix.md | 1 + cylc/flow/clean.py | 17 ++++++++++++----- tests/unit/test_clean.py | 16 ++-------------- 3 files changed, 15 insertions(+), 19 deletions(-) create mode 100644 changes.d/6364.fix.md diff --git a/changes.d/6364.fix.md b/changes.d/6364.fix.md new file mode 100644 index 00000000000..9ff613e2c18 --- /dev/null +++ b/changes.d/6364.fix.md @@ -0,0 +1 @@ +Fixed bug where `cylc clean --rm share` would not take care of removing the target of the `share/cycle` symlink directory. diff --git a/cylc/flow/clean.py b/cylc/flow/clean.py index f73b0364766..fda1d4f1dd8 100644 --- a/cylc/flow/clean.py +++ b/cylc/flow/clean.py @@ -57,6 +57,7 @@ ) from cylc.flow.pathutil import ( get_workflow_run_dir, + is_relative_to, parse_rm_dirs, remove_dir_and_target, remove_dir_or_file, @@ -329,13 +330,19 @@ def _clean_using_glob( LOG.info(f"No files matching '{pattern}' in {run_dir}") return # First clean any matching symlink dirs - for path in abs_symlink_dirs: - if path in matches: - remove_dir_and_target(path) - if path == run_dir: + for symlink_dir in abs_symlink_dirs: + # Note: must clean e.g. share/cycle/ before share/ if the former + # is a symlink even if only the latter was specified. + if ( + any(is_relative_to(symlink_dir, path) for path in matches) + and symlink_dir.is_symlink() + ): + remove_dir_and_target(symlink_dir) + if symlink_dir == run_dir: # We have deleted the run dir return - matches.remove(path) + if symlink_dir in matches: + matches.remove(symlink_dir) # Now clean the rest for path in matches: remove_dir_or_file(path) diff --git a/tests/unit/test_clean.py b/tests/unit/test_clean.py index 5227e9d3876..285bfa6a23f 100644 --- a/tests/unit/test_clean.py +++ b/tests/unit/test_clean.py @@ -669,13 +669,7 @@ def test__clean_using_glob( 'flow.cylc': None, }}}}, 'sym-share': {'cylc-run': {}}, - 'sym-cycle': {'cylc-run': {'foo': {'bar': { - 'share': { - 'cycle': { - 'macklunkey.txt': None - } - } - }}}} + 'sym-cycle': {'cylc-run': {}}, }, id="filetree2 share" ), @@ -701,13 +695,7 @@ def test__clean_using_glob( '.service': {'db': None}, }}}}, 'sym-share': {'cylc-run': {}}, - 'sym-cycle': {'cylc-run': {'foo': {'bar': { - 'share': { - 'cycle': { - 'macklunkey.txt': None - } - } - }}}} + 'sym-cycle': {'cylc-run': {}}, }, id="filetree2 *" ), From aac9f9c1f32dfb40f7229379bc44a2f955567c4d Mon Sep 17 00:00:00 2001 From: Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com> Date: Tue, 10 Sep 2024 18:57:00 +0100 Subject: [PATCH 3/5] Simplify `cylc clean` reinvocation check --- cylc/flow/clean.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cylc/flow/clean.py b/cylc/flow/clean.py index fda1d4f1dd8..72c23e1e3e2 100644 --- a/cylc/flow/clean.py +++ b/cylc/flow/clean.py @@ -117,12 +117,8 @@ def _clean_check(opts: 'Values', id_: str, run_dir: Path) -> None: # Thing to clean must be a dir or broken symlink: if not run_dir.is_dir() and not run_dir.is_symlink(): raise FileNotFoundError(f"No directory to clean at {run_dir}") - db_path = ( - run_dir / WorkflowFiles.Service.DIRNAME / WorkflowFiles.Service.DB - ) - if opts.local_only and not db_path.is_file(): - # Will reach here if this is cylc clean re-invoked on remote host - # (workflow DB only exists on scheduler host); don't need to worry + if opts.no_scan: + # This is cylc clean re-invoked on remote host; don't need to worry # about contact file. return try: From b9504e04052aec26823f34634569cbeebe97d05e Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Wed, 18 Sep 2024 12:53:50 +0100 Subject: [PATCH 4/5] subprocpool: populate missing 255 callback arguments * In some situations, these missing arguments could cause tasks to submit fail as the result of a host outage rather than moving onto the next host for the platform. --- changes.d/6376.fix.md | 1 + cylc/flow/subprocpool.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 changes.d/6376.fix.md diff --git a/changes.d/6376.fix.md b/changes.d/6376.fix.md new file mode 100644 index 00000000000..686f7eef945 --- /dev/null +++ b/changes.d/6376.fix.md @@ -0,0 +1 @@ +Fixes an issue that could cause Cylc to ignore the remaining hosts in a platform in response to an `ssh` error in some niche circumstances. diff --git a/cylc/flow/subprocpool.py b/cylc/flow/subprocpool.py index 864071332b8..5af8b897ac1 100644 --- a/cylc/flow/subprocpool.py +++ b/cylc/flow/subprocpool.py @@ -291,8 +291,17 @@ def process(self): ) continue # Command still running, see if STDOUT/STDERR are readable or not - runnings.append([ - proc, ctx, bad_hosts, callback, callback_args, None, None]) + runnings.append( + [ + proc, + ctx, + bad_hosts, + callback, + callback_args, + callback_255, + callback_255_args, + ] + ) # Unblock proc's STDOUT/STDERR if necessary. Otherwise, a full # STDOUT or STDERR may stop command from proceeding. self._poll_proc_pipes(proc, ctx) From 0a88f7aad9b619e8728f9f3bc0be69101a260ca8 Mon Sep 17 00:00:00 2001 From: David Sutherland Date: Mon, 21 Oct 2024 22:34:14 +1300 Subject: [PATCH 5/5] fix CPF wall_clock offset precision constraint (#6431) --- changes.d/6431.fix.md | 1 + cylc/flow/task_proxy.py | 15 +++++++++------ tests/integration/test_xtrigger_mgr.py | 20 +++++++++++++++++--- 3 files changed, 27 insertions(+), 9 deletions(-) create mode 100644 changes.d/6431.fix.md diff --git a/changes.d/6431.fix.md b/changes.d/6431.fix.md new file mode 100644 index 00000000000..900bfee61d7 --- /dev/null +++ b/changes.d/6431.fix.md @@ -0,0 +1 @@ +The `cycle point format` was imposing an undesirable constraint on `wall_clock` offsets, this has been fixed. diff --git a/cylc/flow/task_proxy.py b/cylc/flow/task_proxy.py index 4e7b60d6e0a..0ba67438fcd 100644 --- a/cylc/flow/task_proxy.py +++ b/cylc/flow/task_proxy.py @@ -48,7 +48,6 @@ from cylc.flow.cycling.iso8601 import ( point_parse, interval_parse, - ISO8601Interval ) if TYPE_CHECKING: @@ -424,14 +423,18 @@ def get_clock_trigger_time( """ offset_str = offset_str if offset_str else 'P0Y' if offset_str not in self.clock_trigger_times: + # Convert ISO8601Point into metomi-isodatetime TimePoint at full + # second precision (N.B. it still dumps at the same precision + # as workflow cycle point format): + point_time = point_parse(str(point)) if offset_str == 'P0Y': - trigger_time = point + trigger_time = point_time else: - trigger_time = point + ISO8601Interval(offset_str) + trigger_time = point_time + interval_parse(offset_str) - offset = int( - point_parse(str(trigger_time)).seconds_since_unix_epoch) - self.clock_trigger_times[offset_str] = offset + self.clock_trigger_times[offset_str] = int( + trigger_time.seconds_since_unix_epoch + ) return self.clock_trigger_times[offset_str] def get_try_num(self): diff --git a/tests/integration/test_xtrigger_mgr.py b/tests/integration/test_xtrigger_mgr.py index d04c202bd43..73dd66fc2c5 100644 --- a/tests/integration/test_xtrigger_mgr.py +++ b/tests/integration/test_xtrigger_mgr.py @@ -29,30 +29,38 @@ def get_task_ids(schd: Scheduler) -> Set[str]: async def test_2_xtriggers(flow, start, scheduler, monkeypatch): - """Test that if an itask has 2 wall_clock triggers with different - offsets that xtrigger manager gets both of them. + """Test that if an itask has 4 wall_clock triggers with different + offsets that xtrigger manager gets all of them. https://github.com/cylc/cylc-flow/issues/5783 n.b. Clock 3 exists to check the memoization path is followed, and causing this test to give greater coverage. + Clock 4 & 5 test higher precision offsets than the CPF. """ task_point = 1588636800 # 2020-05-05 ten_years_ahead = 1904169600 # 2030-05-05 + PT2H35M31S_ahead = 1588646131 # 2020-05-05 02:35:31 + PT2H35M31S_behind = 1588627469 # 2020-05-04 21:24:29 monkeypatch.setattr( 'cylc.flow.xtriggers.wall_clock.time', lambda: ten_years_ahead - 1 ) id_ = flow({ + 'scheduler': { + 'cycle point format': 'CCYY-MM-DD', + }, 'scheduling': { 'initial cycle point': '2020-05-05', 'xtriggers': { 'clock_1': 'wall_clock()', 'clock_2': 'wall_clock(offset=P10Y)', 'clock_3': 'wall_clock(offset=P10Y)', + 'clock_4': 'wall_clock(offset=PT2H35M31S)', + 'clock_5': 'wall_clock(offset=-PT2H35M31S)', }, 'graph': { - 'R1': '@clock_1 & @clock_2 & @clock_3 => foo' + 'R1': '@clock_1 & @clock_2 & @clock_3 & @clock_4 & @clock_5 => foo' } } }) @@ -62,16 +70,22 @@ async def test_2_xtriggers(flow, start, scheduler, monkeypatch): clock_1_ctx = schd.xtrigger_mgr.get_xtrig_ctx(foo_proxy, 'clock_1') clock_2_ctx = schd.xtrigger_mgr.get_xtrig_ctx(foo_proxy, 'clock_2') clock_3_ctx = schd.xtrigger_mgr.get_xtrig_ctx(foo_proxy, 'clock_2') + clock_4_ctx = schd.xtrigger_mgr.get_xtrig_ctx(foo_proxy, 'clock_4') + clock_5_ctx = schd.xtrigger_mgr.get_xtrig_ctx(foo_proxy, 'clock_5') assert clock_1_ctx.func_kwargs['trigger_time'] == task_point assert clock_2_ctx.func_kwargs['trigger_time'] == ten_years_ahead assert clock_3_ctx.func_kwargs['trigger_time'] == ten_years_ahead + assert clock_4_ctx.func_kwargs['trigger_time'] == PT2H35M31S_ahead + assert clock_5_ctx.func_kwargs['trigger_time'] == PT2H35M31S_behind schd.xtrigger_mgr.call_xtriggers_async(foo_proxy) assert foo_proxy.state.xtriggers == { 'clock_1': True, 'clock_2': False, 'clock_3': False, + 'clock_4': True, + 'clock_5': True, }