From c9714c492ecaae8ef8b451b15c7ab100b9479702 Mon Sep 17 00:00:00 2001 From: dennisvang <29799340+dennisvang@users.noreply.github.com> Date: Thu, 7 Mar 2024 17:17:01 +0100 Subject: [PATCH 1/9] handle required updates in Client.check_for_updates --- src/tufup/client.py | 26 +++++++++++++++++++++++--- src/tufup/common.py | 1 + tests/test_common.py | 3 +++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/tufup/client.py b/src/tufup/client.py index b1974c7..e89473a 100644 --- a/src/tufup/client.py +++ b/src/tufup/client.py @@ -12,7 +12,7 @@ from tuf.api.exceptions import DownloadError, UnsignedMetadataError import tuf.ngclient -from tufup.common import Patcher, TargetMeta +from tufup.common import KEY_REQUIRED, Patcher, TargetMeta from tufup.utils.platform_specific import install_update logger = logging.getLogger(__name__) @@ -138,7 +138,10 @@ def download_and_apply_update( ) def check_for_updates( - self, pre: Optional[str] = None, patch: bool = True + self, + pre: Optional[str] = None, + patch: bool = True, + ignore_required: bool = False, ) -> Optional[TargetMeta]: """ Check if any updates are available, based on current app version. @@ -152,6 +155,14 @@ def check_for_updates( candidate, respectively. If `patch` is `False`, a full update is enforced. + + If a new release is marked as "required" (in its custom metadata) this + release will take precedence over any non-required releases, *even* if the + latter are newer. This may be useful e.g. in case of a configuration change. + These "required" releases should be rare, and should preferably be avoided. + However, in the exceedingly rare event that there *are* "required" updates, + yet the user wants to treat them as non-required, they can specify + `ignore_required=True`. """ # invalid pre-release specifiers are ignored, with a warning pre_map = dict(a='abrc', b='brc', rc='rc') @@ -186,7 +197,16 @@ def check_for_updates( new_archive_meta = None if new_archives: logger.debug(f'{len(new_archives)} new *archives* found') - new_archive_meta, self.new_archive_info = sorted(new_archives.items())[-1] + # the "latest" archive is typically just the last one in the sorted list + # of new archives, except when there are new "required" archives, + # in which case we must update to the first "required" archive encountered + for archive_meta, archive_info in sorted(new_archives.items()): + if not ignore_required: + if archive_meta.custom and KEY_REQUIRED in archive_meta.custom: + logger.debug(f'required update found: {archive_meta.version}') + break + new_archive_meta = archive_meta # noqa + self.new_archive_info = archive_info # noqa self.new_archive_local_path = pathlib.Path( self.target_dir, new_archive_meta.path.name ) diff --git a/src/tufup/common.py b/src/tufup/common.py index 5ca2b61..de64855 100644 --- a/src/tufup/common.py +++ b/src/tufup/common.py @@ -10,6 +10,7 @@ logger = logging.getLogger(__name__) +KEY_REQUIRED = 'required' SUFFIX_ARCHIVE = '.tar.gz' SUFFIX_PATCH = '.patch' diff --git a/tests/test_common.py b/tests/test_common.py index e4d99fb..978d76b 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -153,6 +153,9 @@ def test_compose_filename(self): ) self.assertEqual('app-1.0.tar.gz', filename) + def test_custom(self): + self.assertIsNone(TargetMeta().custom) + class PatcherTests(TempDirTestCase): def setUp(self) -> None: From 5f10e416cc7b388952bd93d0b52ecbd4b1c5833c Mon Sep 17 00:00:00 2001 From: dennisvang <29799340+dennisvang@users.noreply.github.com> Date: Thu, 7 Mar 2024 18:33:03 +0100 Subject: [PATCH 2/9] add required arg to Repository.add_bundle() --- src/tufup/common.py | 2 +- src/tufup/repo/__init__.py | 16 +++++++++++++++- tests/test_repo.py | 6 +++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/tufup/common.py b/src/tufup/common.py index de64855..8b63bf5 100644 --- a/src/tufup/common.py +++ b/src/tufup/common.py @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) -KEY_REQUIRED = 'required' +KEY_REQUIRED = '_tufup_flag_required' # unlikely to be identical to user-specified key SUFFIX_ARCHIVE = '.tar.gz' SUFFIX_PATCH = '.patch' diff --git a/src/tufup/repo/__init__.py b/src/tufup/repo/__init__.py index 8f735a9..6bc6e1e 100644 --- a/src/tufup/repo/__init__.py +++ b/src/tufup/repo/__init__.py @@ -38,7 +38,7 @@ ) from tuf.api.serialization.json import JSONSerializer -from tufup.common import Patcher, SUFFIX_PATCH, TargetMeta +from tufup.common import KEY_REQUIRED, Patcher, SUFFIX_PATCH, TargetMeta from tufup.utils.platform_specific import _patched_resolve logger = logging.getLogger(__name__) @@ -735,6 +735,7 @@ def add_bundle( new_version: Optional[str] = None, skip_patch: bool = False, custom_metadata: Optional[dict] = None, # archive only + required: bool = False, ): """ Adds a new application bundle to the local repository. @@ -744,9 +745,22 @@ def add_bundle( a patch file is also created and added to the repository, unless `skip_patch` is True. + If `required=True` (default is `False`), this release will always be + installed, even if newer releases are available. For example, suppose + an app is running at version 1.0, and version 2.0 is required, but version + 3.0 is also available, then tufup will first update to version 2.0, + before updating to 3.0 on the next run. + Note the changes are not published yet: call `publish_changes()` for that. """ + if custom_metadata is None: + custom_metadata = dict() + if required: + # todo: probably better to separate tufup's internal custom metadata from + # user-specified custom_metadata, e.g. using nested dicts, as in + # unrecognized_fields['custom'] = {'tufup': {...}, 'user': custom_metadata} + custom_metadata[KEY_REQUIRED] = True # enforce path object new_bundle_dir = pathlib.Path(new_bundle_dir) # determine new version diff --git a/tests/test_repo.py b/tests/test_repo.py index c98560a..bdd0d43 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -25,7 +25,7 @@ ) from tests import TempDirTestCase, TEST_REPO_DIR -from tufup.common import TargetMeta +from tufup.common import KEY_REQUIRED, TargetMeta import tufup.repo # for patching from tufup.repo import ( Base, @@ -753,10 +753,14 @@ def test_add_bundle(self): new_version=version, new_bundle_dir=bundle_dir, custom_metadata=dict(whatever='something'), + required=True, ) self.assertTrue((repo.metadata_dir / 'targets.json').exists()) target_name = f'{app_name}-{version}.tar.gz' self.assertTrue(repo.roles.targets.signed.targets[target_name].custom) + self.assertIn( + KEY_REQUIRED, repo.roles.targets.signed.targets[target_name].custom + ) def test_add_bundle_no_patch(self): # prepare From e83ab7a7ea88ddf8daa16b39b7547bf505efa20b Mon Sep 17 00:00:00 2001 From: dennisvang <29799340+dennisvang@users.noreply.github.com> Date: Fri, 8 Mar 2024 12:07:10 +0100 Subject: [PATCH 3/9] issue111: warn if bundle is not added due to version --- src/tufup/repo/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/tufup/repo/__init__.py b/src/tufup/repo/__init__.py index 6bc6e1e..a49ad74 100644 --- a/src/tufup/repo/__init__.py +++ b/src/tufup/repo/__init__.py @@ -797,6 +797,11 @@ def add_bundle( self.roles.add_or_update_target( local_path=patch_path, custom=dst_size_and_hash ) + else: + logger.warning( + f'bundle not added: version {new_archive.version} must be greater than' + f'that of latest archive ({latest_archive.version})' + ) def remove_latest_bundle(self): """ From ebb9ecb319fe99cd317d715477f2903d902635f2 Mon Sep 17 00:00:00 2001 From: dennisvang <29799340+dennisvang@users.noreply.github.com> Date: Fri, 8 Mar 2024 12:59:56 +0100 Subject: [PATCH 4/9] issue111: make app version 2.0 required in repo workflow example and update test data and tests accordingly --- examples/repo/repo_workflow_example.py | 8 +++-- tests/data/repository/metadata/1.root.json | 10 +++--- tests/data/repository/metadata/2.root.json | 14 ++++---- tests/data/repository/metadata/root.json | 14 ++++---- tests/data/repository/metadata/snapshot.json | 4 +-- tests/data/repository/metadata/targets.json | 5 +-- tests/data/repository/metadata/timestamp.json | 4 +-- tests/test_client.py | 32 ++++++++++++------- 8 files changed, 52 insertions(+), 39 deletions(-) diff --git a/examples/repo/repo_workflow_example.py b/examples/repo/repo_workflow_example.py index 41d9138..e2179f7 100644 --- a/examples/repo/repo_workflow_example.py +++ b/examples/repo/repo_workflow_example.py @@ -90,7 +90,7 @@ KEY_MAP['root'].append('root_two') # use two keys for root ENCRYPTED_KEYS = ['root', 'root_two', 'targets'] -# Custom metadata +# Custom metadata (for example, a list of changes) DUMMY_METADATA = dict(changes=['this has changed', 'that has changed', '...']) # Create repository instance @@ -169,7 +169,11 @@ repo.add_bundle( new_version=new_version, new_bundle_dir=dummy_bundle_dir, - custom_metadata=DUMMY_METADATA, # just to point out the option + # example of optional custom metadata + custom_metadata=DUMMY_METADATA.copy(), + # "required" updates are exceptional and should be avoided if possible, + # but we include one here just for completeness + required=new_version == '2.0', ) repo.publish_changes(private_key_dirs=[OFFLINE_DIR_1, OFFLINE_DIR_2, ONLINE_DIR]) diff --git a/tests/data/repository/metadata/1.root.json b/tests/data/repository/metadata/1.root.json index 32f7456..8847a9d 100755 --- a/tests/data/repository/metadata/1.root.json +++ b/tests/data/repository/metadata/1.root.json @@ -2,17 +2,17 @@ "signatures": [ { "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", - "sig": "0f634a6e5f82af4447accce63c2987350c9c16fe6f8ce391ed504da106be8a127e1d606424c97a27822038cfd35e4daa96da2ec07a4a75bc2610df3bfc95cd0c" + "sig": "581b30a5a2a927c71479b568908bb517bc1d2c7a627428e8399b7c91ef427f4784357c855698c484d3a2aacc952ce1893ea246a03401315e2a473a5db2429d0f" }, { "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2", - "sig": "678256d67bcf6022f75920ff380dc2111e2d68120af834f1769d694665236a2c7fb57ea5731f4050e1562a8b2be870b6594a2203f52182b1b77fa98ae89ed90c" + "sig": "5b53aa824b1e81ec3d7e1fe383ca41b156715f852e9c59bc3ba53275d95fb2869b778cce351da30b867fdb2ed55334fc22e4f9c13732b8a8557049000056ff09" } ], "signed": { "_type": "root", "consistent_snapshot": false, - "expires": "2051-06-27T21:21:03Z", + "expires": "2051-07-25T11:55:06Z", "keys": { "5ef48ab6f5398d2bf17f1f4c4fc0e0440c4aa3734a05ae523561e02e8a99957a": { "keytype": "ed25519", @@ -53,8 +53,8 @@ "roles": { "root": { "keyids": [ - "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2", - "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568" + "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", + "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2" ], "threshold": 2 }, diff --git a/tests/data/repository/metadata/2.root.json b/tests/data/repository/metadata/2.root.json index c51f327..07a0f2a 100755 --- a/tests/data/repository/metadata/2.root.json +++ b/tests/data/repository/metadata/2.root.json @@ -1,22 +1,22 @@ { "signatures": [ - { - "keyid": "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003", - "sig": "47a42813ae34829c60539dcceba0d4b9a8a9286beaa8d5f07d3de3050d404426c22bc95b271e7c5e7ee529bc3180f009eb31313fb825f76c3ed9ca2c501bd503" - }, { "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", - "sig": "421d85636350a89805abc4561acd3019ecf17246a37e91374a53276b5d56638c83754960c27d038c7d1193bdb33db12faf69b7a19099627c745c569093ee0005" + "sig": "3b5fef2d49edba8b0dfb8fb7781fd5721c3f11f602c5700d9821367862973a65d90d2cdc4fbf9433207475defec1eeea085cf9f1145f1eea54210647cbb34004" }, { "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2", - "sig": "a65dbf32349f1a57dd1dd6fc058c69a98be467f5ad408179da6e3b67abc6f2361415eb70214588d21079a9d0351500808f8c244b69f40b35a41999294461ca00" + "sig": "10f57136bfbbd1b4f6c89ba87e1d63c2c1b4478e2a3ceed9924013dce51e886047e028adef41d0ae326940d228505acf78fc39f69fef03b5edc202b79d099808" + }, + { + "keyid": "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003", + "sig": "6da78dc6741ca4f4d83948d357812f6b7ebd62fc5a0ecbf157cde40b3851f448fae35fad5d9fa2e8453088312f6600083c45dfd2f4f72a4c293809d1aad24508" } ], "signed": { "_type": "root", "consistent_snapshot": false, - "expires": "2051-06-27T21:21:13Z", + "expires": "2051-07-25T11:55:10Z", "keys": { "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003": { "keytype": "ed25519", diff --git a/tests/data/repository/metadata/root.json b/tests/data/repository/metadata/root.json index c51f327..07a0f2a 100755 --- a/tests/data/repository/metadata/root.json +++ b/tests/data/repository/metadata/root.json @@ -1,22 +1,22 @@ { "signatures": [ - { - "keyid": "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003", - "sig": "47a42813ae34829c60539dcceba0d4b9a8a9286beaa8d5f07d3de3050d404426c22bc95b271e7c5e7ee529bc3180f009eb31313fb825f76c3ed9ca2c501bd503" - }, { "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", - "sig": "421d85636350a89805abc4561acd3019ecf17246a37e91374a53276b5d56638c83754960c27d038c7d1193bdb33db12faf69b7a19099627c745c569093ee0005" + "sig": "3b5fef2d49edba8b0dfb8fb7781fd5721c3f11f602c5700d9821367862973a65d90d2cdc4fbf9433207475defec1eeea085cf9f1145f1eea54210647cbb34004" }, { "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2", - "sig": "a65dbf32349f1a57dd1dd6fc058c69a98be467f5ad408179da6e3b67abc6f2361415eb70214588d21079a9d0351500808f8c244b69f40b35a41999294461ca00" + "sig": "10f57136bfbbd1b4f6c89ba87e1d63c2c1b4478e2a3ceed9924013dce51e886047e028adef41d0ae326940d228505acf78fc39f69fef03b5edc202b79d099808" + }, + { + "keyid": "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003", + "sig": "6da78dc6741ca4f4d83948d357812f6b7ebd62fc5a0ecbf157cde40b3851f448fae35fad5d9fa2e8453088312f6600083c45dfd2f4f72a4c293809d1aad24508" } ], "signed": { "_type": "root", "consistent_snapshot": false, - "expires": "2051-06-27T21:21:13Z", + "expires": "2051-07-25T11:55:10Z", "keys": { "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003": { "keytype": "ed25519", diff --git a/tests/data/repository/metadata/snapshot.json b/tests/data/repository/metadata/snapshot.json index c05b5ad..1ea162b 100755 --- a/tests/data/repository/metadata/snapshot.json +++ b/tests/data/repository/metadata/snapshot.json @@ -2,12 +2,12 @@ "signatures": [ { "keyid": "5ef48ab6f5398d2bf17f1f4c4fc0e0440c4aa3734a05ae523561e02e8a99957a", - "sig": "73a146f5e1f12c0a36e88c8d7bf613baa1d528ea0c9480fe0d2ccd74d6da239da04470f68d283738194185cc82289c5f9f1312efea373b51dc8722965ca1fc0b" + "sig": "9ad02e9e365e3ac008643d79e9c953ed6289ae63cfb3f22cf5c3029747f84e97c84f637d66838a35af10a50a5bdf0ca25c86e515129e85467c53e27c23ee690c" } ], "signed": { "_type": "snapshot", - "expires": "2051-06-27T21:21:13Z", + "expires": "2051-07-25T11:55:10Z", "meta": { "targets.json": { "version": 6 diff --git a/tests/data/repository/metadata/targets.json b/tests/data/repository/metadata/targets.json index 122d603..1cb9928 100755 --- a/tests/data/repository/metadata/targets.json +++ b/tests/data/repository/metadata/targets.json @@ -2,12 +2,12 @@ "signatures": [ { "keyid": "cd9930c92ac25c02a2f92ae3128b50459b53d7532ef9c0f364e78f388d5808a5", - "sig": "344b1c779103db5c8462508d7a5e72ef9ae8dea0c5fd303d55cace03a87fd67312ff5ca01fc2e377d7d0dcbbbf3f4dff378f5c9759801590340c0b9e3d23bc07" + "sig": "61b235261b5b9627c6f5bdf155dc2f49ead5deb67c8ee7d5696c92c62c17dbc219ffef3ce35ba48327cf46362f8ec427b37dd6965001a220790e5c41b094790b" } ], "signed": { "_type": "targets", - "expires": "2051-06-27T21:21:13Z", + "expires": "2051-07-25T11:55:10Z", "spec_version": "1.0.31", "targets": { "example_app-1.0.tar.gz": { @@ -29,6 +29,7 @@ }, "example_app-2.0.tar.gz": { "custom": { + "_tufup_flag_required": true, "changes": [ "this has changed", "that has changed", diff --git a/tests/data/repository/metadata/timestamp.json b/tests/data/repository/metadata/timestamp.json index e2ce07a..d970671 100755 --- a/tests/data/repository/metadata/timestamp.json +++ b/tests/data/repository/metadata/timestamp.json @@ -2,12 +2,12 @@ "signatures": [ { "keyid": "eddb87d254d513c1404d71e17620ecf5260e1836babdaa55197916c582f37a00", - "sig": "4f05f9947e1fd704ffb877fa994e8841eb1a34a2ccf021c2eac5a618eca3c11baad8531d154fa5c8aa187e0bfaa57b521901ef502bd7a3dc601bfa1c408a4106" + "sig": "948741ee18e88ac4c4e97f534373ced3b3e640ec7352233b8e92f3732c4e5588e12729bea79b075cb23e3d36031c16571974d9baf4700da33c90096368948005" } ], "signed": { "_type": "timestamp", - "expires": "2051-06-27T21:21:13Z", + "expires": "2051-07-25T11:55:10Z", "meta": { "snapshot.json": { "version": 7 diff --git a/tests/test_client.py b/tests/test_client.py index a2e8c15..cca31a7 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -149,28 +149,36 @@ def test_download_and_apply_update(self): self.assertIn(mock_install, mock_apply.call_args.kwargs.values()) def test_check_for_updates(self): - # expectations (based on targets in tests/data/repository): - # - pre=None, '', or 'invalid': only full releases are included, finds 2.0 patch - # - pre='a': finds all, but total patch size exceeds archive size - # - pre='b': there is no 'b' release, so this finds same as 'rc' - # - pre='rc': finds 2.0 and 3.0rc0, total patch size smaller than archive client = self.get_refreshed_client() with patch.object(client, 'refresh', Mock()): + # expectations (based on targets in tests/data/repository, but ignoring + # required releases): + # - pre=None, '', or 'invalid': + # only full releases are included, so this finds 2.0 (patch) + # - pre='a': finds all, but total patch size exceeds archive size + # - pre='b': there is no 'b' release, so this finds same as 'rc' + # - pre='rc': finds 2.0 and 3.0rc0, total patch size smaller than archive for pre, expected in [ (None, 1), ('', 1), ('a', 1), ('b', 2), ('rc', 2), ('invalid', 1) ]: with self.subTest(msg=pre): + # verify that we always find the required release, unless we + # explicitly set ignore_required=True + required_version = '2.0' target_meta = client.check_for_updates(pre=pre) + self.assertEqual(required_version, str(target_meta.version)) + # for the actual test we want to treat all versions as not-required + target_meta = client.check_for_updates( + pre=pre, ignore_required=True + ) self.assertTrue(expected and target_meta) self.assertEqual(expected, len(client.new_targets)) - if pre == 'a': - self.assertTrue( - all(item.is_archive for item in client.new_targets.keys()) - ) - else: - self.assertTrue( - all(item.is_patch for item in client.new_targets.keys()) + self.assertTrue( + all( + getattr(item, 'is_archive' if pre == 'a' else 'is_patch') + for item in client.new_targets.keys() ) + ) # verify that we can access custom metadata where needed if target_meta.is_patch: self.assertTrue(target_meta.custom) From 27b9f3cc66da79de14fe2eb0e3ea9d70debb8869 Mon Sep 17 00:00:00 2001 From: dennisvang <29799340+dennisvang@users.noreply.github.com> Date: Fri, 8 Mar 2024 15:58:43 +0100 Subject: [PATCH 5/9] issue99: separate user-specified custom metadata from tufup-internal custom metadata backward compatible: can read older metadata that does not make the distinction --- src/tufup/common.py | 38 ++++++++++++++++++++++++++++++++++---- src/tufup/repo/__init__.py | 11 +++++++---- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/src/tufup/common.py b/src/tufup/common.py index 5ca2b61..b46d99e 100644 --- a/src/tufup/common.py +++ b/src/tufup/common.py @@ -3,7 +3,7 @@ import logging import pathlib import re -from typing import Dict, Optional, Union +from typing import Dict, get_type_hints, Optional, TypedDict, Union import bsdiff4 from packaging.version import Version, InvalidVersion @@ -14,6 +14,15 @@ SUFFIX_PATCH = '.patch' +class CustomMetadataDict(TypedDict): + """ + explicitly separate custom metadata into user-specified metadata and metadata + used by tufup internally + """ + user: Optional[dict] + tufup: Optional[dict] + + def _immutable(value): """ Make value immutable, recursively, so the result is hashable. @@ -49,7 +58,7 @@ def __init__( name: Optional[str] = None, version: Optional[str] = None, is_archive: Optional[bool] = True, - custom: Optional[dict] = None, + custom: Optional[CustomMetadataDict] = None, ): """ Initialize either with target_path, or with name, version, archive. @@ -68,7 +77,28 @@ def __init__( logger.critical( f'invalid filename "{self.filename}": whitespace not allowed' ) - self.custom = custom + self._custom = custom + + @property + def custom(self) -> Optional[dict]: + """returns user-specified custom metadata""" + return self._get_custom_metadata('user') + + @property + def custom_internal(self) -> Optional[dict]: + """returns tufup-internal custom metadata""" + return self._get_custom_metadata('tufup') + + def _get_custom_metadata(self, key: str) -> Optional[dict]: + """ + get custom metadata in a backward-compatible manner (older versions did not + distinguish between user-specified and internal metadata) + """ + if isinstance(self._custom, dict): + # check dict keys for backward compatibility + if get_type_hints(CustomMetadataDict).keys() != self._custom.keys(): + return self._custom + return self._custom.get(key) def __str__(self): return str(self.target_path_str) @@ -246,7 +276,7 @@ def patch_and_verify( # verify integrity of the final result (raises exception on failure) cls._verify_tar_size_and_hash( tar_content=tar_bytes, - expected=patch_meta.custom, # noqa + expected=patch_meta.custom_internal, # noqa ) # compress .tar data into destination .tar.gz file with gzip.open(dst_path, mode='wb') as dst_file: diff --git a/src/tufup/repo/__init__.py b/src/tufup/repo/__init__.py index 8f735a9..52ebf28 100644 --- a/src/tufup/repo/__init__.py +++ b/src/tufup/repo/__init__.py @@ -38,7 +38,7 @@ ) from tuf.api.serialization.json import JSONSerializer -from tufup.common import Patcher, SUFFIX_PATCH, TargetMeta +from tufup.common import CustomMetadataDict, Patcher, SUFFIX_PATCH, TargetMeta from tufup.utils.platform_specific import _patched_resolve logger = logging.getLogger(__name__) @@ -375,7 +375,7 @@ def add_or_update_target( self, local_path: Union[pathlib.Path, str], url_path_segments: Optional[List[str]] = None, - custom: Optional[dict] = None, + custom: Optional[CustomMetadataDict] = None, ): # based on python-tuf basic_repo.py local_path = pathlib.Path(local_path) @@ -767,7 +767,9 @@ def add_bundle( if not latest_archive or latest_archive.version < new_archive.version: # register new archive self.roles.add_or_update_target( - local_path=new_archive.path, custom=custom_metadata + local_path=new_archive.path, + # separate user-specified metadata from tufup-internal metadata + custom=dict(user=custom_metadata, tufup=None), ) # create patch, if possible, and register that too if latest_archive and not skip_patch: @@ -781,7 +783,8 @@ def add_bundle( # register patch (size and hash are used by the client to verify the # integrity of the patched archive) self.roles.add_or_update_target( - local_path=patch_path, custom=dst_size_and_hash + local_path=patch_path, + custom=dict(user=None, tufup=dst_size_and_hash), ) def remove_latest_bundle(self): From 4551ed6a4e19dac6f9bcd7947a235240e51f5921 Mon Sep 17 00:00:00 2001 From: dennisvang <29799340+dennisvang@users.noreply.github.com> Date: Fri, 8 Mar 2024 16:34:49 +0100 Subject: [PATCH 6/9] issue99: update test data and test custom metadata separation --- tests/data/repository/metadata/1.root.json | 14 ++-- tests/data/repository/metadata/2.root.json | 12 +-- tests/data/repository/metadata/root.json | 12 +-- tests/data/repository/metadata/snapshot.json | 4 +- tests/data/repository/metadata/targets.json | 74 ++++++++++++------- tests/data/repository/metadata/timestamp.json | 4 +- tests/test_client.py | 2 +- tests/test_common.py | 21 ++++++ 8 files changed, 93 insertions(+), 50 deletions(-) diff --git a/tests/data/repository/metadata/1.root.json b/tests/data/repository/metadata/1.root.json index 32f7456..0c055f7 100755 --- a/tests/data/repository/metadata/1.root.json +++ b/tests/data/repository/metadata/1.root.json @@ -1,18 +1,18 @@ { "signatures": [ { - "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", - "sig": "0f634a6e5f82af4447accce63c2987350c9c16fe6f8ce391ed504da106be8a127e1d606424c97a27822038cfd35e4daa96da2ec07a4a75bc2610df3bfc95cd0c" + "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2", + "sig": "9f80547ca0ab37b5de2ae3869be83b9e1312636c3f932265e1e02f528cd59c5057dee7f7d76f7d4f19241538fc578cd01c15b02e9ecfd9a2b6dd1324a53a0008" }, { - "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2", - "sig": "678256d67bcf6022f75920ff380dc2111e2d68120af834f1769d694665236a2c7fb57ea5731f4050e1562a8b2be870b6594a2203f52182b1b77fa98ae89ed90c" + "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", + "sig": "68a0d73c4327f7ac39dd15476e8d5d11fdedd034e5795e7ee1d4bea0066046e7d07f1508e2df0b7f99f39d66dc3c2b540632bafc121dbfa4d1c43f6f07448504" } ], "signed": { "_type": "root", "consistent_snapshot": false, - "expires": "2051-06-27T21:21:03Z", + "expires": "2051-07-25T14:59:45Z", "keys": { "5ef48ab6f5398d2bf17f1f4c4fc0e0440c4aa3734a05ae523561e02e8a99957a": { "keytype": "ed25519", @@ -53,8 +53,8 @@ "roles": { "root": { "keyids": [ - "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2", - "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568" + "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", + "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2" ], "threshold": 2 }, diff --git a/tests/data/repository/metadata/2.root.json b/tests/data/repository/metadata/2.root.json index c51f327..22c2903 100755 --- a/tests/data/repository/metadata/2.root.json +++ b/tests/data/repository/metadata/2.root.json @@ -2,21 +2,21 @@ "signatures": [ { "keyid": "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003", - "sig": "47a42813ae34829c60539dcceba0d4b9a8a9286beaa8d5f07d3de3050d404426c22bc95b271e7c5e7ee529bc3180f009eb31313fb825f76c3ed9ca2c501bd503" + "sig": "1e4c3c283c26ea9512fe9f6ae89583fbdfaab259d72f62a7f26d945e93755b5ffa334a7a2bda1f6eb6a22d3ad11546bfe790ac7aaca5c2a3389022d9f501a004" }, { - "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", - "sig": "421d85636350a89805abc4561acd3019ecf17246a37e91374a53276b5d56638c83754960c27d038c7d1193bdb33db12faf69b7a19099627c745c569093ee0005" + "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2", + "sig": "fdfb8c18130a5cad203bbe1fecf4de1a4d510c0b5daae7bddf53f3cf47a7ca5478338fa0edf6130a5d1c0a44b70071784be5e901670bd7a004ab61f8c4114c02" }, { - "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2", - "sig": "a65dbf32349f1a57dd1dd6fc058c69a98be467f5ad408179da6e3b67abc6f2361415eb70214588d21079a9d0351500808f8c244b69f40b35a41999294461ca00" + "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", + "sig": "6abb0b0a32fee9d038f9d4301e168a8ba530996d43ecc8a780626415e8cee111659741be61b0094435f3d89699546bb340480bfce637fc2230f355a3155ced0a" } ], "signed": { "_type": "root", "consistent_snapshot": false, - "expires": "2051-06-27T21:21:13Z", + "expires": "2051-07-25T14:59:52Z", "keys": { "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003": { "keytype": "ed25519", diff --git a/tests/data/repository/metadata/root.json b/tests/data/repository/metadata/root.json index c51f327..22c2903 100755 --- a/tests/data/repository/metadata/root.json +++ b/tests/data/repository/metadata/root.json @@ -2,21 +2,21 @@ "signatures": [ { "keyid": "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003", - "sig": "47a42813ae34829c60539dcceba0d4b9a8a9286beaa8d5f07d3de3050d404426c22bc95b271e7c5e7ee529bc3180f009eb31313fb825f76c3ed9ca2c501bd503" + "sig": "1e4c3c283c26ea9512fe9f6ae89583fbdfaab259d72f62a7f26d945e93755b5ffa334a7a2bda1f6eb6a22d3ad11546bfe790ac7aaca5c2a3389022d9f501a004" }, { - "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", - "sig": "421d85636350a89805abc4561acd3019ecf17246a37e91374a53276b5d56638c83754960c27d038c7d1193bdb33db12faf69b7a19099627c745c569093ee0005" + "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2", + "sig": "fdfb8c18130a5cad203bbe1fecf4de1a4d510c0b5daae7bddf53f3cf47a7ca5478338fa0edf6130a5d1c0a44b70071784be5e901670bd7a004ab61f8c4114c02" }, { - "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2", - "sig": "a65dbf32349f1a57dd1dd6fc058c69a98be467f5ad408179da6e3b67abc6f2361415eb70214588d21079a9d0351500808f8c244b69f40b35a41999294461ca00" + "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", + "sig": "6abb0b0a32fee9d038f9d4301e168a8ba530996d43ecc8a780626415e8cee111659741be61b0094435f3d89699546bb340480bfce637fc2230f355a3155ced0a" } ], "signed": { "_type": "root", "consistent_snapshot": false, - "expires": "2051-06-27T21:21:13Z", + "expires": "2051-07-25T14:59:52Z", "keys": { "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003": { "keytype": "ed25519", diff --git a/tests/data/repository/metadata/snapshot.json b/tests/data/repository/metadata/snapshot.json index c05b5ad..b6a8333 100755 --- a/tests/data/repository/metadata/snapshot.json +++ b/tests/data/repository/metadata/snapshot.json @@ -2,12 +2,12 @@ "signatures": [ { "keyid": "5ef48ab6f5398d2bf17f1f4c4fc0e0440c4aa3734a05ae523561e02e8a99957a", - "sig": "73a146f5e1f12c0a36e88c8d7bf613baa1d528ea0c9480fe0d2ccd74d6da239da04470f68d283738194185cc82289c5f9f1312efea373b51dc8722965ca1fc0b" + "sig": "48bdd9a911dc60620c513706bd0140573611a85713844d7ec38d938126ad4088688a6829d819049cd94c8a01ed1448e707800f7b4438a7edaa9edd1919612501" } ], "signed": { "_type": "snapshot", - "expires": "2051-06-27T21:21:13Z", + "expires": "2051-07-25T14:59:52Z", "meta": { "targets.json": { "version": 6 diff --git a/tests/data/repository/metadata/targets.json b/tests/data/repository/metadata/targets.json index 122d603..a64f410 100755 --- a/tests/data/repository/metadata/targets.json +++ b/tests/data/repository/metadata/targets.json @@ -2,15 +2,19 @@ "signatures": [ { "keyid": "cd9930c92ac25c02a2f92ae3128b50459b53d7532ef9c0f364e78f388d5808a5", - "sig": "344b1c779103db5c8462508d7a5e72ef9ae8dea0c5fd303d55cace03a87fd67312ff5ca01fc2e377d7d0dcbbbf3f4dff378f5c9759801590340c0b9e3d23bc07" + "sig": "ba92827c2fbe262ca8dc28dfc6b2629a4ce0c8af747d666b00e1104ccba004e3727a85eb00e957edeadae2a430972210c949723f4091e9f01f676025868bef03" } ], "signed": { "_type": "targets", - "expires": "2051-06-27T21:21:13Z", + "expires": "2051-07-25T14:59:52Z", "spec_version": "1.0.31", "targets": { "example_app-1.0.tar.gz": { + "custom": { + "tufup": null, + "user": null + }, "hashes": { "sha256": "223dfd468edbe36256dc119f8477ac4025279ae3705dec04a32002e81b10fd16" }, @@ -18,9 +22,12 @@ }, "example_app-2.0.patch": { "custom": { - "tar_hash": "855c631eb1a8d756bbad8441b76b5452505d292a162b3d497a60877fee2140b5", - "tar_hash_algorithm": "sha256", - "tar_size": 112640 + "tufup": { + "tar_hash": "855c631eb1a8d756bbad8441b76b5452505d292a162b3d497a60877fee2140b5", + "tar_hash_algorithm": "sha256", + "tar_size": 112640 + }, + "user": null }, "hashes": { "sha256": "f2be4504e464bd23c022772c7f3c011e0082295775a24e3fc986bb2504df0f53" @@ -29,11 +36,14 @@ }, "example_app-2.0.tar.gz": { "custom": { - "changes": [ - "this has changed", - "that has changed", - "..." - ] + "tufup": null, + "user": { + "changes": [ + "this has changed", + "that has changed", + "..." + ] + } }, "hashes": { "sha256": "d85f423a56427e522ac4d093a6ce94abcc2cc32f99b80a39b87832d8e4ba9ad8" @@ -42,9 +52,12 @@ }, "example_app-3.0rc0.patch": { "custom": { - "tar_hash": "3cd260c121d05f4c6ed55b6e87569d3710e539e0a86e6fce98189ddca20c99f5", - "tar_hash_algorithm": "sha256", - "tar_size": 112640 + "tufup": { + "tar_hash": "3cd260c121d05f4c6ed55b6e87569d3710e539e0a86e6fce98189ddca20c99f5", + "tar_hash_algorithm": "sha256", + "tar_size": 112640 + }, + "user": null }, "hashes": { "sha256": "01fa6f30ac54fd405dfb5bd6e39f71af572c8e341675f5b2822b35ec341ce6f9" @@ -53,11 +66,14 @@ }, "example_app-3.0rc0.tar.gz": { "custom": { - "changes": [ - "this has changed", - "that has changed", - "..." - ] + "tufup": null, + "user": { + "changes": [ + "this has changed", + "that has changed", + "..." + ] + } }, "hashes": { "sha256": "d7fa6ddd397282e8fa81924a31f340ebbcb8c082604c0549a38f5882cd3716c6" @@ -66,9 +82,12 @@ }, "example_app-4.0a0.patch": { "custom": { - "tar_hash": "3d3efe43388f3bbae910af39232526ef624d1540cfbb69cf0d4c66b7d5dc4b45", - "tar_hash_algorithm": "sha256", - "tar_size": 112640 + "tufup": { + "tar_hash": "3d3efe43388f3bbae910af39232526ef624d1540cfbb69cf0d4c66b7d5dc4b45", + "tar_hash_algorithm": "sha256", + "tar_size": 112640 + }, + "user": null }, "hashes": { "sha256": "e19ccd94e60d1d817dcc44c481f4fb48f54fa335cc0bd7a7e60377a4f6ccf2ea" @@ -77,11 +96,14 @@ }, "example_app-4.0a0.tar.gz": { "custom": { - "changes": [ - "this has changed", - "that has changed", - "..." - ] + "tufup": null, + "user": { + "changes": [ + "this has changed", + "that has changed", + "..." + ] + } }, "hashes": { "sha256": "05304765a0cb4e40cbd7ca2587aeb5f0db36cd9b617921e0d141bb91d3304e2c" diff --git a/tests/data/repository/metadata/timestamp.json b/tests/data/repository/metadata/timestamp.json index e2ce07a..756c84d 100755 --- a/tests/data/repository/metadata/timestamp.json +++ b/tests/data/repository/metadata/timestamp.json @@ -2,12 +2,12 @@ "signatures": [ { "keyid": "eddb87d254d513c1404d71e17620ecf5260e1836babdaa55197916c582f37a00", - "sig": "4f05f9947e1fd704ffb877fa994e8841eb1a34a2ccf021c2eac5a618eca3c11baad8531d154fa5c8aa187e0bfaa57b521901ef502bd7a3dc601bfa1c408a4106" + "sig": "cbff18b83e93143a98a418dccea7f1d2f4eefe466a955860c3dedd730b8ac08fa01b4eb3f3c96a3cb943e8bca154d5ea52aacdb97e53ea4a96a8441e9145b90e" } ], "signed": { "_type": "timestamp", - "expires": "2051-06-27T21:21:13Z", + "expires": "2051-07-25T14:59:53Z", "meta": { "snapshot.json": { "version": 7 diff --git a/tests/test_client.py b/tests/test_client.py index a2e8c15..740edf9 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -112,7 +112,7 @@ def test_trusted_target_metas(self): self.assertIn(example_key, meta.custom) else: # patches must have tar hash information - self.assertIn('tar_hash', meta.custom) + self.assertIn('tar_hash', meta.custom_internal) def test_get_targetinfo(self): client = self.get_refreshed_client() diff --git a/tests/test_common.py b/tests/test_common.py index e4d99fb..264ab3e 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -153,6 +153,27 @@ def test_compose_filename(self): ) self.assertEqual('app-1.0.tar.gz', filename) + def test_custom_metadata(self): + user_metadata = dict(foo='bar') + internal_metadata = dict(something=True) + target_meta = TargetMeta( + custom=dict(user=user_metadata, tufup=internal_metadata) + ) + self.assertEqual(user_metadata, target_meta.custom) + self.assertEqual(internal_metadata, target_meta.custom_internal) + + def test_custom_metadata_backward_compatibility(self): + # older versions of tufup did not distinguish between user and internal metadata + custom_metadata = dict(foo='bar') + target_meta = TargetMeta(custom=custom_metadata) # noqa + self.assertEqual(custom_metadata, target_meta.custom) + self.assertEqual(custom_metadata, target_meta.custom_internal) + + def test_custom_metadata_not_specified(self): + target_meta = TargetMeta() + self.assertIsNone(target_meta.custom) + self.assertIsNone(target_meta.custom_internal) + class PatcherTests(TempDirTestCase): def setUp(self) -> None: From b8ff6ae938f62ab25a35fc441601f0fe942896e5 Mon Sep 17 00:00:00 2001 From: dennisvang <29799340+dennisvang@users.noreply.github.com> Date: Fri, 8 Mar 2024 17:51:14 +0100 Subject: [PATCH 7/9] issue111: update test data and simplify KEY_REQUIRED value --- src/tufup/common.py | 2 +- tests/data/repository/metadata/1.root.json | 10 +-- tests/data/repository/metadata/2.root.json | 12 +-- tests/data/repository/metadata/root.json | 12 +-- tests/data/repository/metadata/snapshot.json | 4 +- tests/data/repository/metadata/targets.json | 83 +++++++++++++------ tests/data/repository/metadata/timestamp.json | 4 +- 7 files changed, 78 insertions(+), 49 deletions(-) diff --git a/src/tufup/common.py b/src/tufup/common.py index af12af0..b73a27d 100644 --- a/src/tufup/common.py +++ b/src/tufup/common.py @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) -KEY_REQUIRED = '_tufup_flag_required' # unlikely to be identical to user-specified key +KEY_REQUIRED = 'required' # unlikely to be identical to user-specified key SUFFIX_ARCHIVE = '.tar.gz' SUFFIX_PATCH = '.patch' diff --git a/tests/data/repository/metadata/1.root.json b/tests/data/repository/metadata/1.root.json index 8847a9d..910764e 100755 --- a/tests/data/repository/metadata/1.root.json +++ b/tests/data/repository/metadata/1.root.json @@ -1,18 +1,18 @@ { "signatures": [ { - "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", - "sig": "581b30a5a2a927c71479b568908bb517bc1d2c7a627428e8399b7c91ef427f4784357c855698c484d3a2aacc952ce1893ea246a03401315e2a473a5db2429d0f" + "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2", + "sig": "6970cde311f0bdf21335dbb80d1500215febf441b74e800445f25810f48b503f962eea2d314865336ccc83aae308d493707d2eca0f852496a4c1555e9eb7d301" }, { - "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2", - "sig": "5b53aa824b1e81ec3d7e1fe383ca41b156715f852e9c59bc3ba53275d95fb2869b778cce351da30b867fdb2ed55334fc22e4f9c13732b8a8557049000056ff09" + "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", + "sig": "4977d4109d9e7ba2a51e34988c5c45fe0a7e81b0edc64d0de428b362ca8397056cb168c85c764d44dc63124e085d7bfa9907bb702c656040bc912bcc1ec1ac0a" } ], "signed": { "_type": "root", "consistent_snapshot": false, - "expires": "2051-07-25T11:55:06Z", + "expires": "2051-07-25T16:49:47Z", "keys": { "5ef48ab6f5398d2bf17f1f4c4fc0e0440c4aa3734a05ae523561e02e8a99957a": { "keytype": "ed25519", diff --git a/tests/data/repository/metadata/2.root.json b/tests/data/repository/metadata/2.root.json index 07a0f2a..10b0a47 100755 --- a/tests/data/repository/metadata/2.root.json +++ b/tests/data/repository/metadata/2.root.json @@ -1,22 +1,22 @@ { "signatures": [ { - "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", - "sig": "3b5fef2d49edba8b0dfb8fb7781fd5721c3f11f602c5700d9821367862973a65d90d2cdc4fbf9433207475defec1eeea085cf9f1145f1eea54210647cbb34004" + "keyid": "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003", + "sig": "b3edc83b2236ccd0c78882e36431d6f2cfc1ae187d71cdae57864f18608d3b2da0495f843a173090aa0483d61e800e60eb00c55b5478b6aa7cef890cf67a1800" }, { "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2", - "sig": "10f57136bfbbd1b4f6c89ba87e1d63c2c1b4478e2a3ceed9924013dce51e886047e028adef41d0ae326940d228505acf78fc39f69fef03b5edc202b79d099808" + "sig": "e70fba716bac644045b1e76f378e601dd56884ab8976f50cf2076da4643e3de1aeff2fc48b6c19336db39e735ce7dce526f9445d0c11b1331398355cf2929602" }, { - "keyid": "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003", - "sig": "6da78dc6741ca4f4d83948d357812f6b7ebd62fc5a0ecbf157cde40b3851f448fae35fad5d9fa2e8453088312f6600083c45dfd2f4f72a4c293809d1aad24508" + "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", + "sig": "7ab48e365e9ddf4559abf719917fe7a49dbc5e30f5077c6f69d2d4db6e9352b53d5930b70970c70bb5570f2cbeb0ab0ba236c75e9209f689f073633708aeea09" } ], "signed": { "_type": "root", "consistent_snapshot": false, - "expires": "2051-07-25T11:55:10Z", + "expires": "2051-07-25T16:49:51Z", "keys": { "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003": { "keytype": "ed25519", diff --git a/tests/data/repository/metadata/root.json b/tests/data/repository/metadata/root.json index 07a0f2a..10b0a47 100755 --- a/tests/data/repository/metadata/root.json +++ b/tests/data/repository/metadata/root.json @@ -1,22 +1,22 @@ { "signatures": [ { - "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", - "sig": "3b5fef2d49edba8b0dfb8fb7781fd5721c3f11f602c5700d9821367862973a65d90d2cdc4fbf9433207475defec1eeea085cf9f1145f1eea54210647cbb34004" + "keyid": "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003", + "sig": "b3edc83b2236ccd0c78882e36431d6f2cfc1ae187d71cdae57864f18608d3b2da0495f843a173090aa0483d61e800e60eb00c55b5478b6aa7cef890cf67a1800" }, { "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2", - "sig": "10f57136bfbbd1b4f6c89ba87e1d63c2c1b4478e2a3ceed9924013dce51e886047e028adef41d0ae326940d228505acf78fc39f69fef03b5edc202b79d099808" + "sig": "e70fba716bac644045b1e76f378e601dd56884ab8976f50cf2076da4643e3de1aeff2fc48b6c19336db39e735ce7dce526f9445d0c11b1331398355cf2929602" }, { - "keyid": "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003", - "sig": "6da78dc6741ca4f4d83948d357812f6b7ebd62fc5a0ecbf157cde40b3851f448fae35fad5d9fa2e8453088312f6600083c45dfd2f4f72a4c293809d1aad24508" + "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", + "sig": "7ab48e365e9ddf4559abf719917fe7a49dbc5e30f5077c6f69d2d4db6e9352b53d5930b70970c70bb5570f2cbeb0ab0ba236c75e9209f689f073633708aeea09" } ], "signed": { "_type": "root", "consistent_snapshot": false, - "expires": "2051-07-25T11:55:10Z", + "expires": "2051-07-25T16:49:51Z", "keys": { "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003": { "keytype": "ed25519", diff --git a/tests/data/repository/metadata/snapshot.json b/tests/data/repository/metadata/snapshot.json index 1ea162b..27c94d6 100755 --- a/tests/data/repository/metadata/snapshot.json +++ b/tests/data/repository/metadata/snapshot.json @@ -2,12 +2,12 @@ "signatures": [ { "keyid": "5ef48ab6f5398d2bf17f1f4c4fc0e0440c4aa3734a05ae523561e02e8a99957a", - "sig": "9ad02e9e365e3ac008643d79e9c953ed6289ae63cfb3f22cf5c3029747f84e97c84f637d66838a35af10a50a5bdf0ca25c86e515129e85467c53e27c23ee690c" + "sig": "5b6c5ce5c2333b93a8f0e86b68fca178b301ee33db6db1df3ed9620e738c9b097886a5b8b8c7b297083f91ca3ca30167e6d9f576e033f01a36471a64bca22708" } ], "signed": { "_type": "snapshot", - "expires": "2051-07-25T11:55:10Z", + "expires": "2051-07-25T16:49:51Z", "meta": { "targets.json": { "version": 6 diff --git a/tests/data/repository/metadata/targets.json b/tests/data/repository/metadata/targets.json index 1cb9928..38f0df5 100755 --- a/tests/data/repository/metadata/targets.json +++ b/tests/data/repository/metadata/targets.json @@ -2,15 +2,21 @@ "signatures": [ { "keyid": "cd9930c92ac25c02a2f92ae3128b50459b53d7532ef9c0f364e78f388d5808a5", - "sig": "61b235261b5b9627c6f5bdf155dc2f49ead5deb67c8ee7d5696c92c62c17dbc219ffef3ce35ba48327cf46362f8ec427b37dd6965001a220790e5c41b094790b" + "sig": "af58034e8d5e5d525da096ad8946b44a2e82882d5a2f400b84e9ce956412d42c31f75a274b566dfe0f13117988a3fe605512e0924c4e9bd06bc33002b1b0c606" } ], "signed": { "_type": "targets", - "expires": "2051-07-25T11:55:10Z", + "expires": "2051-07-25T16:49:51Z", "spec_version": "1.0.31", "targets": { "example_app-1.0.tar.gz": { + "custom": { + "tufup": { + "required": false + }, + "user": null + }, "hashes": { "sha256": "223dfd468edbe36256dc119f8477ac4025279ae3705dec04a32002e81b10fd16" }, @@ -18,9 +24,12 @@ }, "example_app-2.0.patch": { "custom": { - "tar_hash": "855c631eb1a8d756bbad8441b76b5452505d292a162b3d497a60877fee2140b5", - "tar_hash_algorithm": "sha256", - "tar_size": 112640 + "tufup": { + "tar_hash": "855c631eb1a8d756bbad8441b76b5452505d292a162b3d497a60877fee2140b5", + "tar_hash_algorithm": "sha256", + "tar_size": 112640 + }, + "user": null }, "hashes": { "sha256": "f2be4504e464bd23c022772c7f3c011e0082295775a24e3fc986bb2504df0f53" @@ -29,12 +38,16 @@ }, "example_app-2.0.tar.gz": { "custom": { - "_tufup_flag_required": true, - "changes": [ - "this has changed", - "that has changed", - "..." - ] + "tufup": { + "required": true + }, + "user": { + "changes": [ + "this has changed", + "that has changed", + "..." + ] + } }, "hashes": { "sha256": "d85f423a56427e522ac4d093a6ce94abcc2cc32f99b80a39b87832d8e4ba9ad8" @@ -43,9 +56,12 @@ }, "example_app-3.0rc0.patch": { "custom": { - "tar_hash": "3cd260c121d05f4c6ed55b6e87569d3710e539e0a86e6fce98189ddca20c99f5", - "tar_hash_algorithm": "sha256", - "tar_size": 112640 + "tufup": { + "tar_hash": "3cd260c121d05f4c6ed55b6e87569d3710e539e0a86e6fce98189ddca20c99f5", + "tar_hash_algorithm": "sha256", + "tar_size": 112640 + }, + "user": null }, "hashes": { "sha256": "01fa6f30ac54fd405dfb5bd6e39f71af572c8e341675f5b2822b35ec341ce6f9" @@ -54,11 +70,16 @@ }, "example_app-3.0rc0.tar.gz": { "custom": { - "changes": [ - "this has changed", - "that has changed", - "..." - ] + "tufup": { + "required": false + }, + "user": { + "changes": [ + "this has changed", + "that has changed", + "..." + ] + } }, "hashes": { "sha256": "d7fa6ddd397282e8fa81924a31f340ebbcb8c082604c0549a38f5882cd3716c6" @@ -67,9 +88,12 @@ }, "example_app-4.0a0.patch": { "custom": { - "tar_hash": "3d3efe43388f3bbae910af39232526ef624d1540cfbb69cf0d4c66b7d5dc4b45", - "tar_hash_algorithm": "sha256", - "tar_size": 112640 + "tufup": { + "tar_hash": "3d3efe43388f3bbae910af39232526ef624d1540cfbb69cf0d4c66b7d5dc4b45", + "tar_hash_algorithm": "sha256", + "tar_size": 112640 + }, + "user": null }, "hashes": { "sha256": "e19ccd94e60d1d817dcc44c481f4fb48f54fa335cc0bd7a7e60377a4f6ccf2ea" @@ -78,11 +102,16 @@ }, "example_app-4.0a0.tar.gz": { "custom": { - "changes": [ - "this has changed", - "that has changed", - "..." - ] + "tufup": { + "required": false + }, + "user": { + "changes": [ + "this has changed", + "that has changed", + "..." + ] + } }, "hashes": { "sha256": "05304765a0cb4e40cbd7ca2587aeb5f0db36cd9b617921e0d141bb91d3304e2c" diff --git a/tests/data/repository/metadata/timestamp.json b/tests/data/repository/metadata/timestamp.json index d970671..b5cb81a 100755 --- a/tests/data/repository/metadata/timestamp.json +++ b/tests/data/repository/metadata/timestamp.json @@ -2,12 +2,12 @@ "signatures": [ { "keyid": "eddb87d254d513c1404d71e17620ecf5260e1836babdaa55197916c582f37a00", - "sig": "948741ee18e88ac4c4e97f534373ced3b3e640ec7352233b8e92f3732c4e5588e12729bea79b075cb23e3d36031c16571974d9baf4700da33c90096368948005" + "sig": "5e1b959bf5dea6612ac5ee08bd407379c51bfe9be7b9c87ff3f6c7ce2df3cc2508db3bc57abe9b7807e44485e2f02aaa6cdf511a1887425357ff25b2bb10ba09" } ], "signed": { "_type": "timestamp", - "expires": "2051-07-25T11:55:10Z", + "expires": "2051-07-25T16:49:51Z", "meta": { "snapshot.json": { "version": 7 From 157c3b28bb217a7b7e7e64441c38f851d9454880 Mon Sep 17 00:00:00 2001 From: dennisvang <29799340+dennisvang@users.noreply.github.com> Date: Mon, 11 Mar 2024 14:11:35 +0100 Subject: [PATCH 8/9] issue111: use subparser names in cli commands --- src/tufup/repo/cli.py | 20 ++++++++------- tests/test_repo_cli.py | 56 ++++++++++++++++++++++++++++++------------ 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/src/tufup/repo/cli.py b/src/tufup/repo/cli.py index 347f08a..ef6ef08 100644 --- a/src/tufup/repo/cli.py +++ b/src/tufup/repo/cli.py @@ -63,7 +63,9 @@ def get_parser() -> argparse.ArgumentParser: subparser_targets = subparsers.add_parser('targets', parents=[debug_parser]) subparser_targets.set_defaults(func=_cmd_targets) # we use nested subparsers to deal with mutually dependent arguments - targets_subparsers = subparser_targets.add_subparsers() + targets_subparsers = subparser_targets.add_subparsers( + dest='subcommand', required=True + ) subparser_targets_add = targets_subparsers.add_parser( 'add', help=HELP['targets_add'] ) @@ -98,7 +100,9 @@ def get_parser() -> argparse.ArgumentParser: '-e', '--encrypted', action='store_true', help=HELP['keys_encrypted'] ) # we use nested subparsers to deal with mutually dependent arguments - keys_subparsers = subparser_keys.add_subparsers(help=HELP['keys_subcommands']) + keys_subparsers = subparser_keys.add_subparsers( + dest='subcommand', help=HELP['keys_subcommands'] + ) subparser_keys_add = keys_subparsers.add_parser('add') subparser_keys_add.add_argument( 'role_name', choices=TOP_LEVEL_ROLE_NAMES, help=HELP['keys_role_name'] @@ -237,9 +241,7 @@ def _cmd_keys(options: argparse.Namespace): private_key_path=private_key_path, encrypted=options.encrypted ) _print_info('Key pair created.') - replace = hasattr(options, 'old_key_name') - add = hasattr(options, 'role_name') - if replace: + if options.subcommand == 'replace': _print_info( f'Replacing key {options.old_key_name} by {options.new_key_name}...' ) @@ -248,14 +250,14 @@ def _cmd_keys(options: argparse.Namespace): new_public_key_path=public_key_path, new_private_key_encrypted=options.encrypted, ) - elif add: + elif options.subcommand == 'add': _print_info(f'Adding key {options.new_key_name}...') repository.add_key( role_name=options.role_name, public_key_path=public_key_path, encrypted=options.encrypted, ) - if replace or add: + if options.subcommand in ['add', 'replace']: _print_info('Publishing changes...') repository.publish_changes(private_key_dirs=options.key_dirs) _print_info('Done.') @@ -264,14 +266,14 @@ def _cmd_keys(options: argparse.Namespace): def _cmd_targets(options: argparse.Namespace): logger.debug(f'command targets: {vars(options)}') repository = _get_repo() - if hasattr(options, 'app_version') and hasattr(options, 'bundle_dir'): + if options.subcommand == 'add': _print_info('Adding bundle...') repository.add_bundle( new_version=options.app_version, new_bundle_dir=options.bundle_dir, skip_patch=options.skip_patch, ) - else: + elif options.subcommand == 'remove-latest': _print_info('Removing latest bundle...') repository.remove_latest_bundle() _print_info('Publishing changes...') diff --git a/tests/test_repo_cli.py b/tests/test_repo_cli.py index a2eab77..85a7558 100644 --- a/tests/test_repo_cli.py +++ b/tests/test_repo_cli.py @@ -18,6 +18,7 @@ def test_get_parser(self): 'targets -d add 1.0 c:\\my_bundle_dir c:\\private_keys', 'targets -d add -s 1.0 c:\\my_bundle_dir c:\\private_keys', 'targets remove-latest c:\\private_keys', + 'keys my-key-name', # todo: doesn't do anything... use subcommand? 'keys my-key-name -c -e', 'keys my-key-name add root c:\\private_keys d:\\more_private_keys', 'keys my-key-name -c -e add root c:\\private_keys', @@ -31,8 +32,27 @@ def test_get_parser(self): args = cmd.split() options = parser.parse_args(args) expected_func_name = '_cmd_' + args[0] + self.assertEqual( + args[0] in ['targets', 'keys'], hasattr(options, 'subcommand') + ) self.assertEqual(expected_func_name, options.func.__name__) + def test_get_parser_incomplete_commands(self): + parser = tufup.repo.cli.get_parser() + for cmd in [ + 'targets', + 'targets add', + 'targets remove-latest', + 'keys', + 'keys my-key-name add', + 'keys my-key-name replace', + 'sign', + ]: + with self.subTest(msg=cmd): + args = cmd.split() + with self.assertRaises(SystemExit): + parser.parse_args(args) + class CommandTests(TempDirTestCase): def setUp(self) -> None: @@ -76,13 +96,16 @@ def test__cmd_init(self): self.mock_repo.initialize.assert_called() def test__cmd_keys_create(self): - options = argparse.Namespace(new_key_name='test', encrypted=True, create=True) + options = argparse.Namespace( + subcommand=None, new_key_name='test', encrypted=True, create=True + ) with patch('tufup.repo.cli.Repository', self.mock_repo_class): tufup.repo.cli._cmd_keys(options=options) self.mock_repo.keys.create_key_pair.assert_called() def test__cmd_keys_create_and_add_key(self): options = argparse.Namespace( + subcommand='add', create=True, encrypted=True, key_dirs=['c:\\my_private_keys'], @@ -97,6 +120,7 @@ def test__cmd_keys_create_and_add_key(self): def test__cmd_keys_replace_key(self): options = argparse.Namespace( + subcommand='replace', create=True, encrypted=False, key_dirs=['c:\\my_private_keys'], @@ -110,28 +134,28 @@ def test__cmd_keys_replace_key(self): self.mock_repo.publish_changes.assert_called() def test__cmd_targets_add(self): - version = '1.0' - bundle_dir = 'dummy' - key_dirs = ['c:\\my_private_keys'] - skip_patch = True - options = argparse.Namespace( - app_version=version, - bundle_dir=bundle_dir, - key_dirs=key_dirs, - skip_patch=skip_patch, + kwargs = dict( + subcommand='add', + app_version='1.0', + bundle_dir='dummy', + key_dirs=['c:\\my_private_keys'], + skip_patch=True, ) + options = argparse.Namespace(**kwargs) with patch('tufup.repo.cli.Repository', self.mock_repo_class): tufup.repo.cli._cmd_targets(options=options) self.mock_repo.add_bundle.assert_called_with( - new_version=version, - new_bundle_dir=bundle_dir, - skip_patch=skip_patch, + new_version=kwargs['app_version'], + new_bundle_dir=kwargs['bundle_dir'], + skip_patch=kwargs['skip_patch'], + ) + self.mock_repo.publish_changes.assert_called_with( + private_key_dirs=kwargs['key_dirs'] ) - self.mock_repo.publish_changes.assert_called_with(private_key_dirs=key_dirs) def test__cmd_targets_remove_latest(self): - key_dirs = ['c:\\my_private_keys'] - options = argparse.Namespace(key_dirs=key_dirs) + kwargs = dict(subcommand='remove-latest', key_dirs=['c:\\my_private_keys']) + options = argparse.Namespace(**kwargs) with patch('tufup.repo.cli.Repository', self.mock_repo_class): tufup.repo.cli._cmd_targets(options=options) self.mock_repo.remove_latest_bundle.assert_called() From 9a81e41485f346550165920be176f82261608f38 Mon Sep 17 00:00:00 2001 From: dennisvang <29799340+dennisvang@users.noreply.github.com> Date: Mon, 11 Mar 2024 14:15:07 +0100 Subject: [PATCH 9/9] issue111: add targets add --required CLI option --- src/tufup/repo/cli.py | 9 +++++++++ tests/test_repo_cli.py | 3 +++ 2 files changed, 12 insertions(+) diff --git a/src/tufup/repo/cli.py b/src/tufup/repo/cli.py index ef6ef08..45d0d13 100644 --- a/src/tufup/repo/cli.py +++ b/src/tufup/repo/cli.py @@ -18,6 +18,7 @@ targets_add_app_version='Application version (PEP440 compliant)', targets_add_bundle_dir='Directory containing application bundle.', targets_add_skip_patch='Skip patch creation.', + targets_add_required='Mark release as "required".', targets_remove_latest='Remove latest app bundle from the repository.', keys_subcommands='Optional commands to add or replace keys.', keys_new_key_name='Name of new private key (public key gets .pub suffix).', @@ -84,6 +85,13 @@ def get_parser() -> argparse.ArgumentParser: required=False, help=HELP['targets_add_skip_patch'], ) + subparser_targets_add.add_argument( + '-r', + '--required', + action='store_true', + required=False, + help=HELP['targets_add_required'], + ) subparser_targets_remove = targets_subparsers.add_parser( 'remove-latest', help=HELP['targets_remove_latest'] ) @@ -272,6 +280,7 @@ def _cmd_targets(options: argparse.Namespace): new_version=options.app_version, new_bundle_dir=options.bundle_dir, skip_patch=options.skip_patch, + required=options.required, ) elif options.subcommand == 'remove-latest': _print_info('Removing latest bundle...') diff --git a/tests/test_repo_cli.py b/tests/test_repo_cli.py index 85a7558..dc3ae79 100644 --- a/tests/test_repo_cli.py +++ b/tests/test_repo_cli.py @@ -16,6 +16,7 @@ def test_get_parser(self): 'init --debug', 'targets add 1.0 c:\\my_bundle_dir c:\\private_keys', 'targets -d add 1.0 c:\\my_bundle_dir c:\\private_keys', + 'targets -d add -r 1.0 c:\\my_bundle_dir c:\\private_keys', 'targets -d add -s 1.0 c:\\my_bundle_dir c:\\private_keys', 'targets remove-latest c:\\private_keys', 'keys my-key-name', # todo: doesn't do anything... use subcommand? @@ -140,6 +141,7 @@ def test__cmd_targets_add(self): bundle_dir='dummy', key_dirs=['c:\\my_private_keys'], skip_patch=True, + required=False, ) options = argparse.Namespace(**kwargs) with patch('tufup.repo.cli.Repository', self.mock_repo_class): @@ -148,6 +150,7 @@ def test__cmd_targets_add(self): new_version=kwargs['app_version'], new_bundle_dir=kwargs['bundle_dir'], skip_patch=kwargs['skip_patch'], + required=kwargs['required'], ) self.mock_repo.publish_changes.assert_called_with( private_key_dirs=kwargs['key_dirs']