diff --git a/src/tufup/common.py b/src/tufup/common.py index 5ca2b61..b46d99e 100644 --- a/src/tufup/common.py +++ b/src/tufup/common.py @@ -3,7 +3,7 @@ import logging import pathlib import re -from typing import Dict, Optional, Union +from typing import Dict, get_type_hints, Optional, TypedDict, Union import bsdiff4 from packaging.version import Version, InvalidVersion @@ -14,6 +14,15 @@ SUFFIX_PATCH = '.patch' +class CustomMetadataDict(TypedDict): + """ + explicitly separate custom metadata into user-specified metadata and metadata + used by tufup internally + """ + user: Optional[dict] + tufup: Optional[dict] + + def _immutable(value): """ Make value immutable, recursively, so the result is hashable. @@ -49,7 +58,7 @@ def __init__( name: Optional[str] = None, version: Optional[str] = None, is_archive: Optional[bool] = True, - custom: Optional[dict] = None, + custom: Optional[CustomMetadataDict] = None, ): """ Initialize either with target_path, or with name, version, archive. @@ -68,7 +77,28 @@ def __init__( logger.critical( f'invalid filename "{self.filename}": whitespace not allowed' ) - self.custom = custom + self._custom = custom + + @property + def custom(self) -> Optional[dict]: + """returns user-specified custom metadata""" + return self._get_custom_metadata('user') + + @property + def custom_internal(self) -> Optional[dict]: + """returns tufup-internal custom metadata""" + return self._get_custom_metadata('tufup') + + def _get_custom_metadata(self, key: str) -> Optional[dict]: + """ + get custom metadata in a backward-compatible manner (older versions did not + distinguish between user-specified and internal metadata) + """ + if isinstance(self._custom, dict): + # check dict keys for backward compatibility + if get_type_hints(CustomMetadataDict).keys() != self._custom.keys(): + return self._custom + return self._custom.get(key) def __str__(self): return str(self.target_path_str) @@ -246,7 +276,7 @@ def patch_and_verify( # verify integrity of the final result (raises exception on failure) cls._verify_tar_size_and_hash( tar_content=tar_bytes, - expected=patch_meta.custom, # noqa + expected=patch_meta.custom_internal, # noqa ) # compress .tar data into destination .tar.gz file with gzip.open(dst_path, mode='wb') as dst_file: diff --git a/src/tufup/repo/__init__.py b/src/tufup/repo/__init__.py index 8f735a9..52ebf28 100644 --- a/src/tufup/repo/__init__.py +++ b/src/tufup/repo/__init__.py @@ -38,7 +38,7 @@ ) from tuf.api.serialization.json import JSONSerializer -from tufup.common import Patcher, SUFFIX_PATCH, TargetMeta +from tufup.common import CustomMetadataDict, Patcher, SUFFIX_PATCH, TargetMeta from tufup.utils.platform_specific import _patched_resolve logger = logging.getLogger(__name__) @@ -375,7 +375,7 @@ def add_or_update_target( self, local_path: Union[pathlib.Path, str], url_path_segments: Optional[List[str]] = None, - custom: Optional[dict] = None, + custom: Optional[CustomMetadataDict] = None, ): # based on python-tuf basic_repo.py local_path = pathlib.Path(local_path) @@ -767,7 +767,9 @@ def add_bundle( if not latest_archive or latest_archive.version < new_archive.version: # register new archive self.roles.add_or_update_target( - local_path=new_archive.path, custom=custom_metadata + local_path=new_archive.path, + # separate user-specified metadata from tufup-internal metadata + custom=dict(user=custom_metadata, tufup=None), ) # create patch, if possible, and register that too if latest_archive and not skip_patch: @@ -781,7 +783,8 @@ def add_bundle( # register patch (size and hash are used by the client to verify the # integrity of the patched archive) self.roles.add_or_update_target( - local_path=patch_path, custom=dst_size_and_hash + local_path=patch_path, + custom=dict(user=None, tufup=dst_size_and_hash), ) def remove_latest_bundle(self): diff --git a/tests/data/repository/metadata/1.root.json b/tests/data/repository/metadata/1.root.json index 32f7456..0c055f7 100755 --- a/tests/data/repository/metadata/1.root.json +++ b/tests/data/repository/metadata/1.root.json @@ -1,18 +1,18 @@ { "signatures": [ { - "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", - "sig": "0f634a6e5f82af4447accce63c2987350c9c16fe6f8ce391ed504da106be8a127e1d606424c97a27822038cfd35e4daa96da2ec07a4a75bc2610df3bfc95cd0c" + "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2", + "sig": "9f80547ca0ab37b5de2ae3869be83b9e1312636c3f932265e1e02f528cd59c5057dee7f7d76f7d4f19241538fc578cd01c15b02e9ecfd9a2b6dd1324a53a0008" }, { - "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2", - "sig": "678256d67bcf6022f75920ff380dc2111e2d68120af834f1769d694665236a2c7fb57ea5731f4050e1562a8b2be870b6594a2203f52182b1b77fa98ae89ed90c" + "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", + "sig": "68a0d73c4327f7ac39dd15476e8d5d11fdedd034e5795e7ee1d4bea0066046e7d07f1508e2df0b7f99f39d66dc3c2b540632bafc121dbfa4d1c43f6f07448504" } ], "signed": { "_type": "root", "consistent_snapshot": false, - "expires": "2051-06-27T21:21:03Z", + "expires": "2051-07-25T14:59:45Z", "keys": { "5ef48ab6f5398d2bf17f1f4c4fc0e0440c4aa3734a05ae523561e02e8a99957a": { "keytype": "ed25519", @@ -53,8 +53,8 @@ "roles": { "root": { "keyids": [ - "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2", - "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568" + "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", + "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2" ], "threshold": 2 }, diff --git a/tests/data/repository/metadata/2.root.json b/tests/data/repository/metadata/2.root.json index c51f327..22c2903 100755 --- a/tests/data/repository/metadata/2.root.json +++ b/tests/data/repository/metadata/2.root.json @@ -2,21 +2,21 @@ "signatures": [ { "keyid": "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003", - "sig": "47a42813ae34829c60539dcceba0d4b9a8a9286beaa8d5f07d3de3050d404426c22bc95b271e7c5e7ee529bc3180f009eb31313fb825f76c3ed9ca2c501bd503" + "sig": "1e4c3c283c26ea9512fe9f6ae89583fbdfaab259d72f62a7f26d945e93755b5ffa334a7a2bda1f6eb6a22d3ad11546bfe790ac7aaca5c2a3389022d9f501a004" }, { - "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", - "sig": "421d85636350a89805abc4561acd3019ecf17246a37e91374a53276b5d56638c83754960c27d038c7d1193bdb33db12faf69b7a19099627c745c569093ee0005" + "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2", + "sig": "fdfb8c18130a5cad203bbe1fecf4de1a4d510c0b5daae7bddf53f3cf47a7ca5478338fa0edf6130a5d1c0a44b70071784be5e901670bd7a004ab61f8c4114c02" }, { - "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2", - "sig": "a65dbf32349f1a57dd1dd6fc058c69a98be467f5ad408179da6e3b67abc6f2361415eb70214588d21079a9d0351500808f8c244b69f40b35a41999294461ca00" + "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", + "sig": "6abb0b0a32fee9d038f9d4301e168a8ba530996d43ecc8a780626415e8cee111659741be61b0094435f3d89699546bb340480bfce637fc2230f355a3155ced0a" } ], "signed": { "_type": "root", "consistent_snapshot": false, - "expires": "2051-06-27T21:21:13Z", + "expires": "2051-07-25T14:59:52Z", "keys": { "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003": { "keytype": "ed25519", diff --git a/tests/data/repository/metadata/root.json b/tests/data/repository/metadata/root.json index c51f327..22c2903 100755 --- a/tests/data/repository/metadata/root.json +++ b/tests/data/repository/metadata/root.json @@ -2,21 +2,21 @@ "signatures": [ { "keyid": "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003", - "sig": "47a42813ae34829c60539dcceba0d4b9a8a9286beaa8d5f07d3de3050d404426c22bc95b271e7c5e7ee529bc3180f009eb31313fb825f76c3ed9ca2c501bd503" + "sig": "1e4c3c283c26ea9512fe9f6ae89583fbdfaab259d72f62a7f26d945e93755b5ffa334a7a2bda1f6eb6a22d3ad11546bfe790ac7aaca5c2a3389022d9f501a004" }, { - "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", - "sig": "421d85636350a89805abc4561acd3019ecf17246a37e91374a53276b5d56638c83754960c27d038c7d1193bdb33db12faf69b7a19099627c745c569093ee0005" + "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2", + "sig": "fdfb8c18130a5cad203bbe1fecf4de1a4d510c0b5daae7bddf53f3cf47a7ca5478338fa0edf6130a5d1c0a44b70071784be5e901670bd7a004ab61f8c4114c02" }, { - "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2", - "sig": "a65dbf32349f1a57dd1dd6fc058c69a98be467f5ad408179da6e3b67abc6f2361415eb70214588d21079a9d0351500808f8c244b69f40b35a41999294461ca00" + "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568", + "sig": "6abb0b0a32fee9d038f9d4301e168a8ba530996d43ecc8a780626415e8cee111659741be61b0094435f3d89699546bb340480bfce637fc2230f355a3155ced0a" } ], "signed": { "_type": "root", "consistent_snapshot": false, - "expires": "2051-06-27T21:21:13Z", + "expires": "2051-07-25T14:59:52Z", "keys": { "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003": { "keytype": "ed25519", diff --git a/tests/data/repository/metadata/snapshot.json b/tests/data/repository/metadata/snapshot.json index c05b5ad..b6a8333 100755 --- a/tests/data/repository/metadata/snapshot.json +++ b/tests/data/repository/metadata/snapshot.json @@ -2,12 +2,12 @@ "signatures": [ { "keyid": "5ef48ab6f5398d2bf17f1f4c4fc0e0440c4aa3734a05ae523561e02e8a99957a", - "sig": "73a146f5e1f12c0a36e88c8d7bf613baa1d528ea0c9480fe0d2ccd74d6da239da04470f68d283738194185cc82289c5f9f1312efea373b51dc8722965ca1fc0b" + "sig": "48bdd9a911dc60620c513706bd0140573611a85713844d7ec38d938126ad4088688a6829d819049cd94c8a01ed1448e707800f7b4438a7edaa9edd1919612501" } ], "signed": { "_type": "snapshot", - "expires": "2051-06-27T21:21:13Z", + "expires": "2051-07-25T14:59:52Z", "meta": { "targets.json": { "version": 6 diff --git a/tests/data/repository/metadata/targets.json b/tests/data/repository/metadata/targets.json index 122d603..a64f410 100755 --- a/tests/data/repository/metadata/targets.json +++ b/tests/data/repository/metadata/targets.json @@ -2,15 +2,19 @@ "signatures": [ { "keyid": "cd9930c92ac25c02a2f92ae3128b50459b53d7532ef9c0f364e78f388d5808a5", - "sig": "344b1c779103db5c8462508d7a5e72ef9ae8dea0c5fd303d55cace03a87fd67312ff5ca01fc2e377d7d0dcbbbf3f4dff378f5c9759801590340c0b9e3d23bc07" + "sig": "ba92827c2fbe262ca8dc28dfc6b2629a4ce0c8af747d666b00e1104ccba004e3727a85eb00e957edeadae2a430972210c949723f4091e9f01f676025868bef03" } ], "signed": { "_type": "targets", - "expires": "2051-06-27T21:21:13Z", + "expires": "2051-07-25T14:59:52Z", "spec_version": "1.0.31", "targets": { "example_app-1.0.tar.gz": { + "custom": { + "tufup": null, + "user": null + }, "hashes": { "sha256": "223dfd468edbe36256dc119f8477ac4025279ae3705dec04a32002e81b10fd16" }, @@ -18,9 +22,12 @@ }, "example_app-2.0.patch": { "custom": { - "tar_hash": "855c631eb1a8d756bbad8441b76b5452505d292a162b3d497a60877fee2140b5", - "tar_hash_algorithm": "sha256", - "tar_size": 112640 + "tufup": { + "tar_hash": "855c631eb1a8d756bbad8441b76b5452505d292a162b3d497a60877fee2140b5", + "tar_hash_algorithm": "sha256", + "tar_size": 112640 + }, + "user": null }, "hashes": { "sha256": "f2be4504e464bd23c022772c7f3c011e0082295775a24e3fc986bb2504df0f53" @@ -29,11 +36,14 @@ }, "example_app-2.0.tar.gz": { "custom": { - "changes": [ - "this has changed", - "that has changed", - "..." - ] + "tufup": null, + "user": { + "changes": [ + "this has changed", + "that has changed", + "..." + ] + } }, "hashes": { "sha256": "d85f423a56427e522ac4d093a6ce94abcc2cc32f99b80a39b87832d8e4ba9ad8" @@ -42,9 +52,12 @@ }, "example_app-3.0rc0.patch": { "custom": { - "tar_hash": "3cd260c121d05f4c6ed55b6e87569d3710e539e0a86e6fce98189ddca20c99f5", - "tar_hash_algorithm": "sha256", - "tar_size": 112640 + "tufup": { + "tar_hash": "3cd260c121d05f4c6ed55b6e87569d3710e539e0a86e6fce98189ddca20c99f5", + "tar_hash_algorithm": "sha256", + "tar_size": 112640 + }, + "user": null }, "hashes": { "sha256": "01fa6f30ac54fd405dfb5bd6e39f71af572c8e341675f5b2822b35ec341ce6f9" @@ -53,11 +66,14 @@ }, "example_app-3.0rc0.tar.gz": { "custom": { - "changes": [ - "this has changed", - "that has changed", - "..." - ] + "tufup": null, + "user": { + "changes": [ + "this has changed", + "that has changed", + "..." + ] + } }, "hashes": { "sha256": "d7fa6ddd397282e8fa81924a31f340ebbcb8c082604c0549a38f5882cd3716c6" @@ -66,9 +82,12 @@ }, "example_app-4.0a0.patch": { "custom": { - "tar_hash": "3d3efe43388f3bbae910af39232526ef624d1540cfbb69cf0d4c66b7d5dc4b45", - "tar_hash_algorithm": "sha256", - "tar_size": 112640 + "tufup": { + "tar_hash": "3d3efe43388f3bbae910af39232526ef624d1540cfbb69cf0d4c66b7d5dc4b45", + "tar_hash_algorithm": "sha256", + "tar_size": 112640 + }, + "user": null }, "hashes": { "sha256": "e19ccd94e60d1d817dcc44c481f4fb48f54fa335cc0bd7a7e60377a4f6ccf2ea" @@ -77,11 +96,14 @@ }, "example_app-4.0a0.tar.gz": { "custom": { - "changes": [ - "this has changed", - "that has changed", - "..." - ] + "tufup": null, + "user": { + "changes": [ + "this has changed", + "that has changed", + "..." + ] + } }, "hashes": { "sha256": "05304765a0cb4e40cbd7ca2587aeb5f0db36cd9b617921e0d141bb91d3304e2c" diff --git a/tests/data/repository/metadata/timestamp.json b/tests/data/repository/metadata/timestamp.json index e2ce07a..756c84d 100755 --- a/tests/data/repository/metadata/timestamp.json +++ b/tests/data/repository/metadata/timestamp.json @@ -2,12 +2,12 @@ "signatures": [ { "keyid": "eddb87d254d513c1404d71e17620ecf5260e1836babdaa55197916c582f37a00", - "sig": "4f05f9947e1fd704ffb877fa994e8841eb1a34a2ccf021c2eac5a618eca3c11baad8531d154fa5c8aa187e0bfaa57b521901ef502bd7a3dc601bfa1c408a4106" + "sig": "cbff18b83e93143a98a418dccea7f1d2f4eefe466a955860c3dedd730b8ac08fa01b4eb3f3c96a3cb943e8bca154d5ea52aacdb97e53ea4a96a8441e9145b90e" } ], "signed": { "_type": "timestamp", - "expires": "2051-06-27T21:21:13Z", + "expires": "2051-07-25T14:59:53Z", "meta": { "snapshot.json": { "version": 7 diff --git a/tests/test_client.py b/tests/test_client.py index a2e8c15..740edf9 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -112,7 +112,7 @@ def test_trusted_target_metas(self): self.assertIn(example_key, meta.custom) else: # patches must have tar hash information - self.assertIn('tar_hash', meta.custom) + self.assertIn('tar_hash', meta.custom_internal) def test_get_targetinfo(self): client = self.get_refreshed_client() diff --git a/tests/test_common.py b/tests/test_common.py index e4d99fb..264ab3e 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -153,6 +153,27 @@ def test_compose_filename(self): ) self.assertEqual('app-1.0.tar.gz', filename) + def test_custom_metadata(self): + user_metadata = dict(foo='bar') + internal_metadata = dict(something=True) + target_meta = TargetMeta( + custom=dict(user=user_metadata, tufup=internal_metadata) + ) + self.assertEqual(user_metadata, target_meta.custom) + self.assertEqual(internal_metadata, target_meta.custom_internal) + + def test_custom_metadata_backward_compatibility(self): + # older versions of tufup did not distinguish between user and internal metadata + custom_metadata = dict(foo='bar') + target_meta = TargetMeta(custom=custom_metadata) # noqa + self.assertEqual(custom_metadata, target_meta.custom) + self.assertEqual(custom_metadata, target_meta.custom_internal) + + def test_custom_metadata_not_specified(self): + target_meta = TargetMeta() + self.assertIsNone(target_meta.custom) + self.assertIsNone(target_meta.custom_internal) + class PatcherTests(TempDirTestCase): def setUp(self) -> None: