Separate custom metadata into user-specified and tufup-internal

src/tufup/common.py

@@ -3,7 +3,7 @@
 import logging
 import pathlib
 import re
-from typing import Dict, Optional, Union
+from typing import Dict, get_type_hints, Optional, TypedDict, Union
 
 import bsdiff4
 from packaging.version import Version, InvalidVersion
@@ -14,6 +14,15 @@
 SUFFIX_PATCH = '.patch'
 
 
+class CustomMetadataDict(TypedDict):
+    """
+    explicitly separate custom metadata into user-specified metadata and metadata
+    used by tufup internally
+    """
+    user: Optional[dict]
+    tufup: Optional[dict]
+
+
 def _immutable(value):
     """
     Make value immutable, recursively, so the result is hashable.
@@ -49,7 +58,7 @@ def __init__(
         name: Optional[str] = None,
         version: Optional[str] = None,
         is_archive: Optional[bool] = True,
-        custom: Optional[dict] = None,
+        custom: Optional[CustomMetadataDict] = None,
     ):
         """
         Initialize either with target_path, or with name, version, archive.
@@ -68,7 +77,28 @@ def __init__(
             logger.critical(
                 f'invalid filename "{self.filename}": whitespace not allowed'
             )
-        self.custom = custom
+        self._custom = custom
+
+    @property
+    def custom(self) -> Optional[dict]:
+        """returns user-specified custom metadata"""
+        return self._get_custom_metadata('user')
+
+    @property
+    def custom_internal(self) -> Optional[dict]:
+        """returns tufup-internal custom metadata"""
+        return self._get_custom_metadata('tufup')
+
+    def _get_custom_metadata(self, key: str) -> Optional[dict]:
+        """
+        get custom metadata in a backward-compatible manner (older versions did not
+        distinguish between user-specified and internal metadata)
+        """
+        if isinstance(self._custom, dict):
+            # check dict keys for backward compatibility
+            if get_type_hints(CustomMetadataDict).keys() != self._custom.keys():
+                return self._custom
+            return self._custom.get(key)
 
     def __str__(self):
         return str(self.target_path_str)
@@ -246,7 +276,7 @@ def patch_and_verify(
         # verify integrity of the final result (raises exception on failure)
         cls._verify_tar_size_and_hash(
             tar_content=tar_bytes,
-            expected=patch_meta.custom,  # noqa
+            expected=patch_meta.custom_internal,  # noqa
         )
         # compress .tar data into destination .tar.gz file
         with gzip.open(dst_path, mode='wb') as dst_file:

src/tufup/repo/__init__.py

@@ -38,7 +38,7 @@
 )
 from tuf.api.serialization.json import JSONSerializer
 
-from tufup.common import Patcher, SUFFIX_PATCH, TargetMeta
+from tufup.common import CustomMetadataDict, Patcher, SUFFIX_PATCH, TargetMeta
 from tufup.utils.platform_specific import _patched_resolve
 
 logger = logging.getLogger(__name__)
@@ -375,7 +375,7 @@ def add_or_update_target(
         self,
         local_path: Union[pathlib.Path, str],
         url_path_segments: Optional[List[str]] = None,
-        custom: Optional[dict] = None,
+        custom: Optional[CustomMetadataDict] = None,
     ):
         # based on python-tuf basic_repo.py
         local_path = pathlib.Path(local_path)
@@ -767,7 +767,9 @@ def add_bundle(
         if not latest_archive or latest_archive.version < new_archive.version:
             # register new archive
             self.roles.add_or_update_target(
-                local_path=new_archive.path, custom=custom_metadata
+                local_path=new_archive.path,
+                # separate user-specified metadata from tufup-internal metadata
+                custom=dict(user=custom_metadata, tufup=None),
             )
             # create patch, if possible, and register that too
             if latest_archive and not skip_patch:
@@ -781,7 +783,8 @@ def add_bundle(
                 # register patch (size and hash are used by the client to verify the
                 # integrity of the patched archive)
                 self.roles.add_or_update_target(
-                    local_path=patch_path, custom=dst_size_and_hash
+                    local_path=patch_path,
+                    custom=dict(user=None, tufup=dst_size_and_hash),
                 )
 
     def remove_latest_bundle(self):

tests/data/repository/metadata/1.root.json

@@ -1,18 +1,18 @@
 {
  "signatures": [
   {
-   "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568",
-   "sig": "0f634a6e5f82af4447accce63c2987350c9c16fe6f8ce391ed504da106be8a127e1d606424c97a27822038cfd35e4daa96da2ec07a4a75bc2610df3bfc95cd0c"
+   "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2",
+   "sig": "9f80547ca0ab37b5de2ae3869be83b9e1312636c3f932265e1e02f528cd59c5057dee7f7d76f7d4f19241538fc578cd01c15b02e9ecfd9a2b6dd1324a53a0008"
   },
   {
-   "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2",
-   "sig": "678256d67bcf6022f75920ff380dc2111e2d68120af834f1769d694665236a2c7fb57ea5731f4050e1562a8b2be870b6594a2203f52182b1b77fa98ae89ed90c"
+   "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568",
+   "sig": "68a0d73c4327f7ac39dd15476e8d5d11fdedd034e5795e7ee1d4bea0066046e7d07f1508e2df0b7f99f39d66dc3c2b540632bafc121dbfa4d1c43f6f07448504"
   }
  ],
  "signed": {
   "_type": "root",
   "consistent_snapshot": false,
-  "expires": "2051-06-27T21:21:03Z",
+  "expires": "2051-07-25T14:59:45Z",
   "keys": {
    "5ef48ab6f5398d2bf17f1f4c4fc0e0440c4aa3734a05ae523561e02e8a99957a": {
     "keytype": "ed25519",
@@ -53,8 +53,8 @@
   "roles": {
    "root": {
     "keyids": [
-     "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2",
-     "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568"
+     "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568",
+     "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2"
     ],
     "threshold": 2
    },

tests/data/repository/metadata/2.root.json

@@ -2,21 +2,21 @@
  "signatures": [
   {
    "keyid": "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003",
-   "sig": "47a42813ae34829c60539dcceba0d4b9a8a9286beaa8d5f07d3de3050d404426c22bc95b271e7c5e7ee529bc3180f009eb31313fb825f76c3ed9ca2c501bd503"
+   "sig": "1e4c3c283c26ea9512fe9f6ae89583fbdfaab259d72f62a7f26d945e93755b5ffa334a7a2bda1f6eb6a22d3ad11546bfe790ac7aaca5c2a3389022d9f501a004"
   },
   {
-   "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568",
-   "sig": "421d85636350a89805abc4561acd3019ecf17246a37e91374a53276b5d56638c83754960c27d038c7d1193bdb33db12faf69b7a19099627c745c569093ee0005"
+   "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2",
+   "sig": "fdfb8c18130a5cad203bbe1fecf4de1a4d510c0b5daae7bddf53f3cf47a7ca5478338fa0edf6130a5d1c0a44b70071784be5e901670bd7a004ab61f8c4114c02"
   },
   {
-   "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2",
-   "sig": "a65dbf32349f1a57dd1dd6fc058c69a98be467f5ad408179da6e3b67abc6f2361415eb70214588d21079a9d0351500808f8c244b69f40b35a41999294461ca00"
+   "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568",
+   "sig": "6abb0b0a32fee9d038f9d4301e168a8ba530996d43ecc8a780626415e8cee111659741be61b0094435f3d89699546bb340480bfce637fc2230f355a3155ced0a"
   }
  ],
  "signed": {
   "_type": "root",
   "consistent_snapshot": false,
-  "expires": "2051-06-27T21:21:13Z",
+  "expires": "2051-07-25T14:59:52Z",
   "keys": {
    "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003": {
     "keytype": "ed25519",

tests/data/repository/metadata/root.json

@@ -2,21 +2,21 @@
  "signatures": [
   {
    "keyid": "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003",
-   "sig": "47a42813ae34829c60539dcceba0d4b9a8a9286beaa8d5f07d3de3050d404426c22bc95b271e7c5e7ee529bc3180f009eb31313fb825f76c3ed9ca2c501bd503"
+   "sig": "1e4c3c283c26ea9512fe9f6ae89583fbdfaab259d72f62a7f26d945e93755b5ffa334a7a2bda1f6eb6a22d3ad11546bfe790ac7aaca5c2a3389022d9f501a004"
   },
   {
-   "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568",
-   "sig": "421d85636350a89805abc4561acd3019ecf17246a37e91374a53276b5d56638c83754960c27d038c7d1193bdb33db12faf69b7a19099627c745c569093ee0005"
+   "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2",
+   "sig": "fdfb8c18130a5cad203bbe1fecf4de1a4d510c0b5daae7bddf53f3cf47a7ca5478338fa0edf6130a5d1c0a44b70071784be5e901670bd7a004ab61f8c4114c02"
   },
   {
-   "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2",
-   "sig": "a65dbf32349f1a57dd1dd6fc058c69a98be467f5ad408179da6e3b67abc6f2361415eb70214588d21079a9d0351500808f8c244b69f40b35a41999294461ca00"
+   "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568",
+   "sig": "6abb0b0a32fee9d038f9d4301e168a8ba530996d43ecc8a780626415e8cee111659741be61b0094435f3d89699546bb340480bfce637fc2230f355a3155ced0a"
   }
  ],
  "signed": {
   "_type": "root",
   "consistent_snapshot": false,
-  "expires": "2051-06-27T21:21:13Z",
+  "expires": "2051-07-25T14:59:52Z",
   "keys": {
    "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003": {
     "keytype": "ed25519",

tests/data/repository/metadata/snapshot.json

@@ -2,12 +2,12 @@
  "signatures": [
   {
    "keyid": "5ef48ab6f5398d2bf17f1f4c4fc0e0440c4aa3734a05ae523561e02e8a99957a",
-   "sig": "73a146f5e1f12c0a36e88c8d7bf613baa1d528ea0c9480fe0d2ccd74d6da239da04470f68d283738194185cc82289c5f9f1312efea373b51dc8722965ca1fc0b"
+   "sig": "48bdd9a911dc60620c513706bd0140573611a85713844d7ec38d938126ad4088688a6829d819049cd94c8a01ed1448e707800f7b4438a7edaa9edd1919612501"
   }
  ],
  "signed": {
   "_type": "snapshot",
-  "expires": "2051-06-27T21:21:13Z",
+  "expires": "2051-07-25T14:59:52Z",
   "meta": {
    "targets.json": {
     "version": 6

tests/data/repository/metadata/targets.json

@@ -2,25 +2,32 @@
  "signatures": [
   {
    "keyid": "cd9930c92ac25c02a2f92ae3128b50459b53d7532ef9c0f364e78f388d5808a5",
-   "sig": "344b1c779103db5c8462508d7a5e72ef9ae8dea0c5fd303d55cace03a87fd67312ff5ca01fc2e377d7d0dcbbbf3f4dff378f5c9759801590340c0b9e3d23bc07"
+   "sig": "ba92827c2fbe262ca8dc28dfc6b2629a4ce0c8af747d666b00e1104ccba004e3727a85eb00e957edeadae2a430972210c949723f4091e9f01f676025868bef03"
   }
  ],
  "signed": {
   "_type": "targets",
-  "expires": "2051-06-27T21:21:13Z",
+  "expires": "2051-07-25T14:59:52Z",
   "spec_version": "1.0.31",
   "targets": {
    "example_app-1.0.tar.gz": {
+    "custom": {
+     "tufup": null,
+     "user": null
+    },
     "hashes": {
      "sha256": "223dfd468edbe36256dc119f8477ac4025279ae3705dec04a32002e81b10fd16"
     },
     "length": 101613
    },
    "example_app-2.0.patch": {
     "custom": {
-     "tar_hash": "855c631eb1a8d756bbad8441b76b5452505d292a162b3d497a60877fee2140b5",
-     "tar_hash_algorithm": "sha256",
-     "tar_size": 112640
+     "tufup": {
+      "tar_hash": "855c631eb1a8d756bbad8441b76b5452505d292a162b3d497a60877fee2140b5",
+      "tar_hash_algorithm": "sha256",
+      "tar_size": 112640
+     },
+     "user": null
     },
     "hashes": {
      "sha256": "f2be4504e464bd23c022772c7f3c011e0082295775a24e3fc986bb2504df0f53"
@@ -29,11 +36,14 @@
    },
    "example_app-2.0.tar.gz": {
     "custom": {
-     "changes": [
-      "this has changed",
-      "that has changed",
-      "..."
-     ]
+     "tufup": null,
+     "user": {
+      "changes": [
+       "this has changed",
+       "that has changed",
+       "..."
+      ]
+     }
     },
     "hashes": {
      "sha256": "d85f423a56427e522ac4d093a6ce94abcc2cc32f99b80a39b87832d8e4ba9ad8"
@@ -42,9 +52,12 @@
    },
    "example_app-3.0rc0.patch": {
     "custom": {
-     "tar_hash": "3cd260c121d05f4c6ed55b6e87569d3710e539e0a86e6fce98189ddca20c99f5",
-     "tar_hash_algorithm": "sha256",
-     "tar_size": 112640
+     "tufup": {
+      "tar_hash": "3cd260c121d05f4c6ed55b6e87569d3710e539e0a86e6fce98189ddca20c99f5",
+      "tar_hash_algorithm": "sha256",
+      "tar_size": 112640
+     },
+     "user": null
     },
     "hashes": {
      "sha256": "01fa6f30ac54fd405dfb5bd6e39f71af572c8e341675f5b2822b35ec341ce6f9"
@@ -53,11 +66,14 @@
    },
    "example_app-3.0rc0.tar.gz": {
     "custom": {
-     "changes": [
-      "this has changed",
-      "that has changed",
-      "..."
-     ]
+     "tufup": null,
+     "user": {
+      "changes": [
+       "this has changed",
+       "that has changed",
+       "..."
+      ]
+     }
     },
     "hashes": {
      "sha256": "d7fa6ddd397282e8fa81924a31f340ebbcb8c082604c0549a38f5882cd3716c6"
@@ -66,9 +82,12 @@
    },
    "example_app-4.0a0.patch": {
     "custom": {
-     "tar_hash": "3d3efe43388f3bbae910af39232526ef624d1540cfbb69cf0d4c66b7d5dc4b45",
-     "tar_hash_algorithm": "sha256",
-     "tar_size": 112640
+     "tufup": {
+      "tar_hash": "3d3efe43388f3bbae910af39232526ef624d1540cfbb69cf0d4c66b7d5dc4b45",
+      "tar_hash_algorithm": "sha256",
+      "tar_size": 112640
+     },
+     "user": null
     },
     "hashes": {
      "sha256": "e19ccd94e60d1d817dcc44c481f4fb48f54fa335cc0bd7a7e60377a4f6ccf2ea"
@@ -77,11 +96,14 @@
    },
    "example_app-4.0a0.tar.gz": {
     "custom": {
-     "changes": [
-      "this has changed",
-      "that has changed",
-      "..."
-     ]
+     "tufup": null,
+     "user": {
+      "changes": [
+       "this has changed",
+       "that has changed",
+       "..."
+      ]
+     }
     },
     "hashes": {
      "sha256": "05304765a0cb4e40cbd7ca2587aeb5f0db36cd9b617921e0d141bb91d3304e2c"

tests/data/repository/metadata/timestamp.json

@@ -2,12 +2,12 @@
  "signatures": [
   {
    "keyid": "eddb87d254d513c1404d71e17620ecf5260e1836babdaa55197916c582f37a00",
-   "sig": "4f05f9947e1fd704ffb877fa994e8841eb1a34a2ccf021c2eac5a618eca3c11baad8531d154fa5c8aa187e0bfaa57b521901ef502bd7a3dc601bfa1c408a4106"
+   "sig": "cbff18b83e93143a98a418dccea7f1d2f4eefe466a955860c3dedd730b8ac08fa01b4eb3f3c96a3cb943e8bca154d5ea52aacdb97e53ea4a96a8441e9145b90e"
   }
  ],
  "signed": {
   "_type": "timestamp",
-  "expires": "2051-06-27T21:21:13Z",
+  "expires": "2051-07-25T14:59:53Z",
   "meta": {
    "snapshot.json": {
     "version": 7

tests/test_client.py

@@ -112,7 +112,7 @@ def test_trusted_target_metas(self):
                         self.assertIn(example_key, meta.custom)
                     else:
                         # patches must have tar hash information
-                        self.assertIn('tar_hash', meta.custom)
+                        self.assertIn('tar_hash', meta.custom_internal)
 
     def test_get_targetinfo(self):
         client = self.get_refreshed_client()

tests/test_common.py

@@ -153,6 +153,27 @@ def test_compose_filename(self):
         )
         self.assertEqual('app-1.0.tar.gz', filename)
 
+    def test_custom_metadata(self):
+        user_metadata = dict(foo='bar')
+        internal_metadata = dict(something=True)
+        target_meta = TargetMeta(
+            custom=dict(user=user_metadata, tufup=internal_metadata)
+        )
+        self.assertEqual(user_metadata, target_meta.custom)
+        self.assertEqual(internal_metadata, target_meta.custom_internal)
+
+    def test_custom_metadata_backward_compatibility(self):
+        # older versions of tufup did not distinguish between user and internal metadata
+        custom_metadata = dict(foo='bar')
+        target_meta = TargetMeta(custom=custom_metadata)  # noqa
+        self.assertEqual(custom_metadata, target_meta.custom)
+        self.assertEqual(custom_metadata, target_meta.custom_internal)
+
+    def test_custom_metadata_not_specified(self):
+        target_meta = TargetMeta()
+        self.assertIsNone(target_meta.custom)
+        self.assertIsNone(target_meta.custom_internal)
+
 
 class PatcherTests(TempDirTestCase):
     def setUp(self) -> None: