Add ability to mark releases as "required" (#122)

* handle "required" updates in Client.check_for_updates * add "required" arg to Repository.add_bundle() * warn if bundle is not added due to version * make app version 2.0 required in repo workflow example, and update test data and tests accordingly * update test data and simplify KEY_REQUIRED value * use subparser names in cli commands * add targets add --required CLI option
dennisvang · Mar 11, 2024 · eac3e8a · eac3e8a
1 parent 4e02bfe
commit eac3e8a
Show file tree

Hide file tree

Showing 14 changed files with 174 additions and 67 deletions.
diff --git a/examples/repo/repo_workflow_example.py b/examples/repo/repo_workflow_example.py
@@ -90,7 +90,7 @@
 KEY_MAP['root'].append('root_two')  # use two keys for root
 ENCRYPTED_KEYS = ['root', 'root_two', 'targets']
 
-# Custom metadata
+# Custom metadata (for example, a list of changes)
 DUMMY_METADATA = dict(changes=['this has changed', 'that has changed', '...'])
 
 # Create repository instance
@@ -169,7 +169,11 @@
     repo.add_bundle(
         new_version=new_version,
         new_bundle_dir=dummy_bundle_dir,
-        custom_metadata=DUMMY_METADATA,  # just to point out the option
+        # example of optional custom metadata
+        custom_metadata=DUMMY_METADATA.copy(),
+        # "required" updates are exceptional and should be avoided if possible,
+        # but we include one here just for completeness
+        required=new_version == '2.0',
     )
     repo.publish_changes(private_key_dirs=[OFFLINE_DIR_1, OFFLINE_DIR_2, ONLINE_DIR])
 

diff --git a/src/tufup/client.py b/src/tufup/client.py
@@ -12,7 +12,7 @@
 from tuf.api.exceptions import DownloadError, UnsignedMetadataError
 import tuf.ngclient
 
-from tufup.common import Patcher, TargetMeta
+from tufup.common import KEY_REQUIRED, Patcher, TargetMeta
 from tufup.utils.platform_specific import install_update
 
 logger = logging.getLogger(__name__)
@@ -138,7 +138,10 @@ def download_and_apply_update(
             )
 
     def check_for_updates(
-        self, pre: Optional[str] = None, patch: bool = True
+        self,
+        pre: Optional[str] = None,
+        patch: bool = True,
+        ignore_required: bool = False,
     ) -> Optional[TargetMeta]:
         """
         Check if any updates are available, based on current app version.
@@ -152,6 +155,14 @@ def check_for_updates(
         candidate, respectively.
 
         If `patch` is `False`, a full update is enforced.
+
+        If a new release is marked as "required" (in its custom metadata) this
+        release will take precedence over any non-required releases, *even* if the
+        latter are newer. This may be useful e.g. in case of a configuration change.
+        These "required" releases should be rare, and should preferably be avoided.
+        However, in the exceedingly rare event that there *are* "required" updates,
+        yet the user wants to treat them as non-required, they can specify
+        `ignore_required=True`.
         """
         # invalid pre-release specifiers are ignored, with a warning
         pre_map = dict(a='abrc', b='brc', rc='rc')
@@ -186,7 +197,16 @@ def check_for_updates(
         new_archive_meta = None
         if new_archives:
             logger.debug(f'{len(new_archives)} new *archives* found')
-            new_archive_meta, self.new_archive_info = sorted(new_archives.items())[-1]
+            # the "latest" archive is typically just the last one in the sorted list
+            # of new archives, except when there are new "required" archives,
+            # in which case we must update to the first "required" archive encountered
+            for archive_meta, archive_info in sorted(new_archives.items()):
+                if not ignore_required and archive_meta.custom_internal:
+                    if archive_meta.custom_internal.get(KEY_REQUIRED):
+                        logger.debug(f'required update found: {archive_meta.version}')
+                        break
+            new_archive_meta = archive_meta  # noqa
+            self.new_archive_info = archive_info  # noqa
             self.new_archive_local_path = pathlib.Path(
                 self.target_dir, new_archive_meta.path.name
             )

diff --git a/src/tufup/common.py b/src/tufup/common.py
@@ -10,6 +10,7 @@
 
 logger = logging.getLogger(__name__)
 
+KEY_REQUIRED = 'required'  # unlikely to be identical to user-specified key
 SUFFIX_ARCHIVE = '.tar.gz'
 SUFFIX_PATCH = '.patch'
 
@@ -19,6 +20,7 @@ class CustomMetadataDict(TypedDict):
     explicitly separate custom metadata into user-specified metadata and metadata
     used by tufup internally
     """
+
     user: Optional[dict]
     tufup: Optional[dict]
 

diff --git a/src/tufup/repo/__init__.py b/src/tufup/repo/__init__.py
@@ -38,7 +38,13 @@
 )
 from tuf.api.serialization.json import JSONSerializer
 
-from tufup.common import CustomMetadataDict, Patcher, SUFFIX_PATCH, TargetMeta
+from tufup.common import (
+    CustomMetadataDict,
+    KEY_REQUIRED,
+    Patcher,
+    SUFFIX_PATCH,
+    TargetMeta,
+)
 from tufup.utils.platform_specific import _patched_resolve
 
 logger = logging.getLogger(__name__)
@@ -735,6 +741,7 @@ def add_bundle(
         new_version: Optional[str] = None,
         skip_patch: bool = False,
         custom_metadata: Optional[dict] = None,  # archive only
+        required: bool = False,
     ):
         """
         Adds a new application bundle to the local repository.
@@ -744,6 +751,12 @@ def add_bundle(
         a patch file is also created and added to the repository, unless
         `skip_patch` is True.
 
+        If `required=True` (default is `False`), this release will always be
+        installed, even if newer releases are available. For example, suppose
+        an app is running at version 1.0, and version 2.0 is required, but version
+        3.0 is also available, then tufup will first update to version 2.0,
+        before updating to 3.0 on the next run.
+
         Note the changes are not published yet: call `publish_changes()` for
         that.
         """
@@ -769,7 +782,7 @@ def add_bundle(
             self.roles.add_or_update_target(
                 local_path=new_archive.path,
                 # separate user-specified metadata from tufup-internal metadata
-                custom=dict(user=custom_metadata, tufup=None),
+                custom=dict(user=custom_metadata, tufup={KEY_REQUIRED: required}),
             )
             # create patch, if possible, and register that too
             if latest_archive and not skip_patch:
@@ -786,6 +799,11 @@ def add_bundle(
                     local_path=patch_path,
                     custom=dict(user=None, tufup=dst_size_and_hash),
                 )
+        else:
+            logger.warning(
+                f'bundle not added: version {new_archive.version} must be greater than'
+                f'that of latest archive ({latest_archive.version})'
+            )
 
     def remove_latest_bundle(self):
         """

diff --git a/src/tufup/repo/cli.py b/src/tufup/repo/cli.py
@@ -18,6 +18,7 @@
     targets_add_app_version='Application version (PEP440 compliant)',
     targets_add_bundle_dir='Directory containing application bundle.',
     targets_add_skip_patch='Skip patch creation.',
+    targets_add_required='Mark release as "required".',
     targets_remove_latest='Remove latest app bundle from the repository.',
     keys_subcommands='Optional commands to add or replace keys.',
     keys_new_key_name='Name of new private key (public key gets .pub suffix).',
@@ -63,7 +64,9 @@ def get_parser() -> argparse.ArgumentParser:
     subparser_targets = subparsers.add_parser('targets', parents=[debug_parser])
     subparser_targets.set_defaults(func=_cmd_targets)
     # we use nested subparsers to deal with mutually dependent arguments
-    targets_subparsers = subparser_targets.add_subparsers()
+    targets_subparsers = subparser_targets.add_subparsers(
+        dest='subcommand', required=True
+    )
     subparser_targets_add = targets_subparsers.add_parser(
         'add', help=HELP['targets_add']
     )
@@ -82,6 +85,13 @@ def get_parser() -> argparse.ArgumentParser:
         required=False,
         help=HELP['targets_add_skip_patch'],
     )
+    subparser_targets_add.add_argument(
+        '-r',
+        '--required',
+        action='store_true',
+        required=False,
+        help=HELP['targets_add_required'],
+    )
     subparser_targets_remove = targets_subparsers.add_parser(
         'remove-latest', help=HELP['targets_remove_latest']
     )
@@ -98,7 +108,9 @@ def get_parser() -> argparse.ArgumentParser:
         '-e', '--encrypted', action='store_true', help=HELP['keys_encrypted']
     )
     # we use nested subparsers to deal with mutually dependent arguments
-    keys_subparsers = subparser_keys.add_subparsers(help=HELP['keys_subcommands'])
+    keys_subparsers = subparser_keys.add_subparsers(
+        dest='subcommand', help=HELP['keys_subcommands']
+    )
     subparser_keys_add = keys_subparsers.add_parser('add')
     subparser_keys_add.add_argument(
         'role_name', choices=TOP_LEVEL_ROLE_NAMES, help=HELP['keys_role_name']
@@ -237,9 +249,7 @@ def _cmd_keys(options: argparse.Namespace):
             private_key_path=private_key_path, encrypted=options.encrypted
         )
         _print_info('Key pair created.')
-    replace = hasattr(options, 'old_key_name')
-    add = hasattr(options, 'role_name')
-    if replace:
+    if options.subcommand == 'replace':
         _print_info(
             f'Replacing key {options.old_key_name} by {options.new_key_name}...'
         )
@@ -248,14 +258,14 @@ def _cmd_keys(options: argparse.Namespace):
             new_public_key_path=public_key_path,
             new_private_key_encrypted=options.encrypted,
         )
-    elif add:
+    elif options.subcommand == 'add':
         _print_info(f'Adding key {options.new_key_name}...')
         repository.add_key(
             role_name=options.role_name,
             public_key_path=public_key_path,
             encrypted=options.encrypted,
         )
-    if replace or add:
+    if options.subcommand in ['add', 'replace']:
         _print_info('Publishing changes...')
         repository.publish_changes(private_key_dirs=options.key_dirs)
     _print_info('Done.')
@@ -264,14 +274,15 @@ def _cmd_keys(options: argparse.Namespace):
 def _cmd_targets(options: argparse.Namespace):
     logger.debug(f'command targets: {vars(options)}')
     repository = _get_repo()
-    if hasattr(options, 'app_version') and hasattr(options, 'bundle_dir'):
+    if options.subcommand == 'add':
         _print_info('Adding bundle...')
         repository.add_bundle(
             new_version=options.app_version,
             new_bundle_dir=options.bundle_dir,
             skip_patch=options.skip_patch,
+            required=options.required,
         )
-    else:
+    elif options.subcommand == 'remove-latest':
         _print_info('Removing latest bundle...')
         repository.remove_latest_bundle()
     _print_info('Publishing changes...')

diff --git a/tests/data/repository/metadata/1.root.json b/tests/data/repository/metadata/1.root.json
@@ -2,17 +2,17 @@
  "signatures": [
   {
    "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2",
-   "sig": "9f80547ca0ab37b5de2ae3869be83b9e1312636c3f932265e1e02f528cd59c5057dee7f7d76f7d4f19241538fc578cd01c15b02e9ecfd9a2b6dd1324a53a0008"
+   "sig": "6970cde311f0bdf21335dbb80d1500215febf441b74e800445f25810f48b503f962eea2d314865336ccc83aae308d493707d2eca0f852496a4c1555e9eb7d301"
   },
   {
    "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568",
-   "sig": "68a0d73c4327f7ac39dd15476e8d5d11fdedd034e5795e7ee1d4bea0066046e7d07f1508e2df0b7f99f39d66dc3c2b540632bafc121dbfa4d1c43f6f07448504"
+   "sig": "4977d4109d9e7ba2a51e34988c5c45fe0a7e81b0edc64d0de428b362ca8397056cb168c85c764d44dc63124e085d7bfa9907bb702c656040bc912bcc1ec1ac0a"
   }
  ],
  "signed": {
   "_type": "root",
   "consistent_snapshot": false,
-  "expires": "2051-07-25T14:59:45Z",
+  "expires": "2051-07-25T16:49:47Z",
   "keys": {
    "5ef48ab6f5398d2bf17f1f4c4fc0e0440c4aa3734a05ae523561e02e8a99957a": {
     "keytype": "ed25519",

diff --git a/tests/data/repository/metadata/2.root.json b/tests/data/repository/metadata/2.root.json
@@ -2,21 +2,21 @@
  "signatures": [
   {
    "keyid": "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003",
-   "sig": "1e4c3c283c26ea9512fe9f6ae89583fbdfaab259d72f62a7f26d945e93755b5ffa334a7a2bda1f6eb6a22d3ad11546bfe790ac7aaca5c2a3389022d9f501a004"
+   "sig": "b3edc83b2236ccd0c78882e36431d6f2cfc1ae187d71cdae57864f18608d3b2da0495f843a173090aa0483d61e800e60eb00c55b5478b6aa7cef890cf67a1800"
   },
   {
    "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2",
-   "sig": "fdfb8c18130a5cad203bbe1fecf4de1a4d510c0b5daae7bddf53f3cf47a7ca5478338fa0edf6130a5d1c0a44b70071784be5e901670bd7a004ab61f8c4114c02"
+   "sig": "e70fba716bac644045b1e76f378e601dd56884ab8976f50cf2076da4643e3de1aeff2fc48b6c19336db39e735ce7dce526f9445d0c11b1331398355cf2929602"
   },
   {
    "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568",
-   "sig": "6abb0b0a32fee9d038f9d4301e168a8ba530996d43ecc8a780626415e8cee111659741be61b0094435f3d89699546bb340480bfce637fc2230f355a3155ced0a"
+   "sig": "7ab48e365e9ddf4559abf719917fe7a49dbc5e30f5077c6f69d2d4db6e9352b53d5930b70970c70bb5570f2cbeb0ab0ba236c75e9209f689f073633708aeea09"
   }
  ],
  "signed": {
   "_type": "root",
   "consistent_snapshot": false,
-  "expires": "2051-07-25T14:59:52Z",
+  "expires": "2051-07-25T16:49:51Z",
   "keys": {
    "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003": {
     "keytype": "ed25519",

diff --git a/tests/data/repository/metadata/root.json b/tests/data/repository/metadata/root.json
@@ -2,21 +2,21 @@
  "signatures": [
   {
    "keyid": "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003",
-   "sig": "1e4c3c283c26ea9512fe9f6ae89583fbdfaab259d72f62a7f26d945e93755b5ffa334a7a2bda1f6eb6a22d3ad11546bfe790ac7aaca5c2a3389022d9f501a004"
+   "sig": "b3edc83b2236ccd0c78882e36431d6f2cfc1ae187d71cdae57864f18608d3b2da0495f843a173090aa0483d61e800e60eb00c55b5478b6aa7cef890cf67a1800"
   },
   {
    "keyid": "d4ec748f9476f9f7e1f0a247b917dde4abe8a024de9ba34c7458b41bec8be6b2",
-   "sig": "fdfb8c18130a5cad203bbe1fecf4de1a4d510c0b5daae7bddf53f3cf47a7ca5478338fa0edf6130a5d1c0a44b70071784be5e901670bd7a004ab61f8c4114c02"
+   "sig": "e70fba716bac644045b1e76f378e601dd56884ab8976f50cf2076da4643e3de1aeff2fc48b6c19336db39e735ce7dce526f9445d0c11b1331398355cf2929602"
   },
   {
    "keyid": "b7ad916e4138911155b771d0ede66666e9647e7fb6c85a1904be97dee5653568",
-   "sig": "6abb0b0a32fee9d038f9d4301e168a8ba530996d43ecc8a780626415e8cee111659741be61b0094435f3d89699546bb340480bfce637fc2230f355a3155ced0a"
+   "sig": "7ab48e365e9ddf4559abf719917fe7a49dbc5e30f5077c6f69d2d4db6e9352b53d5930b70970c70bb5570f2cbeb0ab0ba236c75e9209f689f073633708aeea09"
   }
  ],
  "signed": {
   "_type": "root",
   "consistent_snapshot": false,
-  "expires": "2051-07-25T14:59:52Z",
+  "expires": "2051-07-25T16:49:51Z",
   "keys": {
    "1bd53d9d6f08f6efba19477880b348906f5f29a67d78cbca8a44aedfad12d003": {
     "keytype": "ed25519",

diff --git a/tests/data/repository/metadata/snapshot.json b/tests/data/repository/metadata/snapshot.json
@@ -2,12 +2,12 @@
  "signatures": [
   {
    "keyid": "5ef48ab6f5398d2bf17f1f4c4fc0e0440c4aa3734a05ae523561e02e8a99957a",
-   "sig": "48bdd9a911dc60620c513706bd0140573611a85713844d7ec38d938126ad4088688a6829d819049cd94c8a01ed1448e707800f7b4438a7edaa9edd1919612501"
+   "sig": "5b6c5ce5c2333b93a8f0e86b68fca178b301ee33db6db1df3ed9620e738c9b097886a5b8b8c7b297083f91ca3ca30167e6d9f576e033f01a36471a64bca22708"
   }
  ],
  "signed": {
   "_type": "snapshot",
-  "expires": "2051-07-25T14:59:52Z",
+  "expires": "2051-07-25T16:49:51Z",
   "meta": {
    "targets.json": {
     "version": 6

diff --git a/tests/data/repository/metadata/targets.json b/tests/data/repository/metadata/targets.json
@@ -2,17 +2,19 @@
  "signatures": [
   {
    "keyid": "cd9930c92ac25c02a2f92ae3128b50459b53d7532ef9c0f364e78f388d5808a5",
-   "sig": "ba92827c2fbe262ca8dc28dfc6b2629a4ce0c8af747d666b00e1104ccba004e3727a85eb00e957edeadae2a430972210c949723f4091e9f01f676025868bef03"
+   "sig": "af58034e8d5e5d525da096ad8946b44a2e82882d5a2f400b84e9ce956412d42c31f75a274b566dfe0f13117988a3fe605512e0924c4e9bd06bc33002b1b0c606"
   }
  ],
  "signed": {
   "_type": "targets",
-  "expires": "2051-07-25T14:59:52Z",
+  "expires": "2051-07-25T16:49:51Z",
   "spec_version": "1.0.31",
   "targets": {
    "example_app-1.0.tar.gz": {
     "custom": {
-     "tufup": null,
+     "tufup": {
+      "required": false
+     },
      "user": null
     },
     "hashes": {
@@ -36,7 +38,9 @@
    },
    "example_app-2.0.tar.gz": {
     "custom": {
-     "tufup": null,
+     "tufup": {
+      "required": true
+     },
      "user": {
       "changes": [
        "this has changed",
@@ -66,7 +70,9 @@
    },
    "example_app-3.0rc0.tar.gz": {
     "custom": {
-     "tufup": null,
+     "tufup": {
+      "required": false
+     },
      "user": {
       "changes": [
        "this has changed",
@@ -96,7 +102,9 @@
    },
    "example_app-4.0a0.tar.gz": {
     "custom": {
-     "tufup": null,
+     "tufup": {
+      "required": false
+     },
      "user": {
       "changes": [
        "this has changed",

diff --git a/tests/data/repository/metadata/timestamp.json b/tests/data/repository/metadata/timestamp.json
@@ -2,12 +2,12 @@
  "signatures": [
   {
    "keyid": "eddb87d254d513c1404d71e17620ecf5260e1836babdaa55197916c582f37a00",
-   "sig": "cbff18b83e93143a98a418dccea7f1d2f4eefe466a955860c3dedd730b8ac08fa01b4eb3f3c96a3cb943e8bca154d5ea52aacdb97e53ea4a96a8441e9145b90e"
+   "sig": "5e1b959bf5dea6612ac5ee08bd407379c51bfe9be7b9c87ff3f6c7ce2df3cc2508db3bc57abe9b7807e44485e2f02aaa6cdf511a1887425357ff25b2bb10ba09"
   }
  ],
  "signed": {
   "_type": "timestamp",
-  "expires": "2051-07-25T14:59:53Z",
+  "expires": "2051-07-25T16:49:51Z",
   "meta": {
    "snapshot.json": {
     "version": 7