Merge pull request #55 from freedomofpress/migrate-cleanup-to-gha

Migrate cleanup workflow to GitHub Actions

.github/workflows/cleanup.yml

@@ -0,0 +1,31 @@
+name: Remove old packages
+on:
+  schedule:
+    - cron: "0 8 * * *"
+defaults:
+  run:
+    shell: bash
+
+jobs:
+    clean-old-packages:
+        runs-on: ubuntu-latest
+        container: fedora:39
+        steps:
+            - name: Install dependencies
+              run: |
+                dnf install -y rpmdevtools git git-lfs
+            - uses: actions/checkout@v4
+              with:
+                lfs: true
+                token: ${{ secrets.PUSH_TOKEN }}
+            - name: Clean old packages
+              run: |
+                git config --global --add safe.directory '*'
+                git config user.email "[email protected]"
+                git config user.name "sdcibot"
+                # Preserve up to 4 packages for both nightlies and non-nightlies
+                find workstation -mindepth 1 -maxdepth 2 -type d | xargs -I '{}' ./scripts/clean-old-packages '{}' 4
+                git add .
+                # Index will be clean if there are no changes
+                git diff-index --quiet HEAD || git commit -m "Removing old packages"
+                git push origin main

scripts/clean-old-packages

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""
+Clean up old packages, specifying how many to keep
+
+Example:
+    ./clean-old-packages securedrop-yum-test/workstation/buster-nightlies 7
+
+This script is run in CI in a Fedora container. You can spin up a similar
+container locally using podman or docker, e.g.:
+
+    podman run -it --rm -v $(pwd):/workspace:Z fedora:39 bash
+
+The `rpm` module is provided by `python3-rpm`, and `rpmdev-vercmp`
+is provided by `rpmdevtools`.
+"""
+import argparse
+import functools
+import subprocess
+from collections import defaultdict
+from pathlib import Path
+from typing import Tuple
+
+import rpm
+
+
+def sort_rpm_versions(one: Tuple[str, Path], two: Tuple[str, Path]):
+    """sort two RPM package versions"""
+    status = subprocess.run(['rpmdev-vercmp', one[0], two[0]], stdout=subprocess.DEVNULL)
+    if status.returncode == 11:
+        # false, one is bigger
+        return 1
+    else:  # status.returncode == 12
+        # true, two is bigger
+        return -1
+
+
+def fix_name(name: str) -> str:
+    """
+    Linux packages embed the version in the name, so we'd never have multiple
+    packages meet the deletion threshold. Silly string manipulation to drop
+    the version.
+    E.g. "linux-image-5.15.26-grsec-securedrop" -> "linux-image-securedrop"
+    """
+    if name.endswith(('-securedrop', '-workstation')):
+        suffix = name.split('-')[-1]
+    else:
+        return name
+    if name.startswith('linux-image-'):
+        return f'linux-image-{suffix}'
+    elif name.startswith('linux-headers-'):
+        return f'linux-headers-{suffix}'
+    return name
+
+
+def rpm_info(path: Path) -> Tuple[str, str]:
+    """
+    learned this incantation from <https://web.archive.org/web/20120911204323/http://docs.fedoraproject.org/en-US/Fedora_Draft_Documentation/0.1/html/RPM_Guide/ch16s05.html>
+    and help(headers)
+    """
+    ts = rpm.ts()
+    with path.open() as f:
+        headers = ts.hdrFromFdno(f)
+    print(headers[rpm.RPMTAG_VERSION])
+
+    return headers[rpm.RPMTAG_NAME], headers[rpm.RPMTAG_VERSION] + '-' + headers[rpm.RPMTAG_RELEASE]
+
+
+def cleanup(data, to_keep: int, sorter):
+    for name, versions in sorted(data.items()):
+        if len(versions) <= to_keep:
+            # Nothing to delete
+            continue
+        print(f'### {name}')
+        items = sorted(versions.items(), key=functools.cmp_to_key(sorter), reverse=True)
+        keeps = items[:to_keep]
+        print('Keeping:')
+        for _, keep in keeps:
+            print(f'* {keep.name}')
+        delete = items[to_keep:]
+        print('Deleting:')
+        for _, path in delete:
+            print(f'* {path.name}')
+            path.unlink()
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Cleans up old packages"
+    )
+    parser.add_argument(
+        "directory",
+        type=Path,
+        help="Directory to clean up",
+    )
+    parser.add_argument(
+        "keep",
+        type=int,
+        help="Number of packages to keep"
+    )
+    args = parser.parse_args()
+    if not args.directory.is_dir():
+        raise RuntimeError(f"Directory, {args.directory}, doesn't exist")
+    print(f'Only keeping the latest {args.keep} packages')
+    rpms = defaultdict(dict)
+    for rpm in args.directory.glob('*.rpm'):
+        name, version = rpm_info(rpm)
+        rpms[name][version] = rpm
+
+    cleanup(rpms, args.keep, sort_rpm_versions)
+
+
+if __name__ == '__main__':
+    main()