-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[WIP] Migrate cleanup workflow to GitHub Action
The `clean-old-packages` script was adapted from the `securedrop-builder` repository and originally written by Kunal Mehta.
- Loading branch information
Showing
2 changed files
with
142 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
name: Cleanup | ||
on: pull_request | ||
|
||
defaults: | ||
run: | ||
shell: bash | ||
|
||
jobs: | ||
clean-old-packages: | ||
runs-on: ubuntu-latest | ||
container: fedora:39 | ||
steps: | ||
- name: Install dependencies | ||
run: | | ||
dnf install -y rpmdevtools git git-lfs | ||
- uses: actions/checkout@v4 | ||
with: | ||
lfs: true | ||
token: ${{ secrets.PUSH_TOKEN }} | ||
- name: Clean old packages | ||
run: | | ||
git config --global --add safe.directory '*' | ||
git config user.email "[email protected]" | ||
git config user.name "sdcibot" | ||
find workstation -mindepth 1 -maxdepth 2 -type d | xargs -I '{}' ./scripts/clean-old-packages '{}' 4 | ||
git add . | ||
# Index will be clean if there are no changes | ||
git diff-index --quiet HEAD || git commit -m "Removing old packages" | ||
# git push origin main |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,113 @@ | ||
#!/usr/bin/env python3 | ||
""" | ||
Clean up old packages, specifying how many to keep | ||
Example: | ||
./clean-old-packages securedrop-yum-test/workstation/buster-nightlies 7 | ||
This script is run in CI in a Fedora container. You can spin up a similar | ||
container locally using podman or docker, e.g.: | ||
podman run -it --rm -v $(pwd):/workspace:Z fedora:39 bash | ||
The `rpm` module is provided by `python3-rpm`, and `rpmdev-vercmp` | ||
is provided by `rpmdevtools`. | ||
""" | ||
import argparse | ||
import functools | ||
import subprocess | ||
from collections import defaultdict | ||
from pathlib import Path | ||
from typing import Tuple | ||
|
||
import rpm | ||
|
||
|
||
def sort_rpm_versions(one: Tuple[str, Path], two: Tuple[str, Path]): | ||
"""sort two RPM package versions""" | ||
status = subprocess.run(['rpmdev-vercmp', one[0], two[0]], stdout=subprocess.DEVNULL) | ||
if status.returncode == 11: | ||
# false, one is bigger | ||
return 1 | ||
else: # status.returncode == 12 | ||
# true, two is bigger | ||
return -1 | ||
|
||
|
||
def fix_name(name: str) -> str: | ||
""" | ||
Linux packages embed the version in the name, so we'd never have multiple | ||
packages meet the deletion threshold. Silly string manipulation to drop | ||
the version. | ||
E.g. "linux-image-5.15.26-grsec-securedrop" -> "linux-image-securedrop" | ||
""" | ||
if name.endswith(('-securedrop', '-workstation')): | ||
suffix = name.split('-')[-1] | ||
else: | ||
return name | ||
if name.startswith('linux-image-'): | ||
return f'linux-image-{suffix}' | ||
elif name.startswith('linux-headers-'): | ||
return f'linux-headers-{suffix}' | ||
return name | ||
|
||
|
||
def rpm_info(path: Path) -> Tuple[str, str]: | ||
""" | ||
learned this incantation from <https://web.archive.org/web/20120911204323/http://docs.fedoraproject.org/en-US/Fedora_Draft_Documentation/0.1/html/RPM_Guide/ch16s05.html> | ||
and help(headers) | ||
""" | ||
ts = rpm.ts() | ||
with path.open() as f: | ||
headers = ts.hdrFromFdno(f) | ||
print(headers[rpm.RPMTAG_VERSION]) | ||
|
||
return headers[rpm.RPMTAG_NAME], headers[rpm.RPMTAG_VERSION] + '-' + headers[rpm.RPMTAG_RELEASE] | ||
|
||
|
||
def cleanup(data, to_keep: int, sorter): | ||
for name, versions in sorted(data.items()): | ||
if len(versions) <= to_keep: | ||
# Nothing to delete | ||
continue | ||
print(f'### {name}') | ||
items = sorted(versions.items(), key=functools.cmp_to_key(sorter), reverse=True) | ||
keeps = items[:to_keep] | ||
print('Keeping:') | ||
for _, keep in keeps: | ||
print(f'* {keep.name}') | ||
delete = items[to_keep:] | ||
print('Deleting:') | ||
for _, path in delete: | ||
print(f'* {path.name}') | ||
path.unlink() | ||
|
||
|
||
def main(): | ||
parser = argparse.ArgumentParser( | ||
description="Cleans up old packages" | ||
) | ||
parser.add_argument( | ||
"directory", | ||
type=Path, | ||
help="Directory to clean up", | ||
) | ||
parser.add_argument( | ||
"keep", | ||
type=int, | ||
help="Number of packages to keep" | ||
) | ||
args = parser.parse_args() | ||
if not args.directory.is_dir(): | ||
raise RuntimeError(f"Directory, {args.directory}, doesn't exist") | ||
print(f'Only keeping the latest {args.keep} packages') | ||
rpms = defaultdict(dict) | ||
for rpm in args.directory.glob('*.rpm'): | ||
name, version = rpm_info(rpm) | ||
rpms[name][version] = rpm | ||
|
||
cleanup(rpms, args.keep, sort_rpm_versions) | ||
|
||
|
||
if __name__ == '__main__': | ||
main() |