From d1c49aabbfa2d485c55f8c7acf70c40de2c26924 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 20 Feb 2024 15:36:52 -0500 Subject: [PATCH] Make c9s variant contain c9s content only, no OCP content This is a first stab at #799, aimed at the c9s variant to start. In this model, the base (container and disk) images we build in the pipeline do not contain any OCP-specific details. The compose is made up purely of RPMs coming out directly from the c9s pungi composes. Let's go over details of this in bullet form: 1. To emphasize the binding to c9s composes, we change the versioning scheme: the version string is now *exactly* the same version as the pungi compose from which we've built (well, we do add a `.N` field because we want to be able to rebuild multiple times on top of the same base pungi compose). It's almost like if our builds are part of the c9s pungi composes directly. (And maybe one day they will be...) This is implemented using a `versionary` script that queries compose info. 2. We no longer include `packages-openshift.yaml`: this has all the OCP stuff that we want to do in a layered build instead. 3. We no longer completely rewrite `/etc/os-release`. The host *is* image-mode CentOS Stream and e.g. `ID` will now say `centos`. However, we do still inject `VARIANT` and `VARIANT_ID` fields to note that it's of the CoreOS kind. We should probably actually match FCOS here and properly add a CoreOS variant in the `centos-release` package. 4. Tests which have to do with the OpenShift layer now have the required tag `openshift`. This means that it'll no longer run in the default set of kola tests. When building the derived image, we will run just those tests using `kola run --tag openshift --oscontainer ...`. Note that to make this work, OCP itself still needs to actually have that derived image containing the OCP bits. For now, we will build this in the pipelines (as a separate artifact that we push to the repos) but the eventual goal is that we'd split that out of the pipeline and have it be more like how the rest of OCP is built (using Prow/OSBS/Konflux). Note also we don't currently build the c9s variant in the pipelines but this is a long time overdue IMO. --- manifest-c9s.yaml | 105 ++------------ tests/kola/files/openvswitch-hugetlbfs-groups | 1 + tests/kola/version/rhel-major-version | 3 + tests/kola/version/rhel-matches-rhcos-build | 6 + versionary | 134 ++++++++++++++++++ 5 files changed, 157 insertions(+), 92 deletions(-) create mode 100755 versionary diff --git a/manifest-c9s.yaml b/manifest-c9s.yaml index b5e675a6d..dd3291eac 100644 --- a/manifest-c9s.yaml +++ b/manifest-c9s.yaml @@ -1,9 +1,9 @@ -# Manifest for CentOS Stream CoreOS (SCOS) +# Manifest for CentOS Stream 9 CoreOS Base rojig: license: MIT name: scos - summary: OKD 4 + summary: CentOS Stream 9 CoreOS variables: osversion: "c9s" @@ -12,7 +12,6 @@ variables: # common to RHEL 9 & C9S variants include: - common.yaml - - packages-openshift.yaml # Starting from here, everything should be specific to SCOS @@ -20,101 +19,23 @@ include: repos: - baseos - appstream - # CentOS Extras Common repo for SIG RPM GPG keys - - extras-common - # CentOS NFV SIG repo for openvswitch - - sig-nfv - # CentOS Cloud SIG repo for cri-o, cri-tools and conmon-rs - - sig-cloud-okd - # Include RHCOS 9 repo for oc, hyperkube - - rhel-9.2-server-ose-4.16 -# We include hours/minutes to avoid version number reuse -automatic-version-prefix: "416.9." -# This ensures we're semver-compatible which OpenShift wants -automatic-version-suffix: "-" -# Keep this is sync with the version in postprocess -mutate-os-release: "4.16" +# Match the format of c9s compose IDs. This field will be driven in the pipeline +# anyway to match exactly the same compose ID we're composing with so the value +# here is purely for developer builds. +automatic-version-prefix: "9-.dev" +mutate-os-release: "9" + +# Mark the OS as of the CoreOS variant. postprocess: - | - #!/usr/bin/env bash - set -xeo pipefail - - # Tweak /usr/lib/os-release - grep -v "OSTREE_VERSION" /etc/os-release > /usr/lib/os-release.stream - OCP_RELEASE="4.16" - ( - . /etc/os-release - cat > /usr/lib/os-release <> /usr/lib/os-release < /usr/lib/system-release-cpe < /usr/lib/system-release < /usr/lib/issue </ + # /os and we want to get to .../compose + return urlunparse(url._replace(path='/'.join(components[:-3]))) + + +def get_dnf_base(basedir): + base = dnf.Base() + base.conf.reposdir = basedir + base.read_all_repos() + return base + + +def setup_repos(base, treefile): + for repo in base.repos.values(): + repo.disable() + + eprint("Enabled repos:") + for repo in treefile['repos']: + base.repos[repo].enable() + eprint(f"- {repo}") + + +def get_next_iteration(composeid): + try: + with open('builds/builds.json') as f: + builds = json.load(f) + except FileNotFoundError: + builds = {'builds': []} + + if len(builds['builds']) == 0: + eprint("n: 0 (no previous builds)") + return 0 + + last_buildid = builds['builds'][0]['id'] + last_version = parse_version(last_buildid) + if not last_version: + eprint(f"n: 0 (previous version {last_buildid} does not match scheme)") + return 0 + + last_composeid, last_iteration = last_version + + if composeid != last_composeid: + eprint(f"n: 0 (previous version {last_buildid} compose ID does not match)") + return 0 + + n = last_iteration + 1 + eprint(f"n: {n} (incremented from previous version {last_buildid})") + return n + + +def parse_version(version): + m = re.match(r'^([0-9]+-[0-9]{8}\.[0-9]+)\.([0-9]+)$', version) + if m is None: + return None + composeid, iteration = m.groups() + return tuple((composeid, int(iteration))) + + +def get_manifest_path(): + if os.path.exists('src/config.json'): + config_json = json.load(open('src/config.json')) + variant = config_json['coreos-assembler.config-variant'] + return f'src/config/manifest-{variant}.yaml' + return 'src/config/manifest.yaml' + + +def get_flattened_manifest(fn): + return yaml.safe_load( + subprocess.check_output(['rpm-ostree', 'compose', 'tree', + '--print-only', fn])) + + +def eprint(*args): + print(*args, file=sys.stderr) + + +if __name__ == "__main__": + sys.exit(main())