diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 09d976684..fd2bdf158 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -10,7 +10,7 @@ env: jobs: hello-bench: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: HelloBench env: BENCHMARK_LOG_DIR: ${{ github.workspace }}/log/ @@ -29,8 +29,8 @@ jobs: steps: - name: Install tools run: | - sudo apt-get update && sudo apt-get --no-install-recommends install -y gnuplot - pip install numpy + sudo apt-get update && \ + sudo apt-get install -y gnuplot python3-numpy - uses: actions/checkout@v4 - name: Prepare directories run: mkdir "${BENCHMARK_RESULT_DIR}" "${BENCHMARK_LOG_DIR}" diff --git a/.github/workflows/kind-image.yml b/.github/workflows/kind-image.yml index ce28775d5..6b5c34029 100644 --- a/.github/workflows/kind-image.yml +++ b/.github/workflows/kind-image.yml @@ -10,7 +10,7 @@ env: jobs: kind-image: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: Kind image steps: - name: Checkout diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 71105b6b1..05adf55bf 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -22,7 +22,7 @@ env: jobs: integration: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: Integration steps: - name: Install htpasswd for setting up private registry @@ -32,7 +32,7 @@ jobs: run: make integration test-optimize: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: Optimize steps: - name: Install htpasswd for setting up private registry @@ -42,7 +42,7 @@ jobs: run: make test-optimize test-kind: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: Kind steps: - name: Install htpasswd for setting up private registry @@ -52,7 +52,7 @@ jobs: run: make test-kind test-criauth: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: CRIAuth steps: - name: Install htpasswd for setting up private registry @@ -62,7 +62,7 @@ jobs: run: make test-criauth test-cri-containerd: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: CRIValidationContainerd steps: - uses: actions/checkout@v4 @@ -70,7 +70,7 @@ jobs: run: make test-cri-containerd test-cri-o: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: CRIValidationCRIO steps: - name: Install the latest docker @@ -88,7 +88,7 @@ jobs: make test-cri-o test-k3s: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: K3S steps: - uses: actions/setup-go@v5 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 78577bf4a..3e1004103 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,7 +9,7 @@ env: jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: Build strategy: matrix: @@ -42,7 +42,7 @@ jobs: path: ${{ env.OUTPUT_DIR }}/* release: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: Release needs: [build] env: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a68dbe32f..70cdaf43e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,7 @@ env: jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: Build steps: - uses: actions/checkout@v4 @@ -18,7 +18,7 @@ jobs: run: ./script/util/make.sh build -j2 test: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: Test steps: - uses: actions/checkout@v4 @@ -26,7 +26,7 @@ jobs: run: ./script/util/make.sh test-all -j2 linter: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: Linter strategy: fail-fast: false @@ -47,7 +47,7 @@ jobs: working-directory: ${{ matrix.targetdir }} integration: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: Integration strategy: fail-fast: false @@ -76,7 +76,7 @@ jobs: run: make integration test-optimize: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: Optimize strategy: fail-fast: false @@ -94,7 +94,7 @@ jobs: run: make test-optimize test-kind: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: Kind strategy: fail-fast: false @@ -117,7 +117,7 @@ jobs: run: make test-kind test-criauth: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: CRIAuth strategy: fail-fast: false @@ -140,7 +140,7 @@ jobs: run: make test-criauth test-cri-containerd: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: CRIValidationContainerd strategy: fail-fast: false @@ -167,7 +167,7 @@ jobs: run: make test-cri-containerd test-cri-cri-o: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: CRIValidationCRIO strategy: fail-fast: false @@ -190,7 +190,7 @@ jobs: make test-cri-o test-podman: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: PodmanRootless steps: - uses: actions/checkout@v4 @@ -198,7 +198,7 @@ jobs: run: make test-podman test-k3s: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: K3S steps: - uses: actions/setup-go@v5 @@ -218,7 +218,7 @@ jobs: run: make test-k3s test-ipfs: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: IPFS steps: - uses: actions/checkout@v4 @@ -226,7 +226,7 @@ jobs: run: make test-ipfs test-k3s-argo-workflow: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: K3SArgoWorkflow env: RESULT_DIR: ${{ github.workspace }}/argo-workflow/ @@ -273,7 +273,7 @@ jobs: project: name: Project Checks - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 10 steps: - uses: actions/setup-go@v5 diff --git a/Dockerfile b/Dockerfile index 0e40a2b39..3bb40133b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -181,7 +181,7 @@ COPY --from=snapshotter-dev /out/ctr-remote /usr/local/bin/ RUN ln -s /usr/local/bin/ctr-remote /usr/local/bin/ctr # Base image which contains podman with stargz-store -FROM ubuntu:22.04 AS podman-base +FROM ubuntu:24.04 AS podman-base ARG TARGETARCH ARG CNI_PLUGINS_VERSION ARG PODMAN_VERSION diff --git a/script/benchmark/hello-bench/src/hello.py b/script/benchmark/hello-bench/src/hello.py index 50e3eb593..fe19df8dd 100755 --- a/script/benchmark/hello-bench/src/hello.py +++ b/script/benchmark/hello-bench/src/hello.py @@ -36,7 +36,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -import os, sys, subprocess, select, random, urllib2, time, json, tempfile, shutil, itertools +import os, sys, subprocess, select, random, time, json, tempfile, shutil, itertools TMP_DIR = tempfile.mkdtemp() LEGACY_MODE = "legacy" @@ -176,7 +176,7 @@ def __init__(self, repository='docker.io/library', srcrepository='docker.io/libr elif runtime == "podman": self.controller = PodmanController() else: - print 'Unknown runtime mode: '+runtime + print ('Unknown runtime mode: '+runtime) exit(1) self.repository = repository self.srcrepository = srcrepository @@ -236,15 +236,15 @@ def run_cmd_arg_wait(self, image, cid, runargs): stderr=subprocess.STDOUT, stdout=subprocess.PIPE) while True: - l = p.stdout.readline() + l = p.stdout.readline().decode("utf-8") if l == '': continue - print 'out: ' + l.strip() + print ('out: ' + l.strip()) # are we done? if l.find(runargs.waitline) >= 0: runtime = time.time() - startrun # cleanup - print 'DONE' + print ('DONE') cmd = self.controller.task_kill_cmd(cid) rc = os.system(cmd) assert(rc == 0) @@ -262,10 +262,10 @@ def run_cmd_stdin(self, image, cid, runargs): cmd = self.controller.task_start_cmd(cid) startrun = time.time() p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - print runargs.stdin - out, _ = p.communicate(runargs.stdin) + print (runargs.stdin) + out, _ = p.communicate(runargs.stdin.encode()) runtime = time.time() - startrun - print out + print (out) assert(p.returncode == 0) return createtime, runtime @@ -273,7 +273,7 @@ def run(self, bench, cid): name = bench.name image = self.fully_qualify(bench.name) - print "Pulling the image..." + print ("Pulling the image...") pullcmd = self.controller.pull_cmd(image) startpull = time.time() rc = os.system(pullcmd) @@ -291,7 +291,7 @@ def run(self, bench, cid): elif name in BenchRunner.CMD_STDIN: createtime, runtime = self.run_cmd_stdin(image=image, cid=cid, runargs=BenchRunner.CMD_STDIN[name]) else: - print 'Unknown bench: '+name + print ('Unknown bench: '+name) exit(1) return pulltime, createtime, runtime @@ -300,7 +300,7 @@ def convert_echo_hello(self, src, dest, option): period=10 cmd = ('%s %s -cni -period %s -entrypoint \'["/bin/sh", "-c"]\' -args \'["echo hello"]\' %s %s' % (self.optimizer, option, period, src, dest)) - print cmd + print (cmd) rc = os.system(cmd) assert(rc == 0) @@ -312,7 +312,7 @@ def convert_cmd_arg(self, src, dest, runargs, option): entry = '-entrypoint \'["/bin/sh", "-c"]\'' cmd = ('%s %s -cni -period %s %s %s %s %s' % (self.optimizer, option, period, entry, genargs_for_optimization(runargs.arg), src, dest)) - print cmd + print (cmd) rc = os.system(cmd) assert(rc == 0) @@ -323,10 +323,10 @@ def convert_cmd_arg_wait(self, src, dest, runargs, option): a = tmp_copy(a) mounts += '--mount type=bind,src=%s,dst=%s,options=rbind ' % (a,b) period = 90 - env = ' '.join(['-env %s=%s' % (k,v) for k,v in runargs.env.iteritems()]) + env = ' '.join(['-env %s=%s' % (k,v) for k,v in runargs.env.items()]) cmd = ('%s %s -cni -period %s %s %s %s %s %s' % (self.optimizer, option, period, mounts, env, genargs_for_optimization(runargs.arg), src, dest)) - print cmd + print (cmd) rc = os.system(cmd) assert(rc == 0) @@ -339,36 +339,36 @@ def convert_cmd_stdin(self, src, dest, runargs, option): period = 60 cmd = ('%s %s -i -cni -period %s %s -entrypoint \'["/bin/sh", "-c"]\' %s %s %s' % (self.optimizer, option, period, mounts, genargs_for_optimization(runargs.stdin_sh), src, dest)) - print cmd + print (cmd) p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) - print runargs.stdin - out,_ = p.communicate(runargs.stdin) - print out + print (runargs.stdin) + out,_ = p.communicate(runargs.stdin.encode()) + print (out) p.wait() assert(p.returncode == 0) def push_img(self, dest): cmd = '%s %s' % (self.pusher, dest) - print cmd + print (cmd) rc = os.system(cmd) assert(rc == 0) def pull_img(self, src): cmd = '%s %s' % (self.puller, src) - print cmd + print (cmd) rc = os.system(cmd) assert(rc == 0) def copy_img(self, src, dest): cmd = 'crane copy %s %s' % (src, dest) - print cmd + print (cmd) rc = os.system(cmd) assert(rc == 0) def convert_and_push_img(self, src, dest): self.pull_img(src) cmd = '%s --no-optimize %s %s' % (self.optimizer, src, dest) - print cmd + print (cmd) rc = os.system(cmd) assert(rc == 0) self.push_img(dest) @@ -384,7 +384,7 @@ def optimize_img(self, name, src, dest, option): elif name in BenchRunner.CMD_STDIN: self.convert_cmd_stdin(src=src, dest=dest, runargs=BenchRunner.CMD_STDIN[name], option=option) else: - print 'Unknown bench: '+name + print ('Unknown bench: '+name) exit(1) self.push_img(dest) @@ -403,7 +403,7 @@ def operation(self, op, bench, cid): self.prepare(bench) return 0, 0, 0 else: - print 'Unknown operation: '+op + print ('Unknown operation: '+op) exit(1) class ContainerdController: @@ -415,7 +415,7 @@ def pull_cmd(self, image): if self.is_lazypull: base_cmd = "ctr-remote i rpull" cmd = '%s %s' % (base_cmd, image) - print cmd + print (cmd) return cmd def create_echo_hello_cmd(self, image, cid): @@ -423,7 +423,7 @@ def create_echo_hello_cmd(self, image, cid): if self.is_lazypull: snapshotter_opt = "--snapshotter=stargz" cmd = '%s c create --net-host %s -- %s %s echo hello' % (CTR, snapshotter_opt, image, cid) - print cmd + print (cmd) return cmd def create_cmd_arg_cmd(self, image, cid, runargs): @@ -433,21 +433,21 @@ def create_cmd_arg_cmd(self, image, cid, runargs): cmd = '%s c create --net-host %s ' % (CTR, snapshotter_opt) cmd += '-- %s %s ' % (image, cid) cmd += runargs.arg - print cmd + print (cmd) return cmd def create_cmd_arg_wait_cmd(self, image, cid, runargs): snapshotter_opt = "" if self.is_lazypull: snapshotter_opt = "--snapshotter=stargz" - env = ' '.join(['--env %s=%s' % (k,v) for k,v in runargs.env.iteritems()]) + env = ' '.join(['--env %s=%s' % (k,v) for k,v in runargs.env.items()]) cmd = '%s c create --net-host %s %s '% (CTR, snapshotter_opt, env) for a,b in runargs.mount: a = os.path.join(os.path.dirname(os.path.abspath(__file__)), a) a = tmp_copy(a) cmd += '--mount type=bind,src=%s,dst=%s,options=rbind ' % (a,b) cmd += '-- %s %s %s' % (image, cid, runargs.arg) - print cmd + print (cmd) return cmd def create_cmd_stdin_cmd(self, image, cid, runargs): @@ -463,30 +463,30 @@ def create_cmd_stdin_cmd(self, image, cid, runargs): if runargs.stdin_sh: cmd += runargs.stdin_sh # e.g., sh -c - print cmd + print (cmd) return cmd def task_start_cmd(self, cid): cmd = '%s t start %s' % (CTR, cid) - print cmd + print (cmd) return cmd def task_kill_cmd(self, cid): cmd = '%s t kill -s 9 %s' % (CTR, cid) - print cmd + print (cmd) return cmd def cleanup(self, name, image): - print "Cleaning up environment..." + print ("Cleaning up environment...") cmd = '%s t kill -s 9 %s' % (CTR, name) - print cmd + print (cmd) rc = os.system(cmd) # sometimes containers already exit. we ignore the failure. cmd = '%s c rm %s' % (CTR, name) - print cmd + print (cmd) rc = os.system(cmd) assert(rc == 0) cmd = '%s image rm %s' % (CTR, image) - print cmd + print (cmd) rc = os.system(cmd) assert(rc == 0) @@ -500,29 +500,29 @@ def start_profile(self, seconds): class PodmanController: def pull_cmd(self, image): cmd = '%s pull docker://%s' % (PODMAN, image) - print cmd + print (cmd) return cmd def create_echo_hello_cmd(self, image, cid): cmd = '%s create --name %s docker://%s echo hello' % (PODMAN, cid, image) - print cmd + print (cmd) return cmd def create_cmd_arg_cmd(self, image, cid, runargs): cmd = '%s create --name %s docker://%s ' % (PODMAN, cid, image) cmd += runargs.arg - print cmd + print (cmd) return cmd def create_cmd_arg_wait_cmd(self, image, cid, runargs): - env = ' '.join(['--env %s=%s' % (k,v) for k,v in runargs.env.iteritems()]) + env = ' '.join(['--env %s=%s' % (k,v) for k,v in runargs.env.items()]) cmd = '%s create %s --name %s '% (PODMAN, env, cid) for a,b in runargs.mount: a = os.path.join(os.path.dirname(os.path.abspath(__file__)), a) a = tmp_copy(a) cmd += '--mount type=bind,src=%s,dst=%s ' % (a,b) cmd += ' docker://%s %s ' % (image, runargs.arg) - print cmd + print (cmd) return cmd def create_cmd_stdin_cmd(self, image, cid, runargs): @@ -535,51 +535,51 @@ def create_cmd_stdin_cmd(self, image, cid, runargs): if runargs.stdin_sh: cmd += runargs.stdin_sh # e.g., sh -c - print cmd + print (cmd) return cmd def task_start_cmd(self, cid): cmd = '%s start -a %s' % (PODMAN, cid) - print cmd + print (cmd) return cmd def task_kill_cmd(self, cid): cmd = '%s kill -s 9 %s' % (PODMAN, cid) - print cmd + print (cmd) return cmd def cleanup(self, name, image): - print "Cleaning up environment..." + print ("Cleaning up environment...") cmd = '%s kill -s 9 %s' % (PODMAN, name) - print cmd + print (cmd) rc = os.system(cmd) # sometimes containers already exit. we ignore the failure. cmd = '%s rm %s' % (PODMAN, name) - print cmd + print (cmd) rc = os.system(cmd) assert(rc == 0) cmd = '%s image rm %s' % (PODMAN, image) - print cmd + print (cmd) rc = os.system(cmd) assert(rc == 0) cmd = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../reboot_store.sh') - print cmd + print (cmd) rc = os.system(cmd) assert(rc == 0) def main(): if len(sys.argv) == 1: - print 'Usage: bench.py [OPTIONS] [BENCHMARKS]' - print 'OPTIONS:' - print '--repository=' - print '--srcrepository=' - print '--all' - print '--list' - print '--list-json' - print '--experiments' - print '--profile=' - print '--runtime' - print '--op=(prepare|run)' - print '--mode=(%s|%s|%s)' % (LEGACY_MODE, ESTARGZ_NOOPT_MODE, ESTARGZ_MODE, ZSTDCHUNKED_MODE) + print ('Usage: bench.py [OPTIONS] [BENCHMARKS]') + print ('OPTIONS:') + print ('--repository=') + print ('--srcrepository=') + print ('--all') + print ('--list') + print ('--list-json') + print ('--experiments') + print ('--profile=') + print ('--runtime') + print ('--op=(prepare|run)') + print ('--mode=(%s|%s|%s)' % (LEGACY_MODE, ESTARGZ_NOOPT_MODE, ESTARGZ_MODE, ZSTDCHUNKED_MODE)) exit(1) benches = [] @@ -594,11 +594,11 @@ def main(): benches.extend(BenchRunner.ALL.values()) elif parts[0] == 'list': template = '%-16s\t%-20s' - print template % ('CATEGORY', 'NAME') + print (template % ('CATEGORY', 'NAME')) for b in sorted(BenchRunner.ALL.values(), key=lambda b:(b.category, b.name)): - print template % (b.category, b.name) + print (template % (b.category, b.name)) elif parts[0] == 'list-json': - print json.dumps([b.__dict__ for b in BenchRunner.ALL.values()]) + print (json.dumps([b.__dict__ for b in BenchRunner.ALL.values()])) else: benches.append(BenchRunner.ALL[arg]) @@ -633,18 +633,18 @@ def main(): pull_times.append(pulltime) create_times.append(createtime) run_times.append(runtime) - print 'ITERATION %s:' % i - print 'elapsed %s' % elapsed - print 'pull %s' % pulltime - print 'create %s' % createtime - print 'run %s' % runtime + print ('ITERATION %s:' % i) + print ('elapsed %s' % elapsed) + print ('pull %s' % pulltime) + print ('create %s' % createtime) + print ('run %s' % runtime) if profile > 0 and time.time() - for_start > profile: break row = {'mode':'%s' % runner.mode, 'repo':bench.name, 'bench':bench.name, 'elapsed':sum(elapsed_times) / len(elapsed_times), 'elapsed_pull':sum(pull_times) / len(pull_times), 'elapsed_create':sum(create_times) / len(create_times), 'elapsed_run':sum(run_times) / len(run_times)} js = json.dumps(row) - print '%s%s,' % (BENCHMARKOUT_MARK, js) + print ('%s%s,' % (BENCHMARKOUT_MARK, js)) sys.stdout.flush() if __name__ == '__main__': diff --git a/script/benchmark/test.sh b/script/benchmark/test.sh index 5f3ab7383..4cfbe73b0 100755 --- a/script/benchmark/test.sh +++ b/script/benchmark/test.sh @@ -70,8 +70,8 @@ cat < "${TMP_CONTEXT}/Dockerfile" FROM ${BENCHMARKING_BASE_IMAGE_NAME} RUN apt-get update -y && \ - apt-get install -y python2 jq wget && \ - ln -s /usr/bin/python2 /usr/bin/python && \ + apt-get install -y python3 jq wget && \ + ln -s /usr/bin/python3 /usr/bin/python && \ mkdir -p /tmp/crane && \ wget -O - https://github.com/google/go-containerregistry/releases/download/v0.19.1/go-containerregistry_Linux_x86_64.tar.gz | tar -C /tmp/crane/ -zxf - && \ mv /tmp/crane/crane /usr/local/bin/ diff --git a/script/benchmark/tools/table.sh b/script/benchmark/tools/table.sh index 4c5b500e6..a133a7365 100755 --- a/script/benchmark/tools/table.sh +++ b/script/benchmark/tools/table.sh @@ -47,7 +47,7 @@ done cat <&1 | cat" ExecStopPost=podman unshare umount %h/.local/share/stargz-store/store +Restart=always +RestartSec=1 [Install] WantedBy=default.target diff --git a/script/podman/config/test-podman-rootless.sh b/script/podman/config/test-podman-rootless.sh index 40375b7ee..aee0e55c9 100755 --- a/script/podman/config/test-podman-rootless.sh +++ b/script/podman/config/test-podman-rootless.sh @@ -17,7 +17,7 @@ set -eux -o pipefail if [[ "$(id -u)" = "0" ]]; then # Switch to the rootless user via SSH; This is the same approach as done in nerdctl CI - systemctl start sshd + systemctl start ssh exec ssh -o StrictHostKeyChecking=no rootless@localhost "$0" "$@" else systemctl --user start podman-rootless-stargz-store diff --git a/script/podman/run_test.sh b/script/podman/run_test.sh index b25a9b2f6..5c6fd4844 100755 --- a/script/podman/run_test.sh +++ b/script/podman/run_test.sh @@ -18,6 +18,29 @@ set -euo pipefail STARGZ_STORE_SERVICE=podman-rootless-stargz-store +RETRYNUM=100 +RETRYINTERVAL=1 +TIMEOUTSEC=180 +function retry { + local SUCCESS=false + for i in $(seq ${RETRYNUM}) ; do + if eval "timeout ${TIMEOUTSEC} ${@}" ; then + SUCCESS=true + break + fi + echo "Fail(${i}). Retrying..." + sleep ${RETRYINTERVAL} + done + if [ "${SUCCESS}" == "true" ] ; then + return 0 + else + return 1 + fi +} + +echo "podman unshare mount | grep stargzstore" > /tmp/test1.sh +retry bash -euo pipefail /tmp/test1.sh + # Lazy pulling and run podman pull ghcr.io/stargz-containers/alpine:3.10.2-esgz podman run --rm ghcr.io/stargz-containers/alpine:3.10.2-esgz echo hello