Skip to content

Fix the race condition during GC of snapshots when client retries #19431

Fix the race condition during GC of snapshots when client retries

Fix the race condition during GC of snapshots when client retries #19431

Workflow file for this run

name: CI
on:
push:
branches:
- main
- "release/**"
pull_request:
branches:
- main
- "release/**"
permissions: # added using https://github.com/step-security/secure-workflows
contents: read
jobs:
#
# golangci-lint
#
linters:
permissions:
contents: read # for actions/checkout to fetch code
pull-requests: read # for golangci/golangci-lint-action to fetch pull requests
name: Linters
runs-on: ${{ matrix.os }}
timeout-minutes: 10
strategy:
matrix:
os: [ubuntu-20.04, actuated-arm64-4cpu-16gb, macos-12, windows-2019]
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/install-go
- uses: golangci/golangci-lint-action@v4
with:
only-new-issues: true
version: v1.60.1
skip-cache: true
args: --timeout=8m
#
# Project checks
#
project:
name: Project Checks
if: github.repository == 'containerd/containerd'
runs-on: ubuntu-20.04
timeout-minutes: 5
steps:
- uses: actions/checkout@v4
with:
path: src/github.com/containerd/containerd
fetch-depth: 100
- uses: ./src/github.com/containerd/containerd/.github/actions/install-go
- uses: containerd/[email protected]
with:
working-directory: src/github.com/containerd/containerd
repo-access-token: ${{ secrets.GITHUB_TOKEN }}
- name: verify go modules and vendor directory
run: |
sudo apt-get install -y jq
make verify-vendor
working-directory: src/github.com/containerd/containerd
#
# Protobuf checks
#
protos:
name: Protobuf
runs-on: ubuntu-20.04
timeout-minutes: 5
defaults:
run:
working-directory: src/github.com/containerd/containerd
steps:
- uses: actions/checkout@v4
with:
path: src/github.com/containerd/containerd
- uses: ./src/github.com/containerd/containerd/.github/actions/install-go
- name: Set env
shell: bash
run: |
echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
- name: Install protobuf
run: |
sudo -E PATH=$PATH script/setup/install-protobuf
sudo chmod +x /usr/local/bin/protoc
sudo chmod og+rx /usr/local/include/google /usr/local/include/google/protobuf /usr/local/include/google/protobuf/compiler
sudo chmod -R og+r /usr/local/include/google/protobuf/
protoc --version
- run: script/setup/install-dev-tools
- run: make proto-fmt
- run: make check-protos check-api-descriptors
man:
name: Manpages
runs-on: ubuntu-20.04
timeout-minutes: 5
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/install-go
- run: go install github.com/cpuguy83/go-md2man/[email protected]
- run: make man
# Make sure binaries compile with other minor platforms.
# Well-known architectures are covered in release.yml.
crossbuild:
name: Crossbuild Binaries
needs: [project, linters, protos, man]
runs-on: ubuntu-20.04
timeout-minutes: 10
strategy:
fail-fast: false
matrix:
include:
- goos: linux
goarch: arm
goarm: "7"
- goos: linux
goarch: arm
goarm: "5"
- goos: freebsd
goarch: amd64
- goos: freebsd
goarch: arm64
- goos: windows
goarch: arm
goarm: "7"
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/install-go
- run: |
set -e -x
packages=""
platform="${{matrix.goos}}/${{matrix.goarch}}"
if [ -n "${{matrix.goarm}}" ]; then
platform+="/v${{matrix.goarm}}"
fi
case "${platform}" in
linux/arm/v5)
packages+=" crossbuild-essential-armel"
echo "CGO_ENABLED=1" >> $GITHUB_ENV
echo "CC=arm-linux-gnueabi-gcc" >> $GITHUB_ENV
;;
linux/arm/v7)
packages+=" crossbuild-essential-armhf"
echo "CGO_ENABLED=1" >> $GITHUB_ENV
echo "CC=arm-linux-gnueabihf-gcc" >> $GITHUB_ENV
;;
windows/arm/v7)
echo "CGO_ENABLED=0" >> $GITHUB_ENV
;;
esac
if [ -n "${packages}" ]; then
sudo apt-get update && sudo apt-get install -y ${packages}
fi
name: Install deps
- name: Build
env:
GOOS: ${{matrix.goos}}
GOARCH: ${{matrix.goarch}}
GOARM: ${{matrix.goarm}}
run: |
make build
make binaries
#
# Build containerd binaries
#
binaries:
name: Binaries
runs-on: ${{ matrix.os }}
timeout-minutes: 10
needs: [project, linters, protos, man]
strategy:
matrix:
os: [ubuntu-20.04, actuated-arm64-4cpu-16gb, macos-12, windows-2019, windows-2022]
go-version: ["1.22.7", "1.23.1"]
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/install-go
with:
go-version: ${{ matrix.go-version }}
- name: Make
run: |
make build
make binaries
#
# Integration and CRI tests
#
integration-windows:
name: Windows Integration
runs-on: ${{ matrix.os }}
timeout-minutes: 90
needs: [project, linters, protos, man]
env:
GOTEST: gotestsum --
strategy:
fail-fast: false
matrix:
os: [windows-2019, windows-2022]
enable_cri_sandboxes: ["", "sandboxed"]
defaults:
run:
shell: bash
working-directory: src/github.com/containerd/containerd
steps:
- uses: actions/checkout@v4
with:
path: src/github.com/containerd/containerd
- uses: ./src/github.com/containerd/containerd/.github/actions/install-go
- uses: actions/checkout@v4
with:
repository: kubernetes-sigs/cri-tools
path: src/github.com/kubernetes-sigs/cri-tools
fetch-depth: 0
- name: Set env
run: |
echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
echo "${{ github.workspace }}/src/github.com/containerd/containerd/bin" >> $GITHUB_PATH
echo "${{ github.workspace }}/src/github.com/kubernetes-sigs/cri-tools/build/bin/windows/amd64" >> $GITHUB_PATH
- run: script/setup/install-dev-tools
# needs to be a separate step since terminal reload is required to bring in new env variables and PATH
- name: Upgrade Chocolaty on Windows 2019
if: matrix.os == 'windows-2019'
shell: powershell
run: .\script\setup\upgrade_chocolaty_windows_2019.ps1
- name: Upgrade MinGW on Windows 2019
if: matrix.os == 'windows-2019'
shell: powershell
run: .\script\setup\upgrade_mingw_windows_2019.ps1
- name: Binaries
shell: bash
env:
CGO_ENABLED: 1
run: |
set -o xtrace
mingw32-make.exe binaries
CRITEST_VERSION=$(cat script/setup/critools-version)
cd ../../kubernetes-sigs/cri-tools
git checkout "${CRITEST_VERSION}"
make critest
- run: script/setup/install-cni-windows
- env:
TEST_IMAGE_LIST: ${{github.workspace}}/repolist.toml
CRI_TEST_IMAGES: ${{github.workspace}}/cri-test-images.yaml
BUSYBOX_TESTING_IMAGE_REF: "registry.k8s.io/e2e-test-images/busybox:1.29-2"
RESOURCE_CONSUMER_TESTING_IMAGE_REF: "registry.k8s.io/e2e-test-images/resource-consumer:1.10"
WEBSERVER_TESTING_IMAGE_REF: "registry.k8s.io/e2e-test-images/nginx:1.14-2"
run: |
cat > "${{ env.TEST_IMAGE_LIST }}" << EOF
busybox = "${{ env.BUSYBOX_TESTING_IMAGE_REF }}"
ResourceConsumer = "${{ env.RESOURCE_CONSUMER_TESTING_IMAGE_REF }}"
EOF
cat > "${{ env.CRI_TEST_IMAGES }}" << EOF
defaultTestContainerImage: ${{ env.BUSYBOX_TESTING_IMAGE_REF }}
webServerTestImage: ${{ env.WEBSERVER_TESTING_IMAGE_REF }}
EOF
- name: Get crictl tool
shell: powershell
run: |
# Get critctl tool. Used for cri-integration tests
$CRICTL_DOWNLOAD_URL="https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.21.0/crictl-v1.21.0-windows-amd64.tar.gz"
curl.exe -L $CRICTL_DOWNLOAD_URL -o c:\crictl.tar.gz
tar -xvf c:\crictl.tar.gz
mv crictl.exe "${{ github.workspace }}/bin/crictl.exe" # Move crictl somewhere in path
- run: script/setup/install-gotestsum
- run: script/setup/install-teststat
- name: Tests
env:
CGO_ENABLED: 1
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root.xml
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-unit-root-gotest.json
run: mingw32-make.exe test root-test
- run: if [ -f *-gotest.json ]; then echo '# Root Test' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
if: always()
- run: script/test/test2annotation.sh ${TESTFILE}
env:
TESTFILE: ${{github.workspace}}/test-unit-root-gotest.json
if: always()
- name: Integration 1
env:
CGO_ENABLED: 1
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial-gotest.json
EXTRA_TESTFLAGS: "-timeout=20m"
run: mingw32-make.exe integration
- run: if [ -f *-gotest.json ]; then echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
if: always()
- run: script/test/test2annotation.sh ${TESTFILE}
env:
TESTFILE: ${{github.workspace}}/test-integration-serial-gotest.json
if: always()
# Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759
- name: Integration 2
env:
TESTFLAGS_PARALLEL: 1
EXTRA_TESTFLAGS: "-short"
CGO_ENABLED: 1
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel-gotest.json
run: mingw32-make.exe integration
- run: if [ -f *-gotest.json ]; then echo '# Integration 2' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
if: always()
- run: script/test/test2annotation.sh ${TESTFILE}
env:
TESTFILE: ${{github.workspace}}/test-integration-parallel-gotest.json
if: always()
- name: CRI Integration Test
env:
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
TEST_IMAGE_LIST: ${{github.workspace}}/repolist.toml
run: |
make cri-integration
- name: cri-tools critest
env:
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
CRI_TEST_IMAGES: ${{github.workspace}}/cri-test-images.yaml
shell: powershell
run: |
Start-Process -FilePath containerd.exe -NoNewWindow -RedirectStandardError true -PassThru
get-process | sls containerd
start-sleep 5
# This test is exceedingly flaky only on ws2022 so skip for now to keep CI happy.
# Info: https://github.com/containerd/containerd/issues/6652
if( '${{ matrix.os }}' -eq 'windows-2022' )
{
$skip = "-ginkgo.skip=runtime should support exec with tty=true and stdin=true"
}
critest.exe --runtime-endpoint=npipe://.//pipe//containerd-containerd --test-images-file='${{env.CRI_TEST_IMAGES}}' --report-dir='${{github.workspace}}/critestreport' $skip
- uses: actions/upload-artifact@v4
if: always()
with:
name: TestResults ${{ matrix.os }} ${{ matrix.enable_cri_sandboxes }}
path: |
${{github.workspace}}/*-junit.xml
${{github.workspace}}/*-gotest.json
${{github.workspace}}/report/*.log
integration-linux:
name: Linux Integration
runs-on: ${{ matrix.os }}
timeout-minutes: 40
needs: [project, linters, protos, man]
strategy:
fail-fast: false
matrix:
runtime:
- io.containerd.runtime.v1.linux
- io.containerd.runc.v1
- io.containerd.runc.v2
runc: [runc, crun]
enable_cri_sandboxes: ["", "sandboxed"]
os: [ubuntu-20.04, actuated-arm64-4cpu-16gb]
exclude:
- runtime: io.containerd.runc.v1
runc: crun
- runtime: io.containerd.runtime.v1.linux
runc: crun
- runtime: io.containerd.runc.v1
os: actuated-arm64-4cpu-16gb
- runtime: io.containerd.runtime.v1.linux
os: actuated-arm64-4cpu-16gb
env:
GOTEST: gotestsum --
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/install-go
- name: Install containerd dependencies
env:
RUNC_FLAVOR: ${{ matrix.runc }}
run: |
sudo apt-get update
sudo apt-get install -y gperf dmsetup strace xfsprogs
script/setup/install-seccomp
script/setup/install-runc
script/setup/install-cni $(grep containernetworking/plugins go.mod | awk '{print $2}')
script/setup/install-critools
script/setup/install-failpoint-binaries
- name: Install criu
# NOTE: Required actuated enable CONFIG_CHECKPOINT_RESTORE
#
# REF: https://criu.org/Linux_kernel
if: matrix.os != 'actuated-arm64-4cpu-16gb'
run: |
sudo add-apt-repository -y ppa:criu/ppa
sudo apt-get update
sudo apt-get install -y criu
- name: Install containerd
env:
CGO_ENABLED: 1
run: |
make binaries GO_BUILD_FLAGS="-mod=vendor"
sudo -E PATH=$PATH make install
- run: script/setup/install-gotestsum
- run: script/setup/install-teststat
- name: Tests
env:
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root-junit.xml
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-unit-root-gotest.json
run: |
make test
sudo -E PATH=$PATH make root-test
- run: if [ -f *-gotest.json ]; then echo '# Root Test' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
if: always()
- run: script/test/test2annotation.sh *-gotest.json
if: always()
- name: Integration 1
env:
TEST_RUNTIME: ${{ matrix.runtime }}
RUNC_FLAVOR: ${{ matrix.runc }}
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial-gotest.json
run: |
extraflags=""
[ "${RUNC_FLAVOR}" == "crun" ] && {
extraflags="EXTRA_TESTFLAGS=-no-criu";
}
sudo -E PATH=$PATH make integration ${extraflags} TESTFLAGS_RACE=-race
- run: if [ -f *-gotest.json ]; then echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
if: always()
- run: script/test/test2annotation.sh *-gotest.json
if: always()
# Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759
- name: Integration 2
env:
TEST_RUNTIME: ${{ matrix.runtime }}
RUNC_FLAVOR: ${{ matrix.runc }}
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel-gotest.json
run: |
extraflags=""
[ "${RUNC_FLAVOR}" == "crun" ] && {
extraflags="EXTRA_TESTFLAGS=-no-criu";
}
sudo -E PATH=$PATH TESTFLAGS_PARALLEL=1 make integration ${extraflags}
- run: if [ -f *-gotest.json ]; then echo '# Integration 2' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
if: always()
- run: script/test/test2annotation.sh *-gotest.json
if: always()
- name: CRI Integration Test
env:
TEST_RUNTIME: ${{ matrix.runtime }}
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
run: |
CONTAINERD_RUNTIME=$TEST_RUNTIME make cri-integration
- name: cri-tools critest
env:
TEST_RUNTIME: ${{ matrix.runtime }}
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
run: |
sudo -E PATH=$PATH ./script/critest.sh "${{github.workspace}}/report"
# Log the status of this VM to investigate issues like
# https://github.com/containerd/containerd/issues/4969
- name: Host Status
if: always()
run: |
set -x
mount
df
losetup -l
- name: Kernel Message
if: failure()
run: |
sudo lsmod
sudo dmesg -T -f kern
- uses: actions/upload-artifact@v4
if: always()
with:
name: TestResults ${{ matrix.runtime }} ${{matrix.runc}} ${{ matrix.os }} ${{ matrix.enable_cri_sandboxes }}
path: |
*-junit.xml
*-gotest.json
${{github.workspace}}/report/*.xml
${{github.workspace}}/report/*.log
integration-vagrant:
name: Vagrant integration
runs-on: ubuntu-22.04
timeout-minutes: 60
needs: [project, linters, protos, man]
strategy:
fail-fast: false
matrix:
box:
- fedora/39-cloud-base
# We have to keep EL8 to test old glibc, cgroup, kernel, etc.
# The image was changed from rockylinux/8 to almalinux/8,
# as the former one no longer works:
# https://github.com/containerd/containerd/pull/10297
- almalinux/8
- rockylinux/[email protected]
env:
BOX: ${{ matrix.box }}
steps:
- name: Show the host info
run: |
set -x
uname -a
cat /etc/os-release
cat /proc/cpuinfo
free -mt
- uses: actions/checkout@v4
- uses: actions/cache@v4
with:
path: /root/.vagrant.d
key: vagrant-${{ matrix.box }}
- name: Set up vagrant
run: |
# Canonical's Vagrant 2.2.19 dpkg cannot download Fedora 38 image: https://bugs.launchpad.net/vagrant/+bug/2017828
# So we have to install Vagrant >= 2.3.1 from the upstream: https://github.com/opencontainers/runc/blob/v1.1.8/.cirrus.yml#L41-L49
curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
sudo sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list
sudo apt-get update
sudo apt-get install -y libvirt-daemon libvirt-daemon-system vagrant
sudo systemctl enable --now libvirtd
sudo apt-get build-dep -y vagrant ruby-libvirt
sudo apt-get install -y --no-install-recommends libxslt-dev libxml2-dev libvirt-dev ruby-bundler ruby-dev zlib1g-dev
sudo vagrant plugin install vagrant-libvirt
- name: Boot VM
run: sudo BOX=$BOX vagrant up --no-tty
- name: test-integration
run: sudo BOX=$BOX vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-integration
- name: test-cri-integration
run: sudo BOX=$BOX vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri-integration
- name: test-cri
run: sudo BOX=$BOX vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri
tests-cri-in-userns:
name: "CRI-in-UserNS"
runs-on: ubuntu-22.04
timeout-minutes: 40
needs: [project, linters, protos, man]
steps:
- uses: actions/checkout@v4
- name: Set up cgroup v2 delegation
run: |
sudo mkdir -p /etc/systemd/system/[email protected]
cat <<EOF | sudo tee /etc/systemd/system/[email protected]/delegate.conf
[Service]
Delegate=cpu cpuset io memory pids
EOF
sudo systemctl daemon-reload
- name: Build cri-in-userns image
run: podman build --target cri-in-userns -t cri-in-userns -f ./contrib/Dockerfile.test .
- name: Run cri-in-userns image
# Rootless Podman is used for testing CRI-in-UserNS
# (We could use rootless Docker or rootless nerdctl, but we are using Podman here because it is preinstalled)
run: podman run --rm --privileged cri-in-userns
tests-mac-os:
name: MacOS unit tests
runs-on: macos-12
timeout-minutes: 10
needs: [project, linters, protos, man]
env:
GOTEST: gotestsum --
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/install-go
- run: script/setup/install-gotestsum
- run: script/setup/install-teststat
- name: Tests
env:
GOTESTSUM_JUNITFILE: "${{ github.workspace }}/macos-test-junit.xml"
GOTESTSUM_JSONFILE: "${{ github.workspace }}/macos-test-gotest.json"
run: make test
- run: if [ -f *-gotest.json ]; then echo '# Unit Tests' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
if: always()
- run: script/test/test2annotation.sh *-gotest.json
if: always()
- uses: actions/upload-artifact@v4
if: always()
with:
name: TestResults MacOS
path: |
*-junit.xml
*-gotest.json