Skip to content

fix possible wrong merging after ATTACH PART for Collapsing and Repla… #1311

fix possible wrong merging after ATTACH PART for Collapsing and Repla…

fix possible wrong merging after ATTACH PART for Collapsing and Repla… #1311

Workflow file for this run

name: Build
on:
pull_request:
branches:
- "*"
push:
branches:
- master
jobs:
build:
name: Build
runs-on: ubuntu-latest
strategy:
matrix:
golang-version:
- "1.23"
steps:
- name: Checkout project
uses: actions/checkout@v4
- name: Setup golang
id: setup-go
uses: actions/setup-go@v5
with:
go-version: '^${{ matrix.golang-version }}'
- name: Setup musl
id: setup-musl
run: |
sudo apt-get install -y musl-tools musl-dev
- name: Cache golang
id: cache-golang
uses: actions/cache@v4
with:
path: |
~/go/pkg/mod
~/.cache/go-build
key: clickhouse-backup-golang-${{ matrix.golang-version }}-${{ hashFiles('go.mod', '.github/workflows/*.yaml') }}
- name: Install golang dependencies
run: go mod download -x
if: |
steps.cache-golang.outputs.cache-hit != 'true'
- name: Build clickhouse-backup binary
id: make-race
env:
GOROOT: ${{ env.GOROOT_1_23_X64 }}
run: |
make build/linux/amd64/clickhouse-backup build/linux/arm64/clickhouse-backup
make build/linux/amd64/clickhouse-backup-fips build/linux/arm64/clickhouse-backup-fips
make build-race build-race-fips config test
- name: Report unittest coverage
uses: coverallsapp/github-action@v2
with:
file: _coverage_/coverage.out
parallel: true
format: golang
flag-name: unittest-${{ matrix.clickhouse }}
# be careful with encrypt with old OpenSSL - https://habr.com/ru/post/535140/
# openssl enc -base64 -aes-256-cbc -e -in test/integration/credentials.json -out test/integration/credentials.json.enc -md md5 -k ${VAULT_PASSWORD}
- name: Decrypting credentials for Google Cloud Storage
id: secrets
env:
VAULT_PASSWORD: ${{ secrets.VAULT_PASSWORD }}
run: |
if [[ "" != "${VAULT_PASSWORD}" ]]; then
openssl version
openssl enc -base64 -aes-256-cbc -d -in test/integration/credentials.json.enc -out test/integration/credentials.json -md md5 -k ${VAULT_PASSWORD}
fi
echo "GCS_TESTS=$(if [ -z "${{ secrets.VAULT_PASSWORD }}" ]; then echo "false"; else echo "true"; fi)" >> $GITHUB_OUTPUT
- uses: actions/upload-artifact@v4
with:
name: build-gcp-credentials
path: |
test/integration/credentials.json
if-no-files-found: error
retention-days: 1
if: |
steps.secrets.outputs.GCS_TESTS == 'true'
- uses: actions/upload-artifact@v4
with:
name: build-artifacts
path: |
build/linux/amd64/clickhouse-backup
build/linux/arm64/clickhouse-backup
build/linux/amd64/clickhouse-backup-fips
build/linux/arm64/clickhouse-backup-fips
if-no-files-found: error
retention-days: 1
- uses: actions/upload-artifact@v4
with:
name: build-test-artifacts
path: |
clickhouse-backup/clickhouse-backup-race
clickhouse-backup/clickhouse-backup-race-fips
if-no-files-found: error
retention-days: 1
outputs:
GCS_TESTS: ${{ steps.secrets.outputs.GCS_TESTS }}
testflows:
needs: build
name: Testflows
runs-on: ubuntu-latest
strategy:
matrix:
golang-version:
- "1.23"
clickhouse:
- '22.3'
- '22.8'
- '23.3'
- '23.8'
- '24.3'
- '24.8'
steps:
- name: Checkout project
uses: actions/checkout@v4
- name: Setup golang
id: setup-go
uses: actions/setup-go@v5
with:
go-version: '^${{ matrix.golang-version }}'
- uses: actions/download-artifact@v4
with:
name: build-test-artifacts
path: ./clickhouse-backup/
- name: Install python venv
run: |
set -x
(dpkg -l | grep venv) || (apt-get update && apt-get install -y python3-venv)
python3 -m venv ~/venv/qa
- name: Cache python
uses: actions/cache@v4
id: cache-python
with:
path: ~/venv/qa
key: clickhouse-backup-python-${{ hashFiles('test/testflows/requirements.txt','.github/workflows/*.yaml') }}
- name: Install python dependencies
run: |
set -x
~/venv/qa/bin/pip3 install -U -r ./test/testflows/requirements.txt
if: |
steps.cache-python.outputs.cache-hit != 'true'
- name: Running TestFlows tests
env:
CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
QA_AWS_ACCESS_KEY: ${{ secrets.QA_AWS_ACCESS_KEY }}
QA_AWS_ENDPOINT: ${{ secrets.QA_AWS_ENDPOINT }}
QA_AWS_SECRET_KEY: ${{ secrets.QA_AWS_SECRET_KEY }}
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_BUCKET: ${{ secrets.QA_AWS_BUCKET }}
QA_GCS_CRED_JSON: ${{ secrets.QA_GCS_CRED_JSON }}
# don't change it to avoid not working CI/CD
RUN_TESTS: "*"
run: |
set -xe
export CLICKHOUSE_TESTS_DIR=$(pwd)/test/testflows/clickhouse_backup
docker compose -f ${CLICKHOUSE_TESTS_DIR}/docker-compose/docker-compose.yml pull
chmod +x $(pwd)/clickhouse-backup/clickhouse-backup*
source ~/venv/qa/bin/activate
~/venv/qa/bin/python3 ./test/testflows/clickhouse_backup/regression.py --debug --only="${RUN_TESTS:-*}" --log ./test/testflows/raw.log
tfs --debug --no-colors transform compact ./test/testflows/raw.log ./test/testflows/compact.log
tfs --debug --no-colors transform nice ./test/testflows/raw.log ./test/testflows/nice.log.txt
tfs --debug --no-colors transform short ./test/testflows/raw.log ./test/testflows/short.log.txt
tfs --debug --no-colors report results -a "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/" ./test/testflows/raw.log - --confidential --copyright "Altinity LTD" --logo ./test/testflows/altinity.png | ~/venv/qa/bin/tfs --debug --no-colors document convert > ./test/testflows/report.html
sudo chmod -Rv +rx test/testflows/clickhouse_backup/_instances
- name: Format testflows coverage
env:
GOROOT: ${{ env.GOROOT_1_23_X64 }}
run: |
sudo chmod -Rv a+rw test/testflows/_coverage_/
ls -la test/testflows/_coverage_
go env
go tool covdata textfmt -i test/testflows/_coverage_/ -o test/testflows/_coverage_/coverage.out
- name: Report testflows coverage
uses: coverallsapp/github-action@v2
with:
base-path: ./
file: test/testflows/_coverage_/coverage.out
parallel: true
format: golang
flag-name: testflows-${{ matrix.clickhouse }}
# todo possible failures https://github.com/actions/upload-artifact/issues/270
- name: Upload testflows logs
uses: actions/upload-artifact@v4
with:
name: testflows-logs-and-reports-${{ matrix.clickhouse }}-${{ github.run_id }}
path: |
test/testflows/*.log
test/testflows/*.log.txt
test/testflows/clickhouse_backup/_instances/**/*.log
test/testflows/*.html
retention-days: 7
test:
needs: build
name: Test
runs-on: ubuntu-latest
strategy:
matrix:
golang-version:
- "1.23"
clickhouse:
- '1.1.54394'
- '19.17'
- '20.3'
- '20.8'
- '21.3'
- '21.8'
- '22.3'
- '22.8'
- '23.3'
- '23.8'
- '24.3'
- '24.8'
steps:
- name: Checkout project
uses: actions/checkout@v4
- name: Setup golang
id: setup-go
uses: actions/setup-go@v5
with:
go-version: '^${{ matrix.golang-version }}'
# - name: Cache golang
# id: cache-golang
# uses: actions/cache@v4
# with:
# path: |
# ~/go/pkg/mod
# ~/.cache/go-build
# key: clickhouse-backup-golang-${{ matrix.golang-version }}-${{ hashFiles('go.mod', '.github/workflows/*.yaml') }}
- uses: actions/download-artifact@v4
with:
name: build-test-artifacts
path: ./clickhouse-backup/
- uses: actions/download-artifact@v4
with:
name: build-gcp-credentials
path: ./test/integration/
if: |
needs.build.outputs.GCS_TESTS == 'true'
- name: Running integration tests
env:
RUN_PARALLEL: 3
GOROOT: ${{ env.GOROOT_1_23_X64 }}
CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
# options for advanced debug CI/CD
# RUN_TESTS: "TestLongListRemote"
# LOG_LEVEL: "debug"
# TEST_LOG_LEVEL: "debug"
# GCS_DEBUG: "true"
# SFTP_DEBUG: "true"
# AZBLOB_DEBUG: "true"
# FTP_DEBUG: "true"
# S3_DEBUG: "true"
CGO_ENABLED: 0
GCS_TESTS: ${{ needs.build.outputs.GCS_TESTS }}
RUN_ADVANCED_TESTS: 1
AZURE_TESTS: 1
# need for FIPS
QA_AWS_ACCESS_KEY: ${{ secrets.QA_AWS_ACCESS_KEY }}
QA_AWS_SECRET_KEY: ${{ secrets.QA_AWS_SECRET_KEY }}
QA_AWS_BUCKET: ${{ secrets.QA_AWS_BUCKET }}
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
# need for GCP over S3
QA_GCS_OVER_S3_ACCESS_KEY: ${{ secrets.QA_GCS_OVER_S3_ACCESS_KEY }}
QA_GCS_OVER_S3_SECRET_KEY: ${{ secrets.QA_GCS_OVER_S3_SECRET_KEY }}
QA_GCS_OVER_S3_BUCKET: ${{ secrets.QA_GCS_OVER_S3_BUCKET }}
run: |
set -xe
echo "CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION}"
echo "GCS_TESTS=${GCS_TESTS}"
chmod +x $(pwd)/clickhouse-backup/clickhouse-backup*
if [[ "${CLICKHOUSE_VERSION}" =~ 2[2-9]+ ]]; then
export CLICKHOUSE_IMAGE=clickhouse/clickhouse-server
else
export CLICKHOUSE_IMAGE=yandex/clickhouse-server
fi
if [[ "${CLICKHOUSE_VERSION}" == 2* ]]; then
export COMPOSE_FILE=docker-compose_advanced.yml
else
export COMPOSE_FILE=docker-compose.yml
fi
export CUR_DIR="$(pwd)/test/integration"
export CLICKHOUSE_BACKUP_BIN="$(pwd)/clickhouse-backup/clickhouse-backup-race"
docker compose -f "${CUR_DIR}/${COMPOSE_FILE}" --progress=quiet pull
pids=()
project_ids=()
for ((i = 0; i < RUN_PARALLEL; i++)); do
docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name project${i} --progress plain up -d &
pids+=($!)
project_ids+=("project${i}")
done
for index in "${!pids[@]}"; do
pid=${pids[index]}
project_id=${project_ids[index]}
if wait "$pid"; then
echo "$pid docker compose up successful"
else
echo "=== docker ${project_id} state ==="
docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress plain ps -a
echo "=== docker ${project_id} logs ==="
docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress plain logs
echo "$pid the docker compose up failed."
exit 1 # Exit with an error code if any command fails
fi
done
set +e
go test -parallel ${RUN_PARALLEL} -timeout 90m -failfast -tags=integration -run "${RUN_TESTS:-.+}" -v test/integration/integration_test.go
TEST_FAILED=$?
set -e
if [[ "0" != "${TEST_FAILED}" ]]; then
for project_id in "${project_ids[@]}"; do
echo "=== docker ${project_id} state ==="
docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress plain ps -a
echo "=== docker ${project_id} logs ==="
docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress plain logs
done
exit 1
fi
- name: Format integration coverage
env:
GOROOT: ${{ env.GOROOT_1_23_X64 }}
run: |
sudo chmod -Rv a+rw test/integration/_coverage_/
ls -la test/integration/_coverage_
go tool covdata textfmt -i test/integration/_coverage_/ -o test/integration/_coverage_/coverage.out
- name: Report integration coverage
uses: coverallsapp/github-action@v2
with:
base-path: ./
file: test/integration/_coverage_/coverage.out
parallel: true
format: golang
flag-name: integration-${{ matrix.clickhouse }}
coverage:
needs:
- test
- testflows
name: coverage
runs-on: ubuntu-latest
steps:
- name: Coveralls Finished
uses: coverallsapp/github-action@v2
with:
base-path: ./
parallel-finished: true
docker:
needs:
- test
- testflows
name: Docker
runs-on: ubuntu-latest
steps:
- name: Checkout project
uses: actions/checkout@v4
- uses: actions/download-artifact@v4
with:
name: build-artifacts
path: ./build/linux/
- name: Extract DOCKER_TAG version
id: docker_tag
run: |
DOCKER_TAG=${GITHUB_REF##*/}
export DOCKER_TAG=${DOCKER_TAG##*\\}
echo "docker_tag=${DOCKER_TAG:-dev}" >> $GITHUB_OUTPUT
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Building docker image
env:
DOCKER_REPO: ${{ secrets.DOCKER_REPO }}
DOCKER_IMAGE: ${{ secrets.DOCKER_IMAGE }}
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
DOCKER_USER: ${{ secrets.DOCKER_USER }}
DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
DOCKER_TAG: ${{ steps.docker_tag.outputs.docker_tag }}
run: |
if [[ "${DOCKER_TOKEN}" != "" ]]; then
export DOCKER_REGISTRY=${DOCKER_REGISTRY:-docker.io}
echo ${DOCKER_TOKEN} | docker login -u ${DOCKER_USER} --password-stdin ${DOCKER_REGISTRY}
export VERSION=$(git describe --always --tags --abbrev=0 2>/dev/null | sed 's/^v//;s/\([^-]*-g\)/c\1/;s/-/./g')
docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG} --build-arg VERSION=$VERSION --target=image_short --pull --push .
docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG}-fips --build-arg VERSION=$VERSION --target=image_fips --pull --push .
docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}-full:${DOCKER_TAG} --build-arg VERSION=$VERSION --target=image_full --pull --push .
fi
cleanup:
name: Cleanup
needs:
- docker
- coverage
runs-on: ubuntu-latest
if: always()
steps:
- name: delete build-artifacts
uses: geekyeggo/delete-artifact@v5
with:
name: build-artifacts
failOnError: false
- name: delete build-gcp-credentials
uses: geekyeggo/delete-artifact@v5
with:
name: build-gcp-credentials
failOnError: false