ci: updated workflow #465
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Docker Image CI: crawlab" | |
| on: | |
| push: | |
| branches: [ develop, main ] | |
| #pull_request: | |
| # branches: [ main ] | |
| release: | |
| types: [ published ] | |
| workflow_dispatch: | |
| repository_dispatch: | |
| types: [ docker-crawlab ] | |
| env: | |
| IMAGE_PATH_CRAWLAB_BACKEND: backend | |
| IMAGE_PATH_CRAWLAB_FRONTEND: frontend | |
| IMAGE_NAME_CRAWLAB: crawlabteam/crawlab | |
| IMAGE_NAME_CRAWLAB_BACKEND: crawlabteam/crawlab-backend | |
| IMAGE_NAME_CRAWLAB_FRONTEND: crawlabteam/crawlab-frontend | |
| IMAGE_NAME_CRAWLAB_TENCENT: ccr.ccs.tencentyun.com/crawlabteam/crawlab | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| backend_changed: ${{ steps.check_changed_files.outputs.backend_changed }} | |
| frontend_changed: ${{ steps.check_changed_files.outputs.frontend_changed }} | |
| dockerfile_changed: ${{ steps.check_changed_files.outputs.dockerfile_changed }} | |
| version: ${{ steps.version.outputs.version }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Get changed files | |
| id: changed_files | |
| uses: tj-actions/changed-files@v45 | |
| with: | |
| files_yaml: | | |
| backend: | |
| - 'backend/**' | |
| - 'core/**' | |
| - 'db/**' | |
| - 'fs/**' | |
| - 'grpc/**' | |
| - 'vcs/**' | |
| - 'trace/**' | |
| frontend: | |
| - '${{ env.IMAGE_PATH_CRAWLAB_FRONTEND }}/**' | |
| dockerfile: | |
| - 'Dockerfile' | |
| - id: check_changed_files | |
| name: Check changed files | |
| run: | | |
| # set outputs | |
| if [[ "${{ github.ref }}" == "refs/tags/"* ]]; then | |
| echo "backend_changed=true" >> $GITHUB_OUTPUT | |
| echo "frontend_changed=true" >> $GITHUB_OUTPUT | |
| echo "dockerfile_changed=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "backend_changed=${{ steps.changed_files.outputs.backend_any_changed }}" >> $GITHUB_OUTPUT | |
| echo "frontend_changed=${{ steps.changed_files.outputs.frontend_any_changed }}" >> $GITHUB_OUTPUT | |
| echo "dockerfile_changed=${{ steps.changed_files.outputs.dockerfile_any_changed }}" >> $GITHUB_OUTPUT | |
| fi | |
| # Display change status for each component | |
| echo "Backend changed: ${{ steps.changed_files.outputs.backend_any_changed }}" | |
| echo "Frontend changed: ${{ steps.changed_files.outputs.frontend_any_changed }}" | |
| echo "Dockerfile changed: ${{ steps.changed_files.outputs.dockerfile_any_changed }}" | |
| - id: version | |
| name: Get version | |
| run: | | |
| # Strip git ref prefix from version | |
| VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') | |
| # Strip "v" prefix from tag name | |
| [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') | |
| # Use Docker `latest` tag convention | |
| [ "$VERSION" == "main" ] && VERSION=latest | |
| echo "version=$VERSION" >> $GITHUB_OUTPUT | |
| test_backend: | |
| needs: [ setup ] | |
| if: needs.setup.outputs.backend_changed == 'true' | |
| runs-on: ubuntu-latest | |
| services: | |
| mongo: | |
| image: mongo:5 | |
| ports: | |
| - 27017:27017 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: '1.22' | |
| - name: Run tests | |
| run: | | |
| cd core | |
| # Find all directories containing *_test.go files | |
| test_dirs=$(find . -name "*_test.go" -exec dirname {} \; | sort -u) | |
| # Run go test on each directory | |
| for dir in $test_dirs | |
| do | |
| echo "Running tests in $dir" | |
| go test ./$dir | |
| done | |
| build_backend: | |
| needs: [ setup, test_backend ] | |
| if: needs.setup.outputs.backend_changed == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Build image | |
| run: | | |
| cd $IMAGE_PATH_CRAWLAB_BACKEND | |
| docker build . --file Dockerfile --tag image | |
| - name: Log into registry | |
| run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin | |
| - name: Push image | |
| run: | | |
| IMAGE_VERSION=${{needs.setup.outputs.version}} | |
| IMAGE_NAME=$IMAGE_NAME_CRAWLAB_BACKEND | |
| docker tag image $IMAGE_NAME:$IMAGE_VERSION | |
| docker push $IMAGE_NAME:$IMAGE_VERSION | |
| build_frontend: | |
| needs: [ setup ] | |
| if: needs.setup.outputs.frontend_changed == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Build image | |
| run: | | |
| cd $IMAGE_PATH_CRAWLAB_FRONTEND | |
| docker build . --file Dockerfile --tag image | |
| - name: Log into registry | |
| run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin | |
| - name: Push image | |
| run: | | |
| IMAGE_VERSION=${{needs.setup.outputs.version}} | |
| IMAGE_NAME=$IMAGE_NAME_CRAWLAB_FRONTEND | |
| docker tag image $IMAGE_NAME:$IMAGE_VERSION | |
| docker push $IMAGE_NAME:$IMAGE_VERSION | |
| build_crawlab: | |
| needs: [setup, build_backend, build_frontend] | |
| if: | | |
| always() && | |
| ((needs.setup.outputs.backend_changed == 'true' && needs.build_backend.result == 'success') || | |
| (needs.setup.outputs.frontend_changed == 'true' && needs.build_frontend.result == 'success') || | |
| (needs.setup.outputs.dockerfile_changed == 'true')) | |
| runs-on: ubuntu-latest | |
| services: | |
| mongo: | |
| image: mongo:5 | |
| ports: | |
| - 27017:27017 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Update Dockerfile | |
| run: | | |
| IMAGE_VERSION=${{needs.setup.outputs.version}} | |
| if [[ $IMAGE_VERSION != "latest" ]]; then | |
| for n in crawlab-backend crawlab-frontend; do | |
| IMAGE_NAME=$n | |
| sed -i "s/${IMAGE_NAME}:latest/${IMAGE_NAME}:${IMAGE_VERSION}/" Dockerfile | |
| done | |
| fi | |
| - name: Build image | |
| run: docker build . --file Dockerfile --tag image | |
| - name: Test image | |
| run: | | |
| docker run --rm -d --name crawlab_master \ | |
| -e CRAWLAB_NODE_MASTER=true \ | |
| -e CRAWLAB_DEMO=true \ | |
| -e CRAWLAB_MONGO_HOST=localhost \ | |
| -e CRAWLAB_MONGO_PORT=27017 \ | |
| -p 8080:8080 \ | |
| --network host \ | |
| image | |
| docker exec crawlab_master env | |
| docker logs -f crawlab_master & | |
| sleep 10 | |
| docker ps | |
| cmd='curl http://localhost:8080/api/system-info -s' | |
| echo "cmd: ${cmd}" | |
| res=`${cmd}` | |
| echo "res: ${res}" | |
| docker stop crawlab_master | |
| if [[ $res =~ "success" ]]; then | |
| echo "Test passed successfully" | |
| else | |
| echo "Test failed" | |
| exit 1 | |
| fi | |
| push_images: | |
| if: ${{ always() && needs.build_crawlab.result == 'success' }} | |
| needs: [setup, build_crawlab] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| registry: [dockerhub, tencent] | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ (matrix.registry == 'dockerhub' && '') || (matrix.registry == 'tencent' && 'ccr.ccs.tencentyun.com') }} | |
| username: ${{ (matrix.registry == 'dockerhub' && secrets.DOCKER_USERNAME) || (matrix.registry == 'tencent' && secrets.DOCKER_TENCENT_USERNAME) }} | |
| password: ${{ (matrix.registry == 'dockerhub' && secrets.DOCKER_PASSWORD) || (matrix.registry == 'tencent' && secrets.DOCKER_TENCENT_PASSWORD) }} | |
| - name: Push to DockerHub | |
| uses: docker/build-push-action@v6 | |
| with: | |
| push: true | |
| tags: ${{ (matrix.registry == 'dockerhub' && env.IMAGE_NAME_CRAWLAB) || (matrix.registry == 'tencent' && env.IMAGE_NAME_CRAWLAB_TENCENT) }}:${{ needs.setup.outputs.version }} |