feat:【流水线插件】支持历史大版本下的小版本更新 TencentBlueKing#1841

.github/workflows/backend.yml

@@ -20,16 +20,23 @@ jobs:
   build:
     # The type of runner that the job will run on
     runs-on: ubuntu-latest
+    services:
+      mysql:
+        image: mysql:8.0
+        env:
+          MYSQL_ROOT_PASSWORD: root
+        ports:
+          - 3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=10
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
     - uses: actions/checkout@v2
     - name: init mysql
       run: |
-        sudo /etc/init.d/mysql start
         export MYSQL_PWD=root
-        for i in *.sql;do echo $i;mysql -h localhost -uroot < $i;done
+        for i in *.sql;do echo $i;mysql -h 127.0.0.1 --port ${{ job.services.mysql.ports['3306'] }} -uroot < $i;done
       working-directory: support-files/sql
     - name: Set up JDK 1.8
       uses: actions/setup-java@v1
@@ -44,4 +51,7 @@ jobs:
           ${{ runner.os }}-gradle-
     - name: Gradle Build
       working-directory: src/backend/ci
-      run: ./gradlew test clean :boot-assembly:classes -DmavenRepoUrl="http://maven.aliyun.com/nexus/content/groups/public/"
+      run: |
+        ./gradlew test clean :boot-assembly:classes \
+        -DmavenRepoUrl="http://maven.aliyun.com/nexus/content/groups/public/" \
+        -DmysqlURL=127.0.0.1:${{ job.services.mysql.ports['3306'] }} -DmysqlUser=root -DmysqlPasswd=root

.github/workflows/backend_booster.yml

@@ -0,0 +1,40 @@
+# This is a basic workflow to help you get started with Actions
+
+name: Backend Booster Check
+
+# Controls when the action will run. Triggers the workflow on push or pull request
+# events but only for the master branch
+on:
+  push:
+    branches: [ master ]
+    paths:
+    - 'src/backend/booster/**'
+  pull_request:
+    branches: [ master, release-* ]
+    paths:
+    - 'src/backend/booster/**'
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+    - uses: actions/checkout@v2
+    - uses: actions/setup-go@v2
+      with:
+        go-version: 1.13.15
+    - name: GO Mod
+      working-directory: src/backend/booster
+      run: |
+        go mod download
+        go get -u github.com/gobuffalo/packr/v2/packr2
+    - name: GO Build 
+      working-directory: src/backend/booster
+      run: make -j buildbooster dist
+      env:
+        GITTAG: ${{ github.ref }}

.github/workflows/publishMavenJar4CI.yml

@@ -13,13 +13,21 @@ jobs:
 
     runs-on: ubuntu-latest
 
+    services:
+      mysql:
+        image: mysql:8.0
+        env:
+          MYSQL_ROOT_PASSWORD: root
+        ports:
+          - 3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=10
+
     steps:
       - uses: actions/checkout@v2
       - name: init mysql
         run: |
-          sudo /etc/init.d/mysql start
           export MYSQL_PWD=root
-          for i in *.sql;do echo $i;mysql -h localhost -uroot < $i;done
+          for i in *.sql;do echo $i;mysql -h 127.0.0.1 --port ${{ job.services.mysql.ports['3306'] }} -uroot < $i;done
         working-directory: support-files/sql
       - name: Set up JDK 1.8
         uses: actions/setup-java@v1
@@ -61,7 +69,8 @@ jobs:
           -Psigning.keyId=${{ secrets.SIGNING_KEY_ID }} \
           -Psigning.password='${{ secrets.SIGNING_PASSWORD }}' \
           -Psigning.secretKeyRingFile=${{ github.workspace }}/secretKeyRingFile.gpg \
-          -DmavenRepoUrl="http://maven.aliyun.com/nexus/content/groups/public/"
+          -DmavenRepoUrl="http://maven.aliyun.com/nexus/content/groups/public/" \
+          -DmysqlURL=127.0.0.1:${{ job.services.mysql.ports['3306'] }} -DmysqlUser=root -DmysqlPasswd=root
       - name: Clean Up
         if: ${{ always() }}
         working-directory:

.github/workflows/release.yml

@@ -59,13 +59,20 @@ jobs:
   backend:
     name: Build backend and release
     runs-on: ubuntu-latest
+    services:
+      mysql:
+        image: mysql:8.0
+        env:
+          MYSQL_ROOT_PASSWORD: root
+        ports:
+          - 3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=10
     steps:
       - uses: actions/checkout@v2
       - name: init mysql
         run: |
-          sudo /etc/init.d/mysql start
           export MYSQL_PWD=root
-          for i in *.sql;do echo $i;mysql -h localhost -uroot < $i;done
+          for i in *.sql;do echo $i;mysql -h 127.0.0.1 --port ${{ job.services.mysql.ports['3306'] }} -uroot < $i;done
         working-directory: support-files/sql
       - name: Set up JDK 1.8
         uses: actions/setup-java@v1
@@ -80,7 +87,10 @@ jobs:
             ${{ runner.os }}-gradle-
       - name: Gradle Build Backend Service
         working-directory: src/backend/ci
-        run: ./gradlew clean test build :core:worker:worker-agent:shadowJar -DmavenRepoUrl="https://maven.aliyun.com/nexus/content/groups/public/"
+        run: |
+          ./gradlew clean test build :core:worker:worker-agent:shadowJar \
+          -DmavenRepoUrl="https://maven.aliyun.com/nexus/content/groups/public/" \
+          -DmysqlURL=127.0.0.1:${{ job.services.mysql.ports['3306'] }} -DmysqlUser=root -DmysqlPasswd=root
 
       - name: Gradle Build Turbo Service
         working-directory: src/backend/turbo

.github/workflows/sql.yml

@@ -17,13 +17,21 @@ jobs:
     # The type of runner that the job will run on
     runs-on: ubuntu-latest
 
+    services:
+      mysql:
+        image: mysql:8.0
+        env:
+          MYSQL_ROOT_PASSWORD: root
+        ports:
+          - 3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=10
+
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
     - uses: actions/checkout@v2
     - name: init mysql
       run: |
-        sudo /etc/init.d/mysql start
         export MYSQL_PWD=root
-        for i in *.sql;do echo $i;mysql -h localhost -uroot < $i;done
+        for i in *.sql;do echo $i;mysql -h 127.0.0.1 --port ${{ job.services.mysql.ports['3306'] }} -uroot < $i;done
       working-directory: support-files/sql

docs/install/gateway.en.md

@@ -17,26 +17,26 @@ Here the CentOS 7.x environment is used for illustration purpose.
 Before installing and deploying the gateway, upload relevant installation and deployment packages to the corresponding servers. Below are files to upload.
 |   File Name   |   File Description     |
 | ------------ | ---------------- |
-|   [openresty-openssl-1.1.0h-3.el7.centos.x86_64.rpm](https://openresty.org/package/centos/7/x86_64/openresty-openssl-1.1.0h-3.el7.centos.x86_64.rpm)   |  OpenSSL package OpenResty depends on    |
-|   [openresty-pcre-8.42-1.el7.centos.x86_64.rpm](https://openresty.org/package/centos/7/x86_64/openresty-pcre-8.42-1.el7.centos.x86_64.rpm)   | PCRE package OpenResty depends on |
+|   [openresty-openssl111-1.1.1g-3.el7.centos.x86_64.rpm](https://openresty.org/package/centos/7/x86_64/openresty-openssl111-1.1.1g-3.el7.centos.x86_64.rpm)   |  OpenSSL package OpenResty depends on    |
+|   [openresty-pcre-8.44-1.el7.centos.x86_64.rpm](https://openresty.org/package/centos/7/x86_64/openresty-pcre-8.44-1.el7.centos.x86_64.rpm)   | PCRE package OpenResty depends on |
 |   [openresty-zlib-1.2.11-3.el7.centos.x86_64.rpm](https://openresty.org/package/centos/7/x86_64/openresty-zlib-1.2.11-3.el7.centos.x86_64.rpm)   |  zlib package OpenResty depends on |
-|   [openresty-1.13.6.2-1.el7.centos.x86_64.rpm](https://openresty.org/package/centos/7/x86_64/openresty-1.13.6.2-1.el7.centos.x86_64.rpm)   |  OpenResty installation package |
+|   [openresty-1.17.8.2-1.el7.centos.x86_64.rpm](https://openresty.org/package/centos/7/x86_64/openresty-1.17.8.2-1.el7.centos.x86_64.rpm)   |  OpenResty installation package |
 
 - Install OpenResty
 
 Since the gateway deployment depends on Lua scripts for authentication and forwarding, you need to install OpenResty 1.13.6.2. Attached are RPM installation packages. Below are installation commands.
 
 ```shell
 # Modify the permission of installation packages
-chmod 644 openresty-1.13.6.2-1.el7.centos.x86_64.rpm
-chmod 644 openresty-openssl-1.1.0h-3.el7.centos.x86_64.rpm
-chmod 644 openresty-pcre-8.42-1.el7.centos.x86_64.rpm
+chmod 644 openresty-1.17.8.2-1.el7.centos.x86_64.rpm
+chmod 644 openresty-openssl111-1.1.1g-3.el7.centos.x86_64.rpm
+chmod 644 openresty-pcre-8.44-1.el7.centos.x86_64.rpm
 chmod 644 openresty-zlib-1.2.11-3.el7.centos.x86_64.rpm
 # Start installation
-rpm -ivh openresty-pcre-8.42-1.el7.centos.x86_64.rpm
+rpm -ivh openresty-pcre-8.44-1.el7.centos.x86_64.rpm
 rpm -ivh openresty-zlib-1.2.11-3.el7.centos.x86_64.rpm
-rpm -ivh openresty-openssl-1.1.0h-3.el7.centos.x86_64.rpm  --replacefiles
-rpm -ivh openresty-1.13.6.2-1.el7.centos.x86_64.rpm
+rpm -ivh openresty-openssl111-1.1.1g-3.el7.centos.x86_64.rpm  --replacefiles
+rpm -ivh openresty-1.17.8.2-1.el7.centos.x86_64.rpm
 # Check installation status
 cd /usr/local/openresty/nginx && ./sbin/nginx -v
 ```

docs/install/gateway.md

@@ -19,26 +19,26 @@
 
 |   文件名称   |   文件说明     |
 | ------------ | ---------------- |
-|   [openresty-openssl-1.1.0h-3.el7.centos.x86_64.rpm](https://openresty.org/package/centos/7/x86_64/openresty-openssl-1.1.0h-3.el7.centos.x86_64.rpm)   |  openresty依赖的openssl包    |
-|   [openresty-pcre-8.42-1.el7.centos.x86_64.rpm](https://openresty.org/package/centos/7/x86_64/openresty-pcre-8.42-1.el7.centos.x86_64.rpm)   |  openresty依赖的pcre包 |
+|   [openresty-openssl111-1.1.1g-3.el7.centos.x86_64.rpm](https://openresty.org/package/centos/7/x86_64/openresty-openssl111-1.1.1g-3.el7.centos.x86_64.rpm)   |  openresty依赖的openssl包    |
+|   [openresty-pcre-8.44-1.el7.centos.x86_64.rpm](https://openresty.org/package/centos/7/x86_64/openresty-pcre-8.44-1.el7.centos.x86_64.rpm)   |  openresty依赖的pcre包 |
 |   [openresty-zlib-1.2.11-3.el7.centos.x86_64.rpm](https://openresty.org/package/centos/7/x86_64/openresty-zlib-1.2.11-3.el7.centos.x86_64.rpm)   |  openresty依赖的zlib包 |
-|   [openresty-1.13.6.2-1.el7.centos.x86_64.rpm](https://openresty.org/package/centos/7/x86_64/openresty-1.13.6.2-1.el7.centos.x86_64.rpm)   |  openresty安装包 |
+|   [openresty-1.17.8.2-1.el7.centos.x86_64.rpm](https://openresty.org/package/centos/7/x86_64/openresty-1.17.8.2-1.el7.centos.x86_64.rpm)   |  openresty安装包 |
 
 - 安装openresty
 
 网关部署依赖于lua脚本做鉴权和转发，所以这里需要安装openresty，使用的版本：1.13.6.2。rpm安装包请查看附件，下面是安装命令。
 
 ```shell
 # 修改安装包权限
-chmod 644 openresty-1.13.6.2-1.el7.centos.x86_64.rpm
-chmod 644 openresty-openssl-1.1.0h-3.el7.centos.x86_64.rpm
-chmod 644 openresty-pcre-8.42-1.el7.centos.x86_64.rpm
+chmod 644 openresty-1.17.8.2-1.el7.centos.x86_64.rpm
+chmod 644 openresty-openssl111-1.1.1g-3.el7.centos.x86_64.rpm
+chmod 644 openresty-pcre-8.44-1.el7.centos.x86_64.rpm
 chmod 644 openresty-zlib-1.2.11-3.el7.centos.x86_64.rpm
 # 开始安装
-rpm -ivh openresty-pcre-8.42-1.el7.centos.x86_64.rpm
+rpm -ivh openresty-pcre-8.44-1.el7.centos.x86_64.rpm
 rpm -ivh openresty-zlib-1.2.11-3.el7.centos.x86_64.rpm
-rpm -ivh openresty-openssl-1.1.0h-3.el7.centos.x86_64.rpm  --replacefiles
-rpm -ivh openresty-1.13.6.2-1.el7.centos.x86_64.rpm
+rpm -ivh openresty-openssl111-1.1.1g-3.el7.centos.x86_64.rpm  --replacefiles
+rpm -ivh openresty-1.17.8.2-1.el7.centos.x86_64.rpm
 # 检测安装情况
 cd /usr/local/openresty/nginx && ./sbin/nginx -v
 ```

scripts/bkenv.properties

@@ -211,6 +211,8 @@ BK_CI_FQDN_CERT=
 BK_CI_NOTIFY_WEWORK_API_URL=https://qyapi.weixin.qq.com
 # BK_CI_NOTIFY_WEWORK_SEND_CHANNEL 发送rtx的通道: weworkAgent 企业微信应用,blueking 蓝鲸消息
 BK_CI_NOTIFY_WEWORK_SEND_CHANNEL=weworkAgent
+# BK_CI_ARTIFACTORY_REALM 默认值为local,按需修改,制品库存储方式,可选值local, bkrepo
+BK_CI_ARTIFACTORY_REALM=local
 ##########
 # 4-微服务依赖
 ##########

src/agent/build_windows.bat

@@ -20,3 +20,23 @@ go build -o %WORK_DIR%\bin\upgrader.exe %WORK_DIR%\src\cmd\upgrader
 
 go build -o %WORK_DIR%\bin\installer.exe %WORK_DIR%\src\cmd\installer
 
+
+if not exist %WORK_DIR%\bin\devopsDaemon.exe (
+    echo "can not find %WORK_DIR%\bin\devopsDaemon.exe"
+    exit 1
+)
+
+if not exist %WORK_DIR%\bin\devopsAgent.exe (
+    echo "can not find %WORK_DIR%\bin\devopsAgent.exe"
+    exit 1
+)
+
+if not exist %WORK_DIR%\bin\upgrader.exe (
+    echo "can not find %WORK_DIR%\bin\upgrader.exe"
+    exit 1
+)
+
+if not exist %WORK_DIR%\bin\installer.exe (
+    echo "can not find %WORK_DIR%\bin\installer.exe"
+    exit 1
+)

src/agent/src/cmd/daemon/main.go

@@ -51,6 +51,11 @@ const (
 )
 
 func main() {
+	if len(os.Args) == 2 && os.Args[1] == "version" {
+		fmt.Println(config.AgentVersion)
+		systemutil.ExitProcess(0)
+	}
+
 	runtime.GOMAXPROCS(4)
 
 	workDir := systemutil.GetExecutableDir()

src/agent/src/cmd/daemon/main_win.go

@@ -29,12 +29,14 @@
 package main
 
 import (
+	"fmt"
 	"encoding/json"
 	"os"
 	"os/exec"
 	"runtime"
 	"time"
 
+	"github.com/Tencent/bk-ci/src/agent/src/pkg/config"
 	"github.com/Tencent/bk-ci/src/agent/src/pkg/util/fileutil"
 	"github.com/Tencent/bk-ci/src/agent/src/pkg/util/systemutil"
 	"github.com/astaxie/beego/logs"
@@ -44,6 +46,10 @@ import (
 const daemonProcess = "daemon"
 
 func main() {
+	if len(os.Args) == 2 && os.Args[1] == "version" {
+		fmt.Println(config.AgentVersion)
+		systemutil.ExitProcess(0)
+	}
 	runtime.GOMAXPROCS(4)
 
 	workDir := systemutil.GetExecutableDir()

src/agent/src/pkg/config/version.go

@@ -26,4 +26,4 @@
 
 package config
 
-const AgentVersion = "v1.7.0"
+const AgentVersion = "v1.7.20"

src/agent/src/pkg/upgrade/operation.go

@@ -51,14 +51,14 @@ func UninstallAgent() {
 }
 
 func runUpgrader(action string) error {
-	logs.Info("start upgrader process")
+	logs.Info("[agentUpgrade]|start upgrader process")
 
 	scripPath := systemutil.GetUpgradeDir() + "/" + config.GetClientUpgraderFile()
 
 	if !systemutil.IsWindows() {
 		err := os.Chmod(scripPath, 0777)
 		if err != nil {
-			logs.Error("chmod failed: ", err.Error())
+			logs.Error("[agentUpgrade]|chmod failed: ", err.Error())
 			return errors.New("chmod failed: ")
 		}
 	}
@@ -70,52 +70,52 @@ func runUpgrader(action string) error {
 
 	pid, err := command.StartProcess(scripPath, args, systemutil.GetWorkDir(), nil, "")
 	if err != nil {
-		logs.Error("run upgrader failed: ", err.Error())
+		logs.Error("[agentUpgrade]|run upgrader failed: ", err.Error())
 		return errors.New("run upgrader failed")
 	}
-	logs.Info("start process success, pid: ", pid)
+	logs.Info("[agentUpgrade]|start process success, pid: ", pid)
 
-	logs.Warning("agent process exiting")
+	logs.Warning("[agentUpgrade]|agent process exiting")
 	systemutil.ExitProcess(0)
 	return nil
 }
 
 func DoUpgradeOperation(agentChanged bool, workAgentChanged bool) error {
-	logs.Info("start upgrade, agent changed: ", agentChanged, ", work agent changed: ", workAgentChanged)
+	logs.Info("[agentUpgrade]|start upgrade, agent changed: ", agentChanged, ", work agent changed: ", workAgentChanged)
 	config.GIsAgentUpgrading = true
 	defer func() {
 		config.GIsAgentUpgrading = false
 		api.FinishUpgrade(true)
 	}()
 
 	if !agentChanged && !workAgentChanged {
-		logs.Info("no change to upgrade, skip")
+		logs.Info("[agentUpgrade]|no change to upgrade, skip")
 		return nil
 	}
 
 	if workAgentChanged {
-		logs.Info("work agent changed, replace work agent file")
+		logs.Info("[agentUpgrade]|work agent changed, replace work agent file")
 		_, err := fileutil.CopyFile(
 			systemutil.GetUpgradeDir()+"/"+config.WorkAgentFile,
 			systemutil.GetWorkDir()+"/"+config.WorkAgentFile,
 			true)
 		if err != nil {
-			logs.Error("replace work agent file failed: ", err.Error())
+			logs.Error("[agentUpgrade]|replace work agent file failed: ", err.Error())
 			return errors.New("replace work agent file failed")
 		}
-		logs.Info("relace agent file done")
+		logs.Info("[agentUpgrade]|relace agent file done")
 
 		config.GAgentEnv.SlaveVersion = config.DetectWorkerVersion()
 	}
 
 	if agentChanged {
-		logs.Info("agent changed, start upgrader")
+		logs.Info("[agentUpgrade]|agent changed, start upgrader")
 		err := runUpgrader(config.ActionUpgrade)
 		if err != nil {
 			return err
 		}
 	} else {
-		logs.Info("agent not changed, skip agent upgrade")
+		logs.Info("[agentUpgrade]|agent not changed, skip agent upgrade")
 	}
 	return nil
 }