From af2d0f522c726b8c892e6c8c7b1f984737ec5c10 Mon Sep 17 00:00:00 2001 From: Ying Chun Guo Date: Fri, 5 Jul 2024 17:15:06 +0800 Subject: [PATCH] enable GMC system installation on push (#158) Signed-off-by: Yingchun Guo --- .github/workflows/gmc-on-push.yaml | 60 ++++++++++++ .github/workflows/image-build-on-push.yaml | 56 ----------- .github/workflows/reuse-gmc-image-build.yaml | 8 +- .github/workflows/scripts/e2e/gmc_install.sh | 99 ++++++++++++++++++++ .github/workflows/scripts/e2e/utils.sh | 49 ++++++++++ 5 files changed, 212 insertions(+), 60 deletions(-) create mode 100644 .github/workflows/gmc-on-push.yaml delete mode 100644 .github/workflows/image-build-on-push.yaml create mode 100755 .github/workflows/scripts/e2e/gmc_install.sh create mode 100755 .github/workflows/scripts/e2e/utils.sh diff --git a/.github/workflows/gmc-on-push.yaml b/.github/workflows/gmc-on-push.yaml new file mode 100644 index 00000000..9280be04 --- /dev/null +++ b/.github/workflows/gmc-on-push.yaml @@ -0,0 +1,60 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +name: Upgrade GMC system on push event + +on: + push: + branches: ["main"] + paths: + - microservices-connector/** + - manifests/** + - "!**.md" + - "!**.txt" + - "!**.png" + - "!.**" + - .github/workflows/gmc-on-push.yaml + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-on-push + cancel-in-progress: true + +env: + GOSRC_DIR: "microservices-connector" + +jobs: + image-build: + strategy: + matrix: + platform: [xeon, gaudi] + uses: ./.github/workflows/reuse-gmc-image-build.yaml + with: + image_tag: 'latest' + runner_label: 'docker-build-${{ matrix.platform }}' + + gmc-install: + strategy: + matrix: + platform: [xeon, gaudi] + needs: image-build + runs-on: ${{ matrix.platform }} + steps: + - name: Checkout out Repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set variables + run: | + echo "SYSTEM_NAMESPACE=opea-system" >> $GITHUB_ENV + # don't set IMAGE_REPO to the outputs because outputs.image_repo may not be correct + # echo "IMAGE_REPO=${{ needs.image-build.outputs.image_repo }}" >> $GITHUB_ENV + echo "VERSION=${{ needs.image-build.outputs.image_tag }}" >> $GITHUB_ENV + + - name: Cleanup existing GMC + run: | + .github/workflows/scripts/e2e/gmc_install.sh cleanup_gmc + + - name: Install GMC + run: | + .github/workflows/scripts/e2e/gmc_install.sh install_gmc diff --git a/.github/workflows/image-build-on-push.yaml b/.github/workflows/image-build-on-push.yaml deleted file mode 100644 index 87bb4352..00000000 --- a/.github/workflows/image-build-on-push.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright (C) 2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -name: Build latest images on push event - -on: - push: - branches: ["main"] - paths: - - microservices-connector/** - - "!**.md" - - "!**.txt" - - "!**.png" - - "!.**" - - .github/workflows/image-build-on-push.yml - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-on-push - cancel-in-progress: true - -env: - GOSRC_DIR: "microservices-connector" - -jobs: - image-build: - strategy: - matrix: - platform: [xeon, gaudi] - runs-on: docker-build-${{ matrix.platform }} - steps: - - name: Checkout out Repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set variables - id: set_variables - run: | - echo "DOCKER_REGISTRY=${OPEA_IMAGE_REPO}opea" >> $GITHUB_ENV - echo "VERSION=latest" >> $GITHUB_ENV - echo "DOCKER_REGISTRY=${OPEA_IMAGE_REPO}opea" >> $GITHUB_OUTPUT - echo "VERSION=latest" >> $GITHUB_OUTPUT - - - name: Build image and push - run: | - cd $GOSRC_DIR - make docker.build - make docker.push - - - name: Clean up images - if: always() - run: | - # clean up the images - docker rmi ${{ env.DOCKER_REGISTRY }}/gmcrouter:${{ env.VERSION }} - docker rmi ${{ env.DOCKER_REGISTRY }}/gmcmanager:${{ env.VERSION }} - echo y | docker image prune diff --git a/.github/workflows/reuse-gmc-image-build.yaml b/.github/workflows/reuse-gmc-image-build.yaml index 6d4ed293..c3a5da77 100644 --- a/.github/workflows/reuse-gmc-image-build.yaml +++ b/.github/workflows/reuse-gmc-image-build.yaml @@ -15,11 +15,11 @@ on: runner_label: required: false type: string - default: 'xeon' + default: 'docker-build-xeon' outputs: image_repo: description: "The image repository used for the image build" - value: ${{ jobs.image-build.outputs.docker_registry }} + value: ${{ jobs.image-build.outputs.image_repo }} image_tag: description: "The image tag used for the image build" value: ${{ jobs.image-build.outputs.version }} @@ -30,7 +30,7 @@ jobs: image-build: runs-on: ${{ inputs.runner_label }} outputs: - docker_registry: ${{ steps.set_variables.outputs.DOCKER_REGISTRY }} + image_repo: ${{ steps.set_variables.outputs.IMAGE_REPO }} version: ${{ steps.set_variables.outputs.VERSION }} steps: - name: Checkout out Repo @@ -43,7 +43,7 @@ jobs: run: | echo "DOCKER_REGISTRY=${OPEA_IMAGE_REPO}opea" >> $GITHUB_ENV echo "VERSION=${{ inputs.image_tag }}" >> $GITHUB_ENV - echo "DOCKER_REGISTRY=${OPEA_IMAGE_REPO}opea" >> $GITHUB_OUTPUT + echo "IMAGE_REPO=${OPEA_IMAGE_REPO}" >> $GITHUB_OUTPUT echo "VERSION=${{ inputs.image_tag }}" >> $GITHUB_OUTPUT - name: Build image and push diff --git a/.github/workflows/scripts/e2e/gmc_install.sh b/.github/workflows/scripts/e2e/gmc_install.sh new file mode 100755 index 00000000..01933d1d --- /dev/null +++ b/.github/workflows/scripts/e2e/gmc_install.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +source ${DIR}/utils.sh + +USER_ID=$(whoami) +LOG_PATH=/home/$(whoami)/logs +MOUNT_DIR=${KIND_MOUNT_DIR:-"/home/$USER_ID/.cache/huggingface/hub"} +TOKEN_DIR=${KIND_TOKEN_DIR:-"/home/$USER_ID/.cache/huggingface/token"} +IMAGE_REPO=${OPEA_IMAGE_REPO:-""} +VERSION=${VERSION:-"latest"} + +function install_gmc() { + PLATFORM=${1:-"xeon"} + # Make sure you have to use image tag $VERSION for microservice-connector installation + echo "install microservice-connector on $PLATFORM, using repo $IMAGE_REPO and tag $VERSION" + echo "using namespace $SYSTEM_NAMESPACE" + + init_gmc + + kubectl apply -f $(pwd)/config/crd/bases/gmc.opea.io_gmconnectors.yaml + kubectl apply -f $(pwd)/config/rbac/gmc-manager-rbac.yaml + kubectl create configmap gmcyaml -n $SYSTEM_NAMESPACE --from-file $(pwd)/config/manifests + kubectl apply -f $(pwd)/config/manager/gmc-manager.yaml + + # Wait until the gmc controller pod is ready + wait_until_pod_ready "gmc-controller" $SYSTEM_NAMESPACE "gmc-controller" + kubectl get pods -n $SYSTEM_NAMESPACE +} + +function copy_manifests() { + # Copy manifest into gmc + mkdir -p $(pwd)/config/manifests + cp $(dirname $(pwd))/manifests/ChatQnA/*.yaml -p $(pwd)/config/manifests/ + cp $(dirname $(pwd))/manifests/DocSum/xeon/docsum_llm.yaml -p $(pwd)/config/manifests/ + cp $(dirname $(pwd))/manifests/DocSum/gaudi/docsum_gaudi_llm.yaml -p $(pwd)/config/manifests/ +} + +function init_gmc() { + # copy manifests + copy_manifests + + # replace tag with for the gmc-router and gmc-manager image + sed -i "s|opea/\(.*\):latest|opea/\1:$VERSION|g" $(pwd)/config/gmcrouter/gmc-router.yaml + sed -i "s|opea/\(.*\):latest|opea/\1:$VERSION|g" $(pwd)/config/manager/gmc-manager.yaml + # replace the pull policy "IfNotPresent" with "Always" + sed -i "s#pullPolicy: IfNotPresent#pullPolicy: Always#g" $(pwd)/config/gmcrouter/gmc-router.yaml + sed -i "s#pullPolicy: IfNotPresent#pullPolicy: Always#g" $(pwd)/config/manager/gmc-manager.yaml + + cp $(pwd)/config/gmcrouter/gmc-router.yaml -p $(pwd)/config/manifests/ + + # replace namespace for gmc-router and gmc-manager + sed -i "s|namespace: system|namespace: $SYSTEM_NAMESPACE|g" $(pwd)/config/manager/gmc-manager.yaml + sed -i "s|namespace: system|namespace: $SYSTEM_NAMESPACE|g" $(pwd)/config/rbac/gmc-manager-rbac.yaml + sed -i "s|name: system|name: $SYSTEM_NAMESPACE|g" $(pwd)/config/rbac/gmc-manager-rbac.yaml + + # replace the mount dir "path: /mnt/model" with "path: $CHART_MOUNT" + find . -name '*.yaml' -type f -exec sed -i "s#path: /mnt/models#path: $MOUNT_DIR#g" {} \; + # replace the repository "image: opea/*" with "image: ${IMAGE_REPO}opea/" + find . -name '*.yaml' -type f -exec sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" {} \; + find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}opea/#g" {} \; + # set huggingface token + # find . -name '*.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#$(cat $TOKEN_DIR)#g" {} \; + find . -name '*.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#$(cat $TOKEN_DIR)#g" {} \; +} + +function cleanup_gmc() { + echo "clean up microservice-connector" + if kubectl get namespace $SYSTEM_NAMESPACE > /dev/null 2>&1; then + echo "Deleting namespace: $SYSTEM_NAMESPACE" + kubectl delete namespace "$SYSTEM_NAMESPACE" + kubectl delete crd gmconnectors.gmc.opea.io + else + echo "Namespace $SYSTEM_NAMESPACE does not exist" + fi +} + +#------MAIN----------- +if [ $# -eq 0 ]; then + echo "Usage: $0 " + exit 1 +fi +case "$1" in + install_gmc) + pushd microservices-connector + install_gmc + popd + ;; + cleanup_gmc) + cleanup_gmc + ;; + *) + echo "Unknown function: $1" + ;; +esac diff --git a/.github/workflows/scripts/e2e/utils.sh b/.github/workflows/scripts/e2e/utils.sh new file mode 100755 index 00000000..18e2b7cc --- /dev/null +++ b/.github/workflows/scripts/e2e/utils.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +function wait_until_pod_ready() { + echo "Waiting for the $1 to be ready..." + max_retries=30 + retry_count=0 + while ! is_pod_ready $2 $3; do + if [ $retry_count -ge $max_retries ]; then + echo "$1 is not ready after waiting for a significant amount of time" + get_gmc_controller_logs + exit 1 + fi + echo "$1 is not ready yet. Retrying in 30 seconds..." + sleep 30 + output=$(kubectl get pods -n $2) + echo $output + retry_count=$((retry_count + 1)) + done +} + +function is_pod_ready() { + if [ "$2" == "gmc-controller" ]; then + pod_status=$(kubectl get pods -n $1 -o jsonpath='{.items[].status.conditions[?(@.type=="Ready")].status}') + else + pod_status=$(kubectl get pods -n $1 -l app=$2 -o jsonpath='{.items[].status.conditions[?(@.type=="Ready")].status}') + fi + if [ "$pod_status" == "True" ]; then + return 0 + else + return 1 + fi +} + +function get_gmc_controller_logs() { + # Fetch the name of the pod with the app-name gmc-controller in the specified namespace + pod_name=$(kubectl get pods -n $SYSTEM_NAMESPACE -l control-plane=gmc-controller -o jsonpath='{.items[0].metadata.name}') + + # Check if the pod name was found + if [ -z "$pod_name" ]; then + echo "No pod found with app-name gmc-controller in namespace $SYSTEM_NAMESPACE" + return 1 + fi + + # Get the logs of the found pod + echo "Fetching logs for pod $pod_name in namespace $SYSTEM_NAMESPACE..." + kubectl logs $pod_name -n $SYSTEM_NAMESPACE +}