benchmark test with chatqna_benchmark #3
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (C) 2024 Intel Corporation | |
| # SPDX-License-Identifier: Apache-2.0 | |
| name: benchmark test with chatqna_benchmark | |
| on: | |
| # pull_request: | |
| # branches: [main] | |
| # types: [opened, reopened, ready_for_review, synchronize] | |
| # # inputs: | |
| # # variables: | |
| # # hardware: | |
| # # description: 'Enter your param' #gaudi or xeon | |
| # # required: true | |
| # # default: xeon | |
| schedule: | |
| - cron: "35 0 * * 6" | |
| workflow_dispatch: | |
| inputs: | |
| hardware: | |
| description: 'Enter your hardware' #gaudi or xeon | |
| required: true | |
| default: gaudi | |
| jobs: | |
| Example-test: | |
| runs-on: ${{ github.event.inputs.hardware || 'gaudi' }} #xeon #gaudi | |
| steps: | |
| - name: Clean Up Working Directory | |
| run: sudo rm -rf ${{github.workspace}}/* | |
| - name: Checkout out Repo | |
| uses: actions/checkout@v4 | |
| - name: Clone repo GenAIEval | |
| run: | | |
| git clone https://github.com/opea-project/GenAIEval.git | |
| cd GenAIEval && git checkout v0.6 | |
| - name: Run test | |
| env: | |
| HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} | |
| GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }} | |
| GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} | |
| hardware: ${{ github.event.inputs.hardware || 'gaudi' }} #xeon | |
| mode: perf | |
| IMAGE_TAG: latest | |
| IMAGE_REPO_GAUDI: ${{ vars.IMAGE_REPO_GAUDI }} | |
| IMAGE_REPO_XEON: ${{ vars.IMAGE_REPO_XEON }} | |
| run: | | |
| # cd ${{ github.workspace }}/$example/tests | |
| cd ${{ github.workspace }}/ChatQnA/tests | |
| cp ../../GenAIEval/evals/benchmark/chatqna_benchmark.py . | |
| cp ../../GenAIEval/evals/benchmark/data.json ${{ github.workspace }}/ChatQnA/docker/${hardware}/ | |
| if [ "$hardware" == "gaudi" ]; then IMAGE_REPO=$IMAGE_REPO_GAUDI; else IMAGE_REPO=$IMAGE_REPO_XEON; fi | |
| export IMAGE_REPO=${IMAGE_REPO} | |
| # example_l=$(echo $example | tr '[:upper:]' '[:lower:]') | |
| if [ -f test_chatqna_on_${hardware}.sh ]; then timeout 30m bash test_chatqna_on_${hardware}.sh > ${hardware}_output.log; else echo "Test script not found, skip test!"; fi | |
| - name: Process log and save to JSON | |
| env: | |
| hardware: ${{ github.event.inputs.hardware || 'gaudi' }} #xeon | |
| run: | | |
| cd ${{ github.workspace }}/ChatQnA/tests | |
| echo '{}' > ${hardware}_output.json | |
| echo $(grep -a 'Total Requests:' ${hardware}_output.log | awk '{print "{\"total_requests\": \""$3 "\"}"}') > ${hardware}_output.json | |
| echo $(grep -a 'P50 latency is' ${hardware}_output.log | awk '{print "{\"p50_latency\": \""$4 "\"}"}') >> ${hardware}_output.json | |
| echo $(grep -a 'P99 latency is' ${hardware}_output.log | awk '{print "{\"p99_latency\": \""$4 "\"}"}') >> ${hardware}_output.json | |
| jq -s 'add' ${hardware}_output.json > ${hardware}_final_output.json && mv ${hardware}_final_output.json ${hardware}_output.json | |
| - name: Publish pipeline artifact | |
| if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: | | |
| ${{ github.workspace }}/ChatQnA/tests/*.log | |
| ${{ github.workspace }}/ChatQnA/tests/*.json |