This repository has been archived by the owner on Aug 30, 2024. It is now read-only.
migrate codes from itrex #1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: LLM Model Test | |
| on: | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - '.github/workflows/llm-test.yml' | |
| - '.github/workflows/script/models/run_llm.sh' | |
| - "intel_extension_for_transformers/llm/runtime/deprecated/**" | |
| - "!intel_extension_for_transformers/llm/runtime/deprecated/kernels/**" | |
| - "!intel_extension_for_transformers/llm/runtime/graph/**" | |
| - "!intel_extension_for_transformers/llm/runtime/deprecated/test/**" | |
| - "!intel_extension_for_transformers/llm/runtime/deprecated/third_party/**" | |
| - "!intel_extension_for_transformers/llm/runtime/deprecated/docs/**" | |
| workflow_dispatch: | |
| # If there is a new commit, the previous jobs will be canceled | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| OUT_SCRIPT_PATH: ${{ github.workspace }}/.github/workflows/script/models | |
| SCRIPT_PATH: ${{ github.workspace }}/.github/workflows/script | |
| WORKING_DIR: ${{ github.workspace }} | |
| EXTRA_CONTAINER_NAME: "codeScan" | |
| jobs: | |
| LLM-Workflow: | |
| runs-on: spr | |
| strategy: | |
| matrix: | |
| include: | |
| - modelName: "gpt-j-6b" | |
| framework: "engine" | |
| mode: "latency" | |
| precision: "bf16,int8,fp8" | |
| steps: | |
| - name: Checkout out Repo | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: "recursive" | |
| fetch-tags: true | |
| - name: Env build | |
| run: | | |
| bash ${{ github.workspace }}/.github/workflows/script/prepare_env_with_conda.sh "llm-test" "3.8" | |
| - name: Binary build | |
| run: | | |
| cd ${{ github.workspace }} | |
| conda activate llm-test || source activate llm-test | |
| pip install build --upgrade | |
| pip install -r requirements.txt | |
| python setup.py sdist bdist_wheel | |
| pip install dist/intel_extension_for_transformers*.whl | |
| pip list | |
| - name: BF16 Benchmark | |
| run: | | |
| cd ${{ github.workspace }}/.github/workflows/script/models | |
| bash run_llm.sh --model=${{ matrix.modelName }} --framework=${{ matrix.framework }} --mode=${{ matrix.mode }} --conda_env_name=llm-test --precision=bf16 | |
| - name: INT8 Benchmark | |
| run: | | |
| cd ${{ github.workspace }}/.github/workflows/script/models | |
| bash run_llm.sh --model=${{ matrix.modelName }} --framework=${{ matrix.framework }} --mode=${{ matrix.mode }} --conda_env_name=llm-test --precision=int8 | |
| - name: FP8 Benchmark | |
| run: | | |
| cd ${{ github.workspace }}/.github/workflows/script/models | |
| bash run_llm.sh --model=${{ matrix.modelName }} --framework=${{ matrix.framework }} --mode=${{ matrix.mode }} --conda_env_name=llm-test --precision=fp8 | |
| - name: Publish pipeline artifact | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ !cancelled() }} | |
| with: | |
| name: llm | |
| path: ${{ github.workspace }}/*.log | |
| if-no-files-found: ignore # 'warn' or 'ignore' are also available, defaults to `warn` | |
| retention-days: 60 # 1 <= retention-days <= 90 | |
| Genreate-Report: | |
| runs-on: itrex-node-spell | |
| needs: [LLM-Workflow] | |
| steps: | |
| - name: Docker Clean Up | |
| run: | | |
| docker ps -a | |
| if [[ $(docker ps -a | grep -i '${{ env.EXTRA_CONTAINER_NAME }}-${{ runner.name }}'$) ]]; then | |
| docker start ${{ env.EXTRA_CONTAINER_NAME }}-${{ runner.name }} | |
| echo "remove left files through container ..." | |
| docker exec ${{ env.EXTRA_CONTAINER_NAME }}-${{ runner.name }} bash -c "ls -a /intel-extension-for-transformers && rm -fr /intel-extension-for-transformers/* && rm -fr /intel-extension-for-transformers/.* || true" | |
| fi | |
| - name: Checkout out Repo | |
| uses: actions/checkout@v3 | |
| - name: Download Summary Log | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: ${{ env.OUT_SCRIPT_PATH }}/generated/log | |
| - name: Download Reference Artifact | |
| id: download-artifact | |
| uses: dawidd6/action-download-artifact@v2 | |
| with: | |
| workflow: llm-test.yml | |
| name: FinalReport | |
| run_id: ${{ vars.LLM_REF_ID }} | |
| path: ${{ env.OUT_SCRIPT_PATH }} | |
| name_is_regexp: true | |
| repo: ${{ github.repository }} | |
| check_artifacts: false | |
| search_artifacts: false | |
| skip_unpack: false | |
| if_no_artifact_found: warn | |
| - name: Display structure of downloaded files | |
| run: cd ${{ env.OUT_SCRIPT_PATH }} && ls -R | |
| - name: Generate report | |
| run: | | |
| echo "------ Generating final report.html ------" | |
| cd ${{ env.OUT_SCRIPT_PATH }} | |
| /usr/bin/bash generate_report.sh --workflow=deploy | |
| env: | |
| RUN_DISPLAY_URL: https://github.com/VincyZhang/intel-extension-for-transformers/actions/runs/${{ github.run_id }} | |
| BUILD_NUMBER: ${{ github.run_id }} | |
| JOB_STATUS: succeed | |
| MR_source_branch: ${{ github.head_ref }} | |
| ghprbActualCommit: ${{ github.event.pull_request.head.sha }} | |
| - name: Publish Report | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ !cancelled() }} | |
| with: | |
| name: FinalReport | |
| path: ${{ env.OUT_SCRIPT_PATH }}/generated | |
| - name: Specify performance regression | |
| run: | | |
| if [ $(is_perf_reg) == 'true' ]; then | |
| echo "[Performance Regression] Some model performance regression occurred, please check artifacts and reports." | |
| exit 1 | |
| fi | |