migrate CI

Signed-off-by: Hengyu Meng <[email protected]>

.github/pull_request_template.md

@@ -0,0 +1,21 @@
+## Type of Change
+
+feature or bug fix or documentation or others
+API changed or not
+
+## Description
+
+detail description 
+Issues: xxx
+
+## Expected Behavior & Potential Risk
+
+the expected behavior that triggered by this PR 
+
+## How has this PR been tested?
+
+how to reproduce the test (including hardware information)
+
+## Dependency Change?
+
+any library dependency introduced or removed

.github/workflows/copyright_check.yml

@@ -0,0 +1,80 @@
+name: Copyright Check
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - intel_extension_for_transformers/**
+      - setup.py
+      - .github/workflows/format_scan.yml
+  workflow_dispatch:
+
+# If there is a new commit, the previous jobs will be canceled
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  CODE_SCAN_LOG_PATH: "${{ github.workspace }}/log"
+  CONTAINER_NAME: "codeScan"
+
+jobs:
+  format-scan:
+    runs-on: itrex-node-spell
+    strategy:
+      matrix:
+        job_name: ["copyright"]
+      fail-fast: false
+    steps:
+      - name: Docker Clean Up
+        run: |
+          docker ps -a
+          if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}-${{ runner.name }}'$) ]]; then
+              docker start ${{ env.CONTAINER_NAME }}-${{ runner.name }}
+              echo "remove left files through container ..."
+              docker exec ${{ env.CONTAINER_NAME }}-${{ runner.name }} bash -c "ls -a /intel-extension-for-transformers && rm -fr /intel-extension-for-transformers/* && rm -fr /intel-extension-for-transformers/.* || true"
+          fi
+          
+      - name: Checkout out Repo
+        uses: actions/checkout@v3
+
+      - name: CopyRight check
+        run: |
+          source ${{ github.workspace }}/.github/workflows/script/change_color.sh
+          set -e
+          mkdir -p ${{ env.CODE_SCAN_LOG_PATH }}
+          supported_extensions=(py, sh, yaml)
+          git fetch
+          git --no-pager diff --name-only remotes/origin/${{ github.base_ref }} ${{ github.workspace }}/intel_extension_for_transformers> ${{ env.CODE_SCAN_LOG_PATH }}/diff.log
+          files=$(cat ${{ env.CODE_SCAN_LOG_PATH }}/diff.log | awk '!a[$0]++')
+          $LIGHT_PURPLE && echo " ----------------- checking ... --------------------------" && $RESET
+          if [[ -f ${{ env.CODE_SCAN_LOG_PATH }}/copyright_issue_summary.log ]]; then 
+            rm -f ${{ env.CODE_SCAN_LOG_PATH }}/copyright_issue_summary.log
+          fi
+          for file in ${files}
+          do
+              if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then
+                  if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then
+                      echo ${file} >> ${{ env.CODE_SCAN_LOG_PATH }}/copyright_issue_summary.log
+                      $BOLD_YELLOW && echo " -----------------  Current log file output start --------------------------"
+                      cat ${{ env.CODE_SCAN_LOG_PATH }}/copyright_issue_summary.log
+                      $BOLD_YELLOW && echo " -----------------  Current log file output end --------------------------" && $RESET
+                      $BOLD_RED && echo "CopyRight has something wrong! Please click on the artifact button to download and view the error log!" && $RESET
+                  fi
+              else
+                  $LIGHT_PURPLE && echo "Skipping ${file}"  && $RESET
+              fi
+          done
+          if [[ -f ${{ env.CODE_SCAN_LOG_PATH }}/copyright_issue_summary.log ]]; then 
+            $BOLD_YELLOW && echo " -----------------  Current log file output start --------------------------"
+            cat ${{ env.CODE_SCAN_LOG_PATH }}/copyright_issue_summary.log
+            $BOLD_YELLOW && echo " -----------------  Current log file output end --------------------------" && $RESET
+            $BOLD_RED && echo "CopyRight has something wrong! Please click on the artifact button to download and view the error log!" && $RESET && exit 1
+          fi
+
+      - name: Publish pipeline artifact
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.job_name }}
+          path: ${{ env.CODE_SCAN_LOG_PATH }}.*

.github/workflows/cpp-graph-test.yml

@@ -0,0 +1,158 @@
+name: CPP Graph Test
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+       - '.github/workflows/cpp-graph-test.yml'
+       - '.github/workflows/script/models/cpp_graph_inference.sh'
+       - 'intel_extension_for_transformers/llm/runtime/graph/**'
+       - 'intel_extension_for_transformers/llm/library/jblas/**'
+       - '!intel_extension_for_transformers/llm/runtime/graph/README.md'
+  workflow_dispatch:
+    inputs:
+      compiler_version:
+        description: 'compiler_version'
+        required: false
+        type: string
+        default: '13.1.0'
+      models:
+        description: 'models (in json)'
+        required: false
+        type: string
+        default: '["llama-2-7b-chat", "gptj-6b"]'
+      runner:
+        description: 'runner'
+        required: false
+        type: string
+        default: 'spr'
+
+# If there is a new commit, the previous jobs will be canceled
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  OUT_SCRIPT_PATH: ${{ github.workspace }}/.github/workflows/script/models
+  SCRIPT_PATH: ${{ github.workspace }}/.github/workflows/script
+  WORKING_DIR: ${{ github.workspace }}
+  CONTAINER_NAME: "codeScan"
+  INPUT_COMPILER_VERSION: ${{ inputs.compiler_version || '13.1.0' }}
+
+jobs:
+  CPP-Graph-Workflow:
+    runs-on: ${{inputs.runner || 'spr'}}
+    strategy:
+      matrix:
+        modelName: ${{fromJson(inputs.models || '["llama-2-7b-chat", "gptj-6b"]')}}
+    steps:
+      - name: Checkout out Repo
+        uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+          fetch-tags: true
+
+      - name: Env build
+        run: |
+          bash ${{ github.workspace }}/.github/workflows/script/prepare_env_with_conda.sh "cpp-graph-test" "3.8"
+
+      - name: Binary build
+        # cpp model does not requires itrex package
+        if: 0 == 1
+        run: |
+          cd ${{ github.workspace }}
+          conda activate cpp-graph-test || source activate cpp-graph-test
+          pip install build --upgrade
+          pip install -r requirements.txt
+          python setup.py sdist bdist_wheel
+          pip install dist/intel_extension_for_transformers*.whl
+          pip list
+
+      - name: BF16 Benchmark
+        run: |
+          cd ${{ github.workspace }}/.github/workflows/script/models
+          bash cpp_graph_inference.sh cpp-graph-test ${{ matrix.modelName }} ${{ env.INPUT_COMPILER_VERSION }}
+      
+      - name: Rename summary
+        run: |
+          cd ${{ github.workspace }}
+          cp cpp_graph_summary.log cpp_graph_summary_${{matrix.modelName}}.log
+
+      - name: Publish pipeline artifact
+        uses: actions/upload-artifact@v3
+        if: ${{ !cancelled() }}
+        with:
+          name: cpp_graph
+          path: ${{ github.workspace }}/cpp_graph_summary_${{matrix.modelName}}.log
+          if-no-files-found: ignore # 'warn' or 'ignore' are also available, defaults to `warn`
+          retention-days: 60 # 1 <= retention-days <= 90
+
+  Genreate-Report:
+    runs-on: itrex-node-spell
+    needs: [CPP-Graph-Workflow]
+    steps:
+      - name: Docker Clean Up
+        run: |
+          docker ps -a
+          if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}-${{ runner.name }}'$) ]]; then
+              docker start ${{ env.CONTAINER_NAME }}-${{ runner.name }}
+              echo "remove left files through container ..."
+              docker exec ${{ env.CONTAINER_NAME }}-${{ runner.name }} bash -c "ls -a /intel-extension-for-transformers && rm -fr /intel-extension-for-transformers/* && rm -fr /intel-extension-for-transformers/.* || true"
+          fi
+      - name: Checkout out Repo
+        uses: actions/checkout@v3
+
+      - name: Download Summary Log
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ env.OUT_SCRIPT_PATH }}/generated/log
+
+      - name: Merge CPP Graph Summary Log
+        run: |
+          cd ${{ env.OUT_SCRIPT_PATH }}/generated/log/cpp_graph
+          for summary in $(find . -name "cpp_graph_summary_*.log"); do cat $summary >> cpp_graph_summary.log; done
+
+      - name: Download Reference Artifact
+        id: download-artifact
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: cpp-graph-test.yml
+          name: FinalReport
+          run_id: ${{ vars.GRAPH_REF_ID }}
+          path: ${{ env.OUT_SCRIPT_PATH }}
+          name_is_regexp: true
+          repo: ${{ github.repository }}
+          check_artifacts: false
+          search_artifacts: false
+          skip_unpack: false
+          if_no_artifact_found: warn
+
+      - name: Display structure of downloaded files
+        run: cd ${{ env.OUT_SCRIPT_PATH }} && ls -R
+
+      - name: Generate report
+        run: |
+          echo "------ Generating final report.html ------"
+          cd ${{ env.OUT_SCRIPT_PATH }}
+          /usr/bin/bash generate_report.sh --workflow=deploy
+          sed -n '/<body>/,/<\/body>/p' generated/report.html | sed -r '/^$/d' | sed -r 's/^ +//g' >> $GITHUB_STEP_SUMMARY
+        env:
+          RUN_DISPLAY_URL: https://github.com/VincyZhang/intel-extension-for-transformers/actions/runs/${{ github.run_id }}
+          BUILD_NUMBER: ${{ github.run_id }}
+          JOB_STATUS: succeed
+          MR_source_branch: ${{ github.head_ref }}
+          ghprbActualCommit: ${{ github.event.pull_request.head.sha }}
+
+      - name: Publish Report
+        uses: actions/upload-artifact@v3
+        if: ${{ !cancelled() }}
+        with:
+          name: FinalReport
+          path: ${{ env.OUT_SCRIPT_PATH }}/generated
+
+      - name: Specify performance regression
+        run: |
+          if [ $(is_perf_reg) == 'true' ]; then
+            echo "[Performance Regression] Some model performance regression occurred, please check artifacts and reports."
+            exit 1
+          fi

.github/workflows/format_scan.yml

@@ -0,0 +1,84 @@
+name: Format Scan
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - intel_extension_for_transformers/**
+      - neural_chat/**
+      - workflows/**
+      - setup.py
+      - .github/workflows/format_scan.yml
+      - .github/workflows/script/formatScan/**
+  workflow_dispatch:
+
+# If there is a new commit, the previous jobs will be canceled
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  DOCKER_CONFIG_NAME: "commonDockerConfig"
+  REPO_NAME: "code-scan"
+  REPO_TAG: "1.0"
+  DOCKER_FILE_NAME: "codeScan"
+  CONTAINER_NAME: "codeScan"
+
+jobs:
+  format-scan:
+    runs-on: itrex-node-spell
+    strategy:
+      matrix:
+        job_name: [
+            "pylint",
+            "bandit",
+            "clangformat",
+            "cloc",
+            "cpplint",
+            # "pydocstyle",
+            #"pyspelling",
+          ]
+      fail-fast: false
+    steps:
+      - name: Docker Clean Up
+        run: |
+          docker ps -a
+          if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}-${{ runner.name }}'$) ]]; then
+              docker start ${{ env.CONTAINER_NAME }}-${{ runner.name }}
+              echo "remove left files through container ..."
+              docker exec ${{ env.CONTAINER_NAME }}-${{ runner.name }} bash -c "ls -a /intel-extension-for-transformers && rm -fr /intel-extension-for-transformers/* && rm -fr /intel-extension-for-transformers/.* || true"
+          fi
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v3
+
+      - name: Docker Build
+        run: |
+          docker build -f ${{ github.workspace }}/.github/workflows/docker/${{ env.DOCKER_FILE_NAME }}.dockerfile -t ${{ env.REPO_NAME }}:${{ env.REPO_TAG }} .
+
+      - name: Docker Run
+        run: |
+          if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}-${{ runner.name }}'$) ]]; then
+            docker stop ${{ env.CONTAINER_NAME }}-${{ runner.name }}
+            docker rm -vf ${{ env.CONTAINER_NAME }}-${{ runner.name }} || true
+          fi
+          docker run -dit --memory="4g" --memory-reservation="1g" --disable-content-trust --privileged --name=${{ env.CONTAINER_NAME }}-${{ runner.name }} --shm-size="1g" \
+          -v ${{ github.workspace }}:/intel-extension-for-transformers \
+          ${{ env.REPO_NAME }}:${{ env.REPO_TAG }}
+
+      - name: Env build
+        run: |
+          docker exec ${{ env.CONTAINER_NAME }}-${{ runner.name }} \
+          bash /intel-extension-for-transformers/.github/workflows/script/prepare_env.sh
+
+      - name: Code scan check
+        run: |
+          docker exec ${{ env.CONTAINER_NAME }}-${{ runner.name }} \
+          bash -c "bash /intel-extension-for-transformers/.github/workflows/script/formatScan/${{ matrix.job_name }}.sh"
+
+      - name: Publish pipeline artifact
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.job_name }}
+          path: ${{ github.workspace }}/.github/workflows/script/formatScan/${{ matrix.job_name }}.*