selfhost_intergration.yml

name: SELFHOST-INTEGRATION-TEST
on:

  schedule:
    - cron: '0 14 * * *'

  workflow_dispatch:
    inputs:
      EXEC_TEST_TYPE:
        description: 'Which tests need to be executed? The options are all, python, java, batch, cli, standalone-cli, apiserver, yarn'
        required: true
        default: 'all'
      EXEC_VERSION: 
        description: 'Which version needs to be tested?'
        required: true
        default: 'main'   
env:
  GIT_SUBMODULE_STRATEGY: recursive
  HYBRIDSE_SOURCE: local
  E_VERSION: ${{ github.event.inputs.EXEC_VERSION || 'main'}}
  ETYPE: ${{ github.event.inputs.EXEC_TEST_TYPE || 'all'}}
  NPROC: 4

jobs:
  build-openmldb:

    runs-on: [self-hosted,generic]
    if: github.repository == '4paradigm/OpenMLDB'
    container:
      image: ghcr.io/4paradigm/hybridsql:latest
    env:
      OS: linux
    steps:
      - uses: actions/checkout@v2
      - name: build
        if: ${{ env.E_VERSION == 'main' }}
        run: |
         make configure CMAKE_INSTALL_PREFIX=openmldb-linux
         make SQL_JAVASDK_ENABLE=ON && make SQL_JAVASDK_ENABLE=ON install
         mv openmldb-linux openmldb-main-linux
         tar -zcf openmldb-linux.tar.gz openmldb-main-linux
         mv java/openmldb-batch/target/openmldb*-SNAPSHOT.jar  openmldb-batch.jar
         mv java/openmldb-jdbc/target/openmldb*-SNAPSHOT.jar  openmldb-jdbc.jar
         mv java/openmldb-native/target/openmldb*-SNAPSHOT.jar  openmldb-native.jar
      - name: download
        if: ${{ env.E_VERSION != 'main' }}
        run: | 
         curl -SLO https://github.com/4paradigm/OpenMLDB/releases/download/v${{ env.E_VERSION }}/openmldb-${{ env.E_VERSION }}-linux.tar.gz
         tar -zxf openmldb-${{ env.E_VERSION }}-linux.tar.gz 
         mv openmldb-${{ env.E_VERSION }}-linux.tar.gz  openmldb-linux.tar.gz
      - name: upload artifact
        uses: actions/upload-artifact@v3
        with:
          name: openmldb-package
          path: | 
            openmldb-batch.jar
            openmldb-jdbc.jar
            openmldb-native.jar
            openmldb-linux.tar.gz
          
  apiserver-cluster:
    needs: build-openmldb
    
    runs-on: [self-hosted,common-user]
    steps:
      - uses: actions/checkout@v3
      - name:  before test
        if: ${{ env.ETYPE == 'all' || env.ETYPE == 'java' }}
        run: mkdir ${{ github.job }}
      - name: download artifact
        uses: actions/download-artifact@v3
        with:
            name: openmldb-package       
      - name: install openmldb 
        run: |
          tar -zxf openmldb-linux.tar.gz -C ${{ github.job }}/
          bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux ${{ github.job }} 20001 21000 java
          bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh
          bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh
      - name: test
        run: bash test/steps/openmldb-integration-test.sh -j 0.0.0 -d deploy  -m apiserver
      - name: stop openmldb
        if: always()
        run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh
      - name: remove openmldb
        if: always()
        run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh
      - name: TEST Results
        if: always()
        uses: EnricoMi/publish-unit-test-result-action@v1
        with:
          files: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/TEST-*.xml
          check_name: "SRC apiserver-cluster Report"
          comment_title: "SRC apiserver-cluster Report"
      - name: tar test report
        if: ${{ failure() }}
        run: tar -zcvf surefire-reports.tar.gz test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports
      - name: Send Email
        if: ${{ failure() }}
        uses: dawidd6/action-send-mail@master
        with:
          server_address: mail.4paradigm.com
          server_port: 995
          username: ${{ secrets.MAIL_USERNAME }}
          password: ${{ secrets.MAIL_PASSWORD }}
          secure: true
          subject: OpenMLDB Memory Test
          body: OpenMLDB Memory Test Failed
          html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
          to: ${{ secrets.MAIL_TO }}
          from: GitHub Actions
          attachments: surefire-reports.tar.gz
    
  java-sdk-cluster-memory-0:
    needs: build-openmldb
    
    runs-on: [self-hosted,common-user]
    steps:
      - uses: actions/checkout@v3
      - name:  before test
        if: ${{ env.ETYPE == 'all' || env.ETYPE == 'java' }}
        run: mkdir ${{ github.job }}
      - name: download artifact
        uses: actions/download-artifact@v3
        with:
            name: openmldb-package       
      - name: install openmldb
        run: |
          tar -zxf openmldb-linux.tar.gz -C ${{ github.job }}/
          bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux ${{ github.job }} 20001 21000 java
          bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh
          bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh
      - name: test
        run: bash test/steps/openmldb-integration-test.sh -j 0.0.0  -c test_cluster.xml -d deploy -l "0" -s "memory"    
      - name: stop openmldb
        if: always()
        run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh
      - name: remove openmldb
        if: always()
        run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh
      - name: TEST Results
        if: always()
        uses: EnricoMi/publish-unit-test-result-action@v1
        with:
          files: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/TEST-*.xml
          check_name: "SRC java-sdk-cluster-memory-0 Report"
          comment_title: "SRC java-sdk-cluster-memory-0 Report"
      - name: tar test report
        if: ${{ failure() }}
        run: tar -zcvf surefire-reports.tar.gz test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports
      - name: Send Email
        if: ${{ failure() }}
        uses: dawidd6/action-send-mail@master
        with:
          server_address: mail.4paradigm.com
          server_port: 995
          username: ${{ secrets.MAIL_USERNAME }}
          password: ${{ secrets.MAIL_PASSWORD }}
          secure: true
          subject: OpenMLDB Memory Test
          body: OpenMLDB Memory Test Failed
          html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
          to: ${{ secrets.MAIL_TO }}
          from: GitHub Actions
          attachments: surefire-reports.tar.gz

          
          
  java-sdk-cluster-memory-1:
    needs: build-openmldb
    runs-on: [self-hosted,common-user]
    steps:
      - uses: actions/checkout@v3
      - name:  before test
        if: ${{ env.ETYPE == 'all' || env.ETYPE == 'java' }}
        run: mkdir ${{ github.job }}
      - name: download artifact
        uses: actions/download-artifact@v3
        with:
            name: openmldb-package       
      - name: install openmldb
        run: |
          tar -zxf openmldb-linux.tar.gz -C ${{ github.job }}/
          bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux ${{ github.job }} 21001 22000 java
          bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh
          bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh
      - name: test
        run: bash test/steps/openmldb-integration-test.sh -j 0.0.0  -c test_cluster.xml -d deploy -l "1,2,3,4,5" -s "memory"   
      - name: stop openmldb
        if: always()
        run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh
      - name: remove openmldb
        if: always()
        run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh
      - name: TEST Results
        if: always()
        uses: EnricoMi/publish-unit-test-result-action@v1
        with:
          files: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/TEST-*.xml
          check_name: "SRC java-sdk-cluster-memory-1 Report"
          comment_title: "SRC java-sdk-cluster-memory-1 Report"
      - name: tar test report
        if: ${{ failure() }}
        run: tar -zcvf surefire-reports.tar.gz test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports
      - name: Send Email
        if: ${{ failure() }}
        uses: dawidd6/action-send-mail@master
        with:
          server_address: mail.4paradigm.com
          server_port: 995
          username: ${{ secrets.MAIL_USERNAME }}
          password: ${{ secrets.MAIL_PASSWORD }}
          secure: true
          subject: OpenMLDB Memory Test
          body: OpenMLDB Memory Test Failed
          html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
          to: ${{ secrets.MAIL_TO }}
          from: GitHub Actions
          attachments: surefire-reports.tar.gz

        
        
        
  java-sdk-cluster-hdd:
    needs: build-openmldb
    runs-on: [self-hosted,common-user]
    steps:
      - uses: actions/checkout@v3
      - name:  before test
        if: ${{ env.ETYPE == 'all' || env.ETYPE == 'java' }}
        run: mkdir ${{ github.job }}
      - name: download artifact
        uses: actions/download-artifact@v3
        with:
            name: openmldb-package       
      - name: install openmldb
        run: |
          tar -zxf openmldb-linux.tar.gz -C ${{ github.job }}/
          bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux ${{ github.job }} 22001 23000 java
          bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh
          bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh
      - name: test
        run: bash test/steps/openmldb-integration-test.sh -j 0.0.0  -c test_cluster_disk.xml -d deploy -l "0" -s "hdd"
      - name: stop openmldb
        if: always()
        run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh
      - name: remove openmldb
        if: always()
        run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh
      - name: TEST Results
        if: always()
        uses: EnricoMi/publish-unit-test-result-action@v1
        with:
          files: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/TEST-*.xml
          check_name: "SRC java-sdk-cluster-hdd Report"
          comment_title: "SRC java-sdk-cluster-hdd Report"
      - name: tar test report
        if: ${{ failure() }}
        run: tar -zcvf surefire-reports.tar.gz test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports
      - name: Send Email
        if: ${{ failure() }}
        uses: dawidd6/action-send-mail@master
        with:
          server_address: mail.4paradigm.com
          server_port: 995
          username: ${{ secrets.MAIL_USERNAME }}
          password: ${{ secrets.MAIL_PASSWORD }}
          secure: true
          subject: OpenMLDB HDD Test
          body: OpenMLDB HDD Test Failed
          html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
          to: ${{ secrets.MAIL_TO }}
          from: GitHub Actions
          attachments: surefire-reports.tar.gz

          

  java-sdk-cluster-ssd:
    needs: build-openmldb
    runs-on: [self-hosted,common-user]
    steps:
      - uses: actions/checkout@v3
      - name:  before test
        if: ${{ env.ETYPE == 'all' || env.ETYPE == 'java' }}
        run: mkdir ${{ github.job }}
      - name: download artifact
        uses: actions/download-artifact@v3
        with:
            name: openmldb-package       
      - name: install openmldb
        run: |
          tar -zxf openmldb-linux.tar.gz -C ${{ github.job }}/
          bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux ${{ github.job }} 23001 24000 java ssd
          bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh
          bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh
      - name: test
        run: bash test/steps/openmldb-integration-test.sh -j 0.0.0  -c test_cluster_disk.xml -d deploy -l "0" -s "ssd"
      - name: stop openmldb
        if: always()
        run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh
      - name: remove openmldb
        if: always()
        run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh
      - name: TEST Results
        if: always()
        uses: EnricoMi/publish-unit-test-result-action@v1
        with:
          files: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/TEST-*.xml
          check_name: "SRC java-sdk-cluster-ssd Report"
          comment_title: "SRC java-sdk-cluster-ssd Report"
      - name: tar test report
        if: ${{ failure() }}
        run: tar -zcvf surefire-reports.tar.gz test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports
      - name: Send Email
        if: ${{ failure() }}
        uses: dawidd6/action-send-mail@master
        with:
          server_address: mail.4paradigm.com
          server_port: 995
          username: ${{ secrets.MAIL_USERNAME }}
          password: ${{ secrets.MAIL_PASSWORD }}
          secure: true
          subject: OpenMLDB SSD Test
          body: OpenMLDB SSD Test Failed
          html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
          to: ${{ secrets.MAIL_TO }}
          from: GitHub Actions
          attachments: surefire-reports.tar.gz

          
  java-sdk-yarn:
    needs: build-openmldb
    runs-on: [self-hosted,common-user]
    steps:
      - uses: actions/checkout@v3
      - name:  before test
        if: ${{ env.ETYPE == 'all' || env.ETYPE == 'java' }}
        run: mkdir ${{ github.job }}
      - name: download artifact
        uses: actions/download-artifact@v3
        with:
            name: openmldb-package       
      - name: install openmldb
        run: |
          tar -zxf openmldb-linux.tar.gz -C ${{ github.job }}/
          bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux ${{ github.job }} 24001 25000 java hadoop
          bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh
          bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh
          bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-taskmanagers.sh
          HADOOP_CONF_DIR=/mnt/hdd0/denglong/openmldb_runner_work/hadoop ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-taskmanagers.sh
      - name: test
        run: bash test/steps/openmldb-integration-test.sh -j 0.0.0  -c test_yarn.xml -d deploy -l "0" -s "memory"
      - name: stop openmldb
        if: always()
        run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh
      - name: remove openmldb
        if: always()
        run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh
      - name: TEST Results
        if: always()
        uses: EnricoMi/publish-unit-test-result-action@v1
        with:
          files: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/TEST-*.xml
          check_name: "SRC java-sdk-yarn Report"
          comment_title: "SRC java-yarn Report"
      - name: tar test report
        if: ${{ failure() }}
        run: tar -zcvf surefire-reports.tar.gz test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports
      - name: Send Email
        if: ${{ failure() }}
        uses: dawidd6/action-send-mail@master
        with:
          server_address: mail.4paradigm.com
          server_port: 995
          username: ${{ secrets.MAIL_USERNAME }}
          password: ${{ secrets.MAIL_PASSWORD }}
          secure: true
          subject: OpenMLDB yarn Test
          body: OpenMLDB yarn Test Failed
          html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
          to: ${{ secrets.MAIL_TO }}
          from: GitHub Actions
          attachments: surefire-reports.tar.gz



  java-sdk-kafka:
    needs: build-openmldb
    runs-on: [self-hosted,common-user]
    steps:
      - uses: actions/checkout@v3
      - name:  before test
        if: ${{ env.ETYPE == 'all' || env.ETYPE == 'java' }}
        run: mkdir ${{ github.job }}
      - name: download artifact
        uses: actions/download-artifact@v3
        with:
            name: openmldb-package       
      - name: install openmldb
        run: |
          tar -zxf openmldb-linux.tar.gz -C ${{ github.job }}/
          bash test/steps/format_config.sh $(pwd)/${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux ${{ github.job }} 25001 26000 java kafka
          bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/deploy-all.sh
          bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/start-all.sh
      - name: test
        run: echo "a"
      - name: stop openmldb
        if: always()
        run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/stop-all.sh
      - name: remove openmldb
        if: always()
        run: bash ${{ github.job }}/openmldb-${{ env.E_VERSION }}-linux/sbin/clear-all.sh
      # - name: TEST Results
      #   if: always()
      #   uses: EnricoMi/publish-unit-test-result-action@v1
      #   with:
      #     files: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/TEST-*.xml
      #     check_name: "SRC java-sdk-yarn Report"
      #     comment_title: "SRC java-yarn Report"
      # - name: tar test report
      #   if: ${{ failure() }}
      #   run: tar -zcvf surefire-reports.tar.gz test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports
      # - name: Send Email
      #   if: ${{ failure() }}
      #   uses: dawidd6/action-send-mail@master
      #   with:
      #     server_address: mail.4paradigm.com
      #     server_port: 995
      #     secure: true
      #     username: ${{ secrets.MAIL_USERNAME }}
      #     password: ${{ secrets.MAIL_PASSWORD }}
      #     subject: OpenMLDB yarn Test
      #     body: OpenMLDB yarn Test Failed
      #     html_body: test/integration-test/openmldb-test-java/openmldb-sdk-test/target/surefire-reports/html/overview.html
      #     to: ${{ secrets.MAIL_TO }}
      #     from: GitHub Actions
      #     content_type: text/plain
      #     attachments: surefire-reports.tar.gz