hadoop: avoid create too many threads for multiple JuiceFileSystem instance #2566
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "unittests" | |
| on: | |
| push: | |
| branches: | |
| - 'main' | |
| - 'release-*' | |
| paths-ignore: | |
| - 'docs/**' | |
| - '**.md' | |
| pull_request: | |
| #The branches below must be a subset of the branches above | |
| branches: | |
| - 'main' | |
| - 'release-*' | |
| paths-ignore: | |
| - 'docs/**' | |
| - '**.md' | |
| - '.github/**' | |
| workflow_dispatch: | |
| inputs: | |
| debug: | |
| type: boolean | |
| description: "Run the build with tmate debugging enabled" | |
| required: false | |
| default: false | |
| jobs: | |
| unittests: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test: [ 'test.meta.core','test.meta.non-core','test.pkg','test.cmd', 'test.fdb' ] | |
| timeout-minutes: 60 | |
| runs-on: ubuntu-20.04 | |
| env: | |
| MINIO_TEST_BUCKET: 127.0.0.1:9000/testbucket | |
| MINIO_ACCESS_KEY: testUser | |
| MINIO_SECRET_KEY: testUserPassword | |
| DISPLAY_PROGRESSBAR: false | |
| HDFS_ADDR: localhost:8020 | |
| SFTP_HOST: localhost:2222:/upload/ | |
| SFTP_USER: root | |
| SFTP_PASS: password | |
| WEBDAV_TEST_BUCKET: 127.0.0.1:9007 | |
| TIKV_ADDR: 127.0.0.1 | |
| REDIS_ADDR: redis://127.0.0.1:6379/13 | |
| ETCD_ADDR: 127.0.0.1:3379 | |
| MYSQL_ADDR: (127.0.0.1:3306)/dev | |
| MYSQL_USER: root | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 1 | |
| - name: Build | |
| uses: ./.github/actions/build | |
| - name: Install Packages | |
| run: | | |
| sudo .github/scripts/apt_install.sh g++-multilib redis-server libacl1-dev attr | |
| sudo mkdir -p /home/travis/.m2/ | |
| - if: matrix.test == 'test.meta.non-core' | |
| name: Install redis-cluster | |
| uses: vishnudxb/[email protected] | |
| with: | |
| master1-port: 7000 | |
| master2-port: 7001 | |
| master3-port: 7002 | |
| slave1-port: 7003 | |
| slave2-port: 7004 | |
| slave3-port: 7005 | |
| - name: Prepare Database | |
| run: | | |
| case "${{matrix.test}}" in | |
| "test.meta.core") | |
| # tikv | |
| wget -O /home/travis/.m2/install.sh https://tiup-mirrors.pingcap.com/install.sh | |
| bash /home/travis/.m2/install.sh | |
| source /home/runner/.bash_profile | |
| tiup -v | |
| nohup tiup playground --mode tikv-slim >> output.log 2>&1 & | |
| sleep 30 | |
| # mysql | |
| sudo service mysql start | |
| sudo mysql -uroot -proot -e "use mysql;alter user 'root'@'localhost' identified with mysql_native_password by '';" | |
| sudo mysql -e "create database dev;" | |
| for i in {2379,3306}; do | |
| echo "lsof port:"$i | |
| sudo lsof -i :$i && continue || sleep 5 | |
| sudo lsof -i :$i || ( echo "service not ready, port:$i" && exit 1 ) | |
| done | |
| ;; | |
| "test.meta.non-core") | |
| # postgres | |
| sudo service postgresql start | |
| sudo chmod 777 /etc/postgresql/*/main/pg_hba.conf | |
| sudo sed -i "s?local.*all.*postgres.*peer?local all postgres trust?" /etc/postgresql/*/main/pg_hba.conf | |
| sudo sed -i "s?host.*all.*all.*32.*scram-sha-256?host all all 127.0.0.1/32 trust?" /etc/postgresql/*/main/pg_hba.conf | |
| sudo sed -i "s?host.*all.*all.*128.*scram-sha-256?host all all ::1/128 trust?" /etc/postgresql/*/main/pg_hba.conf | |
| cat /etc/postgresql/*/main/pg_hba.conf | |
| sudo service postgresql restart | |
| psql -c "create user runner superuser;" -U postgres | |
| sudo service postgresql restart | |
| psql -c 'create database test;' -U postgres | |
| # etcd | |
| docker run -d \ | |
| -p 3379:2379 \ | |
| -p 3380:2380 \ | |
| --name etcd_3_5_7 \ | |
| quay.io/coreos/etcd:v3.5.7 \ | |
| /usr/local/bin/etcd --data-dir=/etcd-data --name node1 \ | |
| --listen-client-urls http://0.0.0.0:2379 \ | |
| --advertise-client-urls http://0.0.0.0:2379 \ | |
| --listen-peer-urls http://0.0.0.0:2380 \ | |
| --initial-advertise-peer-urls http://0.0.0.0:2380 \ | |
| --initial-cluster node1=http://0.0.0.0:2380 | |
| # keydb | |
| echo "deb https://download.keydb.dev/open-source-dist $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/keydb.list | |
| sudo wget -O /etc/apt/trusted.gpg.d/keydb.gpg https://download.keydb.dev/open-source-dist/keyring.gpg | |
| sudo .github/scripts/apt_install.sh keydb | |
| keydb-server --storage-provider flash /tmp/ --port 6378 --bind 127.0.0.1 --daemonize yes | |
| keydb-server --port 6377 --bind 127.0.0.1 --daemonize yes | |
| for i in {3379,6377,6378}; do | |
| echo "lsof port:"$i | |
| sudo lsof -i :$i && continue || sleep 5 | |
| sudo lsof -i :$i || ( echo "service not ready, port:$i" && exit 1 ) | |
| done | |
| ;; | |
| "test.cmd") | |
| # minio | |
| docker run -d -p 9000:9000 -p 9001:9001 -e "MINIO_ROOT_USER=testUser" -e "MINIO_ROOT_PASSWORD=testUserPassword" quay.io/minio/minio:RELEASE.2022-01-25T19-56-04Z server /data --console-address ":9001" | |
| # mc | |
| go install github.com/minio/[email protected] && mc config host add local http://127.0.0.1:9000 testUser testUserPassword && mc mb local/testbucket | |
| # litmus is used to test webdav | |
| wget -O /home/travis/.m2/litmus-0.13.tar.gz http://www.webdav.org/neon/litmus/litmus-0.13.tar.gz | |
| tar -zxvf /home/travis/.m2/litmus-0.13.tar.gz -C /home/travis/.m2/ | |
| cd /home/travis/.m2/litmus-0.13/ && ./configure && make && cd - | |
| ;; | |
| "test.fdb") | |
| # foundationdb | |
| wget -O /home/travis/.m2/foundationdb-clients_6.3.23-1_amd64.deb https://github.com/apple/foundationdb/releases/download/6.3.23/foundationdb-clients_6.3.23-1_amd64.deb | |
| wget -O /home/travis/.m2/foundationdb-server_6.3.23-1_amd64.deb https://github.com/apple/foundationdb/releases/download/6.3.23/foundationdb-server_6.3.23-1_amd64.deb | |
| sudo dpkg -i /home/travis/.m2/foundationdb-clients_6.3.23-1_amd64.deb /home/travis/.m2/foundationdb-server_6.3.23-1_amd64.deb | |
| for i in {4500,}; do | |
| echo "lsof port:"$i | |
| sudo lsof -i :$i && continue || sleep 5 | |
| sudo lsof -i :$i || ( echo "service not ready, port:$i" && exit 1 ) | |
| done | |
| ;; | |
| "test.pkg") | |
| # mysql | |
| sudo service mysql start | |
| sudo mysql -uroot -proot -e "use mysql;alter user 'root'@'localhost' identified with mysql_native_password by '';" | |
| sudo mysql -e "create database dev;" | |
| # tikv | |
| wget -O /home/travis/.m2/install.sh https://tiup-mirrors.pingcap.com/install.sh | |
| bash /home/travis/.m2/install.sh | |
| source /home/runner/.bash_profile | |
| tiup -v | |
| nohup tiup playground --mode tikv-slim >> output.log 2>&1 & | |
| # minio | |
| docker run -d -p 9000:9000 -p 9001:9001 -e "MINIO_ROOT_USER=testUser" -e "MINIO_ROOT_PASSWORD=testUserPassword" quay.io/minio/minio:RELEASE.2022-01-25T19-56-04Z server /data --console-address ":9001" | |
| # mc | |
| go install github.com/minio/[email protected] && mc config host add local http://127.0.0.1:9000 testUser testUserPassword && mc mb local/testbucket | |
| # webdav | |
| wget -O /home/travis/.m2/rclone-v1.57.0-linux-amd64.zip --no-check-certificate https://downloads.rclone.org/v1.57.0/rclone-v1.57.0-linux-amd64.zip | |
| unzip /home/travis/.m2/rclone-v1.57.0-linux-amd64.zip -d /home/travis/.m2/ | |
| nohup /home/travis/.m2/rclone-v1.57.0-linux-amd64/rclone serve webdav local --addr 127.0.0.1:9007 >> rclone.log 2>&1 & | |
| # sftp | |
| docker run -d --name sftp -p 2222:22 juicedata/ci-sftp | |
| # etcd | |
| docker run -d \ | |
| -p 3379:2379 \ | |
| -p 3380:2380 \ | |
| --name etcd_3_5_7 \ | |
| quay.io/coreos/etcd:v3.5.7 \ | |
| /usr/local/bin/etcd --data-dir=/etcd-data --name node1 \ | |
| --listen-client-urls http://0.0.0.0:2379 \ | |
| --advertise-client-urls http://0.0.0.0:2379 \ | |
| --listen-peer-urls http://0.0.0.0:2380 \ | |
| --initial-advertise-peer-urls http://0.0.0.0:2380 \ | |
| --initial-cluster node1=http://0.0.0.0:2380 | |
| # hdfs | |
| chmod +x .github/scripts/setup-hdfs.sh | |
| ./.github/scripts/setup-hdfs.sh | |
| for i in {2222,8020,9000,9007,2379}; do | |
| echo "lsof port:"$i | |
| sudo lsof -i :$i && continue || sleep 5 | |
| sudo lsof -i :$i || ( echo "service not ready, port:$i" && exit 1 ) | |
| done | |
| ;; | |
| *) | |
| echo "matrix.test: ${{matrix.test}} is not valid" && exit 1 | |
| ;; | |
| esac | |
| - name: Unit Test | |
| timeout-minutes: 30 | |
| run: | | |
| test=${{matrix.test}} | |
| make $test | |
| # - name: Code Coverage | |
| # uses: codecov/codecov-action@v3 | |
| # with: | |
| # files: ./cov.out | |
| - name: Setup upterm session | |
| if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true' }} | |
| timeout-minutes: 120 | |
| uses: lhotari/action-upterm@v1 | |
| success-all-test: | |
| runs-on: ubuntu-latest | |
| needs: [unittests] | |
| if: always() | |
| steps: | |
| - uses: technote-space/workflow-conclusion-action@v3 | |
| - uses: actions/checkout@v3 | |
| - name: Check Failure | |
| if: env.WORKFLOW_CONCLUSION == 'failure' | |
| run: exit 1 | |
| - name: Send Slack Notification | |
| if: ${{ failure() && github.event_name != 'workflow_dispatch' }} | |
| uses: juicedata/slack-notify-action@main | |
| with: | |
| channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}" | |
| slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}" | |
| - name: Success | |
| if: ${{ success() }} | |
| run: echo "All Done" | |