Delete bulk submissions asynchronously #4890
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| pull_request: | |
| branches: ["main"] | |
| push: | |
| branches: ["main"] | |
| concurrency: | |
| group: ci-${{ github.workflow }}-${{ github.actor }}-${{ github.sha }} | |
| cancel-in-progress: true | |
| jobs: | |
| static-analysis: | |
| name: Prospector Static Analysis | |
| runs-on: ubuntu-22.04 | |
| env: | |
| DJANGO_SETTINGS_MODULE: onadata.settings.github_actions_test | |
| strategy: | |
| fail-fast: false | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| architecture: "x64" | |
| cache: "pip" | |
| cache-dependency-path: | | |
| requirements/base.pip | |
| requirements/dev.pip | |
| requirements/azure.pip | |
| - name: Update apt sources | |
| run: sudo apt-get update | |
| - name: Install APT requirements | |
| run: sudo apt-get install -y --no-install-recommends libjpeg-dev zlib1g-dev software-properties-common ghostscript libxslt1-dev binutils libproj-dev gdal-bin memcached libmemcached-dev libxml2-dev libxslt-dev | |
| - name: Install Pip requirements | |
| run: | | |
| pip install -U pip | |
| pip install wheel | |
| pip install -r requirements/base.pip | |
| pip install -r requirements/dev.pip | |
| pip install -r requirements/azure.pip | |
| - name: Install linting tools | |
| run: pip install prospector==1.7.7 pylint==2.14.5 | |
| - name: Run Prospector | |
| run: prospector -X -s veryhigh onadata | |
| unit-tests: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test_path: | |
| - [" Django Unit Tests (Libraries, Main, RestServices, SMS Support, Viewer, Messaging)", "python manage.py test onadata/libs onadata/apps/main onadata/apps/restservice onadata/apps/sms_support onadata/apps/viewer onadata/apps/messaging --noinput --timing --settings=onadata.settings.github_actions_test --verbosity=2 --parallel=4"] | |
| - ["Django Unit Tests API", "python manage.py test onadata/apps/api --noinput --timing --settings=onadata.settings.github_actions_test --verbosity=2 --parallel=4"] | |
| - ["Django Unit Tests Logger", "python manage.py test onadata/apps/logger --noinput --timing --settings=onadata.settings.github_actions_test --verbosity=2 --parallel=4"] | |
| name: "${{ matrix.test_path[0] }}" | |
| runs-on: ubuntu-22.04 | |
| needs: static-analysis | |
| env: | |
| DJANGO_SETTINGS_MODULE: onadata.settings.github_actions_test | |
| services: | |
| postgres: | |
| image: postgis/postgis:13-3.0 | |
| env: | |
| POSTGRES_PASSWORD: onadata | |
| POSTGRES_DB: onadata | |
| POSTGRES_USER: onadata | |
| ports: | |
| - 5432:5432 | |
| # Set health checks to wait until postgres has started | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: "adopt" | |
| java-version: "8" | |
| - name: Setup python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| architecture: "x64" | |
| cache: "pip" | |
| cache-dependency-path: | | |
| requirements/base.pip | |
| requirements/dev.pip | |
| requirements/azure.pip | |
| - name: Update apt sources | |
| run: sudo apt-get update | |
| - name: Install APT requirements | |
| run: sudo apt-get install -y --no-install-recommends libjpeg-dev zlib1g-dev software-properties-common ghostscript libxslt1-dev binutils libproj-dev gdal-bin memcached libmemcached-dev libxml2-dev libxslt-dev | |
| - name: Install Pip requirements | |
| run: | | |
| pip install -U pip | |
| pip install -r requirements/base.pip | |
| pip install -r requirements/dev.pip | |
| pip install -r requirements/azure.pip | |
| - name: Run tests | |
| run: | | |
| ${{ matrix.test_path[1] }} | |
| security-check: | |
| name: Trivy Security Checks | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Update apt sources | |
| run: sudo apt-get update | |
| - name: Get the branch name | |
| id: get-branch-name | |
| if: github.event_name == 'push' | |
| run: echo "version=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV | |
| - name: Build Docker image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: ./docker/onadata-uwsgi/Dockerfile.ubuntu | |
| platforms: linux/amd64 | |
| push: false | |
| tags: | | |
| onaio/onadata:${{ github.head_ref || github.base_ref || env.version }} | |
| cache-from: type=registry,ref=onaio/onadata:${{ github.head_ref || github.base_ref || env.version }} | |
| cache-to: type=inline | |
| - name: Run Trivy vulnerability scanner | |
| uses: aquasecurity/trivy-action@master | |
| if: github.event_name == 'pull_request' | |
| with: | |
| image-ref: onaio/onadata:${{ github.head_ref || github.base_ref || env.version }} | |
| format: sarif | |
| ignore-unfixed: true | |
| severity: "CRITICAL,HIGH" | |
| output: "trivy_results.sarif" | |
| - name: Run Trivy vulnerability scanner | |
| uses: aquasecurity/trivy-action@master | |
| if: github.event_name == 'push' | |
| with: | |
| image-ref: onaio/onadata:${{ github.head_ref || github.base_ref || env.version }} | |
| format: sarif | |
| ignore-unfixed: true | |
| output: "trivy_results.sarif" | |
| - name: Upload vulnerability scan results | |
| uses: github/codeql-action/upload-sarif@v3 | |
| if: github.event_name == 'push' || github.event_name == 'pull_request' | |
| with: | |
| sarif_file: "trivy_results.sarif" | |
| - name: Run Trivy vulnerability for Slack summary | |
| uses: aquasecurity/trivy-action@master | |
| if: github.event_name == 'push' | |
| with: | |
| image-ref: onaio/onadata:${{ github.head_ref || github.base_ref || env.version }} | |
| format: json | |
| ignore-unfixed: true | |
| output: "trivy_results.json" | |
| - name: Create summary of trivy issues | |
| if: github.event_name == 'push' | |
| run: | | |
| summary=$(jq -r '.Results[] | select(.Vulnerabilities) | .Vulnerabilities | group_by(.Severity) | map({Severity: .[0].Severity, Count: length}) | .[] | [.Severity, .Count] | join(": ")' trivy_results.json | awk 'NR > 1 { printf(" | ") } {printf "%s",$0}') | |
| if [ -z $summary ] | |
| then | |
| summary="0 Issues" | |
| fi | |
| echo "SUMMARY=$summary" >> $GITHUB_ENV | |
| - name: Send Slack Notification | |
| uses: slackapi/[email protected] | |
| if: github.event_name == 'push' | |
| with: | |
| payload: | | |
| { | |
| "text": "Trivy scan results for ${{ github.head_ref || github.base_ref || env.version }}", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "[Ona Data] Trivy scan results for ${{ github.head_ref || github.base_ref || env.version }}: ${{ env.SUMMARY }}" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "View scan results: https://github.com/${{ github.repository }}/security/code-scanning?query=branch:${{ github.head_ref || github.base_ref || env.version }}+is:open++" | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK |