diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0dd6f7c4b6..f7e4b426b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,18 +3,64 @@ on: - push - pull_request concurrency: - group: ci-${{ github.ref }}-${{ github.sha }}-${{ github.event_name }} + group: ci-${{ github.workflow }}-${{ github.actor }}-${{ github.sha }} cancel-in-progress: true jobs: - github-action-ci: + static-analysis: + name: Prospector Static Analysis runs-on: ubuntu-20.04 + env: + DJANGO_SETTINGS_MODULE: onadata.settings.github_actions_test strategy: fail-fast: false - matrix: - include: - - testfolder: "onadata/libs onadata/apps/main onadata/apps/restservice onadata/apps/sms_support onadata/apps/viewer onadata/apps/messaging" - - testfolder: "onadata/apps/api onadata/apps/logger" + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Setup python + uses: actions/setup-python@v3 + with: + python-version: 3.9 + architecture: "x64" + + - name: Get pip cache dir + id: pip-cache + run: | + echo "pip_cache_dir=$(pip cache dir)" >> $GITHUB_ENV + + - name: Cache pip + uses: actions/cache@v3 + id: cache-pip + with: + path: ${{ env.pip_cache_dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements/base.pip') }}-${{ hashFiles('requirements/dev.pip') }}-${{ hashFiles('requirements/azure.pip') }} + + - name: Install APT requirements + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends libjpeg-dev zlib1g-dev software-properties-common ghostscript libxslt1-dev binutils libproj-dev gdal-bin memcached libmemcached-dev libxml2-dev libxslt-dev + sudo rm -rf /var/lib/apt/lists/* + + - name: Install Pip requirements + run: | + pip install -U pip + pip install -r requirements/base.pip + pip install -r requirements/dev.pip + pip install -r requirements/azure.pip + + - name: Install linting tools + run: + pip install prospector==1.7.7 pylint==2.14.5 + + - name: Run Prospector + run: prospector -X -s veryhigh onadata + unit-tests-1: + name: Django Unit Tests (Libraries, Main, RestServices, SMS Support, Viewer, Messagin) + runs-on: ubuntu-20.04 + needs: static-analysis + env: + DJANGO_SETTINGS_MODULE: onadata.settings.github_actions_test services: postgres: image: postgis/postgis:13-3.0 @@ -32,7 +78,7 @@ jobs: --health-retries 5 steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Java uses: actions/setup-java@v2 @@ -49,14 +95,14 @@ jobs: - name: Get pip cache dir id: pip-cache run: | - echo "::set-output name=dir::$(pip cache dir)" + echo "pip_cache_dir=$(pip cache dir)" >> $GITHUB_ENV - name: Cache pip - uses: actions/cache@v2 + uses: actions/cache@v3 id: cache-pip with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/base.pip') }} + path: ${{ env.pip_cache_dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements/base.pip') }}-${{ hashFiles('requirements/dev.pip') }}-${{ hashFiles('requirements/azure.pip') }} - name: Install APT requirements run: | @@ -68,21 +114,170 @@ jobs: run: | pip install -U pip pip install -r requirements/base.pip - pip install flake8 - pip install tblib - pip install flaky - pip install httmock - pip install mock - pip install requests-mock - - - name: Python Static Analysis + pip install -r requirements/dev.pip + + - name: Run tests + run: | + python manage.py test onadata/libs onadata/apps/main onadata/apps/restservice onadata/apps/sms_support onadata/apps/viewer onadata/apps/messaging --noinput --timing --settings=onadata.settings.github_actions_test --verbosity=2 --parallel=4 + unit-tests-2: + name: Django Unit Tests (API, Logger) + runs-on: ubuntu-20.04 + needs: static-analysis + env: + DJANGO_SETTINGS_MODULE: onadata.settings.github_actions_test + services: + postgres: + image: postgis/postgis:13-3.0 env: - DJANGO_SETTINGS_MODULE: onadata.settings.github_actions_test + POSTGRES_PASSWORD: onadata + POSTGRES_DB: onadata + POSTGRES_USER: onadata + ports: + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Setup Java + uses: actions/setup-java@v2 + with: + distribution: "adopt" + java-version: "8" + + - name: Setup python + uses: actions/setup-python@v3 + with: + python-version: 3.9 + architecture: "x64" + + - name: Get pip cache dir + id: pip-cache run: | - pip install prospector==1.7.7 pylint==2.14.5 - pip install -r requirements/azure.pip - prospector -X -s veryhigh onadata + echo "pip_cache_dir=$(pip cache dir)" >> $GITHUB_ENV + + - name: Cache pip + uses: actions/cache@v3 + id: cache-pip + with: + path: ${{ env.pip_cache_dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements/base.pip') }}-${{ hashFiles('requirements/dev.pip') }}-${{ hashFiles('requirements/azure.pip') }} + + - name: Install APT requirements + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends libjpeg-dev zlib1g-dev software-properties-common ghostscript libxslt1-dev binutils libproj-dev gdal-bin memcached libmemcached-dev libxml2-dev libxslt-dev + sudo rm -rf /var/lib/apt/lists/* + + - name: Install Pip requirements + run: | + pip install -U pip + pip install -r requirements/base.pip + pip install -r requirements/dev.pip - name: Run tests run: | - python manage.py test ${{ matrix.testfolder }} --noinput --settings=onadata.settings.github_actions_test --verbosity=2 --parallel=4 + python manage.py test onadata/apps/api onadata/apps/logger --noinput --timing --settings=onadata.settings.github_actions_test --verbosity=2 --parallel=4 + security-check: + name: Trivy Security Checks + runs-on: ubuntu-20.04 + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Get the branch name + id: get-branch-name + if: github.event_name == 'push' + run: echo "version=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + + - name: Build Docker image + uses: docker/build-push-action@v3 + with: + context: ./docker/onadata-uwsgi + file: ./docker/onadata-uwsgi/Dockerfile + platforms: linux/amd64 + build-args: | + release_version=${{ github.head_ref || github.base_ref || env.version }} + push: false + tags: | + onaio/onadata:${{ github.head_ref || github.base_ref || env.version }} + cache-from: type=registry,ref=onaio/onadata:${{ github.head_ref || github.base_ref || env.version }} + cache-to: type=inline + + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + if: github.event_name == 'pull_request' + with: + image-ref: onaio/onadata:${{ github.head_ref || github.base_ref || env.version }} + format: sarif + ignore-unfixed: true + severity: 'CRITICAL,HIGH' + exit-code: '1' + output: 'trivy_results.sarif' + + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + if: github.event_name == 'push' + with: + image-ref: onaio/onadata:${{ github.head_ref || github.base_ref || env.version }} + format: sarif + ignore-unfixed: true + output: 'trivy_results.sarif' + + - name: Upload vulnerability scan results + uses: github/codeql-action/upload-sarif@v2 + if: github.event_name == 'push' + with: + sarif_file: 'trivy_results.sarif' + + - name: Run Trivy vulnerability for Slack summary + uses: aquasecurity/trivy-action@master + if: github.event_name == 'push' + with: + image-ref: onaio/onadata:${{ github.head_ref || github.base_ref || env.version }} + format: json + ignore-unfixed: true + output: 'trivy_results.json' + + - name: Create summary of trivy issues + if: github.event_name == 'push' + run: | + summary=$(jq -r '.Results[] | select(.Vulnerabilities) | .Vulnerabilities | group_by(.Severity) | map({Severity: .[0].Severity, Count: length}) | .[] | [.Severity, .Count] | join(": ")' trivy_results.json | awk 'NR > 1 { printf(" | ") } {printf "%s",$0}') + if [ -z $summary ] + then + summary="0 Issues" + fi + echo "SUMMARY=$summary" >> $GITHUB_ENV + + - name: Send Slack Notification + uses: slackapi/slack-github-action@v1.19.0 + if: github.event_name == 'push' + with: + payload: | + { + "text": "Trivy scan results for ${{ github.head_ref || github.base_ref || env.version }}", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "[Ona Data] Trivy scan results for ${{ github.head_ref || github.base_ref || env.version }}: ${{ env.SUMMARY }}" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "View scan results: https://github.com/${{ github.repository }}/security/code-scanning?query=branch:${{ github.head_ref || github.base_ref || env.version }}+is:open++" + } + } + ] + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK diff --git a/.github/workflows/docker-image-build.yml b/.github/workflows/docker-image-build.yml index 29f02a8bca..4b9113d3f1 100644 --- a/.github/workflows/docker-image-build.yml +++ b/.github/workflows/docker-image-build.yml @@ -30,12 +30,12 @@ jobs: - name: Get the version id: get-version if: github.event.inputs.versionTag == '' && github.event_name != 'push' - run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//} + run: echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV - name: Get the branch name id: get-branch-name if: github.event_name == 'push' - run: echo "##[set-output name=BRANCH;]$(echo ${GITHUB_REF#refs/heads/})" + run: echo "version=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - name: Login to DockerHub uses: docker/login-action@v1 @@ -59,11 +59,11 @@ jobs: file: ./docker/onadata-uwsgi/Dockerfile platforms: linux/amd64,linux/arm64 build-args: | - release_version=${{ github.event.inputs.versionTag || steps.get-version.outputs.VERSION || steps.get-branch-name.outputs.BRANCH }} + release_version=${{ github.event.inputs.versionTag || env.version }} optional_packages=PyYAML django-redis push: true tags: | - onaio/onadata:${{ github.event.inputs.versionTag || steps.get-version.outputs.VERSION || steps.get-branch-name.outputs.BRANCH }} + onaio/onadata:${{ github.event.inputs.versionTag || env.version }} - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/ecr-image-build.yml b/.github/workflows/ecr-image-build.yml index f91bff614c..400bdf621e 100644 --- a/.github/workflows/ecr-image-build.yml +++ b/.github/workflows/ecr-image-build.yml @@ -30,12 +30,12 @@ jobs: - name: Get the version id: get-version if: github.event.inputs.versionTag == '' && github.event_name != 'push' - run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//} + run: echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV - name: Get the branch name id: get-branch-name if: github.event_name == 'push' - run: echo "##[set-output name=BRANCH;]$(echo ${GITHUB_REF#refs/heads/})" + run: echo "version=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 @@ -67,11 +67,11 @@ jobs: ssh: | default=/tmp/ssh-agent.sock build-args: | - release_version=${{ github.event.inputs.versionTag || steps.get-version.outputs.VERSION || steps.get-branch-name.outputs.BRANCH }} + release_version=${{ github.event.inputs.versionTag || env.version }} optional_packages=PyYAML django-redis ${{ secrets.ECR_OPTIONAL_PACKAGES }} push: true tags: | - ${{ steps.login-ecr.outputs.registry }}/onaio/onadata:${{ github.event.inputs.versionTag || steps.get-version.outputs.VERSION || steps.get-branch-name.outputs.BRANCH }} + ${{ steps.login-ecr.outputs.registry }}/onaio/onadata:${{ github.event.inputs.versionTag || env.version }} - name: Image digest run: echo ${{ steps.docker-build.outputs.digest }} @@ -79,7 +79,7 @@ jobs: - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@master with: - image-ref: ${{ steps.login-ecr.outputs.registry }}/onaio/onadata:${{ github.event.inputs.versionTag || steps.get-version.outputs.VERSION || steps.get-branch-name.outputs.BRANCH }} + image-ref: ${{ steps.login-ecr.outputs.registry }}/onaio/onadata:${{ github.event.inputs.versionTag || env.version }} format: 'sarif' output: 'trivy-results.sarif' @@ -91,7 +91,7 @@ jobs: - name: Run Trivy vulnerability scanner for Slack uses: aquasecurity/trivy-action@master with: - image-ref: ${{ steps.login-ecr.outputs.registry }}/onaio/onadata:${{ github.event.inputs.versionTag || steps.get-version.outputs.VERSION || steps.get-branch-name.outputs.BRANCH }} + image-ref: ${{ steps.login-ecr.outputs.registry }}/onaio/onadata:${{ github.event.inputs.versionTag || env.version }} format: json output: 'trivy-results.json' diff --git a/onadata/apps/api/tests/viewsets/test_data_viewset.py b/onadata/apps/api/tests/viewsets/test_data_viewset.py index ab046a1772..5152dcd8d4 100644 --- a/onadata/apps/api/tests/viewsets/test_data_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_data_viewset.py @@ -16,6 +16,7 @@ from django.core.cache import cache from django.db.utils import OperationalError from django.test import RequestFactory +from django.test.testcases import SerializeMixin from django.test.utils import override_settings from django.utils import timezone @@ -109,10 +110,11 @@ def _data_instance(dataid): # pylint: disable=too-many-public-methods -class TestDataViewSet(TestBase): +class TestDataViewSet(SerializeMixin, TestBase): """ Test /data API endpoint implementation. """ + lockfile = __file__ def setUp(self): super().setUp() diff --git a/onadata/apps/api/tests/viewsets/test_media_viewset.py b/onadata/apps/api/tests/viewsets/test_media_viewset.py index 771d13b1c5..53fadac3c4 100644 --- a/onadata/apps/api/tests/viewsets/test_media_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_media_viewset.py @@ -18,6 +18,9 @@ def attachment_url(attachment, suffix=None): class TestMediaViewSet(TestAbstractViewSet): + """ + Test the /api/v1/files endpoint + """ def setUp(self): super(TestMediaViewSet, self).setUp() self.retrieve_view = MediaViewSet.as_view({"get": "retrieve"}) @@ -30,7 +33,7 @@ def test_retrieve_view(self): "/", {"filename": self.attachment.media_file.name}, **self.extra ) response = self.retrieve_view(request, self.attachment.pk) - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 200, response) self.assertEqual(type(response.content), bytes) @patch("onadata.libs.utils.image_tools.get_storage_class") diff --git a/onadata/libs/tests/utils/test_image_tools.py b/onadata/libs/tests/utils/test_image_tools.py index 72dea1e256..21a49f4831 100644 --- a/onadata/libs/tests/utils/test_image_tools.py +++ b/onadata/libs/tests/utils/test_image_tools.py @@ -3,6 +3,7 @@ import requests from django.core.files.storage import get_storage_class +from django.test.testcases import SerializeMixin from httmock import urlmatch, HTTMock from onadata.apps.logger.models.attachment import Attachment @@ -20,7 +21,12 @@ def image_url_mock(url, request): return response -class TestImageTools(TestBase): +class TestImageTools(SerializeMixin, TestBase): + """ + Test class for image utility functions + """ + lockfile = __file__ + def test_resize_exception_is_handled(self): with HTTMock(image_url_mock): with self.assertRaises(Exception) as io_error: