Merge pull request #2325 from onaio/include-postgres-client

Add PostgreSQL client to the Docker images
onaio · Oct 17, 2022 · 1af1dd1 · 1af1dd1
2 parents 5723140 + d8a8105
commit 1af1dd1
Show file tree

Hide file tree

Showing 6 changed files with 243 additions and 37 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -3,18 +3,64 @@ on:
   - push
   - pull_request
 concurrency:
-  group: ci-${{ github.ref }}-${{ github.sha }}-${{ github.event_name }}
+  group: ci-${{ github.workflow }}-${{ github.actor }}-${{ github.sha }}
   cancel-in-progress: true
 
 jobs:
-  github-action-ci:
+  static-analysis:
+    name: Prospector Static Analysis
     runs-on: ubuntu-20.04
+    env:
+      DJANGO_SETTINGS_MODULE: onadata.settings.github_actions_test
     strategy:
       fail-fast: false
-      matrix:
-        include:
-          - testfolder: "onadata/libs onadata/apps/main onadata/apps/restservice onadata/apps/sms_support onadata/apps/viewer onadata/apps/messaging"
-          - testfolder: "onadata/apps/api onadata/apps/logger"
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Setup python
+        uses: actions/setup-python@v3
+        with:
+          python-version: 3.9
+          architecture: "x64"
+
+      - name: Get pip cache dir
+        id: pip-cache
+        run: |
+          echo "pip_cache_dir=$(pip cache dir)" >> $GITHUB_ENV
+
+      - name: Cache pip
+        uses: actions/cache@v3
+        id: cache-pip
+        with:
+          path: ${{ env.pip_cache_dir }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements/base.pip') }}-${{ hashFiles('requirements/dev.pip') }}-${{ hashFiles('requirements/azure.pip') }}
+
+      - name: Install APT requirements
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends libjpeg-dev zlib1g-dev software-properties-common ghostscript libxslt1-dev binutils libproj-dev gdal-bin memcached libmemcached-dev libxml2-dev libxslt-dev
+          sudo rm -rf /var/lib/apt/lists/*
+
+      - name: Install Pip requirements
+        run: |
+          pip install -U pip
+          pip install -r requirements/base.pip
+          pip install -r requirements/dev.pip
+          pip install -r requirements/azure.pip
+
+      - name: Install linting tools
+        run:
+          pip install prospector==1.7.7 pylint==2.14.5
+
+      - name: Run Prospector
+        run: prospector -X -s veryhigh onadata
+  unit-tests-1:
+    name: Django Unit Tests (Libraries, Main, RestServices, SMS Support, Viewer, Messagin)
+    runs-on: ubuntu-20.04
+    needs: static-analysis
+    env:
+      DJANGO_SETTINGS_MODULE: onadata.settings.github_actions_test
     services:
       postgres:
         image: postgis/postgis:13-3.0
@@ -32,7 +78,7 @@ jobs:
           --health-retries 5
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Setup Java
         uses: actions/setup-java@v2
@@ -49,14 +95,14 @@ jobs:
       - name: Get pip cache dir
         id: pip-cache
         run: |
-          echo "::set-output name=dir::$(pip cache dir)"
+          echo "pip_cache_dir=$(pip cache dir)" >> $GITHUB_ENV
 
       - name: Cache pip
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         id: cache-pip
         with:
-          path: ${{ steps.pip-cache.outputs.dir }}
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements/base.pip') }}
+          path: ${{ env.pip_cache_dir }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements/base.pip') }}-${{ hashFiles('requirements/dev.pip') }}-${{ hashFiles('requirements/azure.pip') }}
 
       - name: Install APT requirements
         run: |
@@ -68,21 +114,170 @@ jobs:
         run: |
           pip install -U pip
           pip install -r requirements/base.pip
-          pip install flake8
-          pip install tblib
-          pip install flaky
-          pip install httmock
-          pip install mock
-          pip install requests-mock
-
-      - name: Python Static Analysis
+          pip install -r requirements/dev.pip
+
+      - name: Run tests
+        run: |
+          python manage.py test onadata/libs onadata/apps/main onadata/apps/restservice onadata/apps/sms_support onadata/apps/viewer onadata/apps/messaging --noinput --timing --settings=onadata.settings.github_actions_test --verbosity=2 --parallel=4
+  unit-tests-2:
+    name: Django Unit Tests (API, Logger)
+    runs-on: ubuntu-20.04
+    needs: static-analysis
+    env:
+      DJANGO_SETTINGS_MODULE: onadata.settings.github_actions_test
+    services:
+      postgres:
+        image: postgis/postgis:13-3.0
         env:
-          DJANGO_SETTINGS_MODULE: onadata.settings.github_actions_test
+          POSTGRES_PASSWORD: onadata
+          POSTGRES_DB: onadata
+          POSTGRES_USER: onadata
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Setup Java
+        uses: actions/setup-java@v2
+        with:
+          distribution: "adopt"
+          java-version: "8"
+
+      - name: Setup python
+        uses: actions/setup-python@v3
+        with:
+          python-version: 3.9
+          architecture: "x64"
+
+      - name: Get pip cache dir
+        id: pip-cache
         run: |
-          pip install prospector==1.7.7 pylint==2.14.5
-          pip install -r requirements/azure.pip
-          prospector -X -s veryhigh onadata
+          echo "pip_cache_dir=$(pip cache dir)" >> $GITHUB_ENV
+
+      - name: Cache pip
+        uses: actions/cache@v3
+        id: cache-pip
+        with:
+          path: ${{ env.pip_cache_dir }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements/base.pip') }}-${{ hashFiles('requirements/dev.pip') }}-${{ hashFiles('requirements/azure.pip') }}
+
+      - name: Install APT requirements
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends libjpeg-dev zlib1g-dev software-properties-common ghostscript libxslt1-dev binutils libproj-dev gdal-bin memcached libmemcached-dev libxml2-dev libxslt-dev
+          sudo rm -rf /var/lib/apt/lists/*
+
+      - name: Install Pip requirements
+        run: |
+          pip install -U pip
+          pip install -r requirements/base.pip
+          pip install -r requirements/dev.pip
 
       - name: Run tests
         run: |
-          python manage.py test ${{ matrix.testfolder }} --noinput --settings=onadata.settings.github_actions_test --verbosity=2 --parallel=4
+          python manage.py test onadata/apps/api onadata/apps/logger --noinput --timing --settings=onadata.settings.github_actions_test --verbosity=2 --parallel=4
+  security-check:
+    name: Trivy Security Checks
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Get the branch name
+        id: get-branch-name
+        if: github.event_name == 'push'
+        run: echo "version=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV
+
+      - name: Build Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./docker/onadata-uwsgi
+          file: ./docker/onadata-uwsgi/Dockerfile
+          platforms: linux/amd64
+          build-args: |
+            release_version=${{ github.head_ref || github.base_ref || env.version }}
+          push: false
+          tags: |
+            onaio/onadata:${{ github.head_ref || github.base_ref || env.version }}
+          cache-from: type=registry,ref=onaio/onadata:${{ github.head_ref || github.base_ref || env.version }}
+          cache-to: type=inline
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        if: github.event_name == 'pull_request'
+        with:
+          image-ref: onaio/onadata:${{ github.head_ref || github.base_ref || env.version }}
+          format: sarif
+          ignore-unfixed: true
+          severity: 'CRITICAL,HIGH'
+          exit-code: '1'
+          output: 'trivy_results.sarif'
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        if: github.event_name == 'push'
+        with:
+          image-ref: onaio/onadata:${{ github.head_ref || github.base_ref || env.version }}
+          format: sarif
+          ignore-unfixed: true
+          output: 'trivy_results.sarif'
+
+      - name: Upload vulnerability scan results
+        uses: github/codeql-action/upload-sarif@v2
+        if: github.event_name == 'push'
+        with:
+          sarif_file: 'trivy_results.sarif'
+
+      - name: Run Trivy vulnerability for Slack summary
+        uses: aquasecurity/trivy-action@master
+        if: github.event_name == 'push'
+        with:
+          image-ref: onaio/onadata:${{ github.head_ref || github.base_ref || env.version }}
+          format: json
+          ignore-unfixed: true
+          output: 'trivy_results.json'
+
+      - name: Create summary of trivy issues
+        if: github.event_name == 'push'
+        run: |
+          summary=$(jq -r '.Results[] | select(.Vulnerabilities) | .Vulnerabilities | group_by(.Severity) | map({Severity: .[0].Severity, Count: length}) | .[] | [.Severity, .Count] | join(": ")' trivy_results.json | awk 'NR > 1 { printf(" | ") } {printf "%s",$0}')
+          if [ -z $summary ]
+          then
+            summary="0 Issues"
+          fi
+          echo "SUMMARY=$summary" >> $GITHUB_ENV
+
+      - name: Send Slack Notification
+        uses: slackapi/[email protected]
+        if: github.event_name == 'push'
+        with:
+          payload: |
+            {
+              "text": "Trivy scan results for ${{ github.head_ref || github.base_ref || env.version }}",
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "[Ona Data] Trivy scan results for ${{ github.head_ref || github.base_ref || env.version }}: ${{ env.SUMMARY }}"
+                  }
+                },
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "View scan results: https://github.com/${{ github.repository }}/security/code-scanning?query=branch:${{ github.head_ref || github.base_ref || env.version }}+is:open++"
+                  }
+                }
+              ]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
diff --git a/.github/workflows/docker-image-build.yml b/.github/workflows/docker-image-build.yml
@@ -30,12 +30,12 @@ jobs:
       - name: Get the version
         id: get-version
         if: github.event.inputs.versionTag == '' && github.event_name != 'push'
-        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+        run: echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
 
       - name: Get the branch name
         id: get-branch-name
         if: github.event_name == 'push'
-        run: echo "##[set-output name=BRANCH;]$(echo ${GITHUB_REF#refs/heads/})"
+        run: echo "version=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV
 
       - name: Login to DockerHub
         uses: docker/login-action@v1
@@ -59,11 +59,11 @@ jobs:
           file: ./docker/onadata-uwsgi/Dockerfile
           platforms: linux/amd64,linux/arm64
           build-args: |
-            release_version=${{ github.event.inputs.versionTag || steps.get-version.outputs.VERSION || steps.get-branch-name.outputs.BRANCH }}
+            release_version=${{ github.event.inputs.versionTag || env.version }}
             optional_packages=PyYAML django-redis
           push: true
           tags: |
-            onaio/onadata:${{ github.event.inputs.versionTag || steps.get-version.outputs.VERSION || steps.get-branch-name.outputs.BRANCH }}
+            onaio/onadata:${{ github.event.inputs.versionTag || env.version }}
 
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}
diff --git a/.github/workflows/ecr-image-build.yml b/.github/workflows/ecr-image-build.yml
@@ -30,12 +30,12 @@ jobs:
       - name: Get the version
         id: get-version
         if: github.event.inputs.versionTag == '' && github.event_name != 'push'
-        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+        run: echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
 
       - name: Get the branch name
         id: get-branch-name
         if: github.event_name == 'push'
-        run: echo "##[set-output name=BRANCH;]$(echo ${GITHUB_REF#refs/heads/})"
+        run: echo "version=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
@@ -67,19 +67,19 @@ jobs:
           ssh: |
             default=/tmp/ssh-agent.sock
           build-args: |
-            release_version=${{ github.event.inputs.versionTag || steps.get-version.outputs.VERSION || steps.get-branch-name.outputs.BRANCH }}
+            release_version=${{ github.event.inputs.versionTag || env.version }}
             optional_packages=PyYAML django-redis ${{ secrets.ECR_OPTIONAL_PACKAGES }}
           push: true
           tags: |
-            ${{ steps.login-ecr.outputs.registry }}/onaio/onadata:${{ github.event.inputs.versionTag || steps.get-version.outputs.VERSION || steps.get-branch-name.outputs.BRANCH }}
+            ${{ steps.login-ecr.outputs.registry }}/onaio/onadata:${{ github.event.inputs.versionTag || env.version }}
 
       - name: Image digest
         run: echo ${{ steps.docker-build.outputs.digest }}
 
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: ${{ steps.login-ecr.outputs.registry }}/onaio/onadata:${{ github.event.inputs.versionTag || steps.get-version.outputs.VERSION || steps.get-branch-name.outputs.BRANCH }}
+          image-ref: ${{ steps.login-ecr.outputs.registry }}/onaio/onadata:${{ github.event.inputs.versionTag || env.version }}
           format: 'sarif'
           output: 'trivy-results.sarif'
 
@@ -91,7 +91,7 @@ jobs:
       - name: Run Trivy vulnerability scanner for Slack
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: ${{ steps.login-ecr.outputs.registry }}/onaio/onadata:${{ github.event.inputs.versionTag || steps.get-version.outputs.VERSION || steps.get-branch-name.outputs.BRANCH }}
+          image-ref: ${{ steps.login-ecr.outputs.registry }}/onaio/onadata:${{ github.event.inputs.versionTag || env.version }}
           format: json
           output: 'trivy-results.json'
 

diff --git a/onadata/apps/api/tests/viewsets/test_data_viewset.py b/onadata/apps/api/tests/viewsets/test_data_viewset.py
@@ -16,6 +16,7 @@
 from django.core.cache import cache
 from django.db.utils import OperationalError
 from django.test import RequestFactory
+from django.test.testcases import SerializeMixin
 from django.test.utils import override_settings
 from django.utils import timezone
 
@@ -109,10 +110,11 @@ def _data_instance(dataid):
 
 
 # pylint: disable=too-many-public-methods
-class TestDataViewSet(TestBase):
+class TestDataViewSet(SerializeMixin, TestBase):
     """
     Test /data API endpoint implementation.
     """
+    lockfile = __file__
 
     def setUp(self):
         super().setUp()

diff --git a/onadata/apps/api/tests/viewsets/test_media_viewset.py b/onadata/apps/api/tests/viewsets/test_media_viewset.py
@@ -18,6 +18,9 @@ def attachment_url(attachment, suffix=None):
 
 
 class TestMediaViewSet(TestAbstractViewSet):
+    """
+    Test the /api/v1/files endpoint
+    """
     def setUp(self):
         super(TestMediaViewSet, self).setUp()
         self.retrieve_view = MediaViewSet.as_view({"get": "retrieve"})
@@ -30,7 +33,7 @@ def test_retrieve_view(self):
             "/", {"filename": self.attachment.media_file.name}, **self.extra
         )
         response = self.retrieve_view(request, self.attachment.pk)
-        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.status_code, 200, response)
         self.assertEqual(type(response.content), bytes)
 
     @patch("onadata.libs.utils.image_tools.get_storage_class")