Merge pull request #2353 from onaio/security-scan-fix

Update setuptools & futures packages

Dockerfile

@@ -0,0 +1,79 @@
+FROM ubuntu:20.04
+
+# Silence configuration prompts
+ENV DEBIAN_FRONTEND noninteractive
+ENV PYTHONUNBUFFERED 1
+
+ENV DJANGO_SETTINGS_MODULE onadata.settings.docker
+
+# Install service dependencies
+# hadolint ignore=DL3008
+RUN apt-get update -q &&\
+    apt-get install -y --no-install-recommends software-properties-common \
+        binutils \
+        libproj-dev \
+        gdal-bin \
+        memcached \
+        libmemcached-dev \
+        build-essential \
+        supervisor \
+        python3.9 \
+        python3-dev \
+        python3-pip \
+        python3-setuptools \
+        git \
+        libssl-dev \
+        libpq-dev \
+        gfortran \
+        libatlas-base-dev \
+        libjpeg-dev \
+        libxml2-dev \
+        libxslt1-dev \
+        libpython3.9-dev \
+        zlib1g-dev \
+        ghostscript \
+        python3-celery \
+        python3-sphinx \
+        pkg-config \
+        gcc \
+        automake \
+        libtool \
+        openjdk-11-jre-headless \
+        libpcre3 \
+        libpcre3-dev \
+        locales \
+        netcat && \
+    apt-get -y -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' dist-upgrade &&\
+    rm -rf /var/lib/apt/lists/*
+
+# Generate and set en_US.UTF-8 locale
+RUN locale-gen en_US.UTF-8
+ENV LC_ALL en_US.UTF-8
+ENV LC_CTYPE en_US.UTF-8
+RUN dpkg-reconfigure locales
+
+# Create OnaData user and add to tty group
+RUN useradd -G tty -m onadata
+
+# Make app directory
+RUN mkdir -p /srv/onadata && chown -R onadata:onadata /srv
+
+# Copy local codebase
+COPY . /srv/onadata
+
+# Install service requirements
+# hadolint ignore=DL3013
+RUN python3.9 -m pip install --no-cache-dir -U pip && \
+    python3.9 -m pip install --no-cache-dir -r /srv/onadata/requirements/base.pip && \
+    python3.9 -m pip install --no-cache-dir -r /srv/onadata/requirements/s3.pip && \
+    python3.9 -m pip install --no-cache-dir -r /srv/onadata/requirements/ses.pip && \
+    python3.9 -m pip install --no-cache-dir -r /srv/onadata/requirements/azure.pip && \
+    python3.9 -m pip install --no-cache-dir uwsgitop django-silk
+
+WORKDIR /srv/onadata
+
+EXPOSE 8000
+
+USER onadata
+
+CMD ["/usr/local/bin/uwsgi", "--ini", "/srv/onadata/uwsgi.ini"]

docker-compose.yml

@@ -0,0 +1,44 @@
+services:
+  api:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    depends_on:
+      - database
+      - cache
+    stdin_open: true
+    tty: true
+    user: "onadata"
+    volumes:
+      - ./:/srv/onadata
+    ports:
+      - 8000:8000
+    command: /usr/local/bin/uwsgi --ini /srv/onadata/uwsgi.ini
+
+  celery:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    depends_on:
+      - database
+      - cache
+      - api
+    volumes:
+      - ./:/srv/onadata
+    user: "onadata"
+    command: celery -A onadata.celeryapp worker -B -l INFO -E
+
+  database:
+    image: postgis/postgis:13-3.3-alpine
+    environment:
+      POSTGRES_PASSWORD: onadata
+      POSTGRES_USER: onadata
+      POSTGRES_DB: onadata
+    volumes:
+      - dbdata:/var/lib/postgresql/data
+
+  cache:
+    image: redis:alpine
+
+volumes:
+  dbdata:

docker/onadata-uwsgi/Dockerfile

@@ -13,7 +13,7 @@ RUN mkdir -m 0600 ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts
 # hadolint ignore=DL3013
 RUN --mount=type=ssh if [ -n "$optional_packages" ]; then pip install ${optional_packages} ; fi
 
-FROM ubuntu:20.04
+FROM ubuntu:focal-20221130
 COPY --from=intermediate /usr/local/lib/python3.9/site-packages/ /usr/local/lib/python3.9/dist-packages/
 
 ARG release_version=v3.7.1
@@ -61,7 +61,6 @@ RUN apt-get update -q &&\
         libpcre3-dev \
         locales \
         netcat && \
-    apt-get -y -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' dist-upgrade &&\
     rm -rf /var/lib/apt/lists/*
 
 # Generate and set en_US.UTF-8 locale
@@ -87,7 +86,8 @@ RUN python3.9 -m pip install --no-cache-dir -U pip && \
     python3.9 -m pip install --no-cache-dir -r requirements/s3.pip && \
     python3.9 -m pip install --no-cache-dir -r requirements/ses.pip && \
     python3.9 -m pip install --no-cache-dir -r requirements/azure.pip && \
-    python3.9 -m pip install --no-cache-dir pyyaml uwsgitop django-prometheus==v2.2.0
+    python3.9 -m pip install setuptools==65.5.1 && \
+    python3.9 -m pip install --no-cache-dir pyyaml uwsgitop
 
 # Compile API Docs
 RUN make -C docs html

onadata/apps/logger/tests/test_briefcase_client.py

@@ -111,7 +111,9 @@ def submission_list(request, context):
             res.status_code,
             ";".join(media),
             request.url,
-            storage.exists(os.path.join(settings.PROJECT_ROOT, "test_media/", media[0])),
+            storage.exists(
+                os.path.join(settings.PROJECT_ROOT, "test_media/", media[0])
+            ),
         )
         response.encoding = res.get("content-type")
         return get_streaming_content(res)
@@ -129,7 +131,7 @@ def get_streaming_content(res):
     return content
 
 
-@flaky(max_runs=3)
+@flaky()
 class TestBriefcaseClient(TestBase):
     """Test briefcase_client module."""
 
@@ -200,6 +202,7 @@ def test_download_xform_xml(self):
         )
         self.assertTrue(storage.exists(media_path))
 
+    @flaky(max_runs=3, min_passes=2)
     def test_push(self):
         """Test ODK briefcase client push function."""
         xforms = XForm.objects.filter(

onadata/apps/main/tests/test_process.py

@@ -21,6 +21,7 @@
 import pytz
 import requests
 from django_digest.test import Client as DigestClient
+from flaky import flaky
 from mock import patch
 from six import iteritems
 
@@ -36,6 +37,7 @@
 uuid_regex = re.compile(r'(</instance>.*uuid[^//]+="\')([^\']+)(\'".*)', re.DOTALL)
 
 
+@flaky()
 class TestProcess(TestBase, SerializeMixin):
     """
     Test form publishing processes.
@@ -153,6 +155,7 @@ def test_google_url_upload(self, mock_requests):
                 self.assertEqual(response.status_code, 200)
                 self.assertEqual(XForm.objects.count(), pre_count + 1)
 
+    @flaky(max_runs=3, min_passes=2)
     @patch("onadata.apps.main.forms.requests")
     def test_url_upload(self, mock_requests):
         """Test uploading an XLSForm from a URL."""

onadata/settings/docker.py

@@ -21,10 +21,10 @@
 DATABASES = {
     "default": {
         "ENGINE": "django.contrib.gis.db.backends.postgis",
-        "NAME": "onadata",
-        "USER": "onadata",
-        "PASSWORD": "onadata",
-        "HOST": "db",
+        "NAME": os.environ.get("DB_NAME", "onadata"),
+        "USER": os.environ.get("DB_USER", "onadata"),
+        "PASSWORD": os.environ.get("DB_PASSWORD", "onadata"),
+        "HOST": os.environ.get("DB_HOST", "database"),
         "PORT": 5432,
     }
 }
@@ -33,9 +33,9 @@
 SLAVE_DATABASES = []
 
 # Make a unique unique key just for testing, and don't share it with anybody.
-SECRET_KEY = "~&nN9d`bxmJL2[$HhYE9qAk=+4P:cf3b"  # noqa
+SECRET_KEY = os.environ.get("SECRET_KEY", "~&nN9d`bxmJL2[$HhYE9qAk=+4P:cf3b")  # noqa
 
-ALLOWED_HOSTS = ["127.0.0.1", "localhost"]
+ALLOWED_HOSTS = ["*"]
 
 INTERNAL_IPS = ["127.0.0.1"]
 
@@ -50,8 +50,9 @@
 else:
     TESTING_MODE = False
 
-CELERY_BROKER_URL = "redis://queue:6379"
-CELERY_RESULT_BACKEND = "redis://queue:6379"
+REDIS_HOST = os.environ.get("REDIS_HOST", "cache")
+CELERY_BROKER_URL = f"redis://{ REDIS_HOST }:6379"
+CELERY_RESULT_BACKEND = f"redis://{ REDIS_HOST }:6379"
 CELERY_TASK_ALWAYS_EAGER = True
 CELERY_ACCEPT_CONTENT = ["json"]
 CELERY_TASK_SERIALIZER = "json"
@@ -61,24 +62,25 @@
 CACHES = {
     "default": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://queue:6379",
+        "LOCATION": f"redis://{ REDIS_HOST }:6379",
         "OPTIONS": {"CLIENT_CLASS": "django_redis.client.DefaultClient"},
     }
 }
 
-NOTIFICATION_BACKENDS = {
-    "mqtt": {
-        "BACKEND": "onadata.apps.messaging.backends.mqtt.MQTTBackend",
-        "OPTIONS": {
-            "HOST": "notifications",
-            "PORT": 1883,
-            "QOS": 1,
-            "RETAIN": False,
-            "SECURE": False,
-            "TOPIC_BASE": "onadata",
-        },
-    }
-}
+NOTIFICATION_BACKENDS = {}
+# NOTIFICATION_BACKENDS = {
+#    "mqtt": {
+#        "BACKEND": "onadata.apps.messaging.backends.mqtt.MQTTBackend",
+#        "OPTIONS": {
+#            "HOST": "notifications",
+#            "PORT": 1883,
+#            "QOS": 1,
+#            "RETAIN": False,
+#            "SECURE": False,
+#            "TOPIC_BASE": "onadata",
+#        },
+#    }
+# }
 FULL_MESSAGE_PAYLOAD = True
 
 EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"

requirements/base.pip

@@ -180,7 +180,7 @@ et-xmlfile==1.1.0
     # via openpyxl
 fleming==0.7.0
     # via django-query-builder
-future==0.18.2
+future==0.18.3
     # via python-json2xlsclient
 geojson==2.5.0
     # via onadata

requirements/dev.pip

@@ -211,7 +211,7 @@ flaky==3.7.0
     # via -r requirements/dev.in
 fleming==0.7.0
     # via django-query-builder
-future==0.18.2
+future==0.18.3
     # via python-json2xlsclient
 geojson==2.5.0
     # via onadata

uwsgi.ini

@@ -1,9 +1,27 @@
 [uwsgi]
-http=:3030
-module=onadata.apps.main.wsgi:application
-master=True
-processes=2
-vacuum=True                 # clear environment on exit
-max-requests=50          # respawn processes after serving 50 requests
-static-map=/static=onadata/static
-stats=/tmp/onadata-stats.sock
+uid = onadata
+gid = onadata
+
+http           = :8000
+stats          = :8001
+socket-timeout = 360
+http-timeout   = 360
+http-keepalive = true
+post-buffering = 113
+max-requests   = 300
+
+# Set to 0 or remove when not in development env
+py-autoreload           = 1
+honour-stdin            = true
+master                  = true
+vacuum                  = true
+enable-threads          = true
+ignore-sigpipe          = true
+ignore-write-errors     = true
+disable-write-exception = true
+workers                 = 2
+threads                 = 15
+
+chdir      = /srv/onadata
+module     = onadata.apps.main.wsgi:application
+static-map = /static=/srv/onadata/onadata/static