From af66c0f338c97d8563d9eaf63a265d92519b6069 Mon Sep 17 00:00:00 2001 From: Davis Muro Date: Wed, 11 Jan 2023 14:06:43 +0300 Subject: [PATCH 1/4] Remove `dist-upgrade` from Dockerfile & Update setuptools See https://avd.aquasec.com/misconfig/ds024 See https://avd.aquasec.com/nvd/2022/cve-2022-40897/ --- docker/onadata-uwsgi/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/onadata-uwsgi/Dockerfile b/docker/onadata-uwsgi/Dockerfile index 64efbad8b6..be19667094 100644 --- a/docker/onadata-uwsgi/Dockerfile +++ b/docker/onadata-uwsgi/Dockerfile @@ -13,7 +13,7 @@ RUN mkdir -m 0600 ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts # hadolint ignore=DL3013 RUN --mount=type=ssh if [ -n "$optional_packages" ]; then pip install ${optional_packages} ; fi -FROM ubuntu:20.04 +FROM ubuntu:focal-20221130 COPY --from=intermediate /usr/local/lib/python3.9/site-packages/ /usr/local/lib/python3.9/dist-packages/ ARG release_version=v3.7.1 @@ -61,7 +61,6 @@ RUN apt-get update -q &&\ libpcre3-dev \ locales \ netcat && \ - apt-get -y -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' dist-upgrade &&\ rm -rf /var/lib/apt/lists/* # Generate and set en_US.UTF-8 locale @@ -87,7 +86,8 @@ RUN python3.9 -m pip install --no-cache-dir -U pip && \ python3.9 -m pip install --no-cache-dir -r requirements/s3.pip && \ python3.9 -m pip install --no-cache-dir -r requirements/ses.pip && \ python3.9 -m pip install --no-cache-dir -r requirements/azure.pip && \ - python3.9 -m pip install --no-cache-dir pyyaml uwsgitop django-prometheus==v2.2.0 + python3.9 -m pip install setuptools==65.5.1 && \ + python3.9 -m pip install --no-cache-dir pyyaml uwsgitop # Compile API Docs RUN make -C docs html From 5c0c2d076bbbd3b9ad9f915df95155d1cc0c1f06 Mon Sep 17 00:00:00 2001 From: Davis Muro Date: Sat, 7 Jan 2023 20:30:19 +0300 Subject: [PATCH 2/4] Add development docker-compose file --- Dockerfile | 79 ++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 44 +++++++++++++++++++++ onadata/settings/docker.py | 46 +++++++++++----------- uwsgi.ini | 34 ++++++++++++---- 4 files changed, 173 insertions(+), 30 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..ca745b5c25 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,79 @@ +FROM ubuntu:20.04 + +# Silence configuration prompts +ENV DEBIAN_FRONTEND noninteractive +ENV PYTHONUNBUFFERED 1 + +ENV DJANGO_SETTINGS_MODULE onadata.settings.docker + +# Install service dependencies +# hadolint ignore=DL3008 +RUN apt-get update -q &&\ + apt-get install -y --no-install-recommends software-properties-common \ + binutils \ + libproj-dev \ + gdal-bin \ + memcached \ + libmemcached-dev \ + build-essential \ + supervisor \ + python3.9 \ + python3-dev \ + python3-pip \ + python3-setuptools \ + git \ + libssl-dev \ + libpq-dev \ + gfortran \ + libatlas-base-dev \ + libjpeg-dev \ + libxml2-dev \ + libxslt1-dev \ + libpython3.9-dev \ + zlib1g-dev \ + ghostscript \ + python3-celery \ + python3-sphinx \ + pkg-config \ + gcc \ + automake \ + libtool \ + openjdk-11-jre-headless \ + libpcre3 \ + libpcre3-dev \ + locales \ + netcat && \ + apt-get -y -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' dist-upgrade &&\ + rm -rf /var/lib/apt/lists/* + +# Generate and set en_US.UTF-8 locale +RUN locale-gen en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 +ENV LC_CTYPE en_US.UTF-8 +RUN dpkg-reconfigure locales + +# Create OnaData user and add to tty group +RUN useradd -G tty -m onadata + +# Make app directory +RUN mkdir -p /srv/onadata && chown -R onadata:onadata /srv + +# Copy local codebase +COPY . /srv/onadata + +# Install service requirements +# hadolint ignore=DL3013 +RUN python3.9 -m pip install --no-cache-dir -U pip && \ + python3.9 -m pip install --no-cache-dir -r /srv/onadata/requirements/base.pip && \ + python3.9 -m pip install --no-cache-dir -r /srv/onadata/requirements/s3.pip && \ + python3.9 -m pip install --no-cache-dir -r /srv/onadata/requirements/ses.pip && \ + python3.9 -m pip install --no-cache-dir -r /srv/onadata/requirements/azure.pip && \ + python3.9 -m pip install --no-cache-dir uwsgitop django-silk + +WORKDIR /srv/onadata + +EXPOSE 8000 + +USER onadata + +CMD ["/usr/local/bin/uwsgi", "--ini", "/srv/onadata/uwsgi.ini"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000..5a295c925f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,44 @@ +services: + api: + build: + context: . + dockerfile: Dockerfile + depends_on: + - database + - cache + stdin_open: true + tty: true + user: "onadata" + volumes: + - ./:/srv/onadata + ports: + - 8000:8000 + command: /usr/local/bin/uwsgi --ini /srv/onadata/uwsgi.ini + + celery: + build: + context: . + dockerfile: Dockerfile + depends_on: + - database + - cache + - api + volumes: + - ./:/srv/onadata + user: "onadata" + command: celery -A onadata.celeryapp worker -B -l INFO -E + + database: + image: postgis/postgis:13-3.3-alpine + environment: + POSTGRES_PASSWORD: onadata + POSTGRES_USER: onadata + POSTGRES_DB: onadata + volumes: + - dbdata:/var/lib/postgresql/data + + cache: + image: redis:alpine + +volumes: + dbdata: diff --git a/onadata/settings/docker.py b/onadata/settings/docker.py index b53ea61909..2f1f72652b 100644 --- a/onadata/settings/docker.py +++ b/onadata/settings/docker.py @@ -21,10 +21,10 @@ DATABASES = { "default": { "ENGINE": "django.contrib.gis.db.backends.postgis", - "NAME": "onadata", - "USER": "onadata", - "PASSWORD": "onadata", - "HOST": "db", + "NAME": os.environ.get("DB_NAME", "onadata"), + "USER": os.environ.get("DB_USER", "onadata"), + "PASSWORD": os.environ.get("DB_PASSWORD", "onadata"), + "HOST": os.environ.get("DB_HOST", "database"), "PORT": 5432, } } @@ -33,9 +33,9 @@ SLAVE_DATABASES = [] # Make a unique unique key just for testing, and don't share it with anybody. -SECRET_KEY = "~&nN9d`bxmJL2[$HhYE9qAk=+4P:cf3b" # noqa +SECRET_KEY = os.environ.get("SECRET_KEY", "~&nN9d`bxmJL2[$HhYE9qAk=+4P:cf3b") # noqa -ALLOWED_HOSTS = ["127.0.0.1", "localhost"] +ALLOWED_HOSTS = ["*"] INTERNAL_IPS = ["127.0.0.1"] @@ -50,8 +50,9 @@ else: TESTING_MODE = False -CELERY_BROKER_URL = "redis://queue:6379" -CELERY_RESULT_BACKEND = "redis://queue:6379" +REDIS_HOST = os.environ.get("REDIS_HOST", "cache") +CELERY_BROKER_URL = f"redis://{ REDIS_HOST }:6379" +CELERY_RESULT_BACKEND = f"redis://{ REDIS_HOST }:6379" CELERY_TASK_ALWAYS_EAGER = True CELERY_ACCEPT_CONTENT = ["json"] CELERY_TASK_SERIALIZER = "json" @@ -61,24 +62,25 @@ CACHES = { "default": { "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://queue:6379", + "LOCATION": f"redis://{ REDIS_HOST }:6379", "OPTIONS": {"CLIENT_CLASS": "django_redis.client.DefaultClient"}, } } -NOTIFICATION_BACKENDS = { - "mqtt": { - "BACKEND": "onadata.apps.messaging.backends.mqtt.MQTTBackend", - "OPTIONS": { - "HOST": "notifications", - "PORT": 1883, - "QOS": 1, - "RETAIN": False, - "SECURE": False, - "TOPIC_BASE": "onadata", - }, - } -} +NOTIFICATION_BACKENDS = {} +# NOTIFICATION_BACKENDS = { +# "mqtt": { +# "BACKEND": "onadata.apps.messaging.backends.mqtt.MQTTBackend", +# "OPTIONS": { +# "HOST": "notifications", +# "PORT": 1883, +# "QOS": 1, +# "RETAIN": False, +# "SECURE": False, +# "TOPIC_BASE": "onadata", +# }, +# } +# } FULL_MESSAGE_PAYLOAD = True EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" diff --git a/uwsgi.ini b/uwsgi.ini index 8ad1f0b8d0..16102da5ff 100644 --- a/uwsgi.ini +++ b/uwsgi.ini @@ -1,9 +1,27 @@ [uwsgi] -http=:3030 -module=onadata.apps.main.wsgi:application -master=True -processes=2 -vacuum=True # clear environment on exit -max-requests=50 # respawn processes after serving 50 requests -static-map=/static=onadata/static -stats=/tmp/onadata-stats.sock +uid = onadata +gid = onadata + +http = :8000 +stats = :8001 +socket-timeout = 360 +http-timeout = 360 +http-keepalive = true +post-buffering = 113 +max-requests = 300 + +# Set to 0 or remove when not in development env +py-autoreload = 1 +honour-stdin = true +master = true +vacuum = true +enable-threads = true +ignore-sigpipe = true +ignore-write-errors = true +disable-write-exception = true +workers = 2 +threads = 15 + +chdir = /srv/onadata +module = onadata.apps.main.wsgi:application +static-map = /static=/srv/onadata/onadata/static From 9c094c46c457b4f6ef9b98b666e79fa04a0c130d Mon Sep 17 00:00:00 2001 From: Davis Muro Date: Tue, 17 Jan 2023 13:35:52 +0300 Subject: [PATCH 3/4] Bump future to `0.18.3` --- requirements/base.pip | 2 +- requirements/dev.pip | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/base.pip b/requirements/base.pip index f9b1b9b8e5..cc077127fe 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -180,7 +180,7 @@ et-xmlfile==1.1.0 # via openpyxl fleming==0.7.0 # via django-query-builder -future==0.18.2 +future==0.18.3 # via python-json2xlsclient geojson==2.5.0 # via onadata diff --git a/requirements/dev.pip b/requirements/dev.pip index 17e776019e..9933476061 100644 --- a/requirements/dev.pip +++ b/requirements/dev.pip @@ -211,7 +211,7 @@ flaky==3.7.0 # via -r requirements/dev.in fleming==0.7.0 # via django-query-builder -future==0.18.2 +future==0.18.3 # via python-json2xlsclient geojson==2.5.0 # via onadata From d48fbe4dc1576d6a31fc29fb662fddb039fab155 Mon Sep 17 00:00:00 2001 From: Davis Muro Date: Wed, 18 Jan 2023 10:54:38 +0300 Subject: [PATCH 4/4] Re-run flaky tests and ensure they pass at least 2 times --- onadata/apps/logger/tests/test_briefcase_client.py | 7 +++++-- onadata/apps/main/tests/test_process.py | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/onadata/apps/logger/tests/test_briefcase_client.py b/onadata/apps/logger/tests/test_briefcase_client.py index 945aeba4e4..c3ddfe6d95 100644 --- a/onadata/apps/logger/tests/test_briefcase_client.py +++ b/onadata/apps/logger/tests/test_briefcase_client.py @@ -111,7 +111,9 @@ def submission_list(request, context): res.status_code, ";".join(media), request.url, - storage.exists(os.path.join(settings.PROJECT_ROOT, "test_media/", media[0])), + storage.exists( + os.path.join(settings.PROJECT_ROOT, "test_media/", media[0]) + ), ) response.encoding = res.get("content-type") return get_streaming_content(res) @@ -129,7 +131,7 @@ def get_streaming_content(res): return content -@flaky(max_runs=3) +@flaky() class TestBriefcaseClient(TestBase): """Test briefcase_client module.""" @@ -200,6 +202,7 @@ def test_download_xform_xml(self): ) self.assertTrue(storage.exists(media_path)) + @flaky(max_runs=3, min_passes=2) def test_push(self): """Test ODK briefcase client push function.""" xforms = XForm.objects.filter( diff --git a/onadata/apps/main/tests/test_process.py b/onadata/apps/main/tests/test_process.py index a01bbbbf4f..2ab8df2f26 100644 --- a/onadata/apps/main/tests/test_process.py +++ b/onadata/apps/main/tests/test_process.py @@ -21,6 +21,7 @@ import pytz import requests from django_digest.test import Client as DigestClient +from flaky import flaky from mock import patch from six import iteritems @@ -36,6 +37,7 @@ uuid_regex = re.compile(r'(.*uuid[^//]+="\')([^\']+)(\'".*)', re.DOTALL) +@flaky() class TestProcess(TestBase, SerializeMixin): """ Test form publishing processes. @@ -153,6 +155,7 @@ def test_google_url_upload(self, mock_requests): self.assertEqual(response.status_code, 200) self.assertEqual(XForm.objects.count(), pre_count + 1) + @flaky(max_runs=3, min_passes=2) @patch("onadata.apps.main.forms.requests") def test_url_upload(self, mock_requests): """Test uploading an XLSForm from a URL."""