Merge pull request #22 from kartoza/feat-spw

Feat SPW model integration

.devcontainer/devcontainer.json

@@ -7,7 +7,7 @@
       "../deployment/docker-compose.override.devcontainer.yml"
     ],
     "service": "dev",
-    "runServices": ["db", "minio", "redis", "celery_beat", "worker", "dev"],
+    "runServices": ["db", "minio", "plumber", "redis", "celery_beat", "worker", "dev"],
     "workspaceFolder": "/home/web/project",
     "runArgs": [
       "--env-file",
@@ -23,7 +23,7 @@
     "shutdownAction": "stopCompose",
     "customizations": {
       "vscode": {
-        "extensions": ["ms-python.python", "ms-azuretools.vscode-docker"],
+        "extensions": ["ms-python.python", "ms-azuretools.vscode-docker", "njpwerner.autodocstring"],
         "settings": {
           "terminal.integrated.shell.linux": "/bin/bash",
           "python.pythonPath": "/usr/local/bin/python",

codecov.yml

@@ -9,7 +9,7 @@ coverage:
     changes: false
     patch:
       default:
-        threshold: "5%"
+        threshold: "10%"
   ignore:
     - "**/migrations/*.py"
     - "django_project/core/settings/*.py"

deployment/.template.env

@@ -16,9 +16,9 @@ DATABASE_PORT=5432
 REDIS_HOST=redis
 REDIS_PASSWORD=redis_password
 
-RABBITMQ_HOST=rabbitmq
 SENTRY_DSN=
 INITIAL_FIXTURES=
+PLUMBER_PORT=8282
 
 # Minio Variables for django default storages
 MINIO_AWS_ACCESS_KEY_ID=minio_tomorrownow
@@ -42,3 +42,6 @@ SALIENT_AWS_ENDPOINT_URL=
 SALIENT_AWS_BUCKET_NAME=tomorrownow
 SALIENT_AWS_DIR_PREFIX=
 SALIENT_AWS_REGION_NAME=
+
+# API keys
+TOMORROW_IO_API_KEY=

deployment/docker-compose.override.devcontainer.yml

@@ -48,6 +48,7 @@ services:
     links:
       - db
       - worker
+      - plumber
 
   minio:
     image: quay.io/minio/minio:RELEASE.2024-03-30T09-41-56Z.fips
@@ -62,3 +63,18 @@ services:
     volumes:
       - ./volumes/minio_data:/data
     restart: always
+
+  plumber:
+    build:
+      context: ../
+      dockerfile: deployment/plumber/Dockerfile
+    env_file:
+      - .env
+    volumes:
+      - ../:/home/web/project
+      - ../django_project:/home/web/django_project
+    ports:
+      - "8282:8282"
+    links:
+      - db
+      - redis

deployment/docker-compose.override.template.yml

@@ -62,6 +62,18 @@ services:
       - celery_beat
     entrypoint: [ ]
 
+  plumber:
+    build:
+      context: ../
+      dockerfile: deployment/plumber/Dockerfile
+    env_file:
+      - .env
+    volumes:
+      - ../django_project:/home/web/django_project
+    links:
+      - db
+      - redis
+
   minio:
     image: quay.io/minio/minio:RELEASE.2024-03-30T09-41-56Z.fips
     command: minio server /data --console-address ":9001"

deployment/docker-compose.test.yml

@@ -28,7 +28,6 @@ services:
       - DATABASE_USERNAME=docker
       - DATABASE_PASSWORD=docker
       - DATABASE_HOST=db
-      - RABBITMQ_HOST=rabbitmq
       - DJANGO_SETTINGS_MODULE=core.settings.test
       - SECRET_KEY=SECRET_KEY
 

deployment/docker-compose.yml

@@ -54,6 +54,7 @@ services:
     links:
       - db
       - worker
+      - plumber
 
   worker:
     <<: *default-common-django
@@ -63,6 +64,7 @@ services:
       - db
       - redis
       - celery_beat
+      - plumber
 
   celery_beat:
     <<: *default-common-django
@@ -81,3 +83,13 @@ services:
       - nginx-cache:/home/web/nginx_cache
     links:
       - django
+
+  plumber:
+    build:
+      context: ../
+      dockerfile: deployment/plumber/Dockerfile
+    env_file:
+      - .env
+    links:
+      - db
+      - redis

deployment/docker/requirements-dev.txt

@@ -16,4 +16,6 @@ pdbpp
 responses
 mock==4.0.3
 
-pytest-django
+pytest-django
+# mock requests
+requests-mock

deployment/plumber/Dockerfile

@@ -0,0 +1,42 @@
+FROM rstudio/plumber:v1.2.0 AS prod
+
+# install python 3.10
+RUN apt-get update -y && apt-get upgrade -y
+
+RUN apt install software-properties-common -y
+
+RUN add-apt-repository ppa:deadsnakes/ppa
+
+RUN apt install Python3.10
+
+RUN apt-get install -y --no-install-recommends \
+    gcc gettext cron \
+    spatialite-bin libsqlite3-mod-spatialite \
+    python3-dev python3-gdal python3-psycopg2 python3-ldap \
+    python3-pip python3-pil python3-lxml python3-pylibmc \
+    uwsgi uwsgi-plugin-python3 \
+    libfreetype6-dev libpng-dev libtiff5-dev libjpeg-dev
+
+# install R packages
+RUN install2.r --error tidyverse tidygam mgcv ggpubr classInt zoo
+
+# Install pip packages
+ADD deployment/docker/requirements.txt /requirements.txt
+RUN pip3 install --upgrade pip && pip install --upgrade pip
+# Fix uwsgi build failure missing cc1
+ARG CPUCOUNT=1
+RUN pip3 install -r /requirements.txt
+
+# add django project
+ADD django_project /home/web/django_project
+
+# create directories
+RUN mkdir -p /home/web/plumber_data/
+
+# add entrypoint.sh
+ADD django_project/plumber_entrypoint.sh /home/web/plumber_entrypoint.sh
+
+EXPOSE 8181
+
+WORKDIR /home/web/django_project
+ENTRYPOINT [ "/home/web/plumber_entrypoint.sh" ]

django_project/core/settings/project.py

@@ -32,7 +32,8 @@
     'core',
     'frontend',
     'gap',
-    'gap_api'
+    'gap_api',
+    'spw'
 )
 
 TEMPLATES[0]['DIRS'] += [

django_project/gap/admin.py

@@ -67,7 +67,8 @@ class DatasetAdmin(admin.ModelAdmin):
     """Dataset admin."""
 
     list_display = (
-        'name', 'provider', 'type', 'time_step', 'store_type',
+        'name', 'provider', 'type', 'time_step',
+        'store_type', 'is_internal_use'
     )
 
 

django_project/gap/providers/__init__.py

@@ -10,6 +10,10 @@
 from gap.providers.cbam import CBAMNetCDFReader
 from gap.providers.salient import SalientNetCDFReader
 from gap.providers.tahmo import TahmoDatasetReader
+from gap.providers.tio import (
+    TomorrowIODatasetReader,
+    PROVIDER_NAME as TIO_PROVIDER
+)
 
 
 def get_reader_from_dataset(dataset: Dataset):
@@ -27,6 +31,8 @@ def get_reader_from_dataset(dataset: Dataset):
         return SalientNetCDFReader
     elif dataset.provider.name == 'Tahmo':
         return TahmoDatasetReader
+    elif dataset.provider.name == TIO_PROVIDER:
+        return TomorrowIODatasetReader
     else:
         raise TypeError(
             f'Unsupported provider name: {dataset.provider.name}'

django_project/gap/providers/cbam.py

@@ -30,7 +30,6 @@
 )
 
 
-
 class CBAMNetCDFReader(BaseNetCDFReader):
     """Class to read NetCDF file from CBAM provider."""
 

django_project/gap/providers/tahmo.py

@@ -2,7 +2,7 @@
 """
 Tomorrow Now GAP.
 
-.. note:: CBAM Data Reader
+.. note:: Tahmo Data Reader
 """
 
 from typing import List