Merge pull request #46 from kartoza/feat-storages-minio

Feat storage minio

.github/workflows/tests.yaml

@@ -101,7 +101,7 @@ jobs:
         run: |
           cat << EOF | docker-compose exec -T dev bash
           python manage.py collectstatic --noinput --verbosity 0
-          export DJANGO_SETTINGS_MODULE=core.settings.dev && coverage run manage.py test && coverage xml
+          export DJANGO_SETTINGS_MODULE=core.settings.test && coverage run manage.py test && coverage xml
           EOF
           docker cp dev:/home/web/django_project/coverage.xml ../coverage.xml
       - name: Show Coverage

deployment/.template.env

@@ -20,11 +20,12 @@ RABBITMQ_HOST=rabbitmq
 SENTRY_DSN=
 INITIAL_FIXTURES=
 
-# Minio Dev Variables
+# Minio Variables for django default storages
 MINIO_AWS_ACCESS_KEY_ID=minio_tomorrownow
 MINIO_AWS_SECRET_ACCESS_KEY=minio_tomorrownow
 MINIO_AWS_ENDPOINT_URL=http://minio:9000/
 MINIO_AWS_BUCKET_NAME=tomorrownow
+MINIO_AWS_DIR_PREFIX=dev/media
 
 # CBAM S3 Variables
 CBAM_AWS_ACCESS_KEY_ID=minio_tomorrownow

deployment/docker-compose.override.devcontainer.yml

@@ -21,7 +21,6 @@ services:
       - ../:/home/web/project
       - ../django_project:/home/web/django_project
       - ./volumes/static:/home/web/static
-      - ./volumes/media:/home/web/media
 
   celery_beat:
     image: kartoza/${COMPOSE_PROJECT_NAME:-django_project}_dev
@@ -33,7 +32,6 @@ services:
       - ../:/home/web/project
       - ../django_project:/home/web/django_project
       - ./volumes/static:/home/web/static
-      - ./volumes/media:/home/web/media
 
   dev:
     image: kartoza/${COMPOSE_PROJECT_NAME:-django_project}_dev
@@ -47,7 +45,6 @@ services:
     volumes:
       - ../:/home/web/project
       - ./volumes/static:/home/web/static
-      - ./volumes/media:/home/web/media
     links:
       - db
       - worker

deployment/docker-compose.override.template.yml

@@ -15,7 +15,6 @@ services:
     volumes:
       - ../django_project:/home/web/django_project
       - ./volumes/static:/home/web/static
-      - ./volumes/media:/home/web/media
 
   worker:
     build:
@@ -24,7 +23,6 @@ services:
     volumes:
       - ../django_project:/home/web/django_project
       - ./volumes/static:/home/web/static
-      - ./volumes/media:/home/web/media
 
   celery_beat:
     build:
@@ -33,13 +31,11 @@ services:
     volumes:
       - ../django_project:/home/web/django_project
       - ./volumes/static:/home/web/static
-      - ./volumes/media:/home/web/media
 
   nginx:
     volumes:
       - ./nginx/sites-enabled:/etc/nginx/conf.d:ro
       - ./volumes/static:/home/web/static
-      - ./volumes/media:/home/web/media
     ports:
       - "${HTTP_PORT:-8888}:80"
     links:
@@ -54,7 +50,6 @@ services:
     volumes:
       - ../django_project:/home/web/django_project
       - ./volumes/static:/home/web/static
-      - ./volumes/media:/home/web/media
     ports:
       # for django test server
       - "5000:8080"

deployment/docker-compose.yml

@@ -2,7 +2,6 @@ version: '3.9'
 
 volumes:
   static-data:
-  media-data:
   conf-data:
   database:
   nginx-cache:
@@ -17,7 +16,6 @@ x-common-django:
     - .env
   volumes:
     - static-data:/home/web/static
-    - media-data:/home/web/media
   restart: on-failure
 
 services:
@@ -53,7 +51,6 @@ services:
     command: 'uwsgi --ini /uwsgi.conf'
     volumes:
       - static-data:/home/web/static
-      - media-data:/home/web/media
     links:
       - db
       - worker
@@ -81,7 +78,6 @@ services:
     volumes:
       - conf-data:/etc/nginx/conf.d:ro
       - static-data:/home/web/static
-      - media-data:/home/web/media
       - nginx-cache:/home/web/nginx_cache
     links:
       - django

deployment/docker/requirements.txt

@@ -33,6 +33,10 @@ django-celery-results==2.5.0
 # This extension enables you to store the periodic task schedule in the database.
 django-celery-beat==2.5.0
 
+#django storages
+django-storages==1.14.2
+django-storages[s3]
+
 # Python client for Redis database and key-value store
 redis==4.3.4
 

deployment/nginx/sites-enabled/default.conf

@@ -43,15 +43,6 @@ server {
         return 410;
     }
 
-    # Django media
-    location /media  {
-        # your Django project's media files - amend as required
-        add_header 'Access-Control-Allow-Origin' '*' always;
-
-        alias /home/web/media;
-        expires 21d; # cache for 71 days
-    }
-
     location /static {
         # your Django project's static files - amend as required
         alias /home/web/static;

django_project/core/settings/base.py

@@ -135,9 +135,6 @@
 )
 
 SITE_ID = 1
-STATICFILES_STORAGE = (
-    'django.contrib.staticfiles.storage.ManifestStaticFilesStorage'
-)
 
 CACHES = {
     'default': {

django_project/core/settings/project.py

@@ -6,6 +6,7 @@
 """
 import os  # noqa
 
+from boto3.s3.transfer import TransferConfig
 from .contrib import *  # noqa
 from .utils import absolute_path
 
@@ -37,3 +38,33 @@
 TEMPLATES[0]['DIRS'] += [
     absolute_path('frontend', 'templates'),
 ]
+
+MB = 1024 ** 2
+AWS_TRANSFER_CONFIG = TransferConfig(
+    multipart_chunksize=512 * MB,
+    use_threads=True,
+    max_concurrency=10
+)
+STORAGES = {
+    "default": {
+        "BACKEND": "storages.backends.s3.S3Storage",
+        "OPTIONS": {
+            "access_key": os.environ.get("MINIO_AWS_ACCESS_KEY_ID"),
+            "secret_key": os.environ.get("MINIO_AWS_SECRET_ACCESS_KEY"),
+            "bucket_name": os.environ.get("MINIO_AWS_BUCKET_NAME"),
+            "file_overwrite": False,
+            "max_memory_size": 300 * MB,  # 300MB
+            "transfer_config": AWS_TRANSFER_CONFIG,
+            "endpoint_url": os.environ.get("MINIO_AWS_ENDPOINT_URL")
+        },
+    },
+    "staticfiles": {
+        "BACKEND": (
+            "django.contrib.staticfiles.storage.ManifestStaticFilesStorage"
+        )
+    },
+}
+
+STORAGE_DIR_PREFIX = os.environ.get("MINIO_AWS_DIR_PREFIX", "media")
+if STORAGE_DIR_PREFIX and not STORAGE_DIR_PREFIX.endswith("/"):
+    STORAGE_DIR_PREFIX = f"{STORAGE_DIR_PREFIX}/"

django_project/core/settings/test.py

@@ -16,3 +16,17 @@
         'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
     }
 }
+
+STORAGES = {
+    "default": {
+        "BACKEND": "django.core.files.storage.FileSystemStorage",
+        "OPTIONS": {
+            "location": "/home/web/media/default_test",
+        },
+    },
+    "staticfiles": {
+        "BACKEND": (
+            "django.contrib.staticfiles.storage.ManifestStaticFilesStorage"
+        ),
+    }
+}

django_project/gap/factories.py

@@ -138,7 +138,7 @@ class Meta:  # noqa
         model = Country
 
     name = factory.Faker('country')
-    iso_a3 = factory.Faker('country_code')
+    iso_a3 = factory.Faker('text')
     geometry = factory.LazyAttribute(
         lambda _: MultiPolygon(
             Polygon(((0, 0), (1, 0), (1, 1), (0, 1), (0, 0)))

django_project/gap/ingestor/tahmo.py

@@ -9,6 +9,8 @@
 import json
 import os
 import shutil
+import uuid
+import tempfile
 from datetime import datetime, timezone
 from zipfile import ZipFile
 
@@ -220,14 +222,22 @@ def run(self):
             raise FileNotFoundException()
 
         # Extract file
-        dir_path = os.path.splitext(self.session.file.path)[0]
-        with ZipFile(self.session.file.path, 'r') as zip_ref:
+        dir_path = f'/tmp/{str(uuid.uuid4())}'
+        os.makedirs(dir_path, exist_ok=True)
+        with self.session.file.open('rb') as zip_file:
+            # Create a NamedTemporaryFile to store the downloaded file
+            with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
+                tmp_file.write(zip_file.read())
+                tmp_file_path = tmp_file.name
+        with ZipFile(tmp_file_path, 'r') as zip_ref:
             zip_ref.extractall(dir_path)
 
         # Run the ingestion
         try:
             self._run(dir_path)
-            shutil.rmtree(dir_path)
         except Exception as e:
-            shutil.rmtree(dir_path)
             raise Exception(e)
+        finally:
+            shutil.rmtree(dir_path)
+            if tmp_file_path and os.path.exists(tmp_file_path):
+                os.remove(tmp_file_path)

django_project/gap/migrations/0001_initial.py

@@ -1,8 +1,9 @@
-# Generated by Django 4.2.7 on 2024-07-17 10:21
+# Generated by Django 4.2.7 on 2024-07-19 04:05
 
 import django.contrib.gis.db.models.fields
 from django.db import migrations, models
 import django.db.models.deletion
+import gap.models.ingestor
 
 
 class Migration(migrations.Migration):
@@ -70,7 +71,7 @@ class Migration(migrations.Migration):
             fields=[
                 ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('ingestor_type', models.CharField(choices=[('Tahmo', 'Tahmo')], default='Tahmo', max_length=512)),
-                ('file', models.FileField(blank=True, null=True, upload_to='ingestors/')),
+                ('file', models.FileField(blank=True, null=True, upload_to=gap.models.ingestor.ingestor_file_path)),
                 ('status', models.CharField(choices=[('RUNNING', 'RUNNING'), ('SUCCESS', 'SUCCESS'), ('FAILED', 'FAILED')], default='RUNNING', max_length=512)),
                 ('notes', models.TextField(blank=True, null=True)),
                 ('run_at', models.DateTimeField(auto_now_add=True)),

django_project/gap/models/ingestor.py

@@ -5,16 +5,20 @@
 .. note:: Models
 """
 
-import os
-
 from django.contrib.auth import get_user_model
 from django.db import models
-from django.dispatch import receiver
 from django.utils import timezone
+from django.conf import settings
+
 
 User = get_user_model()
 
 
+def ingestor_file_path(instance, filename):
+    """Return upload path for Ingestor files."""
+    return f'{settings.STORAGE_DIR_PREFIX}ingestors/{filename}'
+
+
 class IngestorType:
     """Ingestor type."""
 
@@ -44,7 +48,7 @@ class IngestorSession(models.Model):
         max_length=512
     )
     file = models.FileField(
-        upload_to='ingestors/',
+        upload_to=ingestor_file_path,
         null=True, blank=True
     )
     status = models.CharField(
@@ -115,38 +119,3 @@ class IngestorSessionProgress(models.Model):
     notes = models.TextField(
         blank=True, null=True
     )
-
-
-@receiver(models.signals.post_delete, sender=IngestorSession)
-def auto_delete_file_on_delete(sender, instance, **kwargs):
-    """Delete file from filesystem.
-
-    when corresponding `IngestorSession` object is deleted.
-    """
-    if instance.file:
-        if os.path.isfile(instance.file.path):
-            os.remove(instance.file.path)
-
-
-@receiver(models.signals.pre_save, sender=IngestorSession)
-def auto_delete_file_on_change(sender, instance, **kwargs):
-    """Delete old file from filesystem.
-
-    when corresponding `IngestorSession` object is updated
-    with new file.
-    """
-    if not instance.pk:
-        return False
-
-    try:
-        old_file = IngestorSession.objects.get(pk=instance.pk).file
-    except IngestorSession.DoesNotExist:
-        return False
-
-    new_file = instance.file
-    if not old_file == new_file:
-        try:
-            if os.path.isfile(old_file.path):
-                os.remove(old_file.path)
-        except ValueError:
-            pass