Merge pull request #895 from kobotoolbox/daily-counter-null-xform

Create null xform instances for DailyXFormSubmissionCounter

onadata/apps/logger/management/commands/populate_submission_counters.py

@@ -163,6 +163,7 @@ def build_counters(self, xf: 'logger.XForm') -> tuple[list, dict]:
             submission_date = values['date_created__date']
             daily_counters.append(DailyXFormSubmissionCounter(
                 xform_id=xf.pk,
+                user=xf.user,
                 date=submission_date,
                 counter=values['num_of_submissions'],
             ))

onadata/apps/logger/migrations/0028_add_user_to_daily_submission_counters.py

@@ -0,0 +1,68 @@
+from django.conf import settings
+from django.db import migrations, models
+from django.db.models import Subquery, deletion, OuterRef
+
+
+def add_user_to_daily_submission_counter(apps, schema_editor):
+    DailyXFormSubmissionCounter = apps.get_model('logger', 'DailyXFormSubmissionCounter')
+    # delete any counters where xform and user are None, since we can't associate them with a user
+    DailyXFormSubmissionCounter.objects.filter(xform=None, user=None).delete()
+    # add the user to the counter, based on the xform user
+    DailyXFormSubmissionCounter.objects.all().exclude(xform=None).update(
+        user=Subquery(
+            DailyXFormSubmissionCounter.objects.filter(pk=OuterRef('pk')).values('xform__user')[:1]
+        ),
+    )
+
+
+def delete_null_xform_daily_counters(apps, schema_editor):
+    DailyXFormSubmissionCounter = apps.get_model('logger', 'DailyXFormSubmissionCounter')
+    # to migrate backwards, we need to delete any null xform instances
+    DailyXFormSubmissionCounter.objects.filter(xform=None).delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('logger', '0027_on_delete_cascade_monthlyxformsubmissioncounter'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='DailyXFormSubmissionCounter',
+            name='user',
+            field=models.ForeignKey('auth.User', related_name='daily_users', null=True, on_delete=models.CASCADE),
+        ),
+        migrations.RunPython(
+            add_user_to_daily_submission_counter,
+            migrations.RunPython.noop,
+        ),
+        migrations.AlterField(
+            model_name='dailyxformsubmissioncounter',
+            name='xform',
+            field=models.ForeignKey(null=True, on_delete=deletion.CASCADE,
+                                    related_name='daily_counters', to='logger.xform'),
+        ),
+        migrations.RunPython(
+            migrations.RunPython.noop,
+            delete_null_xform_daily_counters,
+        ),
+        migrations.AlterUniqueTogether(
+            name='dailyxformsubmissioncounter',
+            unique_together=set(),
+        ),
+        migrations.AddIndex(
+            model_name='dailyxformsubmissioncounter',
+            index=models.Index(fields=['date', 'user'], name='logger_dail_date_f738ed_idx'),
+        ),
+        migrations.AddConstraint(
+            model_name='dailyxformsubmissioncounter',
+            constraint=models.UniqueConstraint(fields=('date', 'user', 'xform'), name='daily_unique_with_xform'),
+        ),
+        migrations.AddConstraint(
+            model_name='dailyxformsubmissioncounter',
+            constraint=models.UniqueConstraint(condition=models.Q(('xform', None)), fields=('date', 'user'),
+                                               name='daily_unique_without_xform'),
+        ),
+    ]

onadata/apps/logger/migrations/0028_populate_daily_xform_counters_for_year.py → onadata/apps/logger/migrations/0029_populate_daily_xform_counters_for_year.py

@@ -31,7 +31,7 @@ def populate_daily_counts_for_year(apps, schema_editor):
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('logger', '0027_on_delete_cascade_monthlyxformsubmissioncounter'),
+        ('logger', '0028_add_user_to_daily_submission_counters'),
         ('main', '0012_add_validate_password_flag_to_profile'),
     ]
 

onadata/apps/logger/models/daily_xform_submission_counter.py

@@ -1,18 +1,47 @@
 # coding: utf-8
-import datetime
 
 from django.contrib.auth.models import User
 from django.db import models
-
-from onadata.apps.logger.models.xform import XForm
+from django.db.models import UniqueConstraint, Q, F
 
 
 class DailyXFormSubmissionCounter(models.Model):
     date = models.DateField()
+    user = models.ForeignKey(User, related_name='daily_counts', null=True, on_delete=models.CASCADE)
     xform = models.ForeignKey(
-        XForm, related_name='daily_counters', on_delete=models.CASCADE
+        'logger.XForm', related_name='daily_counters', null=True, on_delete=models.CASCADE
     )
     counter = models.IntegerField(default=0)
 
     class Meta:
-        unique_together = (('date', 'xform'),)
+        constraints = [
+            UniqueConstraint(fields=['date', 'user', 'xform'],
+                             name='daily_unique_with_xform'),
+            UniqueConstraint(fields=['date', 'user'],
+                             condition=Q(xform=None),
+                             name='daily_unique_without_xform')
+        ]
+        indexes = [
+            models.Index(fields=('date', 'user')),
+        ]
+
+    @classmethod
+    def update_catch_all_counter_on_delete(cls, sender, instance, **kwargs):
+        daily_counters = cls.objects.filter(
+            xform_id=instance.pk, counter__gte=1
+        )
+
+        for daily_counter in daily_counters:
+            criteria = dict(
+                date=daily_counter.date,
+                user=daily_counter.user,
+                xform=None,
+            )
+            # make sure an instance exists with `xform = NULL`
+            cls.objects.get_or_create(**criteria)
+            # add the count for the project being deleted to the null-xform
+            # instance, atomically!
+            cls.objects.filter(**criteria).update(
+                counter=F('counter') + daily_counter.counter
+            )
+

onadata/apps/logger/models/instance.py

@@ -142,6 +142,7 @@ def update_xform_daily_counter(sender, instance, created, **kwargs):
     DailyXFormSubmissionCounter.objects.get_or_create(
         date=date_created,
         xform=instance.xform,
+        user=instance.xform.user,
     )
 
     # update the count for the current submission

onadata/apps/logger/models/xform.py

@@ -21,6 +21,7 @@
 from taggit.managers import TaggableManager
 
 from onadata.apps.logger.fields import LazyDefaultBooleanField
+from onadata.apps.logger.models.daily_xform_submission_counter import DailyXFormSubmissionCounter
 from onadata.apps.logger.models.monthly_xform_submission_counter import (
     MonthlyXFormSubmissionCounter,
 )
@@ -36,7 +37,6 @@
 from onadata.libs.models.base_model import BaseModel
 from onadata.libs.utils.hash import get_hash
 
-
 XFORM_TITLE_LENGTH = 255
 title_pattern = re.compile(r"<h:title>([^<]+)</h:title>")
 
@@ -335,3 +335,9 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs):
     sender=XForm,
     dispatch_uid='update_catch_all_monthly_xform_submission_counter',
 )
+
+pre_delete.connect(
+    DailyXFormSubmissionCounter.update_catch_all_counter_on_delete,
+    sender=XForm,
+    dispatch_uid='update_catch_all_daily_xform_submission_counter',
+)

onadata/apps/logger/tests/models/test_xform_submission_counters.py

@@ -47,7 +47,7 @@ def test_data_retrieval(self):
         self._publish_transportation_form_and_submit_instance()
 
         daily_counter = DailyXFormSubmissionCounter.objects.filter(
-            xform__user__username='bob'
+            user__username='bob'
         ).order_by('date').last()
         today = timezone.now().date()
         self.assertEqual(daily_counter.date, today)
@@ -65,7 +65,7 @@ def test_delete_daily_counters(self):
         """
         self._publish_transportation_form_and_submit_instance()
         counter = DailyXFormSubmissionCounter.objects.filter(
-            xform__user__username='bob'
+            user__username='bob'
         ).order_by('date').last()
         counter.date = counter.date - timedelta(
             days=settings.DAILY_COUNTERS_MAX_DAYS + 1
@@ -79,7 +79,7 @@ def test_delete_daily_counters(self):
         daily_counters = DailyXFormSubmissionCounter.objects.count()
         self.assertEqual(daily_counters, 0)
 
-    def test_deleted_xform_counters_are_merged(self):
+    def test_deleted_monthly_xform_counters_are_merged(self):
         """
         Test that the monthly counter with `xform = NULL` contains the sum of
         counters for all xforms deleted within the current month
@@ -104,3 +104,28 @@ def test_deleted_xform_counters_are_merged(self):
         assert (
             MonthlyXFormSubmissionCounter.objects.get(**criteria).counter == 2
         )
+
+    def test_deleted_daily_xform_counters_are_merged(self):
+        """
+        Test that the daily counter with `xform = NULL` contains the sum of
+        counters for all xforms deleted within the current day
+        """
+        today = timezone.now().date()
+        criteria = dict(
+            date=today,
+            user=User.objects.get(username='bob'),
+            xform=None,
+        )
+        assert not DailyXFormSubmissionCounter.objects.filter(
+            **criteria
+        ).exists()
+        self._publish_transportation_form_and_submit_instance()
+        XForm.objects.filter(user__username='bob').first().delete()
+        assert (
+            DailyXFormSubmissionCounter.objects.get(**criteria).counter == 1
+        )
+        self._publish_transportation_form_and_submit_instance()
+        XForm.objects.filter(user__username='bob').first().delete()
+        assert (
+            DailyXFormSubmissionCounter.objects.get(**criteria).counter == 2
+        )