From 48f4843cb3003cc4ad47278171efee9916f66d1e Mon Sep 17 00:00:00 2001 From: Jessica Thomas Date: Wed, 13 Sep 2023 09:22:28 -0500 Subject: [PATCH 1/5] add user to dailyxform counter and update null counter on cascade delete --- .../commands/populate_submission_counters.py | 1 + ...9_add_user_to_daily_submission_counters.py | 58 +++++++++++++++++++ .../models/daily_xform_submission_counter.py | 39 +++++++++++-- onadata/apps/logger/models/instance.py | 1 + onadata/apps/logger/models/xform.py | 8 ++- .../models/test_xform_submission_counters.py | 31 +++++++++- 6 files changed, 129 insertions(+), 9 deletions(-) create mode 100644 onadata/apps/logger/migrations/0029_add_user_to_daily_submission_counters.py diff --git a/onadata/apps/logger/management/commands/populate_submission_counters.py b/onadata/apps/logger/management/commands/populate_submission_counters.py index 49d3b2543..3be1cf68f 100644 --- a/onadata/apps/logger/management/commands/populate_submission_counters.py +++ b/onadata/apps/logger/management/commands/populate_submission_counters.py @@ -163,6 +163,7 @@ def build_counters(self, xf: 'logger.XForm') -> tuple[list, dict]: submission_date = values['date_created__date'] daily_counters.append(DailyXFormSubmissionCounter( xform_id=xf.pk, + user=xf.user, date=submission_date, counter=values['num_of_submissions'], )) diff --git a/onadata/apps/logger/migrations/0029_add_user_to_daily_submission_counters.py b/onadata/apps/logger/migrations/0029_add_user_to_daily_submission_counters.py new file mode 100644 index 000000000..4713d8b2b --- /dev/null +++ b/onadata/apps/logger/migrations/0029_add_user_to_daily_submission_counters.py @@ -0,0 +1,58 @@ +from django.conf import settings +from django.db import migrations, models +from django.db.models import Subquery, deletion + + +def add_user_to_daily_submission_counter(apps, schema_editor): + DailyXFormSubmissionCounter = apps.get_model("logger", "DailyXFormSubmissionCounter") + # delete any counters where xform and user are None, since we can't associate them with a user + DailyXFormSubmissionCounter.objects.filter(xform=None, user=None).delete() + # add the user to the counter, based on the xform user + DailyXFormSubmissionCounter.objects.all().exclude(xform=None).update( + user=Subquery( + DailyXFormSubmissionCounter.objects.all().exclude(xform=None).values('xform__user')[:1] + ), + ) + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('logger', '0028_populate_daily_xform_counters_for_year'), + ] + + operations = [ + migrations.AddField( + model_name='DailyXFormSubmissionCounter', + name='user', + field=models.ForeignKey('auth.User', related_name='daily_users', null=True, on_delete=models.CASCADE), + ), + migrations.RunPython( + add_user_to_daily_submission_counter, + migrations.RunPython.noop, + ), + migrations.AlterField( + model_name='dailyxformsubmissioncounter', + name='xform', + field=models.ForeignKey(null=True, on_delete=deletion.CASCADE, + related_name='daily_counters', to='logger.xform'), + ), + migrations.AlterUniqueTogether( + name='dailyxformsubmissioncounter', + unique_together=set(), + ), + migrations.AddIndex( + model_name='dailyxformsubmissioncounter', + index=models.Index(fields=['date', 'user'], name='logger_dail_date_f738ed_idx'), + ), + migrations.AddConstraint( + model_name='dailyxformsubmissioncounter', + constraint=models.UniqueConstraint(fields=('date', 'user', 'xform'), name='daily_unique_with_xform'), + ), + migrations.AddConstraint( + model_name='dailyxformsubmissioncounter', + constraint=models.UniqueConstraint(condition=models.Q(('xform', None)), fields=('date', 'user'), + name='daily_unique_without_xform'), + ), + ] diff --git a/onadata/apps/logger/models/daily_xform_submission_counter.py b/onadata/apps/logger/models/daily_xform_submission_counter.py index e147dc77f..9000db2fa 100644 --- a/onadata/apps/logger/models/daily_xform_submission_counter.py +++ b/onadata/apps/logger/models/daily_xform_submission_counter.py @@ -1,18 +1,47 @@ # coding: utf-8 -import datetime from django.contrib.auth.models import User from django.db import models - -from onadata.apps.logger.models.xform import XForm +from django.db.models import UniqueConstraint, Q, F class DailyXFormSubmissionCounter(models.Model): date = models.DateField() + user = models.ForeignKey(User, related_name='daily_users', null=True, on_delete=models.CASCADE) xform = models.ForeignKey( - XForm, related_name='daily_counters', on_delete=models.CASCADE + 'logger.XForm', related_name='daily_counters', null=True, on_delete=models.CASCADE ) counter = models.IntegerField(default=0) class Meta: - unique_together = (('date', 'xform'),) + constraints = [ + UniqueConstraint(fields=['date', 'user', 'xform'], + name='daily_unique_with_xform'), + UniqueConstraint(fields=['date', 'user'], + condition=Q(xform=None), + name='daily_unique_without_xform') + ] + indexes = [ + models.Index(fields=('date', 'user')), + ] + + @classmethod + def update_catch_all_counter_on_delete(cls, sender, instance, **kwargs): + daily_counters = cls.objects.filter( + xform_id=instance.pk, counter__gte=1 + ) + + for daily_counter in daily_counters: + criteria = dict( + date=daily_counter.date, + user=daily_counter.user, + xform=None, + ) + # make sure an instance exists with `xform = NULL` + cls.objects.get_or_create(**criteria) + # add the count for the project being deleted to the null-xform + # instance, atomically! + cls.objects.filter(**criteria).update( + counter=F('counter') + daily_counter.counter + ) + diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index 7b6d40342..19437d838 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -142,6 +142,7 @@ def update_xform_daily_counter(sender, instance, created, **kwargs): DailyXFormSubmissionCounter.objects.get_or_create( date=date_created, xform=instance.xform, + user=instance.xform.user, ) # update the count for the current submission diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index e056a4ede..2d075936a 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -21,6 +21,7 @@ from taggit.managers import TaggableManager from onadata.apps.logger.fields import LazyDefaultBooleanField +from onadata.apps.logger.models.daily_xform_submission_counter import DailyXFormSubmissionCounter from onadata.apps.logger.models.monthly_xform_submission_counter import ( MonthlyXFormSubmissionCounter, ) @@ -36,7 +37,6 @@ from onadata.libs.models.base_model import BaseModel from onadata.libs.utils.hash import get_hash - XFORM_TITLE_LENGTH = 255 title_pattern = re.compile(r"([^<]+)") @@ -335,3 +335,9 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): sender=XForm, dispatch_uid='update_catch_all_monthly_xform_submission_counter', ) + +pre_delete.connect( + DailyXFormSubmissionCounter.update_catch_all_counter_on_delete, + sender=XForm, + dispatch_uid='update_catch_all_daily_xform_submission_counter', +) diff --git a/onadata/apps/logger/tests/models/test_xform_submission_counters.py b/onadata/apps/logger/tests/models/test_xform_submission_counters.py index f4a904d1e..a1b0604a2 100644 --- a/onadata/apps/logger/tests/models/test_xform_submission_counters.py +++ b/onadata/apps/logger/tests/models/test_xform_submission_counters.py @@ -47,7 +47,7 @@ def test_data_retrieval(self): self._publish_transportation_form_and_submit_instance() daily_counter = DailyXFormSubmissionCounter.objects.filter( - xform__user__username='bob' + user__username='bob' ).order_by('date').last() today = timezone.now().date() self.assertEqual(daily_counter.date, today) @@ -65,7 +65,7 @@ def test_delete_daily_counters(self): """ self._publish_transportation_form_and_submit_instance() counter = DailyXFormSubmissionCounter.objects.filter( - xform__user__username='bob' + user__username='bob' ).order_by('date').last() counter.date = counter.date - timedelta( days=settings.DAILY_COUNTERS_MAX_DAYS + 1 @@ -79,7 +79,7 @@ def test_delete_daily_counters(self): daily_counters = DailyXFormSubmissionCounter.objects.count() self.assertEqual(daily_counters, 0) - def test_deleted_xform_counters_are_merged(self): + def test_deleted_monthly_xform_counters_are_merged(self): """ Test that the monthly counter with `xform = NULL` contains the sum of counters for all xforms deleted within the current month @@ -104,3 +104,28 @@ def test_deleted_xform_counters_are_merged(self): assert ( MonthlyXFormSubmissionCounter.objects.get(**criteria).counter == 2 ) + + def test_deleted_daily_xform_counters_are_merged(self): + """ + Test that the daily counter with `xform = NULL` contains the sum of + counters for all xforms deleted within the current year + """ + today = timezone.now().date() + criteria = dict( + date=today, + user=User.objects.get(username='bob'), + xform=None, + ) + assert not DailyXFormSubmissionCounter.objects.filter( + **criteria + ).exists() + self._publish_transportation_form_and_submit_instance() + XForm.objects.filter(user__username='bob').first().delete() + assert ( + DailyXFormSubmissionCounter.objects.get(**criteria).counter == 1 + ) + self._publish_transportation_form_and_submit_instance() + XForm.objects.filter(user__username='bob').first().delete() + assert ( + DailyXFormSubmissionCounter.objects.get(**criteria).counter == 2 + ) From 89937cbda7e155842de233a82b1622b9159b40df Mon Sep 17 00:00:00 2001 From: Jessica Thomas Date: Wed, 13 Sep 2023 09:26:49 -0500 Subject: [PATCH 2/5] fix typo in comment --- .../apps/logger/tests/models/test_xform_submission_counters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/apps/logger/tests/models/test_xform_submission_counters.py b/onadata/apps/logger/tests/models/test_xform_submission_counters.py index a1b0604a2..5c7d71f31 100644 --- a/onadata/apps/logger/tests/models/test_xform_submission_counters.py +++ b/onadata/apps/logger/tests/models/test_xform_submission_counters.py @@ -108,7 +108,7 @@ def test_deleted_monthly_xform_counters_are_merged(self): def test_deleted_daily_xform_counters_are_merged(self): """ Test that the daily counter with `xform = NULL` contains the sum of - counters for all xforms deleted within the current year + counters for all xforms deleted within the current day """ today = timezone.now().date() criteria = dict( From 822298d4d8c6048fe6df26f5570a1260f50d1c3c Mon Sep 17 00:00:00 2001 From: Jessica Thomas Date: Wed, 13 Sep 2023 09:48:35 -0500 Subject: [PATCH 3/5] switch order of logger 0028 and 0029 so daily xform counters are populated with users --- ...=> 0028_add_user_to_daily_submission_counters.py} | 12 +++++++++++- ...> 0029_populate_daily_xform_counters_for_year.py} | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) rename onadata/apps/logger/migrations/{0029_add_user_to_daily_submission_counters.py => 0028_add_user_to_daily_submission_counters.py} (82%) rename onadata/apps/logger/migrations/{0028_populate_daily_xform_counters_for_year.py => 0029_populate_daily_xform_counters_for_year.py} (94%) diff --git a/onadata/apps/logger/migrations/0029_add_user_to_daily_submission_counters.py b/onadata/apps/logger/migrations/0028_add_user_to_daily_submission_counters.py similarity index 82% rename from onadata/apps/logger/migrations/0029_add_user_to_daily_submission_counters.py rename to onadata/apps/logger/migrations/0028_add_user_to_daily_submission_counters.py index 4713d8b2b..56cfa90ae 100644 --- a/onadata/apps/logger/migrations/0029_add_user_to_daily_submission_counters.py +++ b/onadata/apps/logger/migrations/0028_add_user_to_daily_submission_counters.py @@ -15,11 +15,17 @@ def add_user_to_daily_submission_counter(apps, schema_editor): ) +def delete_null_xform_daily_counters(apps, schema_editor): + DailyXFormSubmissionCounter = apps.get_model("logger", "DailyXFormSubmissionCounter") + # to migrate backwards, we need to delete any null xform instances + DailyXFormSubmissionCounter.objects.filter(xform=None).delete() + + class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('logger', '0028_populate_daily_xform_counters_for_year'), + ('logger', '0027_on_delete_cascade_monthlyxformsubmissioncounter'), ] operations = [ @@ -38,6 +44,10 @@ class Migration(migrations.Migration): field=models.ForeignKey(null=True, on_delete=deletion.CASCADE, related_name='daily_counters', to='logger.xform'), ), + migrations.RunPython( + migrations.RunPython.noop, + delete_null_xform_daily_counters, + ), migrations.AlterUniqueTogether( name='dailyxformsubmissioncounter', unique_together=set(), diff --git a/onadata/apps/logger/migrations/0028_populate_daily_xform_counters_for_year.py b/onadata/apps/logger/migrations/0029_populate_daily_xform_counters_for_year.py similarity index 94% rename from onadata/apps/logger/migrations/0028_populate_daily_xform_counters_for_year.py rename to onadata/apps/logger/migrations/0029_populate_daily_xform_counters_for_year.py index b91363e80..2f820a6d5 100644 --- a/onadata/apps/logger/migrations/0028_populate_daily_xform_counters_for_year.py +++ b/onadata/apps/logger/migrations/0029_populate_daily_xform_counters_for_year.py @@ -31,7 +31,7 @@ def populate_daily_counts_for_year(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('logger', '0027_on_delete_cascade_monthlyxformsubmissioncounter'), + ('logger', '0028_add_user_to_daily_submission_counters'), ('main', '0012_add_validate_password_flag_to_profile'), ] From 11535eae2a45520130ae82c2967090751194083f Mon Sep 17 00:00:00 2001 From: Jessica Thomas Date: Thu, 14 Sep 2023 14:18:40 -0500 Subject: [PATCH 4/5] fix misleading relation name --- onadata/apps/logger/models/daily_xform_submission_counter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/apps/logger/models/daily_xform_submission_counter.py b/onadata/apps/logger/models/daily_xform_submission_counter.py index 9000db2fa..25abadc3b 100644 --- a/onadata/apps/logger/models/daily_xform_submission_counter.py +++ b/onadata/apps/logger/models/daily_xform_submission_counter.py @@ -7,7 +7,7 @@ class DailyXFormSubmissionCounter(models.Model): date = models.DateField() - user = models.ForeignKey(User, related_name='daily_users', null=True, on_delete=models.CASCADE) + user = models.ForeignKey(User, related_name='daily_counts', null=True, on_delete=models.CASCADE) xform = models.ForeignKey( 'logger.XForm', related_name='daily_counters', null=True, on_delete=models.CASCADE ) From 133855c71d0d184810cb788e7d5237a16824547e Mon Sep 17 00:00:00 2001 From: Jessica Thomas Date: Fri, 15 Sep 2023 09:40:57 -0500 Subject: [PATCH 5/5] correct query to add user to daily xform counters --- .../0028_add_user_to_daily_submission_counters.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/onadata/apps/logger/migrations/0028_add_user_to_daily_submission_counters.py b/onadata/apps/logger/migrations/0028_add_user_to_daily_submission_counters.py index 56cfa90ae..97803dc00 100644 --- a/onadata/apps/logger/migrations/0028_add_user_to_daily_submission_counters.py +++ b/onadata/apps/logger/migrations/0028_add_user_to_daily_submission_counters.py @@ -1,22 +1,22 @@ from django.conf import settings from django.db import migrations, models -from django.db.models import Subquery, deletion +from django.db.models import Subquery, deletion, OuterRef def add_user_to_daily_submission_counter(apps, schema_editor): - DailyXFormSubmissionCounter = apps.get_model("logger", "DailyXFormSubmissionCounter") + DailyXFormSubmissionCounter = apps.get_model('logger', 'DailyXFormSubmissionCounter') # delete any counters where xform and user are None, since we can't associate them with a user DailyXFormSubmissionCounter.objects.filter(xform=None, user=None).delete() # add the user to the counter, based on the xform user DailyXFormSubmissionCounter.objects.all().exclude(xform=None).update( user=Subquery( - DailyXFormSubmissionCounter.objects.all().exclude(xform=None).values('xform__user')[:1] + DailyXFormSubmissionCounter.objects.filter(pk=OuterRef('pk')).values('xform__user')[:1] ), ) def delete_null_xform_daily_counters(apps, schema_editor): - DailyXFormSubmissionCounter = apps.get_model("logger", "DailyXFormSubmissionCounter") + DailyXFormSubmissionCounter = apps.get_model('logger', 'DailyXFormSubmissionCounter') # to migrate backwards, we need to delete any null xform instances DailyXFormSubmissionCounter.objects.filter(xform=None).delete()