Merge pull request #1178 from akvo/feature/1166-monitoring-list-for-m…

…anage-data-api

Showing Monitoring Data-point in the Manage Data Page

backend/api/v1/v1_data/management/commands/fake_data_monitoring_seeder.py

@@ -49,7 +49,7 @@ def create_new_endpoint(index, form, administration, user):
     return data
 
 
-def seed_data(form, user, administrations, repeat):
+def seed_data(form, user, administrations, repeat, approved):
     pendings = []
     for index in range(repeat):
         selected_adm = random.choice(administrations)
@@ -90,12 +90,13 @@ def seed_data(form, user, administrations, repeat):
     for administration_id, items in grouped_data.items():
         [dp] = items[:1]
         batch_name = fake.sentence(nb_words=3)
+        approved_value = fake.pybool() if not approved else approved
         batch = PendingDataBatch.objects.create(
             form=dp['instance'].form,
             administration=dp['instance'].administration,
             user=dp['instance'].created_by,
             name=f"{batch_name}-{administration_id}",
-            approved=fake.pybool(),
+            approved=approved_value,
         )
         batch_items = [i['instance'] for i in items]
         batch.batch_pending_data_batch.add(*batch_items)
@@ -138,10 +139,17 @@ def add_arguments(self, parser):
                             const=False,
                             default=False,
                             type=bool)
+        parser.add_argument("-a",
+                            "--approved",
+                            nargs="?",
+                            const=False,
+                            default=False,
+                            type=bool)
 
     def handle(self, *args, **options):
         test = options.get("test")
         repeat = options.get("repeat")
+        approved = options.get("approved")
         lowest_level = Levels.objects.order_by('-id').first()
         user = SystemUser.objects.filter(
             user_access__role=UserRoleTypes.user,
@@ -167,5 +175,11 @@ def handle(self, *args, **options):
             for user_form in user_forms:
                 if not test:
                     print(f"\nSeeding - {user_form.form.name}")
-                seed_data(user_form.form, user, administrations, repeat)
+                seed_data(
+                    user_form.form,
+                    user,
+                    administrations,
+                    repeat,
+                    approved
+                )
             refresh_materialized_data()

backend/api/v1/v1_data/migrations/0030_alter_formdata_parent.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.0.4 on 2024-02-19 02:31
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('v1_data', '0029_formdata_parent'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='formdata',
+            name='parent',
+            field=models.ForeignKey(default=None, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='children', to='v1_data.formdata'),
+        ),
+    ]

backend/api/v1/v1_data/models.py

@@ -38,6 +38,14 @@ class FormData(models.Model):
     created = models.DateTimeField(auto_now_add=True)
     updated = models.DateTimeField(default=None, null=True)
 
+    parent = models.ForeignKey(
+        'self',
+        on_delete=models.PROTECT,
+        related_name='children',
+        default=None,
+        null=True
+    )
+
     def __str__(self):
         return self.name
 

backend/api/v1/v1_data/serializers.py

@@ -233,12 +233,21 @@ class ListFormDataRequestSerializer(serializers.Serializer):
         child=CustomPrimaryKeyRelatedField(queryset=Questions.objects.none()),
         required=False,
     )
+    parent = CustomPrimaryKeyRelatedField(
+        queryset=FormData.objects.filter(parent=None).none(),
+        required=False
+    )
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         self.fields.get(
             "administration").queryset = Administration.objects.all()
         self.fields.get("questions").child.queryset = Questions.objects.all()
+        form_id = self.context.get('form_id')
+        self.fields.get("parent").queryset = FormData.objects.filter(
+            form_id=form_id,
+            parent=None,
+        ).all()
 
 
 class ListFormDataSerializer(serializers.ModelSerializer):
@@ -248,6 +257,7 @@ class ListFormDataSerializer(serializers.ModelSerializer):
     updated = serializers.SerializerMethodField()
     administration = serializers.ReadOnlyField(source="administration.name")
     pending_data = serializers.SerializerMethodField()
+    children_count = serializers.SerializerMethodField()
 
     @extend_schema_field(OpenApiTypes.STR)
     def get_created_by(self, instance: FormData):
@@ -288,6 +298,10 @@ def get_pending_data(self, instance: FormData):
             }
         return None
 
+    @extend_schema_field(OpenApiTypes.NUMBER)
+    def get_children_count(self, instance: FormData):
+        return instance.children.count()
+
     class Meta:
         model = FormData
         fields = [
@@ -302,6 +316,7 @@ class Meta:
             "created",
             "updated",
             "pending_data",
+            "children_count",
         ]
 
 

backend/api/v1/v1_data/tasks.py

@@ -19,6 +19,7 @@ def seed_approved_data(data):
         form_data.geo = data.geo
         form_data.updated_by = data.created_by
         form_data.updated = timezone.now()
+        form_data.save_to_file
         form_data.save()
 
         for answer in data.pending_data_answer.all():

backend/api/v1/v1_data/tests/tests_data.py

@@ -31,7 +31,8 @@ def test_list_form_data(self):
                          ['current', 'total', 'total_page', 'data'])
         self.assertEqual(list(result['data'][0]), [
             'id', 'uuid', 'name', 'form', 'administration', 'geo',
-            'created_by', 'updated_by', 'created', 'updated', 'pending_data'
+            'created_by', 'updated_by', 'created', 'updated', 'pending_data',
+            'children_count'
         ])
         self.assertIsNotNone(result['data'][0]['uuid'])
 
@@ -203,3 +204,42 @@ def test_datapoint_with_history_deletion(self):
         self.assertEqual(answers, 0)
         hitory = AnswerHistory.objects.filter(data_id=data_id).count()
         self.assertEqual(hitory, 0)
+
+    def test_monitoring_details_by_parent_id(self):
+        call_command("administration_seeder", "--test")
+        user_payload = {"email": "[email protected]", "password": "Test105*"}
+        user_response = self.client.post('/api/v1/login',
+                                         user_payload,
+                                         content_type='application/json')
+        token = user_response.json().get('token')
+        call_command("form_seeder", "--test")
+        call_command("demo_approval_flow")
+        call_command("fake_data_seeder", "-r", 1, "-t", True)
+        call_command(
+            "fake_data_monitoring_seeder",
+            "-r", 5,
+            "-t", True,
+            "-a", True
+        )
+
+        header = {'HTTP_AUTHORIZATION': f'Bearer {token}'}
+
+        monitoring = FormData.objects.filter(parent__isnull=False).first()
+        parent = monitoring.parent
+        form_id = monitoring.form.id
+        url = f"/api/v1/form-data/{form_id}?page=1&parent={parent.id}"
+        data = self.client.get(
+            url,
+            follow=True,
+            **header
+        )
+        result = data.json()
+        self.assertEqual(data.status_code, 200)
+        self.assertEqual(
+            list(result),
+            ['current', 'total', 'total_page', 'data']
+        )
+        # total equal to number of children + the data itself
+        self.assertEqual(result['total'], parent.children.count() + 1)
+        # make sure the last item is parent
+        self.assertEqual(result['data'][-1]['name'], parent.name)

backend/api/v1/v1_data/views.py

@@ -8,7 +8,7 @@
 # from functools import reduce
 
 from django.contrib.postgres.aggregates import StringAgg
-from django.db.models import Count, TextField, Value, F, Sum, Avg
+from django.db.models import Count, TextField, Value, F, Sum, Avg, Max, Q
 from django.db.models.functions import Cast, Coalesce
 from django.http import HttpResponse
 from django_q.tasks import async_task
@@ -96,18 +96,34 @@ class FormDataAddListView(APIView):
                              required=False,
                              type={'type': 'array',
                                    'items': {'type': 'string'}},
+                             location=OpenApiParameter.QUERY),
+            OpenApiParameter(name='parent',
+                             required=False,
+                             type=OpenApiTypes.NUMBER,
                              location=OpenApiParameter.QUERY)],
         summary='To get list of form data')
     def get(self, request, form_id, version):
         form = get_object_or_404(Forms, pk=form_id)
-        serializer = ListFormDataRequestSerializer(data=request.GET)
+        serializer = ListFormDataRequestSerializer(
+            data=request.GET,
+            context={'form_id': form_id}
+        )
         if not serializer.is_valid():
             return Response(
                 {'message': validate_serializers_message(
                     serializer.errors)},
                 status=status.HTTP_400_BAD_REQUEST
             )
+        parent = serializer.validated_data.get('parent')
         filter_data = {}
+        if parent:
+            latest_created_per_uuid = FormData.objects.filter(
+                Q(form_id=form_id, parent=parent) |
+                Q(pk=parent.id)
+            ).values('uuid').annotate(latest_created=Max('created'))
+            filter_data['uuid__in'] = latest_created_per_uuid.values('uuid')
+        else:
+            filter_data['parent'] = None
         if serializer.validated_data.get('administration'):
             filter_administration = serializer.validated_data.get(
                 'administration')
@@ -133,9 +149,28 @@ def get(self, request, form_id, version):
         page_size = REST_FRAMEWORK.get('PAGE_SIZE')
 
         the_past = datetime.now() - timedelta(days=10 * 365)
-        queryset = form.form_form_data.filter(**filter_data).annotate(
-            last_updated=Coalesce('updated', Value(the_past))).order_by(
-            '-last_updated', '-created')
+        queryset = form.form_form_data.filter(**filter_data)
+        if parent:
+            queryset = queryset.annotate(
+                    last_updated=Coalesce('updated', Value(the_past)),
+                    latest_created=Coalesce('updated', 'created') \
+                    # Use updated time if available, otherwise use created time
+                ).filter(
+                    created=F('latest_created') \
+                    # Filter by objects where created equals latest_created
+                ).order_by(
+                    '-last_updated',
+                    '-created'
+                )
+        else:
+            queryset = queryset.annotate(
+                last_updated=Coalesce('updated', Value(the_past)),
+                children_count=Count('children')
+            ).order_by(
+                '-children_count',
+                '-last_updated',
+                '-created'
+            )
 
         paginator = PageNumberPagination()
         instance = paginator.paginate_queryset(queryset, request)

frontend/src/App.js

@@ -42,6 +42,7 @@ import {
   UploadAdministrationData,
   BIDashboard,
   DownloadAdmData,
+  MonitoringDetail,
   // Visualisation,
 } from "./pages";
 import { useCookies } from "react-cookie";
@@ -114,6 +115,10 @@ const RouteList = () => {
           path="data/manage"
           element={<Private element={ManageData} alias="data" />}
         />
+        <Route
+          path="data/monitoring/:parentId"
+          element={<Private element={MonitoringDetail} alias="data" />}
+        />
         <Route
           path="data/export"
           element={<Private element={ExportData} alias="data" />}

frontend/src/lib/__test__/__snapshots__/store.test.js.snap

@@ -70,6 +70,7 @@ Object {
   "questionGroups": Array [],
   "selectedAdministration": null,
   "selectedForm": null,
+  "selectedFormData": null,
   "showAdvancedFilters": false,
   "showContactFormModal": false,
   "user": null,

frontend/src/lib/__test__/__snapshots__/ui-text.test.js.snap

@@ -58,6 +58,7 @@ Object {
     "attrTypeRequired": "Attribute type is required",
     "backBtn": "Back",
     "backHome": "Back to Homepage",
+    "backManageData": "Back to Manage data",
     "backToCenterLabel": "Back to Control Center",
     "batchDatasets": "Batch Datasets",
     "batchHintDesc": "The operation of merging datasets cannot be undone, and will Create a new batch that will require approval from you admin",
@@ -429,6 +430,18 @@ Object {
     "mobileSelectForms": "Select forms...",
     "mobileSuccessAdded": "Mobile assignment added",
     "mobileSuccessUpdated": "Mobile assignment update",
+    "monitoringDataDescription": <React.Fragment>
+      This is where you :
+      <ul>
+        <li>
+          Get the list of forms that were collected for this datapoint (new and update)
+        </li>
+        <li>
+          Edit monitoring data
+        </li>
+      </ul>
+    </React.Fragment>,
+    "monitoringDataTitle": "Monitoring data",
     "month": "Month",
     "myProfile": "My Profile",
     "nameField": "Name",

frontend/src/lib/store.js

@@ -19,6 +19,7 @@ const defaultUIState = {
   forms: window.forms.sort(sortArray),
   levels: window.levels,
   selectedForm: null,
+  selectedFormData: null,
   loadingForm: false,
   questionGroups: [],
   showAdvancedFilters: false,

frontend/src/lib/ui-text.js

@@ -669,6 +669,20 @@ const uiText = {
     questionCol: "Question",
     responseCol: "Response",
     lastResponseCol: "Last Response",
+    backManageData: "Back to Manage data",
+    monitoringDataTitle: "Monitoring data",
+    monitoringDataDescription: (
+      <Fragment>
+        This is where you :
+        <ul>
+          <li>
+            Get the list of forms that were collected for this datapoint (new
+            and update)
+          </li>
+          <li>Edit monitoring data</li>
+        </ul>
+      </Fragment>
+    ),
   },
 
   de: {},

frontend/src/pages/index.js

@@ -40,3 +40,4 @@ export { default as AddEntityData } from "./master-data-entities/entity-data/Add
 export { default as UploadAdministrationData } from "./upload-administration-data/UploadAdministrationData";
 export { default as BIDashboard } from "./bi/BIDashboard";
 export { default as DownloadAdmData } from "./upload-administration-data/DownloadAdmData";
+export { default as MonitoringDetail } from "./manage-data/MonitoringDetail";