diff --git a/backend/api/v1/v1_data/serializers.py b/backend/api/v1/v1_data/serializers.py index 84b8740fe..a997dbc5b 100644 --- a/backend/api/v1/v1_data/serializers.py +++ b/backend/api/v1/v1_data/serializers.py @@ -1153,63 +1153,50 @@ def validate_value(self, value): return value def validate(self, attrs): - if attrs.get("value") == "": + question = attrs.get("question") + value = attrs.get("value") + + if value == "": raise ValidationError( - "Value is required for Question:{0}".format( - attrs.get("question").id - ) + f"Value is required for Question: {question.id}" ) - if ( - isinstance(attrs.get("value"), list) - and len(attrs.get("value")) == 0 - ): + if isinstance(value, list) and len(value) == 0: raise ValidationError( - "Value is required for Question:{0}".format( - attrs.get("question").id - ) + f"Value is required for Question: {question.id}" ) - if not isinstance(attrs.get("value"), list) and attrs.get( - "question" - ).type in [ + if not isinstance(value, list) and question.type in [ QuestionTypes.geo, QuestionTypes.option, QuestionTypes.multiple_option, ]: raise ValidationError( - "Valid list value is required for Question:{0}".format( - attrs.get("question").id - ) + f"Valid list value is required for Question: {question.id}" ) - elif not isinstance(attrs.get("value"), str) and attrs.get( - "question" - ).type in [ + + elif not isinstance(value, str) and question.type in [ QuestionTypes.text, QuestionTypes.photo, QuestionTypes.date, ]: raise ValidationError( - "Valid string value is required for Question:{0}".format( - attrs.get("question").id - ) + f"Valid string value is required for Question: {question.id}" ) + elif not ( - isinstance(attrs.get("value"), int) - or isinstance(attrs.get("value"), float) - ) and attrs.get("question").type in [ + isinstance(value, int) or isinstance(value, float)) \ + and question.type in [ QuestionTypes.number, QuestionTypes.administration, QuestionTypes.cascade, ]: raise ValidationError( - "Valid number value is required for Question:{0}".format( - attrs.get("question").id - ) + f"Valid number value is required for Question: {question.id}" ) - if attrs.get("question").type == QuestionTypes.administration: - attrs["value"] = int(float(attrs.get("value"))) + if question.type == QuestionTypes.administration: + attrs["value"] = int(float(value)) return attrs @@ -1251,7 +1238,7 @@ def create(self, validated_data): data["created_by"] = self.context.get("user") # check user role and form type - user: SystemUser = self.context.get("user") + user = self.context.get("user") is_super_admin = user.user_access.role == UserRoleTypes.super_admin is_county_admin = ( user.user_access.role == UserRoleTypes.admin @@ -1265,6 +1252,7 @@ def create(self, validated_data): ] if data.get("submission_type") in direct_submission_types: direct_to_data = True + # save to pending data if not direct_to_data: obj_data = self.fields.get("data").create(data) @@ -1281,50 +1269,57 @@ def create(self, validated_data): submission_type=data.get("submission_type"), ) + pending_answers = [] + answers = [] + for answer in validated_data.get("answer"): + question = answer.get("question") name = None value = None option = None - if answer.get("question").meta_uuid: + if question.meta_uuid: obj_data.uuid = answer.get("value") obj_data.save() - if answer.get("question").type in [ + if question.type in [ QuestionTypes.geo, QuestionTypes.option, QuestionTypes.multiple_option, ]: option = answer.get("value") - elif answer.get("question").type in [ + elif question.type in [ QuestionTypes.text, QuestionTypes.photo, QuestionTypes.date, QuestionTypes.autofield, ]: name = answer.get("value") - elif answer.get("question").type == QuestionTypes.cascade: + elif question.type == QuestionTypes.cascade: id = answer.get("value") val = None - if answer.get("question").api: - ep = answer.get("question").api.get("endpoint") + if question.api: + ep = question.api.get("endpoint") if "organisation" in ep: - val = Organisation.objects.filter(pk=id).first() - val = val.name + name = Organisation.objects.filter(pk=id).values_list( + 'name', flat=True).first() + val = name if "entity-data" in ep: - val = EntityData.objects.filter(pk=id).first() - val = val.name + name = EntityData.objects.filter(pk=id).values_list( + 'name', flat=True).first() + val = name if "entity-data" not in ep and "organisation" not in ep: ep = ep.split("?")[0] ep = f"{ep}?id={id}" val = requests.get(ep).json() val = val[0].get("name") - if answer.get("question").extra: - cs_type = answer.get("question").extra.get("type") + if question.extra: + cs_type = question.extra.get("type") if cs_type == "entity": - val = EntityData.objects.filter(pk=id).first() - val = val.name + name = EntityData.objects.filter(pk=id).values_list( + 'name', flat=True).first() + val = name name = val else: # for administration,number question type @@ -1332,26 +1327,32 @@ def create(self, validated_data): # save to pending answer if not direct_to_data: - PendingAnswers.objects.create( + pending_answers.append(PendingAnswers( pending_data=obj_data, - question=answer.get("question"), + question=question, name=name, value=value, options=option, created_by=self.context.get("user"), created=data.get("submitedAt") or timezone.now(), - ) + )) # save to form data if direct_to_data: - Answers.objects.create( + answers.append(Answers( data=obj_data, - question=answer.get("question"), + question=question, name=name, value=value, options=option, created_by=self.context.get("user"), - ) + )) + + # bulk create pending answers / answers + if pending_answers: + PendingAnswers.objects.bulk_create(pending_answers) + if answers: + Answers.objects.bulk_create(answers) if direct_to_data: if data.get("uuid"): diff --git a/backend/api/v1/v1_mobile/tests/tests_api_sync.py b/backend/api/v1/v1_mobile/tests/tests_api_sync.py index 0860c9479..ade83d092 100644 --- a/backend/api/v1/v1_mobile/tests/tests_api_sync.py +++ b/backend/api/v1/v1_mobile/tests/tests_api_sync.py @@ -109,18 +109,21 @@ def test_mobile_sync_to_pending_datapoint(self): "submission_type": SubmissionTypes.registration, "answers": answers, } - self.assertEqual(len(answers), len(questions)) - # Submit correct data - response = self.client.post( - "/api/v1/device/sync", - post_data, - follow=True, - content_type="application/json", - **{"HTTP_AUTHORIZATION": f"Bearer {token}"}, - ) - self.assertEqual(response.status_code, status.HTTP_200_OK) + # check N+1 query + def call_route(): + # Submit correct data + response = self.client.post( + "/api/v1/device/sync", + post_data, + follow=True, + content_type="application/json", + **{"HTTP_AUTHORIZATION": f"Bearer {token}"}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + self.assertNumQueries(28, call_route) pending_data = PendingFormData.objects.filter( created_by=self.user @@ -219,16 +222,19 @@ def test_mobile_sync_to_pending_datapoint(self): assignment.save() token = self.get_assignmen_token(self.passcode) - response = self.client.post( - "/api/v1/device/sync", - post_data, - follow=True, - content_type="application/json", - **{"HTTP_AUTHORIZATION": f"Bearer {token}"}, - ) - self.assertEqual(response.status_code, status.HTTP_200_OK) - # submit certification data + def call_second_route(post_data): + response = self.client.post( + "/api/v1/device/sync", + post_data, + follow=True, + content_type="application/json", + **{"HTTP_AUTHORIZATION": f"Bearer {token}"}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + self.assertNumQueries(28, call_second_route(post_data=post_data)) + post_data = { "formId": self.form.id, "name": "testing datapoint for certification", @@ -238,15 +244,19 @@ def test_mobile_sync_to_pending_datapoint(self): "submission_type": SubmissionTypes.certification, "answers": answers, } - response = self.client.post( - "/api/v1/device/sync", - post_data, - follow=True, - content_type="application/json", - **{"HTTP_AUTHORIZATION": f"Bearer {token}"}, - ) - self.assertEqual(response.status_code, status.HTTP_200_OK) + # submit certification data + def call_third_route(): + response = self.client.post( + "/api/v1/device/sync", + post_data, + follow=True, + content_type="application/json", + **{"HTTP_AUTHORIZATION": f"Bearer {token}"}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + self.assertNumQueries(29, call_third_route) data = FormData.objects.all() names = [d.name for d in data] diff --git a/backend/api/v1/v1_mobile/tests/tests_api_sync_empty_payload.py b/backend/api/v1/v1_mobile/tests/tests_api_sync_empty_payload.py index f2ed901eb..c25acf947 100644 --- a/backend/api/v1/v1_mobile/tests/tests_api_sync_empty_payload.py +++ b/backend/api/v1/v1_mobile/tests/tests_api_sync_empty_payload.py @@ -82,6 +82,19 @@ def test_empty_required_text_type_of_question(self): ).count() self.assertEqual(total_null_answers, 0) + # check N+1 query + def call_route(): + response = self.client.post( + "/api/v1/device/sync", + payload, + follow=True, + content_type="application/json", + **{"HTTP_AUTHORIZATION": f"Bearer {self.token}"}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + self.assertNumQueries(22, call_route) + def test_empty_required_number_type_of_question(self): mobile_adm = self.mobile_user.administrations.first() uuid = "22b9ecb2-c400-4b76-bcba-0a70a6942bb6" @@ -131,6 +144,19 @@ def test_empty_required_number_type_of_question(self): ).count() self.assertEqual(total_null_answers, 0) + # check N+1 query + def call_route(): + response = self.client.post( + "/api/v1/device/sync", + payload, + follow=True, + content_type="application/json", + **{"HTTP_AUTHORIZATION": f"Bearer {self.token}"}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + self.assertNumQueries(22, call_route) + def test_allowed_zero_required_number_type_of_question(self): mobile_adm = self.mobile_user.administrations.first() uuid = "32b9ecb2-c400-4b76-bcba-0a70a6942bb6" @@ -182,6 +208,19 @@ def test_allowed_zero_required_number_type_of_question(self): ).count() self.assertEqual(total_null_answers, 0) + # check N+1 query + def call_route(): + response = self.client.post( + "/api/v1/device/sync", + payload, + follow=True, + content_type="application/json", + **{"HTTP_AUTHORIZATION": f"Bearer {self.token}"}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + self.assertNumQueries(23, call_route) + def test_empty_required_option_type_of_question(self): mobile_adm = self.mobile_user.administrations.first() uuid = "42b9ecb2-c400-4b76-bcba-0a70a6942bb6" @@ -230,6 +269,19 @@ def test_empty_required_option_type_of_question(self): ).count() self.assertEqual(total_null_answers, 0) + # check N+1 query + def call_route(): + response = self.client.post( + "/api/v1/device/sync", + payload, + follow=True, + content_type="application/json", + **{"HTTP_AUTHORIZATION": f"Bearer {self.token}"}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + self.assertNumQueries(21, call_route) + def test_empty_required_multiple_options_type_of_question(self): mobile_adm = self.mobile_user.administrations.first() uuid = "52b9ecb2-c400-4b76-bcba-0a70a6942bb6" @@ -278,6 +330,19 @@ def test_empty_required_multiple_options_type_of_question(self): ).count() self.assertEqual(total_null_answers, 0) + # check N+1 query + def call_route(): + response = self.client.post( + "/api/v1/device/sync", + payload, + follow=True, + content_type="application/json", + **{"HTTP_AUTHORIZATION": f"Bearer {self.token}"}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + self.assertNumQueries(22, call_route) + def test_empty_required_geo_type_of_question(self): mobile_adm = self.mobile_user.administrations.first() uuid = "62b9ecb2-c400-4b76-bcba-0a70a6942bb6" @@ -326,6 +391,19 @@ def test_empty_required_geo_type_of_question(self): ).count() self.assertEqual(total_null_answers, 0) + # check N+1 query + def call_route(): + response = self.client.post( + "/api/v1/device/sync", + payload, + follow=True, + content_type="application/json", + **{"HTTP_AUTHORIZATION": f"Bearer {self.token}"}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + self.assertNumQueries(22, call_route) + def test_empty_required_hidden_from_registration_type(self): mobile_adm = self.mobile_user.administrations.first() uuid = "72b9ecb2-c400-4b76-bcba-0a70a6942bb6" @@ -376,6 +454,19 @@ def test_empty_required_hidden_from_registration_type(self): ).count() self.assertEqual(total_null_answers, 0) + # check N+1 query + def call_route(): + response = self.client.post( + "/api/v1/device/sync", + payload, + follow=True, + content_type="application/json", + **{"HTTP_AUTHORIZATION": f"Bearer {self.token}"}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + self.assertNumQueries(24, call_route) + def test_empty_non_required_autofield_type_of_question(self): mobile_adm = self.mobile_user.administrations.first() uuid = "82b9ecb2-c400-4b76-bcba-0a70a6942bb6" @@ -425,6 +516,18 @@ def test_empty_non_required_autofield_type_of_question(self): ).count() self.assertEqual(total_null_answers, 0) + def call_route(): + response = self.client.post( + "/api/v1/device/sync", + payload, + follow=True, + content_type="application/json", + **{"HTTP_AUTHORIZATION": f"Bearer {self.token}"}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + self.assertNumQueries(23, call_route) + def test_empty_required_meta_uuid_type_of_question(self): mobile_adm = self.mobile_user.administrations.first() st = SubmissionTypes.registration @@ -473,6 +576,19 @@ def test_empty_required_meta_uuid_type_of_question(self): ).count() self.assertEqual(total_null_answers, 0) + # check N+1 query + def call_route(): + response = self.client.post( + "/api/v1/device/sync", + payload, + follow=True, + content_type="application/json", + **{"HTTP_AUTHORIZATION": f"Bearer {self.token}"}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + self.assertNumQueries(21, call_route) + def test_empty_required_photo_type_of_question(self): mobile_adm = self.mobile_user.administrations.first() uuid = "92b9ecb2-c400-4b76-bcba-0a70a6942bb6" @@ -521,6 +637,19 @@ def test_empty_required_photo_type_of_question(self): ).count() self.assertEqual(total_null_answers, 0) + # check N+1 query + def call_route(): + response = self.client.post( + "/api/v1/device/sync", + payload, + follow=True, + content_type="application/json", + **{"HTTP_AUTHORIZATION": f"Bearer {self.token}"}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + self.assertNumQueries(22, call_route) + def test_empty_required_date_type_of_question(self): mobile_adm = self.mobile_user.administrations.first() uuid = "93b9ecb2-c400-4b76-bcba-0a10a6942bb6" @@ -569,6 +698,19 @@ def test_empty_required_date_type_of_question(self): ).count() self.assertEqual(total_null_answers, 0) + # check N+1 query + def call_route(): + response = self.client.post( + "/api/v1/device/sync", + payload, + follow=True, + content_type="application/json", + **{"HTTP_AUTHORIZATION": f"Bearer {self.token}"}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + self.assertNumQueries(22, call_route) + def test_valid_pending_answers_for_all_questions(self): mobile_adm = self.mobile_user.administrations.first() uuid = "94b9ecb2-c400-4b76-bcba-0a20a6942bb6" @@ -653,3 +795,16 @@ def test_valid_pending_answers_for_all_questions(self): self.assertEqual(a_109.value, 5.1) self.assertEqual(a_110.name, uuid) self.assertEqual(a_111.name, "10.2") + + # check N+1 query + def call_route(): + response = self.client.post( + "/api/v1/device/sync", + payload, + follow=True, + content_type="application/json", + **{"HTTP_AUTHORIZATION": f"Bearer {self.token}"}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + self.assertNumQueries(24, call_route) diff --git a/backend/api/v1/v1_mobile/urls.py b/backend/api/v1/v1_mobile/urls.py index b94562184..0b1a0f2c4 100644 --- a/backend/api/v1/v1_mobile/urls.py +++ b/backend/api/v1/v1_mobile/urls.py @@ -28,7 +28,11 @@ r"^(?P(v1))/device/form/(?P[0-9]+)", get_mobile_form_details, ), - re_path(r"^(?P(v1))/device/sync", sync_pending_form_data), + re_path( + r"^(?P(v1))/device/sync", + sync_pending_form_data, + name="device-sync" + ), re_path( r"^(?P(v1))/device/sqlite/(?P.*)$", download_sqlite_file, diff --git a/backend/api/v1/v1_mobile/views.py b/backend/api/v1/v1_mobile/views.py index 64a691957..fb4e21c08 100644 --- a/backend/api/v1/v1_mobile/views.py +++ b/backend/api/v1/v1_mobile/views.py @@ -171,7 +171,10 @@ def sync_pending_form_data(request, version): user = assignment.user # TODO : Get certifications administration list certifications = assignment.certifications.count() - administration = Access.objects.filter(user=user).first().administration + # administration = Access.objects.filter(user=user).first().administration + administration = Access.objects.select_related( + 'administration').filter(user=user).first().administration + if not request.data.get("answers"): return Response( {"message": "Answers is required."}, diff --git a/backend/api/v1/v1_profile/views.py b/backend/api/v1/v1_profile/views.py index 55409d040..63419b338 100644 --- a/backend/api/v1/v1_profile/views.py +++ b/backend/api/v1/v1_profile/views.py @@ -351,7 +351,7 @@ def export_prefilled_administrations_template(request: Request, version): OpenApiParameter( name="entity_ids", required=False, - type={"entity_ids": "array", "items": {"type": "number"}}, + type={"type": "array", "items": {"type": "number"}}, location=OpenApiParameter.QUERY, explode=False, ), @@ -397,9 +397,9 @@ def export_entity_data(request: Request, version): summary="Export template for Entities data bulk upload", parameters=[ OpenApiParameter( - name="entity_ids", + name="entity_types", required=True, - type={"entity_ids": "array", "items": {"type": "number"}}, + type={"type": "array", "items": {"type": "number"}}, location=OpenApiParameter.QUERY, explode=False, ), @@ -415,7 +415,7 @@ def export_entities_data_template(request: Request, version): status=status.HTTP_400_BAD_REQUEST, ) entity_ids = clean_array_param( - request.query_params.get("entity_ids", ""), maybe_int + request.query_params.get("entity_types", ""), maybe_int ) filepath = generate_entities_data_excel( cast(SystemUser, request.user), entity_ids @@ -440,7 +440,7 @@ def export_entities_data_template(request: Request, version): OpenApiParameter( name="entity_ids", required=False, - type={"entity_ids": "array", "items": {"type": "number"}}, + type={"type": "array", "items": {"type": "number"}}, location=OpenApiParameter.QUERY, explode=False, ), diff --git a/backend/source/forms/1699353915355.prod.json b/backend/source/forms/1699353915355.prod.json index 1c15389c5..4ce238a11 100644 --- a/backend/source/forms/1699353915355.prod.json +++ b/backend/source/forms/1699353915355.prod.json @@ -7201,7 +7201,7 @@ "G2": "#93D371", "G3": "#4088F4" }, - "fnString": "(!#o_r_no_exposed_human_excreta# || #o_r_no_exposed_human_excreta#.includes(\"G1\")) && (!#o_r_safe_management_of_child_excreta_and_diapers# || #o_r_safe_management_of_child_excreta_and_diapers#.includes(\"G1\")) ? 'G1' : ((!#o_r_no_exposed_human_excreta# || #o_r_no_exposed_human_excreta#.includes(\"G1\")) && (!#o_r_safe_management_of_child_excreta_and_diapers# || #o_r_safe_management_of_child_excreta_and_diapers#.includes(\"G1\")) && (!#o_r_safe_food_hygiene# || #o_r_safe_food_hygiene#.includes(\"G2\")) && (!#o_r_safe_water_management# || #o_r_safe_water_management#.includes(\"G2\")) && (!#o_r_safe_management_of_animals_wastes# || #o_r_safe_management_of_animals_wastes#.includes(\"G2\")) ? 'G2' : ((!#o_r_no_exposed_human_excreta# || #o_r_no_exposed_human_excreta#.includes(\"G1\")) && (!#o_r_safe_management_of_child_excreta_and_diapers# || #o_r_safe_management_of_child_excreta_and_diapers#.includes(\"G1\")) && (!#o_r_safe_food_hygiene# || #o_r_safe_food_hygiene#.includes(\"G2\")) && (!#o_r_safe_water_management# || #o_r_safe_water_management#.includes(\"G2\")) && (!#o_r_safe_management_of_animals_wastes# || #o_r_safe_management_of_animals_wastes#.includes(\"G2\")) && (!#o_r_safe_waste_management# || #o_r_safe_waste_management#.includes(\"G3\")) && (!#o_r_good_personal_hygiene# || #o_r_good_personal_hygiene#.includes(\"G3\")) && (!#o_r_good_nutrition# || #o_r_good_nutrition#.includes(\"G3\")) && (!#o_r_endemic# || #o_r_endemic#.includes(\"G3\")) ? 'G3' : 'G0')", + "fnString": "(!#o_r_safe_management_of_child_excreta_and_diapers# || #o_r_safe_management_of_child_excreta_and_diapers#.includes(\"G0\")) && (!#o_r_safe_food_hygiene# || #o_r_safe_food_hygiene#.includes(\"G0\")) && (!#o_r_safe_water_management# || #o_r_safe_water_management#.includes(\"G0\")) && (!#o_r_safe_management_of_animals_wastes# || #o_r_safe_management_of_animals_wastes#.includes(\"G0\")) && (!#o_r_safe_waste_management# || #o_r_safe_waste_management#.includes(\"G0\")) && (!#o_r_good_personal_hygiene# || #o_r_good_personal_hygiene#.includes(\"G0\")) && (!#o_r_good_nutrition# || #o_r_good_nutrition#.includes(\"G0\")) && (!#o_r_endemic# || #o_r_endemic#.includes(\"G1\")) ? \"G0 Unsafe child excreta and diaper management\" : (!#o_r_safe_management_of_child_excreta_and_diapers# || #o_r_safe_management_of_child_excreta_and_diapers#.includes(\"G0\")) && (!#o_r_safe_food_hygiene# || #o_r_safe_food_hygiene#.includes(\"G1\")) && (!#o_r_safe_water_management# || #o_r_safe_water_management#.includes(\"G1\")) && (!#o_r_safe_management_of_animals_wastes# || #o_r_safe_management_of_animals_wastes#.includes(\"G1\")) && (!#o_r_safe_waste_management# || #o_r_safe_waste_management#.includes(\"G1\")) && (!#o_r_good_personal_hygiene# || #o_r_good_personal_hygiene#.includes(\"G1\")) && (!#o_r_good_nutrition# || #o_r_good_nutrition#.includes(\"G1\")) && (!#o_r_endemic# || #o_r_endemic#.includes(\"G1\")) ? \"G0 Unsafe child excreta and diaper management\" : (!#o_r_safe_management_of_child_excreta_and_diapers# || #o_r_safe_management_of_child_excreta_and_diapers#.includes(\"G1\")) && (!#o_r_safe_food_hygiene# || #o_r_safe_food_hygiene#.includes(\"G1\")) && (!#o_r_safe_water_management# || #o_r_safe_water_management#.includes(\"G1\")) && (!#o_r_safe_management_of_animals_wastes# || #o_r_safe_management_of_animals_wastes#.includes(\"G1\")) && (!#o_r_safe_waste_management# || #o_r_safe_waste_management#.includes(\"G1\")) && (!#o_r_good_personal_hygiene# || #o_r_good_personal_hygiene#.includes(\"G1\")) && (!#o_r_good_nutrition# || #o_r_good_nutrition#.includes(\"G1\")) && (!#o_r_endemic# || #o_r_endemic#.includes(\"G1\")) ? \"G1 Safe management of child excreta and diapers\" : (!#o_r_safe_management_of_child_excreta_and_diapers# || #o_r_safe_management_of_child_excreta_and_diapers#.includes(\"G1\")) && (!#o_r_safe_food_hygiene# || #o_r_safe_food_hygiene#.includes(\"G2\")) && (!#o_r_safe_water_management# || #o_r_safe_water_management#.includes(\"G2\")) && (!#o_r_safe_management_of_animals_wastes# || #o_r_safe_management_of_animals_wastes#.includes(\"G2\")) && (!#o_r_safe_waste_management# || #o_r_safe_waste_management#.includes(\"G1\")) && (!#o_r_good_personal_hygiene# || #o_r_good_personal_hygiene#.includes(\"G1\")) && (!#o_r_good_nutrition# || #o_r_good_nutrition#.includes(\"G1\")) && (!#o_r_endemic# || #o_r_endemic#.includes(\"G1\")) ? \"G2 Safe food, water and animal management\" : \"\"", "multiline": false } }, diff --git a/backend/source/forms/1699354006503.prod.json b/backend/source/forms/1699354006503.prod.json index e029aec11..4a90deaa2 100644 --- a/backend/source/forms/1699354006503.prod.json +++ b/backend/source/forms/1699354006503.prod.json @@ -1156,7 +1156,7 @@ "G2 Inadequate vector control in communal areas": "#93D371", "G3 Good vector control in communal areas": "#4088F4" }, - "fnString": "(#no_standing_water_communal_areas#.includes(\"g0\") ) ? \"G1 Lack of vector control in communal areas\" : ((#no_standing_water_communal_areas#.includes(\"g3\") ) ? \"G3 Good vector control in communal areas\" : \"G2 Inadequate vector control in communal areas\")", + "fnString": "#no_standing_water_communal_areas#.includes(\"g0\") ? \"G1 Lack of vector control in communal areas\" : #no_standing_water_communal_areas#.includes(\"g3\") ? \"G3 Good vector control in communal areas\" : \"G2 Inadequate vector control in communal areas\"", "multiline": false } }, diff --git a/backend/utils/upload_entities.py b/backend/utils/upload_entities.py index b201d97f0..81f3ffc5f 100644 --- a/backend/utils/upload_entities.py +++ b/backend/utils/upload_entities.py @@ -65,6 +65,12 @@ def generate_list_of_entities( return url +def normalize_string(s): + if s is None: + return "" + return s.strip().replace('"', '') + + def validate_entity_data(filename: str): errors = [] last_level = Levels.objects.all().order_by("level").last() @@ -95,6 +101,13 @@ def validate_entity_data(filename: str): administration = None higher_level = None for level in Levels.objects.all().order_by("level"): + if level.name not in row: + errors.append({ + "sheet": entity.name, + "row": 1, + "message": f"Header {level.name} is missing", + }) + continue if row[level.name] != row[level.name]: previous_level = Levels.objects.filter( level=level.level - 1 @@ -102,11 +115,11 @@ def validate_entity_data(filename: str): if not higher_level: higher_level = previous_level else: - row_value = row[level.name] + row_value = normalize_string(row[level.name]) adm_names += [row_value] administration = Administration.objects.filter( + Q(name__iexact=row_value), parent=administration, - name=row_value, level=level ).first() if not administration: @@ -120,7 +133,7 @@ def validate_entity_data(filename: str): "message": f"Invalid Administration for {adm_names}", }) else: - if level == last_level: + if level == last_level and not bool(pd.isnull(row["Name"])): # skip if the entity data already exists entity_name = row["Name"] entity_data = EntityData.objects.filter(