Merge pull request #243 from awf-dbca/filter-fixes

Filter fixes

wildlifecompliance/components/licences/api.py

@@ -94,53 +94,93 @@ def filter_queryset(self, request, queryset, view):
             # apply user selected filters
             category_name = category_name.lower() if category_name else 'all'
             if category_name != 'all':
-                category_name_licence_ids = []
-                for wildlifelicence in queryset:
-                    if category_name in wildlifelicence.current_application.licence_category_name.lower():
-                        category_name_licence_ids.append(wildlifelicence.id)
-                queryset = queryset.filter(id__in=category_name_licence_ids)
+                #category_name_licence_ids = []
+                #for wildlifelicence in queryset:
+                #    if category_name in wildlifelicence.current_application.licence_category_name.lower():
+                #        category_name_licence_ids.append(wildlifelicence.id)
+                #queryset = queryset.filter(id__in=category_name_licence_ids)
+                queryset = queryset.filter(licence_category__name__iexact=category_name)
+
+            #TODO: prior to fix the date filters did not work due to an uncaught nonetype error (fixed in the commented code)
+            #however, even when fixed the filter a) excluded surrendered licences and b) took far too long (recorded for as long as 5 minutes!)
+            #the current implementation does not exclude surrendered records and loads in <1s
+            #however, the displayed issue date does not always match what is filtered in some cases - what is displayed/filtered will need review
             if date_from:
-                date_from_licence_ids = []
-                for wildlifelicence in queryset:
-                    # Order the queryset 'from' by issue date licence purpose.
-                    # if (pytz.timezone('utc').localize(datetime.strptime(date_from, '%Y-%m-%d'))
-                    #         <= wildlifelicence.current_activities.order_by('-issue_date').first().issue_date):
-                    #             date_from_licence_ids.append(wildlifelicence.id)
-                    _date_from = pytz.timezone('utc').localize(
+                _date_from = pytz.timezone('utc').localize(
                         datetime.strptime(date_from, '%Y-%m-%d'))
-
-                    _issue_date = wildlifelicence.current_activities.first(
-                        ).get_issue_date()
-
-                    if _date_from <= _issue_date:
-                        date_from_licence_ids.append(wildlifelicence.id)
-
-                queryset = queryset.filter(id__in=date_from_licence_ids)
+                queryset = queryset.filter(current_application__selected_activities__proposed_purposes__issue_date__gte=_date_from)                   
+                #date_from_licence_ids = []
+                #for wildlifelicence in queryset:
+                #    print("licence activities")
+                #    print(wildlifelicence.current_application)
+                #    # Order the queryset 'from' by issue date licence purpose.
+                #    # if (pytz.timezone('utc').localize(datetime.strptime(date_from, '%Y-%m-%d'))
+                #    #         <= wildlifelicence.current_activities.order_by('-issue_date').first().issue_date):
+                #    #             date_from_licence_ids.append(wildlifelicence.id)
+                #    _date_from = pytz.timezone('utc').localize(
+                #        datetime.strptime(date_from, '%Y-%m-%d'))
+                #    
+                #    if (wildlifelicence.current_activities.count()):
+                #        _issue_date = wildlifelicence.current_activities.first(
+                #            ).get_issue_date()
+#
+                #        if _issue_date and _date_from <= _issue_date:
+                #            date_from_licence_ids.append(wildlifelicence.id)
+#
+                #queryset = queryset.filter(id__in=date_from_licence_ids)
+
             if date_to:
-                date_to_licence_ids = []
-                for wildlifelicence in queryset:
-                    # Order the queryset 'to' by issue date on licence purpose.
-                    # if (pytz.timezone('utc').localize(datetime.strptime(date_to, '%Y-%m-%d')) + timedelta(days=1)
-                    #         >= wildlifelicence.current_activities.order_by('-issue_date').first().issue_date):
-                    #             date_to_licence_ids.append(wildlifelicence.id)
-                    _date_to = pytz.timezone('utc').localize(
+
+                _date_to = pytz.timezone('utc').localize(
                         datetime.strptime(date_to, '%Y-%m-%d')
                         ) + timedelta(days=1)
-
-                    _issue_date = wildlifelicence.current_activities.first(
-                         ).get_issue_date()
-
-                    if _date_to >= _issue_date:
-                        date_to_licence_ids.append(wildlifelicence.id)
-
-                queryset = queryset.filter(id__in=date_to_licence_ids)
+                queryset = queryset.filter(current_application__selected_activities__proposed_purposes__issue_date__lte=_date_to)   
+                #date_to_licence_ids = []
+                #for wildlifelicence in queryset:
+                #    # Order the queryset 'to' by issue date on licence purpose.
+                #    # if (pytz.timezone('utc').localize(datetime.strptime(date_to, '%Y-%m-%d')) + timedelta(days=1)
+                #    #         >= wildlifelicence.current_activities.order_by('-issue_date').first().issue_date):
+                #    #             date_to_licence_ids.append(wildlifelicence.id)
+                #    _date_to = pytz.timezone('utc').localize(
+                #        datetime.strptime(date_to, '%Y-%m-%d')
+                #        ) + timedelta(days=1)
+#
+                #    if (wildlifelicence.current_activities.count()):
+                #        _issue_date = wildlifelicence.current_activities.first(
+                #            ).get_issue_date()
+#
+                #        if _issue_date and _date_to >= _issue_date:
+                #            date_to_licence_ids.append(wildlifelicence.id)
+#
+                #queryset = queryset.filter(id__in=date_to_licence_ids)
             holder = holder.lower() if holder else 'all'
             if holder != 'all':
-                holder_licence_ids = []
-                for wildlifelicence in queryset:
-                    if holder in wildlifelicence.current_application.applicant.lower():
-                        holder_licence_ids.append(wildlifelicence.id)
-                queryset = queryset.filter(id__in=holder_licence_ids)
+                #holder_licence_ids = []
+                #for wildlifelicence in queryset:
+                #    if holder in wildlifelicence.current_application.applicant.lower():
+                #        holder_licence_ids.append(wildlifelicence.id)
+                #queryset = queryset.filter(id__in=holder_licence_ids)
+                queryset = queryset.annotate(
+                    applicant_name=Case(
+                        When(
+                            current_application__proxy_applicant__isnull=False,
+                            then=Concat(
+                                'current_application__proxy_applicant__first_name',
+                                Value(' '),
+                                'current_application__proxy_applicant__last_name',
+                                Value(''),
+                            )
+                        ),
+                        default=Concat(
+                            'current_application__submitter__first_name',
+                            Value(' '),
+                            'current_application__submitter__last_name',
+                            Value(''),
+                        ),
+                        output_field=CharField(),
+                    )
+                ).filter(Q(applicant_name__iexact=holder)
+                |Q(current_application__org_applicant__organisation__name__iexact=holder))
 
         # override queryset ordering, required because the ordering is usually handled
         # in the super call, but is then clobbered by the custom queryset joining above

wildlifecompliance/components/main/utils.py

@@ -428,33 +428,50 @@ def search_keywords(search_words, search_application, search_licence, search_ret
     application_list = []
     licence_list = []
     return_list = []
-    if is_internal:
-        application_list = Application.objects.all()\
-            .computed_exclude(processing_status__in=[
-                Application.PROCESSING_STATUS_DISCARDED
-            ])\
-            .order_by('lodgement_number', '-id')
-        licence_list = WildlifeLicence.objects.all()\
-            .order_by('licence_number', '-id')\
-            .distinct('licence_number')
-        return_list = Return.objects.all()\
-            .order_by('lodgement_number', '-id')
     if search_words:
+        search_words_regex = "(?:"
+        for i in range(0,len(search_words)):
+            search_words_regex = search_words_regex + search_words[i]
+            if i == len(search_words)-1:
+                search_words_regex = search_words_regex + ")"
+            else:
+                search_words_regex = search_words_regex + "|"
+
+        #filter_regex = ".*\".*\":\s\"(\\\\\"|[^\"])*"+search_words_regex+"(\\\\\"|[^\"])*\".*"
+        if is_internal:
+            application_list = Application.objects.all().filter(form_data_records__value__iregex=search_words_regex)\
+            .order_by('lodgement_number', '-id')\
+            .distinct('lodgement_number') 
+                #.computed_exclude(processing_status__in=[
+                #    Application.PROCESSING_STATUS_DISCARDED
+                #])\
+            licence_list = WildlifeLicence.objects\
+            .filter(licence_number__iregex=search_words_regex)\
+            .order_by('licence_number', '-id')\
+            .distinct('licence_number')  
+            return_list = Return.objects.all()\
+            .filter(lodgement_number__iregex=search_words_regex)\
+            .order_by('lodgement_number', '-id')\
+            .distinct('lodgement_number') 
+
         if search_application:
             for app in application_list:
                 if app.data:
                     try:
                         app_data = {'data': []}
                         for record in app.data:
                             if 'thead' in record.value:
-                                app_data.get('data').append({'value': ast.literal_eval(record.value).get('tbody')})
+                                try:
+                                    app_data.get('data').append({'value': ast.literal_eval(record.value).get('tbody')})
+                                except:
+                                    app_data.get('data').append({'value': record.value})
                             else:
                                 app_data.get('data').append({'value': record.value})
                         results = search(app_data, search_words)
                         final_results = {}
                         if results:
                             for r in results:
-                                for key, value in r.iteritems():
+                                for key, value in r.items():
                                     final_results.update({'key': key, 'value': value})
                             res = {
                                 'number': app.lodgement_number,
@@ -479,7 +496,7 @@ def search_keywords(search_words, search_application, search_licence, search_ret
                             })
                     if results:
                         for r in results:
-                            for key, value in r.iteritems():
+                            for key, value in r.items():
                                 final_results.update({'key': key, 'value': value})
                         res = {
                             'number': lic.licence_number,
@@ -504,7 +521,7 @@ def search_keywords(search_words, search_application, search_licence, search_ret
                             })
                     if results:
                         for r in results:
-                            for key, value in r.iteritems():
+                            for key, value in r.items():
                                 final_results.update({'key': key, 'value': value})
                         res = {
                             'number': ret.lodgement_number,
@@ -531,8 +548,9 @@ def search_reference(reference_number):
     from wildlifecompliance.components.inspection.models import Inspection
     from wildlifecompliance.components.sanction_outcome.models import SanctionOutcome
     from wildlifecompliance.components.artifact.models import Artifact
-    application_list = Application.objects.all().computed_exclude(processing_status__in=[
-        Application.PROCESSING_STATUS_DISCARDED])
+    application_list = Application.objects.all()
+    #this makes the ref search go from under a second to up to 2 minutes - it is better to allow discarded applications to be searched
+    #.computed_exclude(processing_status__in=[Application.PROCESSING_STATUS_DISCARDED])
     licence_list = WildlifeLicence.objects.all().order_by('licence_number').distinct('licence_number')
     returns_list = Return.objects.all().exclude(processing_status__in=[Return.RETURN_PROCESSING_STATUS_FUTURE])
     org_access_request_list = OrganisationRequest.objects.all()
@@ -542,43 +560,53 @@ def search_reference(reference_number):
     result = application_list.filter(lodgement_number=reference_number)
     if result:
         url_string = {'url_string': '/internal/application/' + str(result[0].id)}
+        return url_string
     # Licence
     result = licence_list.filter(licence_number=reference_number)
     if result and not url_string:
         url_string = {'url_string': result[0].licence_document._file.url}
+        return url_string
     # Returns
     result = returns_list.filter(lodgement_number=reference_number)
     if result and not url_string:
         url_string = {'url_string': '/internal/return/' + str(result[0].id)}
+        return url_string
     #import ipdb; ipdb.set_trace()
     # Org access reqeust
     result = org_access_request_list.filter(lodgement_number=reference_number)
     if result and not url_string:
         url_string = {'url_string': '/internal/organisations/access/' + str(result[0].id)}
+        return url_string
     # CallEmail
     result = CallEmail.objects.filter(number=reference_number)
     if result and not url_string:
         url_string = {'url_string': '/internal/call_email/' + str(result[0].id)}
+        return url_string
     # Offence
     result = Offence.objects.filter(lodgement_number=reference_number)
     if result and not url_string:
         url_string = {'url_string': '/internal/offence/' + str(result[0].id)}
+        return url_string
     # Legal Case
     result = LegalCase.objects.filter(number=reference_number)
     if result and not url_string:
         url_string = {'url_string': '/internal/legal_case/' + str(result[0].id)}
+        return url_string
     # Inspection
     result = Inspection.objects.filter(number=reference_number)
     if result and not url_string:
         url_string = {'url_string': '/internal/inspection/' + str(result[0].id)}
+        return url_string
     # Sanction Outcome
     result = SanctionOutcome.objects.filter(lodgement_number=reference_number)
     if result and not url_string:
         url_string = {'url_string': '/internal/sanction_outcome/' + str(result[0].id)}
+        return url_string
     # Offence
     result = Artifact.objects.filter(number=reference_number)
     if result and not url_string:
         url_string = {'url_string': '/internal/object/' + str(result[0].id)}
+        return url_string
 
     if url_string:
         return url_string

wildlifecompliance/components/organisations/api.py

@@ -13,6 +13,8 @@
 from django.views.decorators.http import require_http_methods
 from django.views.decorators.csrf import csrf_exempt
 from django.utils import timezone
+from django.db.models import Value
+from django.db.models.functions import Concat
 from rest_framework import viewsets, serializers, status, generics, views
 from rest_framework.decorators import detail_route, list_route, renderer_classes
 from rest_framework.response import Response
@@ -94,6 +96,10 @@ def filter_queryset(self, request, queryset, view):
 
         total_count = queryset.count()
         search_text = request.GET.get('search[value]')
+        organisation_name = request.GET.get('organisation')
+        applicant = request.GET.get('applicant')
+        role = request.GET.get('role')
+        status = request.GET.get('status')
 
         if queryset.model is Organisation:
             # search_text filter, join all custom search columns
@@ -116,13 +122,35 @@ def filter_queryset(self, request, queryset, view):
                 search_text = search_text.lower()
                 # join queries for the search_text search
                 search_text_org_request_ids = []
+                search_text_org_request_ids = OrganisationRequest.objects.filter(
+                Q(requester__first_name__icontains=search_text) |
+                Q(requester__last_name__icontains=search_text) |
+                Q(assigned_officer__first_name__icontains=search_text) |
+                Q(assigned_officer__last_name__icontains=search_text)
+                ).values('id')
                 # for organisation_request in queryset:
                 #     if search_text in organisation_request.address_string.lower():
                 #         search_text_org_request_ids.append(organisation_request.id)
                 # use pipe to join both custom and built-in DRF datatables querysets (returned by super call above)
                 # (otherwise they will filter on top of each other)
                 queryset = queryset.filter(id__in=search_text_org_request_ids).distinct() | super_queryset
 
+            # apply user selected filters
+            organisation_name = organisation_name.lower() if organisation_name else 'all'
+            if organisation_name != 'all':
+                queryset = queryset.filter(name__iexact=organisation_name)
+            applicant = applicant.lower() if applicant else 'all'
+            if applicant != 'all':
+                queryset = queryset.annotate(applicant_name=Concat('requester__first_name',
+                    Value(' '),'requester__last_name')).filter(applicant_name__iexact=applicant)
+            role = role.lower() if role else 'all'
+            if role != 'all':
+                queryset = queryset.filter(role__iexact=role)
+
+            status = status.lower().replace(" ","_") if status else 'all'
+            if status != 'all':
+                queryset = queryset.filter(status__iexact=status)
+
         # override queryset ordering, required because the ordering is usually handled
         # in the super call, but is then clobbered by the custom queryset joining above
         # also needed to disable ordering for all fields for which data is not an

wildlifecompliance/components/returns/api.py

@@ -88,11 +88,11 @@ def filter_queryset(self, request, queryset, view):
             if date_from:
                 date_from = datetime.strptime(
                     date_from, '%Y-%m-%d') + timedelta(days=1)
-                queryset = queryset.filter(lodgement_date__gte=date_from)
+                queryset = queryset.filter(due_date__gte=date_from)
             if date_to:
                 date_to = datetime.strptime(
                     date_to, '%Y-%m-%d') + timedelta(days=1)
-                queryset = queryset.filter(lodgement_date__lte=date_to)
+                queryset = queryset.filter(due_date__lte=date_to)
 
         return queryset