From 291a52029aa750d98d9044730b2b50be14fcf5c8 Mon Sep 17 00:00:00 2001 From: Dennis Wambua Date: Tue, 16 Nov 2021 12:12:45 +0300 Subject: [PATCH 1/7] Set cache for submission stats --- onadata/apps/api/tasks.py | 35 ++++++++++++++ .../api/tests/viewsets/test_stats_viewset.py | 19 ++++++++ .../api/viewsets/submissionstats_viewset.py | 16 ++++++- onadata/libs/serializers/stats_serializer.py | 47 ++++++++++++++++++- onadata/libs/utils/cache_tools.py | 1 + onadata/settings/common.py | 2 + 6 files changed, 118 insertions(+), 2 deletions(-) diff --git a/onadata/apps/api/tasks.py b/onadata/apps/api/tasks.py index b8e1f4ff2f..f94b6bab67 100644 --- a/onadata/apps/api/tasks.py +++ b/onadata/apps/api/tasks.py @@ -4,6 +4,7 @@ from celery.result import AsyncResult from django.core.files.uploadedfile import TemporaryUploadedFile +from django.core.cache import cache from django.core.files.storage import default_storage from django.contrib.auth.models import User from django.utils.datastructures import MultiValueDict @@ -13,6 +14,8 @@ from onadata.libs.utils.email import send_generic_email from onadata.apps.logger.models.xform import XForm from onadata.celery import app +from onadata.libs.utils.cache_tools import XFORM_SUBMISSION_STAT +from onadata.libs.data.query import get_form_submissions_grouped_by_field def recreate_tmp_file(name, path, mime_type): @@ -98,3 +101,35 @@ def send_verification_email(email, message_txt, subject): @app.task() def send_account_lockout_email(email, message_txt, subject): send_generic_email(email, message_txt, subject) + + +@app.task() +def get_form_submissions_stats_async(xform_id, field, name=None, + data_view=None): + xform = XForm.objects.get(pk=xform_id) + data = { + "STATE": "INITIALIZED", + "DATA": [] + } + cache.set('{}{}{}{}'.format(XFORM_SUBMISSION_STAT, xform.pk, field, name), data) + + try: + data = get_form_submissions_grouped_by_field(xform, field, name, data_view) + except ValueError as e: + raise exceptions.ParseError(detail=e) + else: + if data: + element = xform.get_survey_element(field) + + if element and element.type in SELECT_FIELDS: + for record in data: + label = xform.get_choice_label(element, record[name]) + record[name] = label + + data_to_cache = { + "STATE": "PROCESSED", + "DATA": data + } + + # update the cache + cache.set('{}{}{}{}'.format(XFORM_SUBMISSION_STAT, xform.pk, field, name), data_to_cache) \ No newline at end of file diff --git a/onadata/apps/api/tests/viewsets/test_stats_viewset.py b/onadata/apps/api/tests/viewsets/test_stats_viewset.py index ecd453646f..44f301190f 100644 --- a/onadata/apps/api/tests/viewsets/test_stats_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_stats_viewset.py @@ -5,6 +5,7 @@ from django.test import RequestFactory from builtins import open from mock import patch +from django.test.utils import override_settings from onadata.apps.main.tests.test_base import TestBase from onadata.apps.api.viewsets.stats_viewset import StatsViewSet @@ -66,6 +67,23 @@ def test_submissions_stats(self, mock_time): } self.assertDictContainsSubset(data, response.data[0]) + @override_settings(CELERY_TASK_ALWAYS_EAGER=True) + @patch('onadata.apps.logger.models.instance.submission_time') + def test_submissions_stats_async(self, mock_time): + self._set_mock_time(mock_time) + self._publish_transportation_form() + self._make_submissions() + view = SubmissionStatsViewSet.as_view({'get': 'list'}) + request = self.factory.get('/?group=_xform_id_string&async=true', **self.extra) + formid = self.xform.pk + response = view(request, pk=formid) + self.assertEqual(response.status_code, 200) + self.assertIsInstance(response.data, dict) + + data = { + u'count': 4 + } + self.assertDictContainsSubset(data, response.data["DATA"][0]) @patch('onadata.apps.logger.models.instance.submission_time') def test_submissions_stats_with_xform_in_delete_async_queue( @@ -261,3 +279,4 @@ def test_wrong_stat_function_api(self): self.assertNotEqual(response.get('Cache-Control'), None) self.assertEquals(response.status_code, 200) + diff --git a/onadata/apps/api/viewsets/submissionstats_viewset.py b/onadata/apps/api/viewsets/submissionstats_viewset.py index 2b80e28fc9..910b6533a9 100644 --- a/onadata/apps/api/viewsets/submissionstats_viewset.py +++ b/onadata/apps/api/viewsets/submissionstats_viewset.py @@ -1,4 +1,7 @@ from rest_framework import viewsets +from django.conf import settings +from rest_framework import status +from rest_framework.response import Response from onadata.apps.api.permissions import XFormPermissions from onadata.apps.logger.models.xform import XForm @@ -10,8 +13,10 @@ from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.serializers.stats_serializer import ( - SubmissionStatsSerializer, SubmissionStatsInstanceSerializer) + SubmissionStatsSerializer, SubmissionStatsInstanceSerializer, + SubmissionStatsInstanceSerializerAsync) from onadata.apps.api.tools import get_baseviewset_class +from onadata.libs.utils.string import str2bool BaseViewset = get_baseviewset_class() @@ -78,3 +83,12 @@ def get_serializer_class(self): super(SubmissionStatsViewSet, self).get_serializer_class() return serializer_class + + def list(self, request, *args, **kwargs): + xform = self.get_object() + in_async = request.query_params.get("async") + if xform.num_of_submissions > settings.XFORM_SUBMISSION_STAT_ASYNC_TRESHOLD or str2bool(in_async): + serializer = SubmissionStatsInstanceSerializerAsync(xform, context={'request': request}) + return Response(serializer.data, status=status.HTTP_200_OK) + else: + return super(SubmissionStatsViewSet, self).list(request, *args, **kwargs) diff --git a/onadata/libs/serializers/stats_serializer.py b/onadata/libs/serializers/stats_serializer.py index 186d373161..a5764bae26 100644 --- a/onadata/libs/serializers/stats_serializer.py +++ b/onadata/libs/serializers/stats_serializer.py @@ -1,7 +1,10 @@ from django.utils.translation import ugettext as _ +from django.core.cache import cache +from django.conf import settings + from rest_framework import exceptions from rest_framework import serializers -from rest_framework.utils.serializer_helpers import ReturnList +from rest_framework.utils.serializer_helpers import ReturnList, ReturnDict from onadata.libs.data.statistics import\ get_median_for_numeric_fields_in_form,\ @@ -10,6 +13,10 @@ from onadata.apps.logger.models.xform import XForm from onadata.libs.data.query import get_form_submissions_grouped_by_field +from onadata.libs.utils.cache_tools import XFORM_SUBMISSION_STAT +from onadata.apps.api import tasks + + SELECT_FIELDS = ['select one', 'select multiple'] STATS_FUNCTIONS = { @@ -43,6 +50,11 @@ def to_representation(self, obj): raise exceptions.ParseError(_(u"Expecting `group` and `name`" u" query parameters.")) + data = cache.get( + '{}{}{}{}'.format(XFORM_SUBMISSION_STAT, obj.pk, field, name)) + if data: + return data + try: data = get_form_submissions_grouped_by_field( obj, field, name) @@ -57,6 +69,8 @@ def to_representation(self, obj): label = obj.get_choice_label(element, record[name]) record[name] = label + cache.set('{}{}{}{}'.format(XFORM_SUBMISSION_STAT, obj.pk, field, name), data) + return data @property @@ -65,6 +79,37 @@ def data(self): return ReturnList(ret, serializer=self) +class SubmissionStatsInstanceSerializerAsync(serializers.Serializer): + def to_representation(self, obj): + if obj is None: + return super(SubmissionStatsInstanceSerializerAsync, self)\ + .to_representation(obj) + + request = self.context.get('request') + field = request.query_params.get('group') + name = request.query_params.get('name', field) + + + data = cache.get( + '{}{}{}{}'.format(XFORM_SUBMISSION_STAT, obj.pk, field, name)) + if data: + return data + + tasks.get_form_submissions_stats_async(obj.id, field, name) + + data = cache.get( + '{}{}{}{}'.format(XFORM_SUBMISSION_STAT, obj.pk, field, name)) + if data: + return data + + + + @property + def data(self): + ret = super(serializers.Serializer, self).data + + return ReturnDict(ret, serializer=self) + class StatsSerializer(serializers.HyperlinkedModelSerializer): url = serializers.HyperlinkedIdentityField( diff --git a/onadata/libs/utils/cache_tools.py b/onadata/libs/utils/cache_tools.py index 3ab17f88dc..3e0a16d290 100644 --- a/onadata/libs/utils/cache_tools.py +++ b/onadata/libs/utils/cache_tools.py @@ -44,6 +44,7 @@ # Cache names used in XForm Model XFORM_SUBMISSION_COUNT_FOR_DAY = "xfm-get_submission_count-" XFORM_SUBMISSION_COUNT_FOR_DAY_DATE = "xfm-get_submission_count_date-" +XFORM_SUBMISSION_STAT = "xfm-get_form_submissions_grouped_by_field" def safe_delete(key): diff --git a/onadata/settings/common.py b/onadata/settings/common.py index f38ba1ab7b..9f29a1772d 100644 --- a/onadata/settings/common.py +++ b/onadata/settings/common.py @@ -603,3 +603,5 @@ def configure_logging(logger, **kwargs): # Project & XForm Visibility Settings ALLOW_PUBLIC_DATASETS = True + +XFORM_SUBMISSION_STAT_ASYNC_TRESHOLD = 100000 From f34c8544e69bd4fe482999b591d946d82618c5f4 Mon Sep 17 00:00:00 2001 From: Dennis Wambua Date: Tue, 4 Jan 2022 14:16:16 +0300 Subject: [PATCH 2/7] cache submission stats results for one hour --- onadata/apps/api/tasks.py | 32 ---------------- .../api/tests/viewsets/test_stats_viewset.py | 18 --------- .../api/viewsets/submissionstats_viewset.py | 16 +------- onadata/libs/serializers/stats_serializer.py | 37 ++----------------- onadata/settings/common.py | 3 +- 5 files changed, 6 insertions(+), 100 deletions(-) diff --git a/onadata/apps/api/tasks.py b/onadata/apps/api/tasks.py index f94b6bab67..a0efd306b9 100644 --- a/onadata/apps/api/tasks.py +++ b/onadata/apps/api/tasks.py @@ -101,35 +101,3 @@ def send_verification_email(email, message_txt, subject): @app.task() def send_account_lockout_email(email, message_txt, subject): send_generic_email(email, message_txt, subject) - - -@app.task() -def get_form_submissions_stats_async(xform_id, field, name=None, - data_view=None): - xform = XForm.objects.get(pk=xform_id) - data = { - "STATE": "INITIALIZED", - "DATA": [] - } - cache.set('{}{}{}{}'.format(XFORM_SUBMISSION_STAT, xform.pk, field, name), data) - - try: - data = get_form_submissions_grouped_by_field(xform, field, name, data_view) - except ValueError as e: - raise exceptions.ParseError(detail=e) - else: - if data: - element = xform.get_survey_element(field) - - if element and element.type in SELECT_FIELDS: - for record in data: - label = xform.get_choice_label(element, record[name]) - record[name] = label - - data_to_cache = { - "STATE": "PROCESSED", - "DATA": data - } - - # update the cache - cache.set('{}{}{}{}'.format(XFORM_SUBMISSION_STAT, xform.pk, field, name), data_to_cache) \ No newline at end of file diff --git a/onadata/apps/api/tests/viewsets/test_stats_viewset.py b/onadata/apps/api/tests/viewsets/test_stats_viewset.py index 44f301190f..cc917e48e3 100644 --- a/onadata/apps/api/tests/viewsets/test_stats_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_stats_viewset.py @@ -5,7 +5,6 @@ from django.test import RequestFactory from builtins import open from mock import patch -from django.test.utils import override_settings from onadata.apps.main.tests.test_base import TestBase from onadata.apps.api.viewsets.stats_viewset import StatsViewSet @@ -67,23 +66,6 @@ def test_submissions_stats(self, mock_time): } self.assertDictContainsSubset(data, response.data[0]) - @override_settings(CELERY_TASK_ALWAYS_EAGER=True) - @patch('onadata.apps.logger.models.instance.submission_time') - def test_submissions_stats_async(self, mock_time): - self._set_mock_time(mock_time) - self._publish_transportation_form() - self._make_submissions() - view = SubmissionStatsViewSet.as_view({'get': 'list'}) - request = self.factory.get('/?group=_xform_id_string&async=true', **self.extra) - formid = self.xform.pk - response = view(request, pk=formid) - self.assertEqual(response.status_code, 200) - self.assertIsInstance(response.data, dict) - - data = { - u'count': 4 - } - self.assertDictContainsSubset(data, response.data["DATA"][0]) @patch('onadata.apps.logger.models.instance.submission_time') def test_submissions_stats_with_xform_in_delete_async_queue( diff --git a/onadata/apps/api/viewsets/submissionstats_viewset.py b/onadata/apps/api/viewsets/submissionstats_viewset.py index 910b6533a9..2b80e28fc9 100644 --- a/onadata/apps/api/viewsets/submissionstats_viewset.py +++ b/onadata/apps/api/viewsets/submissionstats_viewset.py @@ -1,7 +1,4 @@ from rest_framework import viewsets -from django.conf import settings -from rest_framework import status -from rest_framework.response import Response from onadata.apps.api.permissions import XFormPermissions from onadata.apps.logger.models.xform import XForm @@ -13,10 +10,8 @@ from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.serializers.stats_serializer import ( - SubmissionStatsSerializer, SubmissionStatsInstanceSerializer, - SubmissionStatsInstanceSerializerAsync) + SubmissionStatsSerializer, SubmissionStatsInstanceSerializer) from onadata.apps.api.tools import get_baseviewset_class -from onadata.libs.utils.string import str2bool BaseViewset = get_baseviewset_class() @@ -83,12 +78,3 @@ def get_serializer_class(self): super(SubmissionStatsViewSet, self).get_serializer_class() return serializer_class - - def list(self, request, *args, **kwargs): - xform = self.get_object() - in_async = request.query_params.get("async") - if xform.num_of_submissions > settings.XFORM_SUBMISSION_STAT_ASYNC_TRESHOLD or str2bool(in_async): - serializer = SubmissionStatsInstanceSerializerAsync(xform, context={'request': request}) - return Response(serializer.data, status=status.HTTP_200_OK) - else: - return super(SubmissionStatsViewSet, self).list(request, *args, **kwargs) diff --git a/onadata/libs/serializers/stats_serializer.py b/onadata/libs/serializers/stats_serializer.py index a5764bae26..b0f470d477 100644 --- a/onadata/libs/serializers/stats_serializer.py +++ b/onadata/libs/serializers/stats_serializer.py @@ -4,7 +4,7 @@ from rest_framework import exceptions from rest_framework import serializers -from rest_framework.utils.serializer_helpers import ReturnList, ReturnDict +from rest_framework.utils.serializer_helpers import ReturnList from onadata.libs.data.statistics import\ get_median_for_numeric_fields_in_form,\ @@ -14,7 +14,6 @@ from onadata.libs.data.query import get_form_submissions_grouped_by_field from onadata.libs.utils.cache_tools import XFORM_SUBMISSION_STAT -from onadata.apps.api import tasks SELECT_FIELDS = ['select one', 'select multiple'] @@ -69,7 +68,8 @@ def to_representation(self, obj): label = obj.get_choice_label(element, record[name]) record[name] = label - cache.set('{}{}{}{}'.format(XFORM_SUBMISSION_STAT, obj.pk, field, name), data) + cache.set('{}{}{}{}'.format(XFORM_SUBMISSION_STAT, obj.pk, field, name), data, + settings.XFORM_SUBMISSION_STAT_CACHE_TIME) return data @@ -79,37 +79,6 @@ def data(self): return ReturnList(ret, serializer=self) -class SubmissionStatsInstanceSerializerAsync(serializers.Serializer): - def to_representation(self, obj): - if obj is None: - return super(SubmissionStatsInstanceSerializerAsync, self)\ - .to_representation(obj) - - request = self.context.get('request') - field = request.query_params.get('group') - name = request.query_params.get('name', field) - - - data = cache.get( - '{}{}{}{}'.format(XFORM_SUBMISSION_STAT, obj.pk, field, name)) - if data: - return data - - tasks.get_form_submissions_stats_async(obj.id, field, name) - - data = cache.get( - '{}{}{}{}'.format(XFORM_SUBMISSION_STAT, obj.pk, field, name)) - if data: - return data - - - - @property - def data(self): - ret = super(serializers.Serializer, self).data - - return ReturnDict(ret, serializer=self) - class StatsSerializer(serializers.HyperlinkedModelSerializer): url = serializers.HyperlinkedIdentityField( diff --git a/onadata/settings/common.py b/onadata/settings/common.py index 9f29a1772d..21809e5bcf 100644 --- a/onadata/settings/common.py +++ b/onadata/settings/common.py @@ -604,4 +604,5 @@ def configure_logging(logger, **kwargs): # Project & XForm Visibility Settings ALLOW_PUBLIC_DATASETS = True -XFORM_SUBMISSION_STAT_ASYNC_TRESHOLD = 100000 +# Cache xform submission stat by one hour 60 * 60 +XFORM_SUBMISSION_STAT_CACHE_TIME = 3600 From c98bbfd7f81e67219352b70bb1d2fb48113cd983 Mon Sep 17 00:00:00 2001 From: Dennis Wambua Date: Tue, 4 Jan 2022 15:22:08 +0300 Subject: [PATCH 3/7] cache charts results for one hour --- onadata/apps/api/viewsets/charts_viewset.py | 12 ++++++++++-- onadata/libs/serializers/stats_serializer.py | 8 +++++--- onadata/libs/utils/cache_tools.py | 3 ++- onadata/settings/common.py | 2 ++ 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/onadata/apps/api/viewsets/charts_viewset.py b/onadata/apps/api/viewsets/charts_viewset.py index d415ffa9f4..e99d2f8af4 100644 --- a/onadata/apps/api/viewsets/charts_viewset.py +++ b/onadata/apps/api/viewsets/charts_viewset.py @@ -23,6 +23,7 @@ from onadata.libs.serializers.chart_serializer import (ChartSerializer, FieldsChartSerializer) from onadata.libs.utils.chart_tools import get_chart_data_for_field +from onadata.libs.utils.cache_tools import XFORM_CHARTS def get_form_field_chart_url(url, field): @@ -104,8 +105,15 @@ def retrieve(self, request, *args, **kwargs): return Response(serializer.data) if field_name or field_xpath: - data = get_chart_data_for_field(field_name, xform, fmt, group_by, - field_xpath) + cache_key = '{}{}{}{}{}{}'.format(XFORM_CHARTS, obj.pk, field_xpath, + field_name, group_by,fmt) + + data = cache.get(cache_key) + if not data: + data = get_chart_data_for_field(field_name, xform, fmt, group_by, + field_xpath) + + cache.set(cache_key, data, settings.XFORM_CHARTS_CACHE_TIME) return Response(data, template_name='chart_detail.html') diff --git a/onadata/libs/serializers/stats_serializer.py b/onadata/libs/serializers/stats_serializer.py index b0f470d477..ac329bd0eb 100644 --- a/onadata/libs/serializers/stats_serializer.py +++ b/onadata/libs/serializers/stats_serializer.py @@ -49,8 +49,10 @@ def to_representation(self, obj): raise exceptions.ParseError(_(u"Expecting `group` and `name`" u" query parameters.")) - data = cache.get( - '{}{}{}{}'.format(XFORM_SUBMISSION_STAT, obj.pk, field, name)) + cache_key = '{}{}{}{}'.format(XFORM_SUBMISSION_STAT, obj.pk, + field, name) + + data = cache.get(cache_key) if data: return data @@ -68,7 +70,7 @@ def to_representation(self, obj): label = obj.get_choice_label(element, record[name]) record[name] = label - cache.set('{}{}{}{}'.format(XFORM_SUBMISSION_STAT, obj.pk, field, name), data, + cache.set(cache_key, data, settings.XFORM_SUBMISSION_STAT_CACHE_TIME) return data diff --git a/onadata/libs/utils/cache_tools.py b/onadata/libs/utils/cache_tools.py index 3e0a16d290..4698bf7dcc 100644 --- a/onadata/libs/utils/cache_tools.py +++ b/onadata/libs/utils/cache_tools.py @@ -44,7 +44,8 @@ # Cache names used in XForm Model XFORM_SUBMISSION_COUNT_FOR_DAY = "xfm-get_submission_count-" XFORM_SUBMISSION_COUNT_FOR_DAY_DATE = "xfm-get_submission_count_date-" -XFORM_SUBMISSION_STAT = "xfm-get_form_submissions_grouped_by_field" +XFORM_SUBMISSION_STAT = "xfm-get_form_submissions_grouped_by_field-" +XFORM_CHARTS = "xfm-get_form_charts-" def safe_delete(key): diff --git a/onadata/settings/common.py b/onadata/settings/common.py index 21809e5bcf..77181af38c 100644 --- a/onadata/settings/common.py +++ b/onadata/settings/common.py @@ -606,3 +606,5 @@ def configure_logging(logger, **kwargs): # Cache xform submission stat by one hour 60 * 60 XFORM_SUBMISSION_STAT_CACHE_TIME = 3600 + +XFORM_CHARTS_CACHE_TIME = 3600 From 583c784a43720e02ddf165bf298b5a767559a234 Mon Sep 17 00:00:00 2001 From: Dennis Wambua Date: Tue, 4 Jan 2022 15:59:03 +0300 Subject: [PATCH 4/7] style fixs --- onadata/apps/api/tasks.py | 3 --- .../apps/api/tests/viewsets/test_stats_viewset.py | 1 - onadata/apps/api/viewsets/charts_viewset.py | 12 ++++++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/onadata/apps/api/tasks.py b/onadata/apps/api/tasks.py index a0efd306b9..b8e1f4ff2f 100644 --- a/onadata/apps/api/tasks.py +++ b/onadata/apps/api/tasks.py @@ -4,7 +4,6 @@ from celery.result import AsyncResult from django.core.files.uploadedfile import TemporaryUploadedFile -from django.core.cache import cache from django.core.files.storage import default_storage from django.contrib.auth.models import User from django.utils.datastructures import MultiValueDict @@ -14,8 +13,6 @@ from onadata.libs.utils.email import send_generic_email from onadata.apps.logger.models.xform import XForm from onadata.celery import app -from onadata.libs.utils.cache_tools import XFORM_SUBMISSION_STAT -from onadata.libs.data.query import get_form_submissions_grouped_by_field def recreate_tmp_file(name, path, mime_type): diff --git a/onadata/apps/api/tests/viewsets/test_stats_viewset.py b/onadata/apps/api/tests/viewsets/test_stats_viewset.py index cc917e48e3..ecd453646f 100644 --- a/onadata/apps/api/tests/viewsets/test_stats_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_stats_viewset.py @@ -261,4 +261,3 @@ def test_wrong_stat_function_api(self): self.assertNotEqual(response.get('Cache-Control'), None) self.assertEquals(response.status_code, 200) - diff --git a/onadata/apps/api/viewsets/charts_viewset.py b/onadata/apps/api/viewsets/charts_viewset.py index e99d2f8af4..17af2acf82 100644 --- a/onadata/apps/api/viewsets/charts_viewset.py +++ b/onadata/apps/api/viewsets/charts_viewset.py @@ -4,6 +4,9 @@ """ from django.core.exceptions import ImproperlyConfigured +from django.core.cache import cache +from django.conf import settings + from rest_framework import viewsets from rest_framework.exceptions import ParseError from rest_framework.renderers import (BrowsableAPIRenderer, JSONRenderer, @@ -105,13 +108,14 @@ def retrieve(self, request, *args, **kwargs): return Response(serializer.data) if field_name or field_xpath: - cache_key = '{}{}{}{}{}{}'.format(XFORM_CHARTS, obj.pk, field_xpath, - field_name, group_by,fmt) + cache_key = '{}{}{}{}{}{}'.format(XFORM_CHARTS, xform.pk, + field_xpath, field_name, + group_by, fmt) data = cache.get(cache_key) if not data: - data = get_chart_data_for_field(field_name, xform, fmt, group_by, - field_xpath) + data = get_chart_data_for_field(field_name, xform, fmt, + group_by, field_xpath) cache.set(cache_key, data, settings.XFORM_CHARTS_CACHE_TIME) From 6ad02202176506404038fb02152d7e506459d7e7 Mon Sep 17 00:00:00 2001 From: Dennis Wambua Date: Wed, 5 Jan 2022 10:47:49 +0300 Subject: [PATCH 5/7] Disable cache on tests --- onadata/settings/docker.py | 5 +++++ onadata/settings/github_actions_test.py | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/onadata/settings/docker.py b/onadata/settings/docker.py index 3c80d21c48..6b4ebf36c6 100644 --- a/onadata/settings/docker.py +++ b/onadata/settings/docker.py @@ -101,6 +101,11 @@ ENKETO_SINGLE_SUBMIT_PATH = '/api/v2/survey/single/once' ENKETO_API_INSTANCE_IFRAME_URL = ENKETO_URL + "api_v1/instance/iframe" NOTIFICATION_BACKENDS = {} + CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', + } + } else: MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'media/') # noqa diff --git a/onadata/settings/github_actions_test.py b/onadata/settings/github_actions_test.py index af9a96f3b5..9729fa9a5c 100644 --- a/onadata/settings/github_actions_test.py +++ b/onadata/settings/github_actions_test.py @@ -67,3 +67,9 @@ ODK_TOKEN_FERNET_KEY = 'ROsB4T8s1rCJskAdgpTQEKfH2x2K_EX_YBi3UFyoYng=' # nosec OPENID_CONNECT_PROVIDERS = {} + +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', + } +} From aff9a924ac59461829707f7ecb64220fbe530eb4 Mon Sep 17 00:00:00 2001 From: Dennis Wambua Date: Wed, 5 Jan 2022 11:25:13 +0300 Subject: [PATCH 6/7] Disable cache the specific test --- onadata/apps/api/tests/viewsets/test_charts_viewset.py | 2 ++ onadata/settings/docker.py | 5 ----- onadata/settings/github_actions_test.py | 5 ----- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_charts_viewset.py b/onadata/apps/api/tests/viewsets/test_charts_viewset.py index b98e15f2e2..391f9fe474 100644 --- a/onadata/apps/api/tests/viewsets/test_charts_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_charts_viewset.py @@ -4,6 +4,7 @@ from django.utils import timezone from django.core.cache import cache +from django.test.utils import override_settings from rest_framework.test import APIClient from rest_framework.test import APIRequestFactory from rest_framework.test import force_authenticate @@ -397,6 +398,7 @@ def test_cascading_select(self): ] self.assertEqual(expected, response.data['data']) + @override_settings(XFORM_CHARTS_CACHE_TIME=0) def test_deleted_submission_not_in_chart_endpoint(self): data = {'field_name': 'gender'} request = self.factory.get('/charts', data) diff --git a/onadata/settings/docker.py b/onadata/settings/docker.py index 6b4ebf36c6..3c80d21c48 100644 --- a/onadata/settings/docker.py +++ b/onadata/settings/docker.py @@ -101,11 +101,6 @@ ENKETO_SINGLE_SUBMIT_PATH = '/api/v2/survey/single/once' ENKETO_API_INSTANCE_IFRAME_URL = ENKETO_URL + "api_v1/instance/iframe" NOTIFICATION_BACKENDS = {} - CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', - } - } else: MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'media/') # noqa diff --git a/onadata/settings/github_actions_test.py b/onadata/settings/github_actions_test.py index 9729fa9a5c..8504c71ff1 100644 --- a/onadata/settings/github_actions_test.py +++ b/onadata/settings/github_actions_test.py @@ -68,8 +68,3 @@ ODK_TOKEN_FERNET_KEY = 'ROsB4T8s1rCJskAdgpTQEKfH2x2K_EX_YBi3UFyoYng=' # nosec OPENID_CONNECT_PROVIDERS = {} -CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', - } -} From 34fb36818f94866b6891eb0cd2aa2acad2709dfc Mon Sep 17 00:00:00 2001 From: Dennis Wambua Date: Mon, 10 Jan 2022 15:53:34 +0300 Subject: [PATCH 7/7] Reduce the default chart cache timeout to 10 mins --- onadata/settings/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/onadata/settings/common.py b/onadata/settings/common.py index 77181af38c..22a6756ebe 100644 --- a/onadata/settings/common.py +++ b/onadata/settings/common.py @@ -604,7 +604,7 @@ def configure_logging(logger, **kwargs): # Project & XForm Visibility Settings ALLOW_PUBLIC_DATASETS = True -# Cache xform submission stat by one hour 60 * 60 -XFORM_SUBMISSION_STAT_CACHE_TIME = 3600 +# Cache xform submission stat by 10 min 10 * 60 +XFORM_SUBMISSION_STAT_CACHE_TIME = 600 -XFORM_CHARTS_CACHE_TIME = 3600 +XFORM_CHARTS_CACHE_TIME = 600