diff --git a/hub/admin/extend_user.py b/hub/admin/extend_user.py index ce8a0d602e..0ee48c965a 100644 --- a/hub/admin/extend_user.py +++ b/hub/admin/extend_user.py @@ -21,13 +21,13 @@ USERNAME_INVALID_MESSAGE, username_validators, ) +from kobo.apps.openrosa.apps.logger.models import MonthlyXFormSubmissionCounter from kobo.apps.organizations.models import OrganizationUser from kobo.apps.trash_bin.exceptions import TrashIntegrityError from kobo.apps.trash_bin.models.account import AccountTrash from kobo.apps.trash_bin.utils import move_to_trash -from kpi.deployment_backends.kc_access.shadow_models import ( - KobocatMonthlyXFormSubmissionCounter, -) + + from kpi.models.asset import AssetDeploymentStatus from .filters import UserAdvancedSearchFilter from .mixins import AdvancedSearchMixin @@ -257,7 +257,7 @@ def monthly_submission_count(self, obj): displayed in the Django admin user changelist page """ today = timezone.now().date() - instances = KobocatMonthlyXFormSubmissionCounter.objects.filter( + instances = MonthlyXFormSubmissionCounter.objects.filter( user_id=obj.id, year=today.year, month=today.month, diff --git a/hub/models/extra_user_detail.py b/hub/models/extra_user_detail.py index 260f2e9be6..990f091910 100644 --- a/hub/models/extra_user_detail.py +++ b/hub/models/extra_user_detail.py @@ -1,7 +1,7 @@ from django.conf import settings from django.db import models -from kpi.deployment_backends.kc_access.shadow_models import KobocatUserProfile +from kobo.apps.openrosa.apps.main.models import UserProfile from kpi.fields import KpiUidField from kpi.mixins import StandardizeSearchableFieldMixin @@ -43,9 +43,9 @@ def save( update_fields=update_fields, ) - # Sync `validated_password` field to `KobocatUserProfile` only when + # Sync `validated_password` field to `UserProfile` only when # this object is updated to avoid a race condition and an IntegrityError - # when trying to save `KobocatUserProfile` object whereas the related + # when trying to save `UserProfile` object whereas the related # `KobocatUser` object has not been created yet. if ( not settings.TESTING @@ -55,7 +55,7 @@ def save( or (update_fields and 'validated_password' in update_fields) ) ): - KobocatUserProfile.set_password_details( + UserProfile.set_password_details( self.user.id, self.validated_password, ) diff --git a/kobo/apps/accounts/mfa/models.py b/kobo/apps/accounts/mfa/models.py index 424105e9d9..0891a24a03 100644 --- a/kobo/apps/accounts/mfa/models.py +++ b/kobo/apps/accounts/mfa/models.py @@ -8,9 +8,7 @@ MFAMethodAdmin as TrenchMFAMethodAdmin, ) -from kpi.deployment_backends.kc_access.shadow_models import ( - KobocatUserProfile, -) +from kobo.apps.openrosa.apps.main.models import UserProfile class MfaAvailableToUser(models.Model): @@ -74,7 +72,7 @@ def save( Update user's profile in KoBoCAT database. """ if not settings.TESTING and not created: - KobocatUserProfile.set_mfa_status( + UserProfile.set_mfa_status( user_id=self.user.pk, is_active=self.is_active ) @@ -83,10 +81,10 @@ def delete(self, using=None, keep_parents=False): super().delete(using, keep_parents) """ - Update user's profile in KoBoCAT database. + Update user's profile in KoboCAT database. """ if not settings.TESTING: - KobocatUserProfile.set_mfa_status( + UserProfile.set_mfa_status( user_id=user_id, is_active=False ) diff --git a/kobo/apps/form_disclaimer/models.py b/kobo/apps/form_disclaimer/models.py index 0b1896a181..ce2fe53e8e 100644 --- a/kobo/apps/form_disclaimer/models.py +++ b/kobo/apps/form_disclaimer/models.py @@ -1,5 +1,5 @@ from django.conf import settings -from django.db import models, transaction +from django.db import models from django.db.models import Q from django.db.models.constraints import UniqueConstraint from markdownx.models import MarkdownxField diff --git a/kobo/apps/hook/constants.py b/kobo/apps/hook/constants.py index 00998fc333..7b84fa03bb 100644 --- a/kobo/apps/hook/constants.py +++ b/kobo/apps/hook/constants.py @@ -1,5 +1,6 @@ # coding: utf-8 from enum import Enum +from rest_framework import status HOOK_LOG_FAILED = 0 @@ -16,3 +17,11 @@ class HookLogStatus(Enum): KOBO_INTERNAL_ERROR_STATUS_CODE = None SUBMISSION_PLACEHOLDER = '%SUBMISSION%' + +# Status codes that trigger a retry +RETRIABLE_STATUS_CODES = [ + # status.HTTP_429_TOO_MANY_REQUESTS, + status.HTTP_502_BAD_GATEWAY, + status.HTTP_503_SERVICE_UNAVAILABLE, + status.HTTP_504_GATEWAY_TIMEOUT, +] diff --git a/kobo/apps/hook/exceptions.py b/kobo/apps/hook/exceptions.py new file mode 100644 index 0000000000..1997f47c2e --- /dev/null +++ b/kobo/apps/hook/exceptions.py @@ -0,0 +1,3 @@ + +class HookRemoteServerDownError(Exception): + pass diff --git a/kobo/apps/hook/models/hook_log.py b/kobo/apps/hook/models/hook_log.py index 51c1a8a2f9..00ecd429e0 100644 --- a/kobo/apps/hook/models/hook_log.py +++ b/kobo/apps/hook/models/hook_log.py @@ -1,4 +1,3 @@ -# coding: utf-8 from datetime import timedelta import constance @@ -17,38 +16,43 @@ class HookLog(models.Model): - hook = models.ForeignKey("Hook", related_name="logs", on_delete=models.CASCADE) + hook = models.ForeignKey( + "Hook", related_name="logs", on_delete=models.CASCADE + ) uid = KpiUidField(uid_prefix="hl") - submission_id = models.IntegerField(default=0, db_index=True) # `KoBoCAT.logger.Instance.id` + submission_id = models.IntegerField( # `KoboCAT.logger.Instance.id` + default=0, db_index=True + ) tries = models.PositiveSmallIntegerField(default=0) status = models.PositiveSmallIntegerField( choices=[[e.value, e.name.title()] for e in HookLogStatus], - default=HookLogStatus.PENDING.value + default=HookLogStatus.PENDING.value, ) # Could use status_code, but will speed-up queries - status_code = models.IntegerField(default=KOBO_INTERNAL_ERROR_STATUS_CODE, null=True, blank=True) + status_code = models.IntegerField( + default=KOBO_INTERNAL_ERROR_STATUS_CODE, null=True, blank=True + ) message = models.TextField(default="") date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now_add=True) class Meta: - ordering = ["-date_created"] + ordering = ['-date_created'] + @property def can_retry(self) -> bool: """ Return whether instance can be resent to external endpoint. Notice: even if False is returned, `self.retry()` can be triggered. """ if self.hook.active: - seconds = HookLog.get_elapsed_seconds( - constance.config.HOOK_MAX_RETRIES - ) - threshold = timezone.now() - timedelta(seconds=seconds) - # We can retry only if system has already tried 3 times. - # If log is still pending after 3 times, there was an issue, - # we allow the retry - return ( - self.status == HOOK_LOG_FAILED - or (self.date_modified < threshold and self.status == HOOK_LOG_PENDING) + # If log is still pending after `constance.config.HOOK_MAX_RETRIES` + # times, there was an issue, we allow the retry. + threshold = timezone.now() - timedelta(seconds=120) + + return self.status == HOOK_LOG_FAILED or ( + self.date_modified < threshold + and self.status == HOOK_LOG_PENDING + and self.tries >= constance.config.HOOK_MAX_RETRIES ) return False @@ -66,29 +70,6 @@ def change_status( self.save(reset_status=True) - @staticmethod - def get_elapsed_seconds(retries_count: int) -> int: - """ - Calculate number of elapsed seconds since first try. - Return the number of seconds. - """ - # We need to sum all seconds between each retry - seconds = 0 - for retries_count in range(retries_count): - # Range is zero-indexed - seconds += HookLog.get_remaining_seconds(retries_count) - - return seconds - - @staticmethod - def get_remaining_seconds(retries_count): - """ - Calculate number of remaining seconds before next retry - :param retries_count: int. - :return: int. Number of seconds - """ - return 60 * (10 ** retries_count) - def retry(self): """ Retries to send data to external service @@ -100,7 +81,7 @@ def retry(self): service_definition.send() self.refresh_from_db() except Exception as e: - logging.error("HookLog.retry - {}".format(str(e)), exc_info=True) + logging.error('HookLog.retry - {}'.format(str(e)), exc_info=True) self.change_status(HOOK_LOG_FAILED) return False @@ -110,7 +91,7 @@ def save(self, *args, **kwargs): # Update date_modified each time object is saved self.date_modified = timezone.now() # We don't want to alter tries when we only change the status - if kwargs.pop("reset_status", False) is False: + if kwargs.pop('reset_status', False) is False: self.tries += 1 self.hook.reset_totals() super().save(*args, **kwargs) diff --git a/kobo/apps/hook/models/service_definition_interface.py b/kobo/apps/hook/models/service_definition_interface.py index e721bf45b7..9b2bd1a095 100644 --- a/kobo/apps/hook/models/service_definition_interface.py +++ b/kobo/apps/hook/models/service_definition_interface.py @@ -15,7 +15,9 @@ HOOK_LOG_SUCCESS, HOOK_LOG_FAILED, KOBO_INTERNAL_ERROR_STATUS_CODE, + RETRIABLE_STATUS_CODES, ) +from ..exceptions import HookRemoteServerDownError class ServiceDefinitionInterface(metaclass=ABCMeta): @@ -41,7 +43,8 @@ def _get_data(self): 'service_json.ServiceDefinition._get_data: ' f'Hook #{self._hook.uid} - Data #{self._submission_id} - ' f'{str(e)}', - exc_info=True) + exc_info=True, + ) return None @abstractmethod @@ -71,106 +74,141 @@ def _prepare_request_kwargs(self): """ pass - def send(self): + def send(self) -> bool: """ - Sends data to external endpoint - :return: bool + Sends data to external endpoint. + + Raise an exception if something is wrong. Retries are only allowed + when `HookRemoteServerDownError` is raised. """ - success = False + if not self._data: + self.save_log( + KOBO_INTERNAL_ERROR_STATUS_CODE, 'Submission has been deleted', allow_retries=False + ) + return False + # Need to declare response before requests.post assignment in case of # RequestException response = None - if self._data: - try: - request_kwargs = self._prepare_request_kwargs() - - # Add custom headers - request_kwargs.get("headers").update( - self._hook.settings.get("custom_headers", {})) - - # Add user agent - public_domain = "- {} ".format(os.getenv("PUBLIC_DOMAIN_NAME")) \ - if os.getenv("PUBLIC_DOMAIN_NAME") else "" - request_kwargs.get("headers").update({ - "User-Agent": "KoboToolbox external service {}#{}".format( - public_domain, - self._hook.uid) - }) - - # If the request needs basic authentication with username and - # password, let's provide them - if self._hook.auth_level == Hook.BASIC_AUTH: - request_kwargs.update({ - "auth": (self._hook.settings.get("username"), - self._hook.settings.get("password")) - }) - - ssrf_protect_options = {} - if constance.config.SSRF_ALLOWED_IP_ADDRESS.strip(): - ssrf_protect_options['allowed_ip_addresses'] = constance.\ - config.SSRF_ALLOWED_IP_ADDRESS.strip().split('\r\n') - - if constance.config.SSRF_DENIED_IP_ADDRESS.strip(): - ssrf_protect_options['denied_ip_addresses'] = constance.\ - config.SSRF_DENIED_IP_ADDRESS.strip().split('\r\n') - - SSRFProtect.validate(self._hook.endpoint, - options=ssrf_protect_options) - - response = requests.post(self._hook.endpoint, timeout=30, - **request_kwargs) - response.raise_for_status() - self.save_log(response.status_code, response.text, True) - success = True - except requests.exceptions.RequestException as e: - # If request fails to communicate with remote server. - # Exception is raised before request.post can return something. - # Thus, response equals None - status_code = KOBO_INTERNAL_ERROR_STATUS_CODE - text = str(e) - if response is not None: - text = response.text - status_code = response.status_code - self.save_log(status_code, text) - except SSRFProtectException as e: - logging.error( - 'service_json.ServiceDefinition.send: ' - f'Hook #{self._hook.uid} - ' - f'Data #{self._submission_id} - ' - f'{str(e)}', - exc_info=True) - self.save_log( - KOBO_INTERNAL_ERROR_STATUS_CODE, - f'{self._hook.endpoint} is not allowed') - except Exception as e: - logging.error( - 'service_json.ServiceDefinition.send: ' - f'Hook #{self._hook.uid} - ' - f'Data #{self._submission_id} - ' - f'{str(e)}', - exc_info=True) - self.save_log( - KOBO_INTERNAL_ERROR_STATUS_CODE, - "An error occurred when sending data to external endpoint") - else: - self.save_log( - KOBO_INTERNAL_ERROR_STATUS_CODE, - 'Submission has been deleted' + try: + request_kwargs = self._prepare_request_kwargs() + + # Add custom headers + request_kwargs.get('headers').update( + self._hook.settings.get('custom_headers', {}) ) - return success + # Add user agent + public_domain = ( + '- {} '.format(os.getenv('PUBLIC_DOMAIN_NAME')) + if os.getenv('PUBLIC_DOMAIN_NAME') + else '' + ) + request_kwargs.get('headers').update( + { + 'User-Agent': 'KoboToolbox external service {}#{}'.format( + public_domain, self._hook.uid + ) + } + ) - def save_log(self, status_code: int, message: str, success: bool = False): + # If the request needs basic authentication with username and + # password, let's provide them + if self._hook.auth_level == Hook.BASIC_AUTH: + request_kwargs.update( + { + 'auth': ( + self._hook.settings.get('username'), + self._hook.settings.get('password'), + ) + } + ) + + ssrf_protect_options = {} + if constance.config.SSRF_ALLOWED_IP_ADDRESS.strip(): + ssrf_protect_options[ + 'allowed_ip_addresses' + ] = constance.config.SSRF_ALLOWED_IP_ADDRESS.strip().split( + '\r\n' + ) + + if constance.config.SSRF_DENIED_IP_ADDRESS.strip(): + ssrf_protect_options[ + 'denied_ip_addresses' + ] = constance.config.SSRF_DENIED_IP_ADDRESS.strip().split( + '\r\n' + ) + + SSRFProtect.validate( + self._hook.endpoint, options=ssrf_protect_options + ) + + response = requests.post( + self._hook.endpoint, timeout=30, **request_kwargs + ) + response.raise_for_status() + self.save_log(response.status_code, response.text, success=True) + + return True + + except requests.exceptions.RequestException as e: + # If request fails to communicate with remote server. + # Exception is raised before request.post can return something. + # Thus, response equals None + status_code = KOBO_INTERNAL_ERROR_STATUS_CODE + text = str(e) + if response is not None: + text = response.text + status_code = response.status_code + + if status_code in RETRIABLE_STATUS_CODES: + self.save_log(status_code, text, allow_retries=True) + raise HookRemoteServerDownError + + self.save_log(status_code, text) + raise + except SSRFProtectException as e: + logging.error( + 'service_json.ServiceDefinition.send: ' + f'Hook #{self._hook.uid} - ' + f'Data #{self._submission_id} - ' + f'{str(e)}', + exc_info=True, + ) + self.save_log( + KOBO_INTERNAL_ERROR_STATUS_CODE, + f'{self._hook.endpoint} is not allowed' + ) + raise + except Exception as e: + logging.error( + 'service_json.ServiceDefinition.send: ' + f'Hook #{self._hook.uid} - ' + f'Data #{self._submission_id} - ' + f'{str(e)}', + exc_info=True, + ) + self.save_log( + KOBO_INTERNAL_ERROR_STATUS_CODE, + 'An error occurred when sending ' + f'data to external endpoint: {str(e)}', + ) + raise + + def save_log( + self, + status_code: int, + message: str, + success: bool = False, + allow_retries: bool = False, + ): """ Updates/creates log entry with: - `status_code` as the HTTP status code of the remote server response - `message` as the content of the remote server response """ - fields = { - 'hook': self._hook, - 'submission_id': self._submission_id - } + fields = {'hook': self._hook, 'submission_id': self._submission_id} try: # Try to load the log with a multiple field FK because # we don't know the log `uid` in this context, but we do know @@ -181,7 +219,7 @@ def save_log(self, status_code: int, message: str, success: bool = False): if success: log.status = HOOK_LOG_SUCCESS - elif log.tries >= constance.config.HOOK_MAX_RETRIES: + elif not allow_retries or log.tries >= constance.config.HOOK_MAX_RETRIES: log.status = HOOK_LOG_FAILED log.status_code = status_code diff --git a/kobo/apps/hook/tasks.py b/kobo/apps/hook/tasks.py index 6ff659961f..c87cd21076 100644 --- a/kobo/apps/hook/tasks.py +++ b/kobo/apps/hook/tasks.py @@ -9,44 +9,37 @@ from django.utils import translation, timezone from django_celery_beat.models import PeriodicTask +from kobo.celery import celery_app from kpi.utils.log import logging from .constants import HOOK_LOG_FAILED +from .exceptions import HookRemoteServerDownError from .models import Hook, HookLog - - -@shared_task(bind=True) -def service_definition_task(self, hook_id, submission_id): +from .utils.lazy import LazyMaxRetriesInt + + +@celery_app.task( + autoretry_for=(HookRemoteServerDownError,), + retry_backoff=60, + retry_backoff_max=1200, + max_retries=LazyMaxRetriesInt(), + retry_jitter=True, + queue='kpi_low_priority_queue', +) +def service_definition_task(hook_id: int, submission_id: int) -> bool: """ Tries to send data to the endpoint of the hook It retries n times (n = `constance.config.HOOK_MAX_RETRIES`) - - - after 1 minutes, - - after 10 minutes, - - after 100 minutes - etc ... - - :param self: Celery.Task. - :param hook_id: int. Hook PK - :param submission_id: int. Instance PK """ hook = Hook.objects.get(id=hook_id) # Use camelcase (even if it's not PEP-8 compliant) # because variable represents the class, not the instance. - ServiceDefinition = hook.get_service_definition() + ServiceDefinition = hook.get_service_definition() # noqa service_definition = ServiceDefinition(hook, submission_id) - if not service_definition.send(): - # Countdown is in seconds - countdown = HookLog.get_remaining_seconds(self.request.retries) - raise self.retry(countdown=countdown, max_retries=constance.config.HOOK_MAX_RETRIES) - - return True + return service_definition.send() @shared_task -def retry_all_task(hooklogs_ids): - """ - :param list: . - """ +def retry_all_task(hooklogs_ids: int): hook_logs = HookLog.objects.filter(id__in=hooklogs_ids) for hook_log in hook_logs: hook_log.retry() @@ -71,22 +64,24 @@ def failures_reports(): if failures_reports_period_task: last_run_at = failures_reports_period_task.last_run_at - queryset = HookLog.objects.filter(hook__email_notification=True, - status=HOOK_LOG_FAILED) + queryset = HookLog.objects.filter( + hook__email_notification=True, status=HOOK_LOG_FAILED + ) if last_run_at: queryset = queryset.filter(date_modified__gte=last_run_at) - queryset = queryset.order_by('hook__asset__name', - 'hook__uid', - '-date_modified') + queryset = queryset.order_by( + 'hook__asset__name', 'hook__uid', '-date_modified' + ) # PeriodicTask are updated every 3 minutes (default). # It means, if this task interval is less than 3 minutes, some data can be duplicated in emails. # Setting `beat-sync-every` to 1, makes PeriodicTask to be updated before running the task. # So, we need to update it manually. # see: http://docs.celeryproject.org/en/latest/userguide/configuration.html#beat-sync-every - PeriodicTask.objects.filter(task=beat_schedule.get("task")). \ - update(last_run_at=timezone.now()) + PeriodicTask.objects.filter(task=beat_schedule.get('task')).update( + last_run_at=timezone.now() + ) records = {} max_length = 0 @@ -147,9 +142,12 @@ def failures_reports(): text_content = plain_text_template.render(variables) html_content = html_template.render(variables) - msg = EmailMultiAlternatives(translation.gettext('REST Services Failure Report'), text_content, - constance.config.SUPPORT_EMAIL, - [record.get('email')]) + msg = EmailMultiAlternatives( + translation.gettext('REST Services Failure Report'), + text_content, + constance.config.SUPPORT_EMAIL, + [record.get('email')], + ) msg.attach_alternative(html_content, 'text/html') email_messages.append(msg) diff --git a/kobo/apps/hook/tests/hook_test_case.py b/kobo/apps/hook/tests/hook_test_case.py index b4ea20658e..bbf8799806 100644 --- a/kobo/apps/hook/tests/hook_test_case.py +++ b/kobo/apps/hook/tests/hook_test_case.py @@ -1,6 +1,7 @@ # coding: utf-8 import json +import pytest import responses from django.conf import settings from django.urls import reverse @@ -11,16 +12,10 @@ from kpi.exceptions import BadFormatException from kpi.tests.kpi_test_case import KpiTestCase from ..constants import HOOK_LOG_FAILED +from ..exceptions import HookRemoteServerDownError from ..models import HookLog, Hook -class MockSSRFProtect: - - @staticmethod - def _get_ip_address(url): - return ip_address('1.2.3.4') - - class HookTestCase(KpiTestCase): def setUp(self): @@ -94,26 +89,45 @@ def _send_and_fail(self): :return: dict """ + first_hooklog_response = self._send_and_wait_for_retry() + + # Fakes Celery n retries by forcing status to `failed` + # (where n is `settings.HOOKLOG_MAX_RETRIES`) + first_hooklog = HookLog.objects.get( + uid=first_hooklog_response.get('uid') + ) + first_hooklog.change_status(HOOK_LOG_FAILED) + + return first_hooklog_response + + def _send_and_wait_for_retry(self): self.hook = self._create_hook() ServiceDefinition = self.hook.get_service_definition() submissions = self.asset.deployment.get_submissions(self.asset.owner) submission_id = submissions[0]['_id'] service_definition = ServiceDefinition(self.hook, submission_id) - first_mock_response = {'error': 'not found'} + first_mock_response = {'error': 'gateway timeout'} # Mock first request's try - responses.add(responses.POST, self.hook.endpoint, - json=first_mock_response, status=status.HTTP_404_NOT_FOUND) + responses.add( + responses.POST, + self.hook.endpoint, + json=first_mock_response, + status=status.HTTP_504_GATEWAY_TIMEOUT, + ) # Mock next requests' tries - responses.add(responses.POST, self.hook.endpoint, - status=status.HTTP_200_OK, - content_type='application/json') + responses.add( + responses.POST, + self.hook.endpoint, + status=status.HTTP_200_OK, + content_type='application/json', + ) # Try to send data to external endpoint - success = service_definition.send() - self.assertFalse(success) + with pytest.raises(HookRemoteServerDownError): + service_definition.send() # Retrieve the corresponding log url = reverse('hook-log-list', kwargs={ @@ -126,20 +140,13 @@ def _send_and_fail(self): # Result should match first try self.assertEqual( - first_hooklog_response.get('status_code'), status.HTTP_404_NOT_FOUND + first_hooklog_response.get('status_code'), + status.HTTP_504_GATEWAY_TIMEOUT, ) self.assertEqual( json.loads(first_hooklog_response.get('message')), first_mock_response, ) - - # Fakes Celery n retries by forcing status to `failed` - # (where n is `settings.HOOKLOG_MAX_RETRIES`) - first_hooklog = HookLog.objects.get( - uid=first_hooklog_response.get('uid') - ) - first_hooklog.change_status(HOOK_LOG_FAILED) - return first_hooklog_response def __prepare_submission(self): diff --git a/kobo/apps/hook/tests/test_api_hook.py b/kobo/apps/hook/tests/test_api_hook.py index 511d6486f2..c6368f568d 100644 --- a/kobo/apps/hook/tests/test_api_hook.py +++ b/kobo/apps/hook/tests/test_api_hook.py @@ -1,12 +1,15 @@ # coding: utf-8 import json +import pytest import responses from constance.test import override_config from django.urls import reverse -from mock import patch +from ipaddress import ip_address +from mock import patch, MagicMock from rest_framework import status + from kobo.apps.hook.constants import ( HOOK_LOG_FAILED, HOOK_LOG_PENDING, @@ -21,7 +24,8 @@ PERM_CHANGE_ASSET ) from kpi.utils.datetime import several_minutes_from_now -from .hook_test_case import HookTestCase, MockSSRFProtect +from .hook_test_case import HookTestCase +from ..exceptions import HookRemoteServerDownError class ApiHookTestCase(HookTestCase): @@ -56,42 +60,6 @@ def test_anonymous_access(self): def test_create_hook(self): self._create_hook() - @patch('ssrf_protect.ssrf_protect.SSRFProtect._get_ip_address', - new=MockSSRFProtect._get_ip_address) - @responses.activate - def test_data_submission(self): - # Create first hook - first_hook = self._create_hook(name="dummy external service", - endpoint="http://dummy.service.local/", - settings={}) - responses.add(responses.POST, first_hook.endpoint, - status=status.HTTP_200_OK, - content_type="application/json") - hook_signal_url = reverse("hook-signal-list", kwargs={"parent_lookup_asset": self.asset.uid}) - - submissions = self.asset.deployment.get_submissions(self.asset.owner) - data = {'submission_id': submissions[0]['_id']} - response = self.client.post(hook_signal_url, data=data, format='json') - self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) - - # Create second hook - second_hook = self._create_hook(name="other dummy external service", - endpoint="http://otherdummy.service.local/", - settings={}) - responses.add(responses.POST, second_hook.endpoint, - status=status.HTTP_200_OK, - content_type="application/json") - - response = self.client.post(hook_signal_url, data=data, format='json') - self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) - - response = self.client.post(hook_signal_url, data=data, format='json') - self.assertEqual(response.status_code, status.HTTP_409_CONFLICT) - - data = {'submission_id': 4} # Instance doesn't belong to `self.asset` - response = self.client.post(hook_signal_url, data=data, format='json') - self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) - def test_editor_access(self): hook = self._create_hook() @@ -205,18 +173,20 @@ def test_partial_update_hook(self): self.assertFalse(hook.active) self.assertEqual(hook.name, "some disabled external service") - @patch('ssrf_protect.ssrf_protect.SSRFProtect._get_ip_address', - new=MockSSRFProtect._get_ip_address) + @patch( + 'ssrf_protect.ssrf_protect.SSRFProtect._get_ip_address', + new=MagicMock(return_value=ip_address('1.2.3.4')) + ) @responses.activate def test_send_and_retry(self): first_log_response = self._send_and_fail() # Let's retry through API call - retry_url = reverse("hook-log-retry", kwargs={ - "parent_lookup_asset": self.asset.uid, - "parent_lookup_hook": self.hook.uid, - "uid": first_log_response.get("uid") + retry_url = reverse('hook-log-retry', kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'parent_lookup_hook': self.hook.uid, + 'uid': first_log_response.get('uid') }) # It should be a success @@ -224,17 +194,49 @@ def test_send_and_retry(self): self.assertEqual(response.status_code, status.HTTP_200_OK) # Let's check if logs has 2 tries - detail_url = reverse("hook-log-detail", kwargs={ - "parent_lookup_asset": self.asset.uid, - "parent_lookup_hook": self.hook.uid, - "uid": first_log_response.get("uid") + detail_url = reverse('hook-log-detail', kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'parent_lookup_hook': self.hook.uid, + 'uid': first_log_response.get('uid') + }) + + response = self.client.get(detail_url, format=SUBMISSION_FORMAT_TYPE_JSON) + self.assertEqual(response.data.get('tries'), 2) + + @patch( + 'ssrf_protect.ssrf_protect.SSRFProtect._get_ip_address', + new=MagicMock(return_value=ip_address('1.2.3.4')) + ) + @responses.activate + def test_send_and_cannot_retry(self): + + first_log_response = self._send_and_wait_for_retry() + + # Let's retry through API call + retry_url = reverse('hook-log-retry', kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'parent_lookup_hook': self.hook.uid, + 'uid': first_log_response.get('uid') + }) + + # It should be a failure. The hook log is going to be retried + response = self.client.patch(retry_url, format=SUBMISSION_FORMAT_TYPE_JSON) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + # Let's check if logs has 2 tries + detail_url = reverse('hook-log-detail', kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'parent_lookup_hook': self.hook.uid, + 'uid': first_log_response.get('uid') }) response = self.client.get(detail_url, format=SUBMISSION_FORMAT_TYPE_JSON) - self.assertEqual(response.data.get("tries"), 2) + self.assertEqual(response.data.get('tries'), 1) - @patch('ssrf_protect.ssrf_protect.SSRFProtect._get_ip_address', - new=MockSSRFProtect._get_ip_address) + @patch( + 'ssrf_protect.ssrf_protect.SSRFProtect._get_ip_address', + new=MagicMock(return_value=ip_address('1.2.3.4')) + ) @responses.activate def test_payload_template(self): @@ -317,8 +319,10 @@ def test_payload_template_validation(self): } self.assertEqual(response.data, expected_response) - @patch('ssrf_protect.ssrf_protect.SSRFProtect._get_ip_address', - new=MockSSRFProtect._get_ip_address) + @patch( + 'ssrf_protect.ssrf_protect.SSRFProtect._get_ip_address', + new=MagicMock(return_value=ip_address('1.2.3.4')) + ) @responses.activate def test_hook_log_filter_success(self): # Create success hook @@ -352,17 +356,24 @@ def test_hook_log_filter_success(self): response = self.client.get(f'{hook_log_url}?status={HOOK_LOG_FAILED}', format='json') self.assertEqual(response.data.get('count'), 0) - @patch('ssrf_protect.ssrf_protect.SSRFProtect._get_ip_address', - new=MockSSRFProtect._get_ip_address) + @patch( + 'ssrf_protect.ssrf_protect.SSRFProtect._get_ip_address', + new=MagicMock(return_value=ip_address('1.2.3.4')) + ) @responses.activate def test_hook_log_filter_failure(self): # Create failing hook - hook = self._create_hook(name="failing hook", - endpoint="http://failing.service.local/", - settings={}) - responses.add(responses.POST, hook.endpoint, - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - content_type="application/json") + hook = self._create_hook( + name='failing hook', + endpoint='http://failing.service.local/', + settings={}, + ) + responses.add( + responses.POST, + hook.endpoint, + status=status.HTTP_504_GATEWAY_TIMEOUT, + content_type="application/json", + ) # simulate a submission ServiceDefinition = hook.get_service_definition() @@ -370,8 +381,8 @@ def test_hook_log_filter_failure(self): submission_id = submissions[0]['_id'] service_definition = ServiceDefinition(hook, submission_id) - success = service_definition.send() - self.assertFalse(success) + with pytest.raises(HookRemoteServerDownError): + service_definition.send() # Get log for the failing hook hook_log_url = reverse('hook-log-list', kwargs={ @@ -380,18 +391,24 @@ def test_hook_log_filter_failure(self): }) # There should be no success log for the failing hook - response = self.client.get(f'{hook_log_url}?status={HOOK_LOG_SUCCESS}', format='json') + response = self.client.get( + f'{hook_log_url}?status={HOOK_LOG_SUCCESS}', format='json' + ) self.assertEqual(response.data.get('count'), 0) # There should be a pending log for the failing hook - response = self.client.get(f'{hook_log_url}?status={HOOK_LOG_PENDING}', format='json') + response = self.client.get( + f'{hook_log_url}?status={HOOK_LOG_PENDING}', format='json' + ) self.assertEqual(response.data.get('count'), 1) def test_hook_log_filter_validation(self): # Create hook - hook = self._create_hook(name="success hook", - endpoint="http://hook.service.local/", - settings={}) + hook = self._create_hook( + name='success hook', + endpoint='http://hook.service.local/', + settings={}, + ) # Get log for the success hook hook_log_url = reverse('hook-log-list', kwargs={ @@ -403,14 +420,16 @@ def test_hook_log_filter_validation(self): response = self.client.get(f'{hook_log_url}?status=abc', format='json') self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - @patch('ssrf_protect.ssrf_protect.SSRFProtect._get_ip_address', - new=MockSSRFProtect._get_ip_address) + @patch( + 'ssrf_protect.ssrf_protect.SSRFProtect._get_ip_address', + new=MagicMock(return_value=ip_address('1.2.3.4')) + ) @responses.activate def test_hook_log_filter_date(self): # Create success hook - hook = self._create_hook(name="date hook", - endpoint="http://date.service.local/", - settings={}) + hook = self._create_hook( + name="date hook", endpoint="http://date.service.local/", settings={} + ) responses.add(responses.POST, hook.endpoint, status=status.HTTP_200_OK, content_type="application/json") diff --git a/kobo/apps/hook/tests/test_email.py b/kobo/apps/hook/tests/test_email.py index 2b1f7d0fdf..2a587340cb 100644 --- a/kobo/apps/hook/tests/test_email.py +++ b/kobo/apps/hook/tests/test_email.py @@ -5,9 +5,10 @@ from django.template.loader import get_template from django.utils import translation, dateparse from django_celery_beat.models import PeriodicTask, CrontabSchedule -from mock import patch +from ipaddress import ip_address +from mock import patch, MagicMock -from .hook_test_case import HookTestCase, MockSSRFProtect +from .hook_test_case import HookTestCase from ..tasks import failures_reports @@ -28,8 +29,10 @@ def _create_periodic_task(self): return periodic_task - @patch('ssrf_protect.ssrf_protect.SSRFProtect._get_ip_address', - new=MockSSRFProtect._get_ip_address) + @patch( + 'ssrf_protect.ssrf_protect.SSRFProtect._get_ip_address', + new=MagicMock(return_value=ip_address('1.2.3.4')) + ) @responses.activate def test_notifications(self): self._create_periodic_task() diff --git a/kobo/apps/hook/tests/test_ssrf.py b/kobo/apps/hook/tests/test_ssrf.py index 89a71f4d4c..df3ebd193f 100644 --- a/kobo/apps/hook/tests/test_ssrf.py +++ b/kobo/apps/hook/tests/test_ssrf.py @@ -1,21 +1,24 @@ -# coding: utf-8 - +import pytest import responses from constance.test import override_config -from mock import patch +from ipaddress import ip_address +from mock import patch, MagicMock from rest_framework import status +from ssrf_protect.exceptions import SSRFProtectException from kobo.apps.hook.constants import ( - HOOK_LOG_PENDING, + HOOK_LOG_FAILED, KOBO_INTERNAL_ERROR_STATUS_CODE ) -from .hook_test_case import HookTestCase, MockSSRFProtect +from .hook_test_case import HookTestCase class SSRFHookTestCase(HookTestCase): - @patch('ssrf_protect.ssrf_protect.SSRFProtect._get_ip_address', - new=MockSSRFProtect._get_ip_address) + @patch( + 'ssrf_protect.ssrf_protect.SSRFProtect._get_ip_address', + new=MagicMock(return_value=ip_address('1.2.3.4')) + ) @override_config(SSRF_DENIED_IP_ADDRESS='1.2.3.4') @responses.activate def test_send_with_ssrf_options(self): @@ -34,9 +37,12 @@ def test_send_with_ssrf_options(self): content_type='application/json') # Try to send data to external endpoint - success = service_definition.send() - self.assertFalse(success) + # Note: it should failed because we explicitly deny 1.2.3.4 and + # SSRFProtect._get_ip_address is mocked to return 1.2.3.4 + with pytest.raises(SSRFProtectException): + service_definition.send() + hook_log = hook.logs.all()[0] self.assertEqual(hook_log.status_code, KOBO_INTERNAL_ERROR_STATUS_CODE) - self.assertEqual(hook_log.status, HOOK_LOG_PENDING) + self.assertEqual(hook_log.status, HOOK_LOG_FAILED) self.assertTrue('is not allowed' in hook_log.message) diff --git a/kobo/apps/hook/tests/test_utils.py b/kobo/apps/hook/tests/test_utils.py new file mode 100644 index 0000000000..931f66ce1e --- /dev/null +++ b/kobo/apps/hook/tests/test_utils.py @@ -0,0 +1,53 @@ +import responses +from ipaddress import ip_address +from mock import patch, MagicMock +from rest_framework import status + +from .hook_test_case import HookTestCase +from ..utils.services import call_services + + +class HookUtilsTestCase(HookTestCase): + + @patch( + 'ssrf_protect.ssrf_protect.SSRFProtect._get_ip_address', + new=MagicMock(return_value=ip_address('1.2.3.4')) + ) + @responses.activate + def test_data_submission(self): + # Create first hook + first_hook = self._create_hook( + name='dummy external service', + endpoint='http://dummy.service.local/', + settings={}, + ) + responses.add( + responses.POST, + first_hook.endpoint, + status=status.HTTP_200_OK, + content_type='application/json', + ) + + submissions = self.asset.deployment.get_submissions(self.asset.owner) + submission_id = submissions[0]['_id'] + assert call_services(self.asset.uid, submission_id) is True + + # Create second hook + second_hook = self._create_hook( + name='other dummy external service', + endpoint='http://otherdummy.service.local/', + settings={}, + ) + responses.add( + responses.POST, + second_hook.endpoint, + status=status.HTTP_200_OK, + content_type='application/json', + ) + # Since second hook hasn't received the submission, `call_services` + # should still return True + assert call_services(self.asset.uid, submission_id) is True + + # But if we try again, it should return False (we cannot send the same + # submission twice to the same external endpoint). + assert call_services(self.asset.uid, submission_id) is False diff --git a/kobo/apps/hook/utils.py b/kobo/apps/hook/utils.py deleted file mode 100644 index 55fe5760e0..0000000000 --- a/kobo/apps/hook/utils.py +++ /dev/null @@ -1,32 +0,0 @@ -# coding: utf-8 -from .models.hook_log import HookLog -from .tasks import service_definition_task - - -class HookUtils: - - @staticmethod - def call_services(asset: 'kpi.models.asset.Asset', submission_id: int): - """ - Delegates to Celery data submission to remote servers - """ - # Retrieve `Hook` ids, to send data to their respective endpoint. - hooks_ids = ( - asset.hooks.filter(active=True) - .values_list('id', flat=True) - .distinct() - ) - # At least, one of the hooks must not have a log that corresponds to - # `submission_id` - # to make success equal True - success = False - for hook_id in hooks_ids: - if not HookLog.objects.filter( - submission_id=submission_id, hook_id=hook_id - ).exists(): - success = True - service_definition_task.apply_async( - queue='kpi_low_priority_queue', args=(hook_id, submission_id) - ) - - return success diff --git a/kobo/apps/hook/utils/__init__.py b/kobo/apps/hook/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/kobo/apps/hook/utils/lazy.py b/kobo/apps/hook/utils/lazy.py new file mode 100644 index 0000000000..58a64c9d1a --- /dev/null +++ b/kobo/apps/hook/utils/lazy.py @@ -0,0 +1,44 @@ +import constance + + +class LazyMaxRetriesInt: + """ + constance settings cannot be used as default parameters of a function. + This wrapper helps to return the value of `constance.config.HOOK_MAX_RETRIES` + on demand. + """ + def __call__(self, *args, **kwargs): + return constance.config.HOOK_MAX_RETRIES + + def __repr__(self): + return str(constance.config.HOOK_MAX_RETRIES) + + def __eq__(self, other): + if isinstance(other, int): + return self() == other + return NotImplemented + + def __ne__(self, other): + if isinstance(other, int): + return self() != other + return NotImplemented + + def __lt__(self, other): + if isinstance(other, int): + return self() < other + return NotImplemented + + def __le__(self, other): + if isinstance(other, int): + return self() <= other + return NotImplemented + + def __gt__(self, other): + if isinstance(other, int): + return self() > other + return NotImplemented + + def __ge__(self, other): + if isinstance(other, int): + return self() >= other + return NotImplemented diff --git a/kobo/apps/hook/utils/services.py b/kobo/apps/hook/utils/services.py new file mode 100644 index 0000000000..f0d05c7ad1 --- /dev/null +++ b/kobo/apps/hook/utils/services.py @@ -0,0 +1,27 @@ +from ..models.hook import Hook +from ..models.hook_log import HookLog +from ..tasks import service_definition_task + + +def call_services(asset_uid: str, submission_id: int) -> bool: + """ + Delegates to Celery data submission to remote servers + """ + # Retrieve `Hook` ids, to send data to their respective endpoint. + hooks_ids = ( + Hook.objects.filter(asset__uid=asset_uid, active=True) + .values_list('id', flat=True) + .distinct() + ) + # At least, one of the hooks must not have a log that corresponds to + # `submission_id` + # to make success equal True + success = False + + for hook_id in hooks_ids: + if not HookLog.objects.filter( + submission_id=submission_id, hook_id=hook_id + ).exists(): + success = True + service_definition_task.delay(hook_id, submission_id) + return success diff --git a/kobo/apps/hook/views/v1/__init__.py b/kobo/apps/hook/views/v1/__init__.py index 66a9504388..c3bb54f968 100644 --- a/kobo/apps/hook/views/v1/__init__.py +++ b/kobo/apps/hook/views/v1/__init__.py @@ -1,4 +1,3 @@ # coding: utf-8 from .hook import HookViewSet from .hook_log import HookLogViewSet -from .hook_signal import HookSignalViewSet diff --git a/kobo/apps/hook/views/v1/hook_signal.py b/kobo/apps/hook/views/v1/hook_signal.py deleted file mode 100644 index 37b0a5c5b3..0000000000 --- a/kobo/apps/hook/views/v1/hook_signal.py +++ /dev/null @@ -1,33 +0,0 @@ -# coding: utf-8 -from kobo.apps.hook.views.v2.hook_signal import HookSignalViewSet as HookSignalViewSetV2 - - -class HookSignalViewSet(HookSignalViewSetV2): - """ - ## This document is for a deprecated version of kpi's API. - - **Please upgrade to latest release `/api/v2/assets/hook-signal/`** - - - This endpoint is only used to trigger asset's hooks if any. - - Tells the hooks to post an instance to external servers. -
-    POST /api/v2/assets/{uid}/hook-signal/
-    
- - - > Example - > - > curl -X POST https://[kpi-url]/assets/aSAvYreNzVEkrWg5Gdcvg/hook-signal/ - - - > **Expected payload** - > - > { - > "submission_id": {integer} - > } - - """ - - pass diff --git a/kobo/apps/hook/views/v2/__init__.py b/kobo/apps/hook/views/v2/__init__.py index 66a9504388..c3bb54f968 100644 --- a/kobo/apps/hook/views/v2/__init__.py +++ b/kobo/apps/hook/views/v2/__init__.py @@ -1,4 +1,3 @@ # coding: utf-8 from .hook import HookViewSet from .hook_log import HookLogViewSet -from .hook_signal import HookSignalViewSet diff --git a/kobo/apps/hook/views/v2/hook.py b/kobo/apps/hook/views/v2/hook.py index 9c4f795b0d..1e2a975868 100644 --- a/kobo/apps/hook/views/v2/hook.py +++ b/kobo/apps/hook/views/v2/hook.py @@ -174,13 +174,20 @@ def retry(self, request, uid=None, *args, **kwargs): response = {"detail": t("Task successfully scheduled")} status_code = status.HTTP_200_OK if hook.active: - seconds = HookLog.get_elapsed_seconds(constance.config.HOOK_MAX_RETRIES) - threshold = timezone.now() - timedelta(seconds=seconds) - - records = hook.logs.filter(Q(date_modified__lte=threshold, - status=HOOK_LOG_PENDING) | - Q(status=HOOK_LOG_FAILED)). \ - values_list("id", "uid").distinct() + threshold = timezone.now() - timedelta(seconds=120) + + records = ( + hook.logs.filter( + Q( + date_modified__lte=threshold, + status=HOOK_LOG_PENDING, + tries__gte=constance.config.HOOK_MAX_RETRIES, + ) + | Q(status=HOOK_LOG_FAILED) + ) + .values_list('id', 'uid') + .distinct() + ) # Prepare lists of ids hooklogs_ids = [] hooklogs_uids = [] @@ -190,7 +197,9 @@ def retry(self, request, uid=None, *args, **kwargs): if len(records) > 0: # Mark all logs as PENDING - HookLog.objects.filter(id__in=hooklogs_ids).update(status=HOOK_LOG_PENDING) + HookLog.objects.filter(id__in=hooklogs_ids).update( + status=HOOK_LOG_PENDING + ) # Delegate to Celery retry_all_task.apply_async( queue='kpi_low_priority_queue', args=(hooklogs_ids,) diff --git a/kobo/apps/hook/views/v2/hook_log.py b/kobo/apps/hook/views/v2/hook_log.py index ab6e1e29c9..6b5a8874c1 100644 --- a/kobo/apps/hook/views/v2/hook_log.py +++ b/kobo/apps/hook/views/v2/hook_log.py @@ -108,21 +108,24 @@ def retry(self, request, uid=None, *args, **kwargs): status_code = status.HTTP_200_OK hook_log = self.get_object() - if hook_log.can_retry(): + if hook_log.can_retry: hook_log.change_status() success = hook_log.retry() if success: # Return status_code of remote server too. # `response["status_code"]` is not the same as `status_code` - response["detail"] = hook_log.message - response["status_code"] = hook_log.status_code + response['detail'] = hook_log.message + response['status_code'] = hook_log.status_code else: - response["detail"] = t( - "An error has occurred when sending the data. Please try again later.") + response['detail'] = t( + 'An error has occurred when sending the data. ' + 'Please try again later.' + ) status_code = status.HTTP_500_INTERNAL_SERVER_ERROR else: - response["detail"] = t( - "Data is being or has already been processed") + response['detail'] = t( + 'Data is being or has already been processed' + ) status_code = status.HTTP_400_BAD_REQUEST return Response(response, status=status_code) diff --git a/kobo/apps/hook/views/v2/hook_signal.py b/kobo/apps/hook/views/v2/hook_signal.py deleted file mode 100644 index 5bab71ef46..0000000000 --- a/kobo/apps/hook/views/v2/hook_signal.py +++ /dev/null @@ -1,84 +0,0 @@ -# coding: utf-8 -from django.http import Http404 -from django.utils.translation import gettext_lazy as t -from rest_framework import status, viewsets, serializers -from rest_framework.response import Response -from rest_framework.pagination import _positive_int as positive_int -from rest_framework_extensions.mixins import NestedViewSetMixin - - -from kobo.apps.hook.utils import HookUtils -from kpi.models import Asset -from kpi.utils.viewset_mixins import AssetNestedObjectViewsetMixin - - -class HookSignalViewSet(AssetNestedObjectViewsetMixin, NestedViewSetMixin, - viewsets.ViewSet): - """ - ## - This endpoint is only used to trigger asset's hooks if any. - - Tells the hooks to post an instance to external servers. -
-    POST /api/v2/assets/{uid}/hook-signal/
-    
- - - > Example - > - > curl -X POST https://[kpi-url]/api/v2/assets/aSAvYreNzVEkrWg5Gdcvg/hook-signal/ - - - > **Expected payload** - > - > { - > "submission_id": {integer} - > } - - """ - - parent_model = Asset - - def create(self, request, *args, **kwargs): - """ - It's only used to trigger hook services of the Asset (so far). - - :param request: - :return: - """ - try: - submission_id = positive_int( - request.data.get('submission_id'), strict=True) - except ValueError: - raise serializers.ValidationError( - {'submission_id': t('A positive integer is required.')}) - - # Check if instance really belongs to Asset. - try: - submission = self.asset.deployment.get_submission(submission_id, - request.user) - except ValueError: - raise Http404 - - if not (submission and int(submission['_id']) == submission_id): - raise Http404 - - if HookUtils.call_services(self.asset, submission_id): - # Follow Open Rosa responses by default - response_status_code = status.HTTP_202_ACCEPTED - response = { - "detail": t( - "We got and saved your data, but may not have " - "fully processed it. You should not try to resubmit.") - } - else: - # call_services() refused to launch any task because this - # instance already has a `HookLog` - response_status_code = status.HTTP_409_CONFLICT - response = { - "detail": t( - "Your data for instance {} has been already " - "submitted.".format(submission_id)) - } - - return Response(response, status=response_status_code) diff --git a/kobo/apps/kobo_auth/models.py b/kobo/apps/kobo_auth/models.py index b9b0151a7e..307e7614ac 100644 --- a/kobo/apps/kobo_auth/models.py +++ b/kobo/apps/kobo_auth/models.py @@ -7,7 +7,7 @@ OPENROSA_APP_LABELS, ) from kobo.apps.openrosa.libs.permissions import get_model_permission_codenames -from kpi.utils.database import use_db +from kpi.utils.database import use_db, update_autofield_sequence class User(AbstractUser): @@ -39,3 +39,23 @@ def has_perm(self, perm, obj=None): # Otherwise, check in KPI DB return super().has_perm(perm, obj) + + def sync_to_openrosa_db(self): + User = self.__class__ # noqa + User.objects.using(settings.OPENROSA_DB_ALIAS).bulk_create( + [self], + update_conflicts=True, + update_fields=[ + 'password', + 'last_login', + 'is_superuser', + 'first_name', + 'last_name', + 'email', + 'is_staff', + 'is_active', + 'date_joined', + ], + unique_fields=['pk'] + ) + update_autofield_sequence(User) diff --git a/kobo/apps/openrosa/apps/api/exceptions.py b/kobo/apps/openrosa/apps/api/exceptions.py index 26bdab1ffd..460ae1e544 100644 --- a/kobo/apps/openrosa/apps/api/exceptions.py +++ b/kobo/apps/openrosa/apps/api/exceptions.py @@ -13,7 +13,7 @@ class LegacyAPIException(APIException): default_code = 'legacy_api_exception' -class NoConfirmationProvidedException(APIException): +class NoConfirmationProvidedAPIException(APIException): status_code = HTTP_400_BAD_REQUEST default_detail = t('No confirmation provided') diff --git a/kobo/apps/openrosa/apps/api/tests/viewsets/test_xform_viewset.py b/kobo/apps/openrosa/apps/api/tests/viewsets/test_xform_viewset.py index 2831e70fb3..cd79a93c39 100644 --- a/kobo/apps/openrosa/apps/api/tests/viewsets/test_xform_viewset.py +++ b/kobo/apps/openrosa/apps/api/tests/viewsets/test_xform_viewset.py @@ -477,7 +477,6 @@ def test_xform_serializer_none(self): 'instances_with_geopoints': False, 'num_of_submissions': 0, 'attachment_storage_bytes': 0, - 'has_kpi_hooks': False, 'kpi_asset_uid': '', } self.assertEqual(data, XFormSerializer(None).data) diff --git a/kobo/apps/openrosa/apps/api/tools.py b/kobo/apps/openrosa/apps/api/tools.py index 209b3fabef..4f93cc290b 100644 --- a/kobo/apps/openrosa/apps/api/tools.py +++ b/kobo/apps/openrosa/apps/api/tools.py @@ -15,7 +15,7 @@ HttpResponseRedirect, ) from django.utils.translation import gettext as t -from kobo_service_account.utils import get_real_user, get_request_headers +from kobo_service_account.utils import get_request_headers from rest_framework import exceptions from rest_framework.request import Request from taggit.forms import TagField @@ -136,56 +136,6 @@ class TagForm(forms.Form): instance.save() -def add_validation_status_to_instance( - request: Request, instance: 'Instance' -) -> bool: - """ - Save instance validation status if it is valid. - To be valid, it has to belong to XForm validation statuses - """ - validation_status_uid = request.data.get('validation_status.uid') - success = False - - # Payload must contain validation_status property. - if validation_status_uid: - real_user = get_real_user(request) - validation_status = get_validation_status( - validation_status_uid, instance.xform, real_user.username - ) - if validation_status: - instance.validation_status = validation_status - instance.save() - success = instance.parsed_instance.update_mongo(asynchronous=False) - - return success - - -def get_validation_status(validation_status_uid, asset, username): - # Validate validation_status value It must belong to asset statuses. - available_statuses = {status.get("uid"): status - for status in asset.settings.get("validation_statuses")} - - validation_status = {} - - if validation_status_uid in available_statuses.keys(): - available_status = available_statuses.get(validation_status_uid) - validation_status = { - "timestamp": int(time.time()), - "uid": validation_status_uid, - "by_whom": username, - "color": available_status.get("color"), - "label": available_status.get("label") - } - - return validation_status - - -def remove_validation_status_from_instance(instance): - instance.validation_status = {} - instance.save() - return instance.parsed_instance.update_mongo(asynchronous=False) - - def get_media_file_response( metadata: MetaData, request: Request = None ) -> HttpResponse: diff --git a/kobo/apps/openrosa/apps/api/viewsets/data_viewset.py b/kobo/apps/openrosa/apps/api/viewsets/data_viewset.py index acccae48e1..46ef0320e1 100644 --- a/kobo/apps/openrosa/apps/api/viewsets/data_viewset.py +++ b/kobo/apps/openrosa/apps/api/viewsets/data_viewset.py @@ -1,10 +1,6 @@ -# coding: utf-8 -import logging -import json from typing import Union from django.db.models import Q -from django.db.models.signals import pre_delete, post_delete from django.http import Http404 from django.shortcuts import get_object_or_404 from django.utils.translation import gettext as t @@ -18,27 +14,30 @@ from rest_framework.settings import api_settings from kobo.apps.openrosa.apps.api.exceptions import ( - NoConfirmationProvidedException, + NoConfirmationProvidedAPIException, ) from kobo.apps.openrosa.apps.api.viewsets.xform_viewset import ( custom_response_handler, ) from kobo.apps.openrosa.apps.api.tools import ( add_tags_to_instance, +) +from kobo.apps.openrosa.apps.logger.utils.instance import ( add_validation_status_to_instance, - get_validation_status, + delete_instances, remove_validation_status_from_instance, + set_instance_validation_statuses, +) +from kobo.apps.openrosa.apps.logger.exceptions import ( + BuildDbQueriesAttributeError, + BuildDbQueriesBadArgumentError, + BuildDbQueriesNoConfirmationProvidedError, + MissingValidationStatusPayloadError, ) from kobo.apps.openrosa.apps.logger.models.xform import XForm from kobo.apps.openrosa.apps.logger.models.instance import ( Instance, ) -from kobo.apps.openrosa.apps.logger.signals import ( - nullify_exports_time_of_last_submission, - update_xform_submission_count_delete, -) -from kobo.apps.openrosa.apps.viewer.models.parsed_instance import ParsedInstance -from kobo.apps.openrosa.apps.viewer.signals import remove_from_mongo from kobo.apps.openrosa.libs.renderers import renderers from kobo.apps.openrosa.libs.mixins.anonymous_user_public_forms_mixin import ( AnonymousUserPublicFormsMixin, @@ -419,89 +418,62 @@ def bulk_delete(self, request, *args, **kwargs): Bulk delete instances """ xform = self.get_object() - postgres_query, mongo_query = self.__build_db_queries(xform, request.data) - - # Disconnect signals to speed-up bulk deletion - pre_delete.disconnect(remove_from_mongo, sender=ParsedInstance) - post_delete.disconnect( - nullify_exports_time_of_last_submission, sender=Instance, - dispatch_uid='nullify_exports_time_of_last_submission', - ) - post_delete.disconnect( - update_xform_submission_count_delete, sender=Instance, - dispatch_uid='update_xform_submission_count_delete', - ) try: - # Delete Postgres & Mongo - all_count, results = Instance.objects.filter(**postgres_query).delete() - identifier = f'{Instance._meta.app_label}.Instance' - try: - deleted_records_count = results[identifier] - except KeyError: - # PostgreSQL did not delete any Instance objects. Keep going in case - # they are still present in MongoDB. - logging.warning('Instance objects cannot be found') - deleted_records_count = 0 - - ParsedInstance.bulk_delete(mongo_query) - - # Update xform like signals would do if it was as single object deletion - nullify_exports_time_of_last_submission(sender=Instance, instance=xform) - update_xform_submission_count_delete( - sender=Instance, - instance=xform, value=deleted_records_count - ) - finally: - # Pre_delete signal needs to be re-enabled for parsed instance - pre_delete.connect(remove_from_mongo, sender=ParsedInstance) - post_delete.connect( - nullify_exports_time_of_last_submission, - sender=Instance, - dispatch_uid='nullify_exports_time_of_last_submission', - ) - post_delete.connect( - update_xform_submission_count_delete, - sender=Instance, - dispatch_uid='update_xform_submission_count_delete', + deleted_records_count = delete_instances(xform, request.data) + except BuildDbQueriesBadArgumentError: + raise ValidationError({ + 'payload': t("`query` and `instance_ids` can't be used together") + }) + except BuildDbQueriesAttributeError: + raise ValidationError( + {'payload': t('Invalid `query` or `submission_ids` params')} ) + except BuildDbQueriesNoConfirmationProvidedError: + raise NoConfirmationProvidedAPIException() - return Response({ - 'detail': t('{} submissions have been deleted').format( - deleted_records_count) - }, status.HTTP_200_OK) + return Response( + { + 'detail': t('{} submissions have been deleted').format( + deleted_records_count + ) + }, + status.HTTP_200_OK, + ) def bulk_validation_status(self, request, *args, **kwargs): xform = self.get_object() + real_user = get_real_user(request) try: - new_validation_status_uid = request.data['validation_status.uid'] - except KeyError: + updated_records_count = set_instance_validation_statuses( + xform, request.data, real_user.username + ) + except BuildDbQueriesBadArgumentError: + raise ValidationError({ + 'payload': t("`query` and `instance_ids` can't be used together") + }) + except BuildDbQueriesAttributeError: + raise ValidationError( + {'payload': t('Invalid `query` or `submission_ids` params')} + ) + except BuildDbQueriesNoConfirmationProvidedError: + raise NoConfirmationProvidedAPIException() + except MissingValidationStatusPayloadError: raise ValidationError({ 'payload': t('No `validation_status.uid` provided') }) - # Create new validation_status object - real_user = get_real_user(request) - new_validation_status = get_validation_status( - new_validation_status_uid, xform, real_user.username + return Response( + { + 'detail': t('{} submissions have been updated').format( + updated_records_count + ) + }, + status.HTTP_200_OK, ) - postgres_query, mongo_query = self.__build_db_queries(xform, - request.data) - - # Update Postgres & Mongo - updated_records_count = Instance.objects.filter( - **postgres_query - ).update(validation_status=new_validation_status) - ParsedInstance.bulk_update_validation_statuses(mongo_query, - new_validation_status) - return Response({ - 'detail': t('{} submissions have been updated').format( - updated_records_count) - }, status.HTTP_200_OK) - def get_serializer_class(self): pk_lookup, dataid_lookup = self.lookup_fields pk = self.kwargs.get(pk_lookup) @@ -588,9 +560,13 @@ def validation_status(self, request, *args, **kwargs): data = {} if request.method != 'GET': + username = get_real_user(request).username + validation_status_uid = request.data.get('validation_status.uid') if ( request.method == 'PATCH' - and not add_validation_status_to_instance(request, instance) + and not add_validation_status_to_instance( + username, validation_status_uid, instance + ) ): http_status = status.HTTP_400_BAD_REQUEST elif request.method == 'DELETE': @@ -741,90 +717,3 @@ def list(self, request, *args, **kwargs): return res return custom_response_handler(request, xform, query, export_type) - - @staticmethod - def __build_db_queries(xform_, request_data): - - """ - Gets instance ids based on the request payload. - Useful to narrow down set of instances for bulk actions - - Args: - xform_ (XForm) - request_data (dict) - - Returns: - tuple(, ): PostgreSQL filters, Mongo filters. - They are meant to be used respectively with Django Queryset - and PyMongo query. - - """ - - mongo_query = ParsedInstance.get_base_query(xform_.user.username, - xform_.id_string) - postgres_query = {'xform_id': xform_.id} - instance_ids = None - # Remove empty values - payload = { - key_: value_ for key_, value_ in request_data.items() if value_ - } - ################################################### - # Submissions can be retrieve in 3 different ways # - ################################################### - # First of all, - # users cannot send `query` and `submission_ids` in POST/PATCH request - # - if all(key_ in payload for key_ in ('query', 'submission_ids')): - raise ValidationError({ - 'payload': t("`query` and `instance_ids` can't be used together") - }) - - # First scenario / Get submissions based on user's query - try: - query = payload['query'] - except KeyError: - pass - else: - try: - query.update(mongo_query) # Overrides `_userform_id` if exists - except AttributeError: - raise ValidationError({ - 'payload': t('Invalid query: %(query)s') - % {'query': json.dumps(query)} - }) - - query_kwargs = { - 'query': json.dumps(query), - 'fields': '["_id"]' - } - - cursor = ParsedInstance.query_mongo_no_paging(**query_kwargs) - instance_ids = [record.get('_id') for record in list(cursor)] - - # Second scenario / Get submissions based on list of ids - try: - submission_ids = payload['submission_ids'] - except KeyError: - pass - else: - try: - # Use int() to test if list of integers is valid. - instance_ids = [int(submission_id) - for submission_id in submission_ids] - except ValueError: - raise ValidationError({ - 'payload': t('Invalid submission ids: %(submission_ids)s') - % {'submission_ids': - json.dumps(payload['submission_ids'])} - }) - - if instance_ids is not None: - # Narrow down queries with list of ids. - postgres_query.update({'id__in': instance_ids}) - mongo_query.update({'_id': {'$in': instance_ids}}) - elif payload.get('confirm', False) is not True: - # Third scenario / get all submissions in form, - # but confirmation param must be among payload - raise NoConfirmationProvidedException() - - return postgres_query, mongo_query diff --git a/kobo/apps/openrosa/apps/logger/exceptions.py b/kobo/apps/openrosa/apps/logger/exceptions.py index 74b31a19bd..d8b85c5c94 100644 --- a/kobo/apps/openrosa/apps/logger/exceptions.py +++ b/kobo/apps/openrosa/apps/logger/exceptions.py @@ -1,5 +1,13 @@ -# coding: utf-8 -from django.utils.translation import gettext as t +class BuildDbQueriesAttributeError(Exception): + pass + + +class BuildDbQueriesBadArgumentError(Exception): + pass + + +class BuildDbQueriesNoConfirmationProvidedError(Exception): + pass class DuplicateUUIDError(Exception): @@ -10,5 +18,9 @@ class FormInactiveError(Exception): pass +class MissingValidationStatusPayloadError(Exception): + pass + + class TemporarilyUnavailableError(Exception): pass diff --git a/kobo/apps/openrosa/apps/logger/migrations/0030_backfill_lost_monthly_counters.py b/kobo/apps/openrosa/apps/logger/migrations/0030_backfill_lost_monthly_counters.py index 08b1492386..ad495779f2 100644 --- a/kobo/apps/openrosa/apps/logger/migrations/0030_backfill_lost_monthly_counters.py +++ b/kobo/apps/openrosa/apps/logger/migrations/0030_backfill_lost_monthly_counters.py @@ -6,7 +6,9 @@ from django.db.models.functions import ExtractYear, ExtractMonth from django.utils import timezone -from kobo.apps.openrosa.apps.logger.utils import delete_null_user_daily_counters +from kobo.apps.openrosa.apps.logger.utils.counters import ( + delete_null_user_daily_counters, +) def populate_missing_monthly_counters(apps, schema_editor): diff --git a/kobo/apps/openrosa/apps/logger/migrations/0031_remove_null_user_daily_counters.py b/kobo/apps/openrosa/apps/logger/migrations/0031_remove_null_user_daily_counters.py index 557d20a344..db961696b9 100644 --- a/kobo/apps/openrosa/apps/logger/migrations/0031_remove_null_user_daily_counters.py +++ b/kobo/apps/openrosa/apps/logger/migrations/0031_remove_null_user_daily_counters.py @@ -1,7 +1,9 @@ from django.conf import settings from django.db import migrations -from kobo.apps.openrosa.apps.logger.utils import delete_null_user_daily_counters +from kobo.apps.openrosa.apps.logger.utils.counters import ( + delete_null_user_daily_counters, +) class Migration(migrations.Migration): diff --git a/kobo/apps/openrosa/apps/logger/migrations/0035_remove_xform_has_kpi_hooks_and_instance_posted_to_kpi.py b/kobo/apps/openrosa/apps/logger/migrations/0035_remove_xform_has_kpi_hooks_and_instance_posted_to_kpi.py new file mode 100644 index 0000000000..b6e48241c8 --- /dev/null +++ b/kobo/apps/openrosa/apps/logger/migrations/0035_remove_xform_has_kpi_hooks_and_instance_posted_to_kpi.py @@ -0,0 +1,27 @@ +# Generated by Django 4.2.11 on 2024-07-31 15:59 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import kobo.apps.openrosa.apps.logger.models.attachment +import kobo.apps.openrosa.apps.logger.models.xform +import kpi.deployment_backends.kc_access.storage + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('logger', '0034_set_require_auth_at_project_level'), + ] + + operations = [ + migrations.RemoveField( + model_name='xform', + name='has_kpi_hooks', + ), + migrations.RemoveField( + model_name='instance', + name='posted_to_kpi', + ), + ] diff --git a/kobo/apps/openrosa/apps/logger/models/__init__.py b/kobo/apps/openrosa/apps/logger/models/__init__.py index 01a8162a96..6defed3784 100644 --- a/kobo/apps/openrosa/apps/logger/models/__init__.py +++ b/kobo/apps/openrosa/apps/logger/models/__init__.py @@ -3,7 +3,6 @@ from kobo.apps.openrosa.apps.logger.models.instance import Instance from kobo.apps.openrosa.apps.logger.models.survey_type import SurveyType from kobo.apps.openrosa.apps.logger.models.xform import XForm -from kobo.apps.openrosa.apps.logger.xform_instance_parser import InstanceParseError from kobo.apps.openrosa.apps.logger.models.note import Note from kobo.apps.openrosa.apps.logger.models.daily_xform_submission_counter import ( DailyXFormSubmissionCounter, diff --git a/kobo/apps/openrosa/apps/logger/models/attachment.py b/kobo/apps/openrosa/apps/logger/models/attachment.py index 024ee06b11..5fa0457e96 100644 --- a/kobo/apps/openrosa/apps/logger/models/attachment.py +++ b/kobo/apps/openrosa/apps/logger/models/attachment.py @@ -1,15 +1,24 @@ # coding: utf-8 import mimetypes import os +from typing import Optional +from urllib.parse import quote as urlquote from django.conf import settings +from django.core.files.base import ContentFile from django.db import models from django.utils.http import urlencode +from kobo.apps.openrosa.libs.utils.image_tools import ( + get_optimized_image_path, + resize, +) from kobo.apps.openrosa.libs.utils.hash import get_hash from kpi.deployment_backends.kc_access.storage import ( default_kobocat_storage as default_storage, + KobocatFileSystemStorage, ) +from kpi.mixins.audio_transcoding import AudioTranscodingMixin from .instance import Instance @@ -37,7 +46,7 @@ def get_queryset(self): return super().get_queryset().filter(deleted_at__isnull=True) -class Attachment(models.Model): +class Attachment(models.Model, AudioTranscodingMixin): instance = models.ForeignKey( Instance, related_name='attachments', on_delete=models.CASCADE ) @@ -62,19 +71,32 @@ class Attachment(models.Model): class Meta: app_label = 'logger' - def save(self, *args, **kwargs): - if self.media_file: - self.media_file_basename = self.filename - if self.mimetype == '': - # guess mimetype - mimetype, encoding = mimetypes.guess_type(self.media_file.name) - if mimetype: - self.mimetype = mimetype - # Cache the file size in the database to avoid expensive calls to - # the storage engine when running reports - self.media_file_size = self.media_file.size + @property + def absolute_mp3_path(self): + """ + Return the absolute path on local file system of the converted version of + attachment. Otherwise, return the AWS url (e.g. https://...) + """ - super().save(*args, **kwargs) + if not default_storage.exists(self.mp3_storage_path): + content = self.get_transcoded_audio('mp3') + default_storage.save(self.mp3_storage_path, ContentFile(content)) + + if isinstance(default_storage, KobocatFileSystemStorage): + return f'{self.media_file.path}.mp3' + + return default_storage.url(self.mp3_storage_path) + + @property + def absolute_path(self): + """ + Return the absolute path on local file system of the attachment. + Otherwise, return the AWS url (e.g. https://...) + """ + if isinstance(default_storage, KobocatFileSystemStorage): + return self.media_file.path + + return self.media_file.url @property def file_hash(self): @@ -90,6 +112,71 @@ def file_hash(self): def filename(self): return os.path.basename(self.media_file.name) + @property + def mp3_storage_path(self): + """ + Return the path of file after conversion. It is the exact same name, plus + the conversion audio format extension concatenated. + E.g: file.mp4 and file.mp4.mp3 + """ + return f'{self.storage_path}.mp3' + + def protected_path( + self, format_: Optional[str] = None, suffix: Optional[str] = None + ) -> str: + """ + Return path to be served as protected file served by NGINX + """ + if format_ == 'mp3': + attachment_file_path = self.absolute_mp3_path + else: + attachment_file_path = self.absolute_path + + optimized_image_path = None + if suffix and self.mimetype.startswith('image/'): + optimized_image_path = get_optimized_image_path( + self.media_file.name, suffix + ) + if not default_storage.exists(optimized_image_path): + resize(self.media_file.name) + + if isinstance(default_storage, KobocatFileSystemStorage): + # Django normally sanitizes accented characters in file names during + # save on disk but some languages have extra letters + # (out of ASCII character set) and must be encoded to let NGINX serve + # them + if optimized_image_path: + attachment_file_path = default_storage.path( + optimized_image_path + ) + protected_url = urlquote(attachment_file_path.replace( + settings.KOBOCAT_MEDIA_ROOT, '/protected') + ) + else: + # Double-encode the S3 URL to take advantage of NGINX's + # otherwise troublesome automatic decoding + if optimized_image_path: + attachment_file_path = default_storage.url( + optimized_image_path + ) + protected_url = f'/protected-s3/{urlquote(attachment_file_path)}' + + return protected_url + + def save(self, *args, **kwargs): + if self.media_file: + self.media_file_basename = self.filename + if self.mimetype == '': + # guess mimetype + mimetype, encoding = mimetypes.guess_type(self.media_file.name) + if mimetype: + self.mimetype = mimetype + # Cache the file size in the database to avoid expensive calls to + # the storage engine when running reports + self.media_file_size = self.media_file.size + + super().save(*args, **kwargs) + def secure_url(self, suffix: str = 'original'): """ Returns image URL through KoboCAT redirector. @@ -105,3 +192,7 @@ def secure_url(self, suffix: str = 'original'): suffix=suffix, media_file=urlencode({'media_file': self.media_file.name}) ) + + @property + def storage_path(self): + return str(self.media_file) diff --git a/kobo/apps/openrosa/apps/logger/models/instance.py b/kobo/apps/openrosa/apps/logger/models/instance.py index 86a52500f0..7b685d261b 100644 --- a/kobo/apps/openrosa/apps/logger/models/instance.py +++ b/kobo/apps/openrosa/apps/logger/models/instance.py @@ -106,10 +106,6 @@ class Instance(models.Model): # TODO Don't forget to update all records with command `update_is_sync_with_mongo`. is_synced_with_mongo = LazyDefaultBooleanField(default=False) - # If XForm.has_kpi_hooks` is True, this field should be True either. - # It tells whether the instance has been successfully sent to KPI. - posted_to_kpi = LazyDefaultBooleanField(default=False) - class Meta: app_label = 'logger' diff --git a/kobo/apps/openrosa/apps/logger/models/xform.py b/kobo/apps/openrosa/apps/logger/models/xform.py index be24470ff6..6e34279294 100644 --- a/kobo/apps/openrosa/apps/logger/models/xform.py +++ b/kobo/apps/openrosa/apps/logger/models/xform.py @@ -6,6 +6,7 @@ from io import BytesIO from xml.sax.saxutils import escape as xml_escape +from django.apps import apps from django.conf import settings from django.core.exceptions import ObjectDoesNotExist from django.urls import reverse @@ -25,15 +26,10 @@ CAN_DELETE_DATA_XFORM, CAN_TRANSFER_OWNERSHIP, ) -from kobo.apps.openrosa.libs.utils.guardian import ( - assign_perm, - get_perms_for_model -) from kobo.apps.openrosa.libs.utils.hash import get_hash from kpi.deployment_backends.kc_access.storage import ( default_kobocat_storage as default_storage, ) -from kpi.models.asset import Asset from kpi.utils.xml import XMLFormWithDisclaimer XFORM_TITLE_LENGTH = 255 @@ -100,7 +96,6 @@ class XForm(BaseModel): tags = TaggableManager() - has_kpi_hooks = LazyDefaultBooleanField(default=False) kpi_asset_uid = models.CharField(max_length=32, null=True) pending_delete = models.BooleanField(default=False) @@ -120,9 +115,6 @@ class Meta: objects = XFormWithoutPendingDeletedManager() all_objects = XFormAllManager() - def file_name(self): - return self.id_string + '.xml' - @property def asset(self): """ @@ -131,6 +123,7 @@ def asset(self): Useful to display form disclaimer in Enketo. See kpi.utils.xml.XMLFormWithDisclaimer for more details. """ + Asset = apps.get_model('kpi', 'Asset') # noqa if not hasattr(self, '_cache_asset'): try: asset = Asset.objects.get(uid=self.kpi_asset_uid) @@ -146,6 +139,16 @@ def asset(self): return getattr(self, '_cache_asset') + def file_name(self): + return self.id_string + '.xml' + + @property + def prefixed_hash(self): + """ + Matches what's returned by the KC API + """ + return f'md5:{self.md5_hash}' + def url(self): return reverse( 'download_xform', @@ -169,14 +172,6 @@ def data_dictionary(self, use_cache: bool = False): def has_instances_with_geopoints(self): return self.instances_with_geopoints - @property - def kpi_hook_service(self): - """ - Returns kpi hook service if it exists. XForm should have only one occurrence in any case. - :return: RestService - """ - return self.restservices.filter(name="kpi_hook").first() - def _set_id_string(self): matches = self.instance_id_regex.findall(self.xml) if len(matches) != 1: @@ -305,25 +300,6 @@ def _xls_file_io(self): else: return BytesIO(ff.read()) - @property - def settings(self): - """ - Mimic Asset settings. - :return: Object - """ - # As soon as we need to add custom validation statuses in Asset settings, - # validation in add_validation_status_to_instance - # (kobocat/kobo.apps.openrosa/apps/api/tools.py) should still work - default_validation_statuses = getattr(settings, "DEFAULT_VALIDATION_STATUSES", []) - - # Later purpose, default_validation_statuses could be merged with a custom validation statuses dict - # for example: - # self._validation_statuses.update(default_validation_statuses) - - return { - "validation_statuses": default_validation_statuses - } - @property def xml_with_disclaimer(self): return XMLFormWithDisclaimer(self).get_object().xml diff --git a/kobo/apps/openrosa/apps/logger/tests/test_simple_submission.py b/kobo/apps/openrosa/apps/logger/tests/test_simple_submission.py index c3d845af45..1630ea1a28 100644 --- a/kobo/apps/openrosa/apps/logger/tests/test_simple_submission.py +++ b/kobo/apps/openrosa/apps/logger/tests/test_simple_submission.py @@ -10,6 +10,7 @@ create_instance, safe_create_instance ) + class TempFileProxy: """ create_instance will be looking for a file object, diff --git a/kobo/apps/openrosa/apps/logger/utils/__init__.py b/kobo/apps/openrosa/apps/logger/utils/__init__.py new file mode 100644 index 0000000000..54d1dd56ae --- /dev/null +++ b/kobo/apps/openrosa/apps/logger/utils/__init__.py @@ -0,0 +1,4 @@ +from .counters import delete_null_user_daily_counters +from .database_query import build_db_queries +from .instance import delete_instances +from .instance import set_instance_validation_statuses diff --git a/kobo/apps/openrosa/apps/logger/utils.py b/kobo/apps/openrosa/apps/logger/utils/counters.py similarity index 99% rename from kobo/apps/openrosa/apps/logger/utils.py rename to kobo/apps/openrosa/apps/logger/utils/counters.py index 4aaf1b1e9d..c76dbf42d4 100644 --- a/kobo/apps/openrosa/apps/logger/utils.py +++ b/kobo/apps/openrosa/apps/logger/utils/counters.py @@ -1,3 +1,4 @@ + def delete_null_user_daily_counters(apps, *args): """ Find any DailyXFormCounters without a user, assign them to a user if we can, otherwise delete them diff --git a/kobo/apps/openrosa/apps/logger/utils/database_query.py b/kobo/apps/openrosa/apps/logger/utils/database_query.py new file mode 100644 index 0000000000..b8a81db872 --- /dev/null +++ b/kobo/apps/openrosa/apps/logger/utils/database_query.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import json + +from kobo.apps.openrosa.apps.viewer.models.parsed_instance import ParsedInstance +from ..exceptions import ( + BuildDbQueriesAttributeError, + BuildDbQueriesBadArgumentError, + BuildDbQueriesNoConfirmationProvidedError, +) +from ..models.xform import XForm + + +def build_db_queries(xform: XForm, request_data: dict) -> tuple[dict, dict]: + + """ + Gets instance ids based on the request payload. + Useful to narrow down set of instances for bulk actions + + Args: + xform (XForm) + request_data (dict) + + Returns: + tuple(, ): PostgreSQL filters, Mongo filters. + They are meant to be used respectively with Django Queryset + and PyMongo query. + + """ + + mongo_query = ParsedInstance.get_base_query( + xform.user.username, xform.id_string + ) + postgres_query = {'xform_id': xform.id} + instance_ids = None + # Remove empty values + payload = { + key_: value_ for key_, value_ in request_data.items() if value_ + } + ################################################### + # Submissions can be retrieve in 3 different ways # + ################################################### + # First of all, + # users cannot send `query` and `submission_ids` in POST/PATCH request + # + if all(key_ in payload for key_ in ('query', 'submission_ids')): + raise BuildDbQueriesBadArgumentError + + # First scenario / Get submissions based on user's query + try: + query = payload['query'] + except KeyError: + pass + else: + try: + query.update(mongo_query) # Overrides `_userform_id` if exists + except AttributeError: + raise BuildDbQueriesAttributeError + + query_kwargs = { + 'query': json.dumps(query), + 'fields': '["_id"]' + } + + cursor = ParsedInstance.query_mongo_no_paging(**query_kwargs) + instance_ids = [record.get('_id') for record in list(cursor)] + + # Second scenario / Get submissions based on list of ids + try: + submission_ids = payload['submission_ids'] + except KeyError: + pass + else: + try: + # Use int() to test if list of integers is valid. + instance_ids = [int(submission_id) + for submission_id in submission_ids] + except ValueError: + raise BuildDbQueriesAttributeError + + if instance_ids is not None: + # Narrow down queries with list of ids. + postgres_query.update({'id__in': instance_ids}) + mongo_query.update({'_id': {'$in': instance_ids}}) + elif payload.get('confirm', False) is not True: + # Third scenario / get all submissions in form, + # but confirmation param must be among payload + raise BuildDbQueriesNoConfirmationProvidedError + + return postgres_query, mongo_query diff --git a/kobo/apps/openrosa/apps/logger/utils/instance.py b/kobo/apps/openrosa/apps/logger/utils/instance.py new file mode 100644 index 0000000000..f54d87b0d4 --- /dev/null +++ b/kobo/apps/openrosa/apps/logger/utils/instance.py @@ -0,0 +1,140 @@ +import logging +import time + +from django.conf import settings +from django.db.models.signals import pre_delete, post_delete + +from kobo.apps.openrosa.apps.logger.signals import ( + nullify_exports_time_of_last_submission, + update_xform_submission_count_delete, +) +from kobo.apps.openrosa.apps.viewer.models.parsed_instance import ParsedInstance +from kobo.apps.openrosa.apps.viewer.signals import remove_from_mongo + + +from .database_query import build_db_queries +from ..exceptions import MissingValidationStatusPayloadError +from ..models.xform import XForm +from ..models.instance import Instance + + +def add_validation_status_to_instance( + username: str, validation_status_uid: str, instance: Instance +) -> bool: + """ + Save instance validation status if it is valid. + To be valid, it has to belong to XForm validation statuses + """ + success = False + + # Payload must contain validation_status property. + if validation_status_uid: + + validation_status = get_validation_status( + validation_status_uid, username + ) + if validation_status: + instance.validation_status = validation_status + instance.save(update_fields=['validation_status']) + success = instance.parsed_instance.update_mongo(asynchronous=False) + + return success + + +def delete_instances(xform: XForm, request_data: dict) -> int: + + deleted_records_count = 0 + postgres_query, mongo_query = build_db_queries(xform, request_data) + + # Disconnect signals to speed-up bulk deletion + pre_delete.disconnect(remove_from_mongo, sender=ParsedInstance) + post_delete.disconnect( + nullify_exports_time_of_last_submission, sender=Instance, + dispatch_uid='nullify_exports_time_of_last_submission', + ) + post_delete.disconnect( + update_xform_submission_count_delete, sender=Instance, + dispatch_uid='update_xform_submission_count_delete', + ) + + try: + # Delete Postgres & Mongo + all_count, results = Instance.objects.filter(**postgres_query).delete() + identifier = f'{Instance._meta.app_label}.Instance' + try: + deleted_records_count = results[identifier] + except KeyError: + # PostgreSQL did not delete any Instance objects. Keep going in case + # they are still present in MongoDB. + logging.warning('Instance objects cannot be found') + + ParsedInstance.bulk_delete(mongo_query) + + # Update xform like signals would do if it was as single object deletion + nullify_exports_time_of_last_submission(sender=Instance, instance=xform) + update_xform_submission_count_delete( + sender=Instance, + instance=xform, + value=deleted_records_count + ) + finally: + # Pre_delete signal needs to be re-enabled for parsed instance + pre_delete.connect(remove_from_mongo, sender=ParsedInstance) + post_delete.connect( + nullify_exports_time_of_last_submission, + sender=Instance, + dispatch_uid='nullify_exports_time_of_last_submission', + ) + post_delete.connect( + update_xform_submission_count_delete, + sender=Instance, + dispatch_uid='update_xform_submission_count_delete', + ) + + return deleted_records_count + + +def get_validation_status(validation_status_uid: str, username: str) -> dict: + try: + label = settings.DEFAULT_VALIDATION_STATUSES[validation_status_uid] + except KeyError: + return {} + + return { + 'timestamp': int(time.time()), + 'uid': validation_status_uid, + 'by_whom': username, + 'label': label, + } + + +def remove_validation_status_from_instance(instance: Instance) -> bool: + instance.validation_status = {} + instance.save(update_fields=['validation_status']) + return instance.parsed_instance.update_mongo(asynchronous=False) + + +def set_instance_validation_statuses( + xform: XForm, request_data: dict, request_username: str +) -> int: + + try: + new_validation_status_uid = request_data['validation_status.uid'] + except KeyError: + raise MissingValidationStatusPayloadError + + # Create new validation_status object + new_validation_status = get_validation_status( + new_validation_status_uid, request_username + ) + + postgres_query, mongo_query = build_db_queries(xform, request_data) + + # Update Postgres & Mongo + updated_records_count = Instance.objects.filter( + **postgres_query + ).update(validation_status=new_validation_status) + ParsedInstance.bulk_update_validation_statuses( + mongo_query, new_validation_status + ) + return updated_records_count diff --git a/kobo/apps/openrosa/apps/main/migrations/0015_drop_old_restservice_tables.py b/kobo/apps/openrosa/apps/main/migrations/0015_drop_old_restservice_tables.py new file mode 100644 index 0000000000..1b58d492d7 --- /dev/null +++ b/kobo/apps/openrosa/apps/main/migrations/0015_drop_old_restservice_tables.py @@ -0,0 +1,77 @@ +# Generated by Django 4.2.11 on 2024-07-31 15:59 + +from django.db import migrations, connections +from django.conf import settings + + +KC_REST_SERVICES_TABLES = [ + 'restservice_restservice', +] + + +def get_operations(): + if settings.TESTING or settings.SKIP_HEAVY_MIGRATIONS: + # Skip this migration if running in test environment or because we want + # to voluntarily skip it. + return [] + + tables = KC_REST_SERVICES_TABLES + operations = [] + + sql = """ + SELECT con.conname + FROM pg_catalog.pg_constraint con + INNER JOIN pg_catalog.pg_class rel + ON rel.oid = con.conrelid + INNER JOIN pg_catalog.pg_namespace nsp + ON nsp.oid = connamespace + WHERE nsp.nspname = 'public' + AND rel.relname = %s; + """ + with connections[settings.OPENROSA_DB_ALIAS].cursor() as cursor: + drop_table_queries = [] + for table in tables: + cursor.execute(sql, [table]) + drop_index_queries = [] + for row in cursor.fetchall(): + if not row[0].endswith('_pkey'): + drop_index_queries.append( + f'ALTER TABLE public.{table} DROP CONSTRAINT {row[0]};' + ) + drop_table_queries.append(f'DROP TABLE IF EXISTS {table};') + operations.append( + migrations.RunSQL( + sql=''.join(drop_index_queries), + reverse_sql=migrations.RunSQL.noop, + ) + ) + + operations.append( + migrations.RunSQL( + sql=''.join(drop_table_queries), + reverse_sql=migrations.RunSQL.noop, + ) + ) + + return operations + + +def print_migration_warning(apps, schema_editor): + if settings.TESTING or settings.SKIP_HEAVY_MIGRATIONS: + return + print( + """ + This migration might take a while. If it is too slow, you may want to + re-run migrations with SKIP_HEAVY_MIGRATIONS=True and apply this one + manually from the django shell. + """ + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0014_drop_old_formdisclaimer_tables'), + ] + + operations = [migrations.RunPython(print_migration_warning), *get_operations()] diff --git a/kobo/apps/openrosa/apps/main/models/user_profile.py b/kobo/apps/openrosa/apps/main/models/user_profile.py index 37a191370b..fa0034a825 100644 --- a/kobo/apps/openrosa/apps/main/models/user_profile.py +++ b/kobo/apps/openrosa/apps/main/models/user_profile.py @@ -1,7 +1,10 @@ # coding: utf-8 +import json + from django.conf import settings from django.db import models from guardian.conf import settings as guardian_settings +from rest_framework.authtoken.models import Token from kobo.apps.kobo_auth.shortcuts import User from kobo.apps.openrosa.apps.logger.fields import LazyDefaultBooleanField @@ -35,9 +38,56 @@ class UserProfile(models.Model): is_mfa_active = LazyDefaultBooleanField(default=False) validated_password = models.BooleanField(default=True) + class Meta: + app_label = 'main' + permissions = ( + ('can_add_xform', "Can add/upload an xform to user profile"), + ('view_profile', "Can view user profile"), + ) + def __str__(self): return '%s[%s]' % (self.name, self.user.username) + @classmethod + def to_dict(cls, user_id: int) -> dict: + """ + Retrieve all fields from the user's KC profile and return them in a + dictionary + """ + profile_model, _ = cls.objects.get_or_create(user_id=user_id) + profile = profile_model.__dict__ + + fields = [ + # Use a (kc_name, new_name) tuple to rename a field + 'name', + 'organization', + ('home_page', 'organization_website'), + ('description', 'bio'), + ('phonenumber', 'phone_number'), + 'address', + 'city', + 'country', + 'twitter', + 'metadata', + ] + + result = {} + + for field in fields: + + if isinstance(field, tuple): + kc_name, field = field + else: + kc_name = field + + value = profile.get(kc_name) + # When a field contains JSON (e.g. `metadata`), it gets loaded as a + # `dict`. Convert it back to a string representation + if isinstance(value, dict): + value = json.dumps(value) + result[field] = value + return result + @property def gravatar(self): return get_gravatar_img_link(self.user) @@ -46,17 +96,25 @@ def gravatar(self): def gravatar_exists(self): return gravatar_exists(self.user) - @property - def twitter_clean(self): - if self.twitter.startswith("@"): - return self.twitter[1:] - return self.twitter + @classmethod + def set_mfa_status(cls, user_id: int, is_active: bool): + user_profile, created = cls.objects.get_or_create(user_id=user_id) + user_profile.is_mfa_active = int(is_active) + user_profile.save(update_fields=['is_mfa_active']) - class Meta: - app_label = 'main' - permissions = ( - ('can_add_xform', "Can add/upload an xform to user profile"), - ('view_profile', "Can view user profile"), + @classmethod + def set_password_details( + cls, + user_id: int, + validated: bool, + ): + """ + Update the kobocat user's password_change_date and validated_password fields + """ + user_profile, created = cls.objects.get_or_create(user_id=user_id) + user_profile.validated_password = validated + user_profile.save( + update_fields=['validated_password'] ) diff --git a/kobo/apps/openrosa/apps/restservice/RestServiceInterface.py b/kobo/apps/openrosa/apps/restservice/RestServiceInterface.py deleted file mode 100644 index 28495d5a5c..0000000000 --- a/kobo/apps/openrosa/apps/restservice/RestServiceInterface.py +++ /dev/null @@ -1,4 +0,0 @@ -# coding: utf-8 -class RestServiceInterface: - def send(self, url, data=None): - raise NotImplementedError diff --git a/kobo/apps/openrosa/apps/restservice/__init__.py b/kobo/apps/openrosa/apps/restservice/__init__.py deleted file mode 100644 index 7fe25636ee..0000000000 --- a/kobo/apps/openrosa/apps/restservice/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding: utf-8 -SERVICE_KPI_HOOK = ("kpi_hook", "KPI Hook POST") - -SERVICE_CHOICES = ( - SERVICE_KPI_HOOK, -) - - -default_app_config = "kobo.apps.openrosa.apps.restservice.app.RestServiceConfig" diff --git a/kobo/apps/openrosa/apps/restservice/app.py b/kobo/apps/openrosa/apps/restservice/app.py deleted file mode 100644 index 32c379ee84..0000000000 --- a/kobo/apps/openrosa/apps/restservice/app.py +++ /dev/null @@ -1,12 +0,0 @@ -# coding: utf-8 -from django.apps import AppConfig - - -class RestServiceConfig(AppConfig): - name = 'kobo.apps.openrosa.apps.restservice' - verbose_name = 'restservice' - - def ready(self): - # Register RestService signals - from . import signals - super().ready() diff --git a/kobo/apps/openrosa/apps/restservice/management/__init__.py b/kobo/apps/openrosa/apps/restservice/management/__init__.py deleted file mode 100644 index 57d631c3f0..0000000000 --- a/kobo/apps/openrosa/apps/restservice/management/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# coding: utf-8 diff --git a/kobo/apps/openrosa/apps/restservice/management/commands/__init__.py b/kobo/apps/openrosa/apps/restservice/management/commands/__init__.py deleted file mode 100644 index 57d631c3f0..0000000000 --- a/kobo/apps/openrosa/apps/restservice/management/commands/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# coding: utf-8 diff --git a/kobo/apps/openrosa/apps/restservice/management/commands/update_kpi_hooks_endpoint.py b/kobo/apps/openrosa/apps/restservice/management/commands/update_kpi_hooks_endpoint.py deleted file mode 100644 index 64a3a79053..0000000000 --- a/kobo/apps/openrosa/apps/restservice/management/commands/update_kpi_hooks_endpoint.py +++ /dev/null @@ -1,41 +0,0 @@ -# coding: utf-8 -from django.core.management.base import BaseCommand - -from kobo.apps.openrosa.apps.restservice.models import RestService - - -class Command(BaseCommand): - """ - A faster method is available with PostgreSQL: - UPDATE restservice_restservice - SET service_url = REGEXP_REPLACE( - service_url, - '/assets/([^/]*)/submissions/', - '/api/v2/assets/\1/hook-signal/' - ) - WHERE service_url LIKE '/assets/%'; - """ - - help = 'Updates KPI rest service endpoint' - - def handle(self, *args, **kwargs): - - rest_services = RestService.objects.filter(name='kpi_hook').all() - for rest_service in rest_services: - service_url = rest_service.service_url - do_save = False - if service_url.endswith('/submissions/'): - service_url = service_url.replace('/submissions/', '/hook-signal/') - rest_service.service_url = service_url - do_save = True - rest_service.save(update_fields=["service_url"]) - - if service_url.startswith('/assets/'): - service_url = service_url.replace('/assets/', '/api/v2/assets/') - rest_service.service_url = service_url - do_save = True - - if do_save: - rest_service.save(update_fields=["service_url"]) - - print('Done!') diff --git a/kobo/apps/openrosa/apps/restservice/migrations/0001_initial.py b/kobo/apps/openrosa/apps/restservice/migrations/0001_initial.py deleted file mode 100644 index 0d68804e6d..0000000000 --- a/kobo/apps/openrosa/apps/restservice/migrations/0001_initial.py +++ /dev/null @@ -1,25 +0,0 @@ -# coding: utf-8 -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('logger', '0001_initial'), - ] - - operations = [ - migrations.CreateModel( - name='RestService', - fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - ('service_url', models.URLField(verbose_name='Service URL')), - ('name', models.CharField(max_length=50, choices=[('f2dhis2', 'f2dhis2'), ('generic_json', 'JSON POST'), ('generic_xml', 'XML POST'), ('bamboo', 'bamboo')])), - ('xform', models.ForeignKey(to='logger.XForm', on_delete=models.CASCADE)), - ], - ), - migrations.AlterUniqueTogether( - name='restservice', - unique_together=set([('service_url', 'xform', 'name')]), - ), - ] diff --git a/kobo/apps/openrosa/apps/restservice/migrations/0002_add_related_name_with_delete_on_cascade.py b/kobo/apps/openrosa/apps/restservice/migrations/0002_add_related_name_with_delete_on_cascade.py deleted file mode 100644 index c2d6cf46c3..0000000000 --- a/kobo/apps/openrosa/apps/restservice/migrations/0002_add_related_name_with_delete_on_cascade.py +++ /dev/null @@ -1,22 +0,0 @@ -# coding: utf-8 -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('restservice', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='restservice', - name='name', - field=models.CharField(max_length=50, choices=[('f2dhis2', 'f2dhis2'), ('generic_json', 'JSON POST'), ('generic_xml', 'XML POST'), ('bamboo', 'bamboo'), ('kpi_hook', 'KPI Hook POST')]), - ), - migrations.AlterField( - model_name='restservice', - name='xform', - field=models.ForeignKey(related_name='restservices', to='logger.XForm', on_delete=models.CASCADE), - ), - ] diff --git a/kobo/apps/openrosa/apps/restservice/migrations/0003_remove_deprecated_services.py b/kobo/apps/openrosa/apps/restservice/migrations/0003_remove_deprecated_services.py deleted file mode 100644 index 306e80da8f..0000000000 --- a/kobo/apps/openrosa/apps/restservice/migrations/0003_remove_deprecated_services.py +++ /dev/null @@ -1,17 +0,0 @@ -# coding: utf-8 -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('restservice', '0002_add_related_name_with_delete_on_cascade'), - ] - - operations = [ - migrations.AlterField( - model_name='restservice', - name='name', - field=models.CharField(max_length=50, choices=[('kpi_hook', 'KPI Hook POST')]), - ), - ] diff --git a/kobo/apps/openrosa/apps/restservice/migrations/__init__.py b/kobo/apps/openrosa/apps/restservice/migrations/__init__.py deleted file mode 100644 index 57d631c3f0..0000000000 --- a/kobo/apps/openrosa/apps/restservice/migrations/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# coding: utf-8 diff --git a/kobo/apps/openrosa/apps/restservice/models.py b/kobo/apps/openrosa/apps/restservice/models.py deleted file mode 100644 index f93945a267..0000000000 --- a/kobo/apps/openrosa/apps/restservice/models.py +++ /dev/null @@ -1,31 +0,0 @@ -# coding: utf-8 -from django.db import models -from django.utils.translation import gettext_lazy - -from kobo.apps.openrosa.apps.logger.models.xform import XForm -from kobo.apps.openrosa.apps.restservice import SERVICE_CHOICES - - -class RestService(models.Model): - - class Meta: - app_label = 'restservice' - unique_together = ('service_url', 'xform', 'name') - - service_url = models.URLField(gettext_lazy("Service URL")) - xform = models.ForeignKey(XForm, related_name="restservices", on_delete=models.CASCADE) - name = models.CharField(max_length=50, choices=SERVICE_CHOICES) - - def __str__(self): - return "%s:%s - %s" % (self.xform, self.long_name, self.service_url) - - def get_service_definition(self): - m = __import__(''.join(['kobo.apps.openrosa.apps.restservice.services.', - self.name]), - globals(), locals(), ['ServiceDefinition']) - return m.ServiceDefinition - - @property - def long_name(self): - sv = self.get_service_definition() - return sv.verbose_name diff --git a/kobo/apps/openrosa/apps/restservice/services/__init__.py b/kobo/apps/openrosa/apps/restservice/services/__init__.py deleted file mode 100644 index f6b69c77ea..0000000000 --- a/kobo/apps/openrosa/apps/restservice/services/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# coding: utf-8 -__all__ = ('kpi_hook') diff --git a/kobo/apps/openrosa/apps/restservice/services/kpi_hook.py b/kobo/apps/openrosa/apps/restservice/services/kpi_hook.py deleted file mode 100644 index 4d0f7127fe..0000000000 --- a/kobo/apps/openrosa/apps/restservice/services/kpi_hook.py +++ /dev/null @@ -1,43 +0,0 @@ -# coding: utf-8 -import logging -import re - -import requests -from django.conf import settings -from kobo.apps.openrosa.apps.restservice.RestServiceInterface import RestServiceInterface -from kobo.apps.openrosa.apps.logger.models import Instance - - -class ServiceDefinition(RestServiceInterface): - id = 'kpi_hook' - verbose_name = 'KPI Hook POST' - - def send(self, endpoint, data): - - # Will be used internally by KPI to fetch data with KoBoCatBackend - post_data = { - 'submission_id': data.get('instance_id') - } - headers = {'Content-Type': 'application/json'} - - # Verify if endpoint starts with `/assets/` before sending - # the request to KPI - pattern = r'{}'.format(settings.KPI_HOOK_ENDPOINT_PATTERN.replace( - '{asset_uid}', '[^/]*')) - - # Match v2 and v1 endpoints. - if re.match(pattern, endpoint) or re.match(pattern[7:], endpoint): - # Build the url in the service to avoid saving hardcoded - # domain name in the DB - url = f'{settings.KOBOFORM_INTERNAL_URL}{endpoint}' - response = requests.post(url, headers=headers, json=post_data) - response.raise_for_status() - - # Save successful - Instance.objects.filter(pk=data.get('instance_id')).update( - posted_to_kpi=True - ) - else: - logging.warning( - f'This endpoint: `{endpoint}` is not valid for `KPI Hook`' - ) diff --git a/kobo/apps/openrosa/apps/restservice/signals.py b/kobo/apps/openrosa/apps/restservice/signals.py deleted file mode 100644 index 80ae3b874c..0000000000 --- a/kobo/apps/openrosa/apps/restservice/signals.py +++ /dev/null @@ -1,36 +0,0 @@ -# coding: utf-8 -from django.conf import settings -from django.db.models.signals import post_save -from django.dispatch import receiver - -from kobo.apps.openrosa.apps.restservice import SERVICE_KPI_HOOK -from kobo.apps.openrosa.apps.logger.models import XForm -from kobo.apps.openrosa.apps.restservice.models import RestService - - -@receiver(post_save, sender=XForm) -def save_kpi_hook_service(sender, instance, **kwargs): - """ - Creates/Deletes Kpi hook Rest service related to XForm instance - :param sender: XForm class - :param instance: XForm instance - :param kwargs: dict - """ - kpi_hook_service = instance.kpi_hook_service - if instance.has_kpi_hooks: - # Only register the service if it hasn't been created yet. - if kpi_hook_service is None: - # For retro-compatibility, if `asset_uid` is null, fallback on - # `id_string` - asset_uid = instance.kpi_asset_uid if instance.kpi_asset_uid \ - else instance.id_string - kpi_hook_service = RestService( - service_url=settings.KPI_HOOK_ENDPOINT_PATTERN.format( - asset_uid=asset_uid), - xform=instance, - name=SERVICE_KPI_HOOK[0] - ) - kpi_hook_service.save() - elif kpi_hook_service is not None: - # Only delete the service if it already exists. - kpi_hook_service.delete() diff --git a/kobo/apps/openrosa/apps/restservice/tasks.py b/kobo/apps/openrosa/apps/restservice/tasks.py deleted file mode 100644 index ce67dfd5e2..0000000000 --- a/kobo/apps/openrosa/apps/restservice/tasks.py +++ /dev/null @@ -1,35 +0,0 @@ -# coding: utf-8 -import logging - -from celery import shared_task -from django.conf import settings - -from kobo.apps.openrosa.apps.restservice.models import RestService - - -@shared_task(bind=True) -def service_definition_task(self, rest_service_id, data): - """ - Tries to send data to the endpoint of the hook - It retries 3 times maximum. - - after 2 minutes, - - after 20 minutes, - - after 200 minutes - - :param self: Celery.Task. - :param rest_service_id: RestService primary key. - :param data: dict. - """ - try: - rest_service = RestService.objects.get(pk=rest_service_id) - service = rest_service.get_service_definition()() - service.send(rest_service.service_url, data) - except Exception as e: - logger = logging.getLogger("console_logger") - logger.error("service_definition_task - {}".format(str(e)), exc_info=True) - # Countdown is in seconds - countdown = 120 * (10 ** self.request.retries) - # Max retries is 3 by default. - raise self.retry(countdown=countdown, max_retries=settings.REST_SERVICE_MAX_RETRIES) - - return True diff --git a/kobo/apps/openrosa/apps/restservice/templates/add-service.html b/kobo/apps/openrosa/apps/restservice/templates/add-service.html deleted file mode 100644 index 717082f50e..0000000000 --- a/kobo/apps/openrosa/apps/restservice/templates/add-service.html +++ /dev/null @@ -1,11 +0,0 @@ -{% load i18n %} -{% block content %} -
-

- Please manage REST Services within the new user interface. Go to the - Project Dashboard and navigate to - - Your Project > Settings > REST Services. - -

-{% endblock %} diff --git a/kobo/apps/openrosa/apps/restservice/tests/__init__.py b/kobo/apps/openrosa/apps/restservice/tests/__init__.py deleted file mode 100644 index 57d631c3f0..0000000000 --- a/kobo/apps/openrosa/apps/restservice/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# coding: utf-8 diff --git a/kobo/apps/openrosa/apps/restservice/tests/fixtures/dhisform.xls b/kobo/apps/openrosa/apps/restservice/tests/fixtures/dhisform.xls deleted file mode 100755 index d0b29d74b7..0000000000 Binary files a/kobo/apps/openrosa/apps/restservice/tests/fixtures/dhisform.xls and /dev/null differ diff --git a/kobo/apps/openrosa/apps/restservice/tests/test_restservice.py b/kobo/apps/openrosa/apps/restservice/tests/test_restservice.py deleted file mode 100644 index f27414601e..0000000000 --- a/kobo/apps/openrosa/apps/restservice/tests/test_restservice.py +++ /dev/null @@ -1,40 +0,0 @@ -# coding: utf-8 -import os - -from django.conf import settings - -from kobo.apps.openrosa.apps.main.tests.test_base import TestBase -from kobo.apps.openrosa.apps.logger.models.xform import XForm -from kobo.apps.openrosa.apps.restservice.RestServiceInterface import RestServiceInterface -from kobo.apps.openrosa.apps.restservice.models import RestService - - -class RestServiceTest(TestBase): - - def setUp(self): - super().setUp() - xls_file_path = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - 'fixtures', - 'dhisform.xls' - ) - self._publish_xls_file_and_set_xform(xls_file_path) - - def _create_rest_service(self): - rs = RestService( - service_url=settings.KPI_HOOK_ENDPOINT_PATTERN.format( - asset_uid='aAAAAAAAAAAA'), - xform=XForm.objects.all().reverse()[0], - name='kpi_hook') - rs.save() - self.restservice = rs - - def test_create_rest_service(self): - count = RestService.objects.all().count() - self._create_rest_service() - self.assertEqual(RestService.objects.all().count(), count + 1) - - def test_service_definition(self): - self._create_rest_service() - sv = self.restservice.get_service_definition()() - self.assertEqual(isinstance(sv, RestServiceInterface), True) diff --git a/kobo/apps/openrosa/apps/restservice/utils.py b/kobo/apps/openrosa/apps/restservice/utils.py deleted file mode 100644 index 5e3cbbb5a0..0000000000 --- a/kobo/apps/openrosa/apps/restservice/utils.py +++ /dev/null @@ -1,24 +0,0 @@ -# coding: utf-8 -from kobo.apps.openrosa.apps.restservice.models import RestService -from kobo.apps.openrosa.apps.restservice.tasks import service_definition_task - - -def call_service(parsed_instance): - # lookup service - instance = parsed_instance.instance - rest_services = RestService.objects.filter(xform=instance.xform) - # call service send with url and data parameters - for rest_service in rest_services: - # Celery can't pickle ParsedInstance object, - # let's use build a serializable object instead - # We don't really need `xform_id`, `xform_id_string`, `instance_uuid` - # We use them only for retro compatibility with all services (even if they are deprecated) - data = { - "xform_id": instance.xform.id, - "xform_id_string": instance.xform.id_string, - "instance_uuid": instance.uuid, - "instance_id": instance.id, - "xml": parsed_instance.instance.xml, - "json": parsed_instance.to_dict_for_mongo() - } - service_definition_task.delay(rest_service.pk, data) diff --git a/kobo/apps/openrosa/apps/viewer/models/parsed_instance.py b/kobo/apps/openrosa/apps/viewer/models/parsed_instance.py index 469fae597a..4afd88a3f3 100644 --- a/kobo/apps/openrosa/apps/viewer/models/parsed_instance.py +++ b/kobo/apps/openrosa/apps/viewer/models/parsed_instance.py @@ -12,7 +12,6 @@ from kobo.apps.openrosa.apps.api.mongo_helper import MongoHelper from kobo.apps.openrosa.apps.logger.models import Instance from kobo.apps.openrosa.apps.logger.models import Note -from kobo.apps.openrosa.apps.restservice.utils import call_service from kobo.apps.openrosa.libs.utils.common_tags import ( ID, UUID, @@ -27,6 +26,8 @@ ) from kobo.apps.openrosa.libs.utils.decorators import apply_form_field_names from kobo.apps.openrosa.libs.utils.model_tools import queryset_iterator +from kobo.apps.hook.utils.services import call_services +from kpi.utils.log import logging # this is Mongo Collection where we will store the parsed submissions xform_instances = settings.MONGO_DB.instances @@ -371,7 +372,16 @@ def save(self, asynchronous=False, *args, **kwargs): # Rest Services were called before data was saved in DB. success = self.update_mongo(asynchronous) if success and created: - call_service(self) + records = ParsedInstance.objects.filter( + instance_id=self.instance_id + ).values_list('instance__xform__kpi_asset_uid', flat=True) + if not (asset_uid := records[0]): + logging.warning( + f'ParsedInstance #: {self.pk} - XForm is not linked with Asset' + ) + else: + call_services(asset_uid, self.instance_id) + return success def add_note(self, note): diff --git a/kobo/apps/openrosa/libs/constants.py b/kobo/apps/openrosa/libs/constants.py index 761d96edd5..9e1b580763 100644 --- a/kobo/apps/openrosa/libs/constants.py +++ b/kobo/apps/openrosa/libs/constants.py @@ -39,6 +39,5 @@ 'logger', 'viewer', 'main', - 'restservice', 'guardian', ] diff --git a/kobo/apps/openrosa/libs/serializers/xform_serializer.py b/kobo/apps/openrosa/libs/serializers/xform_serializer.py index cc910c1e41..f9ad5c0685 100644 --- a/kobo/apps/openrosa/libs/serializers/xform_serializer.py +++ b/kobo/apps/openrosa/libs/serializers/xform_serializer.py @@ -27,7 +27,6 @@ class XFormSerializer(serializers.HyperlinkedModelSerializer): lookup_field='pk') users = serializers.SerializerMethodField('get_xform_permissions') hash = serializers.SerializerMethodField() - has_kpi_hooks = serializers.BooleanField() class Meta: model = XForm diff --git a/kobo/apps/openrosa/libs/utils/logger_tools.py b/kobo/apps/openrosa/libs/utils/logger_tools.py index 3d15d8d8b3..d04002dcca 100644 --- a/kobo/apps/openrosa/libs/utils/logger_tools.py +++ b/kobo/apps/openrosa/libs/utils/logger_tools.py @@ -7,6 +7,7 @@ import sys import traceback from datetime import date, datetime, timezone +from typing import Generator, Optional from xml.etree import ElementTree as ET from xml.parsers.expat import ExpatError try: @@ -17,6 +18,7 @@ from dict2xml import dict2xml from django.conf import settings from django.core.exceptions import ValidationError, PermissionDenied +from django.core.files.base import File from django.core.mail import mail_admins from django.db import IntegrityError, transaction from django.db.models import Q @@ -137,12 +139,12 @@ def check_edit_submission_permissions( @transaction.atomic # paranoia; redundant since `ATOMIC_REQUESTS` set to `True` def create_instance( username: str, - xml_file: str, - media_files: list['django.core.files.uploadedfile.UploadedFile'], + xml_file: File, + media_files: Generator[File], status: str = 'submitted_via_web', uuid: str = None, date_created_override: datetime = None, - request: 'rest_framework.request.Request' = None, + request: Optional['rest_framework.request.Request'] = None, ) -> Instance: """ Submission cases: @@ -524,7 +526,13 @@ def response_with_mimetype_and_name( return response -def safe_create_instance(username, xml_file, media_files, uuid, request): +def safe_create_instance( + username, + xml_file, + media_files, + uuid: Optional[str] = None, + request: Optional['rest_framework.request.Request'] = None, +): """Create an instance and catch exceptions. :returns: A list [error, instance] where error is None if there was no @@ -534,7 +542,8 @@ def safe_create_instance(username, xml_file, media_files, uuid, request): try: instance = create_instance( - username, xml_file, media_files, uuid=uuid, request=request) + username, xml_file, media_files, uuid=uuid, request=request + ) except InstanceInvalidUserError: error = OpenRosaResponseBadRequest(t("Username or ID required.")) except InstanceEmptyError: @@ -570,7 +579,7 @@ def safe_create_instance(username, xml_file, media_files, uuid, request): def save_attachments( instance: Instance, - media_files: list['django.core.files.uploadedfile.UploadedFile'], + media_files: Generator[File], defer_counting: bool = False, ) -> tuple[list[Attachment], list[Attachment]]: """ @@ -584,15 +593,19 @@ def save_attachments( which avoids locking any rows in `logger_xform` or `main_userprofile`. """ new_attachments = [] + for f in media_files: - attachment_filename = generate_attachment_filename(instance, f.name) + attachment_filename = generate_attachment_filename( + instance, os.path.basename(f.name) + ) existing_attachment = Attachment.objects.filter( instance=instance, media_file=attachment_filename, mimetype=f.content_type, ).first() - if existing_attachment and (existing_attachment.file_hash == - hash_attachment_contents(f.read())): + if existing_attachment and ( + existing_attachment.file_hash == hash_attachment_contents(f.read()) + ): # We already have this attachment! continue f.seek(0) @@ -616,7 +629,7 @@ def save_submission( request: 'rest_framework.request.Request', xform: XForm, xml: str, - media_files: list['django.core.files.uploadedfile.UploadedFile'], + media_files: Generator[File], new_uuid: str, status: str, date_created_override: datetime, diff --git a/kobo/apps/project_ownership/tests/api/v2/test_api.py b/kobo/apps/project_ownership/tests/api/v2/test_api.py index 58f708f1de..92a531e470 100644 --- a/kobo/apps/project_ownership/tests/api/v2/test_api.py +++ b/kobo/apps/project_ownership/tests/api/v2/test_api.py @@ -1,15 +1,12 @@ import uuid from constance.test import override_config -from datetime import timedelta -from dateutil.parser import isoparse from django.conf import settings from django.contrib.auth import get_user_model from django.utils import timezone from mock import patch, MagicMock from rest_framework import status from rest_framework.reverse import reverse -from unittest.mock import ANY from kobo.apps.project_ownership.models import ( Invite, @@ -441,14 +438,10 @@ def test_account_usage_transferred_to_new_user(self): ), 'assets': [self.asset.uid] } - with patch( - 'kpi.deployment_backends.backends.MockDeploymentBackend.xform', - MagicMock(), - ): - response = self.client.post( - self.invite_url, data=payload, format='json' - ) - assert response.status_code == status.HTTP_201_CREATED + response = self.client.post( + self.invite_url, data=payload, format='json' + ) + assert response.status_code == status.HTTP_201_CREATED # someuser should have no usage reported anymore response = self.client.get(service_usage_url) @@ -504,14 +497,11 @@ def test_data_accessible_to_new_user(self): ), 'assets': [self.asset.uid] } - with patch( - 'kpi.deployment_backends.backends.MockDeploymentBackend.xform', - MagicMock(), - ): - response = self.client.post( - self.invite_url, data=payload, format='json' - ) - assert response.status_code == status.HTTP_201_CREATED + + response = self.client.post( + self.invite_url, data=payload, format='json' + ) + assert response.status_code == status.HTTP_201_CREATED # anotheruser is the owner and should see the project self.client.login(username='anotheruser', password='anotheruser') diff --git a/kobo/apps/project_ownership/utils.py b/kobo/apps/project_ownership/utils.py index 77d5e526a0..f3db74104c 100644 --- a/kobo/apps/project_ownership/utils.py +++ b/kobo/apps/project_ownership/utils.py @@ -4,10 +4,8 @@ from django.apps import apps from django.utils import timezone -from kpi.deployment_backends.kc_access.shadow_models import ( - KobocatAttachment, - KobocatMetadata, -) +from kobo.apps.openrosa.apps.logger.models import Attachment +from kobo.apps.openrosa.apps.main.models import MetaData from kpi.models.asset import AssetFile from .models.choices import TransferStatusChoices, TransferStatusTypeChoices from .exceptions import AsyncTaskException @@ -55,7 +53,7 @@ def move_attachments(transfer: 'project_ownership.Transfer'): _mark_task_as_successful(transfer, async_task_type) return - attachments = KobocatAttachment.all_objects.filter( + attachments = Attachment.all_objects.filter( instance_id__in=submission_ids ).exclude(media_file__startswith=f'{transfer.asset.owner.username}/') @@ -97,7 +95,7 @@ def move_media_files(transfer: 'project_ownership.Transfer'): if transfer.asset.has_deployment: kc_files = { kc_file.file_hash: kc_file - for kc_file in KobocatMetadata.objects.filter( + for kc_file in MetaData.objects.filter( xform_id=transfer.asset.deployment.xform.pk ) } diff --git a/kobo/apps/shadow_model/__init__.py b/kobo/apps/shadow_model/__init__.py deleted file mode 100644 index 1c1c9a2a79..0000000000 --- a/kobo/apps/shadow_model/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# coding: utf-8 -from django.apps import AppConfig -from kpi.constants import SHADOW_MODEL_APP_LABEL - - -class ShadowModelAppConfig(AppConfig): - """ - This app is not in-use but needed because one of shadow models is registered - in Django Admin. - """ - name = 'kobo.apps.shadow_model' - verbose_name = 'KoBoCAT data' - label = SHADOW_MODEL_APP_LABEL diff --git a/kobo/apps/subsequences/tests/test_submission_extras_api_post.py b/kobo/apps/subsequences/tests/test_submission_extras_api_post.py index f530d870ee..1e951ff389 100644 --- a/kobo/apps/subsequences/tests/test_submission_extras_api_post.py +++ b/kobo/apps/subsequences/tests/test_submission_extras_api_post.py @@ -387,7 +387,9 @@ def test_change_language_list(self): class GoogleTranscriptionSubmissionTest(APITestCase): def setUp(self): self.user = User.objects.create_user(username='someuser', email='user@example.com') - self.asset = Asset(content={'survey': [{'type': 'audio', 'name': 'q1'}]}) + self.asset = Asset( + content={'survey': [{'type': 'audio', 'label': 'q1'}]} + ) self.asset.advanced_features = {'transcript': {'values': ['q1']}} self.asset.owner = self.user self.asset.save() @@ -429,7 +431,7 @@ def test_google_transcript_post(self, m1, m2): 'submission': submission_id, 'q1': {GOOGLETS: {'status': 'requested', 'languageCode': ''}} } - with self.assertNumQueries(FuzzyInt(51, 57)): + with self.assertNumQueries(FuzzyInt(210, 215)): res = self.client.post(url, data, format='json') self.assertContains(res, 'complete') with self.assertNumQueries(FuzzyInt(20, 26)): diff --git a/kobo/apps/superuser_stats/migrations/0001_initial.py b/kobo/apps/superuser_stats/migrations/0001_initial.py new file mode 100644 index 0000000000..7b2d303df9 --- /dev/null +++ b/kobo/apps/superuser_stats/migrations/0001_initial.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.11 on 2024-07-03 19:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + """ + Dummy migration to be able to register SuperuserStatsModel with its admin + model without raising an error. + + > django.db.migrations.exceptions.InvalidBasesError: Cannot resolve bases for [] + > This can happen if you are inheriting models from an app with migrations (e.g. contrib.auth) + > in an app with no migrations; see https://docs.djangoproject.com/en/4.2/topics/migrations/#dependencies for more details + """ + dependencies = [ + ('logger', '0034_set_require_auth_at_project_level'), + ] + + operations = [ + ] diff --git a/kobo/apps/superuser_stats/migrations/__init__.py b/kobo/apps/superuser_stats/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/kobo/apps/superuser_stats/models.py b/kobo/apps/superuser_stats/models.py index 036aee12a0..6c8f824cc8 100644 --- a/kobo/apps/superuser_stats/models.py +++ b/kobo/apps/superuser_stats/models.py @@ -1,9 +1,9 @@ -from kpi.deployment_backends.kc_access.shadow_models import ( - KobocatMonthlyXFormSubmissionCounter, +from kobo.apps.openrosa.apps.logger.models import ( + MonthlyXFormSubmissionCounter ) -class SuperuserStatsModel(KobocatMonthlyXFormSubmissionCounter): +class SuperuserStatsModel(MonthlyXFormSubmissionCounter): """ Spoiler: Kludgy! @@ -11,7 +11,7 @@ class SuperuserStatsModel(KobocatMonthlyXFormSubmissionCounter): the superuser section in Django Admin. Django needs a model to register an admin model, so it extends a shadow model (as a proxy) to avoid creating new migrations. - It extends `KobocatMonthlyXFormSubmissionCounter` but it could have + It extends `MonthlyXFormSubmissionCounter` but it could have been anyone of the (shadow) models since we do not add/update/delete objects from the admin interface. The HTML template only lists the available reports. """ diff --git a/kobo/apps/superuser_stats/tasks.py b/kobo/apps/superuser_stats/tasks.py index 543bcccd0c..f858fc6b96 100644 --- a/kobo/apps/superuser_stats/tasks.py +++ b/kobo/apps/superuser_stats/tasks.py @@ -31,13 +31,12 @@ from kobo.apps.trackers.models import NLPUsageCounter from kobo.static_lists import COUNTRIES from kpi.constants import ASSET_TYPE_SURVEY -from kpi.deployment_backends.kc_access.shadow_models import ( - KobocatMonthlyXFormSubmissionCounter, - KobocatXForm, - KobocatUser, - KobocatUserProfile, - ReadOnlyKobocatInstance, +from kobo.apps.openrosa.apps.logger.models import ( + Instance, + MonthlyXFormSubmissionCounter, + XForm, ) +from kobo.apps.openrosa.apps.main.models import UserProfile from kpi.models.asset import Asset, AssetDeploymentStatus @@ -62,7 +61,7 @@ def get_row_for_country(code_: str, label_: str): ) # Doing it this way because this report is focused on crises in # very specific time frames - instances_count = ReadOnlyKobocatInstance.objects.filter( + instances_count = Instance.objects.filter( xform_id__in=list(xform_ids), date_created__date__range=(start_date, end_date), ).count() @@ -111,7 +110,7 @@ def generate_continued_usage_report(output_filename: str, end_date: str): date_created__date__range=(twelve_months_time, end_date), ) submissions_count = ( - KobocatMonthlyXFormSubmissionCounter.objects.annotate( + MonthlyXFormSubmissionCounter.objects.annotate( date=Cast( Concat(F('year'), Value('-'), F('month'), Value('-'), 1), DateField(), @@ -201,7 +200,7 @@ def generate_domain_report(output_filename: str, start_date: str, end_date: str) # get a count of the submissions domain_submissions = { - domain: KobocatMonthlyXFormSubmissionCounter.objects.annotate( + domain: MonthlyXFormSubmissionCounter.objects.annotate( date=Cast( Concat(F('year'), Value('-'), F('month'), Value('-'), 1), DateField(), @@ -271,11 +270,11 @@ def generate_forms_count_by_submission_range(output_filename: str): today = datetime.today() date_ = today - relativedelta(years=1) - no_submissions = KobocatXForm.objects.filter( + no_submissions = XForm.objects.filter( date_created__date__gte=date_, num_of_submissions=0 ) - queryset = ReadOnlyKobocatInstance.objects.values( + queryset = Instance.objects.values( 'xform_id' ).filter( date_created__date__gte=date_, @@ -299,7 +298,7 @@ def generate_forms_count_by_submission_range(output_filename: str): @shared_task def generate_media_storage_report(output_filename: str): - attachments = KobocatUserProfile.objects.all().values( + attachments = UserProfile.objects.all().values( 'user__username', 'attachment_storage_bytes', ) @@ -364,12 +363,12 @@ def format_date(d): else: return d - def get_row_for_user(u: KobocatUser) -> list: + def get_row_for_user(u: 'kobo_auth.User') -> list: row_ = [] try: - profile = KobocatUserProfile.objects.get(user=u) - except KobocatUserProfile.DoesNotExist: + profile = UserProfile.objects.get(user_id=u.pk) + except UserProfile.DoesNotExist: profile = None try: @@ -407,7 +406,7 @@ def get_row_for_user(u: KobocatUser) -> list: else: row_.append('') - row_.append(KobocatXForm.objects.filter(user=u).count()) + row_.append(XForm.objects.filter(user=u).count()) if profile: row_.append(profile.num_of_submissions) @@ -437,9 +436,7 @@ def get_row_for_user(u: KobocatUser) -> list: with default_storage.open(output_filename, 'w') as output_file: writer = csv.writer(output_file) writer.writerow(columns) - kc_users = KobocatUser.objects.exclude( - pk=settings.ANONYMOUS_USER_ID - ).order_by('pk') + kc_users = User.objects.exclude(pk=settings.ANONYMOUS_USER_ID).order_by('pk') for kc_user in kc_users.iterator(CHUNK_SIZE): try: row = get_row_for_user(kc_user) @@ -478,7 +475,7 @@ def generate_user_statistics_report( # Get records from SubmissionCounter records = ( - KobocatMonthlyXFormSubmissionCounter.objects.annotate( + MonthlyXFormSubmissionCounter.objects.annotate( date=Cast( Concat(F('year'), Value('-'), F('month'), Value('-'), 1), DateField(), diff --git a/kobo/apps/trackers/submission_utils.py b/kobo/apps/trackers/submission_utils.py index 51a85bff07..a47118a28e 100644 --- a/kobo/apps/trackers/submission_utils.py +++ b/kobo/apps/trackers/submission_utils.py @@ -5,9 +5,9 @@ from django.utils import timezone from model_bakery import baker -from kpi.deployment_backends.kc_access.shadow_models import ( - KobocatDailyXFormSubmissionCounter, - KobocatXForm, +from kobo.apps.openrosa.apps.logger.models import ( + DailyXFormSubmissionCounter, + XForm, ) from kpi.models import Asset from kpi.urls.router_api_v2 import URL_NAMESPACE as ROUTER_URL_NAMESPACE @@ -61,7 +61,7 @@ def expected_file_size(submissions: int = 1): def update_xform_counters( - asset: Asset, xform: KobocatXForm = None, submissions: int = 1 + asset: Asset, xform: XForm = None, submissions: int = 1 ): """ Create/update the daily submission counter and the shadow xform we use to query it @@ -103,7 +103,7 @@ def update_xform_counters( ) xform.save() - counter = KobocatDailyXFormSubmissionCounter.objects.filter( + counter = DailyXFormSubmissionCounter.objects.filter( date=today.date(), user_id=asset.owner.id, ).first() diff --git a/kobo/apps/trash_bin/models/project.py b/kobo/apps/trash_bin/models/project.py index 04fdeaae5e..e0ee2354a7 100644 --- a/kobo/apps/trash_bin/models/project.py +++ b/kobo/apps/trash_bin/models/project.py @@ -4,12 +4,12 @@ from django.db import models, transaction from django.utils.timezone import now +from kobo.apps.openrosa.apps.logger.models import XForm from kobo.apps.project_ownership.models import ( Invite, InviteStatusChoices, Transfer, ) -from kpi.deployment_backends.kc_access.shadow_models import KobocatUser, KobocatXForm from kpi.deployment_backends.kc_access.utils import kc_transaction_atomic from kpi.fields import KpiUidField from kpi.models.asset import Asset, AssetDeploymentStatus @@ -49,7 +49,7 @@ def toggle_asset_statuses( kc_filter_params = {'kpi_asset_uid__in': asset_uids} filter_params = {'uid__in': asset_uids} else: - kc_filter_params = {'user': KobocatUser.get_kc_user(owner)} + kc_filter_params = {'user_id': owner.pk} filter_params = {'owner': owner} kc_update_params = {'downloadable': active} @@ -90,7 +90,7 @@ def toggle_asset_statuses( ).update(status=InviteStatusChoices.CANCELLED) if not settings.TESTING: - kc_updated = KobocatXForm.objects.filter( + kc_updated = XForm.objects.filter( **kc_filter_params ).update(**kc_update_params) assert updated >= kc_updated diff --git a/kobo/apps/trash_bin/utils.py b/kobo/apps/trash_bin/utils.py index 4f2e96fb22..c5b7d22487 100644 --- a/kobo/apps/trash_bin/utils.py +++ b/kobo/apps/trash_bin/utils.py @@ -287,11 +287,6 @@ def replace_user_with_placeholder( def _delete_submissions(request_author: settings.AUTH_USER_MODEL, asset: 'kpi.Asset'): - ( - app_label, - model_name, - ) = asset.deployment.submission_model.get_app_label_and_model_name() - while True: audit_logs = [] submissions = list(asset.deployment.get_submissions( @@ -315,8 +310,8 @@ def _delete_submissions(request_author: settings.AUTH_USER_MODEL, asset: 'kpi.As submission_ids = [] for submission in submissions: audit_logs.append(AuditLog( - app_label=app_label, - model_name=model_name, + app_label='logger', + model_name='instance', object_id=submission['_id'], user=request_author, user_uid=request_author.extra_details.uid, diff --git a/kobo/settings/base.py b/kobo/settings/base.py index 2aa5561b3b..715d2badc3 100644 --- a/kobo/settings/base.py +++ b/kobo/settings/base.py @@ -126,7 +126,6 @@ 'kobo.apps.external_integrations.ExternalIntegrationsAppConfig', 'markdownx', 'kobo.apps.help', - 'kobo.apps.shadow_model.ShadowModelAppConfig', 'trench', 'kobo.apps.accounts.mfa.apps.MfaAppConfig', 'kobo.apps.languages.LanguageAppConfig', @@ -139,7 +138,6 @@ 'kobo.apps.openrosa.apps.logger.app.LoggerAppConfig', 'kobo.apps.openrosa.apps.viewer.app.ViewerConfig', 'kobo.apps.openrosa.apps.main.app.MainConfig', - 'kobo.apps.openrosa.apps.restservice.app.RestServiceConfig', 'kobo.apps.openrosa.apps.api', 'guardian', 'kobo.apps.openrosa.libs', @@ -1013,7 +1011,7 @@ def __init__(self, *args, **kwargs): KOBOFORM_URL = os.environ.get('KOBOFORM_URL', 'http://kpi') if 'KOBOCAT_URL' in os.environ: - DEFAULT_DEPLOYMENT_BACKEND = 'kobocat' + DEFAULT_DEPLOYMENT_BACKEND = 'openrosa' else: DEFAULT_DEPLOYMENT_BACKEND = 'mock' @@ -1021,6 +1019,7 @@ def __init__(self, *args, **kwargs): ''' Stripe configuration intended for kf.kobotoolbox.org only, tracks usage limit exceptions ''' STRIPE_ENABLED = env.bool("STRIPE_ENABLED", False) + def dj_stripe_request_callback_method(): # This method exists because dj-stripe's documentation doesn't reflect reality. # It claims that DJSTRIPE_SUBSCRIBER_MODEL no longer needs a request callback but @@ -1199,7 +1198,7 @@ def dj_stripe_request_callback_method(): # http://docs.celeryproject.org/en/latest/getting-started/brokers/redis.html#redis-visibility-timeout # TODO figure out how to pass `Constance.HOOK_MAX_RETRIES` or `HookLog.get_remaining_seconds() # Otherwise hardcode `HOOK_MAX_RETRIES` in Settings - "visibility_timeout": 60 * (10 ** 3) # Longest ETA for RestService (seconds) + "visibility_timeout": 60 * (10 ** 2) # Longest ETA for RestService (seconds) } CELERY_TASK_DEFAULT_QUEUE = "kpi_queue" @@ -1707,23 +1706,11 @@ def dj_stripe_request_callback_method(): os.environ.get('SUPPORT_BRIEFCASE_SUBMISSION_DATE') != 'True' ) -DEFAULT_VALIDATION_STATUSES = [ - { - 'uid': 'validation_status_not_approved', - 'color': '#ff0000', - 'label': 'Not Approved' - }, - { - 'uid': 'validation_status_approved', - 'color': '#00ff00', - 'label': 'Approved' - }, - { - 'uid': 'validation_status_on_hold', - 'color': '#0000ff', - 'label': 'On Hold' - }, -] +DEFAULT_VALIDATION_STATUSES = { + 'validation_status_not_approved': 'Not Approved', + 'validation_status_approved': 'Approved', + 'validation_status_on_hold': 'On Hold', +} THUMB_CONF = { 'large': 1280, diff --git a/kpi/constants.py b/kpi/constants.py index faf95ff905..c098a6c6cb 100644 --- a/kpi/constants.py +++ b/kpi/constants.py @@ -61,14 +61,11 @@ ASSET_TYPE_TEMPLATE: [ASSET_TYPE_SURVEY, ASSET_TYPE_TEMPLATE] } -ASSET_TYPE_ARG_NAME = "asset_type" +ASSET_TYPE_ARG_NAME = 'asset_type' -# Main app label for shadow models. -SHADOW_MODEL_APP_LABEL = 'shadow_model' # List of app labels that need to read/write data from KoBoCAT database # Useful in `db_routers.py` SHADOW_MODEL_APP_LABELS = [ - SHADOW_MODEL_APP_LABEL, 'superuser_stats', ] diff --git a/kpi/deployment_backends/backends.py b/kpi/deployment_backends/backends.py index 04e49705a4..1dd514e4fe 100644 --- a/kpi/deployment_backends/backends.py +++ b/kpi/deployment_backends/backends.py @@ -1,8 +1,9 @@ # coding: utf-8 from .mock_backend import MockDeploymentBackend -from .kobocat_backend import KobocatDeploymentBackend +from .openrosa_backend import OpenRosaDeploymentBackend DEPLOYMENT_BACKENDS = { 'mock': MockDeploymentBackend, - 'kobocat': KobocatDeploymentBackend, + 'kobocat': OpenRosaDeploymentBackend, + 'openrosa': OpenRosaDeploymentBackend, } diff --git a/kpi/deployment_backends/base_backend.py b/kpi/deployment_backends/base_backend.py index 9afb004403..f7860fdf71 100644 --- a/kpi/deployment_backends/base_backend.py +++ b/kpi/deployment_backends/base_backend.py @@ -13,7 +13,6 @@ from bson import json_util from django.conf import settings -from django.core.files.storage import default_storage from django.db.models.query import QuerySet from django.utils import timezone from django.utils.translation import gettext_lazy as t @@ -87,7 +86,7 @@ def bulk_assign_mapped_perms(self): pass def bulk_update_submissions( - self, data: dict, user: settings.AUTH_USER_MODEL + self, data: dict, user: settings.AUTH_USER_MODEL, **kwargs ) -> dict: """ Allows for bulk updating (bulk editing) of submissions. A @@ -145,7 +144,7 @@ def bulk_update_submissions( ) } - kc_responses = [] + backend_responses = [] for submission in submissions: xml_parsed = fromstring_preserve_root_xmlns(submission) @@ -173,18 +172,19 @@ def bulk_update_submissions( for path, value in update_data.items(): edit_submission_xml(xml_parsed, path, value) - kc_response = self.store_submission( - user, xml_tostring(xml_parsed), _uuid + backend_response = self.store_submission( + user, + xml_tostring(xml_parsed), + _uuid, + request=kwargs.get('request'), ) - kc_responses.append( + backend_responses.append( { 'uuid': _uuid, - 'response': kc_response, + 'response': backend_response, } ) - - return self.prepare_bulk_update_response(kc_responses) - + return self.prepare_bulk_update_response(backend_responses) @abc.abstractmethod def calculated_submission_count(self, user: settings.AUTH_USER_MODEL, **kwargs): @@ -199,24 +199,31 @@ def connect(self, active=False): def form_uuid(self): pass + @staticmethod @abc.abstractmethod - def nlp_tracking_data(self, start_date: Optional[datetime.date] = None): + def nlp_tracking_data( + asset_ids: list[int], start_date: Optional[datetime.date] = None + ): pass def delete(self): self.asset._deployment_data.clear() # noqa @abc.abstractmethod - def delete_submission(self, submission_id: int, user: settings.AUTH_USER_MODEL) -> dict: + def delete_submission( + self, submission_id: int, user: settings.AUTH_USER_MODEL + ) -> dict: pass @abc.abstractmethod - def delete_submissions(self, data: dict, user: settings.AUTH_USER_MODEL, **kwargs) -> dict: + def delete_submissions( + self, data: dict, user: settings.AUTH_USER_MODEL, **kwargs + ) -> dict: pass @abc.abstractmethod def duplicate_submission( - self, submission_id: int, user: settings.AUTH_USER_MODEL + self, submission_id: int, request: 'rest_framework.request.Request', ) -> dict: pass @@ -439,19 +446,17 @@ def set_enketo_open_rosa_server( ): pass - @abc.abstractmethod - def set_has_kpi_hooks(self): - pass - def set_status(self, status): self.save_to_db({'status': status}) @abc.abstractmethod - def set_validation_status(self, - submission_id: int, - user: settings.AUTH_USER_MODEL, - data: dict, - method: str) -> dict: + def set_validation_status( + self, + submission_id: int, + user: settings.AUTH_USER_MODEL, + data: dict, + method: str, + ) -> dict: pass @abc.abstractmethod @@ -473,10 +478,9 @@ def store_data(self, values: dict): def stored_data_key(self): return self.__stored_data_key - @property @abc.abstractmethod def store_submission( - self, user, xml_submission, submission_uuid, attachments=None + self, user, xml_submission, submission_uuid, attachments=None, **kwargs ): pass @@ -662,7 +666,7 @@ def validate_access_with_partial_perms( perm: str, submission_ids: list = [], query: dict = {}, - ) -> list: + ) -> Optional[list]: """ Validate whether `user` is allowed to perform write actions on submissions with the permission `perm`. @@ -749,10 +753,6 @@ def version(self): def version_id(self): return self.get_data('version') - @property - def _open_rosa_server_storage(self): - return default_storage - def _get_metadata_queryset(self, file_type: str) -> Union[QuerySet, list]: """ Returns a list of objects, or a QuerySet to pass to Celery to diff --git a/kpi/deployment_backends/kc_access/shadow_models.py b/kpi/deployment_backends/kc_access/shadow_models.py deleted file mode 100644 index df01578af9..0000000000 --- a/kpi/deployment_backends/kc_access/shadow_models.py +++ /dev/null @@ -1,662 +0,0 @@ -# coding: utf-8 -from __future__ import annotations - -from typing import Optional -from urllib.parse import quote as urlquote - -from django.conf import settings -from django.contrib.contenttypes.fields import GenericForeignKey -from django.core import checks -from django.core.exceptions import FieldDoesNotExist -from django.core.files.base import ContentFile -from django.db import ( - ProgrammingError, - connections, - models, - transaction, -) -from django.utils import timezone -from django_digest.models import PartialDigest - -from kobo.apps.openrosa.libs.utils.image_tools import ( - get_optimized_image_path, - resize, -) -from kpi.constants import SHADOW_MODEL_APP_LABEL -from kpi.deployment_backends.kc_access.storage import ( - default_kobocat_storage, -) -from kpi.exceptions import ( - BadContentTypeException, -) -from kpi.fields.file import ExtendedFileField -from kpi.mixins.audio_transcoding import AudioTranscodingMixin -from kpi.utils.hash import calculate_hash -from .storage import ( - get_kobocat_storage, - KobocatFileSystemStorage, -) - - -def update_autofield_sequence(model): - """ - Fixes the PostgreSQL sequence for the first (and only?) `AutoField` on - `model`, à la `manage.py sqlsequencereset` - """ - # Updating sequences on fresh environments fails because the only user - # in the DB is django-guardian AnonymousUser and `max(pk)` returns -1. - # Error: - # > setval: value -1 is out of bounds for sequence - # Using abs() and testing if max(pk) equals -1, leaves the sequence alone. - sql_template = ( - "SELECT setval(" - " pg_get_serial_sequence('{table}','{column}'), " - " abs(coalesce(max({column}), 1)), " - " max({column}) IS NOT null and max({column}) != -1" - ") " - "FROM {table};" - ) - autofield = None - for f in model._meta.get_fields(): - if isinstance(f, models.AutoField): - autofield = f - break - if not autofield: - return - query = sql_template.format( - table=model._meta.db_table, column=autofield.column - ) - connection = connections[settings.OPENROSA_DB_ALIAS] - with connection.cursor() as cursor: - cursor.execute(query) - - -class ShadowModel(models.Model): - """ - Allows identification of writeable and read-only shadow models - """ - class Meta: - managed = False - abstract = True - # TODO find out why it raises a warning when user logs in. - # ``` - # RuntimeWarning: Model '...' was already registered. - # Reloading models is not advised as it can lead to inconsistencies, - # most notably with related models - # ``` - # Maybe because `SHADOW_MODEL_APP_LABEL` is not declared in - # `INSTALLED_APP` - # It's just used for `DefaultDatabaseRouter` conditions. - app_label = SHADOW_MODEL_APP_LABEL - - @classmethod - def get_app_label_and_model_name(cls) -> tuple[str, str]: - model_name_mapping = { - 'kobocatxform': ('logger', 'xform'), - 'readonlykobocatinstance': ('logger', 'instance'), - 'kobocatuserprofile': ('main', 'userprofile'), - 'kobocatuserobjectpermission': ('guardian', 'userobjectpermission'), - } - try: - return model_name_mapping[cls._meta.model_name] - except KeyError: - raise NotImplementedError - - @classmethod - def get_content_type(cls) -> KobocatContentType: - app_label, model_name = cls.get_app_label_and_model_name() - return KobocatContentType.objects.get( - app_label=app_label, model=model_name) - - -class KobocatAttachmentManager(models.Manager): - - def get_queryset(self): - return super().get_queryset().exclude(deleted_at__isnull=False) - - -class KobocatAttachment(ShadowModel, AudioTranscodingMixin): - - class Meta(ShadowModel.Meta): - db_table = 'logger_attachment' - - instance = models.ForeignKey( - 'superuser_stats.ReadOnlyKobocatInstance', - related_name='attachments', - on_delete=models.CASCADE, - ) - media_file = ExtendedFileField( - storage=get_kobocat_storage(), max_length=380, db_index=True - ) - media_file_basename = models.CharField( - max_length=260, null=True, blank=True, db_index=True) - # `PositiveIntegerField` will only accommodate 2 GiB, so we should consider - # `PositiveBigIntegerField` after upgrading to Django 3.1+ - media_file_size = models.PositiveIntegerField(blank=True, null=True) - mimetype = models.CharField( - max_length=100, null=False, blank=True, default='' - ) - deleted_at = models.DateTimeField(blank=True, null=True, db_index=True) - objects = KobocatAttachmentManager() - all_objects = models.Manager() - - @property - def absolute_mp3_path(self): - """ - Return the absolute path on local file system of the converted version of - attachment. Otherwise, return the AWS url (e.g. https://...) - """ - - kobocat_storage = get_kobocat_storage() - - if not kobocat_storage.exists(self.mp3_storage_path): - content = self.get_transcoded_audio('mp3') - kobocat_storage.save(self.mp3_storage_path, ContentFile(content)) - - if isinstance(kobocat_storage, KobocatFileSystemStorage): - return f'{self.media_file.path}.mp3' - - return kobocat_storage.url(self.mp3_storage_path) - - @property - def absolute_path(self): - """ - Return the absolute path on local file system of the attachment. - Otherwise, return the AWS url (e.g. https://...) - """ - if isinstance(get_kobocat_storage(), KobocatFileSystemStorage): - return self.media_file.path - - return self.media_file.url - - @property - def mp3_storage_path(self): - """ - Return the path of file after conversion. It is the exact same name, plus - the conversion audio format extension concatenated. - E.g: file.mp4 and file.mp4.mp3 - """ - return f'{self.storage_path}.mp3' - - def protected_path( - self, format_: Optional[str] = None, suffix: Optional[str] = None - ) -> str: - """ - Return path to be served as protected file served by NGINX - """ - if format_ == 'mp3': - attachment_file_path = self.absolute_mp3_path - else: - attachment_file_path = self.absolute_path - - optimized_image_path = None - if suffix and self.mimetype.startswith('image/'): - optimized_image_path = get_optimized_image_path( - self.media_file.name, suffix - ) - if not default_kobocat_storage.exists(optimized_image_path): - resize(self.media_file.name) - - if isinstance(get_kobocat_storage(), KobocatFileSystemStorage): - # Django normally sanitizes accented characters in file names during - # save on disk but some languages have extra letters - # (out of ASCII character set) and must be encoded to let NGINX serve - # them - if optimized_image_path: - attachment_file_path = default_kobocat_storage.path( - optimized_image_path - ) - protected_url = urlquote(attachment_file_path.replace( - settings.KOBOCAT_MEDIA_ROOT, '/protected') - ) - else: - # Double-encode the S3 URL to take advantage of NGINX's - # otherwise troublesome automatic decoding - if optimized_image_path: - attachment_file_path = default_kobocat_storage.url( - optimized_image_path - ) - protected_url = f'/protected-s3/{urlquote(attachment_file_path)}' - - return protected_url - - @property - def storage_path(self): - return str(self.media_file) - - -class KobocatContentType(ShadowModel): - """ - Minimal representation of Django 1.8's - contrib.contenttypes.models.ContentType - """ - app_label = models.CharField(max_length=100) - model = models.CharField('python model class name', max_length=100) - - class Meta(ShadowModel.Meta): - db_table = 'django_content_type' - unique_together = (('app_label', 'model'),) - - def __str__(self): - # Not as nice as the original, which returns a human-readable name - # complete with whitespace. That requires access to the Python model - # class, though - return self.model - - -class KobocatDailyXFormSubmissionCounter(ShadowModel): - - date = models.DateField() - user = models.ForeignKey( - 'shadow_model.KobocatUser', null=True, on_delete=models.CASCADE - ) - xform = models.ForeignKey( - 'shadow_model.KobocatXForm', - related_name='daily_counts', - null=True, - on_delete=models.CASCADE, - ) - counter = models.IntegerField(default=0) - - class Meta(ShadowModel.Meta): - db_table = 'logger_dailyxformsubmissioncounter' - unique_together = [['date', 'xform', 'user'], ['date', 'user']] - - -class KobocatGenericForeignKey(GenericForeignKey): - - def get_content_type(self, obj=None, id=None, using=None): - if obj is not None: - return KobocatContentType.objects.db_manager(obj._state.db).get_for_model( - obj, for_concrete_model=self.for_concrete_model) - elif id is not None: - return KobocatContentType.objects.db_manager(using).get_for_id(id) - else: - # This should never happen. I love comments like this, don't you? - raise Exception("Impossible arguments to GFK.get_content_type!") - - def get_forward_related_filter(self, obj): - """See corresponding method on RelatedField""" - return { - self.fk_field: obj.pk, - self.ct_field: KobocatContentType.objects.get_for_model(obj).pk, - } - - def _check_content_type_field(self): - try: - field = self.model._meta.get_field(self.ct_field) - except FieldDoesNotExist: - return [ - checks.Error( - "The GenericForeignKey content type references the " - "nonexistent field '%s.%s'." % ( - self.model._meta.object_name, self.ct_field - ), - obj=self, - id='contenttypes.E002', - ) - ] - else: - if not isinstance(field, models.ForeignKey): - return [ - checks.Error( - "'%s.%s' is not a ForeignKey." % ( - self.model._meta.object_name, self.ct_field - ), - hint=( - "GenericForeignKeys must use a ForeignKey to " - "'contenttypes.ContentType' as the 'content_type' field." - ), - obj=self, - id='contenttypes.E003', - ) - ] - elif field.remote_field.model != KobocatContentType: - return [ - checks.Error( - "'%s.%s' is not a ForeignKey to 'contenttypes.ContentType'." - % (self.model._meta.object_name, self.ct_field), - hint=( - "GenericForeignKeys must use a ForeignKey to " - "'contenttypes.ContentType' as the 'content_type' field." - ), - obj=self, - id='contenttypes.E004', - ) - ] - else: - return [] - - -class KobocatMetadata(ShadowModel): - - MEDIA_FILES_TYPE = [ - 'media', - 'paired_data', - ] - - xform = models.ForeignKey('shadow_model.KobocatXForm', on_delete=models.CASCADE) - data_type = models.CharField(max_length=255) - data_value = models.CharField(max_length=255) - data_file = ExtendedFileField(storage=get_kobocat_storage(), blank=True, null=True) - data_file_type = models.CharField(max_length=255, blank=True, null=True) - file_hash = models.CharField(max_length=50, blank=True, null=True) - from_kpi = models.BooleanField(default=False) - data_filename = models.CharField(max_length=255, blank=True, null=True) - date_created = models.DateTimeField(default=timezone.now) - date_modified = models.DateTimeField(default=timezone.now) - - class Meta(ShadowModel.Meta): - db_table = 'main_metadata' - - -class KobocatMonthlyXFormSubmissionCounter(ShadowModel): - year = models.IntegerField() - month = models.IntegerField() - user = models.ForeignKey( - 'shadow_model.KobocatUser', - on_delete=models.CASCADE, - ) - xform = models.ForeignKey( - 'shadow_model.KobocatXForm', - related_name='monthly_counts', - null=True, - on_delete=models.SET_NULL, - ) - counter = models.IntegerField(default=0) - - class Meta(ShadowModel.Meta): - app_label = 'superuser_stats' - db_table = 'logger_monthlyxformsubmissioncounter' - verbose_name_plural = 'User Statistics' - - -class KobocatPermission(ShadowModel): - """ - Minimal representation of Django 1.8's contrib.auth.models.Permission - """ - name = models.CharField('name', max_length=255) - content_type = models.ForeignKey(KobocatContentType, on_delete=models.CASCADE) - codename = models.CharField('codename', max_length=100) - - class Meta(ShadowModel.Meta): - db_table = 'auth_permission' - unique_together = (('content_type', 'codename'),) - ordering = ('content_type__app_label', 'content_type__model', - 'codename') - - def __str__(self): - return "%s | %s | %s" % ( - str(self.content_type.app_label), - str(self.content_type), - str(self.name)) - - -class KobocatUser(ShadowModel): - - username = models.CharField('username', max_length=30, unique=True) - password = models.CharField('password', max_length=128) - last_login = models.DateTimeField('last login', blank=True, null=True) - is_superuser = models.BooleanField('superuser status', default=False) - first_name = models.CharField('first name', max_length=30, blank=True) - last_name = models.CharField('last name', max_length=150, blank=True) - email = models.EmailField('email address', blank=True) - is_staff = models.BooleanField('staff status', default=False) - is_active = models.BooleanField('active', default=True) - date_joined = models.DateTimeField('date joined', default=timezone.now) - - class Meta(ShadowModel.Meta): - db_table = 'auth_user' - - @classmethod - @transaction.atomic - def sync(cls, auth_user): - # NB: `KobocatUserObjectPermission` (and probably other things) depend - # upon PKs being synchronized between KPI and KoboCAT - kc_auth_user = cls.get_kc_user(auth_user) - kc_auth_user.password = auth_user.password - kc_auth_user.last_login = auth_user.last_login - kc_auth_user.is_superuser = auth_user.is_superuser - kc_auth_user.first_name = auth_user.first_name - kc_auth_user.last_name = auth_user.last_name - kc_auth_user.email = auth_user.email - kc_auth_user.is_staff = auth_user.is_staff - kc_auth_user.is_active = auth_user.is_active - kc_auth_user.date_joined = auth_user.date_joined - - kc_auth_user.save() - - # We've manually set a primary key, so `last_value` in the sequence - # `auth_user_id_seq` now lags behind `max(id)`. Fix it now! - update_autofield_sequence(cls) - - @classmethod - def get_kc_user(cls, auth_user: settings.AUTH_USER_MODEL) -> KobocatUser: - try: - kc_auth_user = cls.objects.get(pk=auth_user.pk) - assert kc_auth_user.username == auth_user.username - except KobocatUser.DoesNotExist: - kc_auth_user = cls(pk=auth_user.pk, username=auth_user.username) - - return kc_auth_user - - -class KobocatUserObjectPermission(ShadowModel): - """ - For the _sole purpose_ of letting us manipulate KoBoCAT - permissions, this comprises the following django-guardian classes - all condensed into one: - - * UserObjectPermission - * UserObjectPermissionBase - * BaseGenericObjectPermission - * BaseObjectPermission - - CAVEAT LECTOR: The django-guardian custom manager, - UserObjectPermissionManager, is NOT included! - """ - permission = models.ForeignKey(KobocatPermission, on_delete=models.CASCADE) - content_type = models.ForeignKey(KobocatContentType, on_delete=models.CASCADE) - object_pk = models.CharField('object ID', max_length=255) - content_object = KobocatGenericForeignKey(fk_field='object_pk') - user = models.ForeignKey(KobocatUser, on_delete=models.CASCADE) - - class Meta(ShadowModel.Meta): - db_table = 'guardian_userobjectpermission' - unique_together = ['user', 'permission', 'object_pk'] - - def __str__(self): - # `unicode(self.content_object)` fails when the object's model - # isn't known to this Django project. Let's use something more - # benign instead. - content_object_str = '{app_label}_{model} ({pk})'.format( - app_label=self.content_type.app_label, - model=self.content_type.model, - pk=self.object_pk) - return '%s | %s | %s' % ( - # unicode(self.content_object), - content_object_str, - str(getattr(self, 'user', False) or self.group), - str(self.permission.codename)) - - def save(self, *args, **kwargs): - content_type = KobocatContentType.objects.get_for_model( - self.content_object) - if content_type != self.permission.content_type: - raise BadContentTypeException( - f"Cannot persist permission not designed for this " - "class (permission's type is {self.permission.content_type} " - "and object's type is {content_type}") - return super().save(*args, **kwargs) - - -class KobocatUserPermission(ShadowModel): - """ Needed to assign model-level KoBoCAT permissions """ - user = models.ForeignKey('KobocatUser', db_column='user_id', - on_delete=models.CASCADE) - permission = models.ForeignKey('KobocatPermission', - db_column='permission_id', - on_delete=models.CASCADE) - - class Meta(ShadowModel.Meta): - db_table = 'auth_user_user_permissions' - - -class KobocatUserProfile(ShadowModel): - """ - From onadata/apps/main/models/user_profile.py - """ - class Meta(ShadowModel.Meta): - db_table = 'main_userprofile' - verbose_name = 'user profile' - verbose_name_plural = 'user profiles' - - # This field is required. - user = models.OneToOneField(KobocatUser, - related_name='profile', - on_delete=models.CASCADE) - - # Other fields here - name = models.CharField(max_length=255, blank=True) - city = models.CharField(max_length=255, blank=True) - country = models.CharField(max_length=2, blank=True) - organization = models.CharField(max_length=255, blank=True) - home_page = models.CharField(max_length=255, blank=True) - twitter = models.CharField(max_length=255, blank=True) - description = models.CharField(max_length=255, blank=True) - require_auth = models.BooleanField(default=True) - address = models.CharField(max_length=255, blank=True) - phonenumber = models.CharField(max_length=30, blank=True) - num_of_submissions = models.IntegerField(default=0) - attachment_storage_bytes = models.BigIntegerField(default=0) - metadata = models.JSONField(default=dict, blank=True) - # We need to cast `is_active` to an (positive small) integer because KoBoCAT - # is using `LazyBooleanField` which is an integer behind the scene. - # We do not want to port this class to KPI only for one line of code. - is_mfa_active = models.PositiveSmallIntegerField(default=False) - validated_password = models.BooleanField(default=False) - - @classmethod - def set_mfa_status(cls, user_id: int, is_active: bool): - - user_profile, created = cls.objects.get_or_create(user_id=user_id) - user_profile.is_mfa_active = int(is_active) - user_profile.save(update_fields=['is_mfa_active']) - - @classmethod - def set_password_details( - cls, - user_id: int, - validated: bool, - ): - """ - Update the kobocat user's password_change_date and validated_password fields - """ - user_profile, created = cls.objects.get_or_create(user_id=user_id) - user_profile.validated_password = validated - user_profile.save( - update_fields=['validated_password'] - ) - - -class KobocatToken(ShadowModel): - - key = models.CharField("Key", max_length=40, primary_key=True) - user = models.OneToOneField(KobocatUser, - related_name='auth_token', - on_delete=models.CASCADE, verbose_name="User") - created = models.DateTimeField("Created", auto_now_add=True) - - class Meta(ShadowModel.Meta): - db_table = "authtoken_token" - - @classmethod - def sync(cls, auth_token): - try: - # Token use a One-to-One relationship on User. - # Thus, we can retrieve tokens from users' id. - kc_auth_token = cls.objects.get(user_id=auth_token.user_id) - except KobocatToken.DoesNotExist: - kc_auth_token = cls(pk=auth_token.pk, user_id=auth_token.user_id) - - kc_auth_token.save() - - -class KobocatXForm(ShadowModel): - - class Meta(ShadowModel.Meta): - db_table = 'logger_xform' - verbose_name = 'xform' - verbose_name_plural = 'xforms' - - XFORM_TITLE_LENGTH = 255 - xls = ExtendedFileField(null=True) - xml = models.TextField() - user = models.ForeignKey( - KobocatUser, related_name='xforms', null=True, on_delete=models.CASCADE - ) - shared = models.BooleanField(default=False) - shared_data = models.BooleanField(default=False) - downloadable = models.BooleanField(default=True) - id_string = models.SlugField() - title = models.CharField(max_length=XFORM_TITLE_LENGTH) - date_created = models.DateTimeField() - date_modified = models.DateTimeField() - uuid = models.CharField(max_length=32, default='') - last_submission_time = models.DateTimeField(blank=True, null=True) - num_of_submissions = models.IntegerField(default=0) - attachment_storage_bytes = models.BigIntegerField(default=0) - kpi_asset_uid = models.CharField(max_length=32, null=True) - pending_delete = models.BooleanField(default=False) - require_auth = models.BooleanField(default=True) - - @property - def md5_hash(self): - return calculate_hash(self.xml) - - @property - def prefixed_hash(self): - """ - Matches what's returned by the KC API - """ - - return "md5:%s" % self.md5_hash - - -class ReadOnlyModel(ShadowModel): - - read_only = True - - class Meta(ShadowModel.Meta): - abstract = True - - -class ReadOnlyKobocatInstance(ReadOnlyModel): - - class Meta(ReadOnlyModel.Meta): - app_label = 'superuser_stats' - db_table = 'logger_instance' - verbose_name = 'Submissions by Country' - verbose_name_plural = 'Submissions by Country' - - xml = models.TextField() - user = models.ForeignKey(KobocatUser, null=True, on_delete=models.CASCADE) - xform = models.ForeignKey(KobocatXForm, related_name='instances', - on_delete=models.CASCADE) - date_created = models.DateTimeField() - date_modified = models.DateTimeField() - deleted_at = models.DateTimeField(null=True, default=None) - status = models.CharField(max_length=20, - default='submitted_via_web') - uuid = models.CharField(max_length=249, default='') - - -def safe_kc_read(func): - def _wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except ProgrammingError as e: - raise ProgrammingError( - 'kc_access error accessing kobocat tables: {}'.format(str(e)) - ) - return _wrapper diff --git a/kpi/deployment_backends/kc_access/utils.py b/kpi/deployment_backends/kc_access/utils.py index 47aa22a3fd..857be6c590 100644 --- a/kpi/deployment_backends/kc_access/utils.py +++ b/kpi/deployment_backends/kc_access/utils.py @@ -1,118 +1,29 @@ -# coding: utf-8 -import json import logging from contextlib import ContextDecorator from typing import Union -import requests from django.conf import settings -from django.contrib.auth.models import AnonymousUser +from django.contrib.auth.models import AnonymousUser, Permission +from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ImproperlyConfigured -from django.db import transaction +from django.db import ProgrammingError, transaction from django.db.models import Model -from kobo_service_account.utils import get_request_headers -from rest_framework.authtoken.models import Token +from guardian.models import UserObjectPermission from kobo.apps.kobo_auth.shortcuts import User -from kpi.exceptions import KobocatProfileException from kpi.utils.log import logging from kpi.utils.permissions import is_user_anonymous -from .shadow_models import ( - safe_kc_read, - KobocatContentType, - KobocatPermission, - KobocatUser, - KobocatUserObjectPermission, - KobocatUserPermission, - KobocatUserProfile, - KobocatXForm, -) - - -def _trigger_kc_profile_creation(user): - """ - Get the user's profile via the KC API, causing KC to create a KC - UserProfile if none exists already - """ - url = settings.KOBOCAT_INTERNAL_URL + '/api/v1/user' - token, _ = Token.objects.get_or_create(user=user) - response = requests.get( - url, headers={'Authorization': 'Token ' + token.key}) - if not response.status_code == 200: - raise KobocatProfileException( - 'Bad HTTP status code `{}` when retrieving KoBoCAT user profile' - ' for `{}`.'.format(response.status_code, user.username)) - return response - - -@safe_kc_read -def instance_count(xform_id_string, user_id): - try: - return KobocatXForm.objects.only('num_of_submissions').get( - id_string=xform_id_string, - user_id=user_id - ).num_of_submissions - except KobocatXForm.DoesNotExist: - return 0 -@safe_kc_read -def last_submission_time(xform_id_string, user_id): - return KobocatXForm.objects.get( - user_id=user_id, id_string=xform_id_string - ).last_submission_time - - -@safe_kc_read -def get_kc_profile_data(user_id): - """ - Retrieve all fields from the user's KC profile and return them in a - dictionary - """ - try: - profile_model = KobocatUserProfile.objects.get(user_id=user_id) - # Use a dict instead of the object in case we enter the next exception. - # The response will return a json. - # We want the variable to have the same type in both cases. - profile = profile_model.__dict__ - except KobocatUserProfile.DoesNotExist: +def safe_kc_read(func): + def _wrapper(*args, **kwargs): try: - response = _trigger_kc_profile_creation(User.objects.get(pk=user_id)) - profile = response.json() - except KobocatProfileException: - logging.exception('Failed to create KoBoCAT user profile') - return {} - - fields = [ - # Use a (kc_name, new_name) tuple to rename a field - 'name', - 'organization', - ('home_page', 'organization_website'), - ('description', 'bio'), - ('phonenumber', 'phone_number'), - 'address', - 'city', - 'country', - 'twitter', - 'metadata', - ] - - result = {} - - for field in fields: - - if isinstance(field, tuple): - kc_name, field = field - else: - kc_name = field - - value = profile.get(kc_name) - # When a field contains JSON (e.g. `metadata`), it gets loaded as a - # `dict`. Convert it back to a string representation - if isinstance(value, dict): - value = json.dumps(value) - result[field] = value - return result + return func(*args, **kwargs) + except ProgrammingError as e: + raise ProgrammingError( + 'kc_access error accessing KoboCAT tables: {}'.format(str(e)) + ) + return _wrapper def _get_content_type_kwargs_for_related(obj): @@ -167,8 +78,9 @@ def _get_applicable_kc_permissions(obj, kpi_codenames): # This permission doesn't map to anything in KC continue content_type_kwargs = _get_content_type_kwargs_for_related(obj) - permissions = KobocatPermission.objects.filter( - codename__in=kc_codenames, **content_type_kwargs) + permissions = Permission.objects.using(settings.OPENROSA_DB_ALIAS).filter( + codename__in=kc_codenames, **content_type_kwargs + ) return permissions @@ -183,58 +95,43 @@ def _get_xform_id_for_asset(asset): raise -def grant_kc_model_level_perms(user): +def grant_kc_model_level_perms(user: 'kobo_auth.User'): """ Gives `user` unrestricted model-level access to everything listed in settings.KOBOCAT_DEFAULT_PERMISSION_CONTENT_TYPES. Without this, actions on individual instances are immediately denied and object-level permissions are never considered. """ - if not isinstance(user, KobocatUser): - user = KobocatUser.objects.get(pk=user.pk) - content_types = [] for pair in settings.KOBOCAT_DEFAULT_PERMISSION_CONTENT_TYPES: try: content_types.append( - KobocatContentType.objects.get( + ContentType.objects.using(settings.OPENROSA_DB_ALIAS).get( app_label=pair[0], model=pair[1] ) ) - except KobocatContentType.DoesNotExist: + except ContentType.DoesNotExist: # Consider raising `ImproperlyConfigured` here. Anyone running KPI # without KC should change # `KOBOCAT_DEFAULT_PERMISSION_CONTENT_TYPES` appropriately in their # settings logging.error( - 'Could not find KoBoCAT content type for {}.{}'.format(*pair) + 'Could not find KoboCAT content type for {}.{}'.format(*pair) ) - permissions_to_assign = KobocatPermission.objects.filter( - content_type__in=content_types) + permissions_to_assign = Permission.objects.using( + settings.OPENROSA_DB_ALIAS + ).filter(content_type__in=content_types) if content_types and not permissions_to_assign.exists(): raise RuntimeError( - 'No KoBoCAT permissions found! You may need to run the Django ' - 'management command `migrate` in your KoBoCAT environment. ' + 'No KoboCAT permissions found! You may need to run the Django ' + 'management command `migrate` in your KoboCAT environment. ' 'Searched for content types {}.'.format(content_types) ) - # What KC permissions does this user already have? Getting the KC database - # column names right necessitated a custom M2M model, - # `KobocatUserPermission`, which means we can't use Django's tolerant - # `add()`. Prior to Django 2.2, there's no way to make `bulk_create()` - # ignore `IntegrityError`s, so we have to avoid duplication manually: - # https://docs.djangoproject.com/en/2.2/ref/models/querysets/#django.db.models.query.QuerySet.bulk_create - existing_user_perm_pks = KobocatUserPermission.objects.filter( - user=user - ).values_list('permission_id', flat=True) - - KobocatUserPermission.objects.bulk_create([ - KobocatUserPermission(user=user, permission=p) - for p in permissions_to_assign if p.pk not in existing_user_perm_pks - ]) + user.user_permissions.add(*permissions_to_assign) def set_kc_anonymous_permissions_xform_flags( @@ -242,7 +139,7 @@ def set_kc_anonymous_permissions_xform_flags( ): r""" Given a KPI object, one or more KPI permission codenames and the PK of - a KC `XForm`, assume the KPI permisisons have been assigned to or + a KC `XForm`, assume the KPI permissions have been assigned to or removed from the anonymous user. Then, modify any corresponding flags on the `XForm` accordingly. :param obj: Object with `KC_ANONYMOUS_PERMISSIONS_XFORM_FLAGS` @@ -277,7 +174,8 @@ def set_kc_anonymous_permissions_xform_flags( xform_updates.update(flags) # Write to the KC database - KobocatXForm.objects.filter(pk=xform_id).update(**xform_updates) + XForm = obj.deployment.xform.__class__ # noqa - avoid circular imports + XForm.objects.filter(pk=xform_id).update(**xform_updates) def assign_applicable_kc_permissions( @@ -315,21 +213,34 @@ def assign_applicable_kc_permissions( obj, kpi_codenames, xform_id ) - xform_content_type = KobocatContentType.objects.get( - **obj.KC_CONTENT_TYPE_KWARGS) + xform_content_type = ContentType.objects.using( + settings.OPENROSA_DB_ALIAS + ).get(**obj.KC_CONTENT_TYPE_KWARGS) - kc_permissions_already_assigned = KobocatUserObjectPermission.objects.filter( - user_id=user_id, permission__in=permissions, object_pk=xform_id, - ).values_list('permission__codename', flat=True) + kc_permissions_already_assigned = ( + UserObjectPermission.objects.using(settings.OPENROSA_DB_ALIAS) + .filter( + user_id=user_id, + permission__in=permissions, + object_pk=xform_id, + ) + .values_list('permission__codename', flat=True) + ) permissions_to_create = [] for permission in permissions: if permission.codename in kc_permissions_already_assigned: continue - permissions_to_create.append(KobocatUserObjectPermission( - user_id=user_id, permission=permission, object_pk=xform_id, - content_type=xform_content_type - )) - KobocatUserObjectPermission.objects.bulk_create(permissions_to_create) + permissions_to_create.append( + UserObjectPermission( + user_id=user_id, + permission=permission, + object_pk=xform_id, + content_type=xform_content_type, + ) + ) + UserObjectPermission.objects.using(settings.OPENROSA_DB_ALIAS).bulk_create( + permissions_to_create + ) def remove_applicable_kc_permissions( @@ -365,10 +276,11 @@ def remove_applicable_kc_permissions( if user_id == settings.ANONYMOUS_USER_ID: return set_kc_anonymous_permissions_xform_flags( - obj, kpi_codenames, xform_id, remove=True) + obj, kpi_codenames, xform_id, remove=True + ) content_type_kwargs = _get_content_type_kwargs_for_related(obj) - KobocatUserObjectPermission.objects.filter( + UserObjectPermission.objects.using(settings.OPENROSA_DB_ALIAS).filter( user_id=user_id, permission__in=permissions, object_pk=xform_id, # `permission` has a FK to `ContentType`, but I'm paranoid **content_type_kwargs @@ -406,25 +318,23 @@ def reset_kc_permissions( raise NotImplementedError content_type_kwargs = _get_content_type_kwargs_for_related(obj) - KobocatUserObjectPermission.objects.filter( - user_id=user_id, object_pk=xform_id, + UserObjectPermission.objects.using(settings.OPENROSA_DB_ALIAS).filter( + user_id=user_id, + object_pk=xform_id, # `permission` has a FK to `ContentType`, but I'm paranoid **content_type_kwargs ).delete() def delete_kc_user(username: str): - url = settings.KOBOCAT_INTERNAL_URL + f'/api/v1/users/{username}' - - response = requests.delete( - url, headers=get_request_headers(username) - ) - response.raise_for_status() + User.objects.using(settings.OPENROSA_DB_ALIAS).filter( + username=username + ).delete() -def kc_transaction_atomic(using='kobocat', *args, **kwargs): +def kc_transaction_atomic(using=settings.OPENROSA_DB_ALIAS, *args, **kwargs): """ - KoBoCAT database does not exist in testing environment. + KoboCAT database does not exist in testing environment. `transaction.atomic(using='kobocat') cannot be called without raising errors. This utility returns a context manager which does nothing if environment @@ -442,7 +352,7 @@ def __exit__(self, exc_type, exc_value, traceback): pass assert ( - callable(using) or using == 'kobocat' + callable(using) or using == settings.OPENROSA_DB_ALIAS ), "`kc_transaction_atomic` may only be used with the 'kobocat' database" if settings.TESTING: @@ -455,6 +365,6 @@ def __exit__(self, exc_type, exc_value, traceback): # Not in a testing environment; use the real `atomic` if callable(using): - return transaction.atomic('kobocat', *args, **kwargs)(using) + return transaction.atomic(settings.OPENROSA_DB_ALIAS, *args, **kwargs)(using) else: - return transaction.atomic('kobocat', *args, **kwargs) + return transaction.atomic(settings.OPENROSA_DB_ALIAS, *args, **kwargs) diff --git a/kpi/deployment_backends/mixin.py b/kpi/deployment_backends/mixin.py index ae1690bf14..4a448106b3 100644 --- a/kpi/deployment_backends/mixin.py +++ b/kpi/deployment_backends/mixin.py @@ -20,7 +20,6 @@ def sync_media_files_async(self, always=True): file_type=AssetFile.FORM_MEDIA, synced_with_backend=False ).exists(): self.save(create_version=False, adjust_content=False) - # Not using .delay() due to circular import in tasks.py celery.current_app.send_task('kpi.tasks.sync_media_files', (self.uid,)) diff --git a/kpi/deployment_backends/mock_backend.py b/kpi/deployment_backends/mock_backend.py index 0d6012a684..296e8fbb25 100644 --- a/kpi/deployment_backends/mock_backend.py +++ b/kpi/deployment_backends/mock_backend.py @@ -16,12 +16,17 @@ from deepmerge import always_merger from dict2xml import dict2xml as dict2xml_real +from django.db.models import Q from django.conf import settings +from django.core.files.base import ContentFile from django.db.models import Sum from django.db.models.functions import Coalesce from django.urls import reverse from rest_framework import status +from kobo.apps.openrosa.apps.logger.models import Attachment, Instance, XForm +from kobo.apps.openrosa.apps.logger.models.attachment import upload_to +from kobo.apps.openrosa.apps.main.models import UserProfile from kobo.apps.trackers.models import NLPUsageCounter from kpi.constants import ( SUBMISSION_FORMAT_TYPE_JSON, @@ -38,7 +43,6 @@ ) from kpi.interfaces.sync_backend_media import SyncBackendMediaInterface from kpi.models.asset_file import AssetFile -from kpi.tests.utils.mock import MockAttachment from kpi.utils.mongo_helper import MongoHelper, drop_mock_only from kpi.utils.xml import fromstring_preserve_root_xmlns from .base_backend import BaseDeploymentBackend @@ -87,7 +91,6 @@ def generate_uuid_for_form(): 'active': active, 'backend_response': { 'downloadable': active, - 'has_kpi_hook': self.asset.has_active_hooks, 'kpi_asset_uid': self.asset.uid, 'uuid': generate_uuid_for_form(), # TODO use XForm object and get its primary key @@ -101,7 +104,7 @@ def generate_uuid_for_form(): def form_uuid(self): return 'formhub-uuid' # to match existing tests - def nlp_tracking_data(self, start_date=None): + def nlp_tracking_data(asset_ids=None): """ Get the NLP tracking data since a specified date If no date is provided, get all-time data @@ -164,7 +167,7 @@ def delete_submission( } def delete_submissions( - self, data: dict, user: settings.AUTH_USER_MODEL + self, data: dict, user: settings.AUTH_USER_MODEL, **kwargs ) -> dict: """ Bulk delete provided submissions authenticated by `user`'s API token. @@ -213,11 +216,11 @@ def delete_submissions( } def duplicate_submission( - self, submission_id: int, user: settings.AUTH_USER_MODEL + self, submission_id: int, request: 'rest_framework.request.Request', ) -> dict: # TODO: Make this operate on XML somehow and reuse code from # KobocatDeploymentBackend, to catch issues like #3054 - + user = request.user self.validate_access_with_partial_perms( user=user, perm=PERM_CHANGE_SUBMISSIONS, @@ -275,7 +278,7 @@ def get_attachment( user: settings.AUTH_USER_MODEL, attachment_id: Optional[int] = None, xpath: Optional[str] = None, - ) -> MockAttachment: + ) -> 'logger.Attachment': submission_json = None # First try to get the json version of the submission. # It helps to retrieve the id if `submission_id_or_uuid` is a `UUIDv4` @@ -325,7 +328,13 @@ def get_attachment( is_good_file = int(attachment['id']) == int(attachment_id) if is_good_file: - return MockAttachment(pk=attachment_id, **attachment) + return self._get_attachment_object( + attachment_id=attachment['id'], + submission_xml=submission_xml, + submission_id=submission_json['_id'], + filename=filename, + mimetype=attachment.get('mimetype'), + ) raise AttachmentNotFoundException @@ -333,8 +342,18 @@ def get_attachment_objects_from_dict(self, submission: dict) -> list: if not submission.get('_attachments'): return [] attachments = submission.get('_attachments') + submission_xml = self.get_submission( + submission['_id'], self.asset.owner, format_type=SUBMISSION_FORMAT_TYPE_XML + ) + return [ - MockAttachment(pk=attachment['id'], **attachment) + self._get_attachment_object( + attachment_id=attachment['id'], + submission_xml=submission_xml, + submission_id=submission['_id'], + filename=os.path.basename(attachment['filename']), + mimetype=attachment.get('mimetype'), + ) for attachment in attachments ] @@ -505,19 +524,6 @@ def set_enketo_open_rosa_server( ): pass - def set_has_kpi_hooks(self): - """ - Store a boolean which indicates that KPI has active hooks (or not) - and, if it is the case, it should receive notifications when new data - comes in - """ - has_active_hooks = self.asset.has_active_hooks - self.store_data( - { - 'has_kpi_hooks': has_active_hooks, - } - ) - def set_namespace(self, namespace): self.store_data( { @@ -622,7 +628,7 @@ def set_validation_statuses( } def store_submission( - self, user, xml_submission, submission_uuid, attachments=None + self, user, xml_submission, submission_uuid, attachments=None, **kwargs ): """ Return a mock response without actually storing anything @@ -704,13 +710,62 @@ def transfer_submissions_ownership( @property def xform(self): """ - Dummy property, only present to be mocked by unit tests + Create related XForm on the fly """ - pass + if not ( + xform := XForm.objects.filter(id_string=self.asset.uid).first() + ): + UserProfile.objects.get_or_create(user_id=self.asset.owner_id) + xform = XForm() + xform.xml = self.asset.snapshot().xml + xform.user_id = self.asset.owner_id + xform.kpi_asset_uid = self.asset.uid + xform.save() + + return xform @property def xform_id_string(self): - return self.asset.uid + return self.xform.id_string + + def _get_attachment_object( + self, + submission_xml: str, + submission_id: int, + attachment_id: Optional[int, str] = None, + filename: Optional[str] = None, + mimetype: Optional[str] = None, + ): + if not ( + attachment := Attachment.objects.filter( + Q(pk=attachment_id) | Q(media_file_basename=filename) + ).first() + ): + if not ( + instance := Instance.objects.filter(pk=submission_id).first() + ): + instance = Instance.objects.create( + pk=submission_id, xml=submission_xml, xform=self.xform + ) + + attachment = Attachment() + attachment.instance = instance + basename = os.path.basename(filename) + file_ = os.path.join( + settings.BASE_DIR, + 'kpi', + 'tests', + basename + ) + with open(file_, 'rb') as f: + attachment.media_file = ContentFile( + f.read(), name=upload_to(attachment, basename) + ) + if mimetype: + attachment.mimetype = mimetype + attachment.save() + + return attachment @classmethod def __prepare_bulk_update_data(cls, updates: dict) -> dict: diff --git a/kpi/deployment_backends/kobocat_backend.py b/kpi/deployment_backends/openrosa_backend.py similarity index 62% rename from kpi/deployment_backends/kobocat_backend.py rename to kpi/deployment_backends/openrosa_backend.py index a9b0984457..1d89afdb63 100644 --- a/kpi/deployment_backends/kobocat_backend.py +++ b/kpi/deployment_backends/openrosa_backend.py @@ -1,12 +1,9 @@ from __future__ import annotations -import io -import json -import re from collections import defaultdict from contextlib import contextmanager from datetime import date, datetime -from typing import Generator, Optional, Union +from typing import Generator, Optional, Union, Literal from urllib.parse import urlparse try: from zoneinfo import ZoneInfo @@ -17,17 +14,34 @@ import redis.exceptions from defusedxml import ElementTree as DET from django.conf import settings -from django.core.exceptions import ImproperlyConfigured from django.core.files import File +from django.core.files.base import ContentFile from django.db.models import Sum, F from django.db.models.functions import Coalesce from django.db.models.query import QuerySet from django.utils import timezone from django.utils.translation import gettext_lazy as t from django_redis import get_redis_connection -from kobo_service_account.utils import get_request_headers from rest_framework import status +from kobo.apps.openrosa.apps.main.models import MetaData, UserProfile +from kobo.apps.openrosa.apps.logger.models import ( + Attachment, + DailyXFormSubmissionCounter, + MonthlyXFormSubmissionCounter, + Instance, + XForm, +) +from kobo.apps.openrosa.apps.logger.utils.instance import ( + add_validation_status_to_instance, + delete_instances, + remove_validation_status_from_instance, + set_instance_validation_statuses, +) +from kobo.apps.openrosa.libs.utils.logger_tools import ( + safe_create_instance, + publish_xls_form, +) from kobo.apps.subsequences.utils import stream_with_extras from kobo.apps.trackers.models import NLPUsageCounter from kpi.constants import ( @@ -43,7 +57,6 @@ from kpi.exceptions import ( AttachmentNotFoundException, InvalidXFormException, - KobocatCommunicationError, SubmissionIntegrityError, SubmissionNotFoundException, XPathNotFoundException, @@ -53,36 +66,24 @@ from kpi.models.object_permission import ObjectPermission from kpi.models.paired_data import PairedData from kpi.utils.django_orm_helper import UpdateJSONFieldAttributes +from kpi.utils.files import ExtendedContentFile from kpi.utils.log import logging from kpi.utils.mongo_helper import MongoHelper from kpi.utils.object_permission import get_database_user -from kpi.utils.permissions import is_user_anonymous from kpi.utils.xml import fromstring_preserve_root_xmlns, xml_tostring from .base_backend import BaseDeploymentBackend -from .kc_access.shadow_models import ( - KobocatAttachment, - KobocatDailyXFormSubmissionCounter, - KobocatMonthlyXFormSubmissionCounter, - KobocatUserProfile, - KobocatXForm, - ReadOnlyKobocatInstance, -) from .kc_access.utils import ( assign_applicable_kc_permissions, kc_transaction_atomic, - last_submission_time ) from ..exceptions import ( BadFormatException, - KobocatDeploymentException, - KobocatDuplicateSubmissionException, ) -class KobocatDeploymentBackend(BaseDeploymentBackend): +class OpenRosaDeploymentBackend(BaseDeploymentBackend): """ - Used to deploy a project into KoBoCAT. Stores the project identifiers in the - `self.asset._deployment_data` models.JSONField (referred as "deployment data") + Used to deploy a project into KoboCAT. """ SYNCED_DATA_FILE_TYPES = { @@ -90,6 +91,10 @@ class KobocatDeploymentBackend(BaseDeploymentBackend): AssetFile.PAIRED_DATA: 'paired_data', } + def __init__(self, asset): + super().__init__(asset) + self._xform = None + @property def attachment_storage_bytes(self): try: @@ -106,8 +111,10 @@ def bulk_assign_mapped_perms(self): users_with_perms = self.asset.get_users_with_perms(attach_perms=True) # if only the owner has permissions, no need to go further - if len(users_with_perms) == 1 and \ - list(users_with_perms)[0].id == self.asset.owner_id: + if ( + len(users_with_perms) == 1 + and list(users_with_perms)[0].id == self.asset.owner_id + ): return with kc_transaction_atomic(): @@ -130,48 +137,35 @@ def connect(self, active=False): Store results in deployment data. CAUTION: Does not save deployment data to the database! """ - # Use the external URL here; the internal URL will be substituted - # in when appropriate - if not settings.KOBOCAT_URL or not settings.KOBOCAT_INTERNAL_URL: - raise ImproperlyConfigured( - 'Both KOBOCAT_URL and KOBOCAT_INTERNAL_URL must be ' - 'configured before using KobocatDeploymentBackend' - ) - kc_server = settings.KOBOCAT_URL - id_string = self.asset.uid - - url = self.normalize_internal_url('{}/api/v1/forms'.format(kc_server)) xlsx_io = self.asset.to_xlsx_io( versioned=True, append={ 'settings': { - 'id_string': id_string, + 'id_string': self.asset.uid, 'form_title': self.asset.name, } } ) + xlsx_file = ContentFile(xlsx_io.read(), name=f'{self.asset.uid}.xlsx') + + with kc_transaction_atomic(): + self._xform = publish_xls_form(xlsx_file, self.asset.owner) + self._xform.downloadable = active + self._xform.kpi_asset_uid = self.asset.uid + self._xform.save( + update_fields=['downloadable', 'kpi_asset_uid'] + ) - # Payload contains `kpi_asset_uid` and `has_kpi_hook` for two reasons: - # - KC `XForm`'s `id_string` can be different than `Asset`'s `uid`, then - # we can't rely on it to find its related `Asset`. - # - Removing, renaming `has_kpi_hook` will force PostgreSQL to rewrite - # every record of `logger_xform`. It can be also used to filter - # queries as it is faster to query a boolean than string. - payload = { - 'downloadable': active, - 'has_kpi_hook': self.asset.has_active_hooks, - 'kpi_asset_uid': self.asset.uid - } - files = {'xls_file': ('{}.xlsx'.format(id_string), xlsx_io)} - json_response = self._kobocat_request( - 'POST', url, data=payload, files=files - ) - # Store only path - json_response['url'] = urlparse(json_response['url']).path self.store_data( { - 'backend': 'kobocat', - 'active': json_response['downloadable'], - 'backend_response': json_response, + 'backend': 'openrosa', + 'active': active, + 'backend_response': { + 'formid': self._xform.pk, + 'uuid': self._xform.uuid, + 'id_string': self._xform.id_string, + 'kpi_asset_uid': self.asset.uid, + 'hash': self._xform.prefixed_hash, + }, 'version': self.asset.version_id, } ) @@ -182,62 +176,10 @@ def form_uuid(self): return self.backend_response['uuid'] except KeyError: logging.warning( - 'KoboCAT backend response has no `uuid`', exc_info=True + 'OpenRosa backend response has no `uuid`', exc_info=True ) return None - @staticmethod - def nlp_tracking_data(asset_ids, start_date=None): - """ - Get the NLP tracking data since a specified date - If no date is provided, get all-time data - """ - filter_args = {} - if start_date: - filter_args = {'date__gte': start_date} - try: - nlp_tracking = ( - NLPUsageCounter.objects.only('total_asr_seconds', 'total_mt_characters') - .filter( - asset_id__in=asset_ids, - **filter_args - ).aggregate( - total_nlp_asr_seconds=Coalesce(Sum('total_asr_seconds'), 0), - total_nlp_mt_characters=Coalesce(Sum('total_mt_characters'), 0), - ) - ) - except NLPUsageCounter.DoesNotExist: - return { - 'total_nlp_asr_seconds': 0, - 'total_nlp_mt_characters': 0, - } - else: - return nlp_tracking - - def submission_count_since_date(self, start_date=None): - try: - xform_id = self.xform_id - except InvalidXFormException: - return 0 - - today = timezone.now().date() - filter_args = { - 'xform_id': xform_id, - } - if start_date: - filter_args['date__range'] = [start_date, today] - try: - # Note: this is replicating the functionality that was formerly in - # `current_month_submission_count`. `current_month_submission_count` - # didn't account for partial permissions, and this doesn't either - total_submissions = KobocatDailyXFormSubmissionCounter.objects.only( - 'date', 'counter' - ).filter(**filter_args).aggregate(count_sum=Coalesce(Sum('counter'), 0)) - except KobocatDailyXFormSubmissionCounter.DoesNotExist: - return 0 - else: - return total_submissions['count_sum'] - @staticmethod def format_openrosa_datetime(dt: Optional[datetime] = None) -> str: """ @@ -257,27 +199,10 @@ def delete(self): """ WARNING! Deletes all submitted data! """ - url = self.normalize_internal_url(self.backend_response['url']) try: - self._kobocat_request('DELETE', url) - except KobocatDeploymentException as e: - if not hasattr(e, 'response'): - raise - - if e.response.status_code == status.HTTP_404_NOT_FOUND: - # The KC project is already gone! - pass - elif e.response.status_code in [ - status.HTTP_502_BAD_GATEWAY, - status.HTTP_504_GATEWAY_TIMEOUT, - ]: - raise KobocatCommunicationError - elif e.response.status_code == status.HTTP_401_UNAUTHORIZED: - raise KobocatCommunicationError( - 'Could not authenticate to KoBoCAT' - ) - else: - raise + self._xform.delete() + except XForm.DoesNotExist: + pass super().delete() @@ -285,7 +210,7 @@ def delete_submission( self, submission_id: int, user: settings.AUTH_USER_MODEL ) -> dict: """ - Delete a submission through KoBoCAT proxy + Delete a submission It returns a dictionary which can used as Response object arguments """ @@ -296,16 +221,18 @@ def delete_submission( submission_ids=[submission_id] ) - kc_url = self.get_submission_detail_url(submission_id) - kc_request = requests.Request(method='DELETE', url=kc_url) - kc_response = self.__kobocat_proxy_request(kc_request, user) + Instance.objects.filter(pk=submission_id).delete() - return self.__prepare_as_drf_response_signature(kc_response) + return { + 'content_type': 'application/json', + 'status': status.HTTP_204_NO_CONTENT, + } - def delete_submissions(self, data: dict, user: settings.AUTH_USER_MODEL) -> dict: + def delete_submissions( + self, data: dict, user: settings.AUTH_USER_MODEL, **kwargs + ) -> dict: """ - Bulk delete provided submissions through KoBoCAT proxy, - authenticated by `user`'s API token. + Bulk delete provided submissions. `data` should contain the submission ids or the query to get the subset of submissions to delete @@ -314,7 +241,6 @@ def delete_submissions(self, data: dict, user: settings.AUTH_USER_MODEL) -> dict or {"query": {"Question": "response"} """ - submission_ids = self.validate_access_with_partial_perms( user=user, perm=PERM_DELETE_SUBMISSIONS, @@ -330,15 +256,17 @@ def delete_submissions(self, data: dict, user: settings.AUTH_USER_MODEL) -> dict data.pop('query', None) data['submission_ids'] = submission_ids - kc_url = self.submission_list_url - kc_request = requests.Request(method='DELETE', url=kc_url, json=data) - kc_response = self.__kobocat_proxy_request(kc_request, user) + # TODO handle errors + deleted_count = delete_instances(self.xform, data) - drf_response = self.__prepare_as_drf_response_signature(kc_response) - return drf_response + return { + 'data': {'detail': f'{deleted_count} submissions have been deleted'}, + 'content_type': 'application/json', + 'status': status.HTTP_200_OK, + } def duplicate_submission( - self, submission_id: int, user: 'settings.AUTH_USER_MODEL' + self, submission_id: int, request: 'rest_framework.request.Request', ) -> dict: """ Duplicates a single submission proxied through KoBoCAT. The submission @@ -350,7 +278,7 @@ def duplicate_submission( submission if successful """ - + user = request.user self.validate_access_with_partial_perms( user=user, perm=PERM_CHANGE_SUBMISSIONS, @@ -364,14 +292,14 @@ def duplicate_submission( ) # Get attachments for the duplicated submission if there are any - attachment_objects = KobocatAttachment.objects.filter( + attachments = [] + if attachment_objects := Attachment.objects.filter( instance_id=submission_id - ) - attachments = ( - {a.media_file_basename: a.media_file for a in attachment_objects} - if attachment_objects - else None - ) + ): + attachments = ( + ExtendedContentFile(a.media_file.read(), name=a.media_file_basename) + for a in attachment_objects + ) # parse XML string to ET object xml_parsed = fromstring_preserve_root_xmlns(submission) @@ -393,18 +321,22 @@ def duplicate_submission( uuid_formatted ) - kc_response = self.store_submission( - user, xml_tostring(xml_parsed), _uuid, attachments + # TODO Handle errors returned by safe_create_instance + error, instance = safe_create_instance( + username=user.username, + xml_file=ContentFile(xml_tostring(xml_parsed)), + media_files=attachments, + uuid=_uuid, + request=request, + ) + return self._rewrite_json_attachment_urls( + next(self.get_submissions(user, submission_id=instance.pk)), request ) - if kc_response.status_code == status.HTTP_201_CREATED: - return next(self.get_submissions(user, query={'_uuid': _uuid})) - else: - raise KobocatDuplicateSubmissionException def edit_submission( self, xml_submission_file: File, - user: settings.AUTH_USER_MODEL, + request: 'rest_framework.request.Request', attachments: dict = None, ): """ @@ -413,6 +345,8 @@ def edit_submission( The returned Response should be in XML (expected format by Enketo Express) """ + user = request.user + submission_xml = xml_submission_file.read() try: xml_root = fromstring_preserve_root_xmlns(submission_xml) @@ -431,13 +365,14 @@ def edit_submission( ) # Remove UUID prefix deprecated_uuid = deprecated_uuid[len('uuid:'):] + try: - instance = ReadOnlyKobocatInstance.objects.get( + instance = Instance.objects.get( uuid=deprecated_uuid, xform__uuid=xform_uuid, xform__kpi_asset_uid=self.asset.uid, ) - except ReadOnlyKobocatInstance.DoesNotExist: + except Instance.DoesNotExist: raise SubmissionIntegrityError( t( 'The submission you attempted to edit could not be found, ' @@ -455,20 +390,28 @@ def edit_submission( # Set the In-Memory file’s current position to 0 before passing it to # Request. xml_submission_file.seek(0) - files = {'xml_submission_file': xml_submission_file} - - # Combine all files altogether - if attachments: - files.update(attachments) - kc_request = requests.Request( - method='POST', url=self.submission_url, files=files + # Retrieve only File objects to pass to `safe_create_instance` + # TODO remove those files as soon as the view sends request.FILES directly + # See TODO in kpi/views/v2/asset_snapshot.py::submission + media_files = ( + media_file for media_file in attachments.values() ) - kc_response = self.__kobocat_proxy_request(kc_request, user) - return self.__prepare_as_drf_response_signature( - kc_response, expected_response_format='xml' + + # TODO Handle errors returned by safe_create_instance + safe_create_instance( + username=user.username, + xml_file=xml_submission_file, + media_files=media_files, + request=request, ) + return { + 'headers': {}, + 'content_type': 'text/xml; charset=utf-8', + 'status': status.HTTP_201_CREATED, + } + @property def enketo_id(self): if not (enketo_id := self.get_data('enketo_id')): @@ -490,7 +433,7 @@ def get_attachment( user: settings.AUTH_USER_MODEL, attachment_id: Optional[int] = None, xpath: Optional[str] = None, - ) -> KobocatAttachment: + ) -> Attachment: """ Return an object which can be retrieved by its primary key or by XPath. An exception is raised when the submission or the attachment is not found. @@ -554,13 +497,15 @@ def get_attachment( filters['instance__xform_id'] = self.xform_id try: - attachment = KobocatAttachment.objects.get(**filters) - except KobocatAttachment.DoesNotExist: + attachment = Attachment.objects.get(**filters) + except Attachment.DoesNotExist: raise AttachmentNotFoundException return attachment - def get_attachment_objects_from_dict(self, submission: dict) -> QuerySet: + def get_attachment_objects_from_dict( + self, submission: dict + ) -> Union[QuerySet, list]: # First test that there are attachments to avoid a call to the DB for # nothing @@ -575,11 +520,7 @@ def get_attachment_objects_from_dict(self, submission: dict) -> QuerySet: # - XML filename: Screenshot 2022-01-19 222028-13_45_57.jpg # - Mongo: Screenshot_2022-01-19_222028-13_45_57.jpg - # ToDo What about adding the original basename and the question - # name in Mongo to avoid another DB query? - return KobocatAttachment.objects.filter( - instance_id=submission['_id'] - ) + return Attachment.objects.filter(instance_id=submission['_id']) def get_daily_counts( self, user: settings.AUTH_USER_MODEL, timeframe: tuple[date, date] @@ -637,7 +578,7 @@ def get_daily_counts( # Trivial case, user has 'view_permissions' daily_counts = ( - KobocatDailyXFormSubmissionCounter.objects.values( + DailyXFormSubmissionCounter.objects.values( 'date', 'counter' ).filter( xform_id=self.xform_id, @@ -653,13 +594,13 @@ def get_data_download_links(self): settings.KOBOCAT_URL.rstrip('/'), self.asset.owner.username, 'exports', - self.backend_response['id_string'] + self.xform.id_string )) reports_base_url = '/'.join(( settings.KOBOCAT_URL.rstrip('/'), self.asset.owner.username, 'reports', - self.backend_response['id_string'] + self.xform.id_string )) links = { # To be displayed in iframes @@ -682,7 +623,7 @@ def get_enketo_survey_links(self): settings.KOBOCAT_URL.rstrip('/'), self.asset.owner.username ), - 'form_id': self.backend_response['id_string'] + 'form_id': self.xform.id_string } try: @@ -758,7 +699,7 @@ def get_orphan_postgres_submissions(self) -> Optional[QuerySet, bool]: return False try: - return ReadOnlyKobocatInstance.objects.filter(xform_id=self.xform_id) + return Instance.objects.filter(xform_id=self.xform_id) except InvalidXFormException: return None @@ -776,7 +717,7 @@ def get_submissions( self, user: settings.AUTH_USER_MODEL, format_type: str = SUBMISSION_FORMAT_TYPE_JSON, - submission_ids: list = [], + submission_ids: list = None, request: Optional['rest_framework.request.Request'] = None, **mongo_query_params ) -> Union[Generator[dict, None, None], list]: @@ -801,7 +742,9 @@ def get_submissions( See `BaseDeploymentBackend._rewrite_json_attachment_urls()` """ - mongo_query_params['submission_ids'] = submission_ids + mongo_query_params['submission_ids'] = ( + submission_ids if submission_ids else [] + ) params = self.validate_submission_list_params( user, format_type=format_type, **mongo_query_params ) @@ -819,39 +762,71 @@ def get_submissions( def get_validation_status( self, submission_id: int, user: settings.AUTH_USER_MODEL ) -> dict: - url = self.get_submission_validation_status_url(submission_id) - kc_request = requests.Request(method='GET', url=url) - kc_response = self.__kobocat_proxy_request(kc_request, user) + submission = self.get_submission( + submission_id, user, fields=['_validation_status'] + ) - return self.__prepare_as_drf_response_signature(kc_response) + # TODO simplify response when KobocatDeploymentBackend + # and MockDeploymentBackend are gone + if not submission: + return { + 'content_type': 'application/json', + 'status': status.HTTP_404_NOT_FOUND, + 'data': { + 'detail': f'No submission found with ID: {submission_id}' + } + } - @staticmethod - def internal_to_external_url(url): - """ - Replace the value of `settings.KOBOCAT_INTERNAL_URL` with that of - `settings.KOBOCAT_URL` when it appears at the beginning of - `url` - """ - return re.sub( - pattern='^{}'.format(re.escape(settings.KOBOCAT_INTERNAL_URL)), - repl=settings.KOBOCAT_URL, - string=url - ) + return { + 'data': submission['_validation_status'], + 'content_type': 'application/json', + 'status': status.HTTP_200_OK, + } @property def mongo_userform_id(self): return '{}_{}'.format(self.asset.owner.username, self.xform_id_string) + @staticmethod + def nlp_tracking_data( + asset_ids: list[int], start_date: Optional[datetime.date] = None + ): + """ + Get the NLP tracking data since a specified date + If no date is provided, get all-time data + """ + filter_args = {} + if start_date: + filter_args = {'date__gte': start_date} + try: + nlp_tracking = ( + NLPUsageCounter.objects.only('total_asr_seconds', 'total_mt_characters') + .filter( + asset_id__in=asset_ids, + **filter_args + ).aggregate( + total_nlp_asr_seconds=Coalesce(Sum('total_asr_seconds'), 0), + total_nlp_mt_characters=Coalesce(Sum('total_mt_characters'), 0), + ) + ) + except NLPUsageCounter.DoesNotExist: + return { + 'total_nlp_asr_seconds': 0, + 'total_nlp_mt_characters': 0, + } + else: + return nlp_tracking + def redeploy(self, active=None): """ - Replace (overwrite) the deployment, keeping the same identifier, and + Replace (overwrite) the deployment, and optionally changing whether the deployment is active. CAUTION: Does not save deployment data to the database! """ if active is None: active = self.active - url = self.normalize_internal_url(self.backend_response['url']) - id_string = self.backend_response['id_string'] + + id_string = self.xform.id_string xlsx_io = self.asset.to_xlsx_io( versioned=True, append={ 'settings': { @@ -860,22 +835,34 @@ def redeploy(self, active=None): } } ) - payload = { - 'downloadable': active, - 'title': self.asset.name, - 'has_kpi_hook': self.asset.has_active_hooks - } - files = {'xls_file': ('{}.xlsx'.format(id_string), xlsx_io)} - json_response = self._kobocat_request( - 'PATCH', url, data=payload, files=files - ) - self.store_data({ - 'active': json_response['downloadable'], - 'backend_response': json_response, - 'version': self.asset.version_id, - }) + xlsx_file = ContentFile(xlsx_io.read(), name=f'{self.asset.uid}.xlsx') - self.set_asset_uid() + with kc_transaction_atomic(): + XForm.objects.filter(pk=self.xform.id).update( + downloadable=active, + title=self.asset.name, + ) + self.xform.downloadable = active + self.xform.title = self.asset.name + + publish_xls_form(xlsx_file, self.asset.owner, self.xform.id_string) + + # Do not call `save_to_db()`, asset (and its deployment) is saved right + # after calling this method in `DeployableMixin.deploy()` + self.store_data( + { + 'backend': 'openrosa', + 'active': active, + 'backend_response': { + 'formid': self.xform.pk, + 'uuid': self.xform.uuid, + 'id_string': self.xform.id_string, + 'kpi_asset_uid': self.asset.uid, + 'hash': self._xform.prefixed_hash, + }, + 'version': self.asset.version_id, + } + ) def remove_from_kc_only_flag( self, specific_user: Union[int, settings.AUTH_USER_MODEL] = None @@ -925,25 +912,62 @@ def rename_enketo_id_key(self, previous_owner_username: str): # original does not exist, weird but don't raise a 500 for that pass - def set_active(self, active): + @staticmethod + def prepare_bulk_update_response(backend_responses: list) -> dict: """ - `PATCH` active boolean of the survey. - Store results in deployment data + Formatting the response to allow for partial successes to be seen + more explicitly. """ - # self.store_data is an alias for - # self.asset._deployment_data.update(...) - url = self.normalize_internal_url( - self.backend_response['url']) - payload = { - 'downloadable': bool(active) + + results = [] + cpt_successes = 0 + for backend_response in backend_responses: + uuid = backend_response['uuid'] + error, instance = backend_response['response'] + + message = t('Something went wrong') + status_code = status.HTTP_400_BAD_REQUEST + if not error: + cpt_successes += 1 + message = t('Successful submission') + status_code = status.HTTP_201_CREATED + + results.append( + { + 'uuid': uuid, + 'status_code': status_code, + 'message': message, + } + ) + + total_update_attempts = len(results) + total_successes = cpt_successes + + return { + 'status': ( + status.HTTP_200_OK + if total_successes > 0 + else status.HTTP_400_BAD_REQUEST + ), + 'data': { + 'count': total_update_attempts, + 'successes': total_successes, + 'failures': total_update_attempts - total_successes, + 'results': results, + }, } - json_response = self._kobocat_request('PATCH', url, data=payload) - assert json_response['downloadable'] == bool(active) - self.save_to_db({ - 'active': json_response['downloadable'], - 'backend_response': json_response, - }) + def set_active(self, active): + """ + Set deployment as active or not. + Store results in deployment data + """ + # Use `queryset.update()` over `model.save()` because we don't need to + # run the logic of the `model.save()` method and we don't need signals + # to be called. + XForm.objects.filter(pk=self.xform_id).update(downloadable=active) + self.xform.downloadable = active + self.save_to_db({'active': active}) def set_asset_uid(self, force: bool = False) -> bool: """ @@ -962,16 +986,15 @@ def set_asset_uid(self, force: bool = False) -> bool: if is_synchronized: return False - url = self.normalize_internal_url(self.backend_response['url']) - payload = { - 'kpi_asset_uid': self.asset.uid - } - json_response = self._kobocat_request('PATCH', url, data=payload) - is_set = json_response['kpi_asset_uid'] == self.asset.uid - assert is_set - self.store_data({ - 'backend_response': json_response, - }) + # Use `queryset.update()` over `model.save()` because we don't need to + # run the logic of the `model.save()` method and we don't need signals + # to be called. + XForm.objects.filter(pk=self.xform_id).update( + kpi_asset_uid=self.asset.uid + ) + self.xform.kpi_asset_uid = self.asset.uid + self.backend_response['kpi_asset_uid'] = self.asset.uid + self.store_data({'backend_response': self.backend_response}) return True def set_enketo_open_rosa_server( @@ -998,51 +1021,15 @@ def set_enketo_open_rosa_server( server_url, ) - def set_has_kpi_hooks(self): - """ - `PATCH` `has_kpi_hooks` boolean of related KoBoCAT XForm. - It lets KoBoCAT know whether it needs to notify KPI - each time a submission comes in. - - Store results in deployment data - """ - has_active_hooks = self.asset.has_active_hooks - url = self.normalize_internal_url( - self.backend_response['url']) - payload = { - 'has_kpi_hooks': has_active_hooks, - 'kpi_asset_uid': self.asset.uid - } - - try: - json_response = self._kobocat_request('PATCH', url, data=payload) - except KobocatDeploymentException as e: - if ( - has_active_hooks is False - and hasattr(e, 'response') - and e.response.status_code == status.HTTP_404_NOT_FOUND - ): - # It's okay if we're trying to unset the active hooks flag and - # the KoBoCAT project is already gone. See #2497 - pass - else: - raise - else: - assert json_response['has_kpi_hooks'] == has_active_hooks - self.store_data({ - 'backend_response': json_response, - }) - def set_validation_status( self, submission_id: int, user: settings.AUTH_USER_MODEL, data: dict, - method: str, + method: str = Literal['DELETE', 'PATCH'], ) -> dict: """ - Update validation status through KoBoCAT proxy, - authenticated by `user`'s API token. + Update validation status. If `method` is `DELETE`, the status is reset to `None` It returns a dictionary which can used as Response object arguments @@ -1054,26 +1041,61 @@ def set_validation_status( submission_ids=[submission_id], ) - kc_request_params = { - 'method': method, - 'url': self.get_submission_validation_status_url(submission_id), - } + # TODO simplify response when KobocatDeploymentBackend + # and MockDeploymentBackend are gone + try: + instance = Instance.objects.only( + 'validation_status', 'date_modified' + ).get(pk=submission_id) + except Instance.DoesNotExist: + return { + 'content_type': 'application/json', + 'status': status.HTTP_404_NOT_FOUND, + 'data': { + 'detail': f'No submission found with ID: {submission_id}' + } + } + + if method == 'DELETE': + if remove_validation_status_from_instance(instance): + return { + 'content_type': 'application/json', + 'status': status.HTTP_204_NO_CONTENT, + } + else: + return { + 'content_type': 'application/json', + 'status': status.HTTP_500_INTERNAL_SERVER_ERROR, + 'data': { + 'detail': 'Could not update MongoDB' + } + } - if method == 'PATCH': - kc_request_params.update({'json': data}) + validation_status_uid = data.get('validation_status.uid') - kc_request = requests.Request(**kc_request_params) - kc_response = self.__kobocat_proxy_request(kc_request, user) - return self.__prepare_as_drf_response_signature(kc_response) + if not add_validation_status_to_instance( + user.username, validation_status_uid, instance + ): + return { + 'content_type': 'application/json', + 'status': status.HTTP_400_BAD_REQUEST, + 'data': { + 'detail': f'Invalid validation status: `{validation_status_uid}`' + } + } + return { + 'data': instance.validation_status, + 'content_type': 'application/json', + 'status': status.HTTP_200_OK, + } def set_validation_statuses( self, user: settings.AUTH_USER_MODEL, data: dict ) -> dict: """ - Bulk update validation status for provided submissions through - KoBoCAT proxy, authenticated by `user`'s API token. + Bulk update validation status. - `data` should contains either the submission ids or the query to + `data` should contain either the submission ids or the query to retrieve the subset of submissions chosen by then user. If none of them are provided, all the submissions are selected Examples: @@ -1088,31 +1110,40 @@ def set_validation_statuses( ) # If `submission_ids` is not empty, user has partial permissions. - # Otherwise, they have have full access. + # Otherwise, they have full access. if submission_ids: # Remove query from `data` because all the submission ids have been # already retrieved data.pop('query', None) data['submission_ids'] = submission_ids - # `PATCH` KC even if KPI receives `DELETE` - url = self.submission_list_url - kc_request = requests.Request(method='PATCH', url=url, json=data) - kc_response = self.__kobocat_proxy_request(kc_request, user) - return self.__prepare_as_drf_response_signature(kc_response) + # TODO handle errors + update_instances = set_instance_validation_statuses( + self.xform, data, user + ) + + return { + 'data': {'detail': f'{update_instances} submissions have been updated'}, + 'content_type': 'application/json', + 'status': status.HTTP_200_OK, + } def store_submission( - self, user, xml_submission, submission_uuid, attachments=None + self, user, xml_submission, submission_uuid, attachments=None, **kwargs ): - file_tuple = (submission_uuid, io.StringIO(xml_submission)) - files = {'xml_submission_file': file_tuple} + media_files = [] if attachments: - files.update(attachments) - kc_request = requests.Request( - method='POST', url=self.submission_url, files=files + media_files = ( + media_file for media_file in attachments.values() + ) + + return safe_create_instance( + username=user.username, + xml_file=ContentFile(xml_submission), + media_files=media_files, + uuid=submission_uuid, + request=kwargs.get('request'), ) - kc_response = self.__kobocat_proxy_request(kc_request, user=user) - return kc_response @property def submission_count(self): @@ -1121,6 +1152,30 @@ def submission_count(self): except InvalidXFormException: return 0 + def submission_count_since_date(self, start_date=None): + try: + xform_id = self.xform_id + except InvalidXFormException: + return 0 + + today = timezone.now().date() + filter_args = { + 'xform_id': xform_id, + } + if start_date: + filter_args['date__range'] = [start_date, today] + try: + # Note: this is replicating the functionality that was formerly in + # `current_month_submission_count`. `current_month_submission_count` + # didn't account for partial permissions, and this doesn't either + total_submissions = DailyXFormSubmissionCounter.objects.only( + 'date', 'counter' + ).filter(**filter_args).aggregate(count_sum=Coalesce(Sum('counter'), 0)) + except DailyXFormSubmissionCounter.DoesNotExist: + return 0 + else: + return total_submissions['count_sum'] + @property def submission_list_url(self): url = '{kc_base}/api/v1/data/{formid}'.format( @@ -1131,11 +1186,11 @@ def submission_list_url(self): @property def submission_model(self): - return ReadOnlyKobocatInstance + return Instance @property def submission_url(self) -> str: - # Use internal host to secure calls to KoBoCAT API, + # Use internal host to secure calls to KoboCAT API, # kobo-service-account can restrict requests per hosts. url = '{kc_base}/submission'.format( kc_base=settings.KOBOCAT_INTERNAL_URL, @@ -1144,21 +1199,20 @@ def submission_url(self) -> str: def sync_media_files(self, file_type: str = AssetFile.FORM_MEDIA): - url = self.normalize_internal_url(self.backend_response['url']) - response = self._kobocat_request('GET', url) - kc_files = defaultdict(dict) - - # Build a list of KoBoCAT metadata to compare with KPI - for metadata in response.get('metadata', []): - if metadata['data_type'] == self.SYNCED_DATA_FILE_TYPES[file_type]: - kc_files[metadata['data_value']] = { - 'pk': metadata['id'], - 'url': metadata['url'], - 'md5': metadata['file_hash'], - 'from_kpi': metadata['from_kpi'], - } + metadata_files = defaultdict(dict) + + # Build a list of KoboCAT metadata to compare with KPI + for metadata in MetaData.objects.filter( + xform_id=self.xform_id, + data_type=self.SYNCED_DATA_FILE_TYPES[file_type], + ).values(): + metadata_files[metadata['data_value']] = { + 'pk': metadata['id'], + 'md5': metadata['file_hash'], + 'from_kpi': metadata['from_kpi'], + } - kc_filenames = kc_files.keys() + metadata_filenames = metadata_files.keys() queryset = self._get_metadata_queryset(file_type=file_type) @@ -1167,36 +1221,36 @@ def sync_media_files(self, file_type: str = AssetFile.FORM_MEDIA): backend_media_id = media_file.backend_media_id # File does not exist in KC - if backend_media_id not in kc_filenames: + if backend_media_id not in metadata_filenames: if media_file.deleted_at is None: # New file - self.__save_kc_metadata(media_file) + self._save_openrosa_metadata(media_file) else: # Orphan, delete it media_file.delete(force=True) continue # Existing file - if backend_media_id in kc_filenames: - kc_file = kc_files[backend_media_id] + if backend_media_id in metadata_filenames: + metadata_file = metadata_files[backend_media_id] if media_file.deleted_at is None: # If md5 differs, we need to re-upload it. - if media_file.md5_hash != kc_file['md5']: + if media_file.md5_hash != metadata_file['md5']: if media_file.file_type == AssetFile.PAIRED_DATA: - self.__update_kc_metadata_hash( - media_file, kc_file['pk'] + self._update_kc_metadata_hash( + media_file, metadata_file['pk'] ) else: - self.__delete_kc_metadata(kc_file) - self.__save_kc_metadata(media_file) - elif kc_file['from_kpi']: - self.__delete_kc_metadata(kc_file, media_file) + self._delete_openrosa_metadata(metadata_file) + self._save_openrosa_metadata(media_file) + elif metadata_file['from_kpi']: + self._delete_openrosa_metadata(metadata_file, media_file) else: # Remote file has been uploaded directly to KC. We # cannot delete it, but we need to vacuum KPI. media_file.delete(force=True) # Skip deletion of key corresponding to `backend_media_id` - # in `kc_files` to avoid unique constraint failure in case + # in `metadata_files` to avoid unique constraint failure in case # user deleted # and re-uploaded the same file in a row between # two deployments @@ -1210,44 +1264,48 @@ def sync_media_files(self, file_type: str = AssetFile.FORM_MEDIA): # already exists in KC db. continue - # Remove current filename from `kc_files`. + # Remove current filename from `metadata_files`. # All files which will remain in this dict (after this loop) # will be considered obsolete and will be deleted - del kc_files[backend_media_id] + del metadata_files[backend_media_id] - # Remove KC orphan files previously uploaded through KPI - for kc_file in kc_files.values(): - if kc_file['from_kpi']: - self.__delete_kc_metadata(kc_file) + # Remove KoboCAT orphan files previously uploaded through KPI + for metadata_file in metadata_files.values(): + if metadata_file['from_kpi']: + self._delete_openrosa_metadata(metadata_file) @property def xform(self): - if not hasattr(self, '_xform'): - pk = self.backend_response['formid'] - xform = ( - KobocatXForm.objects.filter(pk=pk) - .only( - 'user__username', - 'id_string', - 'num_of_submissions', - 'attachment_storage_bytes', - 'require_auth', - ) - .select_related( - 'user' - ) # Avoid extra query to validate username below - .first() - ) - if not ( - xform - and xform.user.username == self.asset.owner.username - and xform.id_string == self.xform_id_string - ): - raise InvalidXFormException( - 'Deployment links to an unexpected KoBoCAT XForm') - setattr(self, '_xform', xform) + if self._xform is not None: + return self._xform + + pk = self.backend_response['formid'] + xform = ( + XForm.objects.filter(pk=pk) + .only( + 'user__username', + 'id_string', + 'num_of_submissions', + 'attachment_storage_bytes', + 'require_auth', + 'uuid', + ) + .select_related( + 'user' + ) # Avoid extra query to validate username below + .first() + ) + if not ( + xform + and xform.user.username == self.asset.owner.username + and xform.id_string == self.xform_id_string + ): + raise InvalidXFormException( + 'Deployment links to an unexpected KoboCAT XForm' + ) + self._xform = xform return self._xform @property @@ -1258,17 +1316,10 @@ def xform_id(self): def xform_id_string(self): return self.get_data('backend_response.id_string') - @property - def timestamp(self): - try: - return self.backend_response['date_modified'] - except KeyError: - return None - @staticmethod @contextmanager def suspend_submissions(user_ids: list[int]): - KobocatUserProfile.objects.filter( + UserProfile.objects.filter( user_id__in=user_ids ).update( metadata=UpdateJSONFieldAttributes( @@ -1279,7 +1330,7 @@ def suspend_submissions(user_ids: list[int]): try: yield finally: - KobocatUserProfile.objects.filter( + UserProfile.objects.filter( user_id__in=user_ids ).update( metadata=UpdateJSONFieldAttributes( @@ -1314,120 +1365,82 @@ def transfer_counters_ownership(self, new_owner: 'kobo_auth.User'): NLPUsageCounter.objects.filter( asset=self.asset, user=self.asset.owner ).update(user=new_owner) - KobocatDailyXFormSubmissionCounter.objects.filter( + DailyXFormSubmissionCounter.objects.filter( xform=self.xform, user_id=self.asset.owner.pk ).update(user=new_owner) - KobocatMonthlyXFormSubmissionCounter.objects.filter( + MonthlyXFormSubmissionCounter.objects.filter( xform=self.xform, user_id=self.asset.owner.pk ).update(user=new_owner) - KobocatUserProfile.objects.filter(user_id=self.asset.owner.pk).update( + UserProfile.objects.filter(user_id=self.asset.owner.pk).update( attachment_storage_bytes=F('attachment_storage_bytes') - - self.xform.attachment_storage_bytes + - self.xform.attachment_storage_bytes ) - KobocatUserProfile.objects.filter(user_id=self.asset.owner.pk).update( + UserProfile.objects.filter(user_id=self.asset.owner.pk).update( attachment_storage_bytes=F('attachment_storage_bytes') - + self.xform.attachment_storage_bytes + + self.xform.attachment_storage_bytes ) - def _kobocat_request(self, method, url, expect_formid=True, **kwargs): + def _delete_openrosa_metadata( + self, metadata_file_: dict, file_: Union[AssetFile, PairedData] = None + ): """ - Make a POST or PATCH request and return parsed JSON. Keyword arguments, - e.g. `data` and `files`, are passed through to `requests.request()`. - - If `expect_formid` is False, it bypasses the presence of 'formid' - property in KoBoCAT response and returns the KoBoCAT response whatever - it is. - - `kwargs` contains arguments to be passed to KoBoCAT request. + A simple utility to delete metadata in KoBoCAT. + If related KPI file is provided (i.e. `file_`), it is deleted too. """ - - expected_status_codes = { - 'GET': 200, - 'POST': 201, - 'PATCH': 200, - 'DELETE': 204, - } - + # Delete MetaData object and its related file (on storage) try: - expected_status_code = expected_status_codes[method] - except KeyError: - raise NotImplementedError( - 'This backend does not implement the {} method'.format(method) - ) + metadata = MetaData.objects.get(pk=metadata_file_['id']) + except MetaData.DoesNotExist: + pass + else: + # Need to call signals + metadata.delete() - # Make the request to KC - try: - kc_request = requests.Request(method=method, url=url, **kwargs) - response = self.__kobocat_proxy_request(kc_request, - user=self.asset.owner) + if file_ is None: + return - except requests.exceptions.RequestException as e: - # Failed to access the KC API - # TODO: clarify that the user cannot correct this - raise KobocatDeploymentException(detail=str(e)) + # Delete file in KPI if requested + file_.delete(force=True) - # If it's a no-content success, return immediately - if response.status_code == expected_status_code == 204: - return {} + def _last_submission_time(self): + return self.xform.last_submission_time - # Parse the response - try: - json_response = response.json() - except ValueError as e: - # Unparseable KC API output - # TODO: clarify that the user cannot correct this - raise KobocatDeploymentException( - detail=str(e), response=response) - - # Check for failure - if ( - response.status_code != expected_status_code - or json_response.get('type') == 'alert-error' - or (expect_formid and 'formid' not in json_response) - ): - if 'text' in json_response: - # KC API refused us for a specified reason, likely invalid - # input Raise a 400 error that includes the reason - e = KobocatDeploymentException(detail=json_response['text']) - e.status_code = status.HTTP_400_BAD_REQUEST - raise e - else: - # Unspecified failure; raise 500 - raise KobocatDeploymentException( - detail='Unexpected KoBoCAT error {}: {}'.format( - response.status_code, response.content), - response=response - ) + def _save_openrosa_metadata(self, file_: SyncBackendMediaInterface): + """ + Create a MetaData object usable for (KoboCAT) v1 API, related to + AssetFile `file_`. + """ + metadata = { + 'data_value': file_.backend_media_id, + 'xform_id': self.xform_id, + 'data_type': self.SYNCED_DATA_FILE_TYPES[file_.file_type], + 'from_kpi': True, + 'data_filename': file_.filename, + 'data_file_type': file_.mimetype, + 'file_hash': file_.md5_hash, + } - return json_response + if not file_.is_remote_url: + metadata['data_file'] = file_.content - def _last_submission_time(self): - id_string = self.backend_response['id_string'] - return last_submission_time( - xform_id_string=id_string, user_id=self.asset.owner.pk) + MetaData.objects.create(**metadata) - @property - def _open_rosa_server_storage(self): - return default_kobocat_storage + file_.synced_with_backend = True + file_.save(update_fields=['synced_with_backend']) - def __delete_kc_metadata( - self, kc_file_: dict, file_: Union[AssetFile, PairedData] = None + def _update_kc_metadata_hash( + self, file_: SyncBackendMediaInterface, metadata_id: int ): """ - A simple utility to delete metadata in KoBoCAT through proxy. - If related KPI file is provided (i.e. `file_`), it is deleted too. + Update metadata object hash """ - # Delete file in KC - - delete_url = self.normalize_internal_url(kc_file_['url']) - self._kobocat_request('DELETE', url=delete_url, expect_formid=False) - - if file_ is None: - return - - # Delete file in KPI if requested - file_.delete(force=True) + data = {'file_hash': file_.md5_hash} + # MetaData has no signals, use `filter().update()` instead of `.get()` + # and `.save(update_fields='...')` + MetaData.objects.filter(pk=metadata_id).update(**data) + file_.synced_with_backend = True + file_.save(update_fields=['synced_with_backend']) def __get_submissions_in_json( self, @@ -1493,9 +1506,7 @@ def __get_submissions_in_xml( ] self.current_submission_count = count - queryset = ReadOnlyKobocatInstance.objects.filter( - xform_id=self.xform_id, - ) + queryset = Instance.objects.filter(xform_id=self.xform_id) if len(submission_ids) > 0 or use_mongo: queryset = queryset.filter(id__in=submission_ids) @@ -1516,174 +1527,3 @@ def __get_submissions_in_xml( queryset = queryset[offset:limit] return (lazy_instance.xml for lazy_instance in queryset) - - @staticmethod - def __kobocat_proxy_request(kc_request, user=None): - """ - Send `kc_request`, which must specify `method` and `url` at a minimum. - If the incoming request to be proxied is authenticated, - logged-in user's API token will be added to `kc_request.headers` - - :param kc_request: requests.models.Request - :param user: User - :return: requests.models.Response - """ - if not is_user_anonymous(user): - kc_request.headers.update(get_request_headers(user.username)) - - session = requests.Session() - return session.send(kc_request.prepare()) - - @staticmethod - def __prepare_as_drf_response_signature( - requests_response, expected_response_format='json' - ): - """ - Prepares a dict from `Requests` response. - Useful to get response from KoBoCAT and use it as a dict or pass it to - DRF Response - """ - - prepared_drf_response = {} - - # `requests_response` may not have `headers` attribute - content_type = requests_response.headers.get('Content-Type') - content_language = requests_response.headers.get('Content-Language') - if content_type: - prepared_drf_response['content_type'] = content_type - if content_language: - prepared_drf_response['headers'] = { - 'Content-Language': content_language - } - - prepared_drf_response['status'] = requests_response.status_code - - try: - prepared_drf_response['data'] = json.loads( - requests_response.content) - except ValueError as e: - if ( - not requests_response.status_code == status.HTTP_204_NO_CONTENT - and expected_response_format == 'json' - ): - prepared_drf_response['data'] = { - 'detail': t( - 'KoBoCAT returned an unexpected response: {}'.format( - str(e)) - ) - } - - return prepared_drf_response - - @staticmethod - def prepare_bulk_update_response(kc_responses: list) -> dict: - """ - Formatting the response to allow for partial successes to be seen - more explicitly. - - Args: - kc_responses (list): A list containing dictionaries with keys of - `_uuid` from the newly generated uuid and `response`, the response - object received from KoBoCAT - - Returns: - dict: formatted dict to be passed to a Response object and sent to - the client - """ - - OPEN_ROSA_XML_MESSAGE = '{http://openrosa.org/http/response}message' - - # Unfortunately, the response message from OpenRosa is in XML format, - # so it needs to be parsed before extracting the text - results = [] - for response in kc_responses: - message = t('Something went wrong') - try: - xml_parsed = fromstring_preserve_root_xmlns( - response['response'].content - ) - except DET.ParseError: - pass - else: - message_el = xml_parsed.find(OPEN_ROSA_XML_MESSAGE) - if message_el is not None and message_el.text.strip(): - message = message_el.text - - results.append( - { - 'uuid': response['uuid'], - 'status_code': response['response'].status_code, - 'message': message, - } - ) - - total_update_attempts = len(results) - total_successes = [result['status_code'] for result in results].count( - status.HTTP_201_CREATED - ) - - return { - 'status': status.HTTP_200_OK - if total_successes > 0 - # FIXME: If KoboCAT returns something unexpected, like a 404 or a - # 500, then 400 is not the right response to send to the client - else status.HTTP_400_BAD_REQUEST, - 'data': { - 'count': total_update_attempts, - 'successes': total_successes, - 'failures': total_update_attempts - total_successes, - 'results': results, - }, - } - - def __save_kc_metadata(self, file_: SyncBackendMediaInterface): - """ - Prepares request and data corresponding to the kind of media file - (i.e. FileStorage or remote URL) to `POST` to KC through proxy. - """ - server = settings.KOBOCAT_INTERNAL_URL - metadata_url = f'{server}/api/v1/metadata' - - kwargs = { - 'data': { - 'data_value': file_.backend_media_id, - 'xform': self.xform_id, - 'data_type': self.SYNCED_DATA_FILE_TYPES[file_.file_type], - 'from_kpi': True, - 'data_filename': file_.filename, - 'data_file_type': file_.mimetype, - 'file_hash': file_.md5_hash, - } - } - - if not file_.is_remote_url: - kwargs['files'] = { - 'data_file': ( - file_.filename, - file_.content.file, - file_.mimetype, - ) - } - - self._kobocat_request( - 'POST', url=metadata_url, expect_formid=False, **kwargs - ) - - file_.synced_with_backend = True - file_.save(update_fields=['synced_with_backend']) - - def __update_kc_metadata_hash( - self, file_: SyncBackendMediaInterface, kc_metadata_id: int - ): - """ - Update metadata hash in KC - """ - server = settings.KOBOCAT_INTERNAL_URL - metadata_detail_url = f'{server}/api/v1/metadata/{kc_metadata_id}' - data = {'file_hash': file_.md5_hash} - self._kobocat_request( - 'PATCH', url=metadata_detail_url, expect_formid=False, data=data - ) - - file_.synced_with_backend = True - file_.save(update_fields=['synced_with_backend']) diff --git a/kpi/management/commands/copy_kc_profile.py b/kpi/management/commands/copy_kc_profile.py index 38e5750f5a..55c169478d 100644 --- a/kpi/management/commands/copy_kc_profile.py +++ b/kpi/management/commands/copy_kc_profile.py @@ -3,7 +3,7 @@ from hub.models import ExtraUserDetail from kobo.apps.kobo_auth.shortcuts import User -from kpi.deployment_backends.kc_access.utils import get_kc_profile_data +from kobo.apps.openrosa.apps.main import UserProfile class Command(BaseCommand): @@ -51,7 +51,8 @@ def handle(self, *args, **options): user=user) if not extra_details.data.get('copied_kc_profile', False) or \ options.get('again'): - kc_detail = get_kc_profile_data(user.pk) + + kc_detail = UserProfile.to_dict(user_id=user.pk) for k, v in kc_detail.items(): if extra_details.data.get(k, None) is None: extra_details.data[k] = v diff --git a/kpi/management/commands/sync_kobocat_perms.py b/kpi/management/commands/sync_kobocat_perms.py index 03efc0d869..5e7e1b1396 100644 --- a/kpi/management/commands/sync_kobocat_perms.py +++ b/kpi/management/commands/sync_kobocat_perms.py @@ -3,6 +3,7 @@ from django.contrib.postgres.aggregates import ArrayAgg from django.core.exceptions import ImproperlyConfigured from django.core.management.base import BaseCommand +from guardian.models import UserObjectPermission from kpi.constants import PERM_FROM_KC_ONLY from kpi.models import Asset, ObjectPermission @@ -10,9 +11,6 @@ assign_applicable_kc_permissions, kc_transaction_atomic, ) -from kpi.deployment_backends.kc_access.shadow_models import ( - KobocatUserObjectPermission -) from kpi.management.commands.sync_kobocat_xforms import _sync_permissions from kpi.utils.object_permission import get_perm_ids_from_code_names @@ -94,9 +92,11 @@ def _sync_perms(self, **options): with kc_transaction_atomic(): kc_user_obj_perm_qs = ( - KobocatUserObjectPermission.objects.filter( - object_pk=asset.deployment.xform_id - ).exclude(user_id=asset.owner_id) + UserObjectPermission.objects.using( + settings.OPENROSA_DB_ALIAS + ) + .filter(object_pk=asset.deployment.xform_id) + .exclude(user_id=asset.owner_id) ) if kc_user_obj_perm_qs.exists(): if self._verbosity >= 1: diff --git a/kpi/management/commands/sync_kobocat_xforms.py b/kpi/management/commands/sync_kobocat_xforms.py index f38e79f683..32e5fd3214 100644 --- a/kpi/management/commands/sync_kobocat_xforms.py +++ b/kpi/management/commands/sync_kobocat_xforms.py @@ -14,6 +14,7 @@ from django.core.management import call_command from django.core.management.base import BaseCommand from django.db import transaction +from guardian.models import UserObjectPermission from formpack.utils.xls_to_ss_structure import xlsx_to_dicts from pyxform import xls2json_backends from rest_framework.authtoken.models import Token @@ -21,12 +22,8 @@ from kobo.apps.kobo_auth.shortcuts import User from kpi.constants import PERM_FROM_KC_ONLY from kpi.utils.log import logging -from kpi.deployment_backends.kc_access.shadow_models import ( - KobocatPermission, - KobocatUserObjectPermission, - KobocatXForm, -) -from kpi.deployment_backends.kobocat_backend import KobocatDeploymentBackend +from kobo.apps.openrosa.apps.logger.models.xform import XForm +from kpi.deployment_backends.openrosa_backend import OpenRosaDeploymentBackend from kpi.models import Asset, ObjectPermission from kpi.utils.object_permission import get_anonymous_user from kpi.utils.models import _set_auto_field_update @@ -40,17 +37,24 @@ ASSET_CT = ContentType.objects.get_for_model(Asset) FROM_KC_ONLY_PERMISSION = Permission.objects.get( content_type=ASSET_CT, codename=PERM_FROM_KC_ONLY) -XFORM_CT = KobocatXForm.get_content_type() +XFORM_CT = ContentType.objects.using(settings.OPENROSA_DB_ALIAS).get( + app_label=XForm._meta.app_label, model=XForm._meta.model_name +) + ANONYMOUS_USER = get_anonymous_user() # Replace codenames with Permission PKs, remembering the codenames permission_map_copy = dict(PERMISSIONS_MAP) KPI_PKS_TO_CODENAMES = {} for kc_codename, kpi_codename in permission_map_copy.items(): - kc_perm_pk = KobocatPermission.objects.get( - content_type=XFORM_CT, codename=kc_codename).pk + kc_perm_pk = ( + Permission.objects.using(settings.OPENROSA_DB_ALIAS) + .get(content_type=XFORM_CT, codename=kc_codename) + .pk + ) kpi_perm_pk = Permission.objects.get( - content_type=ASSET_CT, codename=kpi_codename).pk + content_type=ASSET_CT, codename=kpi_codename + ).pk del PERMISSIONS_MAP[kc_codename] @@ -175,24 +179,14 @@ def _xform_to_asset_content(xform): return asset_content -def _get_kc_backend_response(xform): - # Get the form data from KC - user = xform.user - response = _kc_forms_api_request(user.auth_token, xform.pk) - if response.status_code == 404: - raise SyncKCXFormsWarning([ - user.username, - xform.id_string, - 'unable to load form data ({})'.format(response.status_code) - ]) - elif response.status_code != 200: - raise SyncKCXFormsError([ - user.username, - xform.id_string, - 'unable to load form data ({})'.format(response.status_code) - ]) - backend_response = response.json() - return backend_response +def _get_backend_response(xform): + return { + 'formid': xform.pk, + 'uuid': xform.uuid, + 'id_string': xform.id_string, + 'kpi_asset_uid': xform.asset.uid, + 'hash': xform.prefixed_hash, + } def _sync_form_content(asset, xform, changes): @@ -242,7 +236,7 @@ def _sync_form_content(asset, xform, changes): # It's important to update `deployment_data` with the new hash from KC; # otherwise, we'll be re-syncing the same content forever (issue #1302) asset.deployment.store_data( - {'backend_response': _get_kc_backend_response(xform)} + {'backend_response': _get_backend_response(xform)} ) return modified @@ -258,11 +252,11 @@ def _sync_form_metadata(asset, xform, changes): if not asset.has_deployment: # A brand-new asset asset.date_created = xform.date_created - kc_deployment = KobocatDeploymentBackend(asset) - kc_deployment.store_data({ - 'backend': 'kobocat', + backend_deployment = OpenRosaDeploymentBackend(asset) + backend_deployment.store_data({ + 'backend': 'openrosa', 'active': xform.downloadable, - 'backend_response': _get_kc_backend_response(xform), + 'backend_response': _get_backend_response(xform), 'version': asset.version_id }) changes.append('CREATE METADATA') @@ -276,10 +270,8 @@ def _sync_form_metadata(asset, xform, changes): modified = False fetch_backend_response = False - backend_response = asset.deployment.backend_response - if (asset.deployment.active != xform.downloadable or - backend_response['downloadable'] != xform.downloadable): + if asset.deployment.active != xform.downloadable: asset.deployment.store_data({'active': xform.downloadable}) modified = True fetch_backend_response = True @@ -297,7 +289,7 @@ def _sync_form_metadata(asset, xform, changes): if fetch_backend_response: asset.deployment.store_data({ - 'backend_response': _get_kc_backend_response(xform) + 'backend_response': _get_backend_response(xform) }) modified = True @@ -318,11 +310,15 @@ def _sync_permissions(asset, xform): return [] # Get all applicable KC permissions set for this xform - xform_user_perms = KobocatUserObjectPermission.objects.filter( - permission_id__in=PERMISSIONS_MAP.keys(), - content_type=XFORM_CT, - object_pk=xform.pk - ).values_list('user', 'permission') + xform_user_perms = ( + UserObjectPermission.objects.using(settings.OPENROSA_DB_ALIAS) + .filter( + permission_id__in=PERMISSIONS_MAP.keys(), + content_type=XFORM_CT, + object_pk=xform.pk, + ) + .values_list('user', 'permission') + ) if not xform_user_perms and not asset.pk: # Nothing to do @@ -473,9 +469,9 @@ def handle(self, *args, **options): sync_kobocat_form_media = options.get('sync_kobocat_form_media') verbosity = options.get('verbosity') users = User.objects.all() - # Do a basic query just to make sure the KobocatXForm model is + # Do a basic query just to make sure the XForm model is # loaded - if not KobocatXForm.objects.exists(): + if not XForm.objects.exists(): return self._print_str('%d total users' % users.count()) # A specific user or everyone? @@ -503,8 +499,8 @@ def handle(self, *args, **options): xform_uuids_to_asset_pks[backend_response['uuid']] = \ existing_survey.pk - # KobocatXForm has a foreign key on KobocatUser, not on User - xforms = KobocatXForm.objects.filter(user_id=user.pk).all() + # XForm has a foreign key on KobocatUser, not on User + xforms = XForm.objects.filter(user_id=user.pk).all() for xform in xforms: try: with transaction.atomic(): diff --git a/kpi/migrations/0011_explode_asset_deployments.py b/kpi/migrations/0011_explode_asset_deployments.py index 2898198a70..0111e83a4f 100644 --- a/kpi/migrations/0011_explode_asset_deployments.py +++ b/kpi/migrations/0011_explode_asset_deployments.py @@ -2,7 +2,7 @@ import sys from django.db import migrations -from kpi.deployment_backends.kobocat_backend import KobocatDeploymentBackend +from kpi.deployment_backends.openrosa_backend import OpenRosaDeploymentBackend from kpi.utils.models import _set_auto_field_update @@ -20,8 +20,8 @@ def explode_assets(apps, schema_editor): for asset in deployed_assets: deployment = asset.assetdeployment_set.last() # Copy the deployment-related data - kc_deployment = KobocatDeploymentBackend(asset) - kc_deployment.store_data({ + backend_deployment = OpenRosaDeploymentBackend(asset) + backend_deployment.store_data({ 'backend': 'kobocat', 'active': deployment.data['downloadable'], 'backend_response': deployment.data, diff --git a/kpi/models/paired_data.py b/kpi/models/paired_data.py index a49bcf7eb8..5487e89f8d 100644 --- a/kpi/models/paired_data.py +++ b/kpi/models/paired_data.py @@ -211,8 +211,6 @@ def md5_hash(self): f'{str(time.time())}.{self.backend_media_id}', prefix=True ) + '-time' - return self.asset_file.md5_hash - @property def is_remote_url(self): """ diff --git a/kpi/serializers/v2/deployment.py b/kpi/serializers/v2/deployment.py index 956568c228..e140800ca9 100644 --- a/kpi/serializers/v2/deployment.py +++ b/kpi/serializers/v2/deployment.py @@ -28,13 +28,15 @@ def create(self, validated_data): asset = self.context['asset'] self._raise_unless_current_version(asset, validated_data) # if no backend is provided, use the installation's default backend - backend_id = validated_data.get('backend', - settings.DEFAULT_DEPLOYMENT_BACKEND) + backend_id = validated_data.get( + 'backend', settings.DEFAULT_DEPLOYMENT_BACKEND + ) # `asset.deploy()` deploys the latest version and updates that versions' # 'deployed' boolean value - asset.deploy(backend=backend_id, - active=validated_data.get('active', False)) + asset.deploy( + backend=backend_id, active=validated_data.get('active', False) + ) return asset.deployment def update(self, instance, validated_data): diff --git a/kpi/serializers/v2/service_usage.py b/kpi/serializers/v2/service_usage.py index 105be87c70..d524443140 100644 --- a/kpi/serializers/v2/service_usage.py +++ b/kpi/serializers/v2/service_usage.py @@ -6,15 +6,15 @@ from rest_framework.fields import empty from kobo.apps.kobo_auth.shortcuts import User +from kobo.apps.openrosa.apps.logger.models import ( + DailyXFormSubmissionCounter, + XForm, +) from kobo.apps.organizations.models import Organization from kobo.apps.organizations.utils import organization_month_start, organization_year_start from kobo.apps.stripe.constants import ACTIVE_STRIPE_STATUSES from kobo.apps.trackers.models import NLPUsageCounter -from kpi.deployment_backends.kc_access.shadow_models import ( - KobocatXForm, - KobocatDailyXFormSubmissionCounter, -) -from kpi.deployment_backends.kobocat_backend import KobocatDeploymentBackend +from kpi.deployment_backends.openrosa_backend import OpenRosaDeploymentBackend from kpi.models.asset import Asset @@ -91,7 +91,7 @@ def _get_nlp_tracking_data(self, asset, start_date=None): 'total_nlp_asr_seconds': 0, 'total_nlp_mt_characters': 0, } - return KobocatDeploymentBackend.nlp_tracking_data( + return OpenRosaDeploymentBackend.nlp_tracking_data( asset_ids=[asset.id], start_date=start_date ) @@ -237,7 +237,7 @@ def _get_storage_usage(self): Users are represented by their ids with `self._user_ids` """ - xforms = KobocatXForm.objects.only('attachment_storage_bytes', 'id').exclude( + xforms = XForm.objects.only('attachment_storage_bytes', 'id').exclude( pending_delete=True ).filter(self._user_id_query) @@ -253,7 +253,7 @@ def _get_submission_counters(self, month_filter, year_filter): Users are represented by their ids with `self._user_ids` """ - submission_count = KobocatDailyXFormSubmissionCounter.objects.only( + submission_count = DailyXFormSubmissionCounter.objects.only( 'counter', 'user_id' ).filter(self._user_id_query).aggregate( all_time=Coalesce(Sum('counter'), 0), diff --git a/kpi/signals.py b/kpi/signals.py index bf25e01eb8..89be23937d 100644 --- a/kpi/signals.py +++ b/kpi/signals.py @@ -10,9 +10,7 @@ from kobo.apps.kobo_auth.shortcuts import User from kobo.apps.hook.models.hook import Hook from kpi.constants import PERM_ADD_SUBMISSIONS -from kpi.deployment_backends.kc_access.shadow_models import ( - KobocatUser, -) + from kpi.deployment_backends.kc_access.utils import ( grant_kc_model_level_perms, kc_transaction_atomic, @@ -63,7 +61,7 @@ def save_kobocat_user(sender, instance, created, raw, **kwargs): if not settings.TESTING: with kc_transaction_atomic(): - KobocatUser.sync(instance) + instance.sync_to_openrosa_db() if created: grant_kc_model_level_perms(instance) @@ -76,16 +74,6 @@ def tag_uid_post_save(sender, instance, created, raw, **kwargs): TagUid.objects.get_or_create(tag=instance) -@receiver(post_save, sender=Hook) -def update_kc_xform_has_kpi_hooks(sender, instance, **kwargs): - """ - Updates KoBoCAT XForm instance as soon as Asset.Hook list is updated. - """ - asset = instance.asset - if asset.has_deployment: - asset.deployment.set_has_kpi_hooks() - - @receiver(post_delete, sender=Asset) def post_delete_asset(sender, instance, **kwargs): # Update parent's languages if this object is a child of another asset. diff --git a/kpi/tasks.py b/kpi/tasks.py index 3006d1fa0f..3da2466091 100644 --- a/kpi/tasks.py +++ b/kpi/tasks.py @@ -1,10 +1,13 @@ # coding: utf-8 +import time + import constance import requests from django.conf import settings from django.core.mail import send_mail from django.core.management import call_command + from kobo.apps.kobo_auth.shortcuts import User from kobo.apps.markdownx_uploader.tasks import remove_unused_markdown_files from kobo.celery import celery_app @@ -73,7 +76,15 @@ def sync_kobocat_xforms( @celery_app.task def sync_media_files(asset_uid): - asset = Asset.objects.get(uid=asset_uid) + asset = Asset.objects.defer('content').get(uid=asset_uid) + if not asset.has_deployment: + # 🙈 Race condition: Celery task starts too fast and does not see + # the deployment data, even if asset has been saved prior to call this + # task + # TODO Find why the race condition happens and remove `time.sleep(1)` + time.sleep(1) + asset.refresh_from_db(fields=['_deployment_data']) + asset.deployment.sync_media_files() @@ -95,7 +106,6 @@ def enketo_flush_cached_preview(server_url, form_id): response.raise_for_status() - @celery_app.task(time_limit=LIMIT_HOURS_23, soft_time_limit=LIMIT_HOURS_23) def perform_maintenance(): """ diff --git a/kpi/tests/api/v2/test_api_attachments.py b/kpi/tests/api/v2/test_api_attachments.py index 3ceb4ae8e9..0f3c891cf7 100644 --- a/kpi/tests/api/v2/test_api_attachments.py +++ b/kpi/tests/api/v2/test_api_attachments.py @@ -95,7 +95,6 @@ def test_convert_mp4_to_mp3(self): ), querystring=query_dict.urlencode() ) - response = self.client.get(url) assert response.status_code == status.HTTP_200_OK assert response['Content-Type'] == 'audio/mpeg' diff --git a/kpi/tests/api/v2/test_api_invalid_password_access.py b/kpi/tests/api/v2/test_api_invalid_password_access.py index 9468156e6e..fc2f2918ef 100644 --- a/kpi/tests/api/v2/test_api_invalid_password_access.py +++ b/kpi/tests/api/v2/test_api_invalid_password_access.py @@ -185,20 +185,3 @@ def _access_endpoints(self, access_granted: bool, headers: dict = {}): # `/environment` response = self.client.get(reverse('environment'), **headers) assert response.status_code == status.HTTP_200_OK - - # Hook signal is a particular case but should not return a 403 - data = {'submission_id': submission_id} - # # `/api/v2/assets//hook-signal/` - response = self.client.post( - reverse( - self._get_endpoint('hook-signal-list'), - kwargs={ - 'format': 'json', - 'parent_lookup_asset': self.asset.uid, - }, - ), - data=data, - **headers, - ) - # return a 202 first time but 409 other attempts. - assert response.status_code != status.HTTP_403_FORBIDDEN diff --git a/kpi/tests/api/v2/test_api_paired_data.py b/kpi/tests/api/v2/test_api_paired_data.py index 5ca5121623..bf4fc5821f 100644 --- a/kpi/tests/api/v2/test_api_paired_data.py +++ b/kpi/tests/api/v2/test_api_paired_data.py @@ -1,6 +1,6 @@ # coding: utf-8 import unittest -from mock import patch, MagicMock +from mock import patch from django.urls import reverse from rest_framework import status @@ -390,13 +390,11 @@ def test_get_external_with_no_auth(self): # When owner's destination asset does not require any authentications, # everybody can see their data self.client.logout() - with patch( - 'kpi.deployment_backends.backends.MockDeploymentBackend.xform', - MagicMock(), - ) as xf_mock: - xf_mock.require_auth = False - response = self.client.get(self.external_xml_url) - self.assertEqual(response.status_code, status.HTTP_200_OK) + xform = self.destination_asset.deployment.xform + xform.require_auth = False + xform.save(update_fields=['require_auth']) + response = self.client.get(self.external_xml_url) + self.assertEqual(response.status_code, status.HTTP_200_OK) @unittest.skip(reason='Skip until mock back end supports XML submissions') def test_get_external_with_changed_source_fields(self): diff --git a/kpi/tests/api/v2/test_api_submissions.py b/kpi/tests/api/v2/test_api_submissions.py index 5a98d72f11..9a1c0409dd 100644 --- a/kpi/tests/api/v2/test_api_submissions.py +++ b/kpi/tests/api/v2/test_api_submissions.py @@ -206,12 +206,8 @@ def test_audit_log_on_bulk_delete(self): self.asset.owner, fields=['_id'] ) ] - ( - app_label, - model_name, - ) = self.asset.deployment.submission_model.get_app_label_and_model_name() audit_log_count = AuditLog.objects.filter( - user=self.someuser, app_label=app_label, model_name=model_name + user=self.someuser, app_label='logger', model_name='instance' ).count() # No submissions have been deleted yet assert audit_log_count == 0 @@ -221,7 +217,7 @@ def test_audit_log_on_bulk_delete(self): # All submissions have been deleted and should be logged deleted_submission_ids = AuditLog.objects.values_list( 'pk', flat=True - ).filter(user=self.someuser, app_label=app_label, model_name=model_name) + ).filter(user=self.someuser, app_label='logger', model_name='instance') assert len(expected_submission_ids) > 0 assert sorted(expected_submission_ids), sorted(deleted_submission_ids) @@ -783,12 +779,8 @@ def test_audit_log_on_delete(self): deleted. """ submission = self.submissions_submitted_by_someuser[0] - ( - app_label, - model_name, - ) = self.asset.deployment.submission_model.get_app_label_and_model_name() audit_log_count = AuditLog.objects.filter( - user=self.someuser, app_label=app_label, model_name=model_name + user=self.someuser, app_label='logger', model_name='instance' ).count() # No submissions have been deleted yet assert audit_log_count == 0 @@ -798,7 +790,7 @@ def test_audit_log_on_delete(self): # All submissions have been deleted and should be logged deleted_submission_ids = AuditLog.objects.values_list( 'pk', flat=True - ).filter(user=self.someuser, app_label=app_label, model_name=model_name) + ).filter(user=self.someuser, app_label='logger', model_name='instance') assert len(deleted_submission_ids) > 0 assert [submission['_id']], deleted_submission_ids diff --git a/kpi/tests/utils/mock.py b/kpi/tests/utils/mock.py index 631c364f6a..1400228e93 100644 --- a/kpi/tests/utils/mock.py +++ b/kpi/tests/utils/mock.py @@ -9,9 +9,14 @@ from django.conf import settings from django.core.files import File + from django.core.files.storage import default_storage from rest_framework import status +from kobo.apps.openrosa.apps.logger.models.attachment import ( + Attachment, + upload_to, +) from kobo.apps.openrosa.libs.utils.image_tools import ( get_optimized_image_path, resize, @@ -107,83 +112,3 @@ def enketo_view_instance_response(request): } headers = {} return status.HTTP_201_CREATED, headers, json.dumps(resp_body) - - -class MockAttachment(AudioTranscodingMixin): - """ - Mock object to simulate KobocatAttachment. - Relationship with ReadOnlyKobocatInstance is ignored but could be implemented - - TODO Remove this class and use `Attachment` model everywhere in tests - """ - def __init__(self, pk: int, filename: str, mimetype: str = None, **kwargs): - - self.id = pk # To mimic Django model instances - self.pk = pk - - # Unit test `test_thumbnail_creation_on_demand()` is using real `Attachment` - # objects while other tests are using `MockAttachment` objects. - # If an Attachment object exists, let's assume unit test is using real - # Attachment objects. Otherwise, use MockAttachment. - from kobo.apps.openrosa.apps.logger.models import Attachment # Avoid circular import - - attachment_object = Attachment.objects.filter(pk=pk).first() - if attachment_object: - self.media_file = attachment_object.media_file - self.media_file_size = attachment_object.media_file_size - self.media_file_basename = attachment_object.media_file_basename - else: - basename = os.path.basename(filename) - file_ = os.path.join( - settings.BASE_DIR, - 'kpi', - 'tests', - basename - ) - self.media_file = File(open(file_, 'rb'), basename) - self.media_file.path = file_ - self.media_file_size = os.path.getsize(file_) - self.media_file_basename = basename - - self.content = self.media_file.read() - - if not mimetype: - self.mimetype, _ = guess_type(file_) - else: - self.mimetype = mimetype - - def __exit__(self, exc_type, exc_val, exc_tb): - self.media_file.close() - - @property - def absolute_path(self): - """ - Return the absolute path on local file system of the attachment. - Otherwise, return the AWS url (e.g. https://...) - """ - if isinstance(default_kobocat_storage, KobocatFileSystemStorage): - return self.media_file.path - - return self.media_file.url - - def protected_path( - self, format_: Optional[str] = None, suffix: Optional[str] = None - ) -> str: - if format_ == 'mp3': - extension = '.mp3' - with NamedTemporaryFile(suffix=extension) as f: - self.content = self.get_transcoded_audio(format_) - return f.name - else: - if suffix and self.mimetype.startswith('image/'): - optimized_image_path = get_optimized_image_path( - self.media_file.name, suffix - ) - if not default_storage.exists(optimized_image_path): - resize(self.media_file.name) - if isinstance(default_kobocat_storage, KobocatFileSystemStorage): - return default_kobocat_storage.path(optimized_image_path) - else: - return default_kobocat_storage.url(optimized_image_path) - else: - return self.absolute_path diff --git a/kpi/urls/router_api_v1.py b/kpi/urls/router_api_v1.py index 4e53c7ae8c..368101a9e3 100644 --- a/kpi/urls/router_api_v1.py +++ b/kpi/urls/router_api_v1.py @@ -3,7 +3,6 @@ from kobo.apps.hook.views.v1.hook import HookViewSet from kobo.apps.hook.views.v1.hook_log import HookLogViewSet -from kobo.apps.hook.views.v1.hook_signal import HookSignalViewSet from kobo.apps.reports.views import ReportsViewSet from kpi.views.v1 import ( @@ -29,11 +28,6 @@ basename='asset-version', parents_query_lookups=['asset'], ) -asset_routes.register(r'hook-signal', - HookSignalViewSet, - basename='hook-signal', - parents_query_lookups=['asset'], - ) asset_routes.register(r'submissions', SubmissionViewSet, basename='submission', diff --git a/kpi/urls/router_api_v2.py b/kpi/urls/router_api_v2.py index 9883884fc4..7c7dbd2575 100644 --- a/kpi/urls/router_api_v2.py +++ b/kpi/urls/router_api_v2.py @@ -5,7 +5,6 @@ from kobo.apps.audit_log.urls import router as audit_log_router from kobo.apps.hook.views.v2.hook import HookViewSet from kobo.apps.hook.views.v2.hook_log import HookLogViewSet -from kobo.apps.hook.views.v2.hook_signal import HookSignalViewSet from kobo.apps.languages.urls import router as language_router from kobo.apps.organizations.views import OrganizationViewSet from kobo.apps.project_ownership.urls import router as project_ownership_router @@ -104,12 +103,6 @@ def get_urls(self, *args, **kwargs): parents_query_lookups=['asset'], ) -asset_routes.register(r'hook-signal', - HookSignalViewSet, - basename='hook-signal', - parents_query_lookups=['asset'], - ) - asset_routes.register(r'paired-data', PairedDataViewset, basename='paired-data', diff --git a/kpi/utils/database.py b/kpi/utils/database.py index 77bf0e70a8..6b1e624b53 100644 --- a/kpi/utils/database.py +++ b/kpi/utils/database.py @@ -1,6 +1,12 @@ import threading from functools import wraps +from django.conf import settings +from django.db import ( + connections, + models, +) + thread_local = threading.local() @@ -26,3 +32,38 @@ def inner(*args, **kwargs): def get_thread_local(attr, default=None): return getattr(thread_local, attr, None) or default + + +def update_autofield_sequence( + model: models.Model, using: str = settings.OPENROSA_DB_ALIAS +): + """ + Fixes the PostgreSQL sequence for the first (and only?) `AutoField` on + `model`, à la `manage.py sqlsequencereset` + """ + # Updating sequences on fresh environments fails because the only user + # in the DB is django-guardian AnonymousUser and `max(pk)` returns -1. + # Error: + # > setval: value -1 is out of bounds for sequence + # Using abs() and testing if max(pk) equals -1, leaves the sequence alone. + sql_template = ( + "SELECT setval(" + " pg_get_serial_sequence('{table}','{column}'), " + " abs(coalesce(max({column}), 1)), " + " max({column}) IS NOT null and max({column}) != -1" + ") " + "FROM {table};" + ) + autofield = None + for f in model._meta.get_fields(): + if isinstance(f, models.AutoField): + autofield = f + break + if not autofield: + return + query = sql_template.format( + table=model._meta.db_table, column=autofield.column + ) + connection = connections[using] + with connection.cursor() as cursor: + cursor.execute(query) diff --git a/kpi/utils/files.py b/kpi/utils/files.py new file mode 100644 index 0000000000..adb911ee1e --- /dev/null +++ b/kpi/utils/files.py @@ -0,0 +1,12 @@ +import os +from mimetypes import guess_type + +from django.core.files.base import ContentFile + + +class ExtendedContentFile(ContentFile): + + @property + def content_type(self): + mimetype, _ = guess_type(os.path.basename(self.name)) + return mimetype diff --git a/kpi/utils/project_view_exports.py b/kpi/utils/project_view_exports.py index 96044fca44..5655f4496f 100644 --- a/kpi/utils/project_view_exports.py +++ b/kpi/utils/project_view_exports.py @@ -2,14 +2,15 @@ from __future__ import annotations import csv from io import StringIO +from typing import Union from django.conf import settings from django.db.models import Count, F, Q from django.db.models.query import QuerySet from kobo.apps.kobo_auth.shortcuts import User +from kobo.apps.openrosa.apps.logger.models.xform import XForm from kpi.constants import ASSET_TYPE_SURVEY -from kpi.deployment_backends.kc_access.shadow_models import KobocatXForm from kpi.models import Asset from kpi.utils.project_views import get_region_for_view @@ -117,7 +118,7 @@ def get_q(countries: list[str], export_type: str) -> QuerySet: def get_submission_count(xform_id: int) -> int: result = ( - KobocatXForm.objects.values('num_of_submissions') + XForm.objects.values('num_of_submissions') .filter(pk=xform_id) .first() ) diff --git a/kpi/views/v2/asset_snapshot.py b/kpi/views/v2/asset_snapshot.py index d3036c7031..c55d4c8855 100644 --- a/kpi/views/v2/asset_snapshot.py +++ b/kpi/views/v2/asset_snapshot.py @@ -1,20 +1,14 @@ -# coding: utf-8 -import re import copy -from xml.dom import Node from typing import Optional import requests -from defusedxml import minidom from django.conf import settings -from django.db.models import Q, F from django.http import HttpResponseRedirect, Http404 from rest_framework import renderers, serializers from rest_framework.decorators import action from rest_framework.response import Response from rest_framework.reverse import reverse -from kobo.apps.form_disclaimer.models import FormDisclaimer from kpi.authentication import DigestAuthentication, EnketoSessionAuthentication from kpi.constants import PERM_VIEW_ASSET from kpi.exceptions import SubmissionIntegrityError @@ -232,17 +226,19 @@ def submission(self, request, *args, **kwargs): # Prepare attachments even if all files are present in `request.FILES` # (i.e.: submission XML and attachments) - attachments = None + attachments = {} # Remove 'xml_submission_file' since it is already handled request.FILES.pop('xml_submission_file') + + # TODO pass request.FILES to `edit_submission()` directly when + # KobocatBackendDeployment is gone if len(request.FILES): - attachments = {} for name, attachment in request.FILES.items(): attachments[name] = attachment try: xml_response = asset_snapshot.asset.deployment.edit_submission( - xml_submission_file, request.user, attachments + xml_submission_file, request, attachments ) except SubmissionIntegrityError as e: raise serializers.ValidationError(str(e)) diff --git a/kpi/views/v2/attachment.py b/kpi/views/v2/attachment.py index 75aa81887f..b35c1dd4d0 100644 --- a/kpi/views/v2/attachment.py +++ b/kpi/views/v2/attachment.py @@ -147,7 +147,7 @@ def _get_response( else None ) return Response( - attachment.content, + attachment.media_file, content_type=content_type, ) diff --git a/kpi/views/v2/data.py b/kpi/views/v2/data.py index 402f541d09..febab500ed 100644 --- a/kpi/views/v2/data.py +++ b/kpi/views/v2/data.py @@ -56,8 +56,9 @@ from kpi.serializers.v2.data import DataBulkActionsValidator -class DataViewSet(AssetNestedObjectViewsetMixin, NestedViewSetMixin, - viewsets.GenericViewSet): +class DataViewSet( + AssetNestedObjectViewsetMixin, NestedViewSetMixin, viewsets.GenericViewSet +): """ ## List of submissions for a specific asset @@ -355,14 +356,10 @@ def bulk(self, request, *args, **kwargs): query=data['query'], fields=['_id', '_uuid'] ) - ( - app_label, - model_name, - ) = deployment.submission_model.get_app_label_and_model_name() for submission in submissions: audit_logs.append(AuditLog( - app_label=app_label, - model_name=model_name, + app_label='logger', + model_name='instance', object_id=submission['_id'], user=request.user, user_uid=request.user.extra_details.uid, @@ -374,7 +371,9 @@ def bulk(self, request, *args, **kwargs): )) # Send request to KC - json_response = action_(bulk_actions_validator.data, request.user) + json_response = action_( + bulk_actions_validator.data, request.user, request=request + ) # If requests has succeeded, let's log deletions (if any) if json_response['status'] == status.HTTP_200_OK and audit_logs: @@ -399,13 +398,9 @@ def destroy(self, request, pk, *args, **kwargs): ) if json_response['status'] == status.HTTP_204_NO_CONTENT: - ( - app_label, - model_name, - ) = deployment.submission_model.get_app_label_and_model_name() AuditLog.objects.create( - app_label=app_label, - model_name=model_name, + app_label='logger', + model_name='instance', object_id=pk, user=request.user, metadata={ @@ -471,10 +466,12 @@ def list(self, request, *args, **kwargs): ) try: - submissions = deployment.get_submissions(request.user, - format_type=format_type, - request=request, - **filters) + submissions = deployment.get_submissions( + request.user, + format_type=format_type, + request=request, + **filters + ) except OperationFailure as err: message = str(err) # Don't show just any raw exception message out of fear of data leaking @@ -537,7 +534,7 @@ def retrieve(self, request, pk, *args, **kwargs): # The `get_submissions()` is a generator in KobocatDeploymentBackend # class but a list in MockDeploymentBackend. We cast the result as a list # no matter what is the deployment back-end class to make it work with - # both. Since the number of submissions is be very small, it should not + # both. Since the number of submissions is very small, it should not # have a big impact on memory (i.e. list vs generator) submissions = list(deployment.get_submissions(**params)) if not submissions: @@ -557,7 +554,7 @@ def duplicate(self, request, pk, *args, **kwargs): # Coerce to int because back end only finds matches with same type submission_id = positive_int(pk) duplicate_response = deployment.duplicate_submission( - submission_id=submission_id, user=request.user + submission_id=submission_id, request=request ) return Response(duplicate_response, status=status.HTTP_201_CREATED)