Code refactoring for project deployment

hub/admin/extend_user.py

@@ -21,13 +21,13 @@
     USERNAME_INVALID_MESSAGE,
     username_validators,
 )
+from kobo.apps.openrosa.apps.logger.models import MonthlyXFormSubmissionCounter
 from kobo.apps.organizations.models import OrganizationUser
 from kobo.apps.trash_bin.exceptions import TrashIntegrityError
 from kobo.apps.trash_bin.models.account import AccountTrash
 from kobo.apps.trash_bin.utils import move_to_trash
-from kpi.deployment_backends.kc_access.shadow_models import (
-    KobocatMonthlyXFormSubmissionCounter,
-)
+
+
 from kpi.models.asset import AssetDeploymentStatus
 from .filters import UserAdvancedSearchFilter
 from .mixins import AdvancedSearchMixin
@@ -257,7 +257,7 @@ def monthly_submission_count(self, obj):
         displayed in the Django admin user changelist page
         """
         today = timezone.now().date()
-        instances = KobocatMonthlyXFormSubmissionCounter.objects.filter(
+        instances = MonthlyXFormSubmissionCounter.objects.filter(
             user_id=obj.id,
             year=today.year,
             month=today.month,

hub/models/extra_user_detail.py

@@ -1,7 +1,7 @@
 from django.conf import settings
 from django.db import models
 
-from kpi.deployment_backends.kc_access.shadow_models import KobocatUserProfile
+from kobo.apps.openrosa.apps.main.models import UserProfile
 from kpi.fields import KpiUidField
 from kpi.mixins import StandardizeSearchableFieldMixin
 
@@ -43,9 +43,9 @@ def save(
             update_fields=update_fields,
         )
 
-        # Sync `validated_password` field to `KobocatUserProfile` only when
+        # Sync `validated_password` field to `UserProfile` only when
         # this object is updated to avoid a race condition and an IntegrityError
-        # when trying to save `KobocatUserProfile` object whereas the related
+        # when trying to save `UserProfile` object whereas the related
         # `KobocatUser` object has not been created yet.
         if (
             not settings.TESTING
@@ -55,7 +55,7 @@ def save(
                 or (update_fields and 'validated_password' in update_fields)
             )
         ):
-            KobocatUserProfile.set_password_details(
+            UserProfile.set_password_details(
                 self.user.id,
                 self.validated_password,
             )

kobo/apps/accounts/mfa/models.py

@@ -8,9 +8,7 @@
     MFAMethodAdmin as TrenchMFAMethodAdmin,
 )
 
-from kpi.deployment_backends.kc_access.shadow_models import (
-    KobocatUserProfile,
-)
+from kobo.apps.openrosa.apps.main.models import UserProfile
 
 
 class MfaAvailableToUser(models.Model):
@@ -74,7 +72,7 @@ def save(
         Update user's profile in KoBoCAT database.
         """
         if not settings.TESTING and not created:
-            KobocatUserProfile.set_mfa_status(
+            UserProfile.set_mfa_status(
                 user_id=self.user.pk, is_active=self.is_active
             )
 
@@ -83,10 +81,10 @@ def delete(self, using=None, keep_parents=False):
         super().delete(using, keep_parents)
 
         """
-        Update user's profile in KoBoCAT database.
+        Update user's profile in KoboCAT database.
         """
         if not settings.TESTING:
-            KobocatUserProfile.set_mfa_status(
+            UserProfile.set_mfa_status(
                 user_id=user_id, is_active=False
             )
 

kobo/apps/form_disclaimer/models.py

@@ -1,5 +1,5 @@
 from django.conf import settings
-from django.db import models, transaction
+from django.db import models
 from django.db.models import Q
 from django.db.models.constraints import UniqueConstraint
 from markdownx.models import MarkdownxField

kobo/apps/hook/constants.py

@@ -1,5 +1,6 @@
 # coding: utf-8
 from enum import Enum
+from rest_framework import status
 
 
 HOOK_LOG_FAILED = 0
@@ -16,3 +17,11 @@ class HookLogStatus(Enum):
 KOBO_INTERNAL_ERROR_STATUS_CODE = None
 
 SUBMISSION_PLACEHOLDER = '%SUBMISSION%'
+
+# Status codes that trigger a retry
+RETRIABLE_STATUS_CODES = [
+    # status.HTTP_429_TOO_MANY_REQUESTS,
+    status.HTTP_502_BAD_GATEWAY,
+    status.HTTP_503_SERVICE_UNAVAILABLE,
+    status.HTTP_504_GATEWAY_TIMEOUT,
+]

kobo/apps/hook/exceptions.py

@@ -0,0 +1,3 @@
+
+class HookRemoteServerDownError(Exception):
+    pass

kobo/apps/hook/models/hook_log.py

@@ -1,4 +1,3 @@
-# coding: utf-8
 from datetime import timedelta
 
 import constance
@@ -17,38 +16,43 @@
 
 class HookLog(models.Model):
 
-    hook = models.ForeignKey("Hook", related_name="logs", on_delete=models.CASCADE)
+    hook = models.ForeignKey(
+        "Hook", related_name="logs", on_delete=models.CASCADE
+    )
     uid = KpiUidField(uid_prefix="hl")
-    submission_id = models.IntegerField(default=0, db_index=True)  # `KoBoCAT.logger.Instance.id`
+    submission_id = models.IntegerField(  # `KoboCAT.logger.Instance.id`
+        default=0, db_index=True
+    )
     tries = models.PositiveSmallIntegerField(default=0)
     status = models.PositiveSmallIntegerField(
         choices=[[e.value, e.name.title()] for e in HookLogStatus],
-        default=HookLogStatus.PENDING.value
+        default=HookLogStatus.PENDING.value,
     )  # Could use status_code, but will speed-up queries
-    status_code = models.IntegerField(default=KOBO_INTERNAL_ERROR_STATUS_CODE, null=True, blank=True)
+    status_code = models.IntegerField(
+        default=KOBO_INTERNAL_ERROR_STATUS_CODE, null=True, blank=True
+    )
     message = models.TextField(default="")
     date_created = models.DateTimeField(auto_now_add=True)
     date_modified = models.DateTimeField(auto_now_add=True)
 
     class Meta:
-        ordering = ["-date_created"]
+        ordering = ['-date_created']
 
+    @property
     def can_retry(self) -> bool:
         """
         Return whether instance can be resent to external endpoint.
         Notice: even if False is returned, `self.retry()` can be triggered.
         """
         if self.hook.active:
-            seconds = HookLog.get_elapsed_seconds(
-                constance.config.HOOK_MAX_RETRIES
-            )
-            threshold = timezone.now() - timedelta(seconds=seconds)
-            # We can retry only if system has already tried 3 times.
-            # If log is still pending after 3 times, there was an issue,
-            # we allow the retry
-            return (
-                self.status == HOOK_LOG_FAILED
-                or (self.date_modified < threshold and self.status == HOOK_LOG_PENDING)
+            # If log is still pending after `constance.config.HOOK_MAX_RETRIES`
+            # times, there was an issue, we allow the retry.
+            threshold = timezone.now() - timedelta(seconds=120)
+
+            return self.status == HOOK_LOG_FAILED or (
+                self.date_modified < threshold
+                and self.status == HOOK_LOG_PENDING
+                and self.tries >= constance.config.HOOK_MAX_RETRIES
             )
 
         return False
@@ -66,29 +70,6 @@ def change_status(
 
         self.save(reset_status=True)
 
-    @staticmethod
-    def get_elapsed_seconds(retries_count: int) -> int:
-        """
-        Calculate number of elapsed seconds since first try.
-        Return the number of seconds.
-        """
-        # We need to sum all seconds between each retry
-        seconds = 0
-        for retries_count in range(retries_count):
-            # Range is zero-indexed
-            seconds += HookLog.get_remaining_seconds(retries_count)
-
-        return seconds
-
-    @staticmethod
-    def get_remaining_seconds(retries_count):
-        """
-        Calculate number of remaining seconds before next retry
-        :param retries_count: int.
-        :return: int. Number of seconds
-        """
-        return 60 * (10 ** retries_count)
-
     def retry(self):
         """
         Retries to send data to external service
@@ -100,7 +81,7 @@ def retry(self):
             service_definition.send()
             self.refresh_from_db()
         except Exception as e:
-            logging.error("HookLog.retry - {}".format(str(e)), exc_info=True)
+            logging.error('HookLog.retry - {}'.format(str(e)), exc_info=True)
             self.change_status(HOOK_LOG_FAILED)
             return False
 
@@ -110,7 +91,7 @@ def save(self, *args, **kwargs):
         # Update date_modified each time object is saved
         self.date_modified = timezone.now()
         # We don't want to alter tries when we only change the status
-        if kwargs.pop("reset_status", False) is False:
+        if kwargs.pop('reset_status', False) is False:
             self.tries += 1
             self.hook.reset_totals()
         super().save(*args, **kwargs)