ci: models: avoid polling jobs waiting more than a week on the backend

squad/ci/models.py

@@ -50,10 +50,18 @@ class Backend(models.Model):
     def poll(self):
         if not self.poll_enabled:
             return
+
+        # There are cases that the backend might be running into issues and there is no
+        # way for SQUAD to know it, causing jobs to this backend/environment to clog the
+        # ci_fetch queue with jobs that are never ready to fetch. In order to avoid this
+        # SQUAD will not poll jobs that didn't get processed after a week. It's hardcoded
+        # for now, but it can become a setting in the backend model.
+        week_ago = timezone.now() - relativedelta(days=7)
         test_jobs = self.test_jobs.filter(
             submitted=True,
             fetched=False,
-            fetch_attempts__lt=self.max_fetch_attempts
+            fetch_attempts__lt=self.max_fetch_attempts,
+            submitted_at__gt=week_ago,
         )
         for test_job in test_jobs:
             last = test_job.last_fetch_attempt

test/ci/test_models.py

@@ -39,6 +39,7 @@ def setUp(self):
         self.build = self.project.builds.create(version='1')
 
     def create_test_job(self, **attrs):
+        attrs['submitted_at'] = attrs.get('submitted_at') or timezone.now()
         return self.backend.test_jobs.create(target=self.project, target_build=self.build, **attrs)
 
 
@@ -81,6 +82,15 @@ def test_poll_gives_up_eventually(self):
         jobs = list(self.backend.poll())
         self.assertEqual([], jobs)
 
+    def test_poll_gives_up_after_a_week(self):
+        # The test above is when the testjob is ready to be fetched
+        # this tests the case where the job has been submitted but
+        # it was never processed by the backend
+        eight_days_ago = timezone.now() - relativedelta(days=8)
+        self.create_test_job(submitted=True, submitted_at=eight_days_ago, fetched=False)
+        jobs = list(self.backend.poll())
+        self.assertEqual([], jobs)
+
 
 class BackendFetchTest(BackendTestBase):
 

test/ci/test_tasks.py

@@ -6,6 +6,7 @@
 
 
 from celery.exceptions import Retry
+from django.utils import timezone
 
 
 from squad.ci import models
@@ -41,7 +42,7 @@ def test_poll_calls_fetch_on_all_test_jobs(self, fetch_method):
         group = core_models.Group.objects.create(slug='testgroup')
         project = group.projects.create(slug='testproject')
         backend = models.Backend.objects.create(name='b1')
-        testjob = backend.test_jobs.create(target=project, submitted=True)
+        testjob = backend.test_jobs.create(target=project, submitted=True, submitted_at=timezone.now())
         poll.apply()
         fetch_method.apply_async.assert_called_with(args=(testjob.id,), task_id=task_id(testjob))