Update the fetch for notifications to take the correct datetimes into account

app/dao/fact_notification_status_dao.py

@@ -1,4 +1,4 @@
-from datetime import datetime, time, timedelta
+from datetime import datetime, time, timedelta, timezone
 
 from flask import current_app
 from sqlalchemy import Date, case, func
@@ -7,7 +7,7 @@
 from sqlalchemy.types import DateTime, Integer
 
 from app import db
-from app.dao.date_util import tz_aware_midnight_n_days_ago, utc_midnight_n_days_ago
+from app.dao.date_util import get_query_date_based_on_retention_period
 from app.models import (
     EMAIL_TYPE,
     KEY_TYPE_NORMAL,
@@ -246,32 +246,41 @@ def fetch_notification_status_for_service_for_day(bst_day, service_id):
     )
 
 
-def fetch_notification_status_for_service_for_today_and_7_previous_days(service_id, by_template=False, limit_days=7):
-    if limit_days == 1:
-        ft_start_date = utc_midnight_n_days_ago(limit_days - 1)
-        # For daily stats, service limits reset at 12:00am UTC each night, so we need to fetch the data from 12:00 UTC to now
-        start = utc_midnight_n_days_ago(0)
-        end = datetime.utcnow()
-    else:
-        ft_start_date = utc_midnight_n_days_ago(limit_days)
-
-        # The nightly task that populates ft_notification_status counts collects notifications from
-        # 5AM the day before to 5AM of the current day. So we need to match that timeframe when
-        # we fetch notifications for the current day.
-        start = (tz_aware_midnight_n_days_ago(1) + timedelta(hours=5)).replace(minute=0, second=0, microsecond=0)
-        end = (tz_aware_midnight_n_days_ago(0) + timedelta(hours=5)).replace(minute=0, second=0, microsecond=0)
+def _stats_for_days_facts(service_id, start_time, by_template=False, notification_type=None):
+    """
+    We want to take the data from the fact_notification_status table for bst_data>=start_date
 
-    stats_for_7_days = db.session.query(
+    Returns:
+        Aggregate data in a certain format for total notifications
+    """
+    stats_from_facts = db.session.query(
         FactNotificationStatus.notification_type.label("notification_type"),
         FactNotificationStatus.notification_status.label("status"),
         *([FactNotificationStatus.template_id.label("template_id")] if by_template else []),
         *([FactNotificationStatus.notification_count.label("count")]),
     ).filter(
         FactNotificationStatus.service_id == service_id,
-        FactNotificationStatus.bst_date >= ft_start_date,
+        FactNotificationStatus.bst_date >= start_time,
         FactNotificationStatus.key_type != KEY_TYPE_TEST,
     )
 
+    if notification_type:
+        stats_from_facts = stats_from_facts.filter(FactNotificationStatus.notification_type == notification_type)
+
+    return stats_from_facts
+
+
+def _timing_notification_table(service_id):
+    max_date_from_facts = (
+        FactNotificationStatus.query.with_entities(func.max(FactNotificationStatus.bst_date))
+        .filter(FactNotificationStatus.service_id == service_id)
+        .scalar()
+    )
+    date_to_use = max_date_from_facts + timedelta(days=1) if max_date_from_facts else datetime.now(timezone.utc)
+    return datetime.combine(date_to_use, time.min)
+
+
+def _stats_for_today(service_id, start_time, by_template=False, notification_type=None):
     stats_for_today = (
         db.session.query(
             Notification.notification_type.cast(db.Text),
@@ -280,8 +289,7 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_
             *([func.count().label("count")]),
         )
         .filter(
-            Notification.created_at >= start,
-            Notification.created_at <= end,
+            Notification.created_at >= start_time,
             Notification.service_id == service_id,
             Notification.key_type != KEY_TYPE_TEST,
         )
@@ -291,8 +299,41 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_
             Notification.status,
         )
     )
+    if notification_type:
+        stats_for_today = stats_for_today.filter(Notification.notification_type == notification_type)
+
+    return stats_for_today
+
+
+def fetch_notification_status_for_service_for_today_and_7_previous_days(
+    service_id, by_template=False, limit_days=7, notification_type=None
+):
+    """
+    We want to take the data from the fact_notification_status table and the notifications table and combine them
+    We will take the data from notifications ONLY for today and the fact_notification_status for the last 6 days.
+    In total we will have 7 days worth of data.
+
+    As the data in facts is populated by a job, instead of
+    keeping track of the job - we will find the max date in the facts table and then use that date to get the
+    data from the notifications table.
+
+    Args:
+        service_id (uuid): service_id
+        by_template (bool, optional): aggregate by template Defaults to False.
+        limit_days (int, optional): Number of days we want to get data for - it can depend on sensitive services.
+            Defaults to 7.
+        notification_type (str, optional): notification type. Defaults to None which means all notification types.
+
+    Returns:
+        Aggregate data in a certain format for total notifications
+    """
+    facts_notification_start_time = get_query_date_based_on_retention_period(limit_days)
+    stats_from_facts = _stats_for_days_facts(service_id, facts_notification_start_time, by_template, notification_type)
+
+    start_time_notify_table = _timing_notification_table(service_id)
+    stats_for_today = _stats_for_today(service_id, start_time_notify_table, by_template, notification_type)
 
-    all_stats_table = stats_for_7_days.union_all(stats_for_today).subquery()
+    all_stats_table = stats_from_facts.union_all(stats_for_today).subquery()
 
     query = db.session.query(
         *(