Bugfix/incident-endpoint-performance

src/dispatch/case/models.py

@@ -183,7 +183,7 @@ class CaseCreate(CaseBase):
     tags: Optional[List[TagRead]] = []
 
 
-class CaseReadNested(CaseBase):
+class CaseReadReadMinimal(CaseBase):
     id: PrimaryKey
     assignee: Optional[UserRead]
     case_priority: CasePriorityRead
@@ -208,14 +208,14 @@ class CaseRead(CaseBase):
     closed_at: Optional[datetime] = None
     created_at: Optional[datetime] = None
     documents: Optional[List[DocumentRead]] = []
-    duplicates: Optional[List[CaseReadNested]] = []
+    duplicates: Optional[List[CaseReadReadMinimal]] = []
     escalated_at: Optional[datetime] = None
     events: Optional[List[EventRead]] = []
     groups: Optional[List[GroupRead]] = []
     incidents: Optional[List[IncidentRead]] = []
     name: Optional[NameStr]
     project: ProjectRead
-    related: Optional[List[CaseReadNested]] = []
+    related: Optional[List[CaseReadReadMinimal]] = []
     reported_at: Optional[datetime] = None
     storage: Optional[StorageRead] = None
     tags: Optional[List[TagRead]] = []
@@ -255,7 +255,7 @@ def find_exclusive(cls, v):
 
 
 class CasePagination(DispatchBase):
-    items: List[CaseRead] = []
+    items: List[CaseReadReadMinimal] = []
     itemsPerPage: int
     page: int
     total: int

src/dispatch/conference/models.py

@@ -39,7 +39,3 @@ def set_description(cls, v, values):
         return Template(INCIDENT_CONFERENCE_DESCRIPTION).render(
             conference_challenge=values["conference_challenge"]
         )
-
-
-class ConferenceNested(ConferenceBase):
-    pass

src/dispatch/conversation/models.py

@@ -35,7 +35,3 @@ class ConversationRead(ConversationBase):
     def set_description(cls, v):
         """Sets the description"""
         return INCIDENT_CONVERSATION_DESCRIPTION
-
-
-class ConversationNested(ConversationBase):
-    pass

src/dispatch/data/source/models.py

@@ -90,7 +90,7 @@ class Source(Base, TimeStampMixin, ProjectMixin):
     search_vector = Column(TSVectorType("name", regconfig="pg_catalog.simple"))
 
 
-class QueryReadNested(DispatchBase):
+class QueryReadMinimal(DispatchBase):
     id: PrimaryKey
     name: str
     description: str
@@ -119,7 +119,7 @@ class SourceBase(DispatchBase):
     links: Optional[List] = []
     tags: Optional[List[TagRead]] = []
     incidents: Optional[List[IncidentRead]] = []
-    queries: Optional[List[QueryReadNested]] = []
+    queries: Optional[List[QueryReadMinimal]] = []
     alerts: Optional[List[AlertRead]] = []
     cost: Optional[float]
     owner: Optional[ServiceRead] = Field(None, nullable=True)

src/dispatch/database/core.py

@@ -1,12 +1,15 @@
-import re
 import functools
+import re
 from typing import Any
-from pydantic.error_wrappers import ErrorWrapper, ValidationError
-from pydantic import BaseModel
+import time
+import logging
 
-from sqlalchemy import create_engine, inspect
+from pydantic import BaseModel
+from pydantic.error_wrappers import ErrorWrapper, ValidationError
+from sqlalchemy import create_engine, event, inspect
+from sqlalchemy.engine import Engine
 from sqlalchemy.ext.declarative import declarative_base, declared_attr
-from sqlalchemy.orm import sessionmaker, object_session
+from sqlalchemy.orm import object_session, sessionmaker
 from sqlalchemy.sql.expression import true
 from sqlalchemy_utils import get_mapper
 from starlette.requests import Request
@@ -15,12 +18,32 @@
 from dispatch.exceptions import NotFoundError
 from dispatch.search.fulltext import make_searchable
 
-
 engine = create_engine(
     config.SQLALCHEMY_DATABASE_URI,
     pool_size=config.DATABASE_ENGINE_POOL_SIZE,
     max_overflow=config.DATABASE_ENGINE_MAX_OVERFLOW,
 )
+
+
+# Useful for identifying slow or n + 1 queries. But doesn't need to be enabled in production.
+# logging.basicConfig()
+# logger = logging.getLogger(__name__)
+# logger.setLevel(logging.DEBUG)
+
+
+# @event.listens_for(Engine, "before_cursor_execute")
+# def before_cursor_execute(conn, cursor, statement, parameters, context, executemany):
+#    conn.info.setdefault("query_start_time", []).append(time.time())
+#    logger.debug("Start Query: %s", statement)
+
+
+# @event.listens_for(Engine, "after_cursor_execute")
+# def after_cursor_execute(conn, cursor, statement, parameters, context, executemany):
+#    total = time.time() - conn.info["query_start_time"].pop(-1)
+#    logger.debug("Query Complete!")
+#    logger.debug("Total Time: %f", total)
+
+
 SessionLocal = sessionmaker(bind=engine)
 
 

src/dispatch/document/models.py

@@ -73,10 +73,6 @@ def set_description(cls, v, values):
         return v
 
 
-class DocumentNested(DocumentRead):
-    pass
-
-
 class DocumentPagination(DispatchBase):
     total: int
     items: List[DocumentRead] = []

src/dispatch/event/models.py

@@ -58,7 +58,3 @@ class EventUpdate(EventBase):
 
 class EventRead(EventBase):
     pass
-
-
-class EventNested(EventBase):
-    id: PrimaryKey

src/dispatch/feedback/models.py

@@ -8,7 +8,7 @@
 
 from dispatch.database.core import Base
 from dispatch.feedback.enums import FeedbackRating
-from dispatch.incident.models import IncidentReadNested
+from dispatch.incident.models import IncidentReadMinimal
 from dispatch.models import DispatchBase, TimeStampMixin, PrimaryKey
 from dispatch.participant.models import ParticipantRead
 from dispatch.project.models import ProjectRead
@@ -42,7 +42,7 @@ class FeedbackBase(DispatchBase):
     created_at: Optional[datetime]
     rating: FeedbackRating = FeedbackRating.very_satisfied
     feedback: Optional[str] = Field(None, nullable=True)
-    incident: Optional[IncidentReadNested]
+    incident: Optional[IncidentReadMinimal]
     participant: Optional[ParticipantRead]
 
 

src/dispatch/group/models.py

@@ -41,7 +41,3 @@ class GroupRead(GroupBase):
     def set_description(cls, v):
         """Sets the description"""
         return TACTICAL_GROUP_DESCRIPTION
-
-
-class GroupNested(GroupBase):
-    id: PrimaryKey

src/dispatch/incident/models.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 from collections import Counter, defaultdict
-from typing import List, Optional
+from typing import List, Optional, Any
 
 from pydantic import validator
 from sqlalchemy import (
@@ -28,23 +28,30 @@
     IncidentPriorityBase,
     IncidentPriorityCreate,
     IncidentPriorityRead,
+    IncidentPriorityReadMinimal,
 )
 from dispatch.incident.severity.models import (
     IncidentSeverityCreate,
     IncidentSeverityRead,
+    IncidentSeverityReadMinimal,
     IncidentSeverityBase,
 )
-from dispatch.incident.type.models import IncidentTypeCreate, IncidentTypeRead, IncidentTypeBase
+from dispatch.incident.type.models import (
+    IncidentTypeCreate,
+    IncidentTypeRead,
+    IncidentTypeReadMinimal,
+    IncidentTypeBase,
+)
 from dispatch.incident_cost.models import IncidentCostRead, IncidentCostUpdate
 from dispatch.messaging.strings import INCIDENT_RESOLUTION_DEFAULT
 from dispatch.models import DispatchBase, ProjectMixin, TimeStampMixin
 from dispatch.models import NameStr, PrimaryKey
 from dispatch.participant.models import Participant
-from dispatch.participant.models import ParticipantRead, ParticipantUpdate
+from dispatch.participant.models import ParticipantRead, ParticipantReadMinimal, ParticipantUpdate
 from dispatch.report.enums import ReportTypes
 from dispatch.report.models import ReportRead
 from dispatch.storage.models import StorageRead
-from dispatch.tag.models import TagRead
+from dispatch.tag.models import TagRead, TagReadMinimal
 from dispatch.term.models import TermRead
 from dispatch.ticket.models import TicketRead
 from dispatch.workflow.models import WorkflowInstanceRead
@@ -124,6 +131,7 @@ def last_executive_report(self):
         "IncidentCost",
         backref="incident",
         cascade="all, delete-orphan",
+        lazy="subquery",
         order_by="IncidentCost.created_at",
     )
 
@@ -166,6 +174,7 @@ def last_executive_report(self):
     tags = relationship(
         "Tag",
         secondary=assoc_incident_tags,
+        lazy="subquery",
         backref="incidents",
     )
     tasks = relationship("Task", backref="incident", cascade="all, delete-orphan")
@@ -176,13 +185,19 @@ def last_executive_report(self):
     )
 
     duplicate_id = Column(Integer, ForeignKey("incident.id"))
-    duplicates = relationship("Incident", remote_side=[id], uselist=True)
+    duplicates = relationship(
+        "Incident", remote_side=[id], lazy="joined", join_depth=2, uselist=True
+    )
 
     commander_id = Column(Integer, ForeignKey("participant.id"))
-    commander = relationship("Participant", foreign_keys=[commander_id], post_update=True)
+    commander = relationship(
+        "Participant", foreign_keys=[commander_id], lazy="subquery", post_update=True
+    )
 
     reporter_id = Column(Integer, ForeignKey("participant.id"))
-    reporter = relationship("Participant", foreign_keys=[reporter_id], post_update=True)
+    reporter = relationship(
+        "Participant", foreign_keys=[reporter_id], lazy="subquery", post_update=True
+    )
 
     liaison_id = Column(Integer, ForeignKey("participant.id"))
     liaison = relationship("Participant", foreign_keys=[liaison_id], post_update=True)
@@ -248,21 +263,6 @@ def description_required(cls, v):
         return v
 
 
-class IncidentReadNested(IncidentBase):
-    id: PrimaryKey
-    closed_at: Optional[datetime] = None
-    commander: Optional[ParticipantRead]
-    created_at: Optional[datetime] = None
-    incident_priority: IncidentPriorityRead
-    incident_severity: IncidentSeverityRead
-    incident_type: IncidentTypeRead
-    name: Optional[NameStr]
-    project: ProjectRead
-    reported_at: Optional[datetime] = None
-    reporter: Optional[ParticipantRead]
-    stable_at: Optional[datetime] = None
-
-
 class IncidentCreate(IncidentBase):
     commander: Optional[ParticipantUpdate]
     incident_priority: Optional[IncidentPriorityCreate]
@@ -273,10 +273,33 @@ class IncidentCreate(IncidentBase):
     tags: Optional[List[TagRead]] = []
 
 
+class IncidentReadMinimal(IncidentBase):
+    id: PrimaryKey
+    closed_at: Optional[datetime] = None
+    commander: Optional[ParticipantReadMinimal]
+    commanders_location: Optional[str]
+    created_at: Optional[datetime] = None
+    incident_priority: IncidentPriorityReadMinimal
+    incident_severity: IncidentSeverityReadMinimal
+    incident_type: IncidentTypeReadMinimal
+    name: Optional[NameStr]
+    participants_location: Optional[str]
+    participants_team: Optional[str]
+    project: ProjectRead
+    reported_at: Optional[datetime] = None
+    reporter: Optional[ParticipantReadMinimal]
+    reporters_location: Optional[str]
+    stable_at: Optional[datetime] = None
+    tags: Optional[List[TagReadMinimal]] = []
+    total_cost: Optional[float]
+    duplicates: Optional[List[Any]] = []
+    incident_costs: Optional[List[IncidentCostRead]] = []
+
+
 class IncidentUpdate(IncidentBase):
     cases: Optional[List[CaseRead]] = []
     commander: Optional[ParticipantUpdate]
-    duplicates: Optional[List[IncidentReadNested]] = []
+    duplicates: Optional[List[IncidentReadMinimal]] = []
     incident_costs: Optional[List[IncidentCostUpdate]] = []
     incident_priority: IncidentPriorityBase
     incident_severity: IncidentSeverityBase
@@ -303,30 +326,6 @@ def find_exclusive(cls, v):
         return v
 
 
-class IncidentReadMinimal(IncidentBase):
-    id: PrimaryKey
-    cases: Optional[List[CaseRead]]
-    closed_at: Optional[datetime] = None
-    commander: Optional[ParticipantRead]
-    commanders_location: Optional[str]
-    created_at: Optional[datetime] = None
-    duplicates: Optional[List[IncidentReadNested]] = []
-    incident_costs: Optional[List[IncidentCostRead]] = []
-    incident_priority: IncidentPriorityRead
-    incident_severity: IncidentSeverityRead
-    incident_type: IncidentTypeRead
-    name: Optional[NameStr]
-    participants_location: Optional[str]
-    participants_team: Optional[str]
-    project: ProjectRead
-    reported_at: Optional[datetime] = None
-    reporter: Optional[ParticipantRead]
-    reporters_location: Optional[str]
-    stable_at: Optional[datetime] = None
-    tags: Optional[List[TagRead]] = []
-    total_cost: Optional[float]
-
-
 class IncidentRead(IncidentReadMinimal):
     conference: Optional[ConferenceRead] = None
     conversation: Optional[ConversationRead] = None
@@ -345,4 +344,4 @@ class IncidentPagination(DispatchBase):
     total: int
     itemsPerPage: int
     page: int
-    items: List[IncidentRead] = []
+    items: List[IncidentReadMinimal] = []

src/dispatch/incident/priority/models.py

@@ -62,6 +62,19 @@ class IncidentPriorityRead(IncidentPriorityBase):
     id: PrimaryKey
 
 
+class IncidentPriorityReadMinimal(DispatchBase):
+    id: PrimaryKey
+    name: NameStr
+    description: Optional[str] = Field(None, nullable=True)
+    page_commander: Optional[StrictBool]
+    tactical_report_reminder: Optional[int]
+    executive_report_reminder: Optional[int]
+    default: Optional[bool]
+    enabled: Optional[bool]
+    view_order: Optional[int]
+    color: Optional[Color] = Field(None, nullable=True)
+
+
 class IncidentPriorityPagination(DispatchBase):
     total: int
-    items: List[IncidentPriorityRead] = []
+    items: List[IncidentPriorityReadMinimal] = []

src/dispatch/incident/severity/models.py

@@ -60,6 +60,15 @@ class IncidentSeverityRead(IncidentSeverityBase):
     id: PrimaryKey
 
 
+class IncidentSeverityReadMinimal(DispatchBase):
+    id: PrimaryKey
+    color: Optional[Color] = Field(None, nullable=True)
+    default: Optional[bool]
+    description: Optional[str] = Field(None, nullable=True)
+    enabled: Optional[bool]
+    name: NameStr
+
+
 class IncidentSeverityPagination(DispatchBase):
     total: int
-    items: List[IncidentSeverityRead] = []
+    items: List[IncidentSeverityReadMinimal] = []

src/dispatch/incident/type/models.py

@@ -111,6 +111,16 @@ class IncidentTypeRead(IncidentTypeBase):
     id: PrimaryKey
 
 
+class IncidentTypeReadMinimal(DispatchBase):
+    id: PrimaryKey
+    name: NameStr
+    visibility: Optional[str] = Field(None, nullable=True)
+    description: Optional[str] = Field(None, nullable=True)
+    enabled: Optional[bool]
+    exclude_from_metrics: Optional[bool] = False
+    default: Optional[bool] = False
+
+
 class IncidentTypePagination(DispatchBase):
     total: int
-    items: List[IncidentTypeRead] = []
+    items: List[IncidentTypeReadMinimal] = []