Show department last updated dates (#214)

* Add Department last updated dates

* Refactor repetitive test code

* Show date_updated for officers, incidents, and assignments

* Update cache TTL to 12 hours

OpenOversight/app/models.py

@@ -1,9 +1,12 @@
+import datetime
 import re
 import time
 from datetime import date
 from decimal import Decimal
 
 from authlib.jose import JoseError, JsonWebToken
+from cachetools import TTLCache, cached
+from cachetools.keys import hashkey
 from flask import current_app
 from flask_login import UserMixin
 from flask_sqlalchemy import SQLAlchemy
@@ -39,6 +42,27 @@
 )
 
 
+date_updated_cache = TTLCache(maxsize=1024, ttl=12 * 60 * 60)
+
+
+def _date_updated_cache_key(update_type: str):
+    """Return a key function to calculate the cache key for Department
+    `latest_*_update` methods using the department id and a given update type.
+
+    Department.id is used instead of a Department obj because the default Python
+    __hash__ is unique per obj instance, meaning multiple instances of the same
+    department will have different hashes.
+
+    Update type is used in the hash to differentiate between the (currently) three
+    update types we compute per department.
+    """
+
+    def _cache_key(dept: "Department"):
+        return hashkey(dept.id, update_type)
+
+    return _cache_key
+
+
 class Department(BaseModel):
     __tablename__ = "departments"
     id = db.Column(db.Integer, primary_key=True)
@@ -59,6 +83,35 @@ def toCustomDict(self):
             "unique_internal_identifier_label": self.unique_internal_identifier_label,
         }
 
+    @cached(cache=date_updated_cache, key=_date_updated_cache_key("incident"))
+    def latest_incident_update(self) -> datetime.date:
+        incident_updated = (
+            db.session.query(func.max(Incident.date_updated))
+            .filter(Incident.department_id == self.id)
+            .scalar()
+        )
+        return incident_updated.date() if incident_updated else None
+
+    @cached(cache=date_updated_cache, key=_date_updated_cache_key("officer"))
+    def latest_officer_update(self) -> datetime.date:
+        officer_updated = (
+            db.session.query(func.max(Officer.date_updated))
+            .filter(Officer.department_id == self.id)
+            .scalar()
+        )
+        return officer_updated.date() if officer_updated else None
+
+    @cached(cache=date_updated_cache, key=_date_updated_cache_key("assignment"))
+    def latest_assignment_update(self) -> datetime.date:
+        assignment_updated = (
+            db.session.query(func.max(Assignment.date_updated))
+            .join(Officer)
+            .filter(Assignment.officer_id == Officer.id)
+            .filter(Officer.department_id == self.id)
+            .scalar()
+        )
+        return assignment_updated.date() if assignment_updated else None
+
 
 class Job(BaseModel):
     __tablename__ = "jobs"
@@ -129,6 +182,10 @@ class Officer(BaseModel):
     unique_internal_identifier = db.Column(
         db.String(50), index=True, unique=True, nullable=True
     )
+    date_created = db.Column(db.DateTime, default=func.now())
+    date_updated = db.Column(
+        db.DateTime, default=func.now(), onupdate=func.now(), index=True
+    )
 
     links = db.relationship(
         "Link", secondary=officer_links, backref=db.backref("officers", lazy=True)
@@ -280,6 +337,10 @@ class Assignment(BaseModel):
     unit = db.relationship("Unit")
     star_date = db.Column(db.Date, index=True, unique=False, nullable=True)
     resign_date = db.Column(db.Date, index=True, unique=False, nullable=True)
+    date_created = db.Column(db.DateTime, default=func.now())
+    date_updated = db.Column(
+        db.DateTime, default=func.now(), onupdate=func.now(), index=True
+    )
 
     def __repr__(self):
         return "<Assignment: ID {} : {}>".format(self.officer_id, self.star_no)
@@ -516,6 +577,10 @@ class Incident(BaseModel):
     last_updated_by = db.relationship(
         "User", backref="incidents_updated", lazy=True, foreign_keys=[last_updated_id]
     )
+    date_created = db.Column(db.DateTime, default=func.now())
+    date_updated = db.Column(
+        db.DateTime, default=func.now(), onupdate=func.now(), index=True
+    )
 
 
 class User(UserMixin, BaseModel):

OpenOversight/app/templates/browse.html

@@ -20,16 +20,27 @@ <h2>{{ department.name }}
             </a>
         {% endif %}
        </h2>
-          <p>
+
+       <div>
+         <i class="dept-{{department.id}}">Officers Updated: {{department.latest_officer_update() | default("No data", true) }}</i>
+         <br>
+         <i class="dept-{{department.id}}">Assignments Updated: {{department.latest_assignment_update() | default("No data", true) }}</i>
+         {% if department.incidents %}
+           <br>
+           <i class="dept-{{department.id}}">Incidents Updated: {{department.latest_incident_update() | default("No data", true) }}</i>
+         {% endif %}
+       </div>
+
+        <p>
           <a class="btn btn-lg btn-primary" href="{{ url_for('main.list_officer', department_id=department.id) }}">
-             Officers
+            Officers
           </a>
           {% if department.incidents %}
             <a class="btn btn-lg btn-primary" href="{{ url_for('main.incident_api', department_id=department.id) }}">
-               Incidents
+              Incidents
             </a>
+          {% endif %}
         </p>
-        {% endif %}
       </div>
     </p>
   {% endfor %}

OpenOversight/migrations/versions/8fc4d110de2c_.py

@@ -0,0 +1,69 @@
+"""Add date_created and date_updated to assignments, incidents, officers tables.
+
+Revision ID: 8fc4d110de2c
+Revises: cd39b33b5360
+Create Date: 2022-10-12 05:26:07.671962
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "8fc4d110de2c"
+down_revision = "cd39b33b5360"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "assignments", sa.Column("date_created", sa.DateTime(), nullable=True)
+    )
+    op.add_column(
+        "assignments", sa.Column("date_updated", sa.DateTime(), nullable=True)
+    )
+    op.create_index(
+        op.f("ix_assignments_date_updated"),
+        "assignments",
+        ["date_updated"],
+        unique=False,
+    )
+
+    op.add_column("incidents", sa.Column("date_created", sa.DateTime(), nullable=True))
+    op.add_column("incidents", sa.Column("date_updated", sa.DateTime(), nullable=True))
+    op.create_index(
+        op.f("ix_incidents_date_updated"), "incidents", ["date_updated"], unique=False
+    )
+
+    op.add_column("officers", sa.Column("date_created", sa.DateTime(), nullable=True))
+    op.add_column("officers", sa.Column("date_updated", sa.DateTime(), nullable=True))
+    op.create_index(
+        op.f("ix_officers_date_updated"), "officers", ["date_updated"], unique=False
+    )
+
+    op.execute(
+        """
+        update officers
+        set date_created='2021-12-28',
+            date_updated='2021-12-28';
+        """
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f("ix_officers_date_updated"), table_name="officers")
+    op.drop_column("officers", "date_updated")
+    op.drop_column("officers", "date_created")
+
+    op.drop_index(op.f("ix_incidents_date_updated"), table_name="incidents")
+    op.drop_column("incidents", "date_updated")
+    op.drop_column("incidents", "date_created")
+
+    op.drop_index(op.f("ix_assignments_date_updated"), table_name="assignments")
+    op.drop_column("assignments", "date_updated")
+    op.drop_column("assignments", "date_created")
+    # ### end Alembic commands ###

OpenOversight/tests/routes/test_officer_and_department.py

@@ -2486,3 +2486,24 @@ def test_ac_cannot_delete_link_from_officer_profile_not_in_their_dept(
         )
 
         assert rv.status_code == 403
+
+
+def test_browse_displays_last_updated_dates(mockdata, client, session):
+    dept = Department.query.first()
+
+    tag = f'<i class="dept-{dept.id}">'
+
+    with current_app.test_request_context():
+        session.execute(
+            Officer.__table__.update().values(date_updated=date(2022, 1, 1))
+        )
+        session.execute(Assignment.__table__.update().values(date_updated=None))
+        session.execute(
+            Incident.__table__.update().values(date_updated=date(2022, 1, 1))
+        )
+        session.commit()
+
+        rv = client.get(url_for("main.browse"))
+        assert tag + "Officers Updated: 2022-01-01" in rv.data.decode("utf-8")
+        assert tag + "Assignments Updated: No data" in rv.data.decode("utf-8")
+        assert tag + "Incidents Updated: 2022-01-01" in rv.data.decode("utf-8")

OpenOversight/tests/test_commands.py

@@ -209,6 +209,7 @@ def test_csv_import_update(csvfile):
 
     assert n_existing > 0
 
+    # Load officers
     n_created, n_updated = bulk_add_officers([csvfile], standalone_mode=False)
 
     assert n_created == 0

requirements.txt

@@ -1,5 +1,6 @@
 Authlib==0.15.5
 boto3==1.24.21
+cachetools==5.2.0
 email-validator==1.2.1
 Fabric3==1.14.post1
 Flask==2.2.2