diff --git a/changelog.d/5103.bugfix b/changelog.d/5103.bugfix
new file mode 100644
index 000000000000..590d80d58f76
--- /dev/null
+++ b/changelog.d/5103.bugfix
@@ -0,0 +1 @@
+Fix bug where presence updates were sent to all servers in a room when a new server joined, rather than to just the new server.
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index bd1285b15c5e..59d53f105059 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -828,6 +828,11 @@ def _handle_state_delta(self, deltas):
             if typ != EventTypes.Member:
                 continue
 
+            if event_id is None:
+                # state has been deleted, so this is not a join. We only care about
+                # joins.
+                continue
+
             event = yield self.store.get_event(event_id)
             if event.content.get("membership") != Membership.JOIN:
                 # We only care about joins
diff --git a/synapse/storage/state_deltas.py b/synapse/storage/state_deltas.py
index 56e42f583db4..31a0279b18e6 100644
--- a/synapse/storage/state_deltas.py
+++ b/synapse/storage/state_deltas.py
@@ -22,6 +22,24 @@
 
 class StateDeltasStore(SQLBaseStore):
     def get_current_state_deltas(self, prev_stream_id):
+        """Fetch a list of room state changes since the given stream id
+
+        Each entry in the result contains the following fields:
+            - stream_id (int)
+            - room_id (str)
+            - type (str): event type
+            - state_key (str):
+            - event_id (str|None): new event_id for this state key. None if the
+                state has been deleted.
+            - prev_event_id (str|None): previous event_id for this state key. None
+                if it's new state.
+
+        Args:
+            prev_stream_id (int): point to get changes since (exclusive)
+
+        Returns:
+            Deferred[list[dict]]: results
+        """
         prev_stream_id = int(prev_stream_id)
         if not self._curr_state_delta_stream_cache.has_any_entity_changed(
             prev_stream_id