Merge remote-tracking branch 'origin/develop' into clokep/gif-err

INSTALL.md

@@ -73,7 +73,7 @@ mkdir -p ~/synapse
 virtualenv -p python3 ~/synapse/env
 source ~/synapse/env/bin/activate
 pip install --upgrade pip
-pip install --upgrade setuptools!=50.0  # setuptools==50.0 fails on some older Python versions
+pip install --upgrade setuptools
 pip install matrix-synapse
 ```
 

changelog.d/8231.misc

@@ -0,0 +1 @@
+Refactor queries for device keys and cross-signatures.

changelog.d/8233.misc

@@ -0,0 +1 @@
+Refactor queries for device keys and cross-signatures.

changelog.d/8237.misc

@@ -0,0 +1 @@
+Fix type hints in `SyncHandler`.

changelog.d/8240.misc

@@ -0,0 +1 @@
+Fix type hints for functions decorated with `@cached`.

changelog.d/8241.misc

@@ -0,0 +1 @@
+Add type hints to `synapse.storage.database`.

changelog.d/8242.feature

@@ -0,0 +1 @@
+Back out experimental support for sharding event persister. **PLEASE REMOVE THIS LINE FROM THE FINAL CHANGELOG**

changelog.d/8244.misc

@@ -0,0 +1 @@
+Add type hints to pagination, initial sync and events handlers.

changelog.d/8245.misc

@@ -0,0 +1 @@
+Remove obsolete `order` field from federation send queues.

changelog.d/8249.misc

@@ -0,0 +1 @@
+Stop sub-classing from object.

contrib/cmdclient/http.py

@@ -24,7 +24,7 @@
 from twisted.web.http_headers import Headers
 
 
-class HttpClient(object):
+class HttpClient:
     """ Interface for talking json over http
     """
 
@@ -169,7 +169,7 @@ def sleep(self, seconds):
         return d
 
 
-class _RawProducer(object):
+class _RawProducer:
     def __init__(self, data):
         self.data = data
         self.body = data
@@ -186,7 +186,7 @@ def stopProducing(self):
         pass
 
 
-class _JsonProducer(object):
+class _JsonProducer:
     """ Used by the twisted http client to create the HTTP body from json
     """
 

contrib/experiments/cursesio.py

@@ -141,7 +141,7 @@ def close(self):
         curses.endwin()
 
 
-class Callback(object):
+class Callback:
     def __init__(self, stdio):
         self.stdio = stdio
 

contrib/experiments/test_messaging.py

@@ -55,7 +55,7 @@ def excpetion_errback(failure):
     logging.exception(failure)
 
 
-class InputOutput(object):
+class InputOutput:
     """ This is responsible for basic I/O so that a user can interact with
     the example app.
     """
@@ -132,7 +132,7 @@ def emit(self, record):
         self.io.print_log(msg)
 
 
-class Room(object):
+class Room:
     """ Used to store (in memory) the current membership state of a room, and
     which home servers we should send PDUs associated with the room to.
     """

mypy.ini

@@ -1,6 +1,6 @@
 [mypy]
 namespace_packages = True
-plugins = mypy_zope:plugin
+plugins = mypy_zope:plugin, scripts-dev/mypy_synapse_plugin.py
 follow_imports = silent
 check_untyped_defs = True
 show_error_codes = True
@@ -17,10 +17,13 @@ files =
   synapse/handlers/auth.py,
   synapse/handlers/cas_handler.py,
   synapse/handlers/directory.py,
+  synapse/handlers/events.py,
   synapse/handlers/federation.py,
   synapse/handlers/identity.py,
+  synapse/handlers/initial_sync.py,
   synapse/handlers/message.py,
   synapse/handlers/oidc_handler.py,
+  synapse/handlers/pagination.py,
   synapse/handlers/presence.py,
   synapse/handlers/room.py,
   synapse/handlers/room_member.py,
@@ -51,6 +54,7 @@ files =
   synapse/storage/util,
   synapse/streams,
   synapse/types.py,
+  synapse/util/caches/descriptors.py,
   synapse/util/caches/stream_change_cache.py,
   synapse/util/metrics.py,
   tests/replication,

scripts-dev/hash_history.py

@@ -15,7 +15,7 @@
 from synapse.storage.signatures import SignatureStore
 
 
-class Store(object):
+class Store:
     _get_pdu_tuples = PduStore.__dict__["_get_pdu_tuples"]
     _get_pdu_content_hashes_txn = SignatureStore.__dict__["_get_pdu_content_hashes_txn"]
     _get_prev_pdu_hashes_txn = SignatureStore.__dict__["_get_prev_pdu_hashes_txn"]

scripts-dev/mypy_synapse_plugin.py

@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This is a mypy plugin for Synpase to deal with some of the funky typing that
+can crop up, e.g the cache descriptors.
+"""
+
+from typing import Callable, Optional
+
+from mypy.plugin import MethodSigContext, Plugin
+from mypy.typeops import bind_self
+from mypy.types import CallableType
+
+
+class SynapsePlugin(Plugin):
+    def get_method_signature_hook(
+        self, fullname: str
+    ) -> Optional[Callable[[MethodSigContext], CallableType]]:
+        if fullname.startswith(
+            "synapse.util.caches.descriptors._CachedFunction.__call__"
+        ):
+            return cached_function_method_signature
+        return None
+
+
+def cached_function_method_signature(ctx: MethodSigContext) -> CallableType:
+    """Fixes the `_CachedFunction.__call__` signature to be correct.
+
+    It already has *almost* the correct signature, except:
+
+        1. the `self` argument needs to be marked as "bound"; and
+        2. any `cache_context` argument should be removed.
+    """
+
+    # First we mark this as a bound function signature.
+    signature = bind_self(ctx.default_signature)
+
+    # Secondly, we remove any "cache_context" args.
+    #
+    # Note: We should be only doing this if `cache_context=True` is set, but if
+    # it isn't then the code will raise an exception when its called anyway, so
+    # its not the end of the world.
+    context_arg_index = None
+    for idx, name in enumerate(signature.arg_names):
+        if name == "cache_context":
+            context_arg_index = idx
+            break
+
+    if context_arg_index:
+        arg_types = list(signature.arg_types)
+        arg_types.pop(context_arg_index)
+
+        arg_names = list(signature.arg_names)
+        arg_names.pop(context_arg_index)
+
+        arg_kinds = list(signature.arg_kinds)
+        arg_kinds.pop(context_arg_index)
+
+        signature = signature.copy_modified(
+            arg_types=arg_types, arg_names=arg_names, arg_kinds=arg_kinds,
+        )
+
+    return signature
+
+
+def plugin(version: str):
+    # This is the entry point of the plugin, and let's us deal with the fact
+    # that the mypy plugin interface is *not* stable by looking at the version
+    # string.
+    #
+    # However, since we pin the version of mypy Synapse uses in CI, we don't
+    # really care.
+    return SynapsePlugin

synapse/api/auth.py

@@ -58,7 +58,7 @@ class _InvalidMacaroonException(Exception):
     pass
 
 
-class Auth(object):
+class Auth:
     """
     FIXME: This class contains a mix of functions for authenticating users
     of our client-server API and authenticating events added to room graphs.

synapse/api/auth_blocking.py

@@ -22,7 +22,7 @@
 logger = logging.getLogger(__name__)
 
 
-class AuthBlocking(object):
+class AuthBlocking:
     def __init__(self, hs):
         self.store = hs.get_datastore()
 

synapse/api/constants.py

@@ -28,7 +28,7 @@
 MAX_USERID_LENGTH = 255
 
 
-class Membership(object):
+class Membership:
 
     """Represents the membership states of a user in a room."""
 
@@ -40,22 +40,22 @@ class Membership(object):
     LIST = (INVITE, JOIN, KNOCK, LEAVE, BAN)
 
 
-class PresenceState(object):
+class PresenceState:
     """Represents the presence state of a user."""
 
     OFFLINE = "offline"
     UNAVAILABLE = "unavailable"
     ONLINE = "online"
 
 
-class JoinRules(object):
+class JoinRules:
     PUBLIC = "public"
     KNOCK = "knock"
     INVITE = "invite"
     PRIVATE = "private"
 
 
-class LoginType(object):
+class LoginType:
     PASSWORD = "m.login.password"
     EMAIL_IDENTITY = "m.login.email.identity"
     MSISDN = "m.login.msisdn"
@@ -65,7 +65,7 @@ class LoginType(object):
     DUMMY = "m.login.dummy"
 
 
-class EventTypes(object):
+class EventTypes:
     Member = "m.room.member"
     Create = "m.room.create"
     Tombstone = "m.room.tombstone"
@@ -96,17 +96,17 @@ class EventTypes(object):
     Presence = "m.presence"
 
 
-class RejectedReason(object):
+class RejectedReason:
     AUTH_ERROR = "auth_error"
 
 
-class RoomCreationPreset(object):
+class RoomCreationPreset:
     PRIVATE_CHAT = "private_chat"
     PUBLIC_CHAT = "public_chat"
     TRUSTED_PRIVATE_CHAT = "trusted_private_chat"
 
 
-class ThirdPartyEntityKind(object):
+class ThirdPartyEntityKind:
     USER = "user"
     LOCATION = "location"
 
@@ -115,7 +115,7 @@ class ThirdPartyEntityKind(object):
 ServerNoticeLimitReached = "m.server_notice.usage_limit_reached"
 
 
-class UserTypes(object):
+class UserTypes:
     """Allows for user type specific behaviour. With the benefit of hindsight
     'admin' and 'guest' users should also be UserTypes. Normal users are type None
     """
@@ -125,7 +125,7 @@ class UserTypes(object):
     ALL_USER_TYPES = (SUPPORT, BOT)
 
 
-class RelationTypes(object):
+class RelationTypes:
     """The types of relations known to this server.
     """
 
@@ -134,14 +134,14 @@ class RelationTypes(object):
     REFERENCE = "m.reference"
 
 
-class LimitBlockingTypes(object):
+class LimitBlockingTypes:
     """Reasons that a server may be blocked"""
 
     MONTHLY_ACTIVE_USER = "monthly_active_user"
     HS_DISABLED = "hs_disabled"
 
 
-class EventContentFields(object):
+class EventContentFields:
     """Fields found in events' content, regardless of type."""
 
     # Labels for the event, cf https://github.com/matrix-org/matrix-doc/pull/2326
@@ -152,6 +152,6 @@ class EventContentFields(object):
     SELF_DESTRUCT_AFTER = "org.matrix.self_destruct_after"
 
 
-class RoomEncryptionAlgorithms(object):
+class RoomEncryptionAlgorithms:
     MEGOLM_V1_AES_SHA2 = "m.megolm.v1.aes-sha2"
     DEFAULT = MEGOLM_V1_AES_SHA2

synapse/api/errors.py

@@ -31,7 +31,7 @@
 logger = logging.getLogger(__name__)
 
 
-class Codes(object):
+class Codes:
     UNRECOGNIZED = "M_UNRECOGNIZED"
     UNAUTHORIZED = "M_UNAUTHORIZED"
     FORBIDDEN = "M_FORBIDDEN"

synapse/api/filtering.py

@@ -130,7 +130,7 @@ def matrix_user_id_validator(user_id_str):
     return UserID.from_string(user_id_str)
 
 
-class Filtering(object):
+class Filtering:
     def __init__(self, hs):
         super(Filtering, self).__init__()
         self.store = hs.get_datastore()
@@ -168,7 +168,7 @@ def check_valid_filter(self, user_filter_json):
             raise SynapseError(400, str(e))
 
 
-class FilterCollection(object):
+class FilterCollection:
     def __init__(self, filter_json):
         self._filter_json = filter_json
 
@@ -249,7 +249,7 @@ def blocks_all_room_timeline(self):
         )
 
 
-class Filter(object):
+class Filter:
     def __init__(self, filter_json):
         self.filter_json = filter_json
 

synapse/api/ratelimiting.py

@@ -21,7 +21,7 @@
 from synapse.util import Clock
 
 
-class Ratelimiter(object):
+class Ratelimiter:
     """
     Ratelimit actions marked by arbitrary keys.