Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

Add admin API to reset connection timeouts for remote server #11639

Merged
merged 19 commits into from
Jan 25, 2022
Merged
Show file tree
Hide file tree
Changes from 17 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/11639.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Add admin API to reset connection timeouts for remote server.
40 changes: 39 additions & 1 deletion docs/usage/administration/admin_api/federation.md
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ The following fields are returned in the JSON response body:
- `next_token`: string representing a positive integer - Indication for pagination. See above.
- `total` - integer - Total number of destinations.

# Destination Details API
## Destination Details API

This API gets the retry timing info for a specific remote server.

Expand All @@ -108,7 +108,45 @@ A response body like the following is returned:
}
```

**Parameters**

The following parameters should be set in the URL:

- `destination` - Name of the remote server.

**Response**

The response fields are the same like in the `destinations` array in
[List of destinations](#list-of-destinations) response.

## Reset connection timeout

Synapse makes federation requests to other homeservers. If a federation request fails,
Synapse will marks the destination homeserver as offline, preventing any future requests
dklimpel marked this conversation as resolved.
Show resolved Hide resolved
to that server for a "cooldown" period. This period grows over time if the server
continues to fail its responses
([exponential backoff](https://en.wikipedia.org/wiki/Exponential_backoff)).

Admins can cancel the cooldown period with this API.

This API resets the retry timing for a specific remote server and tries to connect
dklimpel marked this conversation as resolved.
Show resolved Hide resolved
the remote server again. It does not wait for the next `retry_interval`.
DMRobertson marked this conversation as resolved.
Show resolved Hide resolved
The connection must have previously run into an error and `retry_last_ts`
([Destination Details API](#destination-details-api)) must not be equal to `0`.
DMRobertson marked this conversation as resolved.
Show resolved Hide resolved

The connection attempt is carried out in the background and can take a while
even if the API already returns the http status 200.

The API is:

```
POST /_synapse/admin/v1/federation/destinations/<destination>/reset_connection

{}
```

**Parameters**

The following parameters should be set in the URL:

- `destination` - Name of the remote server.
16 changes: 9 additions & 7 deletions synapse/federation/transport/server/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from typing import Dict, Iterable, List, Optional, Tuple, Type
from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple, Type

from typing_extensions import Literal

Expand All @@ -36,17 +36,19 @@
parse_integer_from_args,
parse_string_from_args,
)
from synapse.server import HomeServer
from synapse.types import JsonDict, ThirdPartyInstanceID
from synapse.util.ratelimitutils import FederationRateLimiter

if TYPE_CHECKING:
from synapse.server import HomeServer

logger = logging.getLogger(__name__)


class TransportLayerServer(JsonResource):
"""Handles incoming federation HTTP requests"""

def __init__(self, hs: HomeServer, servlet_groups: Optional[List[str]] = None):
def __init__(self, hs: "HomeServer", servlet_groups: Optional[List[str]] = None):
"""Initialize the TransportLayerServer

Will by default register all servlets. For custom behaviour, pass in
Expand Down Expand Up @@ -113,7 +115,7 @@ class PublicRoomList(BaseFederationServlet):

def __init__(
self,
hs: HomeServer,
hs: "HomeServer",
authenticator: Authenticator,
ratelimiter: FederationRateLimiter,
server_name: str,
Expand Down Expand Up @@ -203,7 +205,7 @@ class FederationGroupsRenewAttestaionServlet(BaseFederationServlet):

def __init__(
self,
hs: HomeServer,
hs: "HomeServer",
authenticator: Authenticator,
ratelimiter: FederationRateLimiter,
server_name: str,
Expand Down Expand Up @@ -251,7 +253,7 @@ class OpenIdUserInfo(BaseFederationServlet):

def __init__(
self,
hs: HomeServer,
hs: "HomeServer",
authenticator: Authenticator,
ratelimiter: FederationRateLimiter,
server_name: str,
Expand Down Expand Up @@ -297,7 +299,7 @@ async def on_GET(


def register_servlets(
hs: HomeServer,
hs: "HomeServer",
resource: HttpServer,
authenticator: Authenticator,
ratelimiter: FederationRateLimiter,
Expand Down
14 changes: 8 additions & 6 deletions synapse/federation/transport/server/_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
import functools
import logging
import re
from typing import Any, Awaitable, Callable, Optional, Tuple, cast
from typing import TYPE_CHECKING, Any, Awaitable, Callable, Optional, Tuple, cast

from synapse.api.errors import Codes, FederationDeniedError, SynapseError
from synapse.api.urls import FEDERATION_V1_PREFIX
Expand All @@ -29,11 +29,13 @@
start_active_span_follows_from,
whitelisted_homeserver,
)
from synapse.server import HomeServer
from synapse.types import JsonDict
from synapse.util.ratelimitutils import FederationRateLimiter
from synapse.util.stringutils import parse_and_validate_server_name

if TYPE_CHECKING:
from synapse.server import HomeServer

logger = logging.getLogger(__name__)


Expand All @@ -46,7 +48,7 @@ class NoAuthenticationError(AuthenticationError):


class Authenticator:
def __init__(self, hs: HomeServer):
def __init__(self, hs: "HomeServer"):
self._clock = hs.get_clock()
self.keyring = hs.get_keyring()
self.server_name = hs.hostname
Expand Down Expand Up @@ -114,11 +116,11 @@ async def authenticate_request(
# alive
retry_timings = await self.store.get_destination_retry_timings(origin)
if retry_timings and retry_timings.retry_last_ts:
run_in_background(self._reset_retry_timings, origin)
run_in_background(self.reset_retry_timings, origin)

return origin

async def _reset_retry_timings(self, origin: str) -> None:
async def reset_retry_timings(self, origin: str) -> None:
try:
logger.info("Marking origin %r as up", origin)
await self.store.set_destination_retry_timings(origin, None, 0, 0)
Expand Down Expand Up @@ -227,7 +229,7 @@ class BaseFederationServlet:

def __init__(
self,
hs: HomeServer,
hs: "HomeServer",
authenticator: Authenticator,
ratelimiter: FederationRateLimiter,
server_name: str,
Expand Down
24 changes: 18 additions & 6 deletions synapse/federation/transport/server/federation.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from typing import Dict, List, Mapping, Optional, Sequence, Tuple, Type, Union
from typing import (
TYPE_CHECKING,
Dict,
List,
Mapping,
Optional,
Sequence,
Tuple,
Type,
Union,
)

from typing_extensions import Literal

Expand All @@ -30,11 +40,13 @@
parse_string_from_args,
parse_strings_from_args,
)
from synapse.server import HomeServer
from synapse.types import JsonDict
from synapse.util.ratelimitutils import FederationRateLimiter
from synapse.util.versionstring import get_version_string

if TYPE_CHECKING:
from synapse.server import HomeServer

logger = logging.getLogger(__name__)
issue_8631_logger = logging.getLogger("synapse.8631_debug")

Expand All @@ -47,7 +59,7 @@ class BaseFederationServerServlet(BaseFederationServlet):

def __init__(
self,
hs: HomeServer,
hs: "HomeServer",
authenticator: Authenticator,
ratelimiter: FederationRateLimiter,
server_name: str,
Expand Down Expand Up @@ -596,7 +608,7 @@ class FederationSpaceSummaryServlet(BaseFederationServlet):

def __init__(
self,
hs: HomeServer,
hs: "HomeServer",
authenticator: Authenticator,
ratelimiter: FederationRateLimiter,
server_name: str,
Expand Down Expand Up @@ -670,7 +682,7 @@ class FederationRoomHierarchyServlet(BaseFederationServlet):

def __init__(
self,
hs: HomeServer,
hs: "HomeServer",
authenticator: Authenticator,
ratelimiter: FederationRateLimiter,
server_name: str,
Expand Down Expand Up @@ -706,7 +718,7 @@ class RoomComplexityServlet(BaseFederationServlet):

def __init__(
self,
hs: HomeServer,
hs: "HomeServer",
authenticator: Authenticator,
ratelimiter: FederationRateLimiter,
server_name: str,
Expand Down
8 changes: 5 additions & 3 deletions synapse/federation/transport/server/groups_local.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,20 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import Dict, List, Tuple, Type
from typing import TYPE_CHECKING, Dict, List, Tuple, Type

from synapse.api.errors import SynapseError
from synapse.federation.transport.server._base import (
Authenticator,
BaseFederationServlet,
)
from synapse.handlers.groups_local import GroupsLocalHandler
from synapse.server import HomeServer
from synapse.types import JsonDict, get_domain_from_id
from synapse.util.ratelimitutils import FederationRateLimiter

if TYPE_CHECKING:
from synapse.server import HomeServer


class BaseGroupsLocalServlet(BaseFederationServlet):
"""Abstract base class for federation servlet classes which provides a groups local handler.
Expand All @@ -32,7 +34,7 @@ class BaseGroupsLocalServlet(BaseFederationServlet):

def __init__(
self,
hs: HomeServer,
hs: "HomeServer",
authenticator: Authenticator,
ratelimiter: FederationRateLimiter,
server_name: str,
Expand Down
8 changes: 5 additions & 3 deletions synapse/federation/transport/server/groups_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import Dict, List, Tuple, Type
from typing import TYPE_CHECKING, Dict, List, Tuple, Type

from typing_extensions import Literal

Expand All @@ -22,10 +22,12 @@
BaseFederationServlet,
)
from synapse.http.servlet import parse_string_from_args
from synapse.server import HomeServer
from synapse.types import JsonDict, get_domain_from_id
from synapse.util.ratelimitutils import FederationRateLimiter

if TYPE_CHECKING:
from synapse.server import HomeServer


class BaseGroupsServerServlet(BaseFederationServlet):
"""Abstract base class for federation servlet classes which provides a groups server handler.
Expand All @@ -35,7 +37,7 @@ class BaseGroupsServerServlet(BaseFederationServlet):

def __init__(
self,
hs: HomeServer,
hs: "HomeServer",
authenticator: Authenticator,
ratelimiter: FederationRateLimiter,
server_name: str,
Expand Down
6 changes: 4 additions & 2 deletions synapse/rest/admin/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@
EventReportsRestServlet,
)
from synapse.rest.admin.federation import (
DestinationsRestServlet,
DestinationResetConnectionRestServlet,
DestinationRestServlet,
ListDestinationsRestServlet,
)
from synapse.rest.admin.groups import DeleteGroupAdminRestServlet
Expand Down Expand Up @@ -267,7 +268,8 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None:
ListRegistrationTokensRestServlet(hs).register(http_server)
NewRegistrationTokenRestServlet(hs).register(http_server)
RegistrationTokenRestServlet(hs).register(http_server)
DestinationsRestServlet(hs).register(http_server)
DestinationResetConnectionRestServlet(hs).register(http_server)
DestinationRestServlet(hs).register(http_server)
ListDestinationsRestServlet(hs).register(http_server)

# Some servlets only get registered for the main process.
Expand Down
44 changes: 43 additions & 1 deletion synapse/rest/admin/federation.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from typing import TYPE_CHECKING, Tuple

from synapse.api.errors import Codes, NotFoundError, SynapseError
from synapse.federation.transport.server._base import Authenticator
dklimpel marked this conversation as resolved.
Show resolved Hide resolved
from synapse.http.servlet import RestServlet, parse_integer, parse_string
from synapse.http.site import SynapseRequest
from synapse.rest.admin._base import admin_patterns, assert_requester_is_admin
Expand Down Expand Up @@ -90,7 +91,7 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
return HTTPStatus.OK, response


class DestinationsRestServlet(RestServlet):
class DestinationRestServlet(RestServlet):
"""Get details of a destination.
This needs user to have administrator access in Synapse.

Expand Down Expand Up @@ -145,3 +146,44 @@ async def on_GET(
}

return HTTPStatus.OK, response


class DestinationResetConnectionRestServlet(RestServlet):
"""Reset destinations' connection timeouts and wake it up.
This needs user to have administrator access in Synapse.

POST /_synapse/admin/v1/federation/destinations/<destination>/reset_connection
{}

returns:
200 OK otherwise an error.
"""

PATTERNS = admin_patterns(
"/federation/destinations/(?P<destination>[^/]+)/reset_connection$"
)

def __init__(self, hs: "HomeServer"):
self._auth = hs.get_auth()
self._store = hs.get_datastore()
self._authenticator = Authenticator(hs)

async def on_POST(
self, request: SynapseRequest, destination: str
) -> Tuple[int, JsonDict]:
await assert_requester_is_admin(self._auth, request)
DMRobertson marked this conversation as resolved.
Show resolved Hide resolved

if not await self._store.is_destination_known(destination):
raise NotFoundError("Unknown destination")

retry_timings = await self._store.get_destination_retry_timings(destination)
if not (retry_timings and retry_timings.retry_last_ts):
raise SynapseError(
HTTPStatus.BAD_REQUEST,
"The retry timing does not need to be reset for this destination.",
)

# reset timings and wake up
await self._authenticator.reset_retry_timings(destination)

return HTTPStatus.OK, {}
Loading