From 852987863843efe302cef372a7fd52cca387d573 Mon Sep 17 00:00:00 2001 From: Marcus Hoffmann Date: Wed, 13 Jan 2021 21:25:27 +0100 Subject: [PATCH 01/10] allow specifying https:// proxy Signed-off-by: Marcus Hoffmann --- changelog.d/9119.feature | 1 + synapse/http/proxyagent.py | 59 +++++++++++---- tests/http/test_proxyagent.py | 130 ++++++++++++++++++++++++++++++++++ 3 files changed, 175 insertions(+), 15 deletions(-) create mode 100644 changelog.d/9119.feature diff --git a/changelog.d/9119.feature b/changelog.d/9119.feature new file mode 100644 index 000000000000..26e7a8eebc5b --- /dev/null +++ b/changelog.d/9119.feature @@ -0,0 +1 @@ +Add support for https connections to a proxy server. diff --git a/synapse/http/proxyagent.py b/synapse/http/proxyagent.py index 7dfae8b786b9..55712148bd15 100644 --- a/synapse/http/proxyagent.py +++ b/synapse/http/proxyagent.py @@ -22,6 +22,7 @@ from twisted.internet import defer from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS +from twisted.internet.interfaces import IReactorCore from twisted.python.failure import Failure from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _AgentBase from twisted.web.error import SchemeNotSupported @@ -121,11 +122,11 @@ def __init__( self.https_proxy_creds, https_proxy = parse_username_password(https_proxy) self.http_proxy_endpoint = _http_proxy_endpoint( - http_proxy, self.proxy_reactor, **self._endpoint_kwargs + http_proxy, self.proxy_reactor, contextFactory, **self._endpoint_kwargs ) self.https_proxy_endpoint = _http_proxy_endpoint( - https_proxy, self.proxy_reactor, **self._endpoint_kwargs + https_proxy, self.proxy_reactor, contextFactory, **self._endpoint_kwargs ) self.no_proxy = no_proxy @@ -243,7 +244,12 @@ def request(self, method, uri, headers=None, bodyProducer=None): ) -def _http_proxy_endpoint(proxy: Optional[bytes], reactor, **kwargs): +def _http_proxy_endpoint( + proxy: Optional[bytes], + reactor: IReactorCore, + tls_options_factory: Optional[IPolicyForHTTPS], + **kwargs, +): """Parses an http proxy setting and returns an endpoint for the proxy Args: @@ -253,18 +259,31 @@ def _http_proxy_endpoint(proxy: Optional[bytes], reactor, **kwargs): reactor: reactor to be used to connect to the proxy + tls_options_factory: the TLS options to use when connecting through a https proxy kwargs: other args to be passed to HostnameEndpoint Returns: interfaces.IStreamClientEndpoint|None: endpoint to use to connect to the proxy, or None + + Raises: ValueError if given a proxy with a scheme we don't support. """ if proxy is None: return None - # Parse the connection string - host, port = parse_host_port(proxy, default_port=1080) - return HostnameEndpoint(reactor, host, port, **kwargs) + # Note: we can't use urlsplit/urlparse as that is completely broken for things without a scheme:// + scheme, host, port = parse_proxy(proxy) + + if scheme not in (b"http", b"https"): + raise ValueError("Proxy scheme '{}' not supported".format(scheme.decode())) + + proxy_endpoint = HostnameEndpoint(reactor, host, port, **kwargs) + + if scheme == b"https": + tls_options = tls_options_factory.creatorForNetloc(host, port) + proxy_endpoint = wrapClientTLS(tls_options, proxy_endpoint) + + return proxy_endpoint def parse_username_password(proxy: bytes) -> Tuple[Optional[ProxyCredentials], bytes]: @@ -288,25 +307,35 @@ def parse_username_password(proxy: bytes) -> Tuple[Optional[ProxyCredentials], b return None, proxy -def parse_host_port(hostport: bytes, default_port: int = None) -> Tuple[bytes, int]: +def parse_proxy( + proxy: bytes, default_scheme: bytes = b"http", default_port: int = 1080 +) -> Tuple[bytes, bytes, int]: """ Parse the hostname and port from a proxy connection byte string. Args: - hostport: The proxy connection string. Must be in the form 'host[:port]'. - default_port: The default port to return if one is not found in `hostport`. + proxy: The proxy connection string. Must be in the form 'host[:port]'. + default_scheme: The default scheme to return if one is not found in `proxy`. + default_port: The default port to return if one is not found in `proxy`. Returns: - A tuple containing the hostname and port. Uses `default_port` if one was not found. + A tuple containing the scheme, hostname and port. """ - if b":" in hostport: - host, port = hostport.rsplit(b":", 1) + # First check if we have a scheme present + if b"://" in proxy: + scheme, host = proxy.split(b"://", 1) + else: + scheme, host = default_scheme, proxy + # Now check the leftover part for a port + if b":" in host: + new_host, port = host.rsplit(b":", 1) try: port = int(port) - return host, port + return scheme, new_host, port except ValueError: # the thing after the : wasn't a valid port; presumably this is an # IPv6 address. + # TODO: this doesn't work when the last part of the IP is also just a number. + # We probably need to require ipv6's being wrapped in square brackets: [2001:db8:0:0:1::1] pass - - return hostport, default_port + return scheme, host, default_port diff --git a/tests/http/test_proxyagent.py b/tests/http/test_proxyagent.py index fefc8099c9d2..a02670b3e6a8 100644 --- a/tests/http/test_proxyagent.py +++ b/tests/http/test_proxyagent.py @@ -21,10 +21,12 @@ from netaddr import IPSet from twisted.internet import interfaces # noqa: F401 +from twisted.internet.endpoints import HostnameEndpoint, _WrapperEndpoint from twisted.internet.protocol import Factory from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.web.http import HTTPChannel +from synapse.http.proxyagent import parse_proxy from synapse.http.client import BlacklistingReactorWrapper from synapse.http.proxyagent import ProxyAgent @@ -103,6 +105,104 @@ def _make_connection( return http_protocol + def test_parse_proxy_host_only(self): + url = b"localhost" + self.assertEqual((b"http", b"localhost", 1080), parse_proxy(url)) + + def test_parse_proxy_host_port(self): + url = b"localhost:9988" + self.assertEqual((b"http", b"localhost", 9988), parse_proxy(url)) + + def test_parse_proxy_scheme_host(self): + url = b"https://localhost" + self.assertEqual((b"https", b"localhost", 1080), parse_proxy(url)) + + def test_parse_proxy_scheme_host_port(self): + url = b"https://localhost:1234" + self.assertEqual((b"https", b"localhost", 1234), parse_proxy(url)) + + def test_parse_proxy_host_only_ipv4(self): + url = b"1.2.3.4" + self.assertEqual((b"http", b"1.2.3.4", 1080), parse_proxy(url)) + + def test_parse_proxy_host_port_ipv4(self): + url = b"1.2.3.4:9988" + self.assertEqual((b"http", b"1.2.3.4", 9988), parse_proxy(url)) + + def test_parse_proxy_scheme_host_ipv4(self): + url = b"https://1.2.3.4" + self.assertEqual((b"https", b"1.2.3.4", 1080), parse_proxy(url)) + + def test_parse_proxy_scheme_host_port_ipv4(self): + url = b"https://1.2.3.4:9988" + self.assertEqual((b"https", b"1.2.3.4", 9988), parse_proxy(url)) + + def test_parse_proxy_host_ipv6(self): + url = b"2001:0db8:85a3:0000:0000:8a2e:0370:effe" + self.assertEqual( + (b"http", b"2001:0db8:85a3:0000:0000:8a2e:0370:effe", 1080), + parse_proxy(url), + ) + + # currently broken + url = b"2001:0db8:85a3:0000:0000:8a2e:0370:1234" + # self.assertEqual((b"http", b"2001:0db8:85a3:0000:0000:8a2e:0370:1234", 1080), parse_proxy(url)) + + # also broken + url = b"::1" + # self.assertEqual((b"http", b"::1", 1080), parse_proxy(url)) + url = b"::ffff:0.0.0.0" + self.assertEqual((b"http", b"::ffff:0.0.0.0", 1080), parse_proxy(url)) + + def test_parse_proxy_host_port_ipv6(self): + url = b"2001:0db8:85a3:0000:0000:8a2e:0370:effe:9988" + self.assertEqual( + (b"http", b"2001:0db8:85a3:0000:0000:8a2e:0370:effe", 9988), + parse_proxy(url), + ) + + # currently broken + url = b"2001:0db8:85a3:0000:0000:8a2e:0370:1234:9988" + # self.assertEqual((b"http", b"2001:0db8:85a3:0000:0000:8a2e:0370:1234", 9988), parse_proxy(url)) + + url = b"::1:9988" + self.assertEqual((b"http", b"::1", 9988), parse_proxy(url)) + url = b"::ffff:0.0.0.0:9988" + self.assertEqual((b"http", b"::ffff:0.0.0.0", 9988), parse_proxy(url)) + + def test_parse_proxy_scheme_host_ipv6(self): + url = b"https://2001:0db8:85a3:0000:0000:8a2e:0370:effe" + self.assertEqual( + (b"https", b"2001:0db8:85a3:0000:0000:8a2e:0370:effe", 1080), + parse_proxy(url), + ) + + # currently broken + url = b"https://2001:0db8:85a3:0000:0000:8a2e:0370:1234" + # self.assertEqual((b"https", b"2001:0db8:85a3:0000:0000:8a2e:0370:1234", 1080), parse_proxy(url)) + + # also broken + url = b"https://::1" + # self.assertEqual((b"https", b"::1", 1080), parse_proxy(url)) + url = b"https://::ffff:0.0.0.0:1080" + self.assertEqual((b"https", b"::ffff:0.0.0.0", 1080), parse_proxy(url)) + + def test_parse_proxy_scheme_host_port_ipv6(self): + url = b"https://2001:0db8:85a3:0000:0000:8a2e:0370:effe:9988" + self.assertEqual( + (b"https", b"2001:0db8:85a3:0000:0000:8a2e:0370:effe", 9988), + parse_proxy(url), + ) + + # currently broken + url = b"https://2001:0db8:85a3:0000:0000:8a2e:0370:1234:9988" + # self.assertEqual((b"https", b"2001:0db8:85a3:0000:0000:8a2e:0370:1234", 9988), parse_proxy(url)) + + url = b"https://::1:9988" + self.assertEqual((b"https", b"::1", 9988), parse_proxy(url)) + url = b"https://::ffff:0.0.0.0:9988" + self.assertEqual((b"https", b"::ffff:0.0.0.0", 9988), parse_proxy(url)) + def _test_request_direct_connection(self, agent, scheme, hostname, path): """Runs a test case for a direct connection not going through a proxy. @@ -490,6 +590,36 @@ def test_https_request_via_uppercase_proxy_with_blacklist(self): body = self.successResultOf(treq.content(resp)) self.assertEqual(body, b"result") + @patch.dict(os.environ, {"http_proxy": "proxy.com:8888"}) + def test_proxy_with_no_scheme(self): + http_proxy_agent = ProxyAgent(self.reactor, use_proxy=True) + self.assertIsInstance(http_proxy_agent.http_proxy_endpoint, HostnameEndpoint) + self.assertEqual(http_proxy_agent.http_proxy_endpoint._hostStr, "proxy.com") + self.assertEqual(http_proxy_agent.http_proxy_endpoint._port, 8888) + + @patch.dict(os.environ, {"http_proxy": "socks://proxy.com:8888"}) + def test_proxy_with_unsupported_scheme(self): + with self.assertRaises(ValueError): + _ = ProxyAgent(self.reactor, use_proxy=True) + + @patch.dict(os.environ, {"http_proxy": "http://proxy.com:8888"}) + def test_proxy_with_http_scheme(self): + http_proxy_agent = ProxyAgent(self.reactor, use_proxy=True) + self.assertIsInstance(http_proxy_agent.http_proxy_endpoint, HostnameEndpoint) + self.assertEqual(http_proxy_agent.http_proxy_endpoint._hostStr, "proxy.com") + self.assertEqual(http_proxy_agent.http_proxy_endpoint._port, 8888) + + @patch.dict(os.environ, {"http_proxy": "https://proxy.com:8888"}) + def test_proxy_with_https_scheme(self): + https_proxy_agent = ProxyAgent(self.reactor, use_proxy=True) + self.assertIsInstance(https_proxy_agent.http_proxy_endpoint, _WrapperEndpoint) + self.assertEqual( + https_proxy_agent.http_proxy_endpoint._wrappedEndpoint._hostStr, "proxy.com" + ) + self.assertEqual( + https_proxy_agent.http_proxy_endpoint._wrappedEndpoint._port, 8888 + ) + def _wrap_server_factory_for_tls(factory, sanlist=None): """Wrap an existing Protocol Factory with a test TLSMemoryBIOFactory From c9cb0a8394dc3a5adba9769b61570f63c04e2816 Mon Sep 17 00:00:00 2001 From: Marcus Hoffmann Date: Thu, 17 Jun 2021 13:10:12 +0200 Subject: [PATCH 02/10] add back note about supported schemes --- synapse/http/proxyagent.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/synapse/http/proxyagent.py b/synapse/http/proxyagent.py index 55712148bd15..f26e3e14315f 100644 --- a/synapse/http/proxyagent.py +++ b/synapse/http/proxyagent.py @@ -253,9 +253,8 @@ def _http_proxy_endpoint( """Parses an http proxy setting and returns an endpoint for the proxy Args: - proxy: the proxy setting in the form: [:@][:] - Note that compared to other apps, this function currently lacks support - for specifying a protocol schema (i.e. protocol://...). + proxy: the proxy setting in the form: [:@][scheme://][:] + This currently supports http:// and https:// proxies. a hostname without scheme is assumed to be http. reactor: reactor to be used to connect to the proxy @@ -311,12 +310,12 @@ def parse_proxy( proxy: bytes, default_scheme: bytes = b"http", default_port: int = 1080 ) -> Tuple[bytes, bytes, int]: """ - Parse the hostname and port from a proxy connection byte string. + Parse the scheme, hostname and port from a proxy connection byte string. Args: - proxy: The proxy connection string. Must be in the form 'host[:port]'. - default_scheme: The default scheme to return if one is not found in `proxy`. - default_port: The default port to return if one is not found in `proxy`. + proxy: The proxy connection string. Must be in the form '[scheme://]host[:port]'. + default_scheme: The default scheme to return if one is not found in `proxy`. Defaults to http + default_port: The default port to return if one is not found in `proxy`. Defaults to 1080 Returns: A tuple containing the scheme, hostname and port. From f889cdff5d09218e6eb859dab96c8f76158c2486 Mon Sep 17 00:00:00 2001 From: Marcus Hoffmann Date: Thu, 17 Jun 2021 13:23:06 +0200 Subject: [PATCH 03/10] isort --- tests/http/test_proxyagent.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/http/test_proxyagent.py b/tests/http/test_proxyagent.py index a02670b3e6a8..6029599071cd 100644 --- a/tests/http/test_proxyagent.py +++ b/tests/http/test_proxyagent.py @@ -26,9 +26,8 @@ from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.web.http import HTTPChannel -from synapse.http.proxyagent import parse_proxy from synapse.http.client import BlacklistingReactorWrapper -from synapse.http.proxyagent import ProxyAgent +from synapse.http.proxyagent import ProxyAgent, parse_proxy from tests.http import TestServerTLSConnectionFactory, get_test_https_policy from tests.server import FakeTransport, ThreadedMemoryReactorClock From 9326de3d675e3c03fbd9726dc02eab52ebc90eb0 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 23 Jun 2021 12:46:22 +0100 Subject: [PATCH 04/10] Add test for connections to https proxy --- tests/http/test_proxyagent.py | 46 +++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/tests/http/test_proxyagent.py b/tests/http/test_proxyagent.py index 6029599071cd..0f976b8ba309 100644 --- a/tests/http/test_proxyagent.py +++ b/tests/http/test_proxyagent.py @@ -14,7 +14,7 @@ import base64 import logging import os -from typing import Optional +from typing import Iterable, Optional from unittest.mock import patch import treq @@ -43,7 +43,12 @@ def setUp(self): self.reactor = ThreadedMemoryReactorClock() def _make_connection( - self, client_factory, server_factory, ssl=False, expected_sni=None + self, + client_factory, + server_factory, + ssl=False, + expected_sni=None, + tls_sanlist: Optional[Iterable[bytes]] = None, ): """Builds a test server, and completes the outgoing client connection @@ -60,11 +65,14 @@ def _make_connection( expected_sni (bytes|None): the expected SNI value + tls_sanlist: list of SAN entries for the TLS cert presented by the server. + Defaults to [b'DNS:test.com'] + Returns: IProtocol: the server Protocol returned by server_factory """ if ssl: - server_factory = _wrap_server_factory_for_tls(server_factory) + server_factory = _wrap_server_factory_for_tls(server_factory, tls_sanlist) server_protocol = server_factory.buildProtocol(None) @@ -302,10 +310,16 @@ def test_https_request_via_no_proxy_star(self): ) self._test_request_direct_connection(agent, b"https", b"test.com", b"abc") - @patch.dict(os.environ, {"http_proxy": "proxy.com:8888", "no_proxy": "unused.com"}) - def test_http_request_via_proxy(self): - agent = ProxyAgent(self.reactor, use_proxy=True) + def _test_request_proxy_connection( + self, agent: ProxyAgent, ssl: bool = False + ) -> None: + """Send a request via an agent and check that it is correctly received at the proxy + Args: + agent: the Agent to send the request via. It is expected to send requests + to a proxy at 'proxy.com:8888'. + ssl: True if we expect the Agent to connect via https + """ self.reactor.lookups["proxy.com"] = "1.2.3.5" d = agent.request(b"GET", b"http://test.com") @@ -318,7 +332,11 @@ def test_http_request_via_proxy(self): # make a test server, and wire up the client http_server = self._make_connection( - client_factory, _get_test_protocol_factory() + client_factory, + _get_test_protocol_factory(), + ssl=ssl, + tls_sanlist=[b"DNS:proxy.com"] if ssl else None, + expected_sni=b"proxy.com" if ssl else None, ) # the FakeTransport is async, so we need to pump the reactor @@ -340,6 +358,20 @@ def test_http_request_via_proxy(self): body = self.successResultOf(treq.content(resp)) self.assertEqual(body, b"result") + @patch.dict(os.environ, {"http_proxy": "proxy.com:8888", "no_proxy": "unused.com"}) + def test_http_request_via_proxy(self): + agent = ProxyAgent(self.reactor, use_proxy=True) + self._test_request_proxy_connection(agent) + + @patch.dict( + os.environ, {"http_proxy": "https://proxy.com:8888", "no_proxy": "unused.com"} + ) + def test_http_request_via_https_proxy(self): + agent = ProxyAgent( + self.reactor, use_proxy=True, contextFactory=get_test_https_policy() + ) + self._test_request_proxy_connection(agent, ssl=True) + @patch.dict(os.environ, {"https_proxy": "proxy.com", "no_proxy": "unused.com"}) def test_https_request_via_proxy(self): """Tests that TLS-encrypted requests can be made through a proxy""" From ef657bf6fa31f37d44a9be3b3f422995731185d4 Mon Sep 17 00:00:00 2001 From: Marcus Date: Thu, 24 Jun 2021 20:53:51 +0200 Subject: [PATCH 05/10] fix comment wrapping Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --- synapse/http/proxyagent.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/http/proxyagent.py b/synapse/http/proxyagent.py index f26e3e14315f..efd961597139 100644 --- a/synapse/http/proxyagent.py +++ b/synapse/http/proxyagent.py @@ -254,7 +254,8 @@ def _http_proxy_endpoint( Args: proxy: the proxy setting in the form: [:@][scheme://][:] - This currently supports http:// and https:// proxies. a hostname without scheme is assumed to be http. + This currently supports http:// and https:// proxies. + A hostname without scheme is assumed to be http. reactor: reactor to be used to connect to the proxy From 76698d8239b39a9f091ab9057b8eeee0ce0049a3 Mon Sep 17 00:00:00 2001 From: Marcus Hoffmann Date: Thu, 24 Jun 2021 22:01:03 +0200 Subject: [PATCH 06/10] tls_options_factory isn't Optional --- synapse/http/proxyagent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/proxyagent.py b/synapse/http/proxyagent.py index f26e3e14315f..f82363852177 100644 --- a/synapse/http/proxyagent.py +++ b/synapse/http/proxyagent.py @@ -247,7 +247,7 @@ def request(self, method, uri, headers=None, bodyProducer=None): def _http_proxy_endpoint( proxy: Optional[bytes], reactor: IReactorCore, - tls_options_factory: Optional[IPolicyForHTTPS], + tls_options_factory: IPolicyForHTTPS, **kwargs, ): """Parses an http proxy setting and returns an endpoint for the proxy From bca0c048a77f7b2031cf6acb688c70721deb7b00 Mon Sep 17 00:00:00 2001 From: Marcus Hoffmann Date: Thu, 24 Jun 2021 22:04:42 +0200 Subject: [PATCH 07/10] make proxy_parse tests a separate class --- tests/http/test_proxyagent.py | 150 +++++++++++++++++----------------- 1 file changed, 76 insertions(+), 74 deletions(-) diff --git a/tests/http/test_proxyagent.py b/tests/http/test_proxyagent.py index 0f976b8ba309..e70d084befab 100644 --- a/tests/http/test_proxyagent.py +++ b/tests/http/test_proxyagent.py @@ -38,80 +38,7 @@ HTTPFactory = Factory.forProtocol(HTTPChannel) -class MatrixFederationAgentTests(TestCase): - def setUp(self): - self.reactor = ThreadedMemoryReactorClock() - - def _make_connection( - self, - client_factory, - server_factory, - ssl=False, - expected_sni=None, - tls_sanlist: Optional[Iterable[bytes]] = None, - ): - """Builds a test server, and completes the outgoing client connection - - Args: - client_factory (interfaces.IProtocolFactory): the the factory that the - application is trying to use to make the outbound connection. We will - invoke it to build the client Protocol - - server_factory (interfaces.IProtocolFactory): a factory to build the - server-side protocol - - ssl (bool): If true, we will expect an ssl connection and wrap - server_factory with a TLSMemoryBIOFactory - - expected_sni (bytes|None): the expected SNI value - - tls_sanlist: list of SAN entries for the TLS cert presented by the server. - Defaults to [b'DNS:test.com'] - - Returns: - IProtocol: the server Protocol returned by server_factory - """ - if ssl: - server_factory = _wrap_server_factory_for_tls(server_factory, tls_sanlist) - - server_protocol = server_factory.buildProtocol(None) - - # now, tell the client protocol factory to build the client protocol, - # and wire the output of said protocol up to the server via - # a FakeTransport. - # - # Normally this would be done by the TCP socket code in Twisted, but we are - # stubbing that out here. - client_protocol = client_factory.buildProtocol(None) - client_protocol.makeConnection( - FakeTransport(server_protocol, self.reactor, client_protocol) - ) - - # tell the server protocol to send its stuff back to the client, too - server_protocol.makeConnection( - FakeTransport(client_protocol, self.reactor, server_protocol) - ) - - if ssl: - http_protocol = server_protocol.wrappedProtocol - tls_connection = server_protocol._tlsConnection - else: - http_protocol = server_protocol - tls_connection = None - - # give the reactor a pump to get the TLS juices flowing (if needed) - self.reactor.advance(0) - - if expected_sni is not None: - server_name = tls_connection.get_servername() - self.assertEqual( - server_name, - expected_sni, - "Expected SNI %s but got %s" % (expected_sni, server_name), - ) - - return http_protocol - +class ProxyParserTests(TestCase): def test_parse_proxy_host_only(self): url = b"localhost" self.assertEqual((b"http", b"localhost", 1080), parse_proxy(url)) @@ -210,6 +137,81 @@ def test_parse_proxy_scheme_host_port_ipv6(self): url = b"https://::ffff:0.0.0.0:9988" self.assertEqual((b"https", b"::ffff:0.0.0.0", 9988), parse_proxy(url)) + +class MatrixFederationAgentTests(TestCase): + def setUp(self): + self.reactor = ThreadedMemoryReactorClock() + + def _make_connection( + self, + client_factory, + server_factory, + ssl=False, + expected_sni=None, + tls_sanlist: Optional[Iterable[bytes]] = None, + ): + """Builds a test server, and completes the outgoing client connection + + Args: + client_factory (interfaces.IProtocolFactory): the the factory that the + application is trying to use to make the outbound connection. We will + invoke it to build the client Protocol + + server_factory (interfaces.IProtocolFactory): a factory to build the + server-side protocol + + ssl (bool): If true, we will expect an ssl connection and wrap + server_factory with a TLSMemoryBIOFactory + + expected_sni (bytes|None): the expected SNI value + + tls_sanlist: list of SAN entries for the TLS cert presented by the server. + Defaults to [b'DNS:test.com'] + + Returns: + IProtocol: the server Protocol returned by server_factory + """ + if ssl: + server_factory = _wrap_server_factory_for_tls(server_factory, tls_sanlist) + + server_protocol = server_factory.buildProtocol(None) + + # now, tell the client protocol factory to build the client protocol, + # and wire the output of said protocol up to the server via + # a FakeTransport. + # + # Normally this would be done by the TCP socket code in Twisted, but we are + # stubbing that out here. + client_protocol = client_factory.buildProtocol(None) + client_protocol.makeConnection( + FakeTransport(server_protocol, self.reactor, client_protocol) + ) + + # tell the server protocol to send its stuff back to the client, too + server_protocol.makeConnection( + FakeTransport(client_protocol, self.reactor, server_protocol) + ) + + if ssl: + http_protocol = server_protocol.wrappedProtocol + tls_connection = server_protocol._tlsConnection + else: + http_protocol = server_protocol + tls_connection = None + + # give the reactor a pump to get the TLS juices flowing (if needed) + self.reactor.advance(0) + + if expected_sni is not None: + server_name = tls_connection.get_servername() + self.assertEqual( + server_name, + expected_sni, + "Expected SNI %s but got %s" % (expected_sni, server_name), + ) + + return http_protocol + def _test_request_direct_connection(self, agent, scheme, hostname, path): """Runs a test case for a direct connection not going through a proxy. From 9275a29c7cab2a29f54837e7fc331e52f3d7ae00 Mon Sep 17 00:00:00 2001 From: Marcus Hoffmann Date: Thu, 24 Jun 2021 22:09:38 +0200 Subject: [PATCH 08/10] add comment about supported proxy protocols to proxyagent --- synapse/http/proxyagent.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/http/proxyagent.py b/synapse/http/proxyagent.py index 3090f0a384d0..5ef80d8782f3 100644 --- a/synapse/http/proxyagent.py +++ b/synapse/http/proxyagent.py @@ -82,6 +82,10 @@ class ProxyAgent(_AgentBase): use_proxy (bool): Whether proxy settings should be discovered and used from conventional environment variables. + This currently supports http:// and https:// proxies. + A hostname without scheme is assumed to be http. + + Raises: ValueError if given a proxy with a scheme we don't support. """ def __init__( From 59e6f5af7eb7fa4f7e5ebbd546dc82adba895249 Mon Sep 17 00:00:00 2001 From: Marcus Hoffmann Date: Thu, 24 Jun 2021 22:11:14 +0200 Subject: [PATCH 09/10] fix scheme, username:password order confusion in comment --- synapse/http/proxyagent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/proxyagent.py b/synapse/http/proxyagent.py index 5ef80d8782f3..c747b99f3eb6 100644 --- a/synapse/http/proxyagent.py +++ b/synapse/http/proxyagent.py @@ -257,7 +257,7 @@ def _http_proxy_endpoint( """Parses an http proxy setting and returns an endpoint for the proxy Args: - proxy: the proxy setting in the form: [:@][scheme://][:] + proxy: the proxy setting in the form: [scheme://][:@][:] This currently supports http:// and https:// proxies. A hostname without scheme is assumed to be http. From 35378a06de8384459df93d84980e79ef5310a4b7 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 7 Jul 2021 10:50:17 +0100 Subject: [PATCH 10/10] Update synapse/http/proxyagent.py Co-authored-by: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> --- synapse/http/proxyagent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/proxyagent.py b/synapse/http/proxyagent.py index c747b99f3eb6..ffef9d9220f8 100644 --- a/synapse/http/proxyagent.py +++ b/synapse/http/proxyagent.py @@ -258,7 +258,7 @@ def _http_proxy_endpoint( Args: proxy: the proxy setting in the form: [scheme://][:@][:] - This currently supports http:// and https:// proxies. + This currently supports http:// and https:// proxies. A hostname without scheme is assumed to be http. reactor: reactor to be used to connect to the proxy