From d12694ae0c20e1e454ec42cc480aa4c27f9eddf5 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Fri, 3 Sep 2021 10:26:58 -0400 Subject: [PATCH 1/5] More error checking on the URL schemes. --- synapse/config/oembed.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/synapse/config/oembed.py b/synapse/config/oembed.py index 09267b5eefcf..0b77445be9ed 100644 --- a/synapse/config/oembed.py +++ b/synapse/config/oembed.py @@ -93,6 +93,15 @@ def _parse_and_validate_provider( # might have multiple patterns to match. for endpoint in provider["endpoints"]: api_endpoint = endpoint["url"] + + # The API endpoint must be an HTTP(S) URL. + results = urlparse.urlparse(api_endpoint) + if results.scheme not in {"http", "https"}: + raise ConfigError( + f"Insecure oEmbed scheme ({results.scheme}) for endpoint {api_endpoint}", + config_path, + ) + patterns = [ self._glob_to_pattern(glob, config_path) for glob in endpoint["schemes"] @@ -114,9 +123,12 @@ def _glob_to_pattern(self, glob: str, config_path: Iterable[str]) -> Pattern: """ results = urlparse.urlparse(glob) - # Ensure the scheme does not have wildcards (and is a sane scheme). + # The scheme must be HTTP(S) (and cannot contain wildcards). if results.scheme not in {"http", "https"}: - raise ConfigError(f"Insecure oEmbed scheme: {results.scheme}", config_path) + raise ConfigError( + f"Insecure oEmbed scheme ({results.scheme}) for pattern: {glob}", + config_path, + ) pattern = urlparse.urlunparse( [ From 416a8a7499980beeca65d981290e989136dda791 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Fri, 3 Sep 2021 10:52:33 -0400 Subject: [PATCH 2/5] Provide the format as a parameter (and potentially in the URL). --- synapse/rest/media/v1/oembed.py | 8 +++- tests/rest/media/v1/test_url_preview.py | 53 ++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/synapse/rest/media/v1/oembed.py b/synapse/rest/media/v1/oembed.py index afe41823e4f5..67cb3e557bfa 100644 --- a/synapse/rest/media/v1/oembed.py +++ b/synapse/rest/media/v1/oembed.py @@ -86,11 +86,15 @@ async def get_oembed_content(self, endpoint: str, url: str) -> OEmbedResult: """ try: logger.debug("Trying to get oEmbed content for url '%s'", url) + + # Note that only the JSON format is supported, some endpoints want + # this in the URL, others want it as an argument. + endpoint = endpoint.replace("{format}", "json") + result = await self._client.get_json( endpoint, # TODO Specify max height / width. - # Note that only the JSON format is supported. - args={"url": url}, + args={"url": url, "format": "json"}, ) # Ensure there's a version of 1.0. diff --git a/tests/rest/media/v1/test_url_preview.py b/tests/rest/media/v1/test_url_preview.py index 7fa902722770..2b2bfcee93b5 100644 --- a/tests/rest/media/v1/test_url_preview.py +++ b/tests/rest/media/v1/test_url_preview.py @@ -92,7 +92,13 @@ def make_homeserver(self, reactor, clock): url_patterns=[ re.compile(r"http://twitter\.com/.+/status/.+"), ], - ) + ), + OEmbedEndpointConfig( + api_endpoint="http://www.hulu.com/api/oembed.{format}", + url_patterns=[ + re.compile(r"http://www\.hulu\.com/watch/.+"), + ], + ), ] return hs @@ -656,3 +662,48 @@ def test_oembed_rich(self): channel.json_body, {"og:title": None, "og:description": "Content Preview"}, ) + + def test_oembed_format(self): + """Test an oEmbed endpoint which requires the format in the URL.""" + self.lookups["www.hulu.com"] = [(IPv4Address, "10.1.2.3")] + + result = { + "version": "1.0", + "type": "rich", + "html": "
Content Preview
", + } + end_content = json.dumps(result).encode("utf-8") + + channel = self.make_request( + "GET", + "preview_url?url=http://www.hulu.com/watch/12345", + shorthand=False, + await_result=False, + ) + self.pump() + + client = self.reactor.tcpClients[0][2].buildProtocol(None) + server = AccumulatingProtocol() + server.makeConnection(FakeTransport(client, self.reactor)) + client.makeConnection(FakeTransport(server, self.reactor)) + client.dataReceived( + ( + b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n" + b'Content-Type: application/json; charset="utf8"\r\n\r\n' + ) + % (len(end_content),) + + end_content + ) + + self.pump() + + # The {format} should have been turned into json. + self.assertIn(b"/api/oembed.json", server.data) + # A URL parameter of format=json should be provided. + self.assertIn(b"format=json", server.data) + + self.assertEqual(channel.code, 200) + self.assertEqual( + channel.json_body, + {"og:title": None, "og:description": "Content Preview"}, + ) From 628536e8f26805f3ce89a32fa3f6ed5aaa42f922 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Fri, 3 Sep 2021 10:53:35 -0400 Subject: [PATCH 3/5] Ignore XML-only endpoints. --- synapse/config/oembed.py | 8 ++++++-- synapse/rest/media/v1/oembed.py | 17 ++++++++++++++++- tests/rest/media/v1/test_url_preview.py | 2 ++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/synapse/config/oembed.py b/synapse/config/oembed.py index 0b77445be9ed..10da955cdd6f 100644 --- a/synapse/config/oembed.py +++ b/synapse/config/oembed.py @@ -13,7 +13,7 @@ # limitations under the License. import json import re -from typing import Any, Dict, Iterable, List, Pattern +from typing import Any, Dict, Iterable, List, Optional, Pattern from urllib import parse as urlparse import attr @@ -31,6 +31,8 @@ class OEmbedEndpointConfig: api_endpoint: str # The patterns to match. url_patterns: List[Pattern] + # The supported formats. + formats: Optional[List[str]] class OembedConfig(Config): @@ -106,7 +108,9 @@ def _parse_and_validate_provider( self._glob_to_pattern(glob, config_path) for glob in endpoint["schemes"] ] - yield OEmbedEndpointConfig(api_endpoint, patterns) + yield OEmbedEndpointConfig( + api_endpoint, patterns, endpoint.get("formats") + ) def _glob_to_pattern(self, glob: str, config_path: Iterable[str]) -> Pattern: """ diff --git a/synapse/rest/media/v1/oembed.py b/synapse/rest/media/v1/oembed.py index 67cb3e557bfa..aea6b6793d73 100644 --- a/synapse/rest/media/v1/oembed.py +++ b/synapse/rest/media/v1/oembed.py @@ -49,8 +49,23 @@ class OEmbedProvider: def __init__(self, hs: "HomeServer", client: SimpleHttpClient): self._oembed_patterns = {} for oembed_endpoint in hs.config.oembed.oembed_patterns: + api_endpoint = oembed_endpoint.api_endpoint + + # Only JSON is supported at the moment. This could be declared in + # the formats field. Otherwise, if the endpoint ends in .xml assume + # it doesn't support JSON. + if ( + oembed_endpoint.formats is not None + and "json" not in oembed_endpoint.formats + ) or api_endpoint.endswith(".xml"): + logger.info( + f"Ignoring oEmbed endpoint due to not supporting JSON: {api_endpoint}" + ) + continue + + # Iterate through each URL pattern and point it to the endpoint. for pattern in oembed_endpoint.url_patterns: - self._oembed_patterns[pattern] = oembed_endpoint.api_endpoint + self._oembed_patterns[pattern] = api_endpoint self._client = client def get_oembed_url(self, url: str) -> Optional[str]: diff --git a/tests/rest/media/v1/test_url_preview.py b/tests/rest/media/v1/test_url_preview.py index 2b2bfcee93b5..9f6fbfe6def8 100644 --- a/tests/rest/media/v1/test_url_preview.py +++ b/tests/rest/media/v1/test_url_preview.py @@ -92,12 +92,14 @@ def make_homeserver(self, reactor, clock): url_patterns=[ re.compile(r"http://twitter\.com/.+/status/.+"), ], + formats=None, ), OEmbedEndpointConfig( api_endpoint="http://www.hulu.com/api/oembed.{format}", url_patterns=[ re.compile(r"http://www\.hulu\.com/watch/.+"), ], + formats=["json"], ), ] From ef78cf8ae34efba4a9d1535e80ca9b396e5f95d3 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Fri, 3 Sep 2021 13:00:49 -0400 Subject: [PATCH 4/5] Newsfragment --- changelog.d/10759.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/10759.feature diff --git a/changelog.d/10759.feature b/changelog.d/10759.feature new file mode 100644 index 000000000000..7d18f5c1330f --- /dev/null +++ b/changelog.d/10759.feature @@ -0,0 +1 @@ +Allow configuration of the oEmbed URLs used for URL previews. From c7bb3b100ee9f8391bd81a044fcdbd8280c9e7ff Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 7 Sep 2021 09:21:27 -0400 Subject: [PATCH 5/5] Review comments. --- synapse/config/oembed.py | 4 ++-- synapse/rest/media/v1/oembed.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/config/oembed.py b/synapse/config/oembed.py index 10da955cdd6f..ea6ace4729ba 100644 --- a/synapse/config/oembed.py +++ b/synapse/config/oembed.py @@ -100,7 +100,7 @@ def _parse_and_validate_provider( results = urlparse.urlparse(api_endpoint) if results.scheme not in {"http", "https"}: raise ConfigError( - f"Insecure oEmbed scheme ({results.scheme}) for endpoint {api_endpoint}", + f"Unsupported oEmbed scheme ({results.scheme}) for endpoint {api_endpoint}", config_path, ) @@ -130,7 +130,7 @@ def _glob_to_pattern(self, glob: str, config_path: Iterable[str]) -> Pattern: # The scheme must be HTTP(S) (and cannot contain wildcards). if results.scheme not in {"http", "https"}: raise ConfigError( - f"Insecure oEmbed scheme ({results.scheme}) for pattern: {glob}", + f"Unsupported oEmbed scheme ({results.scheme}) for pattern: {glob}", config_path, ) diff --git a/synapse/rest/media/v1/oembed.py b/synapse/rest/media/v1/oembed.py index aea6b6793d73..2e6706dbfa7f 100644 --- a/synapse/rest/media/v1/oembed.py +++ b/synapse/rest/media/v1/oembed.py @@ -59,7 +59,8 @@ def __init__(self, hs: "HomeServer", client: SimpleHttpClient): and "json" not in oembed_endpoint.formats ) or api_endpoint.endswith(".xml"): logger.info( - f"Ignoring oEmbed endpoint due to not supporting JSON: {api_endpoint}" + "Ignoring oEmbed endpoint due to not supporting JSON: %s", + api_endpoint, ) continue