From d7a2fa6d7929e26843c5454d2ce46a654d3c3255 Mon Sep 17 00:00:00 2001 From: Miikka Koskinen Date: Thu, 16 Nov 2023 15:09:22 +0200 Subject: [PATCH 01/14] Fix a memory leak in repeated AWS4Auth initializations --- CHANGELOG.md | 4 ++++ httpx_auth/aws.py | 14 +++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1715210..192f566 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- Fix a memory leak in repeated AWS4Auth initializationa. + ## [0.18.0] - 2023-09-11 ### Changed - Requires [`httpx`](https://www.python-httpx.org)==0.25.\* diff --git a/httpx_auth/aws.py b/httpx_auth/aws.py index 10ea076..e37d059 100644 --- a/httpx_auth/aws.py +++ b/httpx_auth/aws.py @@ -47,13 +47,13 @@ def __init__( self.service = service self.security_token = kwargs.get("security_token") - # TODO Check if we really need to be able to override this default ? - if self.security_token: - # TODO Avoid modifying shared variable - self.default_include_headers.append("x-amz-security-token") - self.include_headers = kwargs.get( - "include_headers", self.default_include_headers - ) + if "include_headers" in kwargs: + self.include_headers = kwargs["include_headers"] + else: + # TODO Check if we really need to be able to override this default ? + self.include_headers = self.default_include_headers.copy() + if self.security_token: + self.include_headers.append("x-amz-security-token") def auth_flow( self, request: httpx.Request From 6e6c3e908f71a3c0889ef815dfbc2d1df72ce7a6 Mon Sep 17 00:00:00 2001 From: Colin-b Date: Sun, 4 Feb 2024 15:35:52 +0100 Subject: [PATCH 02/14] Reproduce the issue with memory leak --- CHANGELOG.md | 5 +- httpx_auth/aws.py | 15 +++--- tests/aws_signature_v4/test_aws4auth_async.py | 51 +++++++++++++++++++ tests/aws_signature_v4/test_aws4auth_sync.py | 50 ++++++++++++++++++ 4 files changed, 110 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52eca34..e29b199 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Fixed - Remove deprecation warnings due to usage of `utcnow` and `utcfromtimestamp`. Thanks to [`Raphael Krupinski`](https://github.com/rafalkrupinski). +- `httpx_auth.AWS4Auth.default_include_headers` value kept growing in size every time a new `httpx_auth.AWS4Auth` instance was created with `security_token` parameter provided. Thanks to [`Miikka Koskinen`](https://github.com/miikka). ## [0.19.0] - 2024-01-09 ### Added @@ -16,10 +17,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Requires [`httpx`](https://www.python-httpx.org)==0.26.\* - Note that this changes the signature sent via AWS auth for URLs containing %. Feel free to open an issue if this is one. -### Fixed - -- Fix a memory leak in repeated AWS4Auth initializationa. - ## [0.18.0] - 2023-09-11 ### Changed - Requires [`httpx`](https://www.python-httpx.org)==0.25.\* diff --git a/httpx_auth/aws.py b/httpx_auth/aws.py index d8bbefb..b6707a6 100644 --- a/httpx_auth/aws.py +++ b/httpx_auth/aws.py @@ -48,13 +48,14 @@ def __init__( self.service = service self.security_token = kwargs.get("security_token") - if "include_headers" in kwargs: - self.include_headers = kwargs["include_headers"] - else: - # TODO Check if we really need to be able to override this default ? - self.include_headers = self.default_include_headers.copy() - if self.security_token: - self.include_headers.append("x-amz-security-token") + + # TODO Check if we really need to be able to override this default ? + if self.security_token: + # TODO Avoid modifying shared variable + self.default_include_headers.append("x-amz-security-token") + self.include_headers = kwargs.get( + "include_headers", self.default_include_headers + ) def auth_flow( self, request: httpx.Request diff --git a/tests/aws_signature_v4/test_aws4auth_async.py b/tests/aws_signature_v4/test_aws4auth_async.py index 9e2cc81..1d8bdb3 100644 --- a/tests/aws_signature_v4/test_aws4auth_async.py +++ b/tests/aws_signature_v4/test_aws4auth_async.py @@ -83,6 +83,57 @@ async def test_aws_auth_with_security_token_and_without_content_in_request( await client.post("https://authorized_only", auth=auth) +@time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) +@pytest.mark.asyncio +async def test_aws_auth_share_security_tokens_between_instances( + httpx_mock: HTTPXMock, +): + httpx_auth.AWS4Auth( + access_id="access_id", + secret_key="wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + region="us-east-1", + service="iam", + security_token="security_token1", + ) + auth2 = httpx_auth.AWS4Auth( + access_id="access_id", + secret_key="wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + region="us-east-1", + service="iam", + security_token="security_token", + ) + assert auth2.include_headers == [ + "host", + "content-type", + "date", + "x-amz-*", + "x-amz-security-token", + "x-amz-security-token", + ] + assert auth2.default_include_headers == [ + "host", + "content-type", + "date", + "x-amz-*", + "x-amz-security-token", + "x-amz-security-token", + ] + + httpx_mock.add_response( + url="https://authorized_only", + method="POST", + match_headers={ + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "Authorization": "AWS4-HMAC-SHA256 Credential=access_id/20181011/us-east-1/iam/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=2ae27ce5e8dcc005736c97ff857e4f44401fc3a33d8358b1d67c079f0f5a8b3e", + "x-amz-date": "20181011T150505Z", + "x-amz-security-token": "security_token", + }, + ) + + async with httpx.AsyncClient() as client: + await client.post("https://authorized_only", auth=auth2) + + @time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) @pytest.mark.asyncio async def test_aws_auth_with_security_token_and_content_in_request( diff --git a/tests/aws_signature_v4/test_aws4auth_sync.py b/tests/aws_signature_v4/test_aws4auth_sync.py index f9fe4d3..65577d5 100644 --- a/tests/aws_signature_v4/test_aws4auth_sync.py +++ b/tests/aws_signature_v4/test_aws4auth_sync.py @@ -79,6 +79,56 @@ def test_aws_auth_with_security_token_and_without_content_in_request( client.post("https://authorized_only", auth=auth) +@time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) +def test_aws_auth_share_security_tokens_between_instances( + httpx_mock: HTTPXMock, +): + httpx_auth.AWS4Auth( + access_id="access_id", + secret_key="wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + region="us-east-1", + service="iam", + security_token="security_token1", + ) + auth2 = httpx_auth.AWS4Auth( + access_id="access_id", + secret_key="wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + region="us-east-1", + service="iam", + security_token="security_token", + ) + assert auth2.include_headers == [ + "host", + "content-type", + "date", + "x-amz-*", + "x-amz-security-token", + "x-amz-security-token", + ] + assert auth2.default_include_headers == [ + "host", + "content-type", + "date", + "x-amz-*", + "x-amz-security-token", + "x-amz-security-token", + ] + + httpx_mock.add_response( + url="https://authorized_only", + method="POST", + match_headers={ + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "Authorization": "AWS4-HMAC-SHA256 Credential=access_id/20181011/us-east-1/iam/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=2ae27ce5e8dcc005736c97ff857e4f44401fc3a33d8358b1d67c079f0f5a8b3e", + "x-amz-date": "20181011T150505Z", + "x-amz-security-token": "security_token", + }, + ) + + with httpx.Client() as client: + client.post("https://authorized_only", auth=auth2) + + @time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) def test_aws_auth_with_security_token_and_content_in_request(httpx_mock: HTTPXMock): auth = httpx_auth.AWS4Auth( From 6bf944df07540704e2503881ca06f929b557fdd1 Mon Sep 17 00:00:00 2001 From: Colin-b Date: Sun, 4 Feb 2024 15:54:24 +0100 Subject: [PATCH 03/14] Cleanup --- httpx_auth/aws.py | 52 +++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/httpx_auth/aws.py b/httpx_auth/aws.py index b6707a6..8f002bc 100644 --- a/httpx_auth/aws.py +++ b/httpx_auth/aws.py @@ -38,6 +38,12 @@ def __init__( http://docs.aws.amazon.com/general/latest/gr/rande.html e.g. elasticbeanstalk. :param security_token: Used for the x-amz-security-token header, for use with STS temporary credentials. + :param include_headers: List of headers to include in the canonical and signed headers. + It's primarily included to allow testing against specific examples from Amazon. + ["host", "content-type", "date", "x-amz-*"] by default. + Specific values: + - "x-amz-*" matches any header starting with 'x-amz-' except for x-amz-client context, which appears to break mobile analytics auth if included. + - "*" will include every provided header. """ self.secret_key = secret_key if not self.secret_key: @@ -148,31 +154,34 @@ def _get_canonical_headers( # single header with lowercase name. Although this is not possible with # Requests, since it uses a case-insensitive dict to hold headers, this # is here just in case you duck type with a regular dict - cano_headers_dict = {} - for hdr, val in headers.items(): - hdr = hdr.strip().lower() - val = cls._amz_norm_whitespace(val).strip() + included_headers = {} + for header, header_value in headers.items(): + header = header.strip().lower() + header_value = cls._amz_norm_whitespace(header_value).strip() if ( - hdr in include + header in include or "*" in include or ( "x-amz-*" in include - and hdr.startswith("x-amz-") - and not hdr == "x-amz-client-context" + and header.startswith("x-amz-") + and not header == "x-amz-client-context" ) ): - vals = cano_headers_dict.setdefault(hdr, []) - vals.append(val) - # Flatten cano_headers dict to string and generate signed_headers - cano_headers = "" - signed_headers_list = [] - for hdr in sorted(cano_headers_dict): - vals = cano_headers_dict[hdr] - val = ",".join(sorted(vals)) - cano_headers += f"{hdr}:{val}\n" - signed_headers_list.append(hdr) - signed_headers = ";".join(signed_headers_list) - return cano_headers, signed_headers + header_values = included_headers.setdefault(header, []) + header_values.append(header_value) + + canonical_headers = "" + signed_headers = [] + for header in sorted(included_headers): + signed_headers.append(header) + + header_values = included_headers[header] + header_values = ",".join(sorted(header_values)) + canonical_headers += f"{header}:{header_values}\n" + + signed_headers = ";".join(signed_headers) + + return canonical_headers, signed_headers @staticmethod def _get_sig_string(req: httpx.Request, cano_req: str, scope: str) -> str: @@ -185,10 +194,9 @@ def _get_sig_string(req: httpx.Request, cano_req: str, scope: str) -> str: amz_date = req.headers["x-amz-date"] hsh = hashlib.sha256(cano_req.encode()) sig_items = ["AWS4-HMAC-SHA256", amz_date, scope, hsh.hexdigest()] - sig_string = "\n".join(sig_items) - return sig_string + return "\n".join(sig_items) - def _amz_cano_path(self, path) -> str: + def _amz_cano_path(self, path: str) -> str: """ Generate the canonical path as per AWS4 auth requirements. Not documented anywhere, determined from aws4_testsuite examples, From 62b6acde704fdd47a6af74f0cac2ef058734b28c Mon Sep 17 00:00:00 2001 From: Colin-b Date: Sun, 4 Feb 2024 16:32:01 +0100 Subject: [PATCH 04/14] Document AWS4Auth changes --- CHANGELOG.md | 15 +++-- README.md | 27 ++++++--- httpx_auth/aws.py | 55 +++++++------------ tests/aws_signature_v4/test_aws4auth_async.py | 9 --- tests/aws_signature_v4/test_aws4auth_sync.py | 9 --- 5 files changed, 47 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e29b199..1a4a4e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Remove deprecation warnings due to usage of `utcnow` and `utcfromtimestamp`. Thanks to [`Raphael Krupinski`](https://github.com/rafalkrupinski). - `httpx_auth.AWS4Auth.default_include_headers` value kept growing in size every time a new `httpx_auth.AWS4Auth` instance was created with `security_token` parameter provided. Thanks to [`Miikka Koskinen`](https://github.com/miikka). +### Changed +- `httpx_auth.AWS4Auth.default_include_headers` is not available anymore, use `httpx_auth.AWS4Auth` `include_headers` parameter instead to change the list of included headers if the default does not fit your need (). + ## [0.19.0] - 2024-01-09 ### Added - Explicit support for Python 3.12 @@ -115,7 +118,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - `get_token` cache method now requires `on_missing_token` function args to be provided as kwargs instead of args. -- `get_token` cache method now requires `on_missing_token` parameter to be provided as a non positional argument. +- `get_token` cache method now requires `on_missing_token` parameter to be provided as a non-positional argument. - `get_token` cache method now expose `early_expiry` parameter, defaulting to 30 seconds. ### Fixed @@ -167,13 +170,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Still under development, subject to breaking changes without notice: `AWS4Auth` authentication class for AWS. Ported from [`requests-aws4auth`](https://github.com/sam-washington/requests-aws4auth) by [`Michael E. Martinka`](https://github.com/martinka). Note that a few changes were made: - - deprecated `amz_date` attribute has been removed. - - it is not possible to provide an `AWSSigningKey` instance, use explicit parameters instead. - - it is not possible to provide a `date`. It will default to now. - - it is not possible to provide `raise_invalid_date` parameter anymore as the date will always be valid. + - Deprecated `amz_date` attribute has been removed. + - It is not possible to provide an `AWSSigningKey` instance, use explicit parameters instead. + - It is not possible to provide a `date`. It will default to now. + - It is not possible to provide `raise_invalid_date` parameter anymore as the date will always be valid. - `include_hdrs` parameter was renamed into `include_headers` - `host` is not considered as a specific Amazon service anymore (no test specific code). - - Each request now has it's own signing key and x-amz-date. Meaning you can use the same auth instance for more than one request. + - Each request now has its own signing key and `x-amz-date`. Meaning you can use the same auth instance for more than one request. - `session_token` was renamed into `security_token` for consistency with the underlying name at Amazon. ## [0.3.0] - 2020-05-26 diff --git a/README.md b/README.md index aef8d58..9da741c 100644 --- a/README.md +++ b/README.md @@ -667,7 +667,7 @@ OAuth2.token_cache = JsonTokenFileCache('path/to/my_token_cache.json') ## AWS Signature v4 -Amazon Web Service Signature version 4 is implemented following [Amazon S3 documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html) and [request-aws4auth](https://github.com/sam-washington/requests-aws4auth). +Amazon Web Service Signature version 4 is implemented following [Amazon S3 documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html) and [request-aws4auth](https://github.com/sam-washington/requests-aws4auth) (with some changes, see below). Use `httpx_auth.AWS4Auth` to configure this kind of authentication. @@ -680,15 +680,26 @@ with httpx.Client() as client: client.get('http://s3-eu-west-1.amazonaws.com', auth=aws) ``` +Note that the following changes were made compared to `requests-aws4auth`: + - Each request now has its own signing key and `x-amz-date`. Meaning **you can use the same auth instance for more than one request**. + - `session_token` was renamed into `security_token` for consistency with the underlying name at Amazon. + - `include_hdrs` parameter was renamed into `include_headers` + - `amz_date` attribute has been removed. + - It is not possible to provide a `date`. It will default to now. + - It is not possible to provide an `AWSSigningKey` instance, use explicit parameters instead. + - It is not possible to provide `raise_invalid_date` parameter anymore as the date will always be valid. + - `host` is not considered as a specific Amazon service anymore (no test specific code). + ### Parameters -| Name | Description | Mandatory | Default value | -|:-----------------|:---------------------------|:----------|:--------------| -| `access_id` | AWS access ID. | Mandatory | | -| `secret_key` | AWS secret access key. | Mandatory | | -| `region` | The region you are connecting to, as per [this list](http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region). For services which do not require a region (e.g. IAM), use us-east-1. | Mandatory | | -| `service` | The name of the service you are connecting to, as per [this list](http://docs.aws.amazon.com/general/latest/gr/rande.html). e.g. elasticbeanstalk. | Mandatory | | -| `security_token` | Used for the `x-amz-security-token` header, for use with STS temporary credentials. | Optional | | +| Name | Description | Mandatory | Default value | +|:-------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------|:--------------| +| `access_id` | AWS access ID. | Mandatory | | +| `secret_key` | AWS secret access key. | Mandatory | | +| `region` | The region you are connecting to, as per [this list](http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region). For services which do not require a region (e.g. IAM), use us-east-1. | Mandatory | | +| `service` | The name of the service you are connecting to, as per [this list](http://docs.aws.amazon.com/general/latest/gr/rande.html). e.g. elasticbeanstalk. | Mandatory | | +| `security_token` | Used for the `x-amz-security-token` header, for use with STS temporary credentials. | Optional | | +| `include_headers` | List of headers to include in the canonical and signed headers. Specific values are "x-amz-*" that matches any header starting with 'x-amz-' (except for x-amz-client context) and "*" that include every provided header. | Optional | ["host", "content-type", "date", "x-amz-*"] if security_token is provided, x-amz-security-token is also included by default. | ## API key in header diff --git a/httpx_auth/aws.py b/httpx_auth/aws.py index 8f002bc..6fb6446 100644 --- a/httpx_auth/aws.py +++ b/httpx_auth/aws.py @@ -22,8 +22,6 @@ class AWS4Auth(httpx.Auth): requires_request_body = True - default_include_headers = ["host", "content-type", "date", "x-amz-*"] - def __init__( self, access_id: str, secret_key: str, region: str, service: str, **kwargs ): @@ -39,10 +37,10 @@ def __init__( e.g. elasticbeanstalk. :param security_token: Used for the x-amz-security-token header, for use with STS temporary credentials. :param include_headers: List of headers to include in the canonical and signed headers. - It's primarily included to allow testing against specific examples from Amazon. ["host", "content-type", "date", "x-amz-*"] by default. + Note that if security_token is provided, x-amz-security-token is also included by default. Specific values: - - "x-amz-*" matches any header starting with 'x-amz-' except for x-amz-client context, which appears to break mobile analytics auth if included. + - "x-amz-*" matches any header starting with 'x-amz-' except for x-amz-client context. - "*" will include every provided header. """ self.secret_key = secret_key @@ -55,13 +53,11 @@ def __init__( self.security_token = kwargs.get("security_token") - # TODO Check if we really need to be able to override this default ? + include_headers = ["host", "content-type", "date", "x-amz-*"] if self.security_token: - # TODO Avoid modifying shared variable - self.default_include_headers.append("x-amz-security-token") - self.include_headers = kwargs.get( - "include_headers", self.default_include_headers - ) + include_headers.append("x-amz-security-token") + + self.include_headers = kwargs.get("include_headers", include_headers) def auth_flow( self, request: httpx.Request @@ -84,9 +80,7 @@ def auth_flow( if self.security_token: request.headers["x-amz-security-token"] = self.security_token - cano_headers, signed_headers = self._get_canonical_headers( - request, self.include_headers - ) + cano_headers, signed_headers = self._get_canonical_headers(request) cano_req = self._get_canonical_request(request, cano_headers, signed_headers) sig_string = self._get_sig_string(request, cano_req, scope) sig_string = sig_string.encode("utf-8") @@ -129,25 +123,13 @@ def _get_canonical_request( ] return "\n".join(req_parts) - @classmethod - def _get_canonical_headers( - cls, req: httpx.Request, include: List[str] - ) -> Tuple[str, str]: + def _get_canonical_headers(self, req: httpx.Request) -> Tuple[str, str]: """ Generate the Canonical Headers section of the Canonical Request. Return the Canonical Headers and the Signed Headers strs as a tuple (canonical_headers, signed_headers). - - :param include: List of headers to include in the canonical and signed - headers. It's primarily included to allow testing against - specific examples from Amazon. If omitted or None it - includes host, content-type and any header starting 'x-amz-' - except for x-amz-client context, which appears to break - mobile analytics auth if included. Except for the - x-amz-client-context exclusion these defaults are per the - AWS documentation. """ - include = [x.lower() for x in include] + include = [x.lower() for x in self.include_headers] headers = req.headers.copy() # Aggregate for upper/lowercase header name collisions in header names, # AMZ requires values of colliding headers be concatenated into a @@ -157,13 +139,14 @@ def _get_canonical_headers( included_headers = {} for header, header_value in headers.items(): header = header.strip().lower() - header_value = cls._amz_norm_whitespace(header_value).strip() + header_value = _amz_norm_whitespace(header_value) if ( header in include or "*" in include or ( "x-amz-*" in include and header.startswith("x-amz-") + # x-amz-client-context break mobile analytics auth if included and not header == "x-amz-client-context" ) ): @@ -242,14 +225,6 @@ def _amz_cano_querystring(qs: str) -> str: qs = "&".join(sorted(qs_strings)) return qs - @staticmethod - def _amz_norm_whitespace(text: str) -> str: - """ - Replace runs of whitespace with a single space. - Ignore text enclosed in quotes. - """ - return " ".join(shlex.split(text, posix=False)) - def generate_key(secret_key: str, region: str, service: str, date: str) -> bytes: init_key = f"AWS4{secret_key}".encode("utf-8") @@ -261,3 +236,11 @@ def generate_key(secret_key: str, region: str, service: str, date: str) -> bytes def sign_sha256(signing_key: bytes, message: str) -> bytes: return hmac.new(signing_key, message.encode("utf-8"), hashlib.sha256).digest() + + +def _amz_norm_whitespace(text: str) -> str: + """ + Replace runs of whitespace with a single space. + Ignore text enclosed in quotes. + """ + return " ".join(shlex.split(text, posix=False)).strip() diff --git a/tests/aws_signature_v4/test_aws4auth_async.py b/tests/aws_signature_v4/test_aws4auth_async.py index 1d8bdb3..d61c2d3 100644 --- a/tests/aws_signature_v4/test_aws4auth_async.py +++ b/tests/aws_signature_v4/test_aws4auth_async.py @@ -108,15 +108,6 @@ async def test_aws_auth_share_security_tokens_between_instances( "date", "x-amz-*", "x-amz-security-token", - "x-amz-security-token", - ] - assert auth2.default_include_headers == [ - "host", - "content-type", - "date", - "x-amz-*", - "x-amz-security-token", - "x-amz-security-token", ] httpx_mock.add_response( diff --git a/tests/aws_signature_v4/test_aws4auth_sync.py b/tests/aws_signature_v4/test_aws4auth_sync.py index 65577d5..7deb4ef 100644 --- a/tests/aws_signature_v4/test_aws4auth_sync.py +++ b/tests/aws_signature_v4/test_aws4auth_sync.py @@ -103,15 +103,6 @@ def test_aws_auth_share_security_tokens_between_instances( "date", "x-amz-*", "x-amz-security-token", - "x-amz-security-token", - ] - assert auth2.default_include_headers == [ - "host", - "content-type", - "date", - "x-amz-*", - "x-amz-security-token", - "x-amz-security-token", ] httpx_mock.add_response( From f3aaab1ba2ead8380c9b9c0e84820cc424dbc229 Mon Sep 17 00:00:00 2001 From: Colin-b Date: Sun, 4 Feb 2024 16:35:26 +0100 Subject: [PATCH 05/14] Document version of requests-aws4auth currently ported --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9da741c..38a8a08 100644 --- a/README.md +++ b/README.md @@ -667,7 +667,7 @@ OAuth2.token_cache = JsonTokenFileCache('path/to/my_token_cache.json') ## AWS Signature v4 -Amazon Web Service Signature version 4 is implemented following [Amazon S3 documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html) and [request-aws4auth](https://github.com/sam-washington/requests-aws4auth) (with some changes, see below). +Amazon Web Service Signature version 4 is implemented following [Amazon S3 documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html) and [request-aws4auth 1.0.1](https://github.com/sam-washington/requests-aws4auth) (with some changes, see below). Use `httpx_auth.AWS4Auth` to configure this kind of authentication. From a37fb58e2166cdeb423cb5ffbac4120f6c95139f Mon Sep 17 00:00:00 2001 From: Colin-b Date: Sun, 4 Feb 2024 16:56:37 +0100 Subject: [PATCH 06/14] httpx headers are lower cased when iterating through them already --- httpx_auth/aws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/httpx_auth/aws.py b/httpx_auth/aws.py index 6fb6446..e5a50df 100644 --- a/httpx_auth/aws.py +++ b/httpx_auth/aws.py @@ -138,7 +138,7 @@ def _get_canonical_headers(self, req: httpx.Request) -> Tuple[str, str]: # is here just in case you duck type with a regular dict included_headers = {} for header, header_value in headers.items(): - header = header.strip().lower() + header = header.strip() header_value = _amz_norm_whitespace(header_value) if ( header in include From 8592e621046cc7e7ec429bb1f1542b2d56d283f9 Mon Sep 17 00:00:00 2001 From: Colin-b Date: Sun, 4 Feb 2024 16:57:02 +0100 Subject: [PATCH 07/14] Ensure the behavior of x-amz-* --- tests/aws_signature_v4/test_aws4auth_async.py | 31 +++++++++++++++++++ tests/aws_signature_v4/test_aws4auth_sync.py | 30 ++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/tests/aws_signature_v4/test_aws4auth_async.py b/tests/aws_signature_v4/test_aws4auth_async.py index d61c2d3..3c577da 100644 --- a/tests/aws_signature_v4/test_aws4auth_async.py +++ b/tests/aws_signature_v4/test_aws4auth_async.py @@ -125,6 +125,37 @@ async def test_aws_auth_share_security_tokens_between_instances( await client.post("https://authorized_only", auth=auth2) +@time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) +@pytest.mark.asyncio +async def test_aws_auth_includes_custom_x_amz_headers( + httpx_mock: HTTPXMock, +): + auth = httpx_auth.AWS4Auth( + access_id="access_id", + secret_key="wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + region="us-east-1", + service="iam", + security_token="security_token", + ) + + httpx_mock.add_response( + url="https://authorized_only", + method="POST", + match_headers={ + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "Authorization": "AWS4-HMAC-SHA256 Credential=access_id/20181011/us-east-1/iam/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-custom;x-amz-date;x-amz-security-token, Signature=533d5180d16f23a2807de5675043e60a439f0a4e929fad4fa09395c0fb3276a4", + "x-amz-date": "20181011T150505Z", + "x-amz-security-token": "security_token", + "X-AmZ-CustoM": "Custom", + }, + ) + + async with httpx.AsyncClient() as client: + await client.post( + "https://authorized_only", headers={"X-AmZ-CustoM": "Custom"}, auth=auth + ) + + @time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) @pytest.mark.asyncio async def test_aws_auth_with_security_token_and_content_in_request( diff --git a/tests/aws_signature_v4/test_aws4auth_sync.py b/tests/aws_signature_v4/test_aws4auth_sync.py index 7deb4ef..32c32e2 100644 --- a/tests/aws_signature_v4/test_aws4auth_sync.py +++ b/tests/aws_signature_v4/test_aws4auth_sync.py @@ -120,6 +120,36 @@ def test_aws_auth_share_security_tokens_between_instances( client.post("https://authorized_only", auth=auth2) +@time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) +def test_aws_auth_includes_custom_x_amz_headers( + httpx_mock: HTTPXMock, +): + auth = httpx_auth.AWS4Auth( + access_id="access_id", + secret_key="wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + region="us-east-1", + service="iam", + security_token="security_token", + ) + + httpx_mock.add_response( + url="https://authorized_only", + method="POST", + match_headers={ + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "Authorization": "AWS4-HMAC-SHA256 Credential=access_id/20181011/us-east-1/iam/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-custom;x-amz-date;x-amz-security-token, Signature=533d5180d16f23a2807de5675043e60a439f0a4e929fad4fa09395c0fb3276a4", + "x-amz-date": "20181011T150505Z", + "x-amz-security-token": "security_token", + "X-AmZ-CustoM": "Custom", + }, + ) + + with httpx.Client() as client: + client.post( + "https://authorized_only", headers={"X-AmZ-CustoM": "Custom"}, auth=auth + ) + + @time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) def test_aws_auth_with_security_token_and_content_in_request(httpx_mock: HTTPXMock): auth = httpx_auth.AWS4Auth( From 969d8bcddfe600910d0ff1a5b33c88576bb2722c Mon Sep 17 00:00:00 2001 From: Colin-b Date: Sun, 4 Feb 2024 17:01:50 +0100 Subject: [PATCH 08/14] Ensure the behavior of x-amz-* --- tests/aws_signature_v4/test_aws4auth_async.py | 31 +++++++++++++++++++ tests/aws_signature_v4/test_aws4auth_sync.py | 30 ++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/tests/aws_signature_v4/test_aws4auth_async.py b/tests/aws_signature_v4/test_aws4auth_async.py index 3c577da..62fc6cf 100644 --- a/tests/aws_signature_v4/test_aws4auth_async.py +++ b/tests/aws_signature_v4/test_aws4auth_async.py @@ -156,6 +156,37 @@ async def test_aws_auth_includes_custom_x_amz_headers( ) +@time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) +@pytest.mark.asyncio +async def test_aws_auth_excludes_x_amz_client_context_header( + httpx_mock: HTTPXMock, +): + auth = httpx_auth.AWS4Auth( + access_id="access_id", + secret_key="wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + region="us-east-1", + service="iam", + ) + + httpx_mock.add_response( + url="https://authorized_only", + method="POST", + match_headers={ + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "Authorization": "AWS4-HMAC-SHA256 Credential=access_id/20181011/us-east-1/iam/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=ce708380ee69b1a9558b9b0dddd4d15f35a2a5e5ea3534b541247f1a746626db", + "x-amz-date": "20181011T150505Z", + "x-amz-Client-Context": "Custom", + }, + ) + + async with httpx.AsyncClient() as client: + await client.post( + "https://authorized_only", + headers={"x-amz-Client-Context": "Custom"}, + auth=auth, + ) + + @time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) @pytest.mark.asyncio async def test_aws_auth_with_security_token_and_content_in_request( diff --git a/tests/aws_signature_v4/test_aws4auth_sync.py b/tests/aws_signature_v4/test_aws4auth_sync.py index 32c32e2..ad9d9dd 100644 --- a/tests/aws_signature_v4/test_aws4auth_sync.py +++ b/tests/aws_signature_v4/test_aws4auth_sync.py @@ -150,6 +150,36 @@ def test_aws_auth_includes_custom_x_amz_headers( ) +@time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) +def test_aws_auth_excludes_x_amz_client_context_header( + httpx_mock: HTTPXMock, +): + auth = httpx_auth.AWS4Auth( + access_id="access_id", + secret_key="wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + region="us-east-1", + service="iam", + ) + + httpx_mock.add_response( + url="https://authorized_only", + method="POST", + match_headers={ + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "Authorization": "AWS4-HMAC-SHA256 Credential=access_id/20181011/us-east-1/iam/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=ce708380ee69b1a9558b9b0dddd4d15f35a2a5e5ea3534b541247f1a746626db", + "x-amz-date": "20181011T150505Z", + "x-amz-Client-Context": "Custom", + }, + ) + + with httpx.Client() as client: + client.post( + "https://authorized_only", + headers={"x-amz-Client-Context": "Custom"}, + auth=auth, + ) + + @time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) def test_aws_auth_with_security_token_and_content_in_request(httpx_mock: HTTPXMock): auth = httpx_auth.AWS4Auth( From f0ab9690d8111cc99d9424121cb801f48fde9922 Mon Sep 17 00:00:00 2001 From: Colin-b Date: Sun, 4 Feb 2024 17:09:51 +0100 Subject: [PATCH 09/14] Ensure the behavior of include_headers --- tests/aws_signature_v4/test_aws4auth_async.py | 40 +++++++++++++++++++ tests/aws_signature_v4/test_aws4auth_sync.py | 39 ++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/tests/aws_signature_v4/test_aws4auth_async.py b/tests/aws_signature_v4/test_aws4auth_async.py index 62fc6cf..09f499d 100644 --- a/tests/aws_signature_v4/test_aws4auth_async.py +++ b/tests/aws_signature_v4/test_aws4auth_async.py @@ -187,6 +187,46 @@ async def test_aws_auth_excludes_x_amz_client_context_header( ) +@time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) +@pytest.mark.asyncio +async def test_aws_auth_allows_to_include_custom_and_default_forbidden_header( + httpx_mock: HTTPXMock, +): + auth = httpx_auth.AWS4Auth( + access_id="access_id", + secret_key="wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + region="us-east-1", + service="iam", + include_headers=[ + "Host", + "content-type", + "date", + "cusTom", + "x-aMz-client-context", + "x-amz-*", + ], + ) + + httpx_mock.add_response( + url="https://authorized_only", + method="POST", + match_headers={ + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "Authorization": "AWS4-HMAC-SHA256 Credential=access_id/20181011/us-east-1/iam/aws4_request, SignedHeaders=custom;host;x-amz-client-context;x-amz-content-sha256;x-amz-date, Signature=215c8030c2f238163ddfb291abcd9e5a02112a0db1363aa7cdb27ba1f646d987", + "x-amz-date": "20181011T150505Z", + "Custom": "Custom", + "x-amz-Client-Context": "Context", + }, + ) + + async with httpx.AsyncClient() as client: + await client.post( + "https://authorized_only", + headers={"Custom": "Custom", "x-amz-Client-Context": "Context"}, + auth=auth, + ) + + @time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) @pytest.mark.asyncio async def test_aws_auth_with_security_token_and_content_in_request( diff --git a/tests/aws_signature_v4/test_aws4auth_sync.py b/tests/aws_signature_v4/test_aws4auth_sync.py index ad9d9dd..5fc759d 100644 --- a/tests/aws_signature_v4/test_aws4auth_sync.py +++ b/tests/aws_signature_v4/test_aws4auth_sync.py @@ -180,6 +180,45 @@ def test_aws_auth_excludes_x_amz_client_context_header( ) +@time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) +def test_aws_auth_allows_to_include_custom_and_default_forbidden_header( + httpx_mock: HTTPXMock, +): + auth = httpx_auth.AWS4Auth( + access_id="access_id", + secret_key="wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + region="us-east-1", + service="iam", + include_headers=[ + "Host", + "content-type", + "date", + "cusTom", + "x-aMz-client-context", + "x-amz-*", + ], + ) + + httpx_mock.add_response( + url="https://authorized_only", + method="POST", + match_headers={ + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "Authorization": "AWS4-HMAC-SHA256 Credential=access_id/20181011/us-east-1/iam/aws4_request, SignedHeaders=custom;host;x-amz-client-context;x-amz-content-sha256;x-amz-date, Signature=215c8030c2f238163ddfb291abcd9e5a02112a0db1363aa7cdb27ba1f646d987", + "x-amz-date": "20181011T150505Z", + "Custom": "Custom", + "x-amz-Client-Context": "Context", + }, + ) + + with httpx.Client() as client: + client.post( + "https://authorized_only", + headers={"Custom": "Custom", "x-amz-Client-Context": "Context"}, + auth=auth, + ) + + @time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) def test_aws_auth_with_security_token_and_content_in_request(httpx_mock: HTTPXMock): auth = httpx_auth.AWS4Auth( From 57d7680cae27380283ffdcf044b7b252262bd079 Mon Sep 17 00:00:00 2001 From: Colin-b Date: Sun, 4 Feb 2024 17:12:40 +0100 Subject: [PATCH 10/14] Only convert value when needed --- httpx_auth/aws.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/httpx_auth/aws.py b/httpx_auth/aws.py index e5a50df..619ce3d 100644 --- a/httpx_auth/aws.py +++ b/httpx_auth/aws.py @@ -139,7 +139,6 @@ def _get_canonical_headers(self, req: httpx.Request) -> Tuple[str, str]: included_headers = {} for header, header_value in headers.items(): header = header.strip() - header_value = _amz_norm_whitespace(header_value) if ( header in include or "*" in include @@ -151,7 +150,7 @@ def _get_canonical_headers(self, req: httpx.Request) -> Tuple[str, str]: ) ): header_values = included_headers.setdefault(header, []) - header_values.append(header_value) + header_values.append(_amz_norm_whitespace(header_value)) canonical_headers = "" signed_headers = [] From 9e6d21b44ee029980b38d20bf0ece539e8750360 Mon Sep 17 00:00:00 2001 From: Colin-b Date: Sun, 4 Feb 2024 17:29:46 +0100 Subject: [PATCH 11/14] Do not strip header names --- CHANGELOG.md | 1 + README.md | 2 +- httpx_auth/aws.py | 1 - tests/aws_signature_v4/test_aws4auth_async.py | 38 +++++++++++++++++++ tests/aws_signature_v4/test_aws4auth_sync.py | 37 ++++++++++++++++++ 5 files changed, 77 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a4a4e0..ec9e89c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - `httpx_auth.AWS4Auth.default_include_headers` is not available anymore, use `httpx_auth.AWS4Auth` `include_headers` parameter instead to change the list of included headers if the default does not fit your need (). +- `httpx_auth.AWS4Auth` `include_headers` values will not be stripped anymore, meaning that you can now include headers prefixed and/or suffixed with blank spaces. ## [0.19.0] - 2024-01-09 ### Added diff --git a/README.md b/README.md index 38a8a08..c31ae66 100644 --- a/README.md +++ b/README.md @@ -683,7 +683,7 @@ with httpx.Client() as client: Note that the following changes were made compared to `requests-aws4auth`: - Each request now has its own signing key and `x-amz-date`. Meaning **you can use the same auth instance for more than one request**. - `session_token` was renamed into `security_token` for consistency with the underlying name at Amazon. - - `include_hdrs` parameter was renamed into `include_headers` + - `include_hdrs` parameter was renamed into `include_headers` and provided values will not be stripped, [WYSIWYG](https://en.wikipedia.org/wiki/WYSIWYG). - `amz_date` attribute has been removed. - It is not possible to provide a `date`. It will default to now. - It is not possible to provide an `AWSSigningKey` instance, use explicit parameters instead. diff --git a/httpx_auth/aws.py b/httpx_auth/aws.py index 619ce3d..7283fdf 100644 --- a/httpx_auth/aws.py +++ b/httpx_auth/aws.py @@ -138,7 +138,6 @@ def _get_canonical_headers(self, req: httpx.Request) -> Tuple[str, str]: # is here just in case you duck type with a regular dict included_headers = {} for header, header_value in headers.items(): - header = header.strip() if ( header in include or "*" in include diff --git a/tests/aws_signature_v4/test_aws4auth_async.py b/tests/aws_signature_v4/test_aws4auth_async.py index 09f499d..a7660a5 100644 --- a/tests/aws_signature_v4/test_aws4auth_async.py +++ b/tests/aws_signature_v4/test_aws4auth_async.py @@ -227,6 +227,44 @@ async def test_aws_auth_allows_to_include_custom_and_default_forbidden_header( ) +@time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) +@pytest.mark.asyncio +async def test_aws_auth_does_not_strips_header_names( + httpx_mock: HTTPXMock, +): + auth = httpx_auth.AWS4Auth( + access_id="access_id", + secret_key="wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + region="us-east-1", + service="iam", + include_headers=[ + "Host", + "content-type", + "date", + " cusTom ", + "x-amz-*", + ], + ) + + httpx_mock.add_response( + url="https://authorized_only", + method="POST", + match_headers={ + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "Authorization": "AWS4-HMAC-SHA256 Credential=access_id/20181011/us-east-1/iam/aws4_request, SignedHeaders= custom ;host;x-amz-content-sha256;x-amz-date, Signature=6156fed4e0764085005828ab8017081e2f8e6d12167c860fe2a9ea2034915987", + "x-amz-date": "20181011T150505Z", + " Custom ": "Custom", + }, + ) + + async with httpx.AsyncClient() as client: + await client.post( + "https://authorized_only", + headers={" Custom ": "Custom"}, + auth=auth, + ) + + @time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) @pytest.mark.asyncio async def test_aws_auth_with_security_token_and_content_in_request( diff --git a/tests/aws_signature_v4/test_aws4auth_sync.py b/tests/aws_signature_v4/test_aws4auth_sync.py index 5fc759d..ac73992 100644 --- a/tests/aws_signature_v4/test_aws4auth_sync.py +++ b/tests/aws_signature_v4/test_aws4auth_sync.py @@ -219,6 +219,43 @@ def test_aws_auth_allows_to_include_custom_and_default_forbidden_header( ) +@time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) +def test_aws_auth_does_not_strips_header_names( + httpx_mock: HTTPXMock, +): + auth = httpx_auth.AWS4Auth( + access_id="access_id", + secret_key="wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + region="us-east-1", + service="iam", + include_headers=[ + "Host", + "content-type", + "date", + " cusTom ", + "x-amz-*", + ], + ) + + httpx_mock.add_response( + url="https://authorized_only", + method="POST", + match_headers={ + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "Authorization": "AWS4-HMAC-SHA256 Credential=access_id/20181011/us-east-1/iam/aws4_request, SignedHeaders= custom ;host;x-amz-content-sha256;x-amz-date, Signature=6156fed4e0764085005828ab8017081e2f8e6d12167c860fe2a9ea2034915987", + "x-amz-date": "20181011T150505Z", + " Custom ": "Custom", + }, + ) + + with httpx.Client() as client: + client.post( + "https://authorized_only", + headers={" Custom ": "Custom"}, + auth=auth, + ) + + @time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) def test_aws_auth_with_security_token_and_content_in_request(httpx_mock: HTTPXMock): auth = httpx_auth.AWS4Auth( From 7a5d9b80678ea694e414b0e4f8147a7156e40bf8 Mon Sep 17 00:00:00 2001 From: Colin-b Date: Sun, 4 Feb 2024 17:38:38 +0100 Subject: [PATCH 12/14] lower case headers only once --- README.md | 16 +++++------ httpx_auth/aws.py | 27 +++++++++---------- tests/aws_signature_v4/test_aws4auth_async.py | 4 +-- tests/aws_signature_v4/test_aws4auth_sync.py | 4 +-- 4 files changed, 24 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index c31ae66..55e50a7 100644 --- a/README.md +++ b/README.md @@ -692,14 +692,14 @@ Note that the following changes were made compared to `requests-aws4auth`: ### Parameters -| Name | Description | Mandatory | Default value | -|:-------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------|:--------------| -| `access_id` | AWS access ID. | Mandatory | | -| `secret_key` | AWS secret access key. | Mandatory | | -| `region` | The region you are connecting to, as per [this list](http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region). For services which do not require a region (e.g. IAM), use us-east-1. | Mandatory | | -| `service` | The name of the service you are connecting to, as per [this list](http://docs.aws.amazon.com/general/latest/gr/rande.html). e.g. elasticbeanstalk. | Mandatory | | -| `security_token` | Used for the `x-amz-security-token` header, for use with STS temporary credentials. | Optional | | -| `include_headers` | List of headers to include in the canonical and signed headers. Specific values are "x-amz-*" that matches any header starting with 'x-amz-' (except for x-amz-client context) and "*" that include every provided header. | Optional | ["host", "content-type", "date", "x-amz-*"] if security_token is provided, x-amz-security-token is also included by default. | +| Name | Description | Mandatory | Default value | +|:-------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------|:---------------------------------------------------------------------------------------------------------------------------------| +| `access_id` | AWS access ID. | Mandatory | | +| `secret_key` | AWS secret access key. | Mandatory | | +| `region` | The region you are connecting to, as per [this list](http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region). For services which do not require a region (e.g. IAM), use us-east-1. | Mandatory | | +| `service` | The name of the service you are connecting to, as per [this list](http://docs.aws.amazon.com/general/latest/gr/rande.html). e.g. elasticbeanstalk. | Mandatory | | +| `security_token` | Used for the `x-amz-security-token` header, for use with STS temporary credentials. | Optional | | +| `include_headers` | Set of headers to include in the canonical and signed headers. Specific values are `x-amz-*` that matches any header starting with `x-amz-` (except for `x-amz-client-context`) and `*` that include every provided header. | Optional | {"host", "content-type", "date", "x-amz-*"} if `security_token` is provided, `x-amz-security-token` is also included by default. | ## API key in header diff --git a/httpx_auth/aws.py b/httpx_auth/aws.py index 7283fdf..71bc19a 100644 --- a/httpx_auth/aws.py +++ b/httpx_auth/aws.py @@ -36,8 +36,8 @@ def __init__( http://docs.aws.amazon.com/general/latest/gr/rande.html e.g. elasticbeanstalk. :param security_token: Used for the x-amz-security-token header, for use with STS temporary credentials. - :param include_headers: List of headers to include in the canonical and signed headers. - ["host", "content-type", "date", "x-amz-*"] by default. + :param include_headers: Set of headers to include in the canonical and signed headers. + {"host", "content-type", "date", "x-amz-*"} by default. Note that if security_token is provided, x-amz-security-token is also included by default. Specific values: - "x-amz-*" matches any header starting with 'x-amz-' except for x-amz-client context. @@ -53,11 +53,13 @@ def __init__( self.security_token = kwargs.get("security_token") - include_headers = ["host", "content-type", "date", "x-amz-*"] + include_headers = {"host", "content-type", "date", "x-amz-*"} if self.security_token: - include_headers.append("x-amz-security-token") + include_headers.add("x-amz-security-token") - self.include_headers = kwargs.get("include_headers", include_headers) + self.include_headers = { + header.lower() for header in kwargs.get("include_headers", include_headers) + } def auth_flow( self, request: httpx.Request @@ -129,7 +131,6 @@ def _get_canonical_headers(self, req: httpx.Request) -> Tuple[str, str]: Return the Canonical Headers and the Signed Headers strs as a tuple (canonical_headers, signed_headers). """ - include = [x.lower() for x in self.include_headers] headers = req.headers.copy() # Aggregate for upper/lowercase header name collisions in header names, # AMZ requires values of colliding headers be concatenated into a @@ -138,15 +139,11 @@ def _get_canonical_headers(self, req: httpx.Request) -> Tuple[str, str]: # is here just in case you duck type with a regular dict included_headers = {} for header, header_value in headers.items(): - if ( - header in include - or "*" in include - or ( - "x-amz-*" in include - and header.startswith("x-amz-") - # x-amz-client-context break mobile analytics auth if included - and not header == "x-amz-client-context" - ) + if (header or "*") in self.include_headers or ( + "x-amz-*" in self.include_headers + and header.startswith("x-amz-") + # x-amz-client-context break mobile analytics auth if included + and not header == "x-amz-client-context" ): header_values = included_headers.setdefault(header, []) header_values.append(_amz_norm_whitespace(header_value)) diff --git a/tests/aws_signature_v4/test_aws4auth_async.py b/tests/aws_signature_v4/test_aws4auth_async.py index a7660a5..cb0333d 100644 --- a/tests/aws_signature_v4/test_aws4auth_async.py +++ b/tests/aws_signature_v4/test_aws4auth_async.py @@ -102,13 +102,13 @@ async def test_aws_auth_share_security_tokens_between_instances( service="iam", security_token="security_token", ) - assert auth2.include_headers == [ + assert auth2.include_headers == { "host", "content-type", "date", "x-amz-*", "x-amz-security-token", - ] + } httpx_mock.add_response( url="https://authorized_only", diff --git a/tests/aws_signature_v4/test_aws4auth_sync.py b/tests/aws_signature_v4/test_aws4auth_sync.py index ac73992..2073c89 100644 --- a/tests/aws_signature_v4/test_aws4auth_sync.py +++ b/tests/aws_signature_v4/test_aws4auth_sync.py @@ -97,13 +97,13 @@ def test_aws_auth_share_security_tokens_between_instances( service="iam", security_token="security_token", ) - assert auth2.include_headers == [ + assert auth2.include_headers == { "host", "content-type", "date", "x-amz-*", "x-amz-security-token", - ] + } httpx_mock.add_response( url="https://authorized_only", From 92a3230616bb7679f12bcaf2f357c08e52e56c73 Mon Sep 17 00:00:00 2001 From: Colin-b Date: Sun, 4 Feb 2024 17:42:35 +0100 Subject: [PATCH 13/14] Headers copy is useless --- httpx_auth/aws.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/httpx_auth/aws.py b/httpx_auth/aws.py index 71bc19a..c0eed73 100644 --- a/httpx_auth/aws.py +++ b/httpx_auth/aws.py @@ -131,14 +131,13 @@ def _get_canonical_headers(self, req: httpx.Request) -> Tuple[str, str]: Return the Canonical Headers and the Signed Headers strs as a tuple (canonical_headers, signed_headers). """ - headers = req.headers.copy() # Aggregate for upper/lowercase header name collisions in header names, # AMZ requires values of colliding headers be concatenated into a # single header with lowercase name. Although this is not possible with # Requests, since it uses a case-insensitive dict to hold headers, this # is here just in case you duck type with a regular dict included_headers = {} - for header, header_value in headers.items(): + for header, header_value in req.headers.items(): if (header or "*") in self.include_headers or ( "x-amz-*" in self.include_headers and header.startswith("x-amz-") From 2678f68fd3327039dcf52b50f565679c09e79d37 Mon Sep 17 00:00:00 2001 From: Colin-b Date: Sun, 4 Feb 2024 18:23:25 +0100 Subject: [PATCH 14/14] Assert multi values header behavior --- README.md | 4 +- httpx_auth/aws.py | 13 +----- tests/aws_signature_v4/test_aws4auth_async.py | 41 +++++++++++++++++++ tests/aws_signature_v4/test_aws4auth_sync.py | 40 ++++++++++++++++++ 4 files changed, 86 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 55e50a7..cfd616a 100644 --- a/README.md +++ b/README.md @@ -683,7 +683,9 @@ with httpx.Client() as client: Note that the following changes were made compared to `requests-aws4auth`: - Each request now has its own signing key and `x-amz-date`. Meaning **you can use the same auth instance for more than one request**. - `session_token` was renamed into `security_token` for consistency with the underlying name at Amazon. - - `include_hdrs` parameter was renamed into `include_headers` and provided values will not be stripped, [WYSIWYG](https://en.wikipedia.org/wiki/WYSIWYG). + - `include_hdrs` parameter was renamed into `include_headers`. When using this parameter: + - Provided values will not be stripped, [WYSIWYG](https://en.wikipedia.org/wiki/WYSIWYG). + - If multiple values are provided for a same header, the computation will be based on the value order you provided and value separated by `, `. Instead of ordered values separated by comma for `requests-aws4auth`. - `amz_date` attribute has been removed. - It is not possible to provide a `date`. It will default to now. - It is not possible to provide an `AWSSigningKey` instance, use explicit parameters instead. diff --git a/httpx_auth/aws.py b/httpx_auth/aws.py index c0eed73..9194e92 100644 --- a/httpx_auth/aws.py +++ b/httpx_auth/aws.py @@ -131,11 +131,6 @@ def _get_canonical_headers(self, req: httpx.Request) -> Tuple[str, str]: Return the Canonical Headers and the Signed Headers strs as a tuple (canonical_headers, signed_headers). """ - # Aggregate for upper/lowercase header name collisions in header names, - # AMZ requires values of colliding headers be concatenated into a - # single header with lowercase name. Although this is not possible with - # Requests, since it uses a case-insensitive dict to hold headers, this - # is here just in case you duck type with a regular dict included_headers = {} for header, header_value in req.headers.items(): if (header or "*") in self.include_headers or ( @@ -144,17 +139,13 @@ def _get_canonical_headers(self, req: httpx.Request) -> Tuple[str, str]: # x-amz-client-context break mobile analytics auth if included and not header == "x-amz-client-context" ): - header_values = included_headers.setdefault(header, []) - header_values.append(_amz_norm_whitespace(header_value)) + included_headers[header] = _amz_norm_whitespace(header_value) canonical_headers = "" signed_headers = [] for header in sorted(included_headers): signed_headers.append(header) - - header_values = included_headers[header] - header_values = ",".join(sorted(header_values)) - canonical_headers += f"{header}:{header_values}\n" + canonical_headers += f"{header}:{included_headers[header]}\n" signed_headers = ";".join(signed_headers) diff --git a/tests/aws_signature_v4/test_aws4auth_async.py b/tests/aws_signature_v4/test_aws4auth_async.py index cb0333d..071f04d 100644 --- a/tests/aws_signature_v4/test_aws4auth_async.py +++ b/tests/aws_signature_v4/test_aws4auth_async.py @@ -265,6 +265,47 @@ async def test_aws_auth_does_not_strips_header_names( ) +@time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) +@pytest.mark.asyncio +async def test_aws_auth_header_with_multiple_values( + httpx_mock: HTTPXMock, +): + auth = httpx_auth.AWS4Auth( + access_id="access_id", + secret_key="wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + region="us-east-1", + service="iam", + include_headers=[ + "Host", + "content-type", + "date", + "cusTom", + "x-amz-*", + ], + ) + + httpx_mock.add_response( + url="https://authorized_only", + method="POST", + match_headers={ + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "Authorization": "AWS4-HMAC-SHA256 Credential=access_id/20181011/us-east-1/iam/aws4_request, SignedHeaders=custom;host;x-amz-content-sha256;x-amz-date, Signature=77fcee19291cb9334678ca7221729baab12848ce49225561477ce95c44222dfb", + "x-amz-date": "20181011T150505Z", + "Custom": "value2, value1", + "custoM": "value3", + }, + ) + + async with httpx.AsyncClient() as client: + await client.post( + "https://authorized_only", + headers=httpx.Headers( + [("Custom", "value2"), ("Custom", "value1"), ("custoM", "value3")] + ), + auth=auth, + ) + + @time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) @pytest.mark.asyncio async def test_aws_auth_with_security_token_and_content_in_request( diff --git a/tests/aws_signature_v4/test_aws4auth_sync.py b/tests/aws_signature_v4/test_aws4auth_sync.py index 2073c89..a5fcaa4 100644 --- a/tests/aws_signature_v4/test_aws4auth_sync.py +++ b/tests/aws_signature_v4/test_aws4auth_sync.py @@ -256,6 +256,46 @@ def test_aws_auth_does_not_strips_header_names( ) +@time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) +def test_aws_auth_header_with_multiple_values( + httpx_mock: HTTPXMock, +): + auth = httpx_auth.AWS4Auth( + access_id="access_id", + secret_key="wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + region="us-east-1", + service="iam", + include_headers=[ + "Host", + "content-type", + "date", + "cusTom", + "x-amz-*", + ], + ) + + httpx_mock.add_response( + url="https://authorized_only", + method="POST", + match_headers={ + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "Authorization": "AWS4-HMAC-SHA256 Credential=access_id/20181011/us-east-1/iam/aws4_request, SignedHeaders=custom;host;x-amz-content-sha256;x-amz-date, Signature=77fcee19291cb9334678ca7221729baab12848ce49225561477ce95c44222dfb", + "x-amz-date": "20181011T150505Z", + "Custom": "value2, value1", + "custoM": "value3", + }, + ) + + with httpx.Client() as client: + client.post( + "https://authorized_only", + headers=httpx.Headers( + [("Custom", "value2"), ("Custom", "value1"), ("custoM", "value3")] + ), + auth=auth, + ) + + @time_machine.travel("2018-10-11T15:05:05.663979+00:00", tick=False) def test_aws_auth_with_security_token_and_content_in_request(httpx_mock: HTTPXMock): auth = httpx_auth.AWS4Auth(