From 6971367cba6c7603eb50a083bf858b706ea11774 Mon Sep 17 00:00:00 2001 From: Alex Kanitz Date: Wed, 10 Nov 2021 17:22:46 +0100 Subject: [PATCH 1/5] feat: support chaining errors in custom handler --- trs_cli/client.py | 24 +++++++++++++----------- trs_cli/errors.py | 17 ++++++++++------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/trs_cli/client.py b/trs_cli/client.py index 41568d4..5c430fa 100644 --- a/trs_cli/client.py +++ b/trs_cli/client.py @@ -188,7 +188,7 @@ def get_service_info( # validate requested content type and get request headers self._validate_content_type( requested_type=accept, - available_types=['application/json', 'text/plain'], + available_types=['application/json'], ) self._get_headers( content_accept=accept, @@ -771,7 +771,7 @@ def get_tool_classes( # validate requested content type and get request headers self._validate_content_type( requested_type=accept, - available_types=['application/json', 'text/plain'], + available_types=['application/json'], ) self._get_headers( content_accept=accept, @@ -848,7 +848,7 @@ def get_tools( # validate requested content type and get request headers self._validate_content_type( requested_type=accept, - available_types=['application/json', 'text/plain'], + available_types=['application/json'], ) self._get_headers( content_accept=accept, @@ -978,7 +978,7 @@ def get_versions( # validate requested content type and get request headers self._validate_content_type( requested_type=accept, - available_types=['application/json', 'text/plain'], + available_types=['application/json'], ) self._get_headers( content_accept=accept, @@ -1040,7 +1040,7 @@ def get_version( # validate requested content type and get request headers self._validate_content_type( requested_type=accept, - available_types=['application/json', 'text/plain'], + available_types=['application/json'], ) self._get_headers( content_accept=accept, @@ -1106,7 +1106,7 @@ def get_containerfiles( # validate requested content type and get request headers self._validate_content_type( requested_type=accept, - available_types=['application/json', 'text/plain'], + available_types=['application/json'], ) self._get_headers( content_accept=accept, @@ -1743,16 +1743,18 @@ def _send_request_and_validate_response( "Could not connect to API endpoint" ) if response.status_code not in [200, 201]: + logger.warning( + f"Received error response: {response.status_code}" + ) try: - logger.warning("Received error response") return validation_class_error(**response.json()) except ( json.decoder.JSONDecodeError, pydantic.ValidationError, - ): + ) as exc: raise InvalidResponseError( "Response could not be validated against API schema" - ) + ) from exc else: try: if validation_type == "list": @@ -1768,7 +1770,7 @@ def _send_request_and_validate_response( except ( json.decoder.JSONDecodeError, pydantic.ValidationError, - ): + ) as exc: raise InvalidResponseError( "Response could not be validated against API schema" - ) + ) from exc diff --git a/trs_cli/errors.py b/trs_cli/errors.py index 5acb53e..f88fe15 100644 --- a/trs_cli/errors.py +++ b/trs_cli/errors.py @@ -12,14 +12,17 @@ def exception_handler( print_traceback: bool = False, ) -> None: """Error handler for all exceptions.""" - msg = "" - if hasattr(_type, "__name__"): - msg = ( - f"{_type.__name__}: {value}" if str(value) else f"{_type.__name__}" - ) - logger.error(msg) if print_traceback: - tb.print_tb(traceback) + tb.print_exception(etype=_type, value=value, tb=traceback) + else: + msg = "" + if hasattr(_type, "__name__"): + msg = ( + f"{_type.__name__}: {value}" + if str(value) + else f"{_type.__name__}" + ) + logger.error(msg) class ContentTypeUnavailable(Exception): From ec8436ba872c96d3e15a6e980a5bc2f4401f2554 Mon Sep 17 00:00:00 2001 From: Alex Kanitz Date: Wed, 10 Nov 2021 17:54:48 +0100 Subject: [PATCH 2/5] fix: do not encode descriptor paths by default --- trs_cli/client.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/trs_cli/client.py b/trs_cli/client.py index 5c430fa..82094c4 100644 --- a/trs_cli/client.py +++ b/trs_cli/client.py @@ -1217,7 +1217,7 @@ def get_descriptor_by_path( path: str, id: str, version_id: Optional[str] = None, - is_encoded: bool = False, + encode_path: bool = False, accept: str = 'application/json', token: Optional[str] = None ) -> Union[FileWrapper, Error]: @@ -1241,7 +1241,8 @@ def get_descriptor_by_path( registry. It is optional if a TRS URI is passed and includes version information. If provided nevertheless, then the `version_id` retrieved from the TRS URI is overridden. - is_encoded: Value of `path` is already percent/URL-encoded. + encode_path: Percent/URL-encode `path` (may be required by some + TRS implementations). accept: Requested content type. token: Bearer token for authentication. Set if required by TRS implementation and if not provided when instatiating client or @@ -1275,7 +1276,7 @@ def get_descriptor_by_path( ) # build request URL - _path = path if is_encoded else quote(path, safe='') + _path = quote(path, safe='') if encode_path else path url = ( f"{self.uri}/tools/{_id}/versions/{_version_id}/{type}/" f"descriptor/{_path}" @@ -1454,7 +1455,7 @@ def retrieve_files( type: str, id: str, version_id: Optional[str] = None, - is_encoded: bool = False, + encode_path: bool = False, token: Optional[str] = None, ) -> Dict[str, List[str]]: """Write tool version file contents for a given descriptor type to @@ -1475,8 +1476,8 @@ def retrieve_files( to this registry. It is optional if version info is included in the TRS URI. If passed, then the existing `version_id` retreived from the TRS URI is overridden. - is_encoded: Values or relative paths of files are already - percent/URL-encoded. + encode_path: Percent/URL-encode relative paths of files (may be + required by some TRS implementations). Returns: Dictionary of `FileType` enumerator values (e.g., `TEST_FILE`, @@ -1520,7 +1521,7 @@ def retrieve_files( path=_f.path, id=id, version_id=version_id, - is_encoded=is_encoded, + encode_path=encode_path, token=token, ) if not isinstance(file_wrapper, FileWrapper): From b36f26849cb70e644f5c43d2f5080f3300deb2c9 Mon Sep 17 00:00:00 2001 From: Alex Kanitz Date: Wed, 10 Nov 2021 19:25:24 +0100 Subject: [PATCH 3/5] feat: allow skipping response validation --- README.md | 13 ++++++----- tests/test_client.py | 9 ++++++++ trs_cli/__init__.py | 2 +- trs_cli/client.py | 51 ++++++++++++++++++++++++++++++++------------ 4 files changed, 55 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index c791e78..1c1145e 100644 --- a/README.md +++ b/README.md @@ -41,20 +41,23 @@ from trs_cli import TRSClient ### Configure client class It is possible to configure the `TRSClient` class with the `.config()` class -method. Currently there is only a single configuration parameter available, -the `debug` flag. Setting it to `True` (default: False) will have the exception -handler print tracebacks for every exception encountered. +method. The following configuration parameters are available: + +| Parameter | Type | Default | Description | +| --- | --- | ---- | --- | +| `debug` | `bool` | `False` | If set, the exception handler prints tracebacks for every exception encountered. | +| `no_validate` | `bool` | `False` | If set, responses are not validated. In that case, unserialized `response` objects are returned. | Example: ```py from trs_cli import TRSClient -TRSClient.config(debug=True) +TRSClient.config(debug=True, no_validate=True) ``` > Note that as a _class method_, the `.config()` method will affect _all_ -> client instances. +> client instances, including existing ones. ### Create client instance diff --git a/tests/test_client.py b/tests/test_client.py index 273051a..47ca736 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1122,6 +1122,15 @@ def test_connection_error(self, monkeypatch): url=MOCK_API, ) + def test_no_validation(self, requests_mock): + TRSClient.config(no_validate=True) + requests_mock.get(self.endpoint, text=MOCK_ID) + response = self.cli._send_request_and_validate_response( + url=MOCK_API, + ) + assert response.text == MOCK_ID + TRSClient.config(no_validate=False) + def test_get_str_validation(self, requests_mock): """Test for getter with string response.""" requests_mock.get(self.endpoint, json=MOCK_ID) diff --git a/trs_cli/__init__.py b/trs_cli/__init__.py index efcdcd7..8932252 100644 --- a/trs_cli/__init__.py +++ b/trs_cli/__init__.py @@ -1,3 +1,3 @@ -__version__ = '0.6.1' +__version__ = '0.7.0' from trs_cli.client import TRSClient # noqa: F401 diff --git a/trs_cli/client.py b/trs_cli/client.py index 82094c4..bd78d63 100644 --- a/trs_cli/client.py +++ b/trs_cli/client.py @@ -87,15 +87,26 @@ class TRSClient(): rf"(\/versions\/(?P{_RE_TRS_ID}))?$" ) + # class configuration variables + no_validate: bool = False + @classmethod def config( cls, debug: bool = False, - ): + no_validate: bool = False, + ) -> None: + """Class configuration. + + Args: + debug: Set to print error tracebacks. + no_validate: Set to skip validation of error responses. + """ if debug: sys.excepthook = partial(exception_handler, print_traceback=True) else: sys.excepthook = partial(exception_handler, print_traceback=False) + cls.no_validate = no_validate def __init__( self, @@ -1681,7 +1692,8 @@ def _validate_content_type( """ if requested_type not in available_types: raise ContentTypeUnavailable( - "Requested content type not provided by the service" + f"Requested content type '{requested_type}' not provided by " + f"the service; available types: {available_types}" ) def _send_request_and_validate_response( @@ -1693,7 +1705,11 @@ def _send_request_and_validate_response( validation_class_error: ModelMetaclass = Error, method: str = 'get', payload: Optional[Dict] = None, - ) -> Optional[Union[str, ModelMetaclass, List[ModelMetaclass]]]: + ) -> Optional[Union[ + str, + requests.models.Response, + ModelMetaclass, List[ModelMetaclass], + ]]: """Send a HTTP equest, validate the response and handle potential exceptions. @@ -1708,17 +1724,10 @@ def _send_request_and_validate_response( method: HTTP method to use for the request. Returns: - Unmarshalled response. + Unmarshalled response (default) or unserialized response if + class configuration flag `TRSClient.no_validate` is set. """ - # Process parameters - validation_type = "model" - if isinstance(validation_class_ok, tuple): - validation_class_ok = validation_class_ok[0] - validation_type = "list" - elif validation_class_ok is None: - validation_type = None - elif validation_class_ok is str: - validation_type = "str" + # Validate HTTP method try: request_func = eval('.'.join(['requests', method])) except AttributeError as e: @@ -1732,7 +1741,7 @@ def _send_request_and_validate_response( if payload is not None: kwargs['json'] = payload - # Send request and manage response + # Send request try: response = request_func(**kwargs) except ( @@ -1743,6 +1752,20 @@ def _send_request_and_validate_response( raise requests.exceptions.ConnectionError( "Could not connect to API endpoint" ) + + # skip validation + if TRSClient.no_validate: + return response + + # set validation parameters + validation_type = "model" + if isinstance(validation_class_ok, tuple): + validation_class_ok = validation_class_ok[0] + validation_type = "list" + elif validation_class_ok is None: + validation_type = None + elif validation_class_ok is str: + validation_type = "str" if response.status_code not in [200, 201]: logger.warning( f"Received error response: {response.status_code}" From 9c4f9a14f8701bd7d209109f97b2bfa4d0c6d7ae Mon Sep 17 00:00:00 2001 From: Alex Kanitz Date: Thu, 11 Nov 2021 19:02:11 +0100 Subject: [PATCH 4/5] feat: fully support non-JSON responses --- README.md | 21 ++-- tests/test_client.py | 131 +++++++++++++++++--- trs_cli/client.py | 289 +++++++++++++++++++++++++++++-------------- 3 files changed, 320 insertions(+), 121 deletions(-) diff --git a/README.md b/README.md index 1c1145e..469f6a3 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,14 @@ including support for additional endpoints defined in [ELIXIR Cloud & AAI's][res-elixir-cloud] generic [TRS-Filer][res-elixir-cloud-trs-filer] TRS implementation. +The TRS API version underlying the client can be found +[here][res-ga4gh-trs-version]. + +TRS-cli has so far been succesfully tested with the +[TRS-Filer][res-elixir-cloud-trs-filer] and +[WorkflowHub][res-eosc-workflow-hub] TRS implementations. WorkflowHub's public +TRS API endpoint can be found here: + ## Table of Contents * [Usage](#usage) @@ -46,7 +54,7 @@ method. The following configuration parameters are available: | Parameter | Type | Default | Description | | --- | --- | ---- | --- | | `debug` | `bool` | `False` | If set, the exception handler prints tracebacks for every exception encountered. | -| `no_validate` | `bool` | `False` | If set, responses are not validated. In that case, unserialized `response` objects are returned. | +| `no_validate` | `bool` | `False` | If set, responses JSON are not validated against the TRS API schemas. In that case, unserialized `response` objects are returned. Set this flag if the TRS implementation you are working with is not fully compliant with the TRS API specification. | Example: @@ -174,15 +182,6 @@ methods for additional endpoints implemented in | [`.delete_version()`][docs-api-delete_version] | `DELETE ​/tools​/{id}​/versions​/{version_id}` | Delete a tool version | | [`.post_service_info()`][docs-api-post_service_info] | `POST ​/service-info` | Register service info | -#### Convenience methods - -Finally, TRS-cli tries to provide convenience methods for common operations -that involve multiple API calls. Currently there is one such method defined: - -| Method | Description | -| --- | --- | -| [`.retrieve_files()`][docs-api-retrieve_files] | Retrieve all files associated with a given tool version and descriptor type. Useful for downloading workflows. | - ### Authorization Authorization [bearer tokens][res-bearer-token] can be provided either during @@ -309,8 +308,10 @@ question etc. [res-elixir-cloud-coc]: [res-elixir-cloud-contributing]: [res-elixir-cloud-trs-filer]: +[res-eosc-workflow-hub]: [res-ga4gh]: [res-ga4gh-trs]: +[res-ga4gh-trs-version]: [res-ga4gh-trs-uri]: [res-pydantic]: [res-pydantic-docs-export]: diff --git a/tests/test_client.py b/tests/test_client.py index 47ca736..a23053c 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -32,6 +32,7 @@ MOCK_API = f"{MOCK_HOST}:{MOCK_PORT}/{MOCK_BASE_PATH}" MOCK_ID = "123456" MOCK_ID_INVALID = "N0T VAL!D" +MOCK_TEXT_PLAIN = "SOME TEXT" MOCK_TRS_URI = f"trs://{MOCK_DOMAIN}/{MOCK_ID}" MOCK_TRS_URI_VERSIONED = f"trs://{MOCK_DOMAIN}/{MOCK_ID}/versions/{MOCK_ID}" MOCK_TOKEN = "MyT0k3n" @@ -489,7 +490,7 @@ class TestDeleteVersion: f"{cli.uri}/tools/{MOCK_ID}/versions/{MOCK_ID}" ) - def test_success(self, monkeypatch, requests_mock): + def test_success(self, requests_mock): """Returns 200 response.""" requests_mock.delete(self.endpoint, json=MOCK_ID) r = self.cli.delete_version( @@ -665,6 +666,10 @@ class TestGetDescriptorByPath: f"{cli.uri}/tools/{MOCK_ID}/versions/{MOCK_ID}/{MOCK_DESCRIPTOR}" f"/descriptor/{MOCK_ID}" ) + endpoint_plain = ( + f"{cli.uri}/tools/{MOCK_ID}/versions/{MOCK_ID}/PLAIN_{MOCK_DESCRIPTOR}" + f"/descriptor/{MOCK_ID}" + ) def test_success(self, requests_mock): """Returns 200 response.""" @@ -677,6 +682,17 @@ def test_success(self, requests_mock): ) assert r.dict() == MOCK_FILE_WRAPPER + def test_success_plain(self, requests_mock): + """Returns 200 response.""" + requests_mock.get(self.endpoint_plain, text=MOCK_TEXT_PLAIN) + r = self.cli.get_descriptor_by_path( + type=f"PLAIN_{MOCK_DESCRIPTOR}", + id=MOCK_ID, + version_id=MOCK_ID, + path=MOCK_ID, + ) + assert r.text == MOCK_TEXT_PLAIN + class TestGetFiles: """Test getter for files of a given descriptor type.""" @@ -713,17 +729,85 @@ def test_ContentTypeUnavailable(self, requests_mock): format=MOCK_ID, ) - def test_success_trs_uri_zip(self, requests_mock): + def test_success_trs_uri_zip(self, requests_mock, tmpdir): """Returns 200 ZIP response with TRS URI.""" - requests_mock.get(self.endpoint, json=[MOCK_TOOL_FILE]) + outfile = tmpdir / 'test.zip' + requests_mock.get(self.endpoint) r = self.cli.get_files( type=MOCK_DESCRIPTOR, id=MOCK_TRS_URI_VERSIONED, format='zip', + outfile=outfile, ) - if not isinstance(r, Error): - assert r[0].file_type.value == MOCK_TOOL_FILE['file_type'] - assert r[0].path == MOCK_TOOL_FILE['path'] + assert r == outfile + + def test_success_trs_uri_zip_default_filename( + self, + requests_mock, + monkeypatch, + ): + """Returns 200 ZIP response with default filename.""" + requests_mock.get(self.endpoint) + monkeypatch.setattr( + 'builtins.open', + lambda *args, **kwargs: _raise(Exception) + ) + with pytest.raises(Exception): + self.cli.get_files( + type=MOCK_DESCRIPTOR, + id=MOCK_TRS_URI_VERSIONED, + format='zip', + ) + + def test_zip_content_type(self, requests_mock, tmpdir): + """Test wrong content type.""" + outfile = tmpdir / 'test.zip' + requests_mock.get( + self.endpoint, + headers={'Content-Type': 'text/plain'}, + ) + r = self.cli.get_files( + type=MOCK_DESCRIPTOR, + id=MOCK_TRS_URI_VERSIONED, + format='zip', + outfile=outfile, + ) + print(r) + print(type(r)) + print(dir(r)) + print(r.headers) + assert isinstance(r, requests.models.Response) + + def test_zip_io_error(self, monkeypatch, requests_mock, tmpdir): + """Test I/O error.""" + outfile = tmpdir / 'test.zip' + requests_mock.get(self.endpoint) + monkeypatch.setattr( + 'builtins.open', + lambda *args, **kwargs: _raise(IOError) + ) + r = self.cli.get_files( + type=MOCK_DESCRIPTOR, + id=MOCK_TRS_URI_VERSIONED, + format='zip', + outfile=outfile, + ) + assert isinstance(r, requests.models.Response) + + def test_zip_connection_error(self, monkeypatch, tmpdir): + """Test connection error.""" + outfile = tmpdir / 'test.zip' + monkeypatch.setattr( + 'requests.get', + lambda *args, **kwargs: _raise(requests.exceptions.ConnectionError) + ) + with pytest.raises(requests.exceptions.ConnectionError): + self.cli.get_files( + type=MOCK_DESCRIPTOR, + id=MOCK_TRS_URI_VERSIONED, + format='zip', + outfile=outfile, + ) class TestGetTests: @@ -1108,6 +1192,9 @@ class TestSendRequestAndValidateRespose: """Test request sending and response validation.""" cli = TRSClient(uri=MOCK_TRS_URI) + cli.headers = { + 'Accept': 'application/json' + } endpoint = MOCK_API payload = {} @@ -1136,7 +1223,7 @@ def test_get_str_validation(self, requests_mock): requests_mock.get(self.endpoint, json=MOCK_ID) response = self.cli._send_request_and_validate_response( url=MOCK_API, - validation_class_ok=str, + json_validation_class=str, ) assert response == MOCK_ID @@ -1145,7 +1232,7 @@ def test_get_none_validation(self, requests_mock): requests_mock.get(self.endpoint, json=MOCK_ID) response = self.cli._send_request_and_validate_response( url=MOCK_API, - validation_class_ok=None, + json_validation_class=None, ) assert response is None @@ -1154,7 +1241,7 @@ def test_get_model_validation(self, requests_mock): requests_mock.get(self.endpoint, json=MOCK_TOOL) response = self.cli._send_request_and_validate_response( url=MOCK_API, - validation_class_ok=Tool, + json_validation_class=Tool, ) assert response == MOCK_TOOL @@ -1163,7 +1250,7 @@ def test_get_list_validation(self, requests_mock): requests_mock.get(self.endpoint, json=[MOCK_TOOL]) response = self.cli._send_request_and_validate_response( url=MOCK_API, - validation_class_ok=(Tool, ), + json_validation_class=(Tool, ), ) assert response == [MOCK_TOOL] @@ -1174,7 +1261,7 @@ def test_post_validation(self, requests_mock): url=MOCK_API, method='post', payload=self.payload, - validation_class_ok=str, + json_validation_class=str, ) assert response == MOCK_ID @@ -1184,7 +1271,7 @@ def test_get_success_invalid(self, requests_mock): with pytest.raises(InvalidResponseError): self.cli._send_request_and_validate_response( url=MOCK_API, - validation_class_ok=Error, + json_validation_class=Error, ) def test_error_response(self, requests_mock): @@ -1197,12 +1284,12 @@ def test_error_response(self, requests_mock): def test_error_response_invalid(self, requests_mock): """Test for error response that fails validation.""" - requests_mock.get(self.endpoint, json=MOCK_ERROR, status_code=400) + requests_mock.get(self.endpoint, json=MOCK_TOOL, status_code=400) with pytest.raises(InvalidResponseError): - self.cli._send_request_and_validate_response( + bla = self.cli._send_request_and_validate_response( url=MOCK_API, - validation_class_error=Tool, ) + print(bla) def test_invalid_http_method(self, requests_mock): """Test for invalid HTTP method.""" @@ -1213,3 +1300,17 @@ def test_invalid_http_method(self, requests_mock): method='non_existing', payload=self.payload, ) + + def test_text_plain_responses(self, requests_mock): + """Test for invalid HTTP method.""" + requests_mock.get( + self.endpoint, + text=MOCK_TEXT_PLAIN, + headers={'Content-Type': 'text/plain'}, + ) + self.cli.headers['Accept'] = 'text/plain' + r = self.cli._send_request_and_validate_response( + url=MOCK_API, + ) + assert r == MOCK_TEXT_PLAIN + self.cli.headers['Accept'] = 'application/json' diff --git a/trs_cli/client.py b/trs_cli/client.py index bd78d63..e6026e4 100644 --- a/trs_cli/client.py +++ b/trs_cli/client.py @@ -5,6 +5,7 @@ import logging from pathlib import Path import re +import shutil import socket import sys from typing import (Dict, List, Optional, Tuple, Type, Union) @@ -213,7 +214,7 @@ def get_service_info( # send request response = self._send_request_and_validate_response( url=url, - validation_class_ok=Service, + json_validation_class=Service, ) logger.info( "Retrieved service info" @@ -269,7 +270,7 @@ def post_tool_class( url=url, method='post', payload=payload, - validation_class_ok=str, + json_validation_class=str, ) logger.info( "Registered tool class" @@ -330,7 +331,7 @@ def put_tool_class( url=url, method='put', payload=payload, - validation_class_ok=str, + json_validation_class=str, ) logger.info( f"Registered tool class with id : {id}" @@ -380,7 +381,7 @@ def delete_tool_class( response = self._send_request_and_validate_response( url=url, method='delete', - validation_class_ok=str, + json_validation_class=str, ) logger.info( "Deleted tool class" @@ -436,7 +437,7 @@ def post_tool( url=url, method='post', payload=payload, - validation_class_ok=str, + json_validation_class=str, ) logger.info( "Registered tool" @@ -497,7 +498,7 @@ def put_tool( url=url, method='put', payload=payload, - validation_class_ok=str, + json_validation_class=str, ) logger.info( f"Registered tool with id: {id}" @@ -551,7 +552,7 @@ def delete_tool( response = self._send_request_and_validate_response( url=url, method='delete', - validation_class_ok=str, + json_validation_class=str, ) logger.info( "Deleted tool" @@ -613,7 +614,7 @@ def post_version( url=url, method='post', payload=payload, - validation_class_ok=str, + json_validation_class=str, ) logger.info( "Registered tool version" @@ -682,7 +683,7 @@ def put_version( url=url, method='put', payload=payload, - validation_class_ok=str, + json_validation_class=str, ) logger.info( f"Registered tool version with id {version_id} for tool {id}" @@ -748,7 +749,7 @@ def delete_version( response = self._send_request_and_validate_response( url=url, method='delete', - validation_class_ok=str, + json_validation_class=str, ) logger.info( "Deleted tool version" @@ -796,7 +797,7 @@ def get_tool_classes( # send request response = self._send_request_and_validate_response( url=url, - validation_class_ok=(ToolClass, ), + json_validation_class=(ToolClass, ), ) logger.info( "Retrieved tool classes" @@ -896,7 +897,7 @@ def get_tools( # send request response = self._send_request_and_validate_response( url=url, - validation_class_ok=(Tool, ), + json_validation_class=(Tool, ), ) logger.info( "Retrieved tools" @@ -908,7 +909,7 @@ def get_tool( id: str, accept: str = 'application/json', token: Optional[str] = None, - ) -> Union[Tool, Error]: + ) -> Union[Error, str, Tool]: """Retrieve tool with the specified identifier. Arguments: @@ -921,9 +922,9 @@ def get_tool( if expired. Returns: - Unmarshalled TRS response as either an instance of `Tool` - in case of a `200` response, or an instance of `Error` for all - other JSON reponses. + Unmarshalled TRS response as either an instance of `Tool` in case + of a `200` or `201` response, an instance of `Error` for all other + JSON reponses, and a string for 'text/plain' responses. Raises: requests.exceptions.ConnectionError: A connection to the provided @@ -951,7 +952,7 @@ def get_tool( # send request response = self._send_request_and_validate_response( url=url, - validation_class_ok=Tool, + json_validation_class=Tool, ) logger.info( "Retrieved tool" @@ -1006,7 +1007,7 @@ def get_versions( # send request response = self._send_request_and_validate_response( url=url, - validation_class_ok=(ToolVersion, ), + json_validation_class=(ToolVersion, ), ) logger.info( "Retrieved tool versions" @@ -1071,7 +1072,7 @@ def get_version( # send request response = self._send_request_and_validate_response( url=url, - validation_class_ok=ToolVersion, + json_validation_class=ToolVersion, ) logger.info( "Retrieved tool version" @@ -1140,7 +1141,7 @@ def get_containerfiles( # send request response = self._send_request_and_validate_response( url=url, - validation_class_ok=(FileWrapper, ), + json_validation_class=(FileWrapper, ), ) logger.info( "Retrieved containerfiles" @@ -1154,7 +1155,7 @@ def get_descriptor( version_id: Optional[str] = None, accept: str = 'application/json', token: Optional[str] = None - ) -> Union[FileWrapper, Error]: + ) -> Union[Error, FileWrapper, str]: """Retrieve the file wrapper for the primary descriptor of a specified tool version and descriptor type. @@ -1180,8 +1181,9 @@ def get_descriptor( Returns: Unmarshalled TRS response as either an instance of `FileWrapper` in - case of a `200` response, or an instance of `Error` for all other - JSON reponses. + case of a `200` or `201` response, an instance of `Error` for all + other JSON reponses, or a string with file contents for + `text/plain` responses. Raises: requests.exceptions.ConnectionError: A connection to the provided @@ -1215,7 +1217,7 @@ def get_descriptor( # send request response = self._send_request_and_validate_response( url=url, - validation_class_ok=FileWrapper, + json_validation_class=FileWrapper, ) logger.info( "Retrieved descriptor" @@ -1229,9 +1231,9 @@ def get_descriptor_by_path( id: str, version_id: Optional[str] = None, encode_path: bool = False, - accept: str = 'application/json', + accept: Optional[str] = None, token: Optional[str] = None - ) -> Union[FileWrapper, Error]: + ) -> Union[Error, FileWrapper, str]: """Retrieve the file wrapper for an indicated file for the specified tool version and descriptor type. @@ -1240,7 +1242,9 @@ def get_descriptor_by_path( the bare descriptor while the "non-plain" types return a descriptor wrapped with metadata. Allowed values include "CWL", "WDL", "NFL", "GALAXY", "SMK", "PLAIN_CWL", "PLAIN_WDL", - "PLAIN_NFL", "PLAIN_GALAXY", "PLAIN_SMK". + "PLAIN_NFL", "PLAIN_GALAXY", "PLAIN_SMK". Setting one of the + "PLAIN_" types will set the default accepted content type to + "text/plain" (usually "application/json"). path: Path, including filename, of descriptor or associated file relative to the primary descriptor file. id: A unique identifier of the tool, scoped to this registry OR @@ -1261,8 +1265,9 @@ def get_descriptor_by_path( Returns: Unmarshalled TRS response as either an instance of `FileWrapper` in - case of a `200` response, or an instance of `Error` for all other - JSON reponses. + case of a `200` or `201` response, an instance of `Error` for all + other JSON reponses, and a string with file contents for + 'text/plain' responses. Raises: requests.exceptions.ConnectionError: A connection to the provided @@ -1271,6 +1276,11 @@ def get_descriptor_by_path( validated against the API schema. """ # validate requested content type and get request headers + if accept is None: + if type.startswith("PLAIN_"): + accept = 'text/plain' + else: + accept = 'application/json' self._validate_content_type( requested_type=accept, available_types=['application/json', 'text/plain'], @@ -1297,7 +1307,7 @@ def get_descriptor_by_path( # send request response = self._send_request_and_validate_response( url=url, - validation_class_ok=FileWrapper, + json_validation_class=FileWrapper, ) logger.info( "Retrieved descriptor" @@ -1310,10 +1320,15 @@ def get_files( id: str, version_id: Optional[str] = None, format: Optional[str] = None, - token: Optional[str] = None - ) -> Union[List[ToolFile], Error]: - """Retrieve file information for the specified tool version and - descriptor type. + outfile: Optional[Path] = None, + token: Optional[str] = None, + ) -> Union[ + List[ToolFile], + Error, + Path, + requests.models.Response, + ]: + """Retrieve file information or ZIP archive of all files. Arguments: type: The output type of the descriptor. Plain types return @@ -1330,20 +1345,26 @@ def get_files( retreived from the TRS URI is overridden. format: Returns a zip file of all files when format=zip is specified. + outfile: Name of zip archive when `format` is set to 'zip'. Ignored + otherwise. If not specified, the filename is set based on the + URL by taking the part after the last '/' and stored in the + current working directory. token: Bearer token for authentication. Set if required by TRS implementation and if not provided when instatiating client or if expired. Returns: Unmarshalled TRS response as either a list of instances of - `ToolFile` in case of a `200` response, or an instance of `Error` - for all other JSON reponses. + `ToolFile` in case of a `200` response, an instance of `Error` + for all other JSON reponses, and the absolute path of the ZIP + archive if `format` is set to 'zip'. Raises: requests.exceptions.ConnectionError: A connection to the provided TRS instance could not be established. trs_cli.errors.InvalidResponseError: The response could not be validated against the API schema. + IOError: The ZIP archive could not be written. """ # validate requested content type and get request headers if format is None: @@ -1375,15 +1396,76 @@ def get_files( ) logger.info(f"Connecting to '{url}'...") - # send request - response = self._send_request_and_validate_response( - url=url, - validation_class_ok=(ToolFile, ), - ) - logger.info( - "Retrieved files" - ) - return response # type: ignore + # send/validate request for 'application/zip' + if format == "zip": + + # set output filename + if outfile is None: + outfile = Path.cwd() / url.split('/')[-1] + outfile = Path(re.sub(r'\?format=zip$', '', str(outfile))) + if not str(outfile).endswith('.zip'): + outfile = outfile.with_suffix(outfile.suffix + '.zip') + + # send request + try: + with requests.get( + url=url, + headers=self.headers, + stream=True, + ) as response: + logger.info( + f"Status code response: {response.status_code}" + ) + + # check content type + try: + content_type = response.headers['Content-Type'] + except (AttributeError, KeyError): + logger.warning( + "No content type set for response; assuming " + "'application/zip'" + ) + content_type = 'application/zip' + logger.info(f"Content type of response: {content_type}") + if not content_type.startswith('application/zip'): + logger.warning( + f"The content type of the response " + "('{content_type}') does not match the requested " + f"content type '{self.headers['Accept']}'; " + "returning the unmarshalled/unserialized response " + "object" + ) + return response + + # copy output + try: + with open(outfile, 'wb') as f: + shutil.copyfileobj(response.raw, f) + except IOError: + logger.warning( + "Could not write output file; returning the " + "unmarshalled/unserialized response object" + ) + return response + except ( + requests.exceptions.ConnectionError, + socket.gaierror, + urllib3.exceptions.NewConnectionError, + ) as exc: + raise requests.exceptions.ConnectionError( + "Could not connect to API endpoint" + ) from exc + logger.info("Retrieved ZIP archive") + return outfile + + # send/validate request for 'applicaton/json' + else: + response = self._send_request_and_validate_response( + url=url, + json_validation_class=(ToolFile, ), + ) + logger.info("Retrieved file info") + return response # type: ignore def get_tests( self, @@ -1392,7 +1474,7 @@ def get_tests( version_id: Optional[str] = None, accept: str = 'application/json', token: Optional[str] = None - ) -> Union[List[FileWrapper], Error]: + ) -> Union[Error, List[FileWrapper], str]: """Retrieve the file wrappers for all tests associated with a specified tool version and descriptor type. @@ -1418,8 +1500,9 @@ def get_tests( Returns: Unmarshalled TRS response as either a list of `FileWrapper` - instances in case of a `200` response, or an instance of `Error` - for all other JSON reponses. + instances in case of a `200` or `201` response, an instance of + `Error` for all other JSON reponses, and a string for 'text/plain' + responses. Raises: requests.exceptions.ConnectionError: A connection to the provided @@ -1453,7 +1536,7 @@ def get_tests( # send request response = self._send_request_and_validate_response( url=url, - validation_class_ok=(FileWrapper, ), + json_validation_class=(FileWrapper, ), ) logger.info( "Retrieved tests" @@ -1472,6 +1555,8 @@ def retrieve_files( """Write tool version file contents for a given descriptor type to files. + DEPRECATED: Use `.get_files` with `format=zip` instead. + Arguments: out_dir: Directory to write requested files to. Will be attempted to create if it does not exist. @@ -1545,7 +1630,7 @@ def retrieve_files( for path, content in file_wrappers.items(): out_path = out_dir / path try: - with open(out_path, "w") as _fp: + with open(out_path, 'w') as _fp: _fp.write(content) except OSError: raise OSError(f"Could not write file '{str(out_path)}'") @@ -1699,16 +1784,17 @@ def _validate_content_type( def _send_request_and_validate_response( self, url: str, - validation_class_ok: Optional[ + json_validation_class: Optional[ Union[ModelMetaclass, Tuple[ModelMetaclass], Type[str]] ] = None, - validation_class_error: ModelMetaclass = Error, method: str = 'get', payload: Optional[Dict] = None, + success_codes: Optional[List] = None, ) -> Optional[Union[ str, requests.models.Response, - ModelMetaclass, List[ModelMetaclass], + ModelMetaclass, + List[ModelMetaclass], ]]: """Send a HTTP equest, validate the response and handle potential exceptions. @@ -1727,11 +1813,13 @@ def _send_request_and_validate_response( Unmarshalled response (default) or unserialized response if class configuration flag `TRSClient.no_validate` is set. """ - # Validate HTTP method + # Validate input parameters try: request_func = eval('.'.join(['requests', method])) except AttributeError as e: - raise AttributeError("Illegal HTTP method provided.") from e + raise AttributeError("Illegal HTTP method provided") from e + if success_codes is None: + success_codes = [200, 201] # Compile request arguments kwargs = { @@ -1752,49 +1840,58 @@ class configuration flag `TRSClient.no_validate` is set. raise requests.exceptions.ConnectionError( "Could not connect to API endpoint" ) + logger.info( + f"Status code response: {response.status_code}" + ) + + # get content type + try: + content_type = response.headers['Content-Type'] + except (AttributeError, KeyError): + logger.warning( + "No content type set for response; assuming 'application/json'" + ) + content_type = 'application/json' + logger.info(f"Content type of response: {content_type}") + if not content_type.startswith(self.headers['Accept']): + logger.warning( + f"The content type of the response ('{content_type}') does " + "not match the requested content type " + f"'{self.headers['Accept']}'; returning the " + "unmarshalled/unserialized response object" + ) + return response # skip validation if TRSClient.no_validate: return response - # set validation parameters - validation_type = "model" - if isinstance(validation_class_ok, tuple): - validation_class_ok = validation_class_ok[0] - validation_type = "list" - elif validation_class_ok is None: - validation_type = None - elif validation_class_ok is str: - validation_type = "str" - if response.status_code not in [200, 201]: - logger.warning( - f"Received error response: {response.status_code}" - ) - try: - return validation_class_error(**response.json()) - except ( - json.decoder.JSONDecodeError, - pydantic.ValidationError, - ) as exc: - raise InvalidResponseError( - "Response could not be validated against API schema" - ) from exc - else: - try: - if validation_type == "list": - return [ - validation_class_ok(**obj) for obj in response.json() - ] # type: ignore - elif validation_type == "str": - return str(response.json()) - elif validation_type is None: - return None - else: - return validation_class_ok(**response.json()) - except ( - json.decoder.JSONDecodeError, - pydantic.ValidationError, - ) as exc: - raise InvalidResponseError( - "Response could not be validated against API schema" - ) from exc + # process 'text/plain' responses + if content_type.startswith("text/plain"): + logger.info("Returning string response") + return response.text + + # validate JSON + try: + if response.status_code not in success_codes: + logger.warning( + f"Received error response: {response.status_code}" + ) + return Error(**response.json()) + if isinstance(json_validation_class, tuple): + return [ + json_validation_class[0](**obj) for obj in response.json() + ] # type: ignore + elif json_validation_class is None: + return None + elif json_validation_class is str: + return str(response.json()) + else: + return json_validation_class(**response.json()) + except ( + json.decoder.JSONDecodeError, + pydantic.ValidationError, + ) as exc: + raise InvalidResponseError( + "Response could not be validated against API schema" + ) from exc From f9babab59769f5767103c891fdb966ba3e27cafb Mon Sep 17 00:00:00 2001 From: Alex Kanitz Date: Thu, 11 Nov 2021 19:10:52 +0100 Subject: [PATCH 5/5] feat: support new versioned TRS URI format See PR: https://github.com/ga4gh/tool-registry-service-schemas/pull/202 --- tests/test_client.py | 2 +- trs_cli/client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_client.py b/tests/test_client.py index a23053c..9f48242 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -34,7 +34,7 @@ MOCK_ID_INVALID = "N0T VAL!D" MOCK_TEXT_PLAIN = "SOME TEXT" MOCK_TRS_URI = f"trs://{MOCK_DOMAIN}/{MOCK_ID}" -MOCK_TRS_URI_VERSIONED = f"trs://{MOCK_DOMAIN}/{MOCK_ID}/versions/{MOCK_ID}" +MOCK_TRS_URI_VERSIONED = f"trs://{MOCK_DOMAIN}/{MOCK_ID}/{MOCK_ID}" MOCK_TOKEN = "MyT0k3n" MOCK_DESCRIPTOR = "CWL" MOCK_RESPONSE_INVALID = {"not": "valid"} diff --git a/trs_cli/client.py b/trs_cli/client.py index e6026e4..7fd660b 100644 --- a/trs_cli/client.py +++ b/trs_cli/client.py @@ -85,7 +85,7 @@ class TRSClient(): ) _RE_TRS_URI_OR_TOOL_ID = ( rf"^(trs:\/\/{_RE_DOMAIN}\/)?(?P{_RE_TRS_ID})" - rf"(\/versions\/(?P{_RE_TRS_ID}))?$" + rf"(\/(?P{_RE_TRS_ID}))?$" ) # class configuration variables