From 51d1925d33f5716373960fd098670e28b19f8199 Mon Sep 17 00:00:00 2001 From: Matthew Barnes Date: Thu, 8 Jun 2017 12:51:45 -0400 Subject: [PATCH] storage: Add CustodiaStoreHandler. Built-in handler for all SecretModel types. Cannot be overridden in service configuration. --- src/commissaire_service/storage/__init__.py | 32 +++- src/commissaire_service/storage/custodia.py | 185 ++++++++++++++++++++ test/test_service_storage.py | 50 +++++- 3 files changed, 262 insertions(+), 5 deletions(-) create mode 100644 src/commissaire_service/storage/custodia.py diff --git a/src/commissaire_service/storage/__init__.py b/src/commissaire_service/storage/__init__.py index bc69d6a..fd53d3e 100644 --- a/src/commissaire_service/storage/__init__.py +++ b/src/commissaire_service/storage/__init__.py @@ -25,6 +25,8 @@ from commissaire_service.service import ( CommissaireService, add_service_arguments) +from .custodia import CustodiaStoreHandler + class StorageService(CommissaireService): """ @@ -80,6 +82,29 @@ def __init__(self, exchange_name, connection_url, config_file=None): if isinstance(v, type) and issubclass(v, models.Model)} + # Prepare CustodiaStoreHandler configuration. + # + # Pick out all the 'custodia_*' items from the root-level JSON + # configuration and discard the prefix for the Custodia handler. + prefix = 'custodia_' + config = {k[len(prefix):]: v + for k, v in self._config_data.items() + if k.startswith(prefix)} + + # Bind SecretModel types to the built-in CustodiaStoreHandler. + # Do this early in case user data tries to specify these types; + # would raise a ConfigurationError in _register_store_handler(). + matched_types = set() + for mt in self._model_types.values(): + if issubclass(mt, models.SecretModel): + matched_types.add(mt) + handler_type = CustodiaStoreHandler + config['name'] = handler_type.__module__ + definition = (handler_type, config, matched_types) + self._definitions_by_name[config['name']] = definition + new_items = {mt: definition for mt in matched_types} + self._definitions_by_model_type.update(new_items) + store_handlers = self._config_data.get('storage_handlers', []) # Configure store handlers from user data. @@ -117,10 +142,13 @@ def _register_store_handler(self, config): handler_type = import_plugin( module_name, 'commissaire.storage', StoreHandlerBase) - # Match model types to type name patterns. + # Match (non-secret) model types to type name patterns. matched_types = set() + configurable_model_names = [ + k for k, v in self._model_types.items() + if not issubclass(v, models.SecretModel)] for pattern in config.pop('models', ['*']): - matches = fnmatch.filter(self._model_types.keys(), pattern) + matches = fnmatch.filter(configurable_model_names, pattern) if not matches: raise ConfigurationError( 'No match for model: {}'.format(pattern)) diff --git a/src/commissaire_service/storage/custodia.py b/src/commissaire_service/storage/custodia.py new file mode 100644 index 0000000..3b8fad3 --- /dev/null +++ b/src/commissaire_service/storage/custodia.py @@ -0,0 +1,185 @@ +# Copyright (C) 2016-2017 Red Hat, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Custodia based StoreHandler. +""" + +import requests + +from urllib.parse import quote + +from commissaire.bus import StorageLookupError +from commissaire.storage import StoreHandlerBase +from commissaire.util.unixadapter import UnixAdapter + + +HTTP_SOCKET_PREFIX = 'http+unix://' +DEFAULT_SOCKET_PATH = '/var/run/custodia/custodia.sock' + + +class CustodiaStoreHandler(StoreHandlerBase): + """ + Handler for securely storing secrets via a local Custodia service. + """ + + # Connection should be nearly instantaneous. + CUSTODIA_TIMEOUT = (1.0, 5.0) # seconds + + @classmethod + def check_config(cls, config): + """ + This store handler has no configuration checks. + """ + pass + + def __init__(self, config): + """ + Creates a new instance of CustodiaStoreHandler. + :param config: Not applicable to this handler + :type config: None + """ + super().__init__(config) + + self.session = requests.Session() + self.session.headers['REMOTE_USER'] = 'commissaire' + self.session.mount(HTTP_SOCKET_PREFIX, UnixAdapter()) + socket_path = config.get('socket_path', DEFAULT_SOCKET_PATH) + self.socket_url = HTTP_SOCKET_PREFIX + quote(socket_path, safe='') + + def _build_key_container_url(self, model_instance): + """ + Builds a Custodia key container URL for the given SecretModel. + + :param model_instance: A SecretModel instance. + :type model_instance: commissaire.model.SecretModel + :returns: A URL string + :rtype: str + """ + return '{}/secrets/{}/'.format( + self.socket_url, model_instance._key_container) + + def _build_key_url(self, model_instance): + """ + Builds a Custodia key URL for the given SecretModel. + + :param model_instance: A SecretModel instance. + :type model_instance: commissaire.model.SecretModel + :returns: A URL string + :rtype: str + """ + base_url = self._build_key_container_url(model_instance) + return base_url + model_instance.primary_key + + def _save(self, model_instance): + """ + Submits a serialized SecretModel string to Custodia and returns the + model instance. + + :param model_instance: SecretModel instance to save. + :type model_instance: commissaire.model.SecretModel + :returns: The saved model instance. + :rtype: commissaire.model.SecretModel + :raises requests.HTTPError: if the request fails + """ + # Create a key container for the model. If it already exists, + # catch the failure and move on. This operation should really + # be idempotent, but Custodia returns a 409 Conflict. + # (see https://github.com/latchset/custodia/issues/206) + try: + url = self._build_key_container_url(model_instance) + + response = self.session.request( + 'POST', url, timeout=self.CUSTODIA_TIMEOUT) + response.raise_for_status() + except requests.HTTPError as error: + # XXX bool(response) defers to response.ok, which is a misfeature. + # Have to explicitly test "if response is None" to know if the + # object is there. + have_response = response is not None + if not (have_response and error.response.status_code == 409): + raise error + + data = model_instance.to_json() + headers = { + 'Content-Type': 'application/octet-stream', + 'Content-Length': str(len(data)) + } + url = self._build_key_url(model_instance) + + response = self.session.request( + 'PUT', url, headers=headers, data=data, + timeout=self.CUSTODIA_TIMEOUT) + response.raise_for_status() + + return model_instance + + def _get(self, model_instance): + """ + Retrieves a serialized SecretModel string from Custodia and constructs + a model instance. + + :param model_instance: SecretModel instance to search and get. + :type model_instance: commissaire.model.SecretModel + :returns: The saved model instance. + :rtype: commissaire.model.SecretModel + :raises StorageLookupError: if data lookup fails (404 Not Found) + :raises requests.HTTPError: if the request fails (other than 404) + """ + headers = { + 'Accept': 'application/octet-stream' + } + url = self._build_key_url(model_instance) + + try: + response = self.session.request( + 'GET', url, headers=headers, + timeout=self.CUSTODIA_TIMEOUT) + response.raise_for_status() + + return model_instance.new(**response.json()) + except requests.HTTPError as error: + # XXX bool(response) defers to response.ok, which is a misfeature. + # Have to explicitly test "if response is None" to know if the + # object is there. + have_response = response is not None + if have_response and error.response.status_code == 404: + raise StorageLookupError(str(error), model_instance) + else: + raise error + + def _delete(self, model_instance): + """ + Deletes a serialized SecretModel string from Custodia. + + :param model_instance: SecretModel instance to delete. + :type model_instance: commissaire.model.SecretModel + :raises StorageLookupError: if data lookup fails (404 Not Found) + :raises requests.HTTPError: if the request fails (other than 404) + """ + url = self._build_key_url(model_instance) + + try: + response = self.session.request( + 'DELETE', url, timeout=self.CUSTODIA_TIMEOUT) + response.raise_for_status() + except requests.HTTPError as error: + # XXX bool(response) defers to response.ok, which is a misfeature. + # Have to explicitly test "if response is None" to know if the + # object is there. + have_response = response is not None + if have_response and error.response.status_code == 404: + raise StorageLookupError(str(error), model_instance) + else: + raise error diff --git a/test/test_service_storage.py b/test/test_service_storage.py index 37253df..e680129 100644 --- a/test/test_service_storage.py +++ b/test/test_service_storage.py @@ -24,6 +24,12 @@ from commissaire.storage import StoreHandlerBase from commissaire.util.config import ConfigurationError from commissaire_service.storage import StorageService +from commissaire_service.storage.custodia import CustodiaStoreHandler + + +SECRET_MODEL_TYPES = ( + models.SecretModel, + models.HostCreds) class StoreHandlerTest(StoreHandlerBase): @@ -83,6 +89,22 @@ def test_register_store_handler(self): definitions_by_model_type = \ self.service_instance._definitions_by_model_type + # Factor builtin definitions in our final checks. + builtin_by_name = len(definitions_by_name) + builtin_by_model_type = len(definitions_by_model_type) + + # Verify SecretModel registrations are rejected. + for model_type in SECRET_MODEL_TYPES: + self.assertTrue(issubclass(model_type, models.SecretModel)) + config = { + 'type': 'test', + 'models': [model_type.__name__] + } + self.assertRaises( + ConfigurationError, + self.service_instance._register_store_handler, + config) + # Valid registration, implicit name. implicit_name = __name__ model_type = models.Host @@ -163,8 +185,12 @@ def test_register_store_handler(self): config) # Verify StoreHandlerManager state. - self.assertEquals(len(definitions_by_name), 4) - self.assertEquals(len(definitions_by_model_type), 6) + self.assertEquals( + len(definitions_by_name), + 4 + builtin_by_name) + self.assertEquals( + len(definitions_by_model_type), + 6 + builtin_by_model_type) expect_handlers = [ (StoreHandlerTest, {'name': __name__}, @@ -183,10 +209,23 @@ def test_register_store_handler(self): ] # Note, actual_handlers is unordered. actual_handlers = list(definitions_by_name.values()) - self.assertEquals(len(actual_handlers), 4) + self.assertEquals(len(actual_handlers), 4 + builtin_by_name) for handler in expect_handlers: self.assertIn(handler, actual_handlers) + def test_register_store_handler_wildcards(self): + """ + Verify wildcard patterns in "models" excludes SecretModels + """ + config = { + 'type': 'test', + 'models': ['*'] + } + # This would throw a ConfigurationError if SecretModel types + # WERE included, since the matched types would conflict with + # pre-registered SecretModel types. + self.service_instance._register_store_handler(config) + def test_get_handler(self): """ Verify StorageService._get_handler() works as intended @@ -241,6 +280,11 @@ def test_get_handler(self): self.service_instance._get_handler, model) + # Verify HostCreds instance returns CustodiaStoreHandler. + model = models.HostCreds.new(address='127.0.0.1') + handler = self.service_instance._get_handler(model) + self.assertIsInstance(handler, CustodiaStoreHandler) + @mock.patch('commissaire_service.storage.StorageService._get_handler') def test_on_get_with_dict(self, get_handler): """