storage: Add CustodiaStoreHandler.

Built-in handler for all SecretModel types.
Cannot be overridden in service configuration.

src/commissaire_service/storage/__init__.py

@@ -25,6 +25,8 @@
 from commissaire_service.service import (
     CommissaireService, add_service_arguments)
 
+from .custodia import CustodiaStoreHandler
+
 
 class StorageService(CommissaireService):
     """
@@ -80,6 +82,29 @@ def __init__(self, exchange_name, connection_url, config_file=None):
                              if isinstance(v, type) and
                              issubclass(v, models.Model)}
 
+        # Prepare CustodiaStoreHandler configuration.
+        #
+        # Pick out all the 'custodia_*' items from the root-level JSON
+        # configuration and discard the prefix for the Custodia handler.
+        prefix = 'custodia_'
+        config = {k[len(prefix):]: v
+                  for k, v in self._config_data.items()
+                  if k.startswith(prefix)}
+
+        # Bind SecretModel types to the built-in CustodiaStoreHandler.
+        # Do this early in case user data tries to specify these types;
+        # would raise a ConfigurationError in _register_store_handler().
+        matched_types = set()
+        for mt in self._model_types.values():
+            if issubclass(mt, models.SecretModel):
+                matched_types.add(mt)
+        handler_type = CustodiaStoreHandler
+        config['name'] = handler_type.__module__
+        definition = (handler_type, config, matched_types)
+        self._definitions_by_name[config['name']] = definition
+        new_items = {mt: definition for mt in matched_types}
+        self._definitions_by_model_type.update(new_items)
+
         store_handlers = self._config_data.get('storage_handlers', [])
 
         # Configure store handlers from user data.
@@ -117,10 +142,13 @@ def _register_store_handler(self, config):
         handler_type = import_plugin(
             module_name, 'commissaire.storage', StoreHandlerBase)
 
-        # Match model types to type name patterns.
+        # Match (non-secret) model types to type name patterns.
         matched_types = set()
+        configurable_model_names = [
+            k for k, v in self._model_types.items()
+            if not issubclass(v, models.SecretModel)]
         for pattern in config.pop('models', ['*']):
-            matches = fnmatch.filter(self._model_types.keys(), pattern)
+            matches = fnmatch.filter(configurable_model_names, pattern)
             if not matches:
                 raise ConfigurationError(
                     'No match for model: {}'.format(pattern))

src/commissaire_service/storage/custodia.py

@@ -0,0 +1,185 @@
+# Copyright (C) 2016-2017  Red Hat, Inc
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Custodia based StoreHandler.
+"""
+
+import requests
+
+from urllib.parse import quote
+
+from commissaire.bus import StorageLookupError
+from commissaire.storage import StoreHandlerBase
+from commissaire.util.unixadapter import UnixAdapter
+
+
+HTTP_SOCKET_PREFIX = 'http+unix://'
+DEFAULT_SOCKET_PATH = '/var/run/custodia/custodia.sock'
+
+
+class CustodiaStoreHandler(StoreHandlerBase):
+    """
+    Handler for securely storing secrets via a local Custodia service.
+    """
+
+    # Connection should be nearly instantaneous.
+    CUSTODIA_TIMEOUT = (1.0, 5.0)  # seconds
+
+    @classmethod
+    def check_config(cls, config):
+        """
+        This store handler has no configuration checks.
+        """
+        pass
+
+    def __init__(self, config):
+        """
+        Creates a new instance of CustodiaStoreHandler.
+        :param config: Not applicable to this handler
+        :type config: None
+        """
+        super().__init__(config)
+
+        self.session = requests.Session()
+        self.session.headers['REMOTE_USER'] = 'commissaire'
+        self.session.mount(HTTP_SOCKET_PREFIX, UnixAdapter())
+        socket_path = config.get('socket_path', DEFAULT_SOCKET_PATH)
+        self.socket_url = HTTP_SOCKET_PREFIX + quote(socket_path, safe='')
+
+    def _build_key_container_url(self, model_instance):
+        """
+        Builds a Custodia key container URL for the given SecretModel.
+
+        :param model_instance: A SecretModel instance.
+        :type model_instance: commissaire.model.SecretModel
+        :returns: A URL string
+        :rtype: str
+        """
+        return '{}/secrets/{}/'.format(
+            self.socket_url, model_instance._key_container)
+
+    def _build_key_url(self, model_instance):
+        """
+        Builds a Custodia key URL for the given SecretModel.
+
+        :param model_instance: A SecretModel instance.
+        :type model_instance: commissaire.model.SecretModel
+        :returns: A URL string
+        :rtype: str
+        """
+        base_url = self._build_key_container_url(model_instance)
+        return base_url + model_instance.primary_key
+
+    def _save(self, model_instance):
+        """
+        Submits a serialized SecretModel string to Custodia and returns the
+        model instance.
+
+        :param model_instance: SecretModel instance to save.
+        :type model_instance: commissaire.model.SecretModel
+        :returns: The saved model instance.
+        :rtype: commissaire.model.SecretModel
+        :raises requests.HTTPError: if the request fails
+        """
+        # Create a key container for the model.  If it already exists,
+        # catch the failure and move on.  This operation should really
+        # be idempotent, but Custodia returns a 409 Conflict.
+        # (see https://github.com/latchset/custodia/issues/206)
+        try:
+            url = self._build_key_container_url(model_instance)
+
+            response = self.session.request(
+                'POST', url, timeout=self.CUSTODIA_TIMEOUT)
+            response.raise_for_status()
+        except requests.HTTPError as error:
+            # XXX bool(response) defers to response.ok, which is a misfeature.
+            #     Have to explicitly test "if response is None" to know if the
+            #     object is there.
+            have_response = response is not None
+            if not (have_response and error.response.status_code == 409):
+                raise error
+
+        data = model_instance.to_json()
+        headers = {
+            'Content-Type': 'application/octet-stream',
+            'Content-Length': str(len(data))
+        }
+        url = self._build_key_url(model_instance)
+
+        response = self.session.request(
+            'PUT', url, headers=headers, data=data,
+            timeout=self.CUSTODIA_TIMEOUT)
+        response.raise_for_status()
+
+        return model_instance
+
+    def _get(self, model_instance):
+        """
+        Retrieves a serialized SecretModel string from Custodia and constructs
+        a model instance.
+
+        :param model_instance: SecretModel instance to search and get.
+        :type model_instance: commissaire.model.SecretModel
+        :returns: The saved model instance.
+        :rtype: commissaire.model.SecretModel
+        :raises StorageLookupError: if data lookup fails (404 Not Found)
+        :raises requests.HTTPError: if the request fails (other than 404)
+        """
+        headers = {
+            'Accept': 'application/octet-stream'
+        }
+        url = self._build_key_url(model_instance)
+
+        try:
+            response = self.session.request(
+                'GET', url, headers=headers,
+                timeout=self.CUSTODIA_TIMEOUT)
+            response.raise_for_status()
+
+            return model_instance.new(**response.json())
+        except requests.HTTPError as error:
+            # XXX bool(response) defers to response.ok, which is a misfeature.
+            #     Have to explicitly test "if response is None" to know if the
+            #     object is there.
+            have_response = response is not None
+            if have_response and error.response.status_code == 404:
+                raise StorageLookupError(str(error), model_instance)
+            else:
+                raise error
+
+    def _delete(self, model_instance):
+        """
+        Deletes a serialized SecretModel string from Custodia.
+
+        :param model_instance: SecretModel instance to delete.
+        :type model_instance: commissaire.model.SecretModel
+        :raises StorageLookupError: if data lookup fails (404 Not Found)
+        :raises requests.HTTPError: if the request fails (other than 404)
+        """
+        url = self._build_key_url(model_instance)
+
+        try:
+            response = self.session.request(
+                'DELETE', url, timeout=self.CUSTODIA_TIMEOUT)
+            response.raise_for_status()
+        except requests.HTTPError as error:
+            # XXX bool(response) defers to response.ok, which is a misfeature.
+            #     Have to explicitly test "if response is None" to know if the
+            #     object is there.
+            have_response = response is not None
+            if have_response and error.response.status_code == 404:
+                raise StorageLookupError(str(error), model_instance)
+            else:
+                raise error

test/test_service_storage.py

@@ -24,6 +24,12 @@
 from commissaire.storage import StoreHandlerBase
 from commissaire.util.config import ConfigurationError
 from commissaire_service.storage import StorageService
+from commissaire_service.storage.custodia import CustodiaStoreHandler
+
+
+SECRET_MODEL_TYPES = (
+    models.SecretModel,
+    models.HostCreds)
 
 
 class StoreHandlerTest(StoreHandlerBase):
@@ -83,6 +89,22 @@ def test_register_store_handler(self):
         definitions_by_model_type = \
             self.service_instance._definitions_by_model_type
 
+        # Factor builtin definitions in our final checks.
+        builtin_by_name = len(definitions_by_name)
+        builtin_by_model_type = len(definitions_by_model_type)
+
+        # Verify SecretModel registrations are rejected.
+        for model_type in SECRET_MODEL_TYPES:
+            self.assertTrue(issubclass(model_type, models.SecretModel))
+            config = {
+                'type': 'test',
+                'models': [model_type.__name__]
+            }
+            self.assertRaises(
+                ConfigurationError,
+                self.service_instance._register_store_handler,
+                config)
+
         # Valid registration, implicit name.
         implicit_name = __name__
         model_type = models.Host
@@ -163,8 +185,12 @@ def test_register_store_handler(self):
             config)
 
         # Verify StoreHandlerManager state.
-        self.assertEquals(len(definitions_by_name), 4)
-        self.assertEquals(len(definitions_by_model_type), 6)
+        self.assertEquals(
+            len(definitions_by_name),
+            4 + builtin_by_name)
+        self.assertEquals(
+            len(definitions_by_model_type),
+            6 + builtin_by_model_type)
         expect_handlers = [
             (StoreHandlerTest,
                 {'name': __name__},
@@ -183,10 +209,23 @@ def test_register_store_handler(self):
         ]
         # Note, actual_handlers is unordered.
         actual_handlers = list(definitions_by_name.values())
-        self.assertEquals(len(actual_handlers), 4)
+        self.assertEquals(len(actual_handlers), 4 + builtin_by_name)
         for handler in expect_handlers:
             self.assertIn(handler, actual_handlers)
 
+    def test_register_store_handler_wildcards(self):
+        """
+        Verify wildcard patterns in "models" excludes SecretModels
+        """
+        config = {
+            'type': 'test',
+            'models': ['*']
+        }
+        # This would throw a ConfigurationError if SecretModel types
+        # WERE included, since the matched types would conflict with
+        # pre-registered SecretModel types.
+        self.service_instance._register_store_handler(config)
+
     def test_get_handler(self):
         """
         Verify StorageService._get_handler() works as intended
@@ -241,6 +280,11 @@ def test_get_handler(self):
             self.service_instance._get_handler,
             model)
 
+        # Verify HostCreds instance returns CustodiaStoreHandler.
+        model = models.HostCreds.new(address='127.0.0.1')
+        handler = self.service_instance._get_handler(model)
+        self.assertIsInstance(handler, CustodiaStoreHandler)
+
     @mock.patch('commissaire_service.storage.StorageService._get_handler')
     def test_on_get_with_dict(self, get_handler):
         """