[WIP] Breaking up of initial TUF services

Makefile

@@ -5,6 +5,7 @@ BRANCH := $(shell echo "$${TRAVIS_BRANCH:-master}")
 DB := example
 IPYTHON := no
 LOCALES := $(shell .state/env/bin/python -c "from warehouse.i18n import KNOWN_LOCALES; print(' '.join(set(KNOWN_LOCALES)-{'en'}))")
+WAREHOUSE_CLI := docker-compose run --rm web python -m warehouse
 
 # set environment variable WAREHOUSE_IPYTHON_SHELL=1 if IPython
 # needed in development environment
@@ -153,6 +154,14 @@ initdb:
 	docker-compose run --rm web python -m warehouse db upgrade head
 	$(MAKE) reindex
 
+inittuf:
+	$(WAREHOUSE_CLI) tuf keypair --rolename root
+	$(WAREHOUSE_CLI) tuf keypair --rolename snapshot
+	$(WAREHOUSE_CLI) tuf keypair --rolename targets
+	$(WAREHOUSE_CLI) tuf keypair --rolename timestamp
+	$(WAREHOUSE_CLI) tuf keypair --rolename bins
+	$(WAREHOUSE_CLI) tuf keypair --rolename bin-n
+
 reindex:
 	docker-compose run --rm web python -m warehouse search reindex
 

requirements/main.in

@@ -18,6 +18,7 @@ google-cloud-bigquery
 google-cloud-storage
 hiredis
 html5lib
+hvac>=0.10.6
 itsdangerous
 Jinja2>=2.8
 limits

requirements/main.txt

@@ -374,6 +374,10 @@ hupper==1.10.2 \
     --hash=sha256:3818f53dabc24da66f65cf4878c1c7a9b5df0c46b813e014abdd7c569eb9a02a \
     --hash=sha256:5de835f3b58324af2a8a16f52270c4d1a3d1734c45eed94b77fd622aea737f29 \
     # via pyramid
+hvac==0.10.6 \
+    --hash=sha256:6e4bea65235bc38b85162a141194d07c857c6722ecd3edb6aeda316e8f4950f5 \
+    --hash=sha256:b0561dbdfecc6a6d7b0cc226d75a800ae9bbc93313a6ad526a1adc97be51eada \
+    # via -r requirements/main.in
 idna==2.10 \
     --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \
     --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 \
@@ -727,7 +731,7 @@ requests-aws4auth==1.0.1 \
 requests==2.25.0 \
     --hash=sha256:7f1a0b932f4a60a1a65caa4263921bb7d9ee911957e0ae4a23a6dd08185ad5f8 \
     --hash=sha256:e786fa28d8c9154e6a4de5d46a1d921b8749f8b74e28bde23768e5e16eece998 \
-    # via -r requirements/main.in, datadog, google-api-core, google-cloud-storage, premailer, requests-aws4auth
+    # via -r requirements/main.in, datadog, google-api-core, google-cloud-storage, hvac, premailer, requests-aws4auth
 rfc3986==1.4.0 \
     --hash=sha256:112398da31a3344dc25dbf477d8df6cb34f9278a94fee2625d89e4514be8bb9d \
     --hash=sha256:af9147e9aceda37c91a05f4deb128d4b4b49d6b199775fd2d2927768abdc8f50 \
@@ -747,7 +751,7 @@ sentry-sdk==0.19.4 \
 six==1.15.0 \
     --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
     --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
-    # via argon2-cffi, automat, bcrypt, bleach, cryptography, elasticsearch-dsl, google-api-core, google-auth, google-cloud-bigquery, google-resumable-media, grpcio, html5lib, limits, packaging, protobuf, pymacaroons, pynacl, pyopenssl, python-dateutil, readme-renderer, structlog, tenacity, webauthn
+    # via argon2-cffi, automat, bcrypt, bleach, cryptography, elasticsearch-dsl, google-api-core, google-auth, google-cloud-bigquery, google-resumable-media, grpcio, html5lib, hvac, limits, packaging, protobuf, pymacaroons, pynacl, pyopenssl, python-dateutil, readme-renderer, structlog, tenacity, webauthn
 sqlalchemy-citext==1.7.0 \
     --hash=sha256:69ba00f5505f92a1455a94eefc6d3fcf72dda3691ab5398a0b4d0d8d85bd6aab \
     # via -r requirements/main.in

tests/unit/cli/test_tuf.py

@@ -0,0 +1,42 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pretend
+
+from warehouse.cli import tuf
+
+
+class TestTufCLI:
+    def test_keypair(self, monkeypatch, cli):
+        response = pretend.stub(raise_for_status=pretend.call_recorder(lambda: None))
+        client = pretend.stub(
+            secrets=pretend.stub(
+                transit=pretend.stub(
+                    create_key=pretend.call_recorder(lambda **kw: response),
+                    read_key=pretend.call_recorder(lambda **kw: "fake key info"),
+                )
+            )
+        )
+        vault = pretend.call_recorder(lambda c: client)
+        monkeypatch.setattr(tuf, "_vault", vault)
+
+        config = pretend.stub()
+
+        result = cli.invoke(tuf.keypair, ["--rolename", "root"], obj=config)
+
+        assert result.exit_code == 0
+        assert vault.calls == [pretend.call(config)]
+        assert client.secrets.transit.create_key.calls == [
+            pretend.call(name="root", exportable=False, key_type="ed25519")
+        ]
+        assert client.secrets.transit.read_key.calls == [pretend.call(name="root")]
+        assert response.raise_for_status.calls == [pretend.call()]

tests/unit/test_config.py

@@ -231,6 +231,8 @@ def __init__(self):
         "token.default.max_age": 21600,
         "warehouse.xmlrpc.client.ratelimit_string": "3600 per hour",
         "warehouse.xmlrpc.search.enabled": True,
+        "vault.verify": environment == config.Environment.production,
+        "vault.cert": None,
     }
 
     if environment == config.Environment.development:

warehouse/cli/tuf.py

@@ -0,0 +1,51 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+import hvac
+
+from warehouse.cli import warehouse
+
+
+# pragma: no branch
+def _vault(config):
+    return hvac.Client(
+        url=config.registry.settings["vault.url"],
+        token=config.registry.settings["vault.token"],
+        cert=config.registry.settings["vault.cert"],
+        verify=config.registry.settings["vault.verify"],
+    )
+
+
+@warehouse.group()
+def tuf():
+    """
+    Manage Warehouse's TUF state.
+    """
+
+
+@tuf.command()
+@click.pass_obj
+@click.option(
+    "--rolename", required=True, help="The name of the TUF role for this keypair"
+)
+def keypair(config, rolename):
+    """
+    Generate a new TUF keypair.
+    """
+    vault = _vault(config)
+    resp = vault.secrets.transit.create_key(
+        name=rolename, exportable=False, key_type="ed25519"
+    )
+    resp.raise_for_status()
+    info = vault.secrets.transit.read_key(name=rolename)
+    print(info)

warehouse/config.py

@@ -214,6 +214,10 @@ def configure(settings=None):
         coercer=int,
         default=21600,  # 6 hours
     )
+    maybe_set(settings, "vault.url", "VAULT_URL")
+    maybe_set(settings, "vault.token", "VAULT_TOKEN")
+    maybe_set(settings, "vault.verify", "VAULT_VERIFY")
+    maybe_set(settings, "vault.cert", "VAULT_VERIFY")
     maybe_set_compound(settings, "files", "backend", "FILES_BACKEND")
     maybe_set_compound(settings, "docs", "backend", "DOCS_BACKEND")
     maybe_set_compound(settings, "origin_cache", "backend", "ORIGIN_CACHE")
@@ -222,6 +226,13 @@ def configure(settings=None):
     maybe_set_compound(settings, "breached_passwords", "backend", "BREACHED_PASSWORDS")
     maybe_set_compound(settings, "malware_check", "backend", "MALWARE_CHECK_BACKEND")
 
+    # Require an encrypted connection to Vault in production.
+    settings.setdefault(
+        "vault.verify", settings["warehouse.env"] == Environment.production
+    )
+    settings.setdefault("vault.cert", None)
+    maybe_set(settings, "vault.vert", "VAULT_CERT")
+
     # Add the settings we use when the environment is set to development.
     if settings["warehouse.env"] == Environment.development:
         settings.setdefault("enforce_https", False)

warehouse/tuf/__init__.py

@@ -0,0 +1,11 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

warehouse/tuf/interfaces.py

@@ -0,0 +1,26 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from zope.interface import Interface
+
+
+class IKeyService(Interface):
+    def create_service(context, request):
+        """
+        Create the service, given the context and request for which it is being
+        created.
+        """
+
+    def keys_for_role(rolename):
+        """
+        Returns a list of TUF `api.key.Key` for the given rolename.
+        """

warehouse/tuf/services.py

@@ -0,0 +1,35 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import hvac
+
+from zope.interface import implementer
+
+from warehouse.tuf.interfaces import IKeyService
+
+
+@implementer(IKeyService)
+class VaultKeyService:
+    def __init__(self, request):
+        self._vault = hvac.Client(
+            url=request.registry.settings["vault.url"],
+            token=request.registry.settings["vault.token"],
+            cert=request.registry.settings["vault.cert"],
+            verify=request.registry.settings["vault.verify"],
+        )
+
+    @classmethod
+    def create_service(cls, _context, request):
+        return cls(request)
+
+    def keys_for_role(self, rolename):
+        raise NotImplementedError