WIP: New tuf api metadata for dev initialization

Signed-off-by: Kairo de Araujo <[email protected]>
lukpueh · Feb 24, 2022 · 50843f6 · 50843f6
1 parent 4fddb8d
commit 50843f6
Show file tree

Hide file tree

Showing 32 changed files with 1,371 additions and 532 deletions.
diff --git a/.gitignore b/.gitignore
@@ -20,7 +20,7 @@ docker-compose.override.yaml
 
 node_modules/
 
-dev/tuf.*
+dev/tufkeys/
 dev/example.sql
 dev/prod.sql
 dev/prod.sql.xz

diff --git a/Makefile b/Makefile
@@ -150,14 +150,15 @@ initdb:
 	docker-compose run web python -m warehouse sponsors populate-db
 
 inittuf:
-	$(WAREHOUSE_CLI) tuf keypair --name root --path /opt/warehouse/src/dev/tuf.root
-	$(WAREHOUSE_CLI) tuf keypair --name snapshot --path /opt/warehouse/src/dev/tuf.snapshot
-	$(WAREHOUSE_CLI) tuf keypair --name targets --path /opt/warehouse/src/dev/tuf.targets
-	$(WAREHOUSE_CLI) tuf keypair --name timestamp --path /opt/warehouse/src/dev/tuf.timestamp
-	$(WAREHOUSE_CLI) tuf keypair --name bins --path /opt/warehouse/src/dev/tuf.bins
-	$(WAREHOUSE_CLI) tuf keypair --name bin-n --path /opt/warehouse/src/dev/tuf.bin-n
-	$(WAREHOUSE_CLI) tuf new-repo
-	$(WAREHOUSE_CLI) tuf build-targets
+	$(WAREHOUSE_CLI) tuf dev keypair --name root --path /opt/warehouse/src/dev/tufkeys/root
+	$(WAREHOUSE_CLI) tuf dev keypair --name snapshot --path /opt/warehouse/src/dev/tufkeys/snapshot
+	$(WAREHOUSE_CLI) tuf dev keypair --name targets --path /opt/warehouse/src/dev/tufkeys/targets
+	$(WAREHOUSE_CLI) tuf dev keypair --name timestamp --path /opt/warehouse/src/dev/tufkeys/timestamp
+	$(WAREHOUSE_CLI) tuf dev keypair --name bins --path /opt/warehouse/src/dev/tufkeys/bins
+	$(WAREHOUSE_CLI) tuf dev keypair --name bin-n --path /opt/warehouse/src/dev/tufkeys/bin-n
+	$(WAREHOUSE_CLI) tuf dev new-repo
+	$(WAREHOUSE_CLI) tuf admin delegate-targets-roles
+	$(WAREHOUSE_CLI) tuf dev add-targets
 
 reindex:
 	docker-compose run --rm web python -m warehouse search reindex
@@ -167,6 +168,7 @@ shell:
 
 clean:
 	rm -rf dev/*.sql
+	rm -rf dev/tufkeys
 
 purge: stop clean
 	rm -rf .state

diff --git a/dev/environment b/dev/environment
@@ -39,12 +39,17 @@ TOKEN_TWO_FACTOR_SECRET="an insecure two-factor auth secret key"
 
 WAREHOUSE_LEGACY_DOMAIN=pypi.python.org
 
-TUF_KEY_BACKEND=warehouse.tuf.services.LocalKeyService key.path=/opt/warehouse/src/dev
+VAULT_URL="http://vault:8200"
+VAULT_TOKEN="an insecure vault access token"
+
+GITHUB_TOKEN_SCANNING_META_API_URL="http://notgithub:8000/meta/public_keys/token_scanning"
+
+TUF_KEY_BACKEND=warehouse.tuf.services.LocalKeyService key.path=/opt/warehouse/src/dev/tufkeys
 TUF_STORAGE_BACKEND=warehouse.tuf.services.LocalStorageService
-TUF_REPO_BACKEND=warehouse.tuf.services.LocalRepositoryService repo.path=/opt/warehouse/src/warehouse/tuf/dist
+TUF_REPO_BACKEND=warehouse.tuf.services.LocalRepositoryService repo.path=/var/opt/warehouse/tuf_metadata/
 TUF_ROOT_SECRET="an insecure private key password"
 TUF_SNAPSHOT_SECRET="an insecure private key password"
 TUF_TARGETS_SECRET="an insecure private key password"
 TUF_TIMESTAMP_SECRET="an insecure private key password"
 TUF_BINS_SECRET="an insecure private key password"
-TUF_BIN_N_SECRET="an insecure private key password"
+TUF_BIN_N_SECRET="an insecure private key password"
diff --git a/dev/tufkeys/bin-n b/dev/tufkeys/bin-n
diff --git a/dev/tufkeys/bin-n.pub b/dev/tufkeys/bin-n.pub
diff --git a/dev/tufkeys/bins b/dev/tufkeys/bins
diff --git a/dev/tufkeys/bins.pub b/dev/tufkeys/bins.pub
diff --git a/dev/tufkeys/root b/dev/tufkeys/root
diff --git a/dev/tufkeys/root.pub b/dev/tufkeys/root.pub
diff --git a/dev/tufkeys/snapshot b/dev/tufkeys/snapshot
diff --git a/dev/tufkeys/snapshot.pub b/dev/tufkeys/snapshot.pub
diff --git a/dev/tufkeys/targets b/dev/tufkeys/targets
diff --git a/dev/tufkeys/targets.pub b/dev/tufkeys/targets.pub
diff --git a/dev/tufkeys/timestamp b/dev/tufkeys/timestamp
diff --git a/dev/tufkeys/timestamp.pub b/dev/tufkeys/timestamp.pub
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -4,6 +4,7 @@ volumes:
   simple:
   packages:
   sponsorlogos:
+  tuf_metadata:
   vault:
 
 services:
@@ -79,6 +80,7 @@ services:
       # The :z option fixes permission issues with SELinux by setting a
       # permissive security context.
       - ./dev:/opt/warehouse/src/dev:z
+      - ./dev/tufkeys:/opt/warehouse/src/dev/tufkeys:z
       - ./docs:/opt/warehouse/src/docs:z
       - ./warehouse:/opt/warehouse/src/warehouse:z
       - ./tests:/opt/warehouse/src/tests:z
@@ -87,6 +89,7 @@ services:
       - packages:/var/opt/warehouse/packages
       - sponsorlogos:/var/opt/warehouse/sponsorlogos
       - simple:/var/opt/warehouse/simple
+      - tuf_metadata:/var/opt/warehouse/tuf_metadata
       - ./bin:/opt/warehouse/src/bin:z
     ports:
       - "${WEB_PORT:-80}:8000"
@@ -102,6 +105,7 @@ services:
       - packages:/var/opt/warehouse/packages
       - sponsorlogos:/var/opt/warehouse/sponsorlogos
       - simple:/var/opt/warehouse/simple
+      - tuf_metadata:/var/opt/warehouse/metadata
     ports:
       - "9001:9001"
 
@@ -114,6 +118,7 @@ services:
     volumes:
       - ./dev:/opt/warehouse/src/dev:z
       - ./warehouse:/opt/warehouse/src/warehouse:z
+      - tuf_metadata:/var/opt/warehouse/tuf_metadata
     env_file: dev/environment
     environment:
       C_FORCE_ROOT: "1"

diff --git a/requirements/main.in b/requirements/main.in
@@ -49,6 +49,7 @@ requests
 requests-aws4auth
 redis>=2.8.0,<5.0.0
 rfc3986
+securesystemslib
 sentry-sdk
 setuptools
 sqlalchemy[asyncio]>=0.9,<1.5.0  # https://github.com/pypa/warehouse/pull/9228
@@ -58,7 +59,7 @@ stdlib-list
 structlog
 transaction
 trove-classifiers
-tuf==0.15.0
+tuf==1.0.0
 typeguard
 webauthn>=1.0.0,<2.0.0
 whitenoise

diff --git a/requirements/main.txt b/requirements/main.txt
@@ -1053,6 +1053,7 @@ requests==2.27.1 \
     #   google-cloud-storage
     #   premailer
     #   requests-aws4auth
+    #   tuf
 requests-aws4auth==1.1.1 \
     --hash=sha256:c0883346ce30b5018903a67da88df72f73ff06e1a320845bba9cd85e811ba0ba \
     --hash=sha256:dfd9f930ffde48a756b72b55698a8522875ea6358dcffbcc44a66700ace31783
@@ -1069,6 +1070,12 @@ s3transfer==0.5.1 \
     --hash=sha256:25c140f5c66aa79e1ac60be50dcd45ddc59e83895f062a3aab263b870102911f \
     --hash=sha256:69d264d3e760e569b78aaa0f22c97e955891cd22e32b10c51f784eeda4d9d10a
     # via boto3
+securesystemslib==0.22.0 \
+    --hash=sha256:2f58ca1ee30fde5401300fe3b3841adcf7b4369674247fa63b258e07e1f52fd2 \
+    --hash=sha256:c3fc41ac32fe8bc9744b89e6ce2ebca45f4417ca737beb766a41c6cb21935662
+    # via
+    #   -r requirements/main.in
+    #   tuf
 sentry-sdk==1.5.5 \
     --hash=sha256:3817274fba2498c8ebf6b896ee98ac916c5598706340573268c07bf2bb30d831 \
     --hash=sha256:98fd155fa5d5fec1dbabed32a1a4ae2705f1edaa5dae4e7f7b62a384ba30e759
@@ -1173,6 +1180,10 @@ trove-classifiers==2022.2.16 \
     --hash=sha256:d985eeadb24c8a40d1fa121317466e2ae73ae8ad9abc8cab69371212c40b085f \
     --hash=sha256:dcf20b1de0f56308dd107caaa5175e72a950bbb12c95deedac6a7cd429ecd694
     # via -r requirements/main.in
+tuf==1.0.0 \
+    --hash=sha256:6d967f992d494678b684a6f5f03d948130e2d9956310c7af74fa2f0296e8ec4d \
+    --hash=sha256:ec74150954d56e206fa72f3b0234ea802649370e528dea5a83e507bd408fda82
+    # via -r requirements/main.in
 typeguard==2.13.3 \
     --hash=sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4 \
     --hash=sha256:5e3e3be01e887e7eafae5af63d1f36c849aaa94e3a0112097312aabfa16284f1

diff --git a/tests/unit/tuf/__init__.py b/tests/unit/tuf/__init__.py
diff --git a/tests/unit/tuf/test_services.py b/tests/unit/tuf/test_services.py
@@ -0,0 +1,47 @@
+import pretend
+
+from tuf.api.metadata import Key
+from zope.interface.verify import verifyClass
+
+from warehouse.tuf import services
+from warehouse.tuf.interfaces import IKeyService
+from warehouse.tuf.services import LocalKeyService
+
+
+class TestLocalLocalKeyService:
+    def test_verify_service(self):
+        assert verifyClass(IKeyService, LocalKeyService)
+
+    def test_create_service(self):
+        request = pretend.stub(
+            registry=pretend.stub(settings={"tuf.key.path": "/tuf/key/path/"})
+        )
+        service = LocalKeyService.create_service(None, request)
+        assert service._key_path == "/tuf/key/path/"
+
+    def test_basic_init(self, db_request):
+        service = LocalKeyService("/opt/warehouse/src/dev/tufkeys", db_request)
+        assert service._key_path == "/opt/warehouse/src/dev/tufkeys"
+
+    def test_pubkeys_for_role(self, monkeypatch, db_request):
+        service = LocalKeyService("/opt/warehouse/src/dev/tufkeys", db_request)
+
+        expected_priv_key_dict = {
+            "keytype": "ed25519",
+            "scheme": "ed25519",
+            "keyval": {
+                "public": "720a9a588deefd533c36da9b071f7c7b4d08984e87bfc5a18f34618e438434c7"
+            },
+            "keyid": "2de4eb9afe9fb7307f1dd0869a7aec2235d3418bd63f4214d3ba7d23b516f23e",
+            "keyid_hash_algorithms": ["sha256", "sha512"],
+        }
+
+        monkeypatch.setattr(
+            "warehouse.tuf.services.import_ed25519_publickey_from_file",
+            lambda *a, **kw: expected_priv_key_dict,
+        )
+
+        root_keyid = service.pubkeys_for_role("root")
+
+        assert isinstance(root_keyid, Key)
+        assert root_keyid.to_dict().get("keyid") == expected_priv_key_dict.get("keyid")