From 16c356844620f986867a236a94240306fba2497e Mon Sep 17 00:00:00 2001 From: Jan Koscielniak Date: Fri, 25 Oct 2024 14:44:14 +0200 Subject: [PATCH] generic fetcher: Generate SBOM components Generic fetcher now generates SBOM components of purl type pkg:generic. It also produces externalReferences of type distribution for each generic component to allow for gating at a later time. Signed-off-by: Jan Koscielniak --- cachi2/core/package_managers/generic/main.py | 31 +++++++++- .../test_data/generic_e2e/bom.json | 37 +++++++++++- tests/unit/package_managers/test_generic.py | 57 ++++++++++++++++++- 3 files changed, 121 insertions(+), 4 deletions(-) diff --git a/cachi2/core/package_managers/generic/main.py b/cachi2/core/package_managers/generic/main.py index ee8c8ab03..15618e590 100644 --- a/cachi2/core/package_managers/generic/main.py +++ b/cachi2/core/package_managers/generic/main.py @@ -5,6 +5,7 @@ from typing import Union import yaml +from packageurl import PackageURL from pydantic import ValidationError from cachi2.core.checksum import must_match_any_checksum @@ -12,7 +13,7 @@ from cachi2.core.errors import PackageRejected from cachi2.core.models.input import Request from cachi2.core.models.output import RequestOutput -from cachi2.core.models.sbom import Component +from cachi2.core.models.sbom import Component, ExternalReference from cachi2.core.package_managers.general import async_download_files from cachi2.core.package_managers.generic.models import GenericLockfileV1 from cachi2.core.rooted_path import RootedPath @@ -69,7 +70,7 @@ def _resolve_generic_lockfile(source_dir: RootedPath, output_dir: RootedPath) -> # verify checksums for artifact in lockfile.artifacts: must_match_any_checksum(artifact.target, artifact.formatted_checksums) - return [] + return _generate_sbom_components(lockfile) def _load_lockfile(lockfile_path: RootedPath, output_dir: RootedPath) -> GenericLockfileV1: @@ -102,3 +103,29 @@ def _load_lockfile(lockfile_path: RootedPath, output_dir: RootedPath) -> Generic ), ) return lockfile + + +def _generate_sbom_components(lockfile: GenericLockfileV1) -> list[Component]: + """Generate a list of SBOM components for a given lockfile.""" + components: list[Component] = [] + + for artifact in lockfile.artifacts: + name = Path(artifact.target).name + url = str(artifact.download_url) + checksums = ",".join([f"{algo}:{digest}" for algo, digest in artifact.checksums.items()]) + component = Component( + name=name, + purl=PackageURL( + type="generic", + name=name, + qualifiers={ + "download_url": url, + "checksums": checksums, + }, + ).to_string(), + type="file", + external_references=[ExternalReference(url=url, type="distribution")], + ) + components.append(component) + + return components diff --git a/tests/integration/test_data/generic_e2e/bom.json b/tests/integration/test_data/generic_e2e/bom.json index 958e49b30..5bf394cab 100644 --- a/tests/integration/test_data/generic_e2e/bom.json +++ b/tests/integration/test_data/generic_e2e/bom.json @@ -1,6 +1,41 @@ { "bomFormat": "CycloneDX", - "components": [], + "components": [ + { + "externalReferences": [ + { + "type": "distribution", + "url": "https://github.com/cachito-testing/cachi2-generic/archive/refs/tags/v2.0.0.zip" + } + ], + "name": "archive.zip", + "properties": [ + { + "name": "cachi2:found_by", + "value": "cachi2" + } + ], + "purl": "pkg:generic/archive.zip?checksums=sha256:386428a82f37345fa24b74068e0e79f4c1f2ff38d4f5c106ea14de4a2926e584&download_url=https://github.com/cachito-testing/cachi2-generic/archive/refs/tags/v2.0.0.zip", + "type": "file" + }, + { + "externalReferences": [ + { + "type": "distribution", + "url": "https://github.com/cachito-testing/cachi2-generic/archive/refs/tags/v1.0.0.zip" + } + ], + "name": "v1.0.0.zip", + "properties": [ + { + "name": "cachi2:found_by", + "value": "cachi2" + } + ], + "purl": "pkg:generic/v1.0.0.zip?checksums=sha256:4fbcaa2a8d17c1f8042578627c122361ab18b7973311e7e9c598696732902f87&download_url=https://github.com/cachito-testing/cachi2-generic/archive/refs/tags/v1.0.0.zip", + "type": "file" + } + ], "metadata": { "tools": [ { diff --git a/tests/unit/package_managers/test_generic.py b/tests/unit/package_managers/test_generic.py index efa3bdbbd..733cae059 100644 --- a/tests/unit/package_managers/test_generic.py +++ b/tests/unit/package_managers/test_generic.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Type +from typing import Any, Type from unittest import mock import pytest @@ -207,6 +207,61 @@ def test_resolve_generic_lockfile_invalid( assert expected_err in str(exc_info.value) +@pytest.mark.parametrize( + ["lockfile_content", "expected_components"], + [ + pytest.param( + LOCKFILE_VALID, + [ + { + "external_references": [ + {"type": "distribution", "url": "https://example.com/artifact"} + ], + "name": "archive.zip", + "properties": [{"name": "cachi2:found_by", "value": "cachi2"}], + "purl": "pkg:generic/archive.zip?checksums=md5:3a18656e1cea70504b905836dee14db0&download_url=https://example.com/artifact", + "type": "file", + "version": None, + }, + { + "external_references": [ + { + "type": "distribution", + "url": "https://example.com/more/complex/path/file.tar.gz?foo=bar#fragment", + } + ], + "name": "file.tar.gz", + "properties": [{"name": "cachi2:found_by", "value": "cachi2"}], + "purl": "pkg:generic/file.tar.gz?checksums=md5:32112bed1914cfe3799600f962750b1d&download_url=https://example.com/more/complex/path/file.tar.gz%3Ffoo%3Dbar%23fragment", + "type": "file", + "version": None, + }, + ], + id="valid_lockfile", + ), + ], +) +@mock.patch("cachi2.core.package_managers.generic.main.asyncio.run") +@mock.patch("cachi2.core.package_managers.generic.main.async_download_files") +@mock.patch("cachi2.core.package_managers.generic.main.must_match_any_checksum") +def test_resolve_generic_lockfile_valid( + mock_checksums: mock.Mock, + mock_download: mock.Mock, + mock_asyncio_run: mock.Mock, + lockfile_content: str, + expected_components: list[dict[str, Any]], + rooted_tmp_path: RootedPath, +) -> None: + # setup lockfile + with open(rooted_tmp_path.join_within_root(DEFAULT_LOCKFILE_NAME), "w") as f: + f.write(lockfile_content) + + assert [ + c.model_dump() for c in _resolve_generic_lockfile(rooted_tmp_path, rooted_tmp_path) + ] == expected_components + mock_checksums.assert_called() + + def test_load_generic_lockfile_valid(rooted_tmp_path: RootedPath) -> None: expected_lockfile = { "metadata": {"version": "1.0"},