diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index e92e2b201c..781c956b8e 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -237,3 +237,38 @@ jobs: pytest -v test/test_cvedb.py test/test_cli.py + + cve_scan: + name: CVE Scan on dependencies + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Get Date + id: get-date + run: | + echo "::set-output name=date::$(/bin/date -u "+%Y%m%d")" + shell: bash + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: get cached python packages + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: get cached database + uses: actions/cache@v2 + with: + path: ~/.cache/cve-bin-tool + key: ${{ runner.os }}-cve-bin-tool-${{ steps.get-date.outputs.date }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt -r doc/requirements.txt + - name: Test to check for CVEs for python requirements and HTML report dependencies + run: | + pytest test/test_requirements.py diff --git a/.gitignore b/.gitignore index d08df3b0e2..13d12ad514 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,5 @@ build/ dist/ doc/_build test/downloads/ +cve_bin_tool_requirements.csv !test/condensed-downloads/*.tar.gz diff --git a/cve_bin_tool/output_engine/html_reports/js/dependencies.csv b/cve_bin_tool/output_engine/html_reports/js/dependencies.csv new file mode 100644 index 0000000000..77c4925a92 --- /dev/null +++ b/cve_bin_tool/output_engine/html_reports/js/dependencies.csv @@ -0,0 +1,4 @@ +vendor,product +getbootstrap,bootstrap +jquery,jquery +plotly,plotly.js diff --git a/doc/requirements.csv b/doc/requirements.csv new file mode 100644 index 0000000000..0497e9fbe2 --- /dev/null +++ b/doc/requirements.csv @@ -0,0 +1,4 @@ +vendor,product +rtfd_not_in_db,recommonmark +sphinx-doc_not_in_db,Sphinx +ryanfox_not_in_db,sphinx_markdown_tables diff --git a/requirements.csv b/requirements.csv new file mode 100644 index 0000000000..70288ab7d2 --- /dev/null +++ b/requirements.csv @@ -0,0 +1,18 @@ +vendor,product +plot,plotly +pocoo,jinja2 +aiohttp_project,aiohttp +pyyaml,pyyaml +reportlab,reportlab +pytest_not_in_db,pytest +pytest_not_in_db,pytest-xdist +pytest_not_in_db,pytest-cov +pytest_not_in_db,pytest-asyncio +pycqa_not_in_db,isort +willmcgugan_not_in_db,rich +crummy_not_in_db,beautifulsoup4 +uiri_not_in_db,toml +jsonschema_not_in_db,jsonschema +python_not_in_db,py +srossross_not_in_db,rpmfile +indygreg_not_in_db,zstandard diff --git a/test/test_requirements.py b/test/test_requirements.py new file mode 100644 index 0000000000..f6f05f0989 --- /dev/null +++ b/test/test_requirements.py @@ -0,0 +1,138 @@ +# Copyright (C) 2021 Intel Corporation +# SPDX-License-Identifier: GPL-3.0-or-later + +import csv +import re +import subprocess +from importlib.metadata import version +from os.path import dirname, join + +ROOT_PATH = join(dirname(__file__), "..") + +REQ_TXT = join(ROOT_PATH, "requirements.txt") +REQ_CSV = join(ROOT_PATH, "requirements.csv") +DOC_TXT = join(ROOT_PATH, "doc", "requirements.txt") +DOC_CSV = join(ROOT_PATH, "doc", "requirements.csv") + +SCAN_CSV = join(ROOT_PATH, "cve_bin_tool_requirements.csv") + +HTML_DEP_PATH = join( + ROOT_PATH, + "cve_bin_tool", + "output_engine", + "html_reports", + "js", +) + +HTML_DEP_CSV = join(HTML_DEP_PATH, "dependencies.csv") + +# Dependencies that currently have CVEs +# Remove from the list once they are updated +ALLOWED_PACKAGES = ["reportlab"] + + +def get_out_of_sync_packages(csv_name, txt_name): + + new_packages = set() + removed_packages = set() + csv_package_names = set() + txt_package_names = set() + + with open(csv_name) as csv_file, open(txt_name) as txt_file: + csv_reader = csv.reader(csv_file) + next(csv_reader) + for (_, product) in csv_reader: + csv_package_names.add(product) + lines = txt_file.readlines() + for line in lines: + txt_package_names.add(re.split(">|\\[|;|=|\n", line)[0]) + new_packages = txt_package_names - csv_package_names + removed_packages = csv_package_names - txt_package_names + + return (new_packages, removed_packages) + + +# Test to check if the requirements.csv files are in sync with requirements.txt files +def test_txt_csv_sync(): + + errors = set() + + ( + req_new_packages, + req_removed_packages, + ) = get_out_of_sync_packages(REQ_CSV, REQ_TXT) + ( + doc_new_packages, + doc_removed_packages, + ) = get_out_of_sync_packages(DOC_CSV, DOC_TXT) + + if doc_removed_packages != set(): + errors.add( + f"The requirements.txt and requirements.csv files of docs are out of sync! Please remove {', '.join(doc_removed_packages)} from the respective requirements.csv file\n" + ) + if doc_new_packages != set(): + errors.add( + f"The requirements.txt and requirements.csv files of docs are out of sync! Please add {', '.join(doc_new_packages)} to the respective requirements.csv file\n" + ) + if req_removed_packages != set(): + errors.add( + f"The requirements.txt and requirements.csv files of cve-bin-tool are out of sync! Please remove {', '.join(req_removed_packages)} from the respective requirements.csv file\n" + ) + if req_new_packages != set(): + errors.add( + f"The requirements.txt and requirements.csv files of cve-bin-tool are out of sync! Please add {', '.join(req_new_packages)} to the respective requirements.csv file\n" + ) + + assert errors == set(), f"The error(s) are:\n {''.join(errors)}" + + +def get_cache_csv_data(file): + + data = [] + + with open(file) as f: + r = csv.reader(f) + next(r) + for (vendor, product) in r: + if file is HTML_DEP_CSV: + file_name = f"{HTML_DEP_PATH}/{product}" + if not file_name.endswith(".js"): + file_name += ".js" + with open(file_name) as f: + file_content = f.read() + html_dep_version = re.search( + r"v([0-9]+\.[0-9]+\.[0-9]+)", file_content + ).group(1) + if product not in ALLOWED_PACKAGES: + data.append((vendor, product, html_dep_version)) + else: + if "_not_in_db" not in vendor and product not in ALLOWED_PACKAGES: + data.append((vendor, product, version(product))) + + return data + + +# Test to check for CVEs in cve-bin-tool requirements/dependencies +def test_requirements(): + + cache_csv_data = ( + get_cache_csv_data(REQ_CSV) + + get_cache_csv_data(DOC_CSV) + + get_cache_csv_data(HTML_DEP_CSV) + ) + + # writes a cache CSV file + with open(SCAN_CSV, "w") as f: + writer = csv.writer(f) + fieldnames = ["vendor", "product", "version"] + writer = csv.writer(f) + writer.writerow(fieldnames) + for row in cache_csv_data: + writer.writerow(row) + + cve_check = subprocess.run( + ["python", "-m", "cve_bin_tool.cli", "--input-file", SCAN_CSV] + ) + assert ( + cve_check.returncode == 0 + ), f"{cve_check.returncode} dependencies/requirements have CVEs"