From 563095f672e67c4352bfcdc05517148e95ed4ded Mon Sep 17 00:00:00 2001 From: BreadGenie Date: Wed, 24 Mar 2021 12:27:02 +0530 Subject: [PATCH 01/38] CSV file for CVE input --- requirements.csv | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 requirements.csv diff --git a/requirements.csv b/requirements.csv new file mode 100644 index 0000000000..3634e59c46 --- /dev/null +++ b/requirements.csv @@ -0,0 +1,6 @@ +vendor,product,version +plot,plotly,4.14.3 +pocoo,jinja2,2.11.3 +aiohttp_project,aiohttp,3.7.4.post0 +pyyaml,pyyaml,5.4.1 +facebook,zstandard,0.15.1 From 3f16c28d533174cbbc7e16489d5b012452f5cb2e Mon Sep 17 00:00:00 2001 From: BreadGenie Date: Thu, 25 Mar 2021 14:51:08 +0530 Subject: [PATCH 02/38] Add github workflow for scanning requirements of cve-bin-tool --- .github/workflows/pythonapp.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index e92e2b201c..1e563da5bd 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -237,3 +237,33 @@ jobs: pytest -v test/test_cvedb.py test/test_cli.py + + cve_scan: + name: CVE Scan of requirements.csv + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: get cached python packages + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: get cached database + uses: actions/cache@v2 + with: + path: ~/.cache/cve-bin-tool + key: ${{ runner.os }}-cve-bin-tool-${{ steps.get-date.outputs.date }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Run CVE Binary tool against requirements.csv + run: | + python -m cve_bin_tool.cli requirements.csv \ No newline at end of file From 5076b9579a7f1700f08f4c78b0e132085e3152da Mon Sep 17 00:00:00 2001 From: BreadGenie Date: Mon, 29 Mar 2021 15:00:29 +0530 Subject: [PATCH 03/38] Remove zstandard --- requirements.csv | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.csv b/requirements.csv index 3634e59c46..8268d7052d 100644 --- a/requirements.csv +++ b/requirements.csv @@ -3,4 +3,3 @@ plot,plotly,4.14.3 pocoo,jinja2,2.11.3 aiohttp_project,aiohttp,3.7.4.post0 pyyaml,pyyaml,5.4.1 -facebook,zstandard,0.15.1 From 48f34454950830ca7a62b3f6ce9f41216b4ed603 Mon Sep 17 00:00:00 2001 From: BreadGenie Date: Wed, 31 Mar 2021 09:22:43 +0530 Subject: [PATCH 04/38] Add script to find versions of packages --- .github/workflows/pythonapp.yml | 7 +++++-- requirements.csv | 10 +++++----- requirements_csv.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 7 deletions(-) create mode 100644 requirements_csv.py diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 1e563da5bd..a2c43eb41e 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -239,7 +239,7 @@ jobs: test/test_cli.py cve_scan: - name: CVE Scan of requirements.csv + name: CVE Scan of requirements runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -264,6 +264,9 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt + - name: Create cache requirements.csv with versions + run: | + python requirements_csv.py - name: Run CVE Binary tool against requirements.csv run: | - python -m cve_bin_tool.cli requirements.csv \ No newline at end of file + python -m cve_bin_tool.cli --input-file ~/.cache/cve-bin-tool/requirements.csv \ No newline at end of file diff --git a/requirements.csv b/requirements.csv index 8268d7052d..a883fd4076 100644 --- a/requirements.csv +++ b/requirements.csv @@ -1,5 +1,5 @@ -vendor,product,version -plot,plotly,4.14.3 -pocoo,jinja2,2.11.3 -aiohttp_project,aiohttp,3.7.4.post0 -pyyaml,pyyaml,5.4.1 +vendor,product +plot,plotly +pocoo,jinja2 +aiohttp_project,aiohttp +pyyaml,pyyaml diff --git a/requirements_csv.py b/requirements_csv.py new file mode 100644 index 0000000000..4d7cd61b30 --- /dev/null +++ b/requirements_csv.py @@ -0,0 +1,31 @@ +# Copyright (C) 2021 Intel Corporation +# SPDX-License-Identifier: GPL-3.0-or-later + +# Script to add versions to requirements.csv + +import csv +import os +import pkg_resources + +REQ_CSV = os.path.abspath(os.path.join(os.path.dirname(__file__), "requirements.csv")) +CACHE_CSV = os.path.join(os.path.expanduser("~"), ".cache", "cve-bin-tool", "requirements.csv") + +def get_cache_csv_data(file): + data = [] + with open(file, "r") as f: + r = csv.reader(f) + next(r) + for row in r: + data.append((row[0], row[1], pkg_resources.get_distribution(row[1]).version)) + return data + +cache_csv_data = get_cache_csv_data(REQ_CSV) + +# writes a cache CSV file +with open(CACHE_CSV, "w") as f: + writer = csv.writer(f) + fieldnames = ["vendor", "product", "version"] + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + for row in cache_csv_data: + writer.writerow({"vendor": row[0], "product": row[1], "version": row[2]}) \ No newline at end of file From 24da96484fcce77ca93f03255fa2c531f3a59065 Mon Sep 17 00:00:00 2001 From: BreadGenie Date: Wed, 31 Mar 2021 13:43:09 +0530 Subject: [PATCH 05/38] Refactor code and black fix --- requirements_csv.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/requirements_csv.py b/requirements_csv.py index 4d7cd61b30..ca64ee14de 100644 --- a/requirements_csv.py +++ b/requirements_csv.py @@ -8,7 +8,10 @@ import pkg_resources REQ_CSV = os.path.abspath(os.path.join(os.path.dirname(__file__), "requirements.csv")) -CACHE_CSV = os.path.join(os.path.expanduser("~"), ".cache", "cve-bin-tool", "requirements.csv") +CACHE_CSV = os.path.join( + os.path.expanduser("~"), ".cache", "cve-bin-tool", "requirements.csv" +) + def get_cache_csv_data(file): data = [] @@ -16,16 +19,19 @@ def get_cache_csv_data(file): r = csv.reader(f) next(r) for row in r: - data.append((row[0], row[1], pkg_resources.get_distribution(row[1]).version)) + data.append( + (row[0], row[1], pkg_resources.get_distribution(row[1]).version) + ) return data + cache_csv_data = get_cache_csv_data(REQ_CSV) # writes a cache CSV file with open(CACHE_CSV, "w") as f: writer = csv.writer(f) fieldnames = ["vendor", "product", "version"] - writer = csv.DictWriter(f, fieldnames=fieldnames) - writer.writeheader() + writer = csv.writer(f) + writer.writerow(fieldnames) for row in cache_csv_data: - writer.writerow({"vendor": row[0], "product": row[1], "version": row[2]}) \ No newline at end of file + writer.writerow(row) From 307a6753edc2def1849e96acdf3d22f0ca28464c Mon Sep 17 00:00:00 2001 From: BreadGenie Date: Thu, 1 Apr 2021 06:29:35 +0530 Subject: [PATCH 06/38] Add CSV for docs --- .github/workflows/pythonapp.yml | 1 + doc/requirements.csv | 4 ++++ requirements.csv | 12 ++++++++++++ requirements_csv.py | 12 ++++++++---- 4 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 doc/requirements.csv diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index a2c43eb41e..7583cf8782 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -264,6 +264,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt + pip install -r doc/requirements.txt - name: Create cache requirements.csv with versions run: | python requirements_csv.py diff --git a/doc/requirements.csv b/doc/requirements.csv new file mode 100644 index 0000000000..5638d099ac --- /dev/null +++ b/doc/requirements.csv @@ -0,0 +1,4 @@ +vendor,product +rtfd_not_in_db,recommonmark +sphinx-doc_not_in_db,sphinx +ryanfox_not_in_db,sphinx_markdown_tables \ No newline at end of file diff --git a/requirements.csv b/requirements.csv index a883fd4076..95bc455104 100644 --- a/requirements.csv +++ b/requirements.csv @@ -3,3 +3,15 @@ plot,plotly pocoo,jinja2 aiohttp_project,aiohttp pyyaml,pyyaml +pytest_not_in_db,pytest +pytest_not_in_db,pytest-xdist +pytest_not_in_db,pytest-cov +pytest_not_in_db,pytest-asyncio +pycqa_not_in_db,isort +willmcgugan_not_in_db,rich +crummy_not_in_db,beautifulsoup4 +aio-libs_not_in_db,aiohttp +uiri_not_in_db,toml +jsonschema_not_in_db,jsonschema +python_not_in_db,py +indygreg_not_in_db,zstandard \ No newline at end of file diff --git a/requirements_csv.py b/requirements_csv.py index ca64ee14de..ef29367b83 100644 --- a/requirements_csv.py +++ b/requirements_csv.py @@ -8,6 +8,9 @@ import pkg_resources REQ_CSV = os.path.abspath(os.path.join(os.path.dirname(__file__), "requirements.csv")) +DOC_CSV = os.path.abspath( + os.path.join(os.path.dirname(__file__), "doc", "requirements.csv") +) CACHE_CSV = os.path.join( os.path.expanduser("~"), ".cache", "cve-bin-tool", "requirements.csv" ) @@ -19,13 +22,14 @@ def get_cache_csv_data(file): r = csv.reader(f) next(r) for row in r: - data.append( - (row[0], row[1], pkg_resources.get_distribution(row[1]).version) - ) + if "_not_in_db" not in row[0]: + data.append( + (row[0], row[1], pkg_resources.get_distribution(row[1]).version) + ) return data -cache_csv_data = get_cache_csv_data(REQ_CSV) +cache_csv_data = get_cache_csv_data(REQ_CSV) + get_cache_csv_data(DOC_CSV) # writes a cache CSV file with open(CACHE_CSV, "w") as f: From d985fee6e52b32d1b7e34d207f70f56436cc97d1 Mon Sep 17 00:00:00 2001 From: BreadGenie Date: Wed, 14 Apr 2021 12:49:15 +0530 Subject: [PATCH 07/38] Swap pkg_resources module for importlib --- requirements_csv.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/requirements_csv.py b/requirements_csv.py index ef29367b83..c5bb7c5dbb 100644 --- a/requirements_csv.py +++ b/requirements_csv.py @@ -5,7 +5,7 @@ import csv import os -import pkg_resources +from importlib.metadata import version REQ_CSV = os.path.abspath(os.path.join(os.path.dirname(__file__), "requirements.csv")) DOC_CSV = os.path.abspath( @@ -23,9 +23,7 @@ def get_cache_csv_data(file): next(r) for row in r: if "_not_in_db" not in row[0]: - data.append( - (row[0], row[1], pkg_resources.get_distribution(row[1]).version) - ) + data.append((row[0], row[1], version(row[1]))) return data From 311249d16f3f7c338522dde95df2d85956764235 Mon Sep 17 00:00:00 2001 From: BreadGenie Date: Wed, 14 Apr 2021 13:02:00 +0530 Subject: [PATCH 08/38] Fix ModuleNotFound Error --- .github/workflows/pythonapp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 7583cf8782..10598964c8 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -247,7 +247,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v1 with: - python-version: 3.7 + python-version: 3.8 - name: get cached python packages uses: actions/cache@v2 with: From 27b72a34f45dff74cd3a5515a009c719506188ca Mon Sep 17 00:00:00 2001 From: BreadGenie Date: Fri, 16 Apr 2021 14:04:20 +0530 Subject: [PATCH 09/38] Add html dependencies and reportlab module --- .github/workflows/pythonapp.yml | 3 ++- cve_bin_tool/output_engine/html_reports/js/dependencies.csv | 4 ++++ requirements.csv | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 cve_bin_tool/output_engine/html_reports/js/dependencies.csv diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 10598964c8..ea24dc428b 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -270,4 +270,5 @@ jobs: python requirements_csv.py - name: Run CVE Binary tool against requirements.csv run: | - python -m cve_bin_tool.cli --input-file ~/.cache/cve-bin-tool/requirements.csv \ No newline at end of file + python -m cve_bin_tool.cli --input-file ~/.cache/cve-bin-tool/requirements.csv + python -m cve_bin_tool.cli --input-file cve_bin_tool/output_engine/html_reports/js/dependencies.csv \ No newline at end of file diff --git a/cve_bin_tool/output_engine/html_reports/js/dependencies.csv b/cve_bin_tool/output_engine/html_reports/js/dependencies.csv new file mode 100644 index 0000000000..4521abb516 --- /dev/null +++ b/cve_bin_tool/output_engine/html_reports/js/dependencies.csv @@ -0,0 +1,4 @@ +vendor,product,version +getbootstrap,bootstrap,4.5.0 +jquery,jquery,3.4.1 +plotly,plotly.js,1.54.1 \ No newline at end of file diff --git a/requirements.csv b/requirements.csv index 95bc455104..3177dc4b7e 100644 --- a/requirements.csv +++ b/requirements.csv @@ -3,6 +3,7 @@ plot,plotly pocoo,jinja2 aiohttp_project,aiohttp pyyaml,pyyaml +reportlab,reportlab pytest_not_in_db,pytest pytest_not_in_db,pytest-xdist pytest_not_in_db,pytest-cov @@ -14,4 +15,5 @@ aio-libs_not_in_db,aiohttp uiri_not_in_db,toml jsonschema_not_in_db,jsonschema python_not_in_db,py +srossross_not_in_db,rpmfile indygreg_not_in_db,zstandard \ No newline at end of file From a9435cb6389e629a1dd62ec57cd46b0d8bded943 Mon Sep 17 00:00:00 2001 From: BreadGenie Date: Fri, 16 Apr 2021 23:00:30 +0530 Subject: [PATCH 10/38] Add get-date step and new lines in every file --- .github/workflows/pythonapp.yml | 10 ++++++++-- .../output_engine/html_reports/js/dependencies.csv | 2 +- doc/requirements.csv | 2 +- requirements.csv | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index ea24dc428b..b52e78a424 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -243,6 +243,11 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: + - name: Get Date + id: get-date + run: | + echo "::set-output name=date::$(/bin/date -u "+%Y%m%d")" + shell: bash - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v1 @@ -268,7 +273,8 @@ jobs: - name: Create cache requirements.csv with versions run: | python requirements_csv.py - - name: Run CVE Binary tool against requirements.csv + - name: Run CVE Binary tool against requirements.csv and HTML report dependencies run: | python -m cve_bin_tool.cli --input-file ~/.cache/cve-bin-tool/requirements.csv - python -m cve_bin_tool.cli --input-file cve_bin_tool/output_engine/html_reports/js/dependencies.csv \ No newline at end of file + python -m cve_bin_tool.cli --input-file cve_bin_tool/output_engine/html_reports/js/dependencies.csv + \ No newline at end of file diff --git a/cve_bin_tool/output_engine/html_reports/js/dependencies.csv b/cve_bin_tool/output_engine/html_reports/js/dependencies.csv index 4521abb516..3c74d48191 100644 --- a/cve_bin_tool/output_engine/html_reports/js/dependencies.csv +++ b/cve_bin_tool/output_engine/html_reports/js/dependencies.csv @@ -1,4 +1,4 @@ vendor,product,version getbootstrap,bootstrap,4.5.0 jquery,jquery,3.4.1 -plotly,plotly.js,1.54.1 \ No newline at end of file +plotly,plotly.js,1.54.1 diff --git a/doc/requirements.csv b/doc/requirements.csv index 5638d099ac..15d37de5da 100644 --- a/doc/requirements.csv +++ b/doc/requirements.csv @@ -1,4 +1,4 @@ vendor,product rtfd_not_in_db,recommonmark sphinx-doc_not_in_db,sphinx -ryanfox_not_in_db,sphinx_markdown_tables \ No newline at end of file +ryanfox_not_in_db,sphinx_markdown_tables diff --git a/requirements.csv b/requirements.csv index 3177dc4b7e..48edbec8f4 100644 --- a/requirements.csv +++ b/requirements.csv @@ -16,4 +16,4 @@ uiri_not_in_db,toml jsonschema_not_in_db,jsonschema python_not_in_db,py srossross_not_in_db,rpmfile -indygreg_not_in_db,zstandard \ No newline at end of file +indygreg_not_in_db,zstandard From 1cf198c42cc558a0202dd96ab87c1f5f054fc1b6 Mon Sep 17 00:00:00 2001 From: BreadGenie Date: Sat, 17 Apr 2021 21:32:52 +0530 Subject: [PATCH 11/38] Add newline --- .github/workflows/pythonapp.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index b52e78a424..d5ef2a3adc 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -277,4 +277,3 @@ jobs: run: | python -m cve_bin_tool.cli --input-file ~/.cache/cve-bin-tool/requirements.csv python -m cve_bin_tool.cli --input-file cve_bin_tool/output_engine/html_reports/js/dependencies.csv - \ No newline at end of file From 40e4214502b7e93074606cc125ade5cb3ab28e24 Mon Sep 17 00:00:00 2001 From: BreadGenie Date: Wed, 21 Apr 2021 18:16:47 +0530 Subject: [PATCH 12/38] Update script to find HTML dependecies' versions --- .../html_reports/js/dependencies.csv | 8 ++--- requirements_csv.py | 30 +++++++++++++++++-- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/cve_bin_tool/output_engine/html_reports/js/dependencies.csv b/cve_bin_tool/output_engine/html_reports/js/dependencies.csv index 3c74d48191..77c4925a92 100644 --- a/cve_bin_tool/output_engine/html_reports/js/dependencies.csv +++ b/cve_bin_tool/output_engine/html_reports/js/dependencies.csv @@ -1,4 +1,4 @@ -vendor,product,version -getbootstrap,bootstrap,4.5.0 -jquery,jquery,3.4.1 -plotly,plotly.js,1.54.1 +vendor,product +getbootstrap,bootstrap +jquery,jquery +plotly,plotly.js diff --git a/requirements_csv.py b/requirements_csv.py index c5bb7c5dbb..47492cb51e 100644 --- a/requirements_csv.py +++ b/requirements_csv.py @@ -5,6 +5,7 @@ import csv import os +import re from importlib.metadata import version REQ_CSV = os.path.abspath(os.path.join(os.path.dirname(__file__), "requirements.csv")) @@ -14,6 +15,12 @@ CACHE_CSV = os.path.join( os.path.expanduser("~"), ".cache", "cve-bin-tool", "requirements.csv" ) +HTML_DEP_PATH = os.path.abspath( + os.path.join( + os.path.dirname(__file__), "cve_bin_tool", "output_engine", "html_reports", "js" + ) +) +HTML_DEP_CSV = os.path.join(HTML_DEP_PATH, "dependencies.csv") def get_cache_csv_data(file): @@ -22,12 +29,29 @@ def get_cache_csv_data(file): r = csv.reader(f) next(r) for row in r: - if "_not_in_db" not in row[0]: - data.append((row[0], row[1], version(row[1]))) + if file is HTML_DEP_CSV: + file_name = ( + "{}/{}".format(HTML_DEP_PATH, row[1]) + if ".js" in row[1] + else "{}/{}.js".format(HTML_DEP_PATH, row[1]) + ) + with open(file_name) as f: + file_content = f.read() + html_dep_version = re.search( + r"v([0-9]+\.[0-9]+\.[0-9]+)", file_content + ).group(1) + data.append((row[0], row[1], html_dep_version)) + else: + if "_not_in_db" not in row[0]: + data.append((row[0], row[1], version(row[1]))) return data -cache_csv_data = get_cache_csv_data(REQ_CSV) + get_cache_csv_data(DOC_CSV) +cache_csv_data = ( + get_cache_csv_data(REQ_CSV) + + get_cache_csv_data(DOC_CSV) + + get_cache_csv_data(HTML_DEP_CSV) +) # writes a cache CSV file with open(CACHE_CSV, "w") as f: From d925d79f20b8ba0b6a248237af241a0ea8201159 Mon Sep 17 00:00:00 2001 From: BreadGenie Date: Wed, 21 Apr 2021 18:34:00 +0530 Subject: [PATCH 13/38] Remove seperate scanning html dependencies --- .github/workflows/pythonapp.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index d5ef2a3adc..84ce961bb9 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -239,7 +239,7 @@ jobs: test/test_cli.py cve_scan: - name: CVE Scan of requirements + name: CVE Scan on dependencies runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -276,4 +276,3 @@ jobs: - name: Run CVE Binary tool against requirements.csv and HTML report dependencies run: | python -m cve_bin_tool.cli --input-file ~/.cache/cve-bin-tool/requirements.csv - python -m cve_bin_tool.cli --input-file cve_bin_tool/output_engine/html_reports/js/dependencies.csv From 50bdcc96b23fcc67b0fa93ba553b254e4aadd529 Mon Sep 17 00:00:00 2001 From: Bread Genie <63963181+BreadGenie@users.noreply.github.com> Date: Wed, 21 Apr 2021 20:03:46 +0530 Subject: [PATCH 14/38] Update requirements_csv.py Co-authored-by: Dmitry Volodin --- requirements_csv.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/requirements_csv.py b/requirements_csv.py index 47492cb51e..8260d5c38d 100644 --- a/requirements_csv.py +++ b/requirements_csv.py @@ -30,11 +30,9 @@ def get_cache_csv_data(file): next(r) for row in r: if file is HTML_DEP_CSV: - file_name = ( - "{}/{}".format(HTML_DEP_PATH, row[1]) - if ".js" in row[1] - else "{}/{}.js".format(HTML_DEP_PATH, row[1]) - ) + file_name = f"{HTML_DEP_PATH}/{row[1]}" + if not file_name.endswith(".js"): + file_name += ".js" with open(file_name) as f: file_content = f.read() html_dep_version = re.search( From a9eefb548408d84cd0d58ae71ba1cbe0ddd00ed6 Mon Sep 17 00:00:00 2001 From: BreadGenie Date: Wed, 21 Apr 2021 21:46:37 +0530 Subject: [PATCH 15/38] Refactor code --- requirements_csv.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements_csv.py b/requirements_csv.py index 8260d5c38d..f01a2b2724 100644 --- a/requirements_csv.py +++ b/requirements_csv.py @@ -28,9 +28,9 @@ def get_cache_csv_data(file): with open(file, "r") as f: r = csv.reader(f) next(r) - for row in r: + for (vendor, product) in r: if file is HTML_DEP_CSV: - file_name = f"{HTML_DEP_PATH}/{row[1]}" + file_name = f"{HTML_DEP_PATH}/{product}" if not file_name.endswith(".js"): file_name += ".js" with open(file_name) as f: @@ -38,10 +38,10 @@ def get_cache_csv_data(file): html_dep_version = re.search( r"v([0-9]+\.[0-9]+\.[0-9]+)", file_content ).group(1) - data.append((row[0], row[1], html_dep_version)) + data.append((vendor, product, html_dep_version)) else: - if "_not_in_db" not in row[0]: - data.append((row[0], row[1], version(row[1]))) + if "_not_in_db" not in vendor: + data.append((vendor, product, version(product))) return data From d9abc333a2502aa1c3100060a023c203048dcdd3 Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Wed, 28 Apr 2021 15:56:26 +0530 Subject: [PATCH 16/38] Move requirements_csv.py to test folder --- .github/workflows/pythonapp.yml | 4 ++-- requirements_csv.py => test/requirements_csv.py | 13 ++++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) rename requirements_csv.py => test/requirements_csv.py (83%) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 84ce961bb9..7e5ee5c496 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -272,7 +272,7 @@ jobs: pip install -r doc/requirements.txt - name: Create cache requirements.csv with versions run: | - python requirements_csv.py - - name: Run CVE Binary tool against requirements.csv and HTML report dependencies + python test/requirements_csv.py + - name: Run CVE Binary tool on requirements.csv run: | python -m cve_bin_tool.cli --input-file ~/.cache/cve-bin-tool/requirements.csv diff --git a/requirements_csv.py b/test/requirements_csv.py similarity index 83% rename from requirements_csv.py rename to test/requirements_csv.py index f01a2b2724..e1ff33af27 100644 --- a/requirements_csv.py +++ b/test/requirements_csv.py @@ -8,16 +8,23 @@ import re from importlib.metadata import version -REQ_CSV = os.path.abspath(os.path.join(os.path.dirname(__file__), "requirements.csv")) +REQ_CSV = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "requirements.csv") +) DOC_CSV = os.path.abspath( - os.path.join(os.path.dirname(__file__), "doc", "requirements.csv") + os.path.join(os.path.dirname(__file__), "..", "doc", "requirements.csv") ) CACHE_CSV = os.path.join( os.path.expanduser("~"), ".cache", "cve-bin-tool", "requirements.csv" ) HTML_DEP_PATH = os.path.abspath( os.path.join( - os.path.dirname(__file__), "cve_bin_tool", "output_engine", "html_reports", "js" + os.path.dirname(__file__), + "..", + "cve_bin_tool", + "output_engine", + "html_reports", + "js", ) ) HTML_DEP_CSV = os.path.join(HTML_DEP_PATH, "dependencies.csv") From 98e0973282cb9e05b60c1974e9f7c03513bf6676 Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Thu, 29 Apr 2021 11:51:34 +0530 Subject: [PATCH 17/38] Add allowable list of packages --- test/requirements_csv.py | 70 +++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/test/requirements_csv.py b/test/requirements_csv.py index e1ff33af27..7dd5099d2f 100644 --- a/test/requirements_csv.py +++ b/test/requirements_csv.py @@ -29,40 +29,44 @@ ) HTML_DEP_CSV = os.path.join(HTML_DEP_PATH, "dependencies.csv") +# Dependencies that currently have CVEs +ALLOWED_PACKAGES = ["jquery", "reportlab"] -def get_cache_csv_data(file): - data = [] - with open(file, "r") as f: - r = csv.reader(f) - next(r) - for (vendor, product) in r: - if file is HTML_DEP_CSV: - file_name = f"{HTML_DEP_PATH}/{product}" - if not file_name.endswith(".js"): - file_name += ".js" - with open(file_name) as f: - file_content = f.read() - html_dep_version = re.search( - r"v([0-9]+\.[0-9]+\.[0-9]+)", file_content - ).group(1) - data.append((vendor, product, html_dep_version)) - else: - if "_not_in_db" not in vendor: - data.append((vendor, product, version(product))) - return data +def test_requirements(): + def get_cache_csv_data(file): + data = [] + with open(file, "r") as f: + r = csv.reader(f) + next(r) + for (vendor, product) in r: + if file is HTML_DEP_CSV: + file_name = f"{HTML_DEP_PATH}/{product}" + if not file_name.endswith(".js"): + file_name += ".js" + with open(file_name) as f: + file_content = f.read() + html_dep_version = re.search( + r"v([0-9]+\.[0-9]+\.[0-9]+)", file_content + ).group(1) + if product not in ALLOWED_PACKAGES: + data.append((vendor, product, html_dep_version)) + else: + if "_not_in_db" not in vendor and product not in ALLOWED_PACKAGES: + data.append((vendor, product, version(product))) + return data -cache_csv_data = ( - get_cache_csv_data(REQ_CSV) - + get_cache_csv_data(DOC_CSV) - + get_cache_csv_data(HTML_DEP_CSV) -) + cache_csv_data = ( + get_cache_csv_data(REQ_CSV) + + get_cache_csv_data(DOC_CSV) + + get_cache_csv_data(HTML_DEP_CSV) + ) -# writes a cache CSV file -with open(CACHE_CSV, "w") as f: - writer = csv.writer(f) - fieldnames = ["vendor", "product", "version"] - writer = csv.writer(f) - writer.writerow(fieldnames) - for row in cache_csv_data: - writer.writerow(row) + # writes a cache CSV file + with open(CACHE_CSV, "w") as f: + writer = csv.writer(f) + fieldnames = ["vendor", "product", "version"] + writer = csv.writer(f) + writer.writerow(fieldnames) + for row in cache_csv_data: + writer.writerow(row) From b518167ba6d04a9ba7e280dd95ccdbf18b860fd8 Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Thu, 29 Apr 2021 11:57:09 +0530 Subject: [PATCH 18/38] Fix workflow --- .github/workflows/pythonapp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 7e5ee5c496..9cf8e98c43 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -272,7 +272,7 @@ jobs: pip install -r doc/requirements.txt - name: Create cache requirements.csv with versions run: | - python test/requirements_csv.py + pytest test/requirements_csv.py - name: Run CVE Binary tool on requirements.csv run: | python -m cve_bin_tool.cli --input-file ~/.cache/cve-bin-tool/requirements.csv From d181e53eb43961d45c7d145fe7ea979163f299c2 Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Thu, 29 Apr 2021 14:25:05 +0530 Subject: [PATCH 19/38] Run CVE scan inside the test --- .github/workflows/pythonapp.yml | 7 ++----- test/{requirements_csv.py => test_requirements.py} | 8 ++++++++ 2 files changed, 10 insertions(+), 5 deletions(-) rename test/{requirements_csv.py => test_requirements.py} (90%) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 9cf8e98c43..26943c70f6 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -270,9 +270,6 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt pip install -r doc/requirements.txt - - name: Create cache requirements.csv with versions + - name: Test to check for CVEs for python requirements and HTML report dependencies run: | - pytest test/requirements_csv.py - - name: Run CVE Binary tool on requirements.csv - run: | - python -m cve_bin_tool.cli --input-file ~/.cache/cve-bin-tool/requirements.csv + pytest test/test_requirements.py diff --git a/test/requirements_csv.py b/test/test_requirements.py similarity index 90% rename from test/requirements_csv.py rename to test/test_requirements.py index 7dd5099d2f..ca7c4ad281 100644 --- a/test/requirements_csv.py +++ b/test/test_requirements.py @@ -6,6 +6,7 @@ import csv import os import re +import subprocess from importlib.metadata import version REQ_CSV = os.path.abspath( @@ -70,3 +71,10 @@ def get_cache_csv_data(file): writer.writerow(fieldnames) for row in cache_csv_data: writer.writerow(row) + + cve_check = subprocess.run( + ["python3", "-m", "cve_bin_tool.cli", "--input-file", CACHE_CSV] + ) + assert ( + cve_check.returncode == 0 + ), f"{cve_check.returncode} dependecies/requirement have CVEs" From a0825d02a3609445a9c1d6a8eeecdf8704cc2f9a Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Fri, 30 Apr 2021 04:30:10 +0530 Subject: [PATCH 20/38] Change command and add comment --- test/test_requirements.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/test_requirements.py b/test/test_requirements.py index ca7c4ad281..5d7ba9a49d 100644 --- a/test/test_requirements.py +++ b/test/test_requirements.py @@ -1,8 +1,6 @@ # Copyright (C) 2021 Intel Corporation # SPDX-License-Identifier: GPL-3.0-or-later -# Script to add versions to requirements.csv - import csv import os import re @@ -34,6 +32,7 @@ ALLOWED_PACKAGES = ["jquery", "reportlab"] +# Test to check for CVEs in cve-bin-tool requirements/dependencies def test_requirements(): def get_cache_csv_data(file): data = [] @@ -73,7 +72,7 @@ def get_cache_csv_data(file): writer.writerow(row) cve_check = subprocess.run( - ["python3", "-m", "cve_bin_tool.cli", "--input-file", CACHE_CSV] + ["python", "-m", "cve_bin_tool.cli", "--input-file", CACHE_CSV] ) assert ( cve_check.returncode == 0 From eb00e33d2f6bdf2cfbc41a0e793ddcee5270e7c4 Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Fri, 30 Apr 2021 22:55:59 +0530 Subject: [PATCH 21/38] Remove nested function --- test/test_requirements.py | 47 +++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/test/test_requirements.py b/test/test_requirements.py index 5d7ba9a49d..66181ac0eb 100644 --- a/test/test_requirements.py +++ b/test/test_requirements.py @@ -29,32 +29,35 @@ HTML_DEP_CSV = os.path.join(HTML_DEP_PATH, "dependencies.csv") # Dependencies that currently have CVEs +# Remove from the list once they are updated ALLOWED_PACKAGES = ["jquery", "reportlab"] +def get_cache_csv_data(file): + data = [] + with open(file, "r") as f: + r = csv.reader(f) + next(r) + for (vendor, product) in r: + if file is HTML_DEP_CSV: + file_name = f"{HTML_DEP_PATH}/{product}" + if not file_name.endswith(".js"): + file_name += ".js" + with open(file_name) as f: + file_content = f.read() + html_dep_version = re.search( + r"v([0-9]+\.[0-9]+\.[0-9]+)", file_content + ).group(1) + if product not in ALLOWED_PACKAGES: + data.append((vendor, product, html_dep_version)) + else: + if "_not_in_db" not in vendor and product not in ALLOWED_PACKAGES: + data.append((vendor, product, version(product))) + return data + + # Test to check for CVEs in cve-bin-tool requirements/dependencies def test_requirements(): - def get_cache_csv_data(file): - data = [] - with open(file, "r") as f: - r = csv.reader(f) - next(r) - for (vendor, product) in r: - if file is HTML_DEP_CSV: - file_name = f"{HTML_DEP_PATH}/{product}" - if not file_name.endswith(".js"): - file_name += ".js" - with open(file_name) as f: - file_content = f.read() - html_dep_version = re.search( - r"v([0-9]+\.[0-9]+\.[0-9]+)", file_content - ).group(1) - if product not in ALLOWED_PACKAGES: - data.append((vendor, product, html_dep_version)) - else: - if "_not_in_db" not in vendor and product not in ALLOWED_PACKAGES: - data.append((vendor, product, version(product))) - return data cache_csv_data = ( get_cache_csv_data(REQ_CSV) @@ -76,4 +79,4 @@ def get_cache_csv_data(file): ) assert ( cve_check.returncode == 0 - ), f"{cve_check.returncode} dependecies/requirement have CVEs" + ), f"{cve_check.returncode} dependecies/requirements have CVEs" From bbdd4d12a10cee8bfe97de394876bb6ee0418cfd Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Fri, 30 Apr 2021 23:05:40 +0530 Subject: [PATCH 22/38] Rerun CI From 68f6fbd530e1c0c638fea21138d4cd98855c5ef2 Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Wed, 5 May 2021 03:15:53 +0530 Subject: [PATCH 23/38] Remove jQuery from allowed list --- test/test_requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_requirements.py b/test/test_requirements.py index 66181ac0eb..5968e1b9a6 100644 --- a/test/test_requirements.py +++ b/test/test_requirements.py @@ -30,7 +30,7 @@ # Dependencies that currently have CVEs # Remove from the list once they are updated -ALLOWED_PACKAGES = ["jquery", "reportlab"] +ALLOWED_PACKAGES = ["reportlab"] def get_cache_csv_data(file): From 758b227dcc031463788dc6a11c268d4ef9f43b5c Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Fri, 14 May 2021 17:45:06 +0530 Subject: [PATCH 24/38] Add test to check if requirements.csv and requirements.txt files are in sync --- test/test_requirements.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test/test_requirements.py b/test/test_requirements.py index 5968e1b9a6..6e015f483b 100644 --- a/test/test_requirements.py +++ b/test/test_requirements.py @@ -7,9 +7,15 @@ import subprocess from importlib.metadata import version +REQ_TXT = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "requirements.txt") +) REQ_CSV = os.path.abspath( os.path.join(os.path.dirname(__file__), "..", "requirements.csv") ) +DOC_TXT = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "doc", "requirements.txt") +) DOC_CSV = os.path.abspath( os.path.join(os.path.dirname(__file__), "..", "doc", "requirements.csv") ) @@ -33,6 +39,35 @@ ALLOWED_PACKAGES = ["reportlab"] +def get_out_of_sync_packages(csvfile, txtfile): + + out_of_sync_packages = [] + csv_package_names = [] + with open(csvfile, "r") as csvf, open(txtfile, "r") as txt: + csvfl = csv.reader(csvf) + next(csvfl) + for (vendor, product) in csvfl: + csv_package_names.append(product) + lines = txt.readlines() + for line in lines: + txt_package = re.split(">|\\[|;|=|\n", line)[0] + if txt_package not in csv_package_names: + out_of_sync_packages.append(txt_package) + return out_of_sync_packages + + +# Test to check if the requirements.csv files are in sync with requirements.txt files +def test_txt_csv_sync(): + + out_of_sync_packages = get_out_of_sync_packages( + REQ_CSV, REQ_TXT + ) + get_out_of_sync_packages(DOC_CSV, DOC_TXT) + + assert ( + out_of_sync_packages == [] + ), f"The CSV file is out of sync! Please add {', '.join(out_of_sync_packages)} in the respective requirements.csv file" + + def get_cache_csv_data(file): data = [] with open(file, "r") as f: From 8a7c53a293f8d10225a80963d0f32d7e9788563a Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Fri, 14 May 2021 17:45:22 +0530 Subject: [PATCH 25/38] Capitalize sphinx to match in requirements.txt --- doc/requirements.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/requirements.csv b/doc/requirements.csv index 15d37de5da..0497e9fbe2 100644 --- a/doc/requirements.csv +++ b/doc/requirements.csv @@ -1,4 +1,4 @@ vendor,product rtfd_not_in_db,recommonmark -sphinx-doc_not_in_db,sphinx +sphinx-doc_not_in_db,Sphinx ryanfox_not_in_db,sphinx_markdown_tables From ec6bfe4e7e32a0ad6f10f2d9d94a9f7dd77a0e7a Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Fri, 14 May 2021 19:47:56 +0530 Subject: [PATCH 26/38] Create cache folder if it doesn't exist --- test/test_requirements.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/test/test_requirements.py b/test/test_requirements.py index 6e015f483b..28180ef0ff 100644 --- a/test/test_requirements.py +++ b/test/test_requirements.py @@ -19,9 +19,11 @@ DOC_CSV = os.path.abspath( os.path.join(os.path.dirname(__file__), "..", "doc", "requirements.csv") ) -CACHE_CSV = os.path.join( - os.path.expanduser("~"), ".cache", "cve-bin-tool", "requirements.csv" -) + +CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "cve-bin-tool") + +CACHE_CSV = os.path.join(CACHE_PATH, "requirements.csv") + HTML_DEP_PATH = os.path.abspath( os.path.join( os.path.dirname(__file__), @@ -43,6 +45,7 @@ def get_out_of_sync_packages(csvfile, txtfile): out_of_sync_packages = [] csv_package_names = [] + with open(csvfile, "r") as csvf, open(txtfile, "r") as txt: csvfl = csv.reader(csvf) next(csvfl) @@ -53,6 +56,7 @@ def get_out_of_sync_packages(csvfile, txtfile): txt_package = re.split(">|\\[|;|=|\n", line)[0] if txt_package not in csv_package_names: out_of_sync_packages.append(txt_package) + return out_of_sync_packages @@ -69,7 +73,9 @@ def test_txt_csv_sync(): def get_cache_csv_data(file): + data = [] + with open(file, "r") as f: r = csv.reader(f) next(r) @@ -88,6 +94,7 @@ def get_cache_csv_data(file): else: if "_not_in_db" not in vendor and product not in ALLOWED_PACKAGES: data.append((vendor, product, version(product))) + return data @@ -100,6 +107,9 @@ def test_requirements(): + get_cache_csv_data(HTML_DEP_CSV) ) + if not os.path.exists(CACHE_PATH): + os.mkdir(CACHE_PATH) + # writes a cache CSV file with open(CACHE_CSV, "w") as f: writer = csv.writer(f) From 8ddf147c51870b5a96d82e42989c5aeccbe000f3 Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Sat, 15 May 2021 11:31:34 +0530 Subject: [PATCH 27/38] Refactor Code --- test/test_requirements.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/test_requirements.py b/test/test_requirements.py index 28180ef0ff..45746253e1 100644 --- a/test/test_requirements.py +++ b/test/test_requirements.py @@ -41,17 +41,17 @@ ALLOWED_PACKAGES = ["reportlab"] -def get_out_of_sync_packages(csvfile, txtfile): +def get_out_of_sync_packages(csv_name, txt_name): out_of_sync_packages = [] csv_package_names = [] - with open(csvfile, "r") as csvf, open(txtfile, "r") as txt: - csvfl = csv.reader(csvf) - next(csvfl) - for (vendor, product) in csvfl: + with open(csv_name) as csv_file, open(txt_name) as txt_file: + csv_reader = csv.reader(csv_file) + next(csv_reader) + for (_, product) in csv_reader: csv_package_names.append(product) - lines = txt.readlines() + lines = txt_file.readlines() for line in lines: txt_package = re.split(">|\\[|;|=|\n", line)[0] if txt_package not in csv_package_names: @@ -76,7 +76,7 @@ def get_cache_csv_data(file): data = [] - with open(file, "r") as f: + with open(file) as f: r = csv.reader(f) next(r) for (vendor, product) in r: @@ -124,4 +124,4 @@ def test_requirements(): ) assert ( cve_check.returncode == 0 - ), f"{cve_check.returncode} dependecies/requirements have CVEs" + ), f"{cve_check.returncode} dependencies/requirements have CVEs" From e567d8114bc7706ca917fbe32a1623035d58c243 Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Sat, 15 May 2021 12:15:08 +0530 Subject: [PATCH 28/38] Improve test case Cover cases for when csv file is not in sync with txt file and split the error messages for improved reading. --- test/test_requirements.py | 52 ++++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/test/test_requirements.py b/test/test_requirements.py index 45746253e1..6cc84488d5 100644 --- a/test/test_requirements.py +++ b/test/test_requirements.py @@ -43,33 +43,57 @@ def get_out_of_sync_packages(csv_name, txt_name): - out_of_sync_packages = [] - csv_package_names = [] + out_of_sync_txt_packages = set() + out_of_sync_csv_packages = set() + csv_package_names = set() + txt_package_names = set() with open(csv_name) as csv_file, open(txt_name) as txt_file: csv_reader = csv.reader(csv_file) next(csv_reader) for (_, product) in csv_reader: - csv_package_names.append(product) + csv_package_names.add(product) lines = txt_file.readlines() for line in lines: - txt_package = re.split(">|\\[|;|=|\n", line)[0] - if txt_package not in csv_package_names: - out_of_sync_packages.append(txt_package) + txt_package_names.add(re.split(">|\\[|;|=|\n", line)[0]) + out_of_sync_txt_packages = txt_package_names - csv_package_names + out_of_sync_csv_packages = csv_package_names - txt_package_names - return out_of_sync_packages + return (out_of_sync_txt_packages, out_of_sync_csv_packages) # Test to check if the requirements.csv files are in sync with requirements.txt files def test_txt_csv_sync(): - out_of_sync_packages = get_out_of_sync_packages( - REQ_CSV, REQ_TXT - ) + get_out_of_sync_packages(DOC_CSV, DOC_TXT) - - assert ( - out_of_sync_packages == [] - ), f"The CSV file is out of sync! Please add {', '.join(out_of_sync_packages)} in the respective requirements.csv file" + errors = set() + + ( + out_of_sync_req_txt_packages, + out_of_sync_req_csv_packages, + ) = get_out_of_sync_packages(REQ_CSV, REQ_TXT) + ( + out_of_sync_doc_txt_packages, + out_of_sync_doc_csv_packages, + ) = get_out_of_sync_packages(DOC_CSV, DOC_TXT) + + if out_of_sync_doc_csv_packages != set(): + errors.add( + f"The requirements.txt and requirements.csv files of docs are out of sync! Please add {', '.join(out_of_sync_doc_csv_packages)} in the respective requirements.txt file\n" + ) + if out_of_sync_doc_txt_packages != set(): + errors.add( + f"The requirements.txt and requirements.csv files of docs are out of sync! Please add {', '.join(out_of_sync_doc_txt_packages)} in the respective requirements.csv file\n" + ) + if out_of_sync_req_csv_packages != set(): + errors.add( + f"The requirements.txt and requirements.csv files of cve-bin-tool are out of sync! Please add {', '.join(out_of_sync_req_csv_packages)} in the respective requirements.txt file\n" + ) + if out_of_sync_req_txt_packages != set(): + errors.add( + f"The requirements.txt and requirements.csv files of cve-bin-tool are out of sync! Please add {', '.join(out_of_sync_req_txt_packages)} in the respective requirements.csv file\n" + ) + + assert errors == set(), f"The errors are {''.join(errors)}" def get_cache_csv_data(file): From 524ad19c96c04b081b2ea02e1a41725de464df64 Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Sat, 15 May 2021 12:26:04 +0530 Subject: [PATCH 29/38] Add single run of cve-bin-tool Add single run of cve-bin-tool to produce a cache folder properly and remove the check for cache directory. --- .github/workflows/pythonapp.yml | 4 ++++ test/test_requirements.py | 3 --- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 26943c70f6..e35e9058ea 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -270,6 +270,10 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt pip install -r doc/requirements.txt + - name: try single cli run of tool + run: | + python -m pip install -e . + NO_EXIT_CVE_NUM=1 python -m cve_bin_tool.cli test/assets - name: Test to check for CVEs for python requirements and HTML report dependencies run: | pytest test/test_requirements.py diff --git a/test/test_requirements.py b/test/test_requirements.py index 6cc84488d5..1203a761fb 100644 --- a/test/test_requirements.py +++ b/test/test_requirements.py @@ -131,9 +131,6 @@ def test_requirements(): + get_cache_csv_data(HTML_DEP_CSV) ) - if not os.path.exists(CACHE_PATH): - os.mkdir(CACHE_PATH) - # writes a cache CSV file with open(CACHE_CSV, "w") as f: writer = csv.writer(f) From 33ad4379784bed7bb30301da516ceff284c036af Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Sat, 15 May 2021 13:25:43 +0530 Subject: [PATCH 30/38] Format assert message --- test/test_requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_requirements.py b/test/test_requirements.py index 1203a761fb..5a9b41a1b9 100644 --- a/test/test_requirements.py +++ b/test/test_requirements.py @@ -93,7 +93,7 @@ def test_txt_csv_sync(): f"The requirements.txt and requirements.csv files of cve-bin-tool are out of sync! Please add {', '.join(out_of_sync_req_txt_packages)} in the respective requirements.csv file\n" ) - assert errors == set(), f"The errors are {''.join(errors)}" + assert errors == set(), f"The error(s) are:\n {''.join(errors)}" def get_cache_csv_data(file): From d1a9cbeb3fda0bc998f923d0ecf9257cadaa1256 Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Sat, 15 May 2021 14:40:38 +0530 Subject: [PATCH 31/38] Install cabextract before scan --- .github/workflows/pythonapp.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index e35e9058ea..2e5fb5e029 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -270,6 +270,8 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt pip install -r doc/requirements.txt + - name: Install cabextract + run: sudo apt-get install cabextract - name: try single cli run of tool run: | python -m pip install -e . From 9a58ea587bc617aa1e1035123dc8673144263553 Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Sat, 15 May 2021 18:28:30 +0530 Subject: [PATCH 32/38] Correct the error message and variables --- test/test_requirements.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/test/test_requirements.py b/test/test_requirements.py index 5a9b41a1b9..82ee50c980 100644 --- a/test/test_requirements.py +++ b/test/test_requirements.py @@ -43,8 +43,8 @@ def get_out_of_sync_packages(csv_name, txt_name): - out_of_sync_txt_packages = set() - out_of_sync_csv_packages = set() + new_packages = set() + removed_packages = set() csv_package_names = set() txt_package_names = set() @@ -56,10 +56,10 @@ def get_out_of_sync_packages(csv_name, txt_name): lines = txt_file.readlines() for line in lines: txt_package_names.add(re.split(">|\\[|;|=|\n", line)[0]) - out_of_sync_txt_packages = txt_package_names - csv_package_names - out_of_sync_csv_packages = csv_package_names - txt_package_names + new_packages = txt_package_names - csv_package_names + removed_packages = csv_package_names - txt_package_names - return (out_of_sync_txt_packages, out_of_sync_csv_packages) + return (new_packages, removed_packages) # Test to check if the requirements.csv files are in sync with requirements.txt files @@ -68,29 +68,29 @@ def test_txt_csv_sync(): errors = set() ( - out_of_sync_req_txt_packages, - out_of_sync_req_csv_packages, + req_new_packages, + req_removed_packages, ) = get_out_of_sync_packages(REQ_CSV, REQ_TXT) ( - out_of_sync_doc_txt_packages, - out_of_sync_doc_csv_packages, + doc_new_packages, + doc_removed_packages, ) = get_out_of_sync_packages(DOC_CSV, DOC_TXT) - if out_of_sync_doc_csv_packages != set(): + if doc_removed_packages != set(): errors.add( - f"The requirements.txt and requirements.csv files of docs are out of sync! Please add {', '.join(out_of_sync_doc_csv_packages)} in the respective requirements.txt file\n" + f"The requirements.txt and requirements.csv files of docs are out of sync! Please remove {', '.join(doc_removed_packages)} from the respective requirements.csv file\n" ) - if out_of_sync_doc_txt_packages != set(): + if doc_new_packages != set(): errors.add( - f"The requirements.txt and requirements.csv files of docs are out of sync! Please add {', '.join(out_of_sync_doc_txt_packages)} in the respective requirements.csv file\n" + f"The requirements.txt and requirements.csv files of docs are out of sync! Please add {', '.join(doc_new_packages)} to the respective requirements.csv file\n" ) - if out_of_sync_req_csv_packages != set(): + if req_removed_packages != set(): errors.add( - f"The requirements.txt and requirements.csv files of cve-bin-tool are out of sync! Please add {', '.join(out_of_sync_req_csv_packages)} in the respective requirements.txt file\n" + f"The requirements.txt and requirements.csv files of cve-bin-tool are out of sync! Please remove {', '.join(req_removed_packages)} from the respective requirements.csv file\n" ) - if out_of_sync_req_txt_packages != set(): + if req_new_packages != set(): errors.add( - f"The requirements.txt and requirements.csv files of cve-bin-tool are out of sync! Please add {', '.join(out_of_sync_req_txt_packages)} in the respective requirements.csv file\n" + f"The requirements.txt and requirements.csv files of cve-bin-tool are out of sync! Please add {', '.join(req_new_packages)} to the respective requirements.csv file\n" ) assert errors == set(), f"The error(s) are:\n {''.join(errors)}" From af9f6a6302da064761fcb606a8969ce563ceabab Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Sat, 15 May 2021 19:20:27 +0530 Subject: [PATCH 33/38] Remove redundant aiohttp entry --- requirements.csv | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.csv b/requirements.csv index 48edbec8f4..70288ab7d2 100644 --- a/requirements.csv +++ b/requirements.csv @@ -11,7 +11,6 @@ pytest_not_in_db,pytest-asyncio pycqa_not_in_db,isort willmcgugan_not_in_db,rich crummy_not_in_db,beautifulsoup4 -aio-libs_not_in_db,aiohttp uiri_not_in_db,toml jsonschema_not_in_db,jsonschema python_not_in_db,py From d7bd5fe1a8e2da0ab9f687ece4beb5c809be9970 Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Sat, 15 May 2021 19:20:34 +0530 Subject: [PATCH 34/38] Remove caching of the .csv file Save the .csv file to the root of the project directory instead of cache directory. --- .github/workflows/pythonapp.yml | 6 ------ test/test_requirements.py | 8 +++----- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 2e5fb5e029..26943c70f6 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -270,12 +270,6 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt pip install -r doc/requirements.txt - - name: Install cabextract - run: sudo apt-get install cabextract - - name: try single cli run of tool - run: | - python -m pip install -e . - NO_EXIT_CVE_NUM=1 python -m cve_bin_tool.cli test/assets - name: Test to check for CVEs for python requirements and HTML report dependencies run: | pytest test/test_requirements.py diff --git a/test/test_requirements.py b/test/test_requirements.py index 82ee50c980..753f895968 100644 --- a/test/test_requirements.py +++ b/test/test_requirements.py @@ -20,9 +20,7 @@ os.path.join(os.path.dirname(__file__), "..", "doc", "requirements.csv") ) -CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "cve-bin-tool") - -CACHE_CSV = os.path.join(CACHE_PATH, "requirements.csv") +SCAN_CSV = os.path.join( os.path.join(os.path.dirname(__file__)), "..", "cve_bin_tool_requirements.csv") HTML_DEP_PATH = os.path.abspath( os.path.join( @@ -132,7 +130,7 @@ def test_requirements(): ) # writes a cache CSV file - with open(CACHE_CSV, "w") as f: + with open(SCAN_CSV, "w") as f: writer = csv.writer(f) fieldnames = ["vendor", "product", "version"] writer = csv.writer(f) @@ -141,7 +139,7 @@ def test_requirements(): writer.writerow(row) cve_check = subprocess.run( - ["python", "-m", "cve_bin_tool.cli", "--input-file", CACHE_CSV] + ["python3", "-m", "cve_bin_tool.cli", "--input-file", SCAN_CSV] ) assert ( cve_check.returncode == 0 From fa142d811f6419acc30a2691d4a9967e5f639927 Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Sat, 15 May 2021 19:24:20 +0530 Subject: [PATCH 35/38] Format with black --- test/test_requirements.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_requirements.py b/test/test_requirements.py index 753f895968..2d32c84077 100644 --- a/test/test_requirements.py +++ b/test/test_requirements.py @@ -20,7 +20,9 @@ os.path.join(os.path.dirname(__file__), "..", "doc", "requirements.csv") ) -SCAN_CSV = os.path.join( os.path.join(os.path.dirname(__file__)), "..", "cve_bin_tool_requirements.csv") +SCAN_CSV = os.path.join( + os.path.join(os.path.dirname(__file__)), "..", "cve_bin_tool_requirements.csv" +) HTML_DEP_PATH = os.path.abspath( os.path.join( @@ -139,7 +141,7 @@ def test_requirements(): writer.writerow(row) cve_check = subprocess.run( - ["python3", "-m", "cve_bin_tool.cli", "--input-file", SCAN_CSV] + ["python", "-m", "cve_bin_tool.cli", "--input-file", SCAN_CSV] ) assert ( cve_check.returncode == 0 From 960cca8854b63475579f65f9a95506c752e66311 Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Sat, 15 May 2021 21:42:13 +0530 Subject: [PATCH 36/38] Add the .csv file used for scan in .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d08df3b0e2..13d12ad514 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,5 @@ build/ dist/ doc/_build test/downloads/ +cve_bin_tool_requirements.csv !test/condensed-downloads/*.tar.gz From bd04c1705892d8691f495a01d4fbcba734bb6866 Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Sat, 15 May 2021 21:46:53 +0530 Subject: [PATCH 37/38] Refactor code --- test/test_requirements.py | 42 +++++++++++++++------------------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/test/test_requirements.py b/test/test_requirements.py index 2d32c84077..f6f05f0989 100644 --- a/test/test_requirements.py +++ b/test/test_requirements.py @@ -2,39 +2,29 @@ # SPDX-License-Identifier: GPL-3.0-or-later import csv -import os import re import subprocess from importlib.metadata import version +from os.path import dirname, join -REQ_TXT = os.path.abspath( - os.path.join(os.path.dirname(__file__), "..", "requirements.txt") -) -REQ_CSV = os.path.abspath( - os.path.join(os.path.dirname(__file__), "..", "requirements.csv") -) -DOC_TXT = os.path.abspath( - os.path.join(os.path.dirname(__file__), "..", "doc", "requirements.txt") -) -DOC_CSV = os.path.abspath( - os.path.join(os.path.dirname(__file__), "..", "doc", "requirements.csv") -) +ROOT_PATH = join(dirname(__file__), "..") -SCAN_CSV = os.path.join( - os.path.join(os.path.dirname(__file__)), "..", "cve_bin_tool_requirements.csv" -) +REQ_TXT = join(ROOT_PATH, "requirements.txt") +REQ_CSV = join(ROOT_PATH, "requirements.csv") +DOC_TXT = join(ROOT_PATH, "doc", "requirements.txt") +DOC_CSV = join(ROOT_PATH, "doc", "requirements.csv") -HTML_DEP_PATH = os.path.abspath( - os.path.join( - os.path.dirname(__file__), - "..", - "cve_bin_tool", - "output_engine", - "html_reports", - "js", - ) +SCAN_CSV = join(ROOT_PATH, "cve_bin_tool_requirements.csv") + +HTML_DEP_PATH = join( + ROOT_PATH, + "cve_bin_tool", + "output_engine", + "html_reports", + "js", ) -HTML_DEP_CSV = os.path.join(HTML_DEP_PATH, "dependencies.csv") + +HTML_DEP_CSV = join(HTML_DEP_PATH, "dependencies.csv") # Dependencies that currently have CVEs # Remove from the list once they are updated From f0cec4f33c1b366ba2f4fad0856f36d7308134db Mon Sep 17 00:00:00 2001 From: Bread Genie Date: Sat, 15 May 2021 22:01:00 +0530 Subject: [PATCH 38/38] Combine pip install command --- .github/workflows/pythonapp.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 26943c70f6..781c956b8e 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -268,8 +268,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r doc/requirements.txt + pip install -r requirements.txt -r doc/requirements.txt - name: Test to check for CVEs for python requirements and HTML report dependencies run: | pytest test/test_requirements.py