Merge branch 'intel:main' into main

.github/workflows/linting.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        tool: ['isort', 'black', 'pyupgrade', 'flake8', 'bandit', 'gitlint']
+        tool: ['isort', 'black', 'pyupgrade', 'flake8', 'bandit', 'gitlint', 'mypy']
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4

.github/workflows/sbom.yml

@@ -1,6 +1,7 @@
 name: SBOM generation
 
 on:
+  workflow_dispatch:
   schedule:
     # Runs at 02:00 UTC every Monday
     - cron: '2 0 * * 1'
@@ -9,45 +10,51 @@ jobs:
   sbom_gen:
     name: Generate SBOM
     runs-on: ubuntu-latest
-    timeout-minutes: 10
-    continue-on-error: true
+    strategy:
+      matrix:
+        python: ['3.7', '3.8', '3.9', '3.10', '3.11']
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.x'
+          python-version: ${{ matrix.python }}
           cache: 'pip'
           cache-dependency-path: '**/requirements.txt'
-      - name: Get date
-        id: get-date
-        run: |
-          echo "date=$(/bin/date -u "+%Y%m%d")" >> $GITHUB_OUTPUT
-      - name: Get cached database
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/cve-bin-tool
-          key: ${{ runner.os }}-cve-bin-tool-${{ steps.get-date.outputs.date }}
       - name: Install dependencies and cve-bin-tool
         run: |
           python -m pip install --upgrade pip
           python -m pip install --upgrade setuptools
           python -m pip install --upgrade wheel
-          python -m pip install --upgrade pytest
           python -m pip install --upgrade sbom4python
-          pip install . -r doc/requirements.txt
+          pip install . --upgrade --upgrade-strategy=eager
       - name: Generate SBOM for cve-bin-tool
         run: |
-          sbom4python --module cve-bin-tool --output sbom/cve-bin-tool.spdx
-          sbom4python --module cve-bin-tool --sbom cyclonedx --format json --output sbom/cve-bin-tool.json
-#      - name: Compare SBOM for cve-bin-tool
-#        # This would fail due to time/date of SBOM generation in SBOM header
-#        # Therefore ignore first 10 lines of file in comparison which is SBOM header
-#        run: |
-#          /bin/tail -n +10 sbom/cve-bin-tool.spdx > orig
-#          /bin/tail -n +10 cve-bin-tool.spdx > new
-#          /bin/diff -b orig new
-#      - name: Display generated SBOM if difference detected
-#        if: ${{ failure() }}
-#        run: |
-#          /bin/cat cve-bin-tool.spdx
-
+          sbom4python --module cve-bin-tool --output cve-bin-tool-py${{ matrix.python }}.spdx
+          sbom4python --module cve-bin-tool --sbom cyclonedx --format json --output cve-bin-tool-py${{ matrix.python }}.json
+      - name: Compare SBOM for cve-bin-tool
+        id: diff-sbom
+        # This would fail due to time/date of SBOM generation in SBOM header
+        # Therefore ignore first 10 lines of file in comparison which is SBOM header
+        run: |
+          /bin/tail -n +10 sbom/cve-bin-tool-py${{ matrix.python }}.spdx > orig
+          /bin/tail -n +10 cve-bin-tool-py${{ matrix.python }}.spdx > new
+          echo "changed=$(/bin/diff -q orig new)" >> $GITHUB_OUTPUT
+      - name: Display generated SBOM if difference detected
+        if: ${{ steps.diff-sbom.outputs.changed }}
+        run: |
+          /bin/cat cve-bin-tool-py${{ matrix.python }}.spdx
+      - name: Update existing SBOM if difference detected
+        if: ${{ steps.diff-sbom.outputs.changed }}
+        run: |
+          cp cve-bin-tool-py${{ matrix.python }}.spdx sbom/cve-bin-tool-py${{ matrix.python }}.spdx
+          cp cve-bin-tool-py${{ matrix.python }}.json sbom/cve-bin-tool-py${{ matrix.python }}.json
+      - name: Create Pull Request
+        if: ${{ steps.diff-sbom.outputs.changed }}
+        uses: peter-evans/create-pull-request@v4
+        with:
+          commit-message: "chore: update SBOM for Python ${{ matrix.python }}"
+          title: "chore: update SBOM for Python ${{ matrix.python }}"
+          branch: chore-sbom-py${{ matrix.python }}
+          delete-branch: true
+          author: GitHub <[email protected]>
+          add-paths: sbom

.github/workflows/testing.yml

@@ -47,7 +47,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: ['3.7', '3.8', '3.9']
+        python: ['3.7', '3.8', '3.9', '3.11']
     timeout-minutes: 20
     steps:
       - uses: actions/checkout@v3
@@ -294,31 +294,6 @@ jobs:
           test/test_cvedb.py
       - name: Run HTML tests
         run: pytest -v -n auto test/test_html.py
-      - name: Cache conda
-        uses: actions/cache@v3
-        env:
-          # Increase to reset cache if requirements.txt file has not changed
-          CACHE_NUMBER: 0
-        with:
-          path: ~/conda_pkgs_dir
-          key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{
-            hashFiles('requirements.txt') }}
-      - uses: conda-incubator/setup-miniconda@v2
-        with:
-          auto-update-conda: true
-          activate-environment: pdftotext
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
-      - name: Install pdftotext, reportlab and cve-bin-tool
-        run: |
-          conda install -c conda-forge python=3.10 poppler pdftotext
-          python -m pip install --upgrade pip
-          python -m pip install --upgrade setuptools
-          python -m pip install --upgrade wheel
-          python -m pip install --upgrade reportlab
-          python -m pip install --upgrade -r dev-requirements.txt
-          python -m pip install --upgrade .
-      - name: Test PDF generation on Windows
-        run: pytest test/test_output_engine.py -k test_output_pdf
 
   windows_long_tests:
     name: Windows long tests
@@ -401,4 +376,4 @@ jobs:
           files: ./coverage.xml
           flags: win-longtests
           name: codecov-umbrella
-          fail_ci_if_error: false
+          fail_ci_if_error: false

.github/workflows/update-js-dependencies.yml

@@ -22,8 +22,7 @@ jobs:
           python-version: '3.x'
 
       - name: Update JS dependencies
-        run: |
-          python .github/workflows/update_js_dependencies.py
+        run: python .github/workflows/update_js_dependencies.py
 
       - name: Get cached Python packages
         uses: actions/cache@v3
@@ -35,15 +34,13 @@ jobs:
 
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev
           python -m pip install --upgrade pip
           python -m pip install --upgrade setuptools wheel
-          python -m pip install --upgrade -r requirements.txt pytest pdftotext
+          python -m pip install --upgrade . pytest-xdist pytest-playwright
+          python -m playwright install chromium --with-deps
 
-      - name: Test HTML report generation
-        run: |
-          pytest test/test_output_engine.py -k test_output_html
+      - name: Run HTML tests
+        run: python -m pytest -v -n auto test/test_html.py
 
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v4
@@ -62,11 +59,10 @@ jobs:
         run: |
           python -c 'from test.test_output_engine import TestOutputEngine; \
           from cve_bin_tool.output_engine.html import output_html; \
-          output_html(TestOutputEngine.MOCK_OUTPUT, "", "", "", 3, 3, 0, None, None, open("test.html", "w"))'
+          output_html(TestOutputEngine.MOCK_OUTPUT, None, "", "", "", 3, 3, 0, None, None, open("test.html", "w"))'
 
       - name: Upload mock report
         uses: actions/upload-artifact@v3
         with:
           name: HTML report
-          path: |
-            test.html
+          path: test.html

.pre-commit-config.yaml

@@ -31,3 +31,38 @@ repos:
     rev: v0.17.0
     hooks:
     - id: gitlint
+
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.991
+    hooks:
+    - id: mypy
+      additional_dependencies:
+        - types-beautifulsoup4
+        - types-jsonschema
+        - types-PyYAML
+        - types-requests
+        - types-setuptools
+        - types-toml
+      files: |
+        (?x)^(
+            cve_bin_tool/parsers/.*|
+            cve_bin_tool/__init__.py|
+            cve_bin_tool/async_utils.py|
+            cve_bin_tool/file.py|
+            cve_bin_tool/linkify.py|
+            cve_bin_tool/log.py|
+            cve_bin_tool/strings.py|
+            cve_bin_tool/theme.py|
+            cve_bin_tool/util.py|
+            cve_bin_tool/validator.py|
+            cve_bin_tool/version.py|
+            doc/.*|
+            test/test_data/.*|
+            test/__init__.py|
+            test/test_file.py|s
+            test/test_requirements.py|
+            test/test_strings.py|
+            test/test_triage.py|
+            test/test_version.py|
+            test/utils.py|
+        )$

cve_bin_tool/checkers/__init__.py

@@ -34,6 +34,7 @@
     "busybox",
     "bzip2",
     "c_ares",
+    "capnproto",
     "chess",
     "chrony",
     "clamav",
@@ -318,7 +319,7 @@ def guess_contains(self, lines):
     def get_version(self, lines, filename):
         version_info = dict()
 
-        if any(pattern.search(filename) for pattern in self.FILENAME_PATTERNS):
+        if any(pattern.match(filename) for pattern in self.FILENAME_PATTERNS):
             version_info["is_or_contains"] = "is"
 
         if "is_or_contains" not in version_info and self.guess_contains(lines):

cve_bin_tool/checkers/capnproto.py

@@ -0,0 +1,21 @@
+# Copyright (C) 2023 Orange
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+
+"""
+CVE checker for capnproto:
+
+https://www.cvedetails.com/product/37224/Capnproto-Capnproto.html?vendor_id=16364
+
+"""
+from cve_bin_tool.checkers import Checker
+
+
+class CapnprotoChecker(Checker):
+    CONTAINS_PATTERNS: list[str] = []
+    FILENAME_PATTERNS: list[str] = []
+    VERSION_PATTERNS = [
+        r"Cap'n Proto version ([0-9]+\.[0-9]+\.[0-9]+)",
+        r"libcapnp-([0-9]+\.[0-9]+\.[0-9]+)",
+    ]
+    VENDOR_PRODUCT = [("capnproto", "capnproto")]

cve_bin_tool/checkers/sqlite.py

@@ -61,8 +61,8 @@ class SqliteChecker(Checker):
     VENDOR_PRODUCT = [("sqlite", "sqlite")]
     VERSION_PATTERNS = [
         r"Id: SQLite version (\d+\.\d+\.\d+)",
-        r"sqlite(\d+)\.debug",
-    ]  # patterns like the second one aren't ideal (check the end of the file)
+        r"(\d{4}-\d{2}-\d{2} \d+:\d+:\d+ [\w]+)\r?\n(?:SQLite|SQLITE|DESC)",
+    ]
     FILENAME_PATTERNS = [r"sqlite", r"sqlite3"]
 
     mapdb = VersionSignatureDb("sqlite", get_version_map, 30)
@@ -90,7 +90,6 @@ def get_version(self, lines, filename):
 
         The most correct way to do this is to search for the sha1 sums per release.
         Fedora rpms have a simpler SQLite version string.
-        If neither of those work, try to at least guess the major version
         """
 
         version_info = super().get_version(lines, filename)
@@ -105,10 +104,3 @@ def get_version(self, lines, filename):
                 version_info["version"] = mapping[0]
 
         return version_info
-
-
-"""
-Using filenames (containing patterns like '.so' etc.) in the binaries as VERSION_PATTERNS aren't ideal.
-The reason behind this is that these might depend on who packages the file (like it
-might work on fedora but not on ubuntu)
-"""

cve_bin_tool/data_sources/gad_source.py

@@ -211,6 +211,9 @@ def format_data(self, all_cve_entries):
         for cve_item in all_cve_entries:
             cve_in_identifier = None
 
+            if not cve_item:
+                continue
+
             for cve in cve_item.get("identifiers"):
                 if "CVE" in cve:
                     cve_in_identifier = cve

cve_bin_tool/error_handler.py

@@ -130,7 +130,9 @@ class ErrorMode(Enum):
 def excepthook(exc_type, exc_val, exc_tb):
     trace = Traceback.from_exception(exc_type, exc_val, exc_tb)
     CONSOLE.print(trace)
-    if ERROR_CODES.get(exc_type):
+    if isinstance(exc_val, SystemExit):
+        sys.exit(exc_val.code)
+    elif ERROR_CODES.get(exc_type):
         sys.exit(ERROR_CODES[exc_type])
 
 
@@ -164,7 +166,10 @@ def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        if isinstance(exc_val, BaseException):
+        if isinstance(exc_val, SystemExit):
+            self.exit_code = exc_val.code
+            self.exc_val = exc_val
+        elif isinstance(exc_val, BaseException):
             self.exit_code = ERROR_CODES.get(exc_type, -1)
             self.exc_val = exc_val
         if self.mode == ErrorMode.Ignore:
@@ -186,7 +191,6 @@ def __exit__(self, exc_type, exc_val, exc_tb):
 # Error code 3 is reserved for "we found negative cves" (should be impossible)
 # Error code 4-20 are reserved just in case
 ERROR_CODES = {
-    SystemExit: 2,
     FileNotFoundError: 21,
     InvalidCsvError: 22,
     InvalidJsonError: 22,

cve_bin_tool/output_engine/__init__.py

@@ -786,7 +786,7 @@ def output_file(self, output_type="console"):
         mode = "w"
         if output_type == "pdf":
             mode = "wb"
-        with open(self.filename, mode) as f:
+        with open(self.filename, mode, encoding="utf8") as f:
             self.output_cves(f, output_type)
 
     def check_file_path(self, filepath: str, output_type: str, prefix: str = "output"):

dev-requirements.txt

@@ -4,6 +4,7 @@ pre-commit==2.20.0
 flake8==5.0.4
 bandit==1.7.4
 gitlint== v0.17.0
+mypy==v0.991
 py>=1.10.0
 pytest
 pytest-xdist