Merge branch 'master' of github.com:oasis-open/cti-stix-validator

.github/workflows/python-ci-tests.yml

@@ -6,10 +6,20 @@ on: [push, pull_request]
 jobs:
   build:
 
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: [3.8, 3.9, '3.10', '3.11', '3.12']
+        os: [ubuntu-latest, windows-latest]
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        exclude:
+          - os: windows-latest
+            python-version: '3.8'
+          - os: windows-latest
+            python-version: '3.9'
+          - os: windows-latest
+            python-version: '3.10'
+          - os: windows-latest
+            python-version: '3.11'
 
     name: Python ${{ matrix.python-version }} Build
     steps:

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
         exclude: ^stix2validator/(v20|v21)/assets/.*.csv$
     -   id: check-merge-conflict
 -   repo: https://github.com/PyCQA/flake8
-    rev: 3.8.4
+    rev: 5.0.4
     hooks:
     -   id: flake8
         name: Check project styling

.readthedocs.yaml

@@ -0,0 +1,24 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Build all formats (incl. pdf, epub)
+formats: all
+
+# Declare the Python requirements required to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: requirements.txt

CHANGELOG

@@ -1,6 +1,22 @@
 CHANGELOG
 =========
 
+3.2.0 - 2024-04-05
+
+* Updated jsonschema dependency version, removed deprecated refResolver
+  (@ostefano)
+* Enforce requirement that network-traffic.http-request-ext.request_header is a
+  list of strings instead of a singular string
+* Allow -ext extensions in SDOs
+* Fixed bug when loading files containing unicode characters on certain
+  platforms
+* Fixed bug with duplicate log entries when importing the validator script as a
+  library (@ostefano)
+* Switched to including IETF data as package data instead of pulling and
+  caching locally, removed caching options, and removed requests-cache and
+  appdirs dependencies (@vEpiphyte)
+* Dropped support for Python 3.7
+
 3.1.4 - 2023-07-24
 
 * Allow latest attrs package to fix community reported dependency problems

docs/conf.py

@@ -10,8 +10,8 @@
 copyright = '2018-2022, OASIS Open'
 author = 'OASIS Open'
 
-version = '3.1.4'
-release = '3.1.4'
+version = '3.2.0'
+release = '3.2.0'
 
 language = None
 exclude_patterns = ['_build', '_templates', 'Thumbs.db', '.DS_Store']

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.1.4
+current_version = 3.2.0
 commit = True
 tag = True
 

setup.py

@@ -19,8 +19,9 @@ def get_version():
 install_requires = [
     'colorama',
     'cpe',
-    'jsonschema[format-nongpl]>=4.6.0,<4.18.0',
+    'jsonschema[format-nongpl]>=4.20.0',
     'python-dateutil',
+    'requests',
     'simplejson',
     'stix2-patterns>=0.4.1',
 ]

stix2validator/scripts/stix2_validator.py

@@ -10,17 +10,18 @@
 from stix2validator import (ValidationError, codes, output, parse_args,
                             print_results, run_validation)
 
-logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s')
-logger = logging.getLogger(__name__)
-
 
 def main():
     # Parse command line arguments
     options = parse_args(sys.argv[1:], is_script=True)
 
+    # Initialize the logger
+    logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s')
+    logger = logging.getLogger(__name__)
+
     # Only print prompt if script is run on cmdline and no input is piped in
     if options.files == sys.stdin and os.isatty(0):
-        logging.info('Input STIX content, then press Ctrl+D: ')
+        logger.info('Input STIX content, then press Ctrl+D: ')
 
     try:
         # Validate input documents

stix2validator/test/v20/misc_tests.py

@@ -41,7 +41,7 @@ def test_run_validation(caplog):
 
 
 def test_run_validation_nonexistent_file():
-    options = ValidationOptions(files='asdf.json', version="2.0")
+    options = ValidationOptions(files=['asdf.json'], version="2.0")
     with pytest.raises(NoJSONFileFoundError):
         run_validation(options)
 

stix2validator/test/v21/misc_tests.py

@@ -16,10 +16,20 @@
 EXAMPLE = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                        '..', '..', 'schemas-2.1', 'examples',
                        'indicator-to-campaign-relationship.json')
+CUSTOM = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                      'test_examples', 'tlp-amber.json')
+CUSTOM_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                          'test_schemas')
+RELATIVE = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                        'test_examples', 'tool.json')
+RELATIVE_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                            'test_schemas')
 IDENTITY = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                         'test_examples', 'identity.json')
 IDENTITY_CUSTOM = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                'test_examples', 'identity_custom.json')
+IDENTITY_UNICODE = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                                'test_examples', 'identity_unicode.json')
 INVALID_BRACES = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                               'test_examples', 'invalid_braces.json')
 INVALID_COMMA = os.path.join(os.path.dirname(os.path.realpath(__file__)),
@@ -63,6 +73,24 @@ def test_validate_file(caplog):
     assert 'STIX JSON: Valid' in caplog.text
 
 
+def test_validate_file_custom(caplog):
+    caplog.set_level('INFO')
+    results = validate_file(CUSTOM, options=ValidationOptions(schema_dir=CUSTOM_DIR))
+    assert results.is_valid
+
+    print_results(results)
+    assert 'STIX JSON: Valid' in caplog.text
+
+
+def test_validate_file_custom_relative(caplog):
+    caplog.set_level('INFO')
+    results = validate_file(RELATIVE, options=ValidationOptions(schema_dir=RELATIVE_DIR))
+    assert results.is_valid
+
+    print_results(results)
+    assert 'STIX JSON: Valid' in caplog.text
+
+
 def test_validate_file_warning(caplog):
     results = validate_file(IDENTITY_CUSTOM)
     assert results.is_valid
@@ -71,6 +99,11 @@ def test_validate_file_warning(caplog):
     assert re.search("Custom property .+ should ", caplog.text)
 
 
+def test_validate_file_unicode(caplog):
+    results = validate_file(IDENTITY_UNICODE)
+    assert results.is_valid
+
+
 def test_validate_file_invalid_brace(caplog):
     results = validate_file(INVALID_BRACES)
     assert not results.is_valid

stix2validator/test/v21/network_traffic_tests.py

@@ -82,9 +82,9 @@ def test_network_traffic_http_request_header(self):
                 "request_value": "/download.html",
                 "request_version": "http/1.1",
                 "request_header": {
-                    "Accept-Encoding": "gzip,deflate",
-                    "Host": "www.example.com",
-                    "x-foobar": "something"
+                    "Accept-Encoding": ["gzip,deflate"],
+                    "Host": ["www.example.com"],
+                    "x-foobar": ["something"]
                 }
             }
         }

stix2validator/test/v21/test_examples/identity_unicode.json

@@ -0,0 +1,9 @@
+{
+  "type": "identity",
+  "spec_version": "2.1",
+  "id": "identity--8c6af861-7b20-41ef-9b59-6344fd872a8f",
+  "created": "2016-08-08T15:50:10.983Z",
+  "modified": "2016-08-08T15:50:10.983Z",
+  "name": "Heizölrückstoßabdämpfung",
+  "identity_class": "organization"
+}

stix2validator/test/v21/test_examples/tlp-amber.json

@@ -0,0 +1,19 @@
+{
+    "type": "bundle",
+    "id": "bundle--63ab8e67-acac-4817-845a-d09f0e86954c",
+    "objects": [
+        {
+            "type": "marking-definition",
+            "spec_version": "2.1",
+            "id": "marking-definition--55d920b0-5e8b-4f79-9ee9-91f868d9b421",
+            "created": "2022-10-01T00:00:00.000Z",
+            "name": "TLP:AMBER",
+            "extensions": {
+                "extension-definition--60a3c5c5-0d10-413e-aab3-9e08dde9e88d": {
+                    "extension_type": "property-extension",
+                    "tlp_2_0" : "amber"
+                }
+            }
+        }
+    ]
+}

stix2validator/test/v21/test_examples/tool.json

@@ -0,0 +1,12 @@
+{
+  "type": "tool",
+  "spec_version": "2.1",
+  "id": "tool--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f",
+  "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff",
+  "created": "2016-04-06T20:03:48.000Z",
+  "modified": "2016-04-06T20:03:48.000Z",
+  "tool_types": [ "remote-access"],
+  "name": "VNC",
+  "foo_value": "bizz",
+  "bar_value": "buzz"
+}

stix2validator/test/v21/test_schemas/bar.json

@@ -0,0 +1,7 @@
+{
+  "properties": {
+    "bar_value": {
+      "type": "string"
+    }
+  }
+}