Merge branch 'master' of https://github.com/oasis-open/cti-stix-valid…

…ator

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,72 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "master" ]
+  schedule:
+    - cron: '15 19 * * 6'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #   echo "Run, Build Application using script"
+    #   ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2

.github/workflows/python-ci-tests.yml

@@ -13,11 +13,11 @@ jobs:
 
     name: Python ${{ matrix.python-version }} Build
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         submodules: true
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install and update essential dependencies
@@ -29,7 +29,7 @@ jobs:
       run: |
         tox
     - name: Upload coverage information to Codecov
-      uses: codecov/codecov-action@v1
+      uses: codecov/codecov-action@v3
       with:
         fail_ci_if_error: true # optional (default = false)
         verbose: true # optional (default = false)

.pre-commit-config.yaml

@@ -12,7 +12,7 @@ repos:
         args:
         - --max-line-length=160
 -   repo: https://github.com/PyCQA/isort
-    rev: 5.7.0
+    rev: 5.11.5
     hooks:
     -   id: isort
         name: Sort python imports (shows diff)

CHANGELOG

@@ -1,6 +1,24 @@
 CHANGELOG
 =========
 
+3.1.2 - 2022-11-30
+
+* Minor fixes to extension definitions
+
+3.1.1 - 2022-11-22
+
+* Add schema in released package
+
+3.1.0 - 2022-11-22
+
+* Allow custom schemas to use extension definition ids so that users can use
+  multiple extensions for the same object type or extensions that extend
+  multiple object types.
+* Prevent KeyError in 2.0 bundles with no id
+* Interop validation
+* Improved error messages
+* Deprecated Python 3.6 support
+
 3.0.2 - 2021-12-17
 
 * Updated internal schemas to be more lenient with windows registry key format

CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 ## <a id="openParticipation">Public Participation Invited</a>
 
-This [OASIS TC Open Repository](https://www.oasis-open.org/resources/open-repositories) ( **[github.com/oasis-open/cti-stix-validator](https://github.com/oasis-open/cti-stix-validator)** ) is a community public repository that supports participation by anyone, whether affiliated with OASIS or not. Substantive contributions (repository "code") and related feedback is invited from all parties, following the common conventions for participation in GitHub public repository projects. Participation is expected to be consistent with the [OASIS TC Open Repository Guidelines and Procedures](https://www.oasis-open.org/policies-guidelines/open-repositories), the [LICENSE](https://www.oasis-open.org/sites/www.oasis-open.org/files/BSD-3-Clause.txt) designated for this particular repository (BSD-3-Clause License), and the requirement for an [Individual Contributor License Agreement](https://www.oasis-open.org/resources/open-repositories/cla/individual-cla). Please see the repository [README](https://github.com/oasis-open/cti-stix-validator/blob/master/README.md) document for other details.
+This [OASIS TC Open Repository](https://www.oasis-open.org/resources/open-repositories) ( **[github.com/oasis-open/cti-stix-validator](https://github.com/oasis-open/cti-stix-validator)** ) is a community public repository that supports participation by anyone, whether affiliated with OASIS or not. Substantive contributions (repository "code") and related feedback is invited from all parties, following the common conventions for participation in GitHub public repository projects. Participation is expected to be consistent with the [OASIS TC Open Repository Guidelines and Procedures](https://www.oasis-open.org/policies-guidelines/open-repositories), the [LICENSE](https://www.oasis-open.org/sites/www.oasis-open.org/files/BSD-3-Clause.txt) designated for this particular repository (BSD-3-Clause License), and the requirement for an [Individual Contributor License Agreement](https://cla-assistant.io/oasis-open/Open-Repo-admin). Please see the repository [README](https://github.com/oasis-open/cti-stix-validator/blob/master/README.md) document for other details.
 
 ## <a id="distinctRules">Governance Distinct from OASIS TC Process</a>
 
@@ -14,7 +14,7 @@ Because different licenses apply to the OASIS TC's specification work, and this
 
 ## <a id="contributionDefined">Contributions Subject to Individual CLA</a>
 
-Formally, <a id="openRepoContribution">"contribution"</a> to this TC Open Repository refers to content merged into the "Code" repository (repository changes represented by code [commits](https://github.com/oasis-open/cti-stix-validator/commits/master)), following the GitHub definition of _[contributor](https://help.github.com/articles/github-glossary/#contributor)_: "someone who has contributed to a project by having a pull request merged but does not have collaborator [_i.e._, direct write] access." Anyone who signs the TC Open Repository [Individual Contributor License Agreement (CLA)](https://www.oasis-open.org/resources/open-repositories/cla/individual-cla), signifying agreement with the licensing requirement, may contribute substantive content — subject to evaluation of a GitHub pull request. The main web page for this repository, as with any GitHub public repository, displays a link to a document listing contributions to the repository's default branch (filtered by Commits, Additions, and Deletions).
+Formally, <a id="openRepoContribution">"contribution"</a> to this TC Open Repository refers to content merged into the "Code" repository (repository changes represented by code [commits](https://github.com/oasis-open/cti-stix-validator/commits/master)), following the GitHub definition of _[contributor](https://help.github.com/articles/github-glossary/#contributor)_: "someone who has contributed to a project by having a pull request merged but does not have collaborator [_i.e._, direct write] access." Anyone who signs the TC Open Repository [Individual Contributor License Agreement (CLA)](https://cla-assistant.io/oasis-open/Open-Repo-admin), signifying agreement with the licensing requirement, may contribute substantive content — subject to evaluation of a GitHub pull request. The main web page for this repository, as with any GitHub public repository, displays a link to a document listing contributions to the repository's default branch (filtered by Commits, Additions, and Deletions).
 
 This TC Open Repository, as with GitHub public repositories generally, also accepts public feedback from any GitHub user. Public feedback includes opening issues, authoring and editing comments, participating in conversations, making wiki edits, creating repository stars, and making suggestions via pull requests. Such feedback does not constitute an OASIS TC Open Repository [contribution](#openRepoContribution). Some details are presented under "Read permissions" in the table of [permission levels](https://help.github.com/articles/repository-permission-levels-for-an-organization/) for a GitHub organization. Technical content intended as a substantive contribution (repository "Code") to an TC Open Repository is subject to evaluation, and requires a signed Individual CLA.
 

README.rst

@@ -12,7 +12,7 @@ requirements specified in the STIX 2.1 specification. In addition to
 checking conformance with the `JSON schemas <https://github.com/oasis-
 open/cti-stix2-json-schemas>`_, the validator checks conformance with
 requirements that cannot be specified in JSON schema, as well as with
-established "best practices." This validator is non-normative; in
+established "best practices". This validator is non-normative; in
 cases of conflict with the STIX 2.1 specification, the specification
 takes precedence.
 
@@ -141,9 +141,10 @@ repositories/maintainers-guide#additionalMaintainers>`_.
 
 **Current Maintainers of this TC Open Repository**
 
-.. Initial Maintainers: Greg Back & Ivan Kirillov
+.. Initial Maintainers: Greg Back & Ivan Kirillov & Chris Lenk
 
-*  `Chris Lenk <mailto:[email protected]>`_; GitHub ID: `https://github.com/clenk <https://github.com/clenk>`_; WWW: `MITRE <https://www.mitre.org>`__
+* `Emily Ratliff <mailto:[email protected]>`__; GitHub ID: `https://github.com/ejratl <https://github.com/ejratl>`_; WWW: `IBM <http://www.ibm.com/>`__
+* `Duncan Sparrell <mailto:[email protected]>`__; GitHub ID: `https://github.com/sparrell <https://github.com/sparrell>`_; WWW: `sFractal <http://sfractal.com/>`__
 *  `Jason Keirstead <mailto:[email protected]>`__; GitHub ID: `https://github.com/JasonKeirstead <https://github.com/JasonKeirstead>`_; WWW: `IBM <http://www.ibm.com/>`__
 
 .. _aboutOpenRepos:

docs/conf.py

@@ -7,11 +7,11 @@
 master_doc = 'index'
 
 project = 'stix2-validator'
-copyright = '2018, OASIS Open'
+copyright = '2018-2022, OASIS Open'
 author = 'OASIS Open'
 
-version = '3.0.2'
-release = '3.0.2'
+version = '3.1.3'
+release = '3.1.3'
 
 language = None
 exclude_patterns = ['_build', '_templates', 'Thumbs.db', '.DS_Store']

docs/release.md

@@ -0,0 +1,70 @@
+# Releasing a new version of stix2_validator
+
+These steps will upgrade the \'stix-validator\' package to version
+3.1.3. Substitute values as necessary.
+
+1.  Make sure that the README file is up-to-date and is consistent with any similar documentation (ReadTheDocs).
+
+2.  Make sure your git working directory is clean (no unstaged changes, or un-committed added/removed files), that the code is up-to-date, and that the code is passing all tests (run with nose, pytest, and/or tox, etc.).
+
+```bash
+git status
+git pull
+tox -r
+```
+
+3.  Update the CHANGELOG (sometimes CHANGES.txt) file with changes since the last release. To help, you can run one of the following:
+
+```bash
+git diff \<old tag\>..master
+git log \<old tag\>..master
+gitk \<old tag\>..master
+```
+
+4.  Run bumpversion patch, bumpversion minor, or bumpversion major (as appropriate).
+
+5.  Create a single commit with all of these changes. This helps improve clarity down the road. Push this commit to GitHub.
+
+```bash
+git commit -m \"Bump version to 3.1.3\"
+git push origin master
+```
+
+Ensure the new commit passes in CI. If the build on CI fails, make any
+changes necessary for it to pass, then commit and push them before
+continuing.
+
+6.  Once Github Actions shows a passing build for the master branch, create a tag for the new release. Push the tag to GitHub as well. (bumpversion may create the tag locally.)
+
+```bash
+git tag -a \"v3.1.3\" -m \"Version 3.1.3\"
+git push origin \--tags
+```
+
+7.  Copy the release notes into a new \"Release\" on GitHub. Don\'t forget to publish the \"Release\".
+
+8.  Make sure your release directory is \"clean\".
+
+```bash
+git clean -x -f -d
+```
+
+9.  Build and publish the new package. This method requires that you have a .pypirc file containing your PyPI credentials, and that your account has appropriate permissions on the PyPI project. See [the Python documentation](http://docs.python.org/2/distutils/packageindex.html#the-pypirc-file) for more information.
+
+```bash
+python3 setup.py sdist bdist_wheel
+twine upload dist/\*
+```
+
+If the filename does not end with -py2.py3-none-any.whl, it is not
+correctly configured as a \"Universal\" wheel.
+
+10. After the release is up on PyPI, create a temporary virtualenv and was downloaded and installed. If installation works fine, try running some samples or unit tests to make sure everything is working as expected.
+
+```bash
+virtualenv mktmpenv
+source mktmpenv/bin/activate
+pip install stix2_validator
+./mktmpenv/bin/stix2_validator cti-documentation/examples/example_json/poisonivy.json
+deactivate
+```

docs/usage.rst

@@ -85,7 +85,7 @@ Additional Schemas
 The validator uses the `STIX 2 JSON schemas <https://github.com/oasis-open/cti-stix2-
 json-schemas>`_ as the basis for its validation, but you can also validate with
 your own additional schemas. This can help if you want to validate STIX content
-using custom objects, properties, observables, or extensions.
+using extensions or (now deprecated) custom objects, properties, or observables.
 
 To do this use the ``--schemas`` argument:
 
@@ -106,10 +106,22 @@ or in Python, using ``schema_dir``:
 You can see some examples of custom schemas `here <https://github.com/oasis-open/cti-stix-validator/tree/master/stix2validator/test/v21/test_schemas>`_.
 
 .. note::
-  The schema's filename must match the type name of the STIX object type so the
-  validator can parse it correctly. For example, a schema defining a new
-  extension on Indicators should be named `indicator.json`. A schema defining a
-  new object type, "my-cool-thing", would need to be named `my-cool-thing.json`.
+  The schema's filename must match the extension definition id of the extension
+  it describes so the validator can apply it correctly. For example, a schema
+  defining a new extension with an id of
+  ``extension-definition--bfaece0b-efa6-4dfa-8248-3d340e2030f8`` should be named
+  `extension-definition--bfaece0b-efa6-4dfa-8248-3d340e2030f8.json`.
 
 .. note::
-  If you want to add a custom property to an existing object type, your custom schema only needs to contain that property; the validator's built-in schemas are still checked against and will handle the rest.
+  Custom objects and properties using the ``x_`` and ``x-`` prefixes have been
+  deprecated in STIX 2.1. However, if you need a schema for validating them, the
+  validator can parse it as long as the schema's filename matches the type name
+  of the STIX object type it should apply to. For example, a schema defining a
+  new property on Indicators should be named ``indicator.json``. A schema
+  defining a new object type, “my-cool-thing”, would need to be named
+  ``my-cool-thing.json``.
+
+.. note::
+  When using additional schemas, the validator's built-in schemas are still
+  checked against. Thus custom schemas only need to contain the properties that
+  differ from the standard.

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.0.2
+current_version = 3.1.3
 commit = True
 tag = True
 

stix2validator/errors.py

@@ -130,6 +130,8 @@ def pretty_error(error, verbose=False):
     elif error.validator == 'pattern' and 'where_sighted_refs' in error.schema_path:
         msg = "'where_sighted_refs' must refer to Identity Objects"
 
+    elif error.validator == 'maxContains' and 'marking-definition' in error.schema['description']:
+        msg = "More than one tlp marking found in marking-definitions. Only one tlp marking is allowed."
     # Reword empty array errors
     elif type(error.instance) is list and len(error.instance) == 0:
         msg = re.sub(r"\[\] is not valid .+$", 'empty arrays are not allowed',
@@ -201,9 +203,14 @@ def pretty_error(error, verbose=False):
                     msg = "'definition' must contain a valid statement, TLP, or "\
                           "custom marking definition"
                 else:
-                    msg = "marking definitions must use a valid extension, or "\
-                          "the 'definition_type' and 'definition' properties "\
-                          "with a valid marking definition"
+                    msg = "marking definitions must use the 'definition_type' "\
+                          "and 'definition' properties with a valid marking "\
+                          "definition, or use an extension and the properties "\
+                          "it requires"
+                    if verbose:
+                        msg += ".\nComplete error: {}".format(remove_u(error.message))
+                    else:
+                        msg += " (for more details use --verbose)"
             elif 'type' in error.instance and error.instance['type'] == 'file':
                 if (('is_encrypted' not in error.instance or
                         error.instance['is_encrypted'] is False) and

stix2validator/output.py

@@ -56,7 +56,7 @@ def info(msg):
     if not _VERBOSE:
         return
 
-    logger.debug("[-] %s" % msg)
+    logger.info("[-] %s" % msg)
 
 
 def print_level(log_function, fmt, level, *args):

stix2validator/test/v20/bundle_tests.py

@@ -3,6 +3,8 @@
 
 import pytest
 
+from stix2validator import validate_instance
+
 from . import ValidatorTest
 from ... import ValidationError
 
@@ -66,3 +68,27 @@ def test_bundle_sdo_missing_type(self):
         del bundle['objects'][0]['type']
         with pytest.raises(ValidationError):
             self.assertFalseWithOptions(bundle)
+
+    def test_bundle_sdo_missing_id(self):
+        bundle = copy.deepcopy(self.valid_bundle)
+        del bundle['objects'][0]['id']
+        self.assertFalseWithOptions(bundle)
+
+    def test_bundle_missing_id(self):
+        bundle = copy.deepcopy(self.valid_bundle)
+        del bundle['id']
+        with pytest.raises(ValidationError):
+            self.assertFalseWithOptions(bundle)
+
+    def test_bundle_sdo_missing_id_with_no_options(self):
+        bundle = copy.deepcopy(self.valid_bundle)
+        del bundle['objects'][0]['id']
+
+        results = validate_instance(bundle)
+        assert not results.is_valid
+
+    def test_bundle_missing_id_with_no_options(self):
+        bundle = copy.deepcopy(self.valid_bundle)
+        del bundle['id']
+        with pytest.raises(ValidationError):
+            validate_instance(bundle)