Add Support for MongoDB 5, 6, 7 PYMONGO 4.6.1 Drop Python 3.6

.circleci/config.yml

@@ -43,7 +43,7 @@ jobs:
   # Run st2 Integration tests
   integration:
     docker:
-      - image: circleci/python:3.6
+      - image: circleci/python:3.8
       - image: mongo:4.0
       - image: rabbitmq:3
     working_directory: ~/st2
@@ -79,7 +79,7 @@ jobs:
   # Run st2 Lint Checks
   lint:
     docker:
-      - image: circleci/python:3.6
+      - image: circleci/python:3.8
       - image: mongo:4.0
       - image: rabbitmq:3
     working_directory: ~/st2
@@ -234,7 +234,7 @@ jobs:
   deploy:
     docker:
       # The primary container is an instance of the first list image listed. Your build commands run in this container.
-      - image: circleci/ruby:2.7
+      - image: circleci/ruby:3.2.2
     working_directory: /tmp/deploy
     environment:
       - DISTROS: "focal el8 el9"

CHANGELOG.rst

@@ -104,6 +104,7 @@ Changed
 * Remove `distutils` dependencies across the project. #5992
   Contributed by @AndroxxTraxxon
 
+
 3.8.0 - November 18, 2022
 -------------------------
 

conf/st2.conf.sample

@@ -153,8 +153,6 @@ ssl_cert_reqs = None
 ssl_certfile = None
 # Private keyfile used to identify the local connection against MongoDB.
 ssl_keyfile = None
-# If True and `ssl_cert_reqs` is not None, enables hostname verification
-ssl_match_hostname = True
 # username for db login
 username = None
 # Compression level when compressors is set to zlib. Valid values are -1 to 9. Defaults to 6.

contrib/packs/actions/pack_mgmt/unload.py

@@ -65,7 +65,6 @@ def initialize(self):
             ssl_cert_reqs=cfg.CONF.database.ssl_cert_reqs,
             ssl_ca_certs=cfg.CONF.database.ssl_ca_certs,
             authentication_mechanism=cfg.CONF.database.authentication_mechanism,
-            ssl_match_hostname=cfg.CONF.database.ssl_match_hostname,
         )
 
     def run(self, packs):

fixed-requirements.txt

@@ -27,7 +27,7 @@ lockfile==0.12.2
 # Fix MarkupSafe to < 2.1.0 as 2.1.0 removes soft_unicode
 # >=0.23 was from jinja2
 MarkupSafe<2.1.0,>=0.23
-mongoengine==0.23.0
+mongoengine==0.27.0
 # required by orquesta (networkx<2.6 for py3.6, networkx<3 for py3.8)
 networkx<3
 # networkx requires decorator>=4.3,<5 which should resolve to version 4.4.2
@@ -43,7 +43,7 @@ paramiko==2.11.0
 passlib==1.7.4
 prompt-toolkit==1.0.15
 pyinotify==0.9.6 ; platform_system=="Linux"
-pymongo==3.11.3
+pymongo==4.6.1
 pyparsing<3
 zstandard==0.15.2
 # pyOpenSSL 23.1.0 supports cryptography up to 40.0.x

requirements.txt

@@ -31,7 +31,7 @@ kombu==5.0.2
 lockfile==0.12.2
 logshipper@ git+https://github.com/StackStorm/logshipper.git@stackstorm_patched ; platform_system=="Linux"
 mock==4.0.3
-mongoengine==0.23.0
+mongoengine==0.27.0
 networkx<3
 nose
 nose-parallel==0.4.0
@@ -47,7 +47,7 @@ prompt-toolkit==1.0.15
 psutil==5.8.0
 pyOpenSSL==23.1.0
 pyinotify==0.9.6 ; platform_system=="Linux"
-pymongo==3.11.3
+pymongo==4.6.1
 pyparsing<3
 pyrabbit
 pysocks

st2api/requirements.txt

@@ -9,10 +9,10 @@ eventlet==0.33.3
 gunicorn==21.2.0
 jsonschema==3.2.0
 kombu==5.0.2
-mongoengine==0.23.0
+mongoengine==0.27.0
 oslo.config>=1.12.1,<1.13
 oslo.utils<5.0,>=4.0.0
-pymongo==3.11.3
+pymongo==4.6.1
 pyparsing<3
 simplejson
 six==1.13.0

st2auth/requirements.txt

@@ -10,7 +10,7 @@ eventlet==0.33.3
 gunicorn==21.2.0
 oslo.config>=1.12.1,<1.13
 passlib==1.7.4
-pymongo==3.11.3
+pymongo==4.6.1
 six==1.13.0
 st2-auth-backend-flat-file@ git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master
 st2-auth-ldap@ git+https://github.com/StackStorm/st2-auth-ldap.git@master

st2common/requirements.txt

@@ -24,14 +24,14 @@ jsonpath-rw==1.4.0
 jsonschema==3.2.0
 kombu==5.0.2
 lockfile==0.12.2
-mongoengine==0.23.0
+mongoengine==0.27.0
 networkx<3
 orjson==3.5.2
 orquesta@ git+https://github.com/StackStorm/[email protected]
 oslo.config>=1.12.1,<1.13
 paramiko==2.11.0
 pyOpenSSL==23.1.0
-pymongo==3.11.3
+pymongo==4.6.1
 python-dateutil==2.8.1
 python-statsd==2.1.0
 pyyaml==5.4.1

st2common/st2common/config.py

@@ -231,11 +231,6 @@ def register_opts(ignore_errors=False):
             help="ca_certs file contains a set of concatenated CA certificates, which are "
             "used to validate certificates passed from MongoDB.",
         ),
-        cfg.BoolOpt(
-            "ssl_match_hostname",
-            default=True,
-            help="If True and `ssl_cert_reqs` is not None, enables hostname verification",
-        ),
         cfg.StrOpt(
             "authentication_mechanism",
             default=None,

st2common/st2common/database_setup.py

@@ -42,7 +42,6 @@ def db_config():
         "ssl_cert_reqs": cfg.CONF.database.ssl_cert_reqs,
         "ssl_ca_certs": cfg.CONF.database.ssl_ca_certs,
         "authentication_mechanism": cfg.CONF.database.authentication_mechanism,
-        "ssl_match_hostname": cfg.CONF.database.ssl_match_hostname,
     }
 
 

st2common/st2common/models/db/__init__.py

@@ -133,7 +133,6 @@ def _db_connect(
     ssl_cert_reqs=None,
     ssl_ca_certs=None,
     authentication_mechanism=None,
-    ssl_match_hostname=True,
 ):
 
     if "://" in db_host:
@@ -168,7 +167,6 @@ def _db_connect(
         ssl_cert_reqs=ssl_cert_reqs,
         ssl_ca_certs=ssl_ca_certs,
         authentication_mechanism=authentication_mechanism,
-        ssl_match_hostname=ssl_match_hostname,
     )
 
     compressor_kwargs = {}
@@ -237,7 +235,6 @@ def db_setup(
     ssl_cert_reqs=None,
     ssl_ca_certs=None,
     authentication_mechanism=None,
-    ssl_match_hostname=True,
 ):
 
     connection = _db_connect(
@@ -252,7 +249,6 @@ def db_setup(
         ssl_cert_reqs=ssl_cert_reqs,
         ssl_ca_certs=ssl_ca_certs,
         authentication_mechanism=authentication_mechanism,
-        ssl_match_hostname=ssl_match_hostname,
     )
 
     # Create all the indexes upfront to prevent race-conditions caused by
@@ -403,7 +399,6 @@ def db_cleanup(
     ssl_cert_reqs=None,
     ssl_ca_certs=None,
     authentication_mechanism=None,
-    ssl_match_hostname=True,
 ):
 
     connection = _db_connect(
@@ -418,7 +413,6 @@ def db_cleanup(
         ssl_cert_reqs=ssl_cert_reqs,
         ssl_ca_certs=ssl_ca_certs,
         authentication_mechanism=authentication_mechanism,
-        ssl_match_hostname=ssl_match_hostname,
     )
 
     LOG.info(
@@ -440,7 +434,6 @@ def _get_ssl_kwargs(
     ssl_cert_reqs=None,
     ssl_ca_certs=None,
     authentication_mechanism=None,
-    ssl_match_hostname=True,
 ):
     # NOTE: In pymongo 3.9.0 some of the ssl related arguments have been renamed -
     # https://api.mongodb.com/python/current/changelog.html#changes-in-version-3-9-0
@@ -468,10 +461,6 @@ def _get_ssl_kwargs(
     if authentication_mechanism:
         ssl_kwargs["ssl"] = True
         ssl_kwargs["authentication_mechanism"] = authentication_mechanism
-    if ssl_kwargs.get("ssl", False):
-        # pass in ssl_match_hostname only if ssl is True. The right default value
-        # for ssl_match_hostname in almost all cases is True.
-        ssl_kwargs["ssl_match_hostname"] = ssl_match_hostname
     return ssl_kwargs
 
 

st2common/st2common/persistence/cleanup.py

@@ -50,7 +50,6 @@ def db_cleanup_with_retry(
     ssl_cert_reqs=None,
     ssl_ca_certs=None,
     authentication_mechanism=None,
-    ssl_match_hostname=True,
 ):
     """
     This method is a retry version of db_cleanup.
@@ -68,7 +67,6 @@ def db_cleanup_with_retry(
         ssl_cert_reqs=ssl_cert_reqs,
         ssl_ca_certs=ssl_ca_certs,
         authentication_mechanism=authentication_mechanism,
-        ssl_match_hostname=ssl_match_hostname,
     )
 
 

st2common/st2common/persistence/db_init.py

@@ -71,7 +71,6 @@ def db_setup_with_retry(
     ssl_cert_reqs=None,
     ssl_ca_certs=None,
     authentication_mechanism=None,
-    ssl_match_hostname=True,
 ):
     """
     This method is a retry version of db_setup.
@@ -90,5 +89,4 @@ def db_setup_with_retry(
         ssl_cert_reqs=ssl_cert_reqs,
         ssl_ca_certs=ssl_ca_certs,
         authentication_mechanism=authentication_mechanism,
-        ssl_match_hostname=ssl_match_hostname,
     )

st2common/tests/unit/test_db.py

@@ -21,7 +21,6 @@
 
 monkey_patch()
 
-import ssl
 import time
 
 import jsonschema
@@ -231,80 +230,16 @@ def test_get_ssl_kwargs(self):
         ssl_kwargs = _get_ssl_kwargs()
         self.assertEqual(ssl_kwargs, {"ssl": False})
 
-        # 2. ssl kwarg provided
-        ssl_kwargs = _get_ssl_kwargs(ssl=True)
-        self.assertEqual(ssl_kwargs, {"ssl": True, "ssl_match_hostname": True})
-
         # 2. authentication_mechanism kwarg provided
         ssl_kwargs = _get_ssl_kwargs(authentication_mechanism="MONGODB-X509")
         self.assertEqual(
             ssl_kwargs,
             {
                 "ssl": True,
-                "ssl_match_hostname": True,
                 "authentication_mechanism": "MONGODB-X509",
             },
         )
 
-        # 3. ssl_keyfile provided
-        ssl_kwargs = _get_ssl_kwargs(ssl_keyfile="/tmp/keyfile")
-        self.assertEqual(
-            ssl_kwargs,
-            {"ssl": True, "ssl_keyfile": "/tmp/keyfile", "ssl_match_hostname": True},
-        )
-
-        # 4. ssl_certfile provided
-        ssl_kwargs = _get_ssl_kwargs(ssl_certfile="/tmp/certfile")
-        self.assertEqual(
-            ssl_kwargs,
-            {"ssl": True, "ssl_certfile": "/tmp/certfile", "ssl_match_hostname": True},
-        )
-
-        # 5. ssl_ca_certs provided
-        ssl_kwargs = _get_ssl_kwargs(ssl_ca_certs="/tmp/ca_certs")
-        self.assertEqual(
-            ssl_kwargs,
-            {"ssl": True, "ssl_ca_certs": "/tmp/ca_certs", "ssl_match_hostname": True},
-        )
-
-        # 6. ssl_ca_certs and ssl_cert_reqs combinations
-        ssl_kwargs = _get_ssl_kwargs(ssl_ca_certs="/tmp/ca_certs", ssl_cert_reqs="none")
-        self.assertEqual(
-            ssl_kwargs,
-            {
-                "ssl": True,
-                "ssl_ca_certs": "/tmp/ca_certs",
-                "ssl_cert_reqs": ssl.CERT_NONE,
-                "ssl_match_hostname": True,
-            },
-        )
-
-        ssl_kwargs = _get_ssl_kwargs(
-            ssl_ca_certs="/tmp/ca_certs", ssl_cert_reqs="optional"
-        )
-        self.assertEqual(
-            ssl_kwargs,
-            {
-                "ssl": True,
-                "ssl_ca_certs": "/tmp/ca_certs",
-                "ssl_cert_reqs": ssl.CERT_OPTIONAL,
-                "ssl_match_hostname": True,
-            },
-        )
-
-        ssl_kwargs = _get_ssl_kwargs(
-            ssl_ca_certs="/tmp/ca_certs", ssl_cert_reqs="required"
-        )
-        self.assertEqual(
-            ssl_kwargs,
-            {
-                "ssl": True,
-                "ssl_ca_certs": "/tmp/ca_certs",
-                "ssl_cert_reqs": ssl.CERT_REQUIRED,
-                "ssl_match_hostname": True,
-            },
-        )
-
     @mock.patch("st2common.models.db.mongoengine")
     def test_db_setup(self, mock_mongoengine):
         db_setup(
@@ -331,7 +266,6 @@ def test_db_setup(self, mock_mongoengine):
                 "tz_aware": True,
                 "authentication_mechanism": "MONGODB-X509",
                 "ssl": True,
-                "ssl_match_hostname": True,
                 "connectTimeoutMS": 3000,
                 "serverSelectionTimeoutMS": 3000,
             },
@@ -571,7 +505,7 @@ def test_cleanup(self):
         """
         Tests dropping the database. Requires the db server to be running.
         """
-        self.assertIn(cfg.CONF.database.db_name, self.db_connection.database_names())
+        self.assertIn(cfg.CONF.database.db_name, self.db_connection.list_database_names())
 
         connection = db_cleanup()
 

st2reactor/st2reactor/container/sensor_wrapper.py

@@ -235,7 +235,6 @@ def __init__(
             ssl_cert_reqs=cfg.CONF.database.ssl_cert_reqs,
             ssl_ca_certs=cfg.CONF.database.ssl_ca_certs,
             authentication_mechanism=cfg.CONF.database.authentication_mechanism,
-            ssl_match_hostname=cfg.CONF.database.ssl_match_hostname,
         )
 
         # 3. Instantiate the watcher

st2stream/requirements.txt

@@ -9,9 +9,9 @@ eventlet==0.33.3
 gunicorn==21.2.0
 jsonschema==3.2.0
 kombu==5.0.2
-mongoengine==0.23.0
+mongoengine==0.27.0
 oslo.config>=1.12.1,<1.13
 oslo.utils<5.0,>=4.0.0
-pymongo==3.11.3
+pymongo==4.6.1
 pyparsing<3
 six==1.13.0

st2tests/st2tests/base.py

@@ -273,7 +273,6 @@ def _drop_db(cls):
             cls.db_connection.drop_database(cfg.CONF.database.db_name)
 
         db_teardown()
-        cls.db_connection = None
 
     @classmethod
     def _drop_collections(cls):