[WIP][From @nzlosh] Interoperability updates for py3.8, py3.9 and py3.10

.github/workflows/test.yaml

@@ -38,6 +38,9 @@ jobs:
           - name: 'Test (pants runs: pytest)'
             python-version-short: '3.9'
             python-version: '3.9.14'
+          - name: 'Test (pants runs: pytest)'
+            python-version-short: '3.10'
+            python-version: '3.10.12'
 
     services:
       mongo:

CHANGELOG.rst

@@ -24,6 +24,9 @@ Changed
 * Update st2client deps: editor and prompt-toolkit. #6189 (by @nzlosh)
 * Updated dependency oslo.config to prepare for python 3.10 support. #6193 (by @nzlosh)
 
+* Support for Python3.8, Python3.9 and Python3.10.  Update underlying modules and
+  remove modules that weren't compatible with newer versions of Python.  #6157
+
 Added
 ~~~~~
 * Continue introducing `pants <https://www.pantsbuild.org/docs>`_ to improve DX (Developer Experience)

Makefile

@@ -53,8 +53,13 @@ COVERAGE_GLOBS_QUOTED := $(foreach glob,$(COVERAGE_GLOBS),'$(glob)')
 
 REQUIREMENTS := test-requirements.txt requirements.txt
 
+# Redis config for testing
+ST2_OVERRIDE_COORDINATOR_REDIS_HOST := ${REDIS_HOST:-"127.0.0.1"}
+ST2_OVERRIDE_COORDINATOR_REDIS_PORT := ${REDIS_PORT:-"6379"} 
+
 # Pin common pip version here across all the targets
 # Note! Periodic maintenance pip upgrades are required to be up-to-date with the latest pip security fixes and updates
+# 202403: Use pip 24.0 which is the highest supported version across Python 3.8 - 3.10
 PIP_VERSION ?= 24.0
 SETUPTOOLS_VERSION ?= 69.5.1
 PIP_OPTIONS := $(ST2_PIP_OPTIONS)
@@ -73,6 +78,9 @@ endif
 # The minus in front of st2.st2common.bootstrap filters out logging statements from that module.
 # See https://nose.readthedocs.io/en/latest/usage.html#cmdoption-logging-filter
 NOSE_OPTS := --rednose --immediate --with-parallel --parallel-strategy=FILE --nocapture --logging-filter=-st2.st2common.bootstrap
+# https://github.com/pytest-dev/pytest-xdist/issues/71
+#PYTEST_OPTS := -n auto --tx 2*popen//execmodel=eventlet
+PYTEST_OPTS := ""
 
 ifndef NOSE_TIME
 	NOSE_TIME := yes
@@ -486,7 +494,7 @@ flake8: requirements .flake8
 	. $(VIRTUALENV_DIR)/bin/activate; flake8 --config ./lint-configs/python/.flake8 pylint_plugins/
 
 # Make task which verifies st2client README will parse pypi checks
-. PHONY: .st2client-pypi-check
+.PHONY: .st2client-pypi-check
 .st2client-pypi-check:
 	@echo
 	@echo "==================== st2client pypi check ===================="
@@ -824,8 +832,7 @@ unit-tests: requirements .unit-tests
 		echo "Running tests in" $$component; \
 		echo "-----------------------------------------------------------"; \
 		. $(VIRTUALENV_DIR)/bin/activate; \
-		    nosetests $(NOSE_OPTS) -s -v \
-		    $$component/tests/unit || ((failed+=1)); \
+		    pytest -rx --verbose $$component/tests/unit || exit 1; \
 		echo "-----------------------------------------------------------"; \
 		echo "Done running tests in" $$component; \
 		echo "==========================================================="; \
@@ -849,9 +856,8 @@ endif
 		echo "-----------------------------------------------------------"; \
 		. $(VIRTUALENV_DIR)/bin/activate; \
 		    COVERAGE_FILE=.coverage.unit.$$(echo $$component | tr '/' '.') \
-		    nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) \
-		    $(NOSE_COVERAGE_PACKAGES) \
-		    $$component/tests/unit || ((failed+=1)); \
+		    pytest --capture=no --verbose $(PYTEST_OPTS) --cov=$$component --cov-branch \
+		    $$component/tests/unit || exit 1; \
 		echo "-----------------------------------------------------------"; \
 		echo "Done running tests in" $$component; \
 		echo "==========================================================="; \
@@ -907,8 +913,8 @@ itests: requirements .itests
 		echo "Running integration tests in" $$component; \
 		echo "-----------------------------------------------------------"; \
 		. $(VIRTUALENV_DIR)/bin/activate; \
-		    nosetests $(NOSE_OPTS) -s -v \
-		    $$component/tests/integration || ((failed+=1)); \
+		    pytest --capture=no --verbose $(PYTEST_OPTS) \
+		    $$component/tests/integration || exit 1; \
 		echo "-----------------------------------------------------------"; \
 		echo "Done running integration tests in" $$component; \
 		echo "==========================================================="; \
@@ -932,9 +938,8 @@ endif
 		echo "-----------------------------------------------------------"; \
 		. $(VIRTUALENV_DIR)/bin/activate; \
 		    COVERAGE_FILE=.coverage.integration.$$(echo $$component | tr '/' '.') \
-		    nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) \
-		    $(NOSE_COVERAGE_PACKAGES) \
-		    $$component/tests/integration || ((failed+=1)); \
+		    pytest --capture=no --verbose $(PYTEST_OPTS) --cov=$$component --cov-branch \
+		    $$component/tests/integration || exit 1; \
 		echo "-----------------------------------------------------------"; \
 		echo "Done integration running tests in" $$component; \
 		echo "==========================================================="; \
@@ -1032,16 +1037,15 @@ orquesta-itests: requirements .orquesta-itests
 	@echo "==================== Orquesta integration tests ===================="
 	@echo "The tests assume st2 is running on 127.0.0.1."
 	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; nosetests $(NOSE_OPTS) -s -v st2tests/integration/orquesta || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; pytest --capture=no --verbose $(PYTEST_OPTS) st2tests/integration/orquesta || exit 1;
 
 .PHONY: .orquesta-itests-coverage-html
 .orquesta-itests-coverage-html:
 	@echo
 	@echo "==================== Orquesta integration tests with coverage (HTML reports) ===================="
 	@echo "The tests assume st2 is running on 127.0.0.1."
 	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; nosetests $(NOSE_OPTS) -s -v --with-coverage \
-        --cover-inclusive --cover-html st2tests/integration/orquesta || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; pytest --capture=no --verbose $(PYTEST_OPTS) --cov=orquesta --cov-branch  st2tests/integration/orquesta || exit 1;
 
 .PHONY: packs-tests
 packs-tests: requirements .packs-tests
@@ -1071,7 +1075,7 @@ runners-tests: requirements .runners-tests
 		echo "==========================================================="; \
 		echo "Running tests in" $$component; \
 		echo "==========================================================="; \
-		. $(VIRTUALENV_DIR)/bin/activate; nosetests $(NOSE_OPTS) -s -v $$component/tests/unit || ((failed+=1)); \
+		. $(VIRTUALENV_DIR)/bin/activate; pytest --capture=no --verbose $(PYTEST_OPTS) $$component/tests/unit || exit 1; \
 	done; \
 	if [ $$failed -gt 0 ]; then exit 1; fi
 
@@ -1089,7 +1093,7 @@ runners-itests: requirements .runners-itests
 		echo "==========================================================="; \
 		echo "Running integration tests in" $$component; \
 		echo "==========================================================="; \
-		. $(VIRTUALENV_DIR)/bin/activate; nosetests $(NOSE_OPTS) -s -v $$component/tests/integration || ((failed+=1)); \
+		. $(VIRTUALENV_DIR)/bin/activate; pytest --capture=no --verbose $(PYTEST_OPTS) $$component/tests/integration || exit 1; \
 	done; \
 	if [ $$failed -gt 0 ]; then exit 1; fi
 
@@ -1104,8 +1108,8 @@ runners-itests: requirements .runners-itests
 		echo "==========================================================="; \
 		echo "Running integration tests in" $$component; \
 		echo "==========================================================="; \
-		. $(VIRTUALENV_DIR)/bin/activate; nosetests $(NOSE_OPTS) -s -v --with-coverage \
-			--cover-inclusive --cover-html $$component/tests/integration || ((failed+=1)); \
+		. $(VIRTUALENV_DIR)/bin/activate; pytest --capture=no --verbose $(PYTEST_OPTS) --cov=$$component --cov-report=html \
+			$$component/tests/integration || exit 1; \
 	done; \
 	if [ $$failed -gt 0 ]; then exit 1; fi
 

contrib/examples/actions/ubuntu_pkg_info/ubuntu_pkg_info.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Copyright 2020 The StackStorm Authors.
+# Copyright 2020-2024 The StackStorm Authors.
 # Copyright 2019 Extreme Networks, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

contrib/packs/actions/pack_mgmt/get_pack_dependencies.py

@@ -1,4 +1,4 @@
-# Copyright 2020 The StackStorm Authors.
+# Copyright 2020-2024 The StackStorm Authors.
 # Copyright 2019 Extreme Networks, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

contrib/runners/local_runner/tests/unit/BUILD

@@ -2,3 +2,7 @@ __defaults__(
     {(python_test, python_tests): dict(tags=["unit"])},
     extend=True,
 )
+
+python_tests(
+    name="tests",
+)

contrib/runners/local_runner/tests/unit/test_local_runner.py

@@ -0,0 +1,20 @@
+# Copyright 2024 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+
+@pytest.mark.skip(reason="TEST TO BE IMPLEMENTED.")
+def test_todo():
+    pass

contrib/runners/orquesta_runner/in-requirements.txt

@@ -1 +1,2 @@
-orquesta@ git+https://github.com/StackStorm/[email protected]
+#orquesta@ git+https://github.com/StackStorm/[email protected]
+orquesta@ git+https://github.com/nzlosh/orquesta.git@st2v39_compat

contrib/runners/orquesta_runner/requirements.txt

@@ -5,4 +5,4 @@
 # If you want to update depdencies for a single component, modify the
 # in-requirements.txt for that component and then run 'make requirements' to
 # update the component requirements.txt
-orquesta@ git+https://github.com/StackStorm/orquesta.git@v1.6.0
+orquesta@ git+https://github.com/nzlosh/orquesta.git@st2v39_compat

contrib/runners/remote_runner/tests/unit/BUILD

@@ -2,3 +2,7 @@ __defaults__(
     {(python_test, python_tests): dict(tags=["unit"])},
     extend=True,
 )
+
+python_tests(
+    name="tests",
+)

contrib/runners/remote_runner/tests/unit/test_remote_runner.py

@@ -0,0 +1,20 @@
+# Copyright 2024 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+
+@pytest.mark.skip(reason="TEST TO BE IMPLEMENTED.")
+def test_todo():
+    pass

fixed-requirements.txt

@@ -1,32 +1,46 @@
-# Packages versions fixed for the whole st2 stack
+# Package versions fixed (pinned) for the whole st2 project.
+# Important: Keep version constraints synchronised with the below repositories:
+# - https://github.com/StackStorm/st2docs
+# - https://github.com/StackStorm/orquesta
+# - https://github.com/StackStorm/st2-auth-ldap
+# - https://github.com/StackStorm/st2-rbac-backend
+# ----------------------------------------------------------------------
 # Note: amqp is used by kombu
 amqp==5.2.0
 apscheduler==3.10.4
-chardet==3.0.4
+# 202404: Use latest chardet for tox / and cross project dependency Orquesta.
+# 202405: chardet is a transtive dependency. Just accept whatever pip picks.
+#chardet==5.2.0
 cffi==1.16.0
 # NOTE: 2.0 version breaks pymongo work with hosts
-dnspython==1.16.0
-cryptography==42.0.5
+dnspython==2.6.1
+cryptography==42.0.7
 eventlet==0.36.1
 flex==6.14.1
 # Note: installs gitpython==3.1.37 (security fixed) under py3.8 and gitpython==3.1.18 (latest available, vulnerable) under py3.6
 # TODO: Pin to 3.1.37 or higher after dropping python3.6 support
 gitpython==3.1.43
 # Needed by gitpython, old versions used to bundle it
 gitdb==4.0.11
-# Note: greenlet is used by eventlet
+# Note: greenlet is used by eventlet (202403: 3.0.3 support for py3.10)
 greenlet==3.0.3
 gunicorn==22.0.0
 jsonpath-rw==1.4.0
 jsonschema==3.2.0
+# 202403: Update to 5.3.5 for py3.10 support
 kombu==5.3.7
 lockfile==0.12.2
 # Fix MarkupSafe to < 2.1.0 as 2.1.0 removes soft_unicode
-# >=0.23 was from jinja2
-MarkupSafe==2.0.1
+# 202403: Bumped Jinja2 and MarkupSafe to add py3.10 support
+jinja2==3.1.4
+# 202405: MarkupSafe is a transitive dependency. Just accept whatever pip picks.
+#MarkupSafe==2.1.5
+# mongoengine 0.24.0 has breaking changes to support pymongo 4.0
 mongoengine==0.23.1
 # required by orquesta (networkx<2.6 for py3.6, networkx<3 for py3.8)
-networkx==2.8.8
+# 202404: networkx >3.2 doesn't support py3.8
+#networkx<3.2
+networkx==3.1
 # networkx dropped its dep on decorator in version 2.6, so the old pin is unneeded.
 # now jsonpath-rw is the only thing that depends on decorator (a transitive dep)
 decorator==5.1.1
@@ -39,6 +53,7 @@ passlib==1.7.4
 # 202403: bump to 3.0.43 for py3.10 support
 prompt-toolkit==3.0.43
 pyinotify==0.9.6 ; platform_system=="Linux"
+# pymongo 3.13 has backports of APIs from pymongo 4 to help w/ migration
 pymongo==3.12.3
 pyparsing==3.1.2
 zstandard==0.22.0
@@ -47,41 +62,41 @@ zstandard==0.22.0
 # 202403: switch from python-editor to editor for py3.10 support
 editor==1.6.6
 # editor dependency, required here for inclusion in st2client setup.py
-pygments==2.17.2
+pygments==2.18.0
 python-keyczar==0.716
 pytz==2024.1
 pywinrm==0.4.3
 pyyaml==6.0.1
 redis==5.0.4
+# 202404: Bumped to latest since py3.6 support has been dropped.
 requests==2.31.0
 retrying==1.3.4
 routes==2.5.1
 semver==3.0.2
+# 202404: Bumped for compatibility prance 23.6.21.0 requires six~=1.15.
 six==1.16.0
-argparse==1.4.0
+# 202403: Bumped to latest for py3.10 support
 argcomplete==3.3.0
 prettytable==3.10.0
+# 202404: Relaxed pinning since py3.6 support is dropped.
 importlib-metadata==7.1.0
 typing-extensions==4.11.0
 # NOTE: sseclient has various issues which sometimes hang the connection for a long time, etc.
 sseclient-py==1.8.0
 stevedore==5.2.0
-tenacity==8.2.3
+tenacity==8.3.0
 tooz==6.1.0
 # Note: virtualenv embeds wheels for pip, wheel, and setuptools. So pinning virtualenv pins those as well.
 # virtualenv==20.26.0 (<21) has pip==24.0 wheel==0.43.0 setuptools==69.5.1
 # lockfiles/st2.lock has pip==24.0 wheel==0.43.0 setuptools==69.5.1
-virtualenv==20.26.1
+virtualenv==20.26.2
 webob==1.8.7
 zake==0.2.2
 # test requirements below
-bcrypt==4.1.2
-jinja2==3.1.3
+bcrypt==4.1.3
 mock==5.1.0
-nose-timer==1.0.1
-nose-parallel==0.4.0
 psutil==5.9.8
-python-dateutil==2.9.0
+python-dateutil==2.9.0.post0
 python-statsd==2.1.0
-orjson==3.10.1
-zipp==3.18.1
+orjson==3.10.3
+zipp==3.18.2