From ff599bc301c746e4346c5f375db8d6a2faceb4fa Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 6 Nov 2024 13:32:48 -0600 Subject: [PATCH 1/9] pants ci: filter remote cache warnings --- pants.ci.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pants.ci.toml b/pants.ci.toml index 428d3b8de3..158b7b3c8a 100644 --- a/pants.ci.toml +++ b/pants.ci.toml @@ -11,6 +11,13 @@ remote_provider = "experimental-github-actions-cache" remote_cache_read = true remote_cache_write = true +# https://www.pantsbuild.org/stable/reference/global-options#ignore_warnings +ignore_warnings = [ + # remote cache errors caused by GitHub rate-limits are not helpful + "Failed to read from remote cache", + "Failed to write to remote cache", +] + [stats] # "print metrics of your cache's performance at the end of the run, # including the number of cache hits and the total time saved thanks From e01e799737f8813c922add2395548a6f28bf7d98 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 2 Nov 2024 00:28:12 -0500 Subject: [PATCH 2/9] pants: add BUILD metadata for st2actions itests --- st2actions/tests/integration/BUILD | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/st2actions/tests/integration/BUILD b/st2actions/tests/integration/BUILD index 2d782aaea0..51c5b56d14 100644 --- a/st2actions/tests/integration/BUILD +++ b/st2actions/tests/integration/BUILD @@ -5,4 +5,8 @@ __defaults__( python_tests( name="tests", + uses=["rabbitmq"], + stevedore_namespaces=[ + "st2common.metrics.driver", + ], ) From f6bc9dfe284ff0ff4b780b579922615e0631458a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 2 Nov 2024 00:48:56 -0500 Subject: [PATCH 3/9] pants: add BUILD metadata for st2api itests --- st2api/tests/integration/BUILD | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/st2api/tests/integration/BUILD b/st2api/tests/integration/BUILD index e1fda4d067..5302bad1b6 100644 --- a/st2api/tests/integration/BUILD +++ b/st2api/tests/integration/BUILD @@ -7,5 +7,13 @@ python_tests( name="tests", dependencies=[ "conf/st2.tests.conf:st2_tests_conf", + "st2api/st2api/wsgi.py", + "st2auth/st2auth/wsgi.py", ], + stevedore_namespaces=[ + "st2auth.sso.backends", + "st2common.metrics.driver", + "st2common.rbac.backend", + ], + uses=["mongo", "rabbitmq", "redis"], ) From b6e06a0d952e81111a55e5e2f185fc0e25010ac6 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 2 Nov 2024 12:13:44 -0500 Subject: [PATCH 4/9] pants: add BUILD metadata for st2reactor itests --- st2reactor/tests/integration/BUILD | 48 +++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/st2reactor/tests/integration/BUILD b/st2reactor/tests/integration/BUILD index 3fea4cca37..147337ac47 100644 --- a/st2reactor/tests/integration/BUILD +++ b/st2reactor/tests/integration/BUILD @@ -3,10 +3,50 @@ __defaults__( extend=True, ) +_conf_deps = [ + "conf/st2.tests.conf:st2_tests_conf", + "conf/st2.tests2.conf:st2_tests_conf", +] + python_tests( name="tests", - dependencies=[ - "conf/st2.tests.conf:st2_tests_conf", - "conf/st2.tests2.conf:st2_tests_conf", - ], + dependencies=_conf_deps, + uses=["mongo", "rabbitmq", "redis"], + overrides={ + "test_garbage_collector.py": dict( + dependencies=[ + *_conf_deps, + "st2reactor/bin/st2garbagecollector", + ], + stevedore_namespaces=[ + "st2common.metrics.driver", + ], + entry_point_dependencies={ + "contrib/runners/inquirer_runner": ["st2common.runners.runner"], + }, + ), + "test_rules_engine.py": dict( + dependencies=[ + *_conf_deps, + "st2reactor/bin/st2timersengine", + ], + stevedore_namespaces=[ + "st2common.metrics.driver", + ], + ), + "test_sensor_container.py": dict( + dependencies=[ + *_conf_deps, + "st2reactor/bin/st2sensorcontainer", + "contrib/examples/sensors", + "contrib/examples:metadata", + ], + stevedore_namespaces=[ + "st2common.metrics.driver", + ], + ), + "test_sensor_watcher.py": dict( + uses=["rabbitmq"], + ), + }, ) From ec5c060afa0c2122fa033f54ceaa735d568f6c43 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 2 Nov 2024 22:09:35 -0500 Subject: [PATCH 5/9] pants: add BUILD metadata for st2common itests --- st2common/tests/fixtures/BUILD | 3 ++- st2common/tests/integration/BUILD | 37 ++++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/st2common/tests/fixtures/BUILD b/st2common/tests/fixtures/BUILD index 3ba6abf422..01dbde214a 100644 --- a/st2common/tests/fixtures/BUILD +++ b/st2common/tests/fixtures/BUILD @@ -1,5 +1,6 @@ python_sources() -shell_sources( +st2_shell_sources_and_resources( name="shell", + sources=["*.sh"], ) diff --git a/st2common/tests/integration/BUILD b/st2common/tests/integration/BUILD index 6587940ceb..275336dae7 100644 --- a/st2common/tests/integration/BUILD +++ b/st2common/tests/integration/BUILD @@ -5,19 +5,46 @@ __defaults__( python_tests( name="tests", - dependencies=[ - # used by test_register_content_script - "conf/st2.tests.conf:st2_tests_conf", - "conf/st2.tests1.conf:st2_tests_conf", - ], stevedore_namespaces=[ "orquesta.expressions.functions", "st2common.runners.runner", "st2common.rbac.backend", "st2common.metrics.driver", ], + uses=["mongo", "rabbitmq", "redis"], + overrides={ + "test_logging.py": dict( + dependencies=[ + "./log_unicode_data.py", + ], + ), + "test_register_content_script.py": dict( + dependencies=[ + "conf/st2.tests.conf:st2_tests_conf", + "conf/st2.tests1.conf:st2_tests_conf", + "st2common/bin/st2-register-content", + ], + ), + "test_service_setup_log_level_filtering.py": dict( + dependencies=[ + "st2api/bin/st2api", + ], + ), + "test_util_green.py": dict( + dependencies=[ + "st2common/tests/fixtures/print_to_stdout_stderr_sleep.sh:shell_resources", + ], + ), + }, ) python_test_utils( sources=["*.py", "!test_*.py"], + overrides={ + "log_unicode_data.py": dict( + dependencies=[ + "st2tests/st2tests/fixtures/conf:st2.tests.conf", + ], + ), + }, ) From c1c4c7c963a4a4b472e44497ccfc463eb78ecead Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 5 Nov 2024 19:16:18 -0600 Subject: [PATCH 6/9] pants: add BUILD metadata for orquesta itests --- contrib/runners/orquesta_runner/tests/integration/BUILD | 2 +- st2tests/integration/orquesta/BUILD | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/runners/orquesta_runner/tests/integration/BUILD b/contrib/runners/orquesta_runner/tests/integration/BUILD index 54651017ff..8d33e316f4 100644 --- a/contrib/runners/orquesta_runner/tests/integration/BUILD +++ b/contrib/runners/orquesta_runner/tests/integration/BUILD @@ -5,5 +5,5 @@ __defaults__( python_tests( name="tests", - uses=["redis"], + uses=["mongo", "rabbitmq", "redis", "system_user"], ) diff --git a/st2tests/integration/orquesta/BUILD b/st2tests/integration/orquesta/BUILD index 5b6c97ec1f..8073d3f7a5 100644 --- a/st2tests/integration/orquesta/BUILD +++ b/st2tests/integration/orquesta/BUILD @@ -1,6 +1,6 @@ python_tests( name="tests", - uses=["redis"], + uses=["mongo", "rabbitmq", "redis", "system_user"], ) python_test_utils( From dddedf012b687e26983732ff99ac7379ccbbe0e9 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 6 Nov 2024 20:59:32 -0600 Subject: [PATCH 7/9] skip tests that require passwordless sudo for local development Only run them when ST2_CI==true to minimize requirements for local development. Also fix a typo in a comment. --- .../tests/integration/test_localrunner.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/contrib/runners/local_runner/tests/integration/test_localrunner.py b/contrib/runners/local_runner/tests/integration/test_localrunner.py index 1ca4f4275f..f165a72ff3 100644 --- a/contrib/runners/local_runner/tests/integration/test_localrunner.py +++ b/contrib/runners/local_runner/tests/integration/test_localrunner.py @@ -15,6 +15,7 @@ from __future__ import absolute_import import os +import unittest import uuid import mock @@ -45,6 +46,8 @@ __all__ = ["LocalShellCommandRunnerTestCase", "LocalShellScriptRunnerTestCase"] +ST2_CI = os.environ.get("ST2_CI", "false").lower() == "true" + MOCK_EXECUTION = mock.Mock() MOCK_EXECUTION.id = "598dbf0c0640fd54bffc688b" @@ -94,6 +97,11 @@ def test_shell_command_action_basic(self): self.assertEqual(output_dbs[0].output_type, "stdout") self.assertEqual(output_dbs[0].data, "10\n") + # This test depends on passwordless sudo. Don't require that for local development. + @unittest.skipIf( + not ST2_CI, + 'Skipping tests because ST2_CI environment variable is not set to "true"', + ) def test_timeout(self): models = self.fixtures_loader.load_models( fixtures_pack=GENERIC_PACK, fixtures_dict={"actions": ["local.yaml"]} @@ -141,8 +149,13 @@ def test_common_st2_env_vars_are_available_to_the_action(self): self.assertEqual(status, action_constants.LIVEACTION_STATUS_SUCCEEDED) self.assertEqual(result["stdout"].strip(), "mock-token") + # This test depends on passwordless sudo. Don't require that for local development. + @unittest.skipIf( + not ST2_CI, + 'Skipping tests because ST2_CI environment variable is not set to "true"', + ) def test_sudo_and_env_variable_preservation(self): - # Verify that the environment environment are correctly preserved when running as a + # Verify that the environment vars are correctly preserved when running as a # root / non-system user # Note: This test will fail if SETENV option is not present in the sudoers file models = self.fixtures_loader.load_models( @@ -297,6 +310,11 @@ def test_action_stdout_and_stderr_is_stored_in_the_db_short_running_action( self.assertEqual(output_dbs[db_index_1].data, mock_stderr[0]) self.assertEqual(output_dbs[db_index_2].data, mock_stderr[1]) + # This test depends on passwordless sudo. Don't require that for local development. + @unittest.skipIf( + not ST2_CI, + 'Skipping tests because ST2_CI environment variable is not set to "true"', + ) def test_shell_command_sudo_password_is_passed_to_sudo_binary(self): # Verify that sudo password is correctly passed to sudo binary via stdin models = self.fixtures_loader.load_models( From 082ac9b9b18e936f247d81754384b22c70282226 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 12 Nov 2024 11:34:03 -0600 Subject: [PATCH 8/9] pants: [test].timeout_default=10min --- pants.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pants.toml b/pants.toml index 72e5fcb6c8..6dc6bba87f 100644 --- a/pants.toml +++ b/pants.toml @@ -244,6 +244,8 @@ extra_env_vars = [ "ST2TESTS_REDIS_HOST", "ST2TESTS_REDIS_PORT", ] +# 10 min should be more than enough even for integration tests. +timeout_default = 600 # seconds [twine] install_from_resolve = "twine" From 098759fae52921c13b02f77719b078303a6234e6 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 12 Nov 2024 15:01:53 -0600 Subject: [PATCH 9/9] update changelog entry --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index abd62f9cc8..613f01a3a8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -69,7 +69,7 @@ Added working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253 - #6254 #6258 #6259 #6260 #6269 + #6254 #6258 #6259 #6260 #6269 #6275 Contributed by @cognifloyd * Build of ST2 EL9 packages #6153 Contributed by @amanda11