diff --git a/.github/workflows/build-test-release.yml b/.github/workflows/build-test-release.yml index 5d10a5ac..00a852bb 100644 --- a/.github/workflows/build-test-release.yml +++ b/.github/workflows/build-test-release.yml @@ -186,6 +186,7 @@ jobs: "splunk_app_cim_broken", "splunk_fiction_indextime", "splunk_fiction_indextime_broken", + "splunk_fiction_indextime_wrong_hec_token", "splunk_setup_fixture", "splunk_app_req", "splunk_app_req_broken", diff --git a/.gitignore b/.gitignore index 5b5acd24..f6d1481b 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ test_report.md *.pickle *_events.lock *_events +.venv diff --git a/pytest_splunk_addon/plugin.py b/pytest_splunk_addon/plugin.py index a4cc6efe..220f5f6e 100644 --- a/pytest_splunk_addon/plugin.py +++ b/pytest_splunk_addon/plugin.py @@ -25,6 +25,8 @@ test_generator = None +EXC_MAP = [Exception] + def pytest_configure(config): """ @@ -93,6 +95,10 @@ def pytest_configure(config): "markers", "splunk_requirements_unit: Test checking if all fields for datamodel are defined in cim_fields and missing_recommended_fields", ) + if config.getoption("splunk_hec_token") == "9b741d03-43e9-4164-908b-e09102327d22": + LOGGER.warning( + "Using the default value for --splunk-hec-token which is going to be deprecated. Please provide --splunk-hec-token argument when executing tests." + ) cim_report = config.getoption("cim_report") if cim_report and not hasattr(config, "slaveinput"): @@ -115,6 +121,7 @@ def pytest_sessionstart(session): SampleXdistGenerator.tokenized_event_source = session.config.getoption( "tokenized_event_source" ).lower() + session.__exc_limits = EXC_MAP if ( SampleXdistGenerator.tokenized_event_source == "store_new" and session.config.getoption("ingest_events").lower() @@ -187,12 +194,16 @@ def init_pytest_splunk_addon_logger(): """ fh = logging.FileHandler(LOG_FILE) fh.setLevel(logging.INFO) + ch = logging.StreamHandler() + ch.setLevel(logging.WARNING) formatter = logging.Formatter( "%(asctime)s - %(levelname)s - %(filename)s - %(funcName)s - %(message)s" ) fh.setFormatter(formatter) + ch.setFormatter(formatter) logger = logging.getLogger("pytest-splunk-addon") logger.addHandler(fh) + logger.addHandler(ch) logging.root.propagate = False logger.setLevel(logging.INFO) return logger @@ -200,3 +211,14 @@ def init_pytest_splunk_addon_logger(): init_pytest_splunk_addon_logger() LOGGER = logging.getLogger("pytest-splunk-addon") + + +def pytest_exception_interact(node, call, report): + """ + Hook called when an exception is raised during a test. + If the number of occurrences for a specific exception exceeds the limit in session.__exc_limits, pytest exits + https://docs.pytest.org/en/stable/reference/reference.html#pytest.hookspec.pytest_exception_interact + """ + if call.excinfo.type in node.session.__exc_limits: + # pytest exits only for exceptions defined in EXC_MAP + pytest.exit(f"Exiting pytest due to: {call.excinfo.type}") diff --git a/pytest_splunk_addon/splunk.py b/pytest_splunk_addon/splunk.py index 6c0f7607..724381c4 100644 --- a/pytest_splunk_addon/splunk.py +++ b/pytest_splunk_addon/splunk.py @@ -110,7 +110,7 @@ def pytest_addoption(parser): action="store", dest="splunk_hec_token", default="9b741d03-43e9-4164-908b-e09102327d22", - help='Splunk HTTP event collector token. default is "9b741d03-43e9-4164-908b-e09102327d22" If an external forwarder is used provide HEC token of forwarder.', + help="Splunk HTTP event collector token. If an external forwarder is used provide HEC token of forwarder. Please specify it as default value is going to be deprecated.", ) group.addoption( "--splunk-port", @@ -552,6 +552,7 @@ def splunk_docker( # get the temp directory shared by all workers root_tmp_dir = tmp_path_factory.getbasetemp().parent fn = root_tmp_dir / "pytest_docker" + # if you encounter docker-compose not found modify shell path in your IDE to use /bin/bash with FileLock(str(fn) + ".lock"): docker_services.start("splunk") @@ -576,11 +577,12 @@ def splunk_docker( docker_services.port_for("splunk", 9997), ) - docker_services.wait_until_responsive( - timeout=180.0, - pause=0.5, - check=lambda: is_responsive_splunk(splunk_info), - ) + for _ in range(RESPONSIVE_SPLUNK_TIMEOUT): + if is_responsive_splunk(splunk_info) and is_responsive_hec( + request, splunk_info + ): + break + sleep(1) return splunk_info @@ -610,9 +612,9 @@ def splunk_external(request): ) for _ in range(RESPONSIVE_SPLUNK_TIMEOUT): - if is_responsive_splunk(splunk_info): - break - if is_responsive_hec(request, splunk_info): + if is_responsive_splunk(splunk_info) and is_responsive_hec( + request, splunk_info + ): break sleep(1) diff --git a/tests/e2e/incorrect_hec_token_conftest.py b/tests/e2e/incorrect_hec_token_conftest.py new file mode 100644 index 00000000..a914a53b --- /dev/null +++ b/tests/e2e/incorrect_hec_token_conftest.py @@ -0,0 +1,44 @@ +import os +import pytest +import requests + +pytest_plugins = "pytester" + + +def pytest_configure(config): + config.addinivalue_line("markers", "external: Test search time only") + config.addinivalue_line("markers", "docker: Test search time only") + config.addinivalue_line("markers", "doc: Test Sphinx docs") + + +@pytest.fixture(scope="session") +def docker_compose_files(request): + """ + Get an absolute path to the `docker-compose.yml` file. Override this + fixture in your tests if you need a custom location. + + Returns: + string: the path of the `docker-compose.yml` file + + """ + docker_compose_path = os.path.join( + str(request.config.invocation_dir), "docker-compose.yml" + ) + # LOGGER.info("docker-compose path: %s", docker_compose_path) + + return [docker_compose_path] + + +@pytest.fixture(scope="session") +def docker_services_project_name(pytestconfig): + rootdir = str(pytestconfig.rootdir) + docker_compose_v2_rootdir = rootdir.lower().replace("/", "") + return f"pytest{docker_compose_v2_rootdir}" + + +@pytest.fixture(scope="session") +def splunk_hec_uri(splunk, request): + splunk_session = requests.Session() + splunk_session.headers = {"Authorization": f"Splunk test-token"} + uri = f'{request.config.getoption("splunk_hec_scheme")}://{splunk["forwarder_host"]}:{splunk["port_hec"]}/services/collector' + return splunk_session, uri diff --git a/tests/e2e/test_splunk_addon.py b/tests/e2e/test_splunk_addon.py index fd1dc59b..25dc68a4 100644 --- a/tests/e2e/test_splunk_addon.py +++ b/tests/e2e/test_splunk_addon.py @@ -166,6 +166,57 @@ def empty_method(): assert result.ret == 0 +@pytest.mark.docker +@pytest.mark.splunk_fiction_indextime_wrong_hec_token +def test_splunk_fiction_indextime_wrong_hec_token(testdir): + """Make sure that pytest accepts our fixture.""" + + testdir.makepyfile( + """ + from pytest_splunk_addon.standard_lib.addon_basic import Basic + class Test_App(Basic): + def empty_method(): + pass + + """ + ) + + shutil.copytree( + os.path.join(testdir.request.fspath.dirname, "addons/TA_fiction_indextime"), + os.path.join(testdir.tmpdir, "package"), + ) + + shutil.copytree( + os.path.join(testdir.request.fspath.dirname, "test_data_models"), + os.path.join(testdir.tmpdir, "tests/data_models"), + ) + + setup_test_dir(testdir) + SampleGenerator.clean_samples() + Rule.clean_rules() + with open( + os.path.join(testdir.request.fspath.dirname, "incorrect_hec_token_conftest.py") + ) as conf_test_file: + testdir.makeconftest(conf_test_file.read()) + + # run pytest with the following cmd args + result = testdir.runpytest( + "--splunk-type=docker", + "-v", + "--search-interval=0", + "--search-retry=0", + "--splunk-data-generator=tests/addons/TA_fiction_indextime/default", + "--search-index=*,_internal", + ) + + result.assert_outcomes(errors=1, passed=0, failed=0, xfailed=0) + result.stdout.fnmatch_lines( + "!!!!!! _pytest.outcomes.Exit: Exiting pytest due to: !!!!!!!" + ) + + assert result.ret != 0 + + @pytest.mark.docker @pytest.mark.splunk_app_broken def test_splunk_app_broken(testdir):