From 0c2c3e32c3c87dda0f6839008bd998eb9ecb2b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Wed, 21 Aug 2024 12:33:42 +0100 Subject: [PATCH 01/14] docs: update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d529d94d7..d89a3f6658 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file. ### Added +- Adding jobflow templates ([#5680](https://github.com/wazuh/wazuh-qa/pull/5680/)) \- (Framework) - Add RockyLinux 8.10 to Allocator module ([#5524](https://github.com/wazuh/wazuh-qa/pull/5524)) \- (Framework) - Add Deployability testing tier 1 ([#5190](https://github.com/wazuh/wazuh-qa/pull/5190)) \- (Framework + Documentation + Tests) - Add Workflow module to Wazuh-qa repository ([#4990](https://github.com/wazuh/wazuh-qa/pull/4990)) \- (Tests) @@ -32,6 +33,7 @@ All notable changes to this project will be documented in this file. ### Fixed +- Fix playbook cleanup.yaml ([#5672](https://github.com/wazuh/wazuh-qa/pull/5672)) \- (Tests) - Fixed unnecesary reference to debian file in dashboard provisioning task ([#5643](https://github.com/wazuh/wazuh-qa/pull/5643)) \- (Framework) - Changed 'Ensure that the manager version is' expected warning to an agnostic version of regex ([#5630](https://github.com/wazuh/wazuh-qa/pull/5630)) \- (Tests) - Adding fixed and dynamic waits to port status checks ([#5627](https://github.com/wazuh/wazuh-qa/pull/5627)) (Framework) From 9af7b7243b9bbbbeb52ffcb9ae0a5b11fa0caa7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Wed, 21 Aug 2024 13:01:36 +0100 Subject: [PATCH 02/14] feat: bump 4.9.0 revision to 40906 --- version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.json b/version.json index 68a33aea49..5a3ccead7c 100644 --- a/version.json +++ b/version.json @@ -1,4 +1,4 @@ { "version": "4.9.0", - "revision": "40905" + "revision": "40906" } From a92ba7cdb7ce5920966fc354dd9228008c959482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Fri, 30 Aug 2024 09:01:10 +0100 Subject: [PATCH 03/14] feat: bump 4.9.0 revision to 40907 --- version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.json b/version.json index 5a3ccead7c..ea7228044a 100644 --- a/version.json +++ b/version.json @@ -1,4 +1,4 @@ { "version": "4.9.0", - "revision": "40906" + "revision": "40907" } From 5afd21531e2aa73810c31dec78714108638a20c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Tue, 3 Sep 2024 17:03:43 +0100 Subject: [PATCH 04/14] feat: increase result windows in VD E2E tests --- .../end_to_end/check_validators.py | 8 ++-- .../wazuh_testing/end_to_end/indexer_api.py | 1 + .../end_to_end/remote_operations_handler.py | 4 +- .../end_to_end/vulnerability_detector.py | 7 ++-- .../test_vulnerability_detector/conftest.py | 7 +++- .../test_vulnerability_detector.py | 41 +++++++++++++------ 6 files changed, 45 insertions(+), 23 deletions(-) diff --git a/deps/wazuh_testing/wazuh_testing/end_to_end/check_validators.py b/deps/wazuh_testing/wazuh_testing/end_to_end/check_validators.py index aa33e5761f..21abfa06c5 100644 --- a/deps/wazuh_testing/wazuh_testing/end_to_end/check_validators.py +++ b/deps/wazuh_testing/wazuh_testing/end_to_end/check_validators.py @@ -28,7 +28,7 @@ def compare_expected_found_vulnerabilities(vulnerabilities, expected_vulnerabili for vulnerability in expected_vulns: if vulnerability not in vulnerabilities.get(agent, []): logging.critical(f"Vulnerability not found for {agent}: {vulnerability}") - if agent not in vulnerabilities_not_found: + if agent not in vulnerabilities_not_found.keys(): vulnerabilities_not_found[agent] = [] failed_agents.append(agent) @@ -39,9 +39,10 @@ def compare_expected_found_vulnerabilities(vulnerabilities, expected_vulnerabili for vulnerability in agent_vulnerabilities: if vulnerability not in expected_vulnerabilities.get(agent, []): logging.critical(f"Vulnerability unexpected found for {agent}: {vulnerability}") - if agent not in vulnerabilities_unexpected: + if agent not in vulnerabilities_unexpected.keys(): vulnerabilities_unexpected[agent] = [] - failed_agents.append(agent) + if agent not in failed_agents: + failed_agents.append(agent) result = False vulnerabilities_unexpected[agent].append(vulnerability) @@ -161,4 +162,3 @@ def equals_but_not_empty(x, y): not any(x[host][level] for level in ["ERROR", "CRITICAL", "WARNING"]) for host in x ) - diff --git a/deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py b/deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py index 1ca658f186..4167f12b00 100644 --- a/deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py +++ b/deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py @@ -31,6 +31,7 @@ WAZUH_STATES_VULNERABILITIES_INDEXNAME_TEMPLATE = 'wazuh-states-vulnerabilities-{cluster_name}' +INDEXER_RESULT_WINDOWS_VULN_E2E = 50000 def get_wazuh_states_vulnerabilities_indexname(cluster_name: str = 'wazuh') -> str: diff --git a/deps/wazuh_testing/wazuh_testing/end_to_end/remote_operations_handler.py b/deps/wazuh_testing/wazuh_testing/end_to_end/remote_operations_handler.py index 4cab9687f0..618b2ae9a3 100644 --- a/deps/wazuh_testing/wazuh_testing/end_to_end/remote_operations_handler.py +++ b/deps/wazuh_testing/wazuh_testing/end_to_end/remote_operations_handler.py @@ -243,9 +243,9 @@ def get_vulnerability_alerts(host_manager: HostManager, agent_list, packages_dat def get_vulnerabilities_index(host_manager: HostManager, agent_list, packages_data: List[Dict], - greater_than_timestamp: str = "") -> Dict: + greater_than_timestamp: str = "", size=10000) -> Dict: vulnerabilities = get_vulnerabilities_from_states_by_agent(host_manager, agent_list, - greater_than_timestamp=greater_than_timestamp) + greater_than_timestamp=greater_than_timestamp, size=size) package_vulnerabilities = filter_vulnerabilities_by_packages(host_manager, vulnerabilities, packages_data) return package_vulnerabilities diff --git a/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py b/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py index 871b07ffeb..1e0c8eea04 100644 --- a/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py +++ b/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py @@ -275,7 +275,8 @@ def parse_vulnerability_from_state(state): def get_vulnerabilities_from_states_by_agent(host_manager: HostManager, agents: List[str], - greater_than_timestamp: str = None, cluster_name='wazuh') -> dict: + greater_than_timestamp: str = None, cluster_name='wazuh', + size=10000) -> dict: """Get vulnerabilities from the vulnerability state index by agent. Args: @@ -309,8 +310,8 @@ def get_vulnerabilities_from_states_by_agent(host_manager: HostManager, agents: filter=states_filter, index=index, credentials={'user': indexer_user, - 'password': indexer_password} - )['hits']['hits'] + 'password': indexer_password}, + size=size)['hits']['hits'] except KeyError as e: logging.error(f"No vulnerabilities were obtained for {agent}. Exception {str(e)}") diff --git a/tests/end_to_end/test_vulnerability_detector/conftest.py b/tests/end_to_end/test_vulnerability_detector/conftest.py index 181e641ec6..7467e339e6 100644 --- a/tests/end_to_end/test_vulnerability_detector/conftest.py +++ b/tests/end_to_end/test_vulnerability_detector/conftest.py @@ -52,7 +52,7 @@ def test_example(host_manager): restore_configuration, save_indexer_credentials_into_keystore) from wazuh_testing.end_to_end.indexer_api import ( - get_wazuh_states_vulnerabilities_indexname, delete_index) + get_wazuh_states_vulnerabilities_indexname, delete_index, extend_result_window, INDEXER_RESULT_WINDOWS_VULN_E2E) from wazuh_testing.end_to_end.logs import (get_hosts_alerts, get_hosts_logs, truncate_remote_host_group_files) from wazuh_testing.end_to_end.remote_operations_handler import ( @@ -320,8 +320,11 @@ def setup(preconditions, teardown, host_manager) -> Generator[Dict, None, None]: timeout_vulnerabilities_detected = len(agents_to_check) * PACKAGE_VULNERABILITY_SCAN_TIME time.sleep(timeout_syscollector_scan + timeout_vulnerabilities_detected) + vuln_index = get_wazuh_states_vulnerabilities_indexname() + extend_result_window(host_manager, index=vuln_index, new_max_result_window=INDEXER_RESULT_WINDOWS_VULN_E2E) - vulnerabilities = get_vulnerabilities_index(host_manager, agents_to_check, package_data) + vulnerabilities = get_vulnerabilities_index(host_manager, agents_to_check, package_data, + size=INDEXER_RESULT_WINDOWS_VULN_E2E) vulnerabilities_from_alerts = get_vulnerability_alerts(host_manager, agents_to_check, package_data, test_timestamp) diff --git a/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py b/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py index 8b2f73b6f6..1005f7f5c4 100644 --- a/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py +++ b/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py @@ -70,7 +70,9 @@ get_vulnerabilities_from_states_by_agent) from wazuh_testing.end_to_end.waiters import wait_until_vd_is_updated from wazuh_testing.tools.system import HostManager - +from wazuh_testing.end_to_end.indexer_api import (INDEXER_RESULT_WINDOWS_VULN_E2E, + extend_result_window, + get_wazuh_states_vulnerabilities_indexname) pytestmark = [pytest.mark.e2e, pytest.mark.vulnerability_detector, pytest.mark.tier0] @@ -192,6 +194,11 @@ def configure_vulnerability_detection_test_environment( yield test_timestamp +def max_result_window(host_manager): + vuln_index = get_wazuh_states_vulnerabilities_indexname() + extend_result_window(host_manager, index=vuln_index, new_max_result_window=INDEXER_RESULT_WINDOWS_VULN_E2E) + + @pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning") class TestInitialScans: # Checks definition @@ -254,7 +261,7 @@ def test_first_syscollector_scan( configure_vulnerability_detection_test_environment, record_property, clean_environment_logs, - delete_states_vulnerability_index, + delete_states_vulnerability_index ): """ description: Validates the initiation of the first Syscollector scans across all agents in the environment. @@ -326,11 +333,13 @@ def test_first_syscollector_scan( logging.critical("Waiting until agent all agents have been scanned.") time.sleep(TIMEOUT_PER_AGENT_VULNERABILITY_FIRST_SCAN * len(AGENTS_SCANNED_FIRST_SCAN)) + max_result_window(host_manager) logging.critical("Checking vulnerabilities in the index") vuln_by_agent_index = get_vulnerabilities_from_states_by_agent( host_manager, AGENTS_SCANNED_FIRST_SCAN, greater_than_timestamp=FIRST_SCAN_TIME, + size=INDEXER_RESULT_WINDOWS_VULN_E2E ) # Store the vulnerabilities in the global variable to make the comparision in test_consistency_initial_scans @@ -371,7 +380,7 @@ def test_first_syscollector_scan( logging.critical("Checking for errors in the environment") unexpected_errors = check_errors_in_environment( - host_manager, expected_errors=VULNERABILITY_DETECTION_E2E_EXPECTED_ERRORS + host_manager, expected_errors=VULNERABILITY_DETECTION_E2E_EXPECTED_ERRORS, error_levels=['ERROR', 'CRITICAL'] ) test_result.validate_check( @@ -501,10 +510,12 @@ def test_syscollector_second_scan( global FIRST_SCAN_TIME + max_result_window(host_manager) + logging.critical("Checking vulnerabilities in the index") vuln_by_agent_index = get_vulnerabilities_from_states_by_agent( - host_manager, agents_to_check_vulns, greater_than_timestamp=FIRST_SCAN_TIME - ) + host_manager, agents_to_check_vulns, greater_than_timestamp=FIRST_SCAN_TIME, + size=INDEXER_RESULT_WINDOWS_VULN_E2E) logging.critical( "Checking that all agents has been scanned and generated vulnerabilities in the index" @@ -522,7 +533,7 @@ def test_syscollector_second_scan( unexpected_errors = check_errors_in_environment( host_manager, expected_errors=VULNERABILITY_DETECTION_E2E_EXPECTED_ERRORS, - greater_than_timestamp=get_timestamp, + greater_than_timestamp=get_timestamp, error_levels=['ERROR','CRITICAL'] ) test_result.validate_check( @@ -707,8 +718,10 @@ def test_install_vulnerable_package_when_agent_down(self, host_manager, request, time.sleep(VD_E2E_TIMEOUT_SYSCOLLECTOR_SCAN + PACKAGE_VULNERABILITY_SCAN_TIME * len(AGENTS_SCANNED_FIRST_SCAN)) package_data = [body["package"]] + max_result_window(host_manager) - vulnerabilities = get_vulnerabilities_index(host_manager, AGENTS_SCANNED_FIRST_SCAN, package_data) + vulnerabilities = get_vulnerabilities_index(host_manager, AGENTS_SCANNED_FIRST_SCAN, package_data, + size=INDEXER_RESULT_WINDOWS_VULN_E2E) expected_vulnerabilities = get_expected_index(host_manager, AGENTS_SCANNED_FIRST_SCAN, body["operation"], body["package"]) duplicated_vulnerabilities = get_duplicated_vulnerabilities(vulnerabilities) @@ -747,7 +760,7 @@ def test_install_vulnerable_package_when_agent_down(self, host_manager, request, errors_environment = check_errors_in_environment( host_manager, expected_errors=VULNERABILITY_DETECTION_E2E_EXPECTED_ERRORS, - greater_than_timestamp=test_timestamp, + greater_than_timestamp=test_timestamp, error_levels=['ERROR','CRITICAL'] ) test_result.validate_check("no_errors", [Evidence("error_level_messages", errors_environment)]) @@ -796,8 +809,10 @@ def test_change_agent_manager(self, permutate_agents_managers, request, precondi time.sleep(VD_E2E_TIMEOUT_SYSCOLLECTOR_SCAN + PACKAGE_VULNERABILITY_SCAN_TIME * len(AGENTS_SCANNED_FIRST_SCAN)) package_data = [body["package"]] + max_result_window(host_manager) - vulnerabilities = get_vulnerabilities_index(host_manager, AGENTS_SCANNED_FIRST_SCAN, package_data) + vulnerabilities = get_vulnerabilities_index(host_manager, AGENTS_SCANNED_FIRST_SCAN, package_data, + size=INDEXER_RESULT_WINDOWS_VULN_E2E) expected_vulnerabilities = get_expected_index(host_manager, AGENTS_SCANNED_FIRST_SCAN, body["operation"], body["package"]) @@ -872,7 +887,7 @@ def test_change_agent_manager(self, permutate_agents_managers, request, precondi errors_environment = check_errors_in_environment( host_manager, expected_errors=VULNERABILITY_DETECTION_E2E_EXPECTED_ERRORS, - greater_than_timestamp=test_timestamp, + greater_than_timestamp=test_timestamp, error_levels=['ERROR','CRITICAL'] ) test_result.validate_check("no_errors", [Evidence("error_level_messages", errors_environment)]) @@ -928,8 +943,10 @@ def test_vulnerability_detector_scans_cases(self, request, preconditions, body, package_data = [body["package"]["to"], body["package"]["from"]] else: package_data = [body["package"]] + max_result_window(host_manager) - vulnerabilities = get_vulnerabilities_index(host_manager, AGENTS_SCANNED_FIRST_SCAN, package_data) + vulnerabilities = get_vulnerabilities_index(host_manager, AGENTS_SCANNED_FIRST_SCAN, package_data, + size=INDEXER_RESULT_WINDOWS_VULN_E2E) expected_vulnerabilities = get_expected_index(host_manager, AGENTS_SCANNED_FIRST_SCAN, body["operation"], body["package"]) duplicated_vulnerabilities = get_duplicated_vulnerabilities(vulnerabilities) @@ -1001,7 +1018,7 @@ def test_vulnerability_detector_scans_cases(self, request, preconditions, body, errors_environment = check_errors_in_environment( host_manager, expected_errors=VULNERABILITY_DETECTION_E2E_EXPECTED_ERRORS, - greater_than_timestamp=test_timestamp, + greater_than_timestamp=test_timestamp, error_levels=['ERROR','CRITICAL'] ) test_result.validate_check("no_errors", [Evidence("error_level_messages", errors_environment)]) From e3a5cec8275f236e6536493fda4101682a2fc465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Tue, 3 Sep 2024 17:57:50 +0100 Subject: [PATCH 05/14] feat: include extend windows in indexer api module --- .../wazuh_testing/end_to_end/indexer_api.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py b/deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py index 4167f12b00..d13cf54726 100644 --- a/deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py +++ b/deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py @@ -177,3 +177,36 @@ def delete_index(host_manager: HostManager, credentials: dict = {'user': 'admin' requests.delete(url=url, verify=False, auth=requests.auth.HTTPBasicAuth(credentials['user'], credentials['password']), headers=headers) + + +def extend_result_window(host_manager: HostManager, credentials: dict = {'user': 'admin', 'password': 'changeme'}, + index: str = 'wazuh-alerts*', new_max_result_window: int = 100000): + """Extend the max_result_window setting for a Wazuh Indexer index. + + Args: + host_manager: An instance of the HostManager class containing information about hosts. + credentials (Optional): A dictionary containing the Indexer credentials. Defaults to + {'user': 'admin', 'password': 'changeme'}. + index (Optional): The Indexer index name. Defaults to 'wazuh-alerts*'. + new_max_result_window (Optional): The new maximum result window size. Defaults to 100,000. + """ + logging.info(f"Extending max_result_window for {index} index to {new_max_result_window}") + + url = f"https://{host_manager.get_master_ip()}:9200/{index}/_settings" + headers = { + 'Content-Type': 'application/json', + } + data = { + "index": { + "max_result_window": new_max_result_window + } + } + + response = requests.put(url=url, json=data, verify=False, + auth=requests.auth.HTTPBasicAuth(credentials['user'], credentials['password']), + headers=headers) + + if response.status_code == 200: + logging.info(f"Successfully updated max_result_window for {index} index.") + else: + logging.error(f"Failed to update max_result_window for {index} index. Response: {response.text}") \ No newline at end of file From d543627880ddd563d83a7ab2f390bf66ebec082d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Tue, 3 Sep 2024 18:02:37 +0100 Subject: [PATCH 06/14] fix: revert error level change --- .../test_vulnerability_detector.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py b/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py index 1005f7f5c4..e769ac515a 100644 --- a/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py +++ b/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py @@ -380,7 +380,7 @@ def test_first_syscollector_scan( logging.critical("Checking for errors in the environment") unexpected_errors = check_errors_in_environment( - host_manager, expected_errors=VULNERABILITY_DETECTION_E2E_EXPECTED_ERRORS, error_levels=['ERROR', 'CRITICAL'] + host_manager, expected_errors=VULNERABILITY_DETECTION_E2E_EXPECTED_ERRORS ) test_result.validate_check( @@ -533,7 +533,7 @@ def test_syscollector_second_scan( unexpected_errors = check_errors_in_environment( host_manager, expected_errors=VULNERABILITY_DETECTION_E2E_EXPECTED_ERRORS, - greater_than_timestamp=get_timestamp, error_levels=['ERROR','CRITICAL'] + greater_than_timestamp=get_timestamp ) test_result.validate_check( @@ -760,7 +760,7 @@ def test_install_vulnerable_package_when_agent_down(self, host_manager, request, errors_environment = check_errors_in_environment( host_manager, expected_errors=VULNERABILITY_DETECTION_E2E_EXPECTED_ERRORS, - greater_than_timestamp=test_timestamp, error_levels=['ERROR','CRITICAL'] + greater_than_timestamp=test_timestamp ) test_result.validate_check("no_errors", [Evidence("error_level_messages", errors_environment)]) @@ -887,7 +887,7 @@ def test_change_agent_manager(self, permutate_agents_managers, request, precondi errors_environment = check_errors_in_environment( host_manager, expected_errors=VULNERABILITY_DETECTION_E2E_EXPECTED_ERRORS, - greater_than_timestamp=test_timestamp, error_levels=['ERROR','CRITICAL'] + greater_than_timestamp=test_timestamp ) test_result.validate_check("no_errors", [Evidence("error_level_messages", errors_environment)]) @@ -1018,7 +1018,7 @@ def test_vulnerability_detector_scans_cases(self, request, preconditions, body, errors_environment = check_errors_in_environment( host_manager, expected_errors=VULNERABILITY_DETECTION_E2E_EXPECTED_ERRORS, - greater_than_timestamp=test_timestamp, error_levels=['ERROR','CRITICAL'] + greater_than_timestamp=test_timestamp ) test_result.validate_check("no_errors", [Evidence("error_level_messages", errors_environment)]) From 11e783c36b83f72b5af4453cfba830070c912699 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Wed, 4 Sep 2024 10:46:11 +0100 Subject: [PATCH 07/14] docs: include 5712 changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 991299ed9f..5eb8fcbfc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ All notable changes to this project will be documented in this file. - Added support for macOS 14.6 to the Allocation module (Vagrant) ([#5671](https://github.com/wazuh/wazuh-qa/pull/5671)) \- (Framework) +### Fixed + +- Increase results windows in E2E Vulnerability detection ([#5712](https://github.com/wazuh/wazuh-qa/pull/5712/)) \- (Framework + Tests) + ## [4.9.0] - TBD ### Added From 50b5adf29605c64653002ce7f2022e868cd96157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Wed, 4 Sep 2024 11:06:27 +0100 Subject: [PATCH 08/14] docs: update docstring with size parameter --- .../wazuh_testing/end_to_end/vulnerability_detector.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py b/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py index 1e0c8eea04..adf303c63d 100644 --- a/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py +++ b/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py @@ -283,6 +283,8 @@ def get_vulnerabilities_from_states_by_agent(host_manager: HostManager, agents: host_manager (HostManager): Host manager object. agents (list): List of agents. greater_than_timestamp (str, optional): Greater than timestamp. Defaults to None. + size (int, optional): Maximun number of vulnerabilities to collect. + More information in https://opensearch.org/docs/latest/search-plugins/searching-data/paginate Returns: dict: Dictionary of vulnerabilities by agent. From 842bbec49b140bd493c5a407ca8386f12e78c9e4 Mon Sep 17 00:00:00 2001 From: fcaffieri Date: Thu, 5 Sep 2024 18:07:00 -0300 Subject: [PATCH 09/14] Remove cleanup.yml --- deployability/modules/testing/main.py | 1 - deployability/modules/testing/models.py | 1 - .../modules/testing/playbooks/cleanup.yml | 11 ------- deployability/modules/testing/testing.py | 32 +++---------------- 4 files changed, 5 insertions(+), 40 deletions(-) delete mode 100644 deployability/modules/testing/playbooks/cleanup.yml diff --git a/deployability/modules/testing/main.py b/deployability/modules/testing/main.py index 2c9988971d..2c7520838b 100755 --- a/deployability/modules/testing/main.py +++ b/deployability/modules/testing/main.py @@ -18,7 +18,6 @@ def parse_arguments(): parser.add_argument("--tests", required=True) parser.add_argument("--component", choices=['manager', 'agent', 'central_components'], required=True) parser.add_argument("--dependencies", action='append', default=[], required=False) - parser.add_argument("--cleanup", required=False, default=True) parser.add_argument("--wazuh-version", required=True) parser.add_argument("--wazuh-revision", required=True) parser.add_argument("--wazuh-branch", required=False) diff --git a/deployability/modules/testing/models.py b/deployability/modules/testing/models.py index 522ce9e07b..d8d6fc0c5e 100644 --- a/deployability/modules/testing/models.py +++ b/deployability/modules/testing/models.py @@ -19,7 +19,6 @@ class InputPayload(ExtraVars): tests: list[str] targets: list[str] dependencies: list[str] | None = None - cleanup: bool = True live: bool = False diff --git a/deployability/modules/testing/playbooks/cleanup.yml b/deployability/modules/testing/playbooks/cleanup.yml deleted file mode 100644 index 880d28c86e..0000000000 --- a/deployability/modules/testing/playbooks/cleanup.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (C) 2015, Wazuh Inc. -# Created by Wazuh, Inc. . -# This program is a free software; you can redistribute it and/or modify it under the terms of GPLv2 - -- hosts: localhost - become: true - tasks: - - name: Clean test directory - file: - path: "{{ working_dir }}" - state: absent diff --git a/deployability/modules/testing/testing.py b/deployability/modules/testing/testing.py index 0522ae4ef6..67aee5632c 100644 --- a/deployability/modules/testing/testing.py +++ b/deployability/modules/testing/testing.py @@ -15,7 +15,6 @@ class Tester: _playbooks_dir = Path(__file__).parent / 'playbooks' _setup_playbook = _playbooks_dir / 'setup.yml' - _cleanup_playbook = _playbooks_dir / 'cleanup.yml' _test_template = _playbooks_dir / 'test.yml' @classmethod @@ -60,14 +59,6 @@ def run(cls, payload: InputPayload) -> None: cls._setup(ansible, extra_vars) cls._run_tests(payload.tests, ansible, extra_vars) - # Clean up if required - if payload.cleanup: - for target_path in payload.targets: - target_value = eval(target_path).values() - target_inventory = Inventory(**Utils.load_from_yaml(str(list(target_value)[0]))) - logger.info("Cleaning up") - cls._cleanup(ansible, extra_vars['working_dir']) - @classmethod def _get_extra_vars(cls, payload: InputPayload) -> ExtraVars: """ @@ -96,13 +87,13 @@ def _run_tests(cls, test_list: list[str], ansible: Ansible, extra_vars: ExtraVar rendering_var = {**extra_vars, 'test': test} template = str(cls._test_template) result = ansible.run_playbook(template, rendering_var) + for event in result.events: - logger.info(f"{event['stdout']}") - if result.stats["failures"]: - for event in result.events: + if result.stats["failures"]: if "fatal" in event['stdout']: - raise Exception(f"Test {test} failed with error") - + raise Exception(f"Test {test} failed with error: {event['stdout']}") + else: + logger.info(f"Test {test} Finished with: {event['stdout']}") @classmethod def _setup(cls, ansible: Ansible, extra_vars: ExtraVars) -> None: @@ -120,16 +111,3 @@ def _setup(cls, ansible: Ansible, extra_vars: ExtraVars) -> None: if "fatal" in event['stdout']: raise Exception(f"Setup {template} failed with error: {event['stdout']}") - - @classmethod - def _cleanup(cls, ansible: Ansible, remote_working_dir: str = '/tmp') -> None: - """ - Cleanup the environment after the tests. - - Args: - ansible (Ansible): The Ansible object to run the cleanup. - remote_working_dir (str): The remote working directory. - """ - extra_vars = {'working_dir': remote_working_dir} - playbook = str(cls._cleanup_playbook) - ansible.run_playbook(playbook, extra_vars) From 9e388bdc489d69b08ca5c2c59675d23ecd997974 Mon Sep 17 00:00:00 2001 From: GGP1 Date: Tue, 10 Sep 2024 11:39:41 -0300 Subject: [PATCH 10/14] fix: Remove change log --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d155161e1..89a80833ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,7 +102,6 @@ s - Added tests for checking agent status upon ungraceful closure.([#4146](https://github.com/wazuh/wazuh-qa/pull/4146)) \- (Tests) - Agent syncronization testing after group deleting ([#4143](https://github.com/wazuh/wazuh-qa/pull/4143)) \- (Tests) - Add test for AWS Custom Logs. ([#4675](https://github.com/wazuh/wazuh-qa/pull/4675)) \- (Tests) -- Add new behaviour for endpoints marked as xfail in api_endpoints_performance test ([#4657](https://github.com/wazuh/wazuh-qa/pull/4657)) \ (Tests) ### Changed From c843dd6f9de4d4d73aa0fabeb88e251a9074c0be Mon Sep 17 00:00:00 2001 From: GGP1 Date: Tue, 10 Sep 2024 11:41:05 -0300 Subject: [PATCH 11/14] fix: Revert test changes --- .../test_api_endpoints_performance.py | 23 +++++-------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/tests/performance/test_api/test_api_endpoints_performance.py b/tests/performance/test_api/test_api_endpoints_performance.py index 4185be1a91..629c596e2d 100755 --- a/tests/performance/test_api/test_api_endpoints_performance.py +++ b/tests/performance/test_api/test_api_endpoints_performance.py @@ -2,7 +2,6 @@ from os.path import join, dirname, realpath from time import sleep -import warnings import pytest import requests from yaml import safe_load @@ -33,6 +32,12 @@ def test_api_endpoints(test_case, set_api_test_environment, api_healthcheck): set_api_test_environment (fixture): Fixture that modifies the API security options. api_healthcheck (fixture): Fixture used to check that the API is ready to respond requests. """ + # Apply xfails + if test_case['endpoint'] in xfailed_items.keys() and \ + test_case['method'] == xfailed_items[test_case['endpoint']]['method']: + pytest.xfail(xfailed_items[test_case['endpoint']]['message']) + + base_url = api_details['base_url'] headers = api_details['auth_headers'] response = None @@ -44,21 +49,6 @@ def test_api_endpoints(test_case, set_api_test_environment, api_healthcheck): assert response.status_code == 200 assert response.json()['error'] == 0 - except AssertionError as e: - # If the assertion fails, and is marked as xfail - if test_case['endpoint'] in xfailed_items.keys() and \ - test_case['method'] == xfailed_items[test_case['endpoint']]['method']: - pytest.xfail(xfailed_items[test_case['endpoint']]['message']) - - raise e - - else: - # If the test does not fail and is marked as xfail, issue a warning - if test_case['endpoint'] in xfailed_items.keys() and \ - test_case['method'] == xfailed_items[test_case['endpoint']]['method']: - warnings.warn(f"Test {test_case['endpoint']} should have failed due " - f"to {xfailed_items[test_case['endpoint']]['message']}") - finally: # Add useful information to report as stdout try: @@ -68,6 +58,5 @@ def test_api_endpoints(test_case, set_api_test_environment, api_healthcheck): except KeyError: print('No response available') - # Restart logic as before if test_case['method'] == 'put' and test_case['restart']: sleep(restart_delay) From 595a2479093b4bb3efe180d340886406cefa87a0 Mon Sep 17 00:00:00 2001 From: c-bordon Date: Wed, 11 Sep 2024 13:24:31 -0300 Subject: [PATCH 12/14] Updated Debian 12 AMIs and Box to 12.7 version --- deployability/modules/allocation/static/specs/os.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deployability/modules/allocation/static/specs/os.yml b/deployability/modules/allocation/static/specs/os.yml index 6a351b744f..4db94ad76d 100644 --- a/deployability/modules/allocation/static/specs/os.yml +++ b/deployability/modules/allocation/static/specs/os.yml @@ -35,7 +35,7 @@ vagrant: virtualizer: virtualbox linux-debian-12-amd64: box: debian/bookworm64 - box_version: 12.20231211.1 + box_version: 12.20240905.1 virtualizer: virtualbox # Oracle Linux linux-oracle-7-amd64: @@ -268,11 +268,11 @@ aws: zone: us-east-1 user: admin linux-debian-12-amd64: - ami: ami-055c8118725fe3a84 + ami: ami-014124f30c18be425 zone: us-east-1 user: admin linux-debian-12-arm64: - ami: ami-06703877c23c4ddf1 + ami: ami-027a194fc587a2e82 zone: us-east-1 user: admin # Oracle Linux @@ -422,7 +422,7 @@ aws: zone: us-east-1 user: ec2-user macos-ventura-13-arm64: - ami: ami-01aa3973cdaf40134 + ami: ami-01aa3973cdaf40134 zone: us-east-1 user: ec2-user macos-sonoma-14-amd64: From cb27e54f457ea8941e9ebf1984329c7522c84813 Mon Sep 17 00:00:00 2001 From: c-bordon Date: Wed, 11 Sep 2024 13:30:20 -0300 Subject: [PATCH 13/14] Changelog updated --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d155161e1..201f98b2cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ All notable changes to this project will be documented in this file. ### Changed +- Updated Debian 12 AMIs and Box to 12.7 version ([#5735](https://github.com/wazuh/wazuh-qa/pull/5735)) \- (Framework) - Increase Feed update timeout in waiters.py ([#5668](https://github.com/wazuh/wazuh-qa/pull/5668)) \- (Framework) - Set `/active-response` as xfail ([#5660](https://github.com/wazuh/wazuh-qa/pull/5660)) \- (Tests) - Modify the directory name for machines deployed in AWS ([#5635](https://github.com/wazuh/wazuh-qa/pull/5635)) \- (Framework) From 5133cc42ed94ec4b62db7be1606e56f9d5c12ed0 Mon Sep 17 00:00:00 2001 From: GGP1 Date: Fri, 13 Sep 2024 08:33:21 -0300 Subject: [PATCH 14/14] fix: Update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89a80833ea..928e907120 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ All notable changes to this project will be documented in this file. - Increase results windows in E2E Vulnerability detection ([#5712](https://github.com/wazuh/wazuh-qa/pull/5712/)) \- (Framework + Tests) +### Deleted + +- Reverted an xfail behaviour change in the API performance test ([#5734](https://github.com/wazuh/wazuh-qa/pull/5734)) \ (Tests) + ## [4.9.0] - TBD ### Added @@ -102,6 +106,7 @@ s - Added tests for checking agent status upon ungraceful closure.([#4146](https://github.com/wazuh/wazuh-qa/pull/4146)) \- (Tests) - Agent syncronization testing after group deleting ([#4143](https://github.com/wazuh/wazuh-qa/pull/4143)) \- (Tests) - Add test for AWS Custom Logs. ([#4675](https://github.com/wazuh/wazuh-qa/pull/4675)) \- (Tests) +- Add new behaviour for endpoints marked as xfail in api_endpoints_performance test ([#4657](https://github.com/wazuh/wazuh-qa/pull/4657)) \ (Tests) ### Changed