From 8001fc7ba2c83926b4b26b1821bdae4afc5c0d96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Mon, 4 Mar 2024 18:26:19 +0000 Subject: [PATCH 1/9] fix: remove VD consistency and alerts tests --- .../wazuh_testing/end_to_end/waiters.py | 2 +- .../test_vulnerability_detector.py | 274 ------------------ 2 files changed, 1 insertion(+), 275 deletions(-) diff --git a/deps/wazuh_testing/wazuh_testing/end_to_end/waiters.py b/deps/wazuh_testing/wazuh_testing/end_to_end/waiters.py index e2208ba93e..8d2be19bc4 100644 --- a/deps/wazuh_testing/wazuh_testing/end_to_end/waiters.py +++ b/deps/wazuh_testing/wazuh_testing/end_to_end/waiters.py @@ -31,7 +31,7 @@ from wazuh_testing.modules.syscollector import TIMEOUT_SYSCOLLECTOR_SHORT_SCAN -VD_FEED_UPDATE_TIMEOUT = 300 +VD_FEED_UPDATE_TIMEOUT = 600 VD_INITIAL_SCAN_PER_AGENT_TIMEOUT = 15 diff --git a/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py b/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py index 00bf1678fc..8983bc97e9 100644 --- a/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py +++ b/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py @@ -253,98 +253,6 @@ def test_syscollector_first_scan(self, request, host_manager, setup_vulnerabilit logger.critical("All agents has been scanned") - def test_syscollector_first_scan_alerts(self, request, host_manager, setup_vulnerability_tests, get_results): - """ - description: Validates that the Vulnerability Detector detects vulnerabilities within the environment in the - first scan. - - This test ensures that the Vulnerability Detector accurately detects vulnerabilities within the environment. - It is assumed that provided hosts will have at least one vulnerability. - - tier: 0 - - parameters: - - request: pytest request object - - host_manager: - type: fixture - brief: Get the host manager of the environment - - setup_vulnerability_tests: - type: fixture - brief: Setup the environment to proceed with the testing - - get_results: fixture to get the results of global class tests - - assertions: - - Verify that all agents has been scanned - - Verify that all agents has generated vulnerabilities - - cases: None - - tags: - - syscollector - - vulnerability_detector - """ - test_result = { - 'checks': { - 'all_successfull': True, - }, - 'evidences': { - 'agents_not_detected_alerts': [], - 'vulnerabilities_alerts_first_scan': [] - } - } - - results = get_results - test_name = request.node.name - - # Filter agents that has not been scanned - agents_to_check = [agent for agent in host_manager.get_group_hosts('agent') if agent not in - results['test_syscollector_first_scan']['evidences']['agents_not_scanned_first_scan']] - - if len(agents_to_check) == 0: - pytest.skip("Syscollector scan not started in any agent. Skipping test") - - # Wait until all agents has been scanned. Timeout: 60 seconds per agent - for agent in host_manager.get_group_hosts('agent'): - logger.critical(f"Waiting until agent {agent} has been scanned." - f"Waiting: {TIMEOUT_PER_AGENT_VULNERABILITY_SCAN}") - time.sleep(TIMEOUT_PER_AGENT_VULNERABILITY_SCAN) - - logger.critical("Check agent's vulnerabilities") - - vuln_by_agent_alerts = {} - for agent in agents_to_check: - agent_all_alerts = get_indexer_values(host_manager, - greater_than_timestamp=setup_vulnerability_tests, - agent=agent)['hits']['hits'] - - # Only is expected alert of affected vulnerabilities - vuln_by_agent_alerts[agent] = parse_vulnerability_detector_alerts(agent_all_alerts)['affected'] - - test_result['evidences']['vulnerabilities_alerts_first_scan'] = vuln_by_agent_alerts - - # Check that it has been triggered vulnerability detector alerts - logger.critical("Checking that all agents has been scanned") - for agent in agents_to_check: - if agent not in list(vuln_by_agent_alerts.keys()) or \ - len(vuln_by_agent_alerts[agent]) == 0: - logger.critical(f"Agent {agent} has not been scanned. Continuing with remaining agents") - test_result['checks']['all_successfull'] = False - test_result['evidences']['agents_not_detected_alerts'].append(agent) - - results[test_name] = test_result - - # Store full alert list in global results. It is needed for the next test - results['vulnerabilities_alerts_first_scan'] = vuln_by_agent_alerts - - - if not test_result['checks']['all_successfull']: - logging_message = 'Some agents has not been scanned:' \ - f"{test_result['evidences']['agents_not_detected_alerts']}." - logger.critical(logging_message) - pytest.fail(logging_message) - else: - logger.critical("All agents has been scanned") - def test_syscollector_first_scan_index(self, request, host_manager, setup_vulnerability_tests, get_results): """ description: Validates that the Vulnerability Detector detects vulnerabilities within the environment in the @@ -429,74 +337,6 @@ def test_syscollector_first_scan_index(self, request, host_manager, setup_vulner logger.critical("All agents has been scanned and updated states index") - def tests_syscollector_vulnerabilities_index_alerts_consistency(self, request, - setup_vulnerability_tests, get_results): - """ - description: Ensure the consistency of the agent's vulnerabilities between the index and the alerts. - - This test ensure that alerts in the index are consistent with the alerts in the alerts index after the first - scan. - - tier: 0 - - parameters: - - request: pytest request object - - setup_vulnerability_tests: - type: fixture - brief: Setup the environment to proceed with the testing - - get_results: fixture to get the results of global class tests - - assertions: - - Verify that the index is consistent with the alerts - - cases: None - - tags: - - syscollector - - vulnerability_detector - """ - results = get_results - test_name = request.node.name - test_result = { - 'checks': { - 'all_successfull': True, - }, - 'evidences': { - 'alerts_not_in_states': [], - 'states_not_in_alerts': [], - 'alerts_first_scan': results['vulnerabilities_alerts_first_scan'], - 'states_first_scan': results['vulnerabilities_index_first_scan'] - } - } - - # Check that the index is consistent with the alerts - logging.critical("Checking index state consistency") - inconsistencies_between_alerts_indexer = \ - check_vuln_state_consistency(results['vulnerabilities_alerts_first_scan'], - results['vulnerabilities_index_first_scan']) - - test_result['evidences']['alerts_not_in_states'] = \ - inconsistencies_between_alerts_indexer['alerts_not_in_states'] - - test_result['evidences']['states_not_in_alerts'] = \ - inconsistencies_between_alerts_indexer['states_not_in_alerts'] - - if len(test_result['evidences']['alerts_not_in_states']) > 0 or \ - len(test_result['evidences']['states_not_in_alerts']) > 0: - logger.critical("Index state is not consistent with the alerts") - test_result['checks']['all_successfull'] = False - - results[test_name] = test_result - - if not test_result['checks']['all_successfull']: - logging_message = "Index state is not consistent with the alerts" - logger.critical(logging_message) - logger.critical(f"Alerts not in states: {test_result['evidences']['alerts_not_in_states']}") - logger.critical(f"States not in alerts: {test_result['evidences']['states_not_in_alerts']}") - pytest.fail(logging_message) - else: - logger.critical("Index state is consistent with the alerts") - def test_syscollector_second_scan(self, request, host_manager, setup_vulnerability_tests, get_results): """ description: Validates the initiation of the second Syscollector scans across all agents in the environment. @@ -570,120 +410,6 @@ def test_syscollector_second_scan(self, request, host_manager, setup_vulnerabili else: logger.critical("Syscollector scan started in all agents") - def tests_syscollector_first_second_scan_consistency_alerts(self, request, host_manager, setup_vulnerability_tests, - get_results): - """ - description: Ensure the consistency of the agent's vulnerabilities between the first and second scans. - - This test ensure that alerts in the first scan are consistent with the alerts in the second scan. - - tier: 0 - - parameters: - - request: pytest request object - - host_manager: - type: fixture - brief: Get the host manager of the environment - - setup_vulnerability_tests: - type: fixture - brief: Setup the environment to proceed with the testing - - get_results: fixture to get the results of global class tests - - assertions: - - Verify that the number of vulnerabilities is the same between scans - - cases: None - - tags: - - syscollector - - vulnerability_detector - """ - - test_name = request.node.name - results = get_results - request = request - test_result = { - 'checks': { - 'all_successfull': True, - }, - 'evidences': { - 'vulnerabilities_not_equal_between_scans_alerts': [], - 'agents_different_between_scans': [], - 'vulnerabilities_alerts_first_scan': results['vulnerabilities_alerts_first_scan'], - 'vulnerabilities_alerts_second_scan': [] - } - } - - # Filter agents that has not been scanned - agents_to_check = [agent for agent in host_manager.get_group_hosts('agent') if agent not in - results['test_syscollector_first_scan']['evidences']['agents_not_scanned_first_scan']] - - if len(agents_to_check) == 0: - pytest.skip("Syscollector scan not started in any agent. Skipping test") - - logger.critical("Waiting until agent's VD scan is over") - # Only wait for the first agent. Any possible discrepancy will be detected in this interval - time.sleep(TIMEOUT_PER_AGENT_VULNERABILITY_SCAN) - - logger.critical("Checking vulnerabilities in the second scan") - - vuln_by_agent_alert_second_scan = {} - for agent in agents_to_check: - agent_all_alerts = get_indexer_values(host_manager, - greater_than_timestamp=setup_vulnerability_tests, - agent=agent)['hits']['hits'] - # Only is expected alert of affected vulnerabilities - vuln_by_agent_alert_second_scan[agent] = parse_vulnerability_detector_alerts(agent_all_alerts)['affected'] - - # Store full alert list in global results. It is needed for the next test - results['vulnerabilities_alerts_second_scan'] = vuln_by_agent_alert_second_scan - test_result['evidences']['vulnerabilities_alerts_second_scan'] = vuln_by_agent_alert_second_scan - - alert_present_in_first_scan_not_in_second_scan = [] - alert_present_in_second_scan_not_in_first_scan = [] - - logger.critical("Checking that all agents has been scanned") - # Check if the number of agents for each scan is the same - if len(vuln_by_agent_alert_second_scan.keys()) != len(results['vulnerabilities_alerts_first_scan'].keys()): - test_result['checks']['all_successfull'] = False - logging.critical(f"Agents with vulnerabilities changed between scans: " - f"First scan: {list(results['vulnerabilities_alerts_first_scan'].keys())}" - f"Second scan: {list(vuln_by_agent_alert_second_scan.keys())}") - - test_result['evidences']['agents_different_between_scans'] = \ - list(set(list(results['vulnerabilities_alerts_first_scan'].keys())) ^ - set(list(results['vulnerabilities_alerts_second_scan'].keys()))) - - logger.critical("Checking the number of vulnerabilities for each agent") - for agent in agents_to_check: - if agent in list(results['vulnerabilities_alerts_first_scan'].keys()) and \ - len(results['vulnerabilities_alerts_first_scan'][agent]): - for alert in list(vuln_by_agent_alert_second_scan[agent][0]): - if alert not in results['vulnerabilities_alerts_first_scan'][agent][0]: - alert_present_in_second_scan_not_in_first_scan.append(alert) - - for alert in list(results['vulnerabilities_alerts_first_scan'][agent][0]): - if alert not in vuln_by_agent_alert_second_scan[agent][0]: - alert_present_in_first_scan_not_in_second_scan.append(alert) - - logger.critical("Checking that all agents has been scanned") - if alert_present_in_first_scan_not_in_second_scan or alert_present_in_second_scan_not_in_first_scan: - test_result['checks']['all_successfull'] = False - test_result['evidences']['vulnerabilities_not_equal_between_scans_alerts'] = { - 'alert_present_in_first_scan_not_in_second_scan': alert_present_in_first_scan_not_in_second_scan, - 'alert_present_in_second_scan_not_in_first_scan': alert_present_in_second_scan_not_in_first_scan - } - - results[test_name] = test_result - - if not test_result['checks']['all_successfull']: - logging_message = "Inconsistencies found between first and second scan." \ - "Check evidences for more information" - logger.critical(logging_message) - pytest.fail(logging_message) - else: - logger.critical("The number of vulnerabilities is the same between scans") - def tests_syscollector_first_second_scan_consistency_index(self, request, host_manager, setup_vulnerability_tests, get_results): From 6c24e5d900c32f7e4f2ffb391f0f9660c909b49f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Wed, 6 Mar 2024 19:07:42 +0000 Subject: [PATCH 2/9] style: minor style fix in conf module --- deps/wazuh_testing/wazuh_testing/end_to_end/configuration.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deps/wazuh_testing/wazuh_testing/end_to_end/configuration.py b/deps/wazuh_testing/wazuh_testing/end_to_end/configuration.py index 8e3b736ac4..ce3c0a980c 100644 --- a/deps/wazuh_testing/wazuh_testing/end_to_end/configuration.py +++ b/deps/wazuh_testing/wazuh_testing/end_to_end/configuration.py @@ -27,7 +27,7 @@ from wazuh_testing.tools.system import HostManager -def backup_configurations(host_manager: HostManager) -> Dict[str, str]: +def backup_configurations(host_manager: HostManager) -> Dict[str, List]: """ Backup configurations for all hosts in the specified host manager. @@ -45,6 +45,7 @@ def backup_configurations(host_manager: HostManager) -> Dict[str, str]: """ logging.info("Backing up configurations") backup_configurations = {} + for host in host_manager.get_group_hosts('all'): host_os_name = host_manager.get_host_variables(host)['os_name'] configuration_filepath = configuration_filepath_os[host_os_name] @@ -52,6 +53,7 @@ def backup_configurations(host_manager: HostManager) -> Dict[str, str]: backup_configurations[host] = host_manager.get_file_content(str(host), configuration_filepath) logging.info("Configurations backed up") + return backup_configurations From 84cc5bcc537995ddca81cac38c58f363bb977bf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Wed, 6 Mar 2024 19:08:27 +0000 Subject: [PATCH 3/9] feat: include logging in gathering evidences --- tests/end_to_end/test_vulnerability_detector/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/end_to_end/test_vulnerability_detector/conftest.py b/tests/end_to_end/test_vulnerability_detector/conftest.py index 3142d4a967..646ae67040 100644 --- a/tests/end_to_end/test_vulnerability_detector/conftest.py +++ b/tests/end_to_end/test_vulnerability_detector/conftest.py @@ -90,6 +90,7 @@ def collect_evidences(test_name, evidences) -> None: if evidences: logging.info(f"Collecting custom evidences for {test_name}") for evidence, content in evidences.items(): + logging.info(f"Collecting {evidence} for {test_name}") if content is not None and content != [] and content != {}: evidence_file = os.path.join(tests_evidences_directory, evidence + ".log") with open(evidence_file, 'w') as evidence_file: From c011ff12bd5eadb03836fe2d47ecb8f32768d3f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Wed, 6 Mar 2024 19:08:59 +0000 Subject: [PATCH 4/9] feat: change syscollector to 2 minutes --- .../test_vulnerability_detector/configurations/agent.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/end_to_end/test_vulnerability_detector/configurations/agent.yaml b/tests/end_to_end/test_vulnerability_detector/configurations/agent.yaml index 17626543de..b965697a4e 100644 --- a/tests/end_to_end/test_vulnerability_detector/configurations/agent.yaml +++ b/tests/end_to_end/test_vulnerability_detector/configurations/agent.yaml @@ -18,5 +18,5 @@ - disabled: value: 'no' - interval: - value: 1m + value: 2m From 01f90209d4d1207032d61f025724f978f4c9d99e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Wed, 6 Mar 2024 19:09:52 +0000 Subject: [PATCH 5/9] feat: change syscollector scan time --- .../wazuh_testing/modules/syscollector/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/wazuh_testing/wazuh_testing/modules/syscollector/__init__.py b/deps/wazuh_testing/wazuh_testing/modules/syscollector/__init__.py index 1ea8364666..ffd9e0d16e 100644 --- a/deps/wazuh_testing/wazuh_testing/modules/syscollector/__init__.py +++ b/deps/wazuh_testing/wazuh_testing/modules/syscollector/__init__.py @@ -1,4 +1,4 @@ -TIMEOUT_SYSCOLLECTOR_SCAN = 360 +TIMEOUT_SYSCOLLECTOR_SCAN = 130 TIMEOUT_SYSCOLLECTOR_SHORT_SCAN = 90 SYSCOLLECTOR_DELTA_EVENT_TYPES = ['packages', 'hotfix', 'hwinfo', 'ports', 'osinfo', 'network', 'process'] From 24f81b9ac67bbf3093b428f76afad7190f059454 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Wed, 6 Mar 2024 19:11:52 +0000 Subject: [PATCH 6/9] fix: consistency tests for second scan --- .../end_to_end/vulnerability_detector.py | 32 +++++++ .../test_vulnerability_detector.py | 91 +++++++++++++++---- 2 files changed, 103 insertions(+), 20 deletions(-) diff --git a/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py b/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py index 039dbfdf26..2ea390238f 100644 --- a/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py +++ b/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py @@ -25,6 +25,9 @@ from wazuh_testing.tools.system import HostManager from wazuh_testing.end_to_end.indexer_api import get_indexer_values from wazuh_testing.end_to_end.regex import REGEX_PATTERNS +from collections import namedtuple + +Vulnerability = namedtuple('Vulnerability', ['cve', 'package_name', 'package_version', 'type', 'architecture']) def load_packages_metadata() -> Dict: @@ -284,3 +287,32 @@ def check_vuln_state_consistency(vulnerabilities_alerts, vulnerabilities_states) 'alerts_not_in_states': alerts_not_in_states, 'states_not_in_alerts': states_not_in_alerts } + + +def get_vulnerabilities_from_states(vulnerabilities_states: List) -> List: + """ Get vulnerabilities from the vulnerability state index. + + Args: + vulnerabilities_states (list): List of vulnerabilities from the vulnerability state index. + + Returns: + list: List of vulnerabilities. + """ + vulnerabilities = [] + + for state_vulnerability in vulnerabilities_states: + try: + vulnerability = Vulnerability( + cve=state_vulnerability['_source']['vulnerability']['id'], + package_name=state_vulnerability['_source']['package']['name'], + package_version=state_vulnerability['_source']['package']['version'], + type=state_vulnerability['_source']['vulnerability']['type'] if 'type' in state_vulnerability['_source']['vulnerability'] else None, + architecture=state_vulnerability['_source']['vulnerability']['architecture'] if 'architecture' in state_vulnerability['_source']['vulnerability'] else None + ) + vulnerabilities.append(vulnerability) + except KeyError: + logging.error(f"Error parsing vulnerability: {state_vulnerability}") + + vulnerabilities = sorted(vulnerabilities, key=lambda x: (x.cve, x.package_name, x.package_version, x.architecture)) + + return vulnerabilities diff --git a/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py b/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py index 8983bc97e9..17c84941e1 100644 --- a/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py +++ b/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py @@ -15,11 +15,11 @@ The verification of vulnerabilities is conducted through Vulnerabilities Index and API endpoint Additionally, the tests ensure the consistency of these values. -Tests: - - test_syscollector_initial_agent_scan: - Validates the initiation of Syscollector scans across all agents in the environment. - Subsequently, it ensures that the Vulnerability Detector detects vulnerabilities within the environment. - The Agent's Vulnerability Indexer index is expected to be updated with the detected vulnerabilities. +Tests: + - TestInitialScans: Validates the initiation of Syscollector scans across all agents in the environment. + - test_syscollector_first_scan: Validates the initiation of the first Syscollector scans across all agents in the environment. + - test_syscollector_first_scan_index: Validates that the Vulnerability Detector detects vulnerabilities within the environment in the first scan in the index. + Issue: https://github.com/wazuh/wazuh-qa/issues/4369 @@ -55,8 +55,7 @@ from wazuh_testing.tools.configuration import load_configuration_template from wazuh_testing.tools.system import HostManager from wazuh_testing.end_to_end.remote_operations_handler import launch_parallel_operations -from wazuh_testing.end_to_end.vulnerability_detector import parse_vulnerability_detector_alerts, \ - check_vuln_state_consistency +from wazuh_testing.end_to_end.vulnerability_detector import get_vulnerabilities_from_states from wazuh_testing.modules.syscollector import TIMEOUT_SYSCOLLECTOR_SCAN @@ -177,6 +176,7 @@ def setup_vulnerability_tests(host_manager: HostManager) -> Generator: host_manager.control_environment('restart', ['agent']) host_manager.control_environment('restart', ['manager']) + logger.error("End") @pytest.mark.filterwarnings('ignore::urllib3.exceptions.InsecureRequestWarning') class TestInitialScans(): @@ -306,11 +306,15 @@ def test_syscollector_first_scan_index(self, request, host_manager, setup_vulner logger.critical("Checking vulnerabilities in the index") vuln_by_agent_index = {} + time.sleep(TIMEOUT_PER_AGENT_VULNERABILITY_SCAN * len(agents_to_check)) + for agent in agents_to_check: agent_all_vulnerabilities = get_indexer_values(host_manager, greater_than_timestamp=setup_vulnerability_tests, agent=agent, index='wazuh-states-vulnerabilities',)['hits']['hits'] vuln_by_agent_index[agent] = agent_all_vulnerabilities + # Substract agent2 key - TESTING CHANGE + agent2_vulnerabilities = vuln_by_agent_index.pop('agent2', None) test_result['evidences']['vulnerabilities_index_first_scan'] = vuln_by_agent_index @@ -390,6 +394,7 @@ def test_syscollector_second_scan(self, request, host_manager, setup_vulnerabili host_manager.get_group_hosts('agent'), 2) monitoring_results = monitoring_events_multihost(host_manager, monitoring_data) + logger.critical(f"Value of monitoring results is: {monitoring_results}") logger.critical("Checking that all agents has been scanned") for agent in monitoring_results: @@ -447,9 +452,12 @@ def tests_syscollector_first_second_scan_consistency_index(self, request, host_m 'all_successfull': True, }, 'evidences': { - 'vulnerabilities_not_equal_between_scans_indexer': [], 'vulnerabilities_index_first_scan': results['vulnerabilities_index_first_scan'], - 'vulnerabilities_index_second_scan': [] + 'vulnerabilities_index_second_scan': [], + 'vulnerabilities_not_found_in_second_scan': [], + 'vulnerabilities_not_found_in_first_scan': [], + 'agent_not_found_in_first_scan': [], + 'agent_not_found_in_second_scan': [] } } @@ -464,22 +472,65 @@ def tests_syscollector_first_second_scan_consistency_index(self, request, host_m # Only is expected alert of affected vulnerabilities vuln_by_agent_index_second_scan[agent] = agent_all_vulnerabilities - results['vulnerabilities_index_second_scan'] = vuln_by_agent_index_second_scan + # TESTING CHANGE + vuln_by_agent_index_second_scan['agent1'] = vuln_by_agent_index_second_scan['agent1'][2:] + test_result['evidences']['vulnerabilities_index_second_scan'] = vuln_by_agent_index_second_scan - differences = list(set(results['vulnerabilities_index_first_scan']).symmetric_difference( - set(results['vulnerabilities_index_second_scan']))) + # Calculate differences between first and second scan + agent_not_found_in_first_scan = list(set(vuln_by_agent_index_second_scan.keys()) - set(results['vulnerabilities_index_first_scan'].keys())) + agent_not_found_in_second_scan = list(set(results['vulnerabilities_index_first_scan'].keys()) - set(vuln_by_agent_index_second_scan.keys())) - results[test_name] = test_result + agent_found_in_all_scans = set(vuln_by_agent_index_second_scan.keys()) & set(results['vulnerabilities_index_first_scan'].keys()) + + vulnerabilities_not_found_in_first_scan = {} + vulnerabilities_not_found_in_second_scan = {} + + for agent in agent_found_in_all_scans: + vulnerabilities_second_scan = get_vulnerabilities_from_states(vuln_by_agent_index_second_scan[agent]) + vulnerabilities_first_scan = get_vulnerabilities_from_states(results['vulnerabilities_index_first_scan'][agent]) + + vulnerabilities_not_found_second_scan = list(set(vulnerabilities_first_scan) - set(vulnerabilities_second_scan)) + vulnerabilities_not_found_first_scan = list(set(vulnerabilities_second_scan) - set(vulnerabilities_first_scan)) - if differences: + # Change to dict to be able to serialize + vulnerabilities_not_found_first_scan = [vuln._asdict() for vuln in vulnerabilities_not_found_first_scan] + vulnerabilities_not_found_second_scan = [vuln._asdict() for vuln in vulnerabilities_not_found_second_scan] + + if len(vulnerabilities_not_found_second_scan) > 0: + vulnerabilities_not_found_in_second_scan[agent] = vulnerabilities_not_found_second_scan + if len(vulnerabilities_not_found_first_scan) > 0: + vulnerabilities_not_found_in_first_scan[agent] = vulnerabilities_not_found_first_scan + + # Check if agents are the same in both scans + if len(agent_found_in_all_scans) != len(vuln_by_agent_index_second_scan) != len(results['vulnerabilities_index_first_scan']): test_result['checks']['all_successfull'] = False - results[test_name]['evidences']['vulnerabilities_not_equal_between_scans_indexer'] = differences - logging.critical("Inconsistencies found between first and second scan in the index." - f"Check evidences for more information: {differences}") - pytest.fail('The number of vulnerabilities is not the same between scans') - else: - logger.critical("The number of vulnerabilities is the same between scans") + logging.critical("Inconsistencies found between first and second scan in the index. Different agents found") + if len(agent_not_found_in_first_scan) > 0: + logging.critical(f"Agents not found in first scan: {agent_not_found_in_first_scan}") + test_result['evidences']['agent_not_found_in_first_scan'] = agent_not_found_in_first_scan + if len(agent_not_found_in_second_scan) > 0: + logging.critical(f"Agents not found in second scan: {agent_not_found_in_second_scan}") + test_result['evidences']['agent_not_found_in_second_scan'] = agent_not_found_in_second_scan + + # Check if vulnerabilities are the same in both scans + if vulnerabilities_not_found_in_first_scan or vulnerabilities_not_found_in_second_scan: + test_result['checks']['all_successfull'] = False + if vulnerabilities_not_found_in_first_scan: + logging.critical(f"Vulnerabilities not found in first scan: {vulnerabilities_not_found_in_first_scan}") + test_result['evidences']['vulnerabilities_not_found_in_first_scan'] = vulnerabilities_not_found_in_first_scan + + if vulnerabilities_not_found_in_second_scan: + logging.critical(f"Vulnerabilities not found in second scan: {vulnerabilities_not_found_in_second_scan}") + test_result['evidences']['vulnerabilities_not_found_in_second_scan'] = vulnerabilities_not_found_in_second_scan + + import pdb; pdb.set_trace() + results[test_name] = test_result + + if not test_result['checks']['all_successfull']: + logging_message = "Inconsistencies found between first and second scan in the index. Check evidences for more information" + logger.critical(logging_message) + pytest.fail(logging_message) # ------------------------- From a1ebdb29ec92b19252a5fb8551820e82cd534340 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Wed, 6 Mar 2024 19:26:43 +0000 Subject: [PATCH 7/9] fix: parsing of vulnerabilities --- .../wazuh_testing/end_to_end/vulnerability_detector.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py b/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py index 2ea390238f..675a0c9bf6 100644 --- a/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py +++ b/deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py @@ -27,6 +27,7 @@ from wazuh_testing.end_to_end.regex import REGEX_PATTERNS from collections import namedtuple + Vulnerability = namedtuple('Vulnerability', ['cve', 'package_name', 'package_version', 'type', 'architecture']) @@ -290,13 +291,13 @@ def check_vuln_state_consistency(vulnerabilities_alerts, vulnerabilities_states) def get_vulnerabilities_from_states(vulnerabilities_states: List) -> List: - """ Get vulnerabilities from the vulnerability state index. + """Parse vulnerabilities from the vulnerability state index. Args: vulnerabilities_states (list): List of vulnerabilities from the vulnerability state index. Returns: - list: List of vulnerabilities. + list: List of vulnerabilities sorted by cve, package_name, package_version, and architecture. """ vulnerabilities = [] @@ -306,8 +307,8 @@ def get_vulnerabilities_from_states(vulnerabilities_states: List) -> List: cve=state_vulnerability['_source']['vulnerability']['id'], package_name=state_vulnerability['_source']['package']['name'], package_version=state_vulnerability['_source']['package']['version'], - type=state_vulnerability['_source']['vulnerability']['type'] if 'type' in state_vulnerability['_source']['vulnerability'] else None, - architecture=state_vulnerability['_source']['vulnerability']['architecture'] if 'architecture' in state_vulnerability['_source']['vulnerability'] else None + type=state_vulnerability['_source']['pacakge']['type'] if 'type' in state_vulnerability['_source']['vulnerability'] else None, + architecture=state_vulnerability['_source']['package']['architecture'] if 'architecture' in state_vulnerability['_source']['vulnerability'] else None ) vulnerabilities.append(vulnerability) except KeyError: From b6954c9539e90c914fcbef972955c3e1dd70431e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Wed, 6 Mar 2024 19:31:04 +0000 Subject: [PATCH 8/9] docs: complete list of tests in module docstring --- .../test_vulnerability_detector.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py b/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py index 17c84941e1..b335693fc6 100644 --- a/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py +++ b/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py @@ -19,7 +19,9 @@ - TestInitialScans: Validates the initiation of Syscollector scans across all agents in the environment. - test_syscollector_first_scan: Validates the initiation of the first Syscollector scans across all agents in the environment. - test_syscollector_first_scan_index: Validates that the Vulnerability Detector detects vulnerabilities within the environment in the first scan in the index. - + - test_syscollector_second_scan: Validates the initiation of the second Syscollector scans across all agents in the environment. + - tests_syscollector_first_second_scan_consistency_index: Ensure the consistency of the agent's vulnerabilities between the first and second scans in index. + - TestScanSyscollectorCases: Validates the Vulnerability Detector's ability to detect new vulnerabilities in the environment. Issue: https://github.com/wazuh/wazuh-qa/issues/4369 From 538c1ec4b186744e22569c6fd8d96c481083e7d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rebollo=20P=C3=A9rez?= Date: Wed, 6 Mar 2024 19:34:15 +0000 Subject: [PATCH 9/9] fix: remove changes to force failure of tests --- .../test_vulnerability_detector.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py b/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py index b335693fc6..3756bc15a2 100644 --- a/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py +++ b/tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py @@ -178,7 +178,6 @@ def setup_vulnerability_tests(host_manager: HostManager) -> Generator: host_manager.control_environment('restart', ['agent']) host_manager.control_environment('restart', ['manager']) - logger.error("End") @pytest.mark.filterwarnings('ignore::urllib3.exceptions.InsecureRequestWarning') class TestInitialScans(): @@ -315,9 +314,6 @@ def test_syscollector_first_scan_index(self, request, host_manager, setup_vulner vuln_by_agent_index[agent] = agent_all_vulnerabilities - # Substract agent2 key - TESTING CHANGE - agent2_vulnerabilities = vuln_by_agent_index.pop('agent2', None) - test_result['evidences']['vulnerabilities_index_first_scan'] = vuln_by_agent_index logger.critical("Checking that all agents has been scanned and generated vulnerabilities in the index") @@ -474,9 +470,6 @@ def tests_syscollector_first_second_scan_consistency_index(self, request, host_m # Only is expected alert of affected vulnerabilities vuln_by_agent_index_second_scan[agent] = agent_all_vulnerabilities - # TESTING CHANGE - vuln_by_agent_index_second_scan['agent1'] = vuln_by_agent_index_second_scan['agent1'][2:] - test_result['evidences']['vulnerabilities_index_second_scan'] = vuln_by_agent_index_second_scan # Calculate differences between first and second scan @@ -491,7 +484,8 @@ def tests_syscollector_first_second_scan_consistency_index(self, request, host_m for agent in agent_found_in_all_scans: vulnerabilities_second_scan = get_vulnerabilities_from_states(vuln_by_agent_index_second_scan[agent]) vulnerabilities_first_scan = get_vulnerabilities_from_states(results['vulnerabilities_index_first_scan'][agent]) - + + # Calculate differences between first and second scan vulnerabilities_not_found_second_scan = list(set(vulnerabilities_first_scan) - set(vulnerabilities_second_scan)) vulnerabilities_not_found_first_scan = list(set(vulnerabilities_second_scan) - set(vulnerabilities_first_scan)) @@ -526,7 +520,6 @@ def tests_syscollector_first_second_scan_consistency_index(self, request, host_m logging.critical(f"Vulnerabilities not found in second scan: {vulnerabilities_not_found_in_second_scan}") test_result['evidences']['vulnerabilities_not_found_in_second_scan'] = vulnerabilities_not_found_in_second_scan - import pdb; pdb.set_trace() results[test_name] = test_result if not test_result['checks']['all_successfull']: