Merge branch '4.9.1' into bug/5698-include-timeout-prevent-network-er…

…rors

CHANGELOG.md

@@ -10,11 +10,17 @@ All notable changes to this project will be documented in this file.
 
 ### Fixed
 - Fix unexpected warnings in E2E vulnerability detection tests ([#5711](https://github.com/wazuh/wazuh-qa/pull/5711/)) \- (Tests)
+- Increase results windows in E2E Vulnerability detection ([#5712](https://github.com/wazuh/wazuh-qa/pull/5712/)) \- (Framework + Tests)
+
+### Deleted
+
+- Reverted an xfail behaviour change in the API performance test ([#5734](https://github.com/wazuh/wazuh-qa/pull/5734)) \ (Tests)
 
 ## [4.9.0] - TBD
 
 ### Added
 
+- Adding jobflow templates ([#5680](https://github.com/wazuh/wazuh-qa/pull/5680/)) \- (Framework)
 - Add RockyLinux 8.10 to Allocator module ([#5524](https://github.com/wazuh/wazuh-qa/pull/5524)) \- (Framework)
 - Add Deployability testing tier 1 ([#5190](https://github.com/wazuh/wazuh-qa/pull/5190)) \- (Framework + Documentation + Tests)
 - Add Workflow module to Wazuh-qa repository ([#4990](https://github.com/wazuh/wazuh-qa/pull/4990)) \- (Tests)
@@ -23,6 +29,7 @@ All notable changes to this project will be documented in this file.
 
 ### Changed
 
+- Updated Debian 12 AMIs and Box to 12.7 version ([#5735](https://github.com/wazuh/wazuh-qa/pull/5735)) \- (Framework)
 - Increase Feed update timeout in waiters.py ([#5668](https://github.com/wazuh/wazuh-qa/pull/5668)) \- (Framework)
 - Set `/active-response` as xfail ([#5660](https://github.com/wazuh/wazuh-qa/pull/5660)) \- (Tests)
 - Modify the directory name for machines deployed in AWS ([#5635](https://github.com/wazuh/wazuh-qa/pull/5635)) \- (Framework)
@@ -41,6 +48,7 @@ All notable changes to this project will be documented in this file.
 
 ### Fixed
 
+- Fix playbook cleanup.yaml ([#5672](https://github.com/wazuh/wazuh-qa/pull/5672)) \- (Tests)
 - Fixed unnecesary reference to debian file in dashboard provisioning task ([#5643](https://github.com/wazuh/wazuh-qa/pull/5643)) \- (Framework)
 - Changed 'Ensure that the manager version is' expected warning to an agnostic version of regex ([#5630](https://github.com/wazuh/wazuh-qa/pull/5630)) \- (Tests)
 - Adding fixed and dynamic waits to port status checks ([#5627](https://github.com/wazuh/wazuh-qa/pull/5627)) (Framework)

deployability/modules/allocation/static/specs/os.yml

@@ -35,7 +35,7 @@ vagrant:
     virtualizer: virtualbox
   linux-debian-12-amd64:
     box: debian/bookworm64
-    box_version: 12.20231211.1
+    box_version: 12.20240905.1
     virtualizer: virtualbox
   # Oracle Linux
   linux-oracle-7-amd64:
@@ -268,11 +268,11 @@ aws:
     zone: us-east-1
     user: admin
   linux-debian-12-amd64:
-    ami: ami-055c8118725fe3a84
+    ami: ami-014124f30c18be425
     zone: us-east-1
     user: admin
   linux-debian-12-arm64:
-    ami: ami-06703877c23c4ddf1
+    ami: ami-027a194fc587a2e82
     zone: us-east-1
     user: admin
   # Oracle Linux
@@ -422,7 +422,7 @@ aws:
     zone: us-east-1
     user: ec2-user
   macos-ventura-13-arm64:
-    ami: ami-01aa3973cdaf40134 
+    ami: ami-01aa3973cdaf40134
     zone: us-east-1
     user: ec2-user
   macos-sonoma-14-amd64:

deployability/modules/testing/main.py

@@ -18,7 +18,6 @@ def parse_arguments():
     parser.add_argument("--tests", required=True)
     parser.add_argument("--component", choices=['manager', 'agent', 'central_components'], required=True)
     parser.add_argument("--dependencies", action='append', default=[], required=False)
-    parser.add_argument("--cleanup", required=False, default=True)
     parser.add_argument("--wazuh-version", required=True)
     parser.add_argument("--wazuh-revision", required=True)
     parser.add_argument("--wazuh-branch", required=False)

deployability/modules/testing/models.py

@@ -19,7 +19,6 @@ class InputPayload(ExtraVars):
     tests: list[str]
     targets: list[str]
     dependencies: list[str] | None = None
-    cleanup: bool = True
     live: bool = False
 
 

deployability/modules/testing/testing.py

@@ -15,7 +15,6 @@
 class Tester:
     _playbooks_dir = Path(__file__).parent / 'playbooks'
     _setup_playbook = _playbooks_dir / 'setup.yml'
-    _cleanup_playbook = _playbooks_dir / 'cleanup.yml'
     _test_template = _playbooks_dir / 'test.yml'
 
     @classmethod
@@ -60,14 +59,6 @@ def run(cls, payload: InputPayload) -> None:
         cls._setup(ansible, extra_vars)
         cls._run_tests(payload.tests, ansible, extra_vars)
 
-        # Clean up if required
-        if payload.cleanup:
-            for target_path in payload.targets:
-                target_value = eval(target_path).values()
-                target_inventory = Inventory(**Utils.load_from_yaml(str(list(target_value)[0])))
-                logger.info("Cleaning up")
-                cls._cleanup(ansible, extra_vars['working_dir'])
-
     @classmethod
     def _get_extra_vars(cls, payload: InputPayload) -> ExtraVars:
         """
@@ -96,13 +87,13 @@ def _run_tests(cls, test_list: list[str], ansible: Ansible, extra_vars: ExtraVar
             rendering_var = {**extra_vars, 'test': test}
             template = str(cls._test_template)
             result = ansible.run_playbook(template, rendering_var)
+
             for event in result.events:
-                logger.info(f"{event['stdout']}")
-            if result.stats["failures"]:
-                for event in result.events:
+                if result.stats["failures"]:
                     if "fatal" in event['stdout']:
-                        raise Exception(f"Test {test} failed with error")
-
+                        raise Exception(f"Test {test} failed with error: {event['stdout']}")
+                else:
+                    logger.info(f"Test {test} Finished with: {event['stdout']}")
 
     @classmethod
     def _setup(cls, ansible: Ansible, extra_vars: ExtraVars) -> None:
@@ -120,16 +111,3 @@ def _setup(cls, ansible: Ansible, extra_vars: ExtraVars) -> None:
                 if "fatal" in event['stdout']:
                     raise Exception(f"Setup {template} failed with error: {event['stdout']}")
 
-
-    @classmethod
-    def _cleanup(cls, ansible: Ansible, remote_working_dir: str = '/tmp') -> None:
-        """
-        Cleanup the environment after the tests.
-
-        Args:
-            ansible (Ansible): The Ansible object to run the cleanup.
-            remote_working_dir (str): The remote working directory.
-        """
-        extra_vars = {'working_dir': remote_working_dir}
-        playbook = str(cls._cleanup_playbook)
-        ansible.run_playbook(playbook, extra_vars)

deps/wazuh_testing/wazuh_testing/end_to_end/check_validators.py

@@ -28,7 +28,7 @@ def compare_expected_found_vulnerabilities(vulnerabilities, expected_vulnerabili
         for vulnerability in expected_vulns:
             if vulnerability not in vulnerabilities.get(agent, []):
                 logging.critical(f"Vulnerability not found for {agent}: {vulnerability}")
-                if agent not in vulnerabilities_not_found:
+                if agent not in vulnerabilities_not_found.keys():
                     vulnerabilities_not_found[agent] = []
                     failed_agents.append(agent)
 
@@ -39,9 +39,10 @@ def compare_expected_found_vulnerabilities(vulnerabilities, expected_vulnerabili
         for vulnerability in agent_vulnerabilities:
             if vulnerability not in expected_vulnerabilities.get(agent, []):
                 logging.critical(f"Vulnerability unexpected found for {agent}: {vulnerability}")
-                if agent not in vulnerabilities_unexpected:
+                if agent not in vulnerabilities_unexpected.keys():
                     vulnerabilities_unexpected[agent] = []
-                    failed_agents.append(agent)
+                    if agent not in failed_agents:
+                        failed_agents.append(agent)
 
                 result = False
                 vulnerabilities_unexpected[agent].append(vulnerability)
@@ -161,4 +162,3 @@ def equals_but_not_empty(x, y):
     not any(x[host][level] for level in ['ERROR', 'CRITICAL'])
     for host in x
 )
-

deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py

@@ -31,6 +31,7 @@
 
 
 WAZUH_STATES_VULNERABILITIES_INDEXNAME_TEMPLATE = 'wazuh-states-vulnerabilities-{cluster_name}'
+INDEXER_RESULT_WINDOWS_VULN_E2E = 50000
 
 
 def get_wazuh_states_vulnerabilities_indexname(cluster_name: str = 'wazuh') -> str:
@@ -176,3 +177,36 @@ def delete_index(host_manager: HostManager, credentials: dict = {'user': 'admin'
 
     requests.delete(url=url, verify=False,
                     auth=requests.auth.HTTPBasicAuth(credentials['user'], credentials['password']), headers=headers)
+
+
+def extend_result_window(host_manager: HostManager, credentials: dict = {'user': 'admin', 'password': 'changeme'},
+                         index: str = 'wazuh-alerts*', new_max_result_window: int = 100000):
+    """Extend the max_result_window setting for a Wazuh Indexer index.
+
+    Args:
+        host_manager: An instance of the HostManager class containing information about hosts.
+        credentials (Optional): A dictionary containing the Indexer credentials. Defaults to
+                                 {'user': 'admin', 'password': 'changeme'}.
+        index (Optional): The Indexer index name. Defaults to 'wazuh-alerts*'.
+        new_max_result_window (Optional): The new maximum result window size. Defaults to 100,000.
+    """
+    logging.info(f"Extending max_result_window for {index} index to {new_max_result_window}")
+
+    url = f"https://{host_manager.get_master_ip()}:9200/{index}/_settings"
+    headers = {
+        'Content-Type': 'application/json',
+    }
+    data = {
+        "index": {
+            "max_result_window": new_max_result_window
+        }
+    }
+
+    response = requests.put(url=url, json=data, verify=False,
+                            auth=requests.auth.HTTPBasicAuth(credentials['user'], credentials['password']),
+                            headers=headers)
+
+    if response.status_code == 200:
+        logging.info(f"Successfully updated max_result_window for {index} index.")
+    else:
+        logging.error(f"Failed to update max_result_window for {index} index. Response: {response.text}")

deps/wazuh_testing/wazuh_testing/end_to_end/remote_operations_handler.py

@@ -243,9 +243,9 @@ def get_vulnerability_alerts(host_manager: HostManager, agent_list, packages_dat
 
 
 def get_vulnerabilities_index(host_manager: HostManager, agent_list, packages_data: List[Dict],
-                              greater_than_timestamp: str = "") -> Dict:
+                              greater_than_timestamp: str = "", size=10000) -> Dict:
     vulnerabilities = get_vulnerabilities_from_states_by_agent(host_manager, agent_list,
-                                                               greater_than_timestamp=greater_than_timestamp)
+                                                               greater_than_timestamp=greater_than_timestamp, size=size)
     package_vulnerabilities = filter_vulnerabilities_by_packages(host_manager, vulnerabilities, packages_data)
 
     return package_vulnerabilities

deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py

@@ -275,13 +275,16 @@ def parse_vulnerability_from_state(state):
 
 
 def get_vulnerabilities_from_states_by_agent(host_manager: HostManager, agents: List[str],
-                                             greater_than_timestamp: str = None, cluster_name='wazuh') -> dict:
+                                             greater_than_timestamp: str = None, cluster_name='wazuh',
+                                             size=10000) -> dict:
     """Get vulnerabilities from the vulnerability state index by agent.
 
     Args:
         host_manager (HostManager): Host manager object.
         agents (list): List of agents.
         greater_than_timestamp (str, optional): Greater than timestamp. Defaults to None.
+        size (int, optional): Maximun number of vulnerabilities to collect.
+            More information in https://opensearch.org/docs/latest/search-plugins/searching-data/paginate
 
     Returns:
         dict: Dictionary of vulnerabilities by agent.
@@ -309,8 +312,8 @@ def get_vulnerabilities_from_states_by_agent(host_manager: HostManager, agents:
                                                            filter=states_filter,
                                                            index=index,
                                                            credentials={'user': indexer_user,
-                                                                        'password': indexer_password}
-                                                           )['hits']['hits']
+                                                                        'password': indexer_password},
+                                                           size=size)['hits']['hits']
         except KeyError as e:
             logging.error(f"No vulnerabilities were obtained for {agent}. Exception {str(e)}")
 

tests/end_to_end/test_vulnerability_detector/conftest.py

@@ -52,7 +52,7 @@ def test_example(host_manager):
                                                     restore_configuration,
                                                     save_indexer_credentials_into_keystore)
 from wazuh_testing.end_to_end.indexer_api import (
-    get_wazuh_states_vulnerabilities_indexname, delete_index)
+    get_wazuh_states_vulnerabilities_indexname, delete_index, extend_result_window, INDEXER_RESULT_WINDOWS_VULN_E2E)
 from wazuh_testing.end_to_end.logs import (get_hosts_alerts, get_hosts_logs,
                                            truncate_remote_host_group_files)
 from wazuh_testing.end_to_end.remote_operations_handler import (
@@ -320,8 +320,11 @@ def setup(preconditions, teardown, host_manager) -> Generator[Dict, None, None]:
         timeout_vulnerabilities_detected = len(agents_to_check) * PACKAGE_VULNERABILITY_SCAN_TIME
 
         time.sleep(timeout_syscollector_scan + timeout_vulnerabilities_detected)
+        vuln_index = get_wazuh_states_vulnerabilities_indexname()
+        extend_result_window(host_manager, index=vuln_index, new_max_result_window=INDEXER_RESULT_WINDOWS_VULN_E2E)
 
-        vulnerabilities = get_vulnerabilities_index(host_manager, agents_to_check, package_data)
+        vulnerabilities = get_vulnerabilities_index(host_manager, agents_to_check, package_data,
+                                                    size=INDEXER_RESULT_WINDOWS_VULN_E2E)
         vulnerabilities_from_alerts = get_vulnerability_alerts(host_manager, agents_to_check, package_data,
                                                                test_timestamp)
 

tests/end_to_end/test_vulnerability_detector/test_vulnerability_detector.py

@@ -70,7 +70,9 @@
                                                              get_vulnerabilities_from_states_by_agent)
 from wazuh_testing.end_to_end.waiters import wait_until_vd_is_updated
 from wazuh_testing.tools.system import HostManager
-
+from wazuh_testing.end_to_end.indexer_api import (INDEXER_RESULT_WINDOWS_VULN_E2E,
+                                                 extend_result_window,
+                                                 get_wazuh_states_vulnerabilities_indexname)
 
 pytestmark = [pytest.mark.e2e, pytest.mark.vulnerability_detector, pytest.mark.tier0]
 
@@ -197,6 +199,11 @@ def configure_vulnerability_detection_test_environment(
     yield test_timestamp
 
 
+def max_result_window(host_manager):
+    vuln_index = get_wazuh_states_vulnerabilities_indexname()
+    extend_result_window(host_manager, index=vuln_index, new_max_result_window=INDEXER_RESULT_WINDOWS_VULN_E2E)
+
+
 @pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
 class TestInitialScans:
     # Checks definition
@@ -259,7 +266,7 @@ def test_first_syscollector_scan(
         configure_vulnerability_detection_test_environment,
         record_property,
         clean_environment_logs,
-        delete_states_vulnerability_index,
+        delete_states_vulnerability_index
     ):
         """
         description: Validates the initiation of the first Syscollector scans across all agents in the environment.
@@ -331,11 +338,13 @@ def test_first_syscollector_scan(
         logging.critical("Waiting until agent all agents have been scanned.")
         time.sleep(TIMEOUT_PER_AGENT_VULNERABILITY_FIRST_SCAN * len(AGENTS_SCANNED_FIRST_SCAN))
 
+        max_result_window(host_manager)
         logging.critical("Checking vulnerabilities in the index")
         vuln_by_agent_index = get_vulnerabilities_from_states_by_agent(
             host_manager,
             AGENTS_SCANNED_FIRST_SCAN,
             greater_than_timestamp=FIRST_SCAN_TIME,
+            size=INDEXER_RESULT_WINDOWS_VULN_E2E
         )
 
         # Store the vulnerabilities in the global variable to make the comparision in test_consistency_initial_scans
@@ -506,10 +515,12 @@ def test_syscollector_second_scan(
 
         global FIRST_SCAN_TIME
 
+        max_result_window(host_manager)
+
         logging.critical("Checking vulnerabilities in the index")
         vuln_by_agent_index = get_vulnerabilities_from_states_by_agent(
-            host_manager, agents_to_check_vulns, greater_than_timestamp=FIRST_SCAN_TIME
-        )
+            host_manager, agents_to_check_vulns, greater_than_timestamp=FIRST_SCAN_TIME,
+            size=INDEXER_RESULT_WINDOWS_VULN_E2E)
 
         logging.critical(
             "Checking that all agents has been scanned and generated vulnerabilities in the index"
@@ -712,8 +723,10 @@ def test_install_vulnerable_package_when_agent_down(self, host_manager, request,
         time.sleep(VD_E2E_TIMEOUT_SYSCOLLECTOR_SCAN + PACKAGE_VULNERABILITY_SCAN_TIME * len(AGENTS_SCANNED_FIRST_SCAN))
 
         package_data = [body["package"]]
+        max_result_window(host_manager)
 
-        vulnerabilities = get_vulnerabilities_index(host_manager, AGENTS_SCANNED_FIRST_SCAN, package_data)
+        vulnerabilities = get_vulnerabilities_index(host_manager, AGENTS_SCANNED_FIRST_SCAN, package_data,
+                                                    size=INDEXER_RESULT_WINDOWS_VULN_E2E)
         expected_vulnerabilities = get_expected_index(host_manager, AGENTS_SCANNED_FIRST_SCAN, body["operation"],
                                                       body["package"])
         duplicated_vulnerabilities = get_duplicated_vulnerabilities(vulnerabilities)
@@ -801,8 +814,10 @@ def test_change_agent_manager(self, permutate_agents_managers, request, precondi
         time.sleep(VD_E2E_TIMEOUT_SYSCOLLECTOR_SCAN + PACKAGE_VULNERABILITY_SCAN_TIME * len(AGENTS_SCANNED_FIRST_SCAN))
 
         package_data = [body["package"]]
+        max_result_window(host_manager)
 
-        vulnerabilities = get_vulnerabilities_index(host_manager, AGENTS_SCANNED_FIRST_SCAN, package_data)
+        vulnerabilities = get_vulnerabilities_index(host_manager, AGENTS_SCANNED_FIRST_SCAN, package_data,
+                                                    size=INDEXER_RESULT_WINDOWS_VULN_E2E)
         expected_vulnerabilities = get_expected_index(host_manager, AGENTS_SCANNED_FIRST_SCAN, body["operation"],
                                                       body["package"])
 
@@ -933,8 +948,10 @@ def test_vulnerability_detector_scans_cases(self, request, preconditions, body,
             package_data = [body["package"]["to"], body["package"]["from"]]
         else:
             package_data = [body["package"]]
+        max_result_window(host_manager)
 
-        vulnerabilities = get_vulnerabilities_index(host_manager, AGENTS_SCANNED_FIRST_SCAN, package_data)
+        vulnerabilities = get_vulnerabilities_index(host_manager, AGENTS_SCANNED_FIRST_SCAN, package_data,
+                                                    size=INDEXER_RESULT_WINDOWS_VULN_E2E)
         expected_vulnerabilities = get_expected_index(host_manager, AGENTS_SCANNED_FIRST_SCAN,
                                                       body["operation"], body["package"])
         duplicated_vulnerabilities = get_duplicated_vulnerabilities(vulnerabilities)