Filebeat pytest improvements (#30103)

* Include the test_file as wait_until name
* Support TESTING_FILEBEAT_FILEPATTERN to test a single ingest file
* Include full event on "not documented" case

CHANGELOG-developer.next.asciidoc

@@ -131,6 +131,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Whitelist `GCP_*` environment variables in dev tools {pull}28364[28364]
 - Add support for `credentials_json` in `gcp` module, all metricsets {pull}29584[29584]
 - Add gcp firestore metricset. {pull}29918[29918]
+- Added TESTING_FILEBEAT_FILEPATTERN option for filebeat module pytests {pull}30103[30103]
 
 ==== Deprecated
 

filebeat/tests/system/test_modules.py

@@ -107,7 +107,7 @@ def load_fileset_test_cases():
                 continue
 
             test_files = glob.glob(os.path.join(modules_dir, module,
-                                                fileset, "test", "*.log"))
+                                                fileset, "test", os.getenv("TESTING_FILEBEAT_FILEPATTERN", "*.log")))
             for test_file in test_files:
                 test_cases.append([module, fileset, test_file])
 
@@ -216,7 +216,8 @@ def run_on_file(self, module, fileset, test_file, cfgfile):
             error_line)
 
         # Make sure index exists
-        self.wait_until(lambda: self.es.indices.exists(self.index_name))
+        self.wait_until(lambda: self.es.indices.exists(self.index_name),
+                        name="indices present for {}".format(test_file))
 
         self.es.indices.refresh(index=self.index_name)
         # Loads the first 100 events to be checked

libbeat/tests/system/beat/beat.py

@@ -776,7 +776,7 @@ def is_documented(key, docs):
             # Range keys as used in 'date_range' etc will not have docs of course
             isRangeKey = key.split('.')[-1] in ['gte', 'gt', 'lte', 'lt']
             if not(is_documented(key, expected_fields) or metaKey or isRangeKey):
-                raise Exception("Key '{}' found in event is not documented!".format(key))
+                raise Exception("Key '{}' found in event ({}) is not documented!".format(key, str(evt)))
             if is_documented(key, aliases):
                 raise Exception("Key '{}' found in event is documented as an alias!".format(key))