From 798978f3be63a0906b2e5eead9950b3f7b60efaf Mon Sep 17 00:00:00 2001 From: Michael Ihde Date: Tue, 5 Jul 2022 13:55:40 -0400 Subject: [PATCH 01/12] fix: allow generator to be run from other folders --- scripts/generators/beats.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py index b6422a86fd..0d006f050f 100644 --- a/scripts/generators/beats.py +++ b/scripts/generators/beats.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -from os.path import join +from os.path import join, dirname from collections import OrderedDict from typing import ( Dict, @@ -29,6 +29,8 @@ FieldNestedEntry, ) +BEATS_DEFAULT_FIELDS = join(dirname(ecs_helpers.__file__), "beats_default_fields_allowlist.yml") + def generate( ecs_nested: Dict[str, FieldNestedEntry], @@ -55,7 +57,7 @@ def generate( beats_fields.append(beats_field) # Load temporary allowlist for default_fields workaround. - df_allowlist = ecs_helpers.yaml_load('scripts/generators/beats_default_fields_allowlist.yml') + df_allowlist = ecs_helpers.yaml_load(BEATS_DEFAULT_FIELDS) # Set default_field configuration. set_default_field(beats_fields, df_allowlist) From 369f0606188426a1b073bc4a45f4d1f7563dacf8 Mon Sep 17 00:00:00 2001 From: Michael Ihde Date: Tue, 5 Jul 2022 13:55:59 -0400 Subject: [PATCH 02/12] fix: allow asciidoc without event fields --- scripts/generators/asciidoc_fields.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 8d4200bf54..adbcd92fd8 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -197,6 +197,7 @@ def page_field_values(nested, template_name='field_values_template.j2'): category_fields = ['event.kind', 'event.category', 'event.type', 'event.outcome'] nested_fields = [] for cat_field in category_fields: - nested_fields.append(nested['event']['fields'][cat_field]) + if nested.get("event", {}).get("fields", {}).get(cat_field) is not None: + nested_fields.append(nested['event']['fields'][cat_field]) return dict(fields=nested_fields) From fe04b23eba17ab301108af1f06182d496a87ea90 Mon Sep 17 00:00:00 2001 From: Michael Ihde Date: Thu, 7 Jul 2022 13:03:06 -0400 Subject: [PATCH 03/12] feat: allow the entire ECS schema folder to be easily ignored --- scripts/generator.py | 8 +++++++- scripts/schema/loader.py | 13 +++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/scripts/generator.py b/scripts/generator.py index a394978d16..d835a393ca 100644 --- a/scripts/generator.py +++ b/scripts/generator.py @@ -65,7 +65,11 @@ def main() -> None: ecs_generated_version += "+exp" print('Experimental ECS version ' + ecs_generated_version) - fields: dict[str, FieldEntry] = loader.load_schemas(ref=args.ref, included_files=args.include) + fields: dict[str, FieldEntry] = loader.load_schemas( + ref=args.ref, + included_files=args.include, + no_ecs=args.no_ecs + ) cleaner.clean(fields, strict=args.strict) finalizer.finalize(fields) fields, docs_only_fields = subset_filter.filter(fields, args.subset, out_dir) @@ -109,6 +113,8 @@ def argument_parser() -> argparse.Namespace: help='enforce strict checking at schema cleanup') parser.add_argument('--intermediate-only', action='store_true', help='generate intermediary files only') + parser.add_argument('--no-ecs', action='store_true', + help='do not include ECS schemas') parser.add_argument('--force-docs', action='store_true', help='generate ECS docs even if --subset, --include, or --exclude are set') args = parser.parse_args() diff --git a/scripts/schema/loader.py b/scripts/schema/loader.py index ef73805e5e..3e09c32cda 100644 --- a/scripts/schema/loader.py +++ b/scripts/schema/loader.py @@ -77,13 +77,18 @@ def load_schemas( ref: Optional[str] = None, - included_files: Optional[List[str]] = [] + included_files: Optional[List[str]] = [], + no_ecs: Optional[bool] = False ) -> Dict[str, FieldEntry]: """Loads ECS and custom schemas. They are returned deeply nested and merged.""" # ECS fields (from git ref or not) - schema_files_raw: Dict[str, FieldNestedEntry] = load_schemas_from_git( - ref) if ref else load_schema_files(ecs_helpers.ecs_files()) - fields: Dict[str, FieldEntry] = deep_nesting_representation(schema_files_raw) + if not no_ecs: + schema_files_raw: Dict[str, FieldNestedEntry] = load_schemas_from_git( + ref) if ref else load_schema_files(ecs_helpers.ecs_files()) + fields: Dict[str, FieldEntry] = deep_nesting_representation(schema_files_raw) + else: + print('Not loading ECS schemas') + fields = {} # Custom additional files if included_files and len(included_files) > 0: From 0a8be31e206fae9d4c88f7f5742d816b58d2fb48 Mon Sep 17 00:00:00 2001 From: Michael Ihde Date: Thu, 7 Jul 2022 13:15:37 -0400 Subject: [PATCH 04/12] fix: allow subsets to warn (but not fail) when a fieldset is referenced but not loaded --- scripts/schema/subset_filter.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/schema/subset_filter.py b/scripts/schema/subset_filter.py index 8b5f0d1762..75d54faf78 100644 --- a/scripts/schema/subset_filter.py +++ b/scripts/schema/subset_filter.py @@ -179,8 +179,15 @@ def extract_matching_fields( subset_definitions: Dict[str, Any] ) -> Dict[str, FieldEntry]: """Removes fields that are not in the subset definition. Returns a copy without modifying the input fields dict.""" - retained_fields: Dict[str, FieldEntry] = {x: fields[x].copy() for x in subset_definitions} + retained_fields: Dict[str, FieldEntry] = {} + for x in subset_definitions: + if x not in fields: + print('{0} included in subset but has not been loaded'.format(x)) + else: + retained_fields[x] = fields[x].copy() for key, val in subset_definitions.items(): + if key not in fields: + continue retained_fields[key]['field_details'] = fields[key]['field_details'].copy() for option in val: if option != 'fields': From 1ef397e0ad6a91c404015df94cb6a04cc5903adb Mon Sep 17 00:00:00 2001 From: Michael Ihde Date: Thu, 7 Jul 2022 13:16:14 -0400 Subject: [PATCH 05/12] fix: beats generator only works if base fieldset is loaded --- scripts/generators/beats.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py index 0d006f050f..495792578e 100644 --- a/scripts/generators/beats.py +++ b/scripts/generators/beats.py @@ -38,7 +38,11 @@ def generate( out_dir: str ) -> None: # base first - beats_fields: List[OrderedDict] = fieldset_field_array(ecs_nested['base']['fields'], ecs_nested['base']['prefix']) + if 'base' in ecs_nested: + beats_fields: List[OrderedDict] = fieldset_field_array( + ecs_nested['base']['fields'], ecs_nested['base']['prefix']) + else: + beats_fields = [] allowed_fieldset_keys: List[str] = ['name', 'title', 'group', 'description', 'footnote', 'type'] # other fieldsets From 46968b09545d850d59edd381f9c29e93d77227c4 Mon Sep 17 00:00:00 2001 From: Michael Ihde Date: Fri, 8 Jul 2022 15:37:40 -0400 Subject: [PATCH 06/12] feat: allow projects to create their own ASCIIDOC templates --- scripts/generators/asciidoc_fields.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index adbcd92fd8..47848d4d58 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -141,8 +141,14 @@ def save_asciidoc(f, text): # jinja2 setup +cur_dir = path.abspath(path.curdir) local_dir = path.dirname(path.abspath(__file__)) -TEMPLATE_DIR = path.join(local_dir, '../templates') +CUR_TEMPLATE_DIR = path.join(cur_dir, 'templates') +LOCAL_TEMPLATE_DIR = path.join(local_dir, '../templates') +if path.exists(CUR_TEMPLATE_DIR): + TEMPLATE_DIR = CUR_TEMPLATE_DIR +elif path.exists(LOCAL_TEMPLATE_DIR): + TEMPLATE_DIR = LOCAL_TEMPLATE_DIR template_loader = jinja2.FileSystemLoader(searchpath=TEMPLATE_DIR) template_env = jinja2.Environment(loader=template_loader, keep_trailing_newline=True) From 708f9d1dbe8feeefb22c53821f4198c0e646d1a7 Mon Sep 17 00:00:00 2001 From: Michael Ihde Date: Wed, 10 Aug 2022 18:56:56 -0400 Subject: [PATCH 07/12] fix: allow component name prefix to be customized --- scripts/generator.py | 5 ++++- scripts/generators/es_template.py | 8 +++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/generator.py b/scripts/generator.py index d835a393ca..085c2587df 100644 --- a/scripts/generator.py +++ b/scripts/generator.py @@ -80,7 +80,8 @@ def main() -> None: exit() csv_generator.generate(flat, ecs_generated_version, out_dir) - es_template.generate(nested, ecs_generated_version, out_dir, args.mapping_settings, args.template_settings) + es_template.generate(nested, ecs_generated_version, args.component_name_prefix, out_dir, + args.mapping_settings, args.template_settings) es_template.generate_legacy(flat, ecs_generated_version, out_dir, args.mapping_settings, args.template_settings_legacy) beats.generate(nested, ecs_generated_version, out_dir) @@ -115,6 +116,8 @@ def argument_parser() -> argparse.Namespace: help='generate intermediary files only') parser.add_argument('--no-ecs', action='store_true', help='do not include ECS schemas') + parser.add_argument('--component-name-prefix', action='store', default="ecs", + help='prefix to use for component names') parser.add_argument('--force-docs', action='store_true', help='generate ECS docs even if --subset, --include, or --exclude are set') args = parser.parse_args() diff --git a/scripts/generators/es_template.py b/scripts/generators/es_template.py index 555892cef1..a4a7811433 100644 --- a/scripts/generators/es_template.py +++ b/scripts/generators/es_template.py @@ -38,13 +38,14 @@ def generate( ecs_nested: Dict[str, FieldNestedEntry], ecs_version: str, + ecs_component_name_prefix: str, out_dir: str, mapping_settings_file: str, template_settings_file: str ) -> None: """This generates all artifacts for the composable template approach""" all_component_templates(ecs_nested, ecs_version, out_dir) - component_names = component_name_convention(ecs_version, ecs_nested) + component_names = component_name_convention(ecs_version, ecs_nested, ecs_component_name_prefix) save_composable_template(ecs_version, component_names, out_dir, mapping_settings_file, template_settings_file) @@ -100,12 +101,13 @@ def save_component_template( def component_name_convention( ecs_version: str, - ecs_nested: Dict[str, FieldNestedEntry] + ecs_nested: Dict[str, FieldNestedEntry], + ecs_component_name_prefix: str ) -> List[str]: version: str = ecs_version.replace('+', '-') names: List[str] = [] for (fieldset_name, fieldset) in candidate_components(ecs_nested).items(): - names.append("ecs_{}_{}".format(version, fieldset_name.lower())) + names.append("{}_{}_{}".format(ecs_component_name_prefix, version, fieldset_name.lower())) return names From a9e6c4f7d884010edb286646ac631bc71f503566 Mon Sep 17 00:00:00 2001 From: Sean Sullivan Date: Fri, 9 Feb 2024 16:00:00 -0500 Subject: [PATCH 08/12] Fix the merge error --- scripts/generators/es_template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generators/es_template.py b/scripts/generators/es_template.py index 7c6d077076..1ac5e14af2 100644 --- a/scripts/generators/es_template.py +++ b/scripts/generators/es_template.py @@ -106,7 +106,7 @@ def component_name_convention( ) -> List[str]: version: str = ecs_version.replace('+', '-') names: List[str] = [] - for (fieldset_name, fieldset) in candidate_components(ecs_nested).items(): + for (fieldset_name, fieldset) in ecs_helpers.remove_top_level_reusable_false(ecs_nested).items(): names.append("{}_{}_{}".format(ecs_component_name_prefix, version, fieldset_name.lower())) return names From 649b2151b32710b40571a14454b53cbe4022863d Mon Sep 17 00:00:00 2001 From: Sean Sullivan Date: Fri, 9 Feb 2024 16:12:16 -0500 Subject: [PATCH 09/12] Add default so test will pass --- scripts/generators/es_template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generators/es_template.py b/scripts/generators/es_template.py index 1ac5e14af2..0433fdfb3d 100644 --- a/scripts/generators/es_template.py +++ b/scripts/generators/es_template.py @@ -102,7 +102,7 @@ def save_component_template( def component_name_convention( ecs_version: str, ecs_nested: Dict[str, FieldNestedEntry], - ecs_component_name_prefix: str + ecs_component_name_prefix: str="ecs" ) -> List[str]: version: str = ecs_version.replace('+', '-') names: List[str] = [] From 3ce137302c92c54d4623acf593009a161fa71ed1 Mon Sep 17 00:00:00 2001 From: Sean Sullivan Date: Fri, 9 Feb 2024 16:15:11 -0500 Subject: [PATCH 10/12] add Unit test for custom prefix --- scripts/tests/test_es_template.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/tests/test_es_template.py b/scripts/tests/test_es_template.py index 24db749028..bbfc8a89a9 100644 --- a/scripts/tests/test_es_template.py +++ b/scripts/tests/test_es_template.py @@ -286,6 +286,18 @@ def test_component_composable_template_name(self): exp = ["ecs_{}_acme".format(version)] self.assertEqual(es_template.component_name_convention(version, test_map), exp) + def test_component_composable_template_name_with_custom_prefix(self): + version = "1.8" + prefix="custom" + test_map = { + "Acme": { + "name": "Acme", + } + } + + exp = ["{}_{}_acme".format(prefix,version)] + self.assertEqual(es_template.component_name_convention(version, test_map,prefix), exp) + def test_legacy_template_settings_override(self): ecs_version = 100 default = es_template.default_legacy_template_settings(ecs_version) From 2096c6b2a0974303ae9af3b5921a47c4c03ef6fe Mon Sep 17 00:00:00 2001 From: Sean Sullivan Date: Fri, 16 Feb 2024 11:14:04 -0500 Subject: [PATCH 11/12] Move component prefix to kwarg --- scripts/generator.py | 4 ++-- scripts/generators/es_template.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/generator.py b/scripts/generator.py index 085c2587df..9cde50cde2 100644 --- a/scripts/generator.py +++ b/scripts/generator.py @@ -80,8 +80,8 @@ def main() -> None: exit() csv_generator.generate(flat, ecs_generated_version, out_dir) - es_template.generate(nested, ecs_generated_version, args.component_name_prefix, out_dir, - args.mapping_settings, args.template_settings) + es_template.generate(nested, ecs_generated_version, out_dir, + args.mapping_settings, args.template_settings,ecs_component_name_prefix=args.component_name_prefix) es_template.generate_legacy(flat, ecs_generated_version, out_dir, args.mapping_settings, args.template_settings_legacy) beats.generate(nested, ecs_generated_version, out_dir) diff --git a/scripts/generators/es_template.py b/scripts/generators/es_template.py index 0433fdfb3d..64ff6fa801 100644 --- a/scripts/generators/es_template.py +++ b/scripts/generators/es_template.py @@ -38,10 +38,10 @@ def generate( ecs_nested: Dict[str, FieldNestedEntry], ecs_version: str, - ecs_component_name_prefix: str, out_dir: str, mapping_settings_file: str, - template_settings_file: str + template_settings_file: str, + ecs_component_name_prefix: str = "ecs" ) -> None: """This generates all artifacts for the composable template approach""" all_component_templates(ecs_nested, ecs_version, out_dir) From bef27cdf5639f2e932c5dd74158a04c7fb59f8f3 Mon Sep 17 00:00:00 2001 From: Sean Sullivan Date: Tue, 20 Feb 2024 12:49:25 -0500 Subject: [PATCH 12/12] Add changes to changelog.next --- CHANGELOG.next.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index b67cd5af89..f4f1635b07 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -21,6 +21,10 @@ Thanks, you're awesome :-) --> * Advanced `process.io` and `process.tty` fields to GA. #2317 #### Improvements +* Allow ECS from any directory #2019 +* Added ability to specify a prefix for es_templates #2019 +* Allow projects to create their own ACSIIDOC templates #2019 + #### Deprecated