Feature: make ecs more flexible

CHANGELOG.next.md

@@ -23,6 +23,10 @@ Thanks, you're awesome :-) -->
 * Added `threat.indicator.id`. #2324
 
 #### Improvements
+* Allow ECS from any directory #2019
+* Added ability to specify a prefix for es_templates #2019
+* Allow projects to create their own ACSIIDOC templates #2019
+
 
 #### Deprecated
 

scripts/generator.py

@@ -65,7 +65,11 @@ def main() -> None:
         ecs_generated_version += "+exp"
         print('Experimental ECS version ' + ecs_generated_version)
 
-    fields: dict[str, FieldEntry] = loader.load_schemas(ref=args.ref, included_files=args.include)
+    fields: dict[str, FieldEntry] = loader.load_schemas(
+        ref=args.ref,
+        included_files=args.include,
+        no_ecs=args.no_ecs
+    )
     cleaner.clean(fields, strict=args.strict)
     finalizer.finalize(fields)
     fields, docs_only_fields = subset_filter.filter(fields, args.subset, out_dir)
@@ -76,7 +80,8 @@ def main() -> None:
         exit()
 
     csv_generator.generate(flat, ecs_generated_version, out_dir)
-    es_template.generate(nested, ecs_generated_version, out_dir, args.mapping_settings, args.template_settings)
+    es_template.generate(nested, ecs_generated_version, out_dir,
+                         args.mapping_settings, args.template_settings,ecs_component_name_prefix=args.component_name_prefix)
     es_template.generate_legacy(flat, ecs_generated_version, out_dir,
                                 args.mapping_settings, args.template_settings_legacy)
     beats.generate(nested, ecs_generated_version, out_dir)
@@ -109,6 +114,10 @@ def argument_parser() -> argparse.Namespace:
                         help='enforce strict checking at schema cleanup')
     parser.add_argument('--intermediate-only', action='store_true',
                         help='generate intermediary files only')
+    parser.add_argument('--no-ecs', action='store_true',
+                        help='do not include ECS schemas')
+    parser.add_argument('--component-name-prefix', action='store', default="ecs",
+                        help='prefix to use for component names')
     parser.add_argument('--force-docs', action='store_true',
                         help='generate ECS docs even if --subset, --include, or --exclude are set')
     args = parser.parse_args()

scripts/generators/asciidoc_fields.py

@@ -143,8 +143,14 @@ def save_asciidoc(f, text):
 # jinja2 setup
 
 
+cur_dir = path.abspath(path.curdir)
 local_dir = path.dirname(path.abspath(__file__))
-TEMPLATE_DIR = path.join(local_dir, '../templates')
+CUR_TEMPLATE_DIR = path.join(cur_dir, 'templates')
+LOCAL_TEMPLATE_DIR = path.join(local_dir, '../templates')
+if path.exists(CUR_TEMPLATE_DIR):
+    TEMPLATE_DIR = CUR_TEMPLATE_DIR
+elif path.exists(LOCAL_TEMPLATE_DIR):
+    TEMPLATE_DIR = LOCAL_TEMPLATE_DIR
 template_loader = jinja2.FileSystemLoader(searchpath=TEMPLATE_DIR)
 template_env = jinja2.Environment(loader=template_loader, keep_trailing_newline=True)
 
@@ -199,6 +205,7 @@ def page_field_values(nested, template_name='field_values_template.j2'):
     category_fields = ['event.kind', 'event.category', 'event.type', 'event.outcome']
     nested_fields = []
     for cat_field in category_fields:
-        nested_fields.append(nested['event']['fields'][cat_field])
+        if nested.get("event", {}).get("fields", {}).get(cat_field) is not None:
+            nested_fields.append(nested['event']['fields'][cat_field])
 
     return dict(fields=nested_fields)

scripts/generators/beats.py

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from os.path import join
+from os.path import join, dirname
 from collections import OrderedDict
 from typing import (
     Dict,
@@ -29,15 +29,21 @@
     FieldNestedEntry,
 )
 
+BEATS_DEFAULT_FIELDS = join(dirname(ecs_helpers.__file__), "beats_default_fields_allowlist.yml")
+
 
 def generate(
     ecs_nested: Dict[str, FieldNestedEntry],
     ecs_version: str,
     out_dir: str
 ) -> None:
     # base first
-    ecs_nested = ecs_helpers.remove_top_level_reusable_false(ecs_nested)
-    beats_fields: List[OrderedDict] = fieldset_field_array(ecs_nested['base']['fields'], ecs_nested['base']['prefix'])
+    if 'base' in ecs_nested:
+        beats_fields: List[OrderedDict] = fieldset_field_array(
+            ecs_nested['base']['fields'], ecs_nested['base']['prefix'])
+    else:
+        beats_fields = []
+
 
     allowed_fieldset_keys: List[str] = ['name', 'title', 'group', 'description', 'footnote', 'type']
     # other fieldsets
@@ -56,7 +62,7 @@ def generate(
         beats_fields.append(beats_field)
 
     # Load temporary allowlist for default_fields workaround.
-    df_allowlist = ecs_helpers.yaml_load('scripts/generators/beats_default_fields_allowlist.yml')
+    df_allowlist = ecs_helpers.yaml_load(BEATS_DEFAULT_FIELDS)
     # Set default_field configuration.
     set_default_field(beats_fields, df_allowlist)
 

scripts/generators/es_template.py

@@ -40,11 +40,12 @@ def generate(
     ecs_version: str,
     out_dir: str,
     mapping_settings_file: str,
-    template_settings_file: str
+    template_settings_file: str,
+    ecs_component_name_prefix: str = "ecs"
 ) -> None:
     """This generates all artifacts for the composable template approach"""
     all_component_templates(ecs_nested, ecs_version, out_dir)
-    component_names = component_name_convention(ecs_version, ecs_nested)
+    component_names = component_name_convention(ecs_version, ecs_nested, ecs_component_name_prefix)
     save_composable_template(ecs_version, component_names, out_dir, mapping_settings_file, template_settings_file)
 
 
@@ -100,12 +101,13 @@ def save_component_template(
 
 def component_name_convention(
     ecs_version: str,
-    ecs_nested: Dict[str, FieldNestedEntry]
+    ecs_nested: Dict[str, FieldNestedEntry],
+    ecs_component_name_prefix: str="ecs"
 ) -> List[str]:
     version: str = ecs_version.replace('+', '-')
     names: List[str] = []
     for (fieldset_name, fieldset) in ecs_helpers.remove_top_level_reusable_false(ecs_nested).items():
-        names.append("ecs_{}_{}".format(version, fieldset_name.lower()))
+        names.append("{}_{}_{}".format(ecs_component_name_prefix, version, fieldset_name.lower()))
     return names
 
 

scripts/schema/loader.py

@@ -77,13 +77,18 @@
 
 def load_schemas(
     ref: Optional[str] = None,
-    included_files: Optional[List[str]] = []
+    included_files: Optional[List[str]] = [],
+    no_ecs: Optional[bool] = False
 ) -> Dict[str, FieldEntry]:
     """Loads ECS and custom schemas. They are returned deeply nested and merged."""
     # ECS fields (from git ref or not)
-    schema_files_raw: Dict[str, FieldNestedEntry] = load_schemas_from_git(
-        ref) if ref else load_schema_files(ecs_helpers.ecs_files())
-    fields: Dict[str, FieldEntry] = deep_nesting_representation(schema_files_raw)
+    if not no_ecs:
+        schema_files_raw: Dict[str, FieldNestedEntry] = load_schemas_from_git(
+            ref) if ref else load_schema_files(ecs_helpers.ecs_files())
+        fields: Dict[str, FieldEntry] = deep_nesting_representation(schema_files_raw)
+    else:
+        print('Not loading ECS schemas')
+        fields = {}
 
     # Custom additional files
     if included_files and len(included_files) > 0:

scripts/schema/subset_filter.py

@@ -179,8 +179,15 @@ def extract_matching_fields(
     subset_definitions: Dict[str, Any]
 ) -> Dict[str, FieldEntry]:
     """Removes fields that are not in the subset definition. Returns a copy without modifying the input fields dict."""
-    retained_fields: Dict[str, FieldEntry] = {x: fields[x].copy() for x in subset_definitions}
+    retained_fields: Dict[str, FieldEntry] = {}
+    for x in subset_definitions:
+        if x not in fields:
+            print('{0} included in subset but has not been loaded'.format(x))
+        else:
+            retained_fields[x] = fields[x].copy()
     for key, val in subset_definitions.items():
+        if key not in fields:
+            continue
         retained_fields[key]['field_details'] = fields[key]['field_details'].copy()
         for option in val:
             if option != 'fields':

scripts/tests/test_es_template.py

@@ -286,6 +286,18 @@ def test_component_composable_template_name(self):
         exp = ["ecs_{}_acme".format(version)]
         self.assertEqual(es_template.component_name_convention(version, test_map), exp)
 
+    def test_component_composable_template_name_with_custom_prefix(self):
+        version = "1.8"
+        prefix="custom"
+        test_map = {
+            "Acme": {
+                "name": "Acme",
+            }
+        }
+
+        exp = ["{}_{}_acme".format(prefix,version)]
+        self.assertEqual(es_template.component_name_convention(version, test_map,prefix), exp)
+
     def test_legacy_template_settings_override(self):
         ecs_version = 100
         default = es_template.default_legacy_template_settings(ecs_version)