Remove ignore_above parameter when indexing is disabled on field (e…

…lastic#1483) * remove ignore_above if indexing is disabled for a field * update artifacts * linting * changelog * update artifacts
ebeahan · Jun 29, 2021 · 96fd70d · 96fd70d
1 parent 4508cd0
commit 96fd70d
Show file tree

Hide file tree

Showing 19 changed files with 15 additions and 39 deletions.
diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md
@@ -40,6 +40,8 @@ Thanks, you're awesome :-) -->
 
 #### Bugfixes
 
+* Remove `ignore_above` when `index: false` and `doc_values: false`. #1483
+
 #### Added
 
 * Support `match_only_text` data type in Go code generator. #1418

diff --git a/experimental/generated/beats/fields.ecs.yml b/experimental/generated/beats/fields.ecs.yml
@@ -1925,7 +1925,6 @@
     - name: original
       level: core
       type: keyword
-      ignore_above: 1024
       description: 'Raw text message of entire event. Used to demonstrate log integrity  or
         where the full log message (before splitting it up in multiple  parts) may
         be required, e.g. for reindex.
@@ -3626,7 +3625,6 @@
     - name: original
       level: core
       type: keyword
-      ignore_above: 1024
       description: 'Deprecated for removal in next major version release. This field
         is superseded by  `event.original`.
 
@@ -8450,7 +8448,6 @@
     - name: enrichments.event.original
       level: core
       type: keyword
-      ignore_above: 1024
       description: 'Raw text message of entire event. Used to demonstrate log integrity  or
         where the full log message (before splitting it up in multiple  parts) may
         be required, e.g. for reindex.

diff --git a/experimental/generated/ecs/ecs_flat.yml b/experimental/generated/ecs/ecs_flat.yml
@@ -2655,7 +2655,6 @@ event.original:
   example: Sep 19 08:26:10 host CEF:0&#124;Security&#124; threatmanager&#124;1.0&#124;100&#124;
     worm successfully stopped&#124;10&#124;src=10.0.0.1 dst=2.1.2.2spt=1232
   flat_name: event.original
-  ignore_above: 1024
   index: false
   level: core
   name: original
@@ -5347,7 +5346,6 @@ log.original:
   doc_values: false
   example: Sep 19 08:26:10 localhost My log
   flat_name: log.original
-  ignore_above: 1024
   index: false
   level: core
   name: original
@@ -12930,7 +12928,6 @@ threat.enrichments.event.original:
   example: Sep 19 08:26:10 host CEF:0&#124;Security&#124; threatmanager&#124;1.0&#124;100&#124;
     worm successfully stopped&#124;10&#124;src=10.0.0.1 dst=2.1.2.2spt=1232
   flat_name: threat.enrichments.event.original
-  ignore_above: 1024
   index: false
   level: core
   name: original

diff --git a/experimental/generated/ecs/ecs_nested.yml b/experimental/generated/ecs/ecs_nested.yml
@@ -3433,7 +3433,6 @@ event:
       example: Sep 19 08:26:10 host CEF:0&#124;Security&#124; threatmanager&#124;1.0&#124;100&#124;
         worm successfully stopped&#124;10&#124;src=10.0.0.1 dst=2.1.2.2spt=1232
       flat_name: event.original
-      ignore_above: 1024
       index: false
       level: core
       name: original
@@ -6556,7 +6555,6 @@ log:
       doc_values: false
       example: Sep 19 08:26:10 localhost My log
       flat_name: log.original
-      ignore_above: 1024
       index: false
       level: core
       name: original
@@ -14985,7 +14983,6 @@ threat:
       example: Sep 19 08:26:10 host CEF:0&#124;Security&#124; threatmanager&#124;1.0&#124;100&#124;
         worm successfully stopped&#124;10&#124;src=10.0.0.1 dst=2.1.2.2spt=1232
       flat_name: threat.enrichments.event.original
-      ignore_above: 1024
       index: false
       level: core
       name: original

diff --git a/experimental/generated/elasticsearch/7/template.json b/experimental/generated/elasticsearch/7/template.json
@@ -904,7 +904,6 @@
           },
           "original": {
             "doc_values": false,
-            "ignore_above": 1024,
             "index": false,
             "type": "keyword"
           },
@@ -1850,7 +1849,6 @@
           },
           "original": {
             "doc_values": false,
-            "ignore_above": 1024,
             "index": false,
             "type": "keyword"
           },
@@ -4546,7 +4544,6 @@
                   },
                   "original": {
                     "doc_values": false,
-                    "ignore_above": 1024,
                     "index": false,
                     "type": "keyword"
                   },

diff --git a/experimental/generated/elasticsearch/component/event.json b/experimental/generated/elasticsearch/component/event.json
@@ -58,7 +58,6 @@
             },
             "original": {
               "doc_values": false,
-              "ignore_above": 1024,
               "index": false,
               "type": "keyword"
             },

diff --git a/experimental/generated/elasticsearch/component/log.json b/experimental/generated/elasticsearch/component/log.json
@@ -43,7 +43,6 @@
             },
             "original": {
               "doc_values": false,
-              "ignore_above": 1024,
               "index": false,
               "type": "keyword"
             },

diff --git a/experimental/generated/elasticsearch/component/threat.json b/experimental/generated/elasticsearch/component/threat.json
@@ -82,7 +82,6 @@
                     },
                     "original": {
                       "doc_values": false,
-                      "ignore_above": 1024,
                       "index": false,
                       "type": "keyword"
                     },

diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml
@@ -1540,7 +1540,6 @@
     - name: stack_trace
       level: extended
       type: keyword
-      ignore_above: 1024
       multi_fields:
       - name: text
         type: text
@@ -1737,7 +1736,6 @@
     - name: original
       level: core
       type: keyword
-      ignore_above: 1024
       description: 'Raw text message of entire event. Used to demonstrate log integrity  or
         where the full log message (before splitting it up in multiple  parts) may
         be required, e.g. for reindex.
@@ -3247,7 +3245,6 @@
     - name: original
       level: core
       type: keyword
-      ignore_above: 1024
       description: 'Deprecated for removal in next major version release. This field
         is superseded by  `event.original`.
 
@@ -6122,7 +6119,6 @@
     - name: enrichments.event.original
       level: core
       type: keyword
-      ignore_above: 1024
       description: 'Raw text message of entire event. Used to demonstrate log integrity  or
         where the full log message (before splitting it up in multiple  parts) may
         be required, e.g. for reindex.

diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml
@@ -1827,7 +1827,6 @@ error.stack_trace:
   description: The stack trace of this error in plain text.
   doc_values: false
   flat_name: error.stack_trace
-  ignore_above: 1024
   index: false
   level: extended
   multi_fields:
@@ -2306,7 +2305,6 @@ event.original:
   example: Sep 19 08:26:10 host CEF:0&#124;Security&#124; threatmanager&#124;1.0&#124;100&#124;
     worm successfully stopped&#124;10&#124;src=10.0.0.1 dst=2.1.2.2spt=1232
   flat_name: event.original
-  ignore_above: 1024
   index: false
   level: core
   name: original
@@ -4645,7 +4643,6 @@ log.original:
   doc_values: false
   example: Sep 19 08:26:10 localhost My log
   flat_name: log.original
-  ignore_above: 1024
   index: false
   level: core
   name: original
@@ -9178,7 +9175,6 @@ threat.enrichments.event.original:
   example: Sep 19 08:26:10 host CEF:0&#124;Security&#124; threatmanager&#124;1.0&#124;100&#124;
     worm successfully stopped&#124;10&#124;src=10.0.0.1 dst=2.1.2.2spt=1232
   flat_name: threat.enrichments.event.original
-  ignore_above: 1024
   index: false
   level: core
   name: original

diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml
@@ -2579,7 +2579,6 @@ error:
       description: The stack trace of this error in plain text.
       doc_values: false
       flat_name: error.stack_trace
-      ignore_above: 1024
       index: false
       level: extended
       multi_fields:
@@ -3084,7 +3083,6 @@ event:
       example: Sep 19 08:26:10 host CEF:0&#124;Security&#124; threatmanager&#124;1.0&#124;100&#124;
         worm successfully stopped&#124;10&#124;src=10.0.0.1 dst=2.1.2.2spt=1232
       flat_name: event.original
-      ignore_above: 1024
       index: false
       level: core
       name: original
@@ -5854,7 +5852,6 @@ log:
       doc_values: false
       example: Sep 19 08:26:10 localhost My log
       flat_name: log.original
-      ignore_above: 1024
       index: false
       level: core
       name: original
@@ -10868,7 +10865,6 @@ threat:
       example: Sep 19 08:26:10 host CEF:0&#124;Security&#124; threatmanager&#124;1.0&#124;100&#124;
         worm successfully stopped&#124;10&#124;src=10.0.0.1 dst=2.1.2.2spt=1232
       flat_name: threat.enrichments.event.original
-      ignore_above: 1024
       index: false
       level: core
       name: original

diff --git a/generated/elasticsearch/6/template.json b/generated/elasticsearch/6/template.json
@@ -732,7 +732,6 @@
                   "type": "text"
                 }
               },
-              "ignore_above": 1024,
               "index": false,
               "type": "keyword"
             },
@@ -794,7 +793,6 @@
             },
             "original": {
               "doc_values": false,
-              "ignore_above": 1024,
               "index": false,
               "type": "keyword"
             },
@@ -1622,7 +1620,6 @@
             },
             "original": {
               "doc_values": false,
-              "ignore_above": 1024,
               "index": false,
               "type": "keyword"
             },
@@ -3232,7 +3229,6 @@
                     },
                     "original": {
                       "doc_values": false,
-                      "ignore_above": 1024,
                       "index": false,
                       "type": "keyword"
                     },

diff --git a/generated/elasticsearch/7/template.json b/generated/elasticsearch/7/template.json
@@ -728,7 +728,6 @@
                 "type": "text"
               }
             },
-            "ignore_above": 1024,
             "index": false,
             "type": "keyword"
           },
@@ -790,7 +789,6 @@
           },
           "original": {
             "doc_values": false,
-            "ignore_above": 1024,
             "index": false,
             "type": "keyword"
           },
@@ -1618,7 +1616,6 @@
           },
           "original": {
             "doc_values": false,
-            "ignore_above": 1024,
             "index": false,
             "type": "keyword"
           },
@@ -3228,7 +3225,6 @@
                   },
                   "original": {
                     "doc_values": false,
-                    "ignore_above": 1024,
                     "index": false,
                     "type": "keyword"
                   },

diff --git a/generated/elasticsearch/component/error.json b/generated/elasticsearch/component/error.json
@@ -28,7 +28,6 @@
                   "type": "text"
                 }
               },
-              "ignore_above": 1024,
               "index": false,
               "type": "keyword"
             },

diff --git a/generated/elasticsearch/component/event.json b/generated/elasticsearch/component/event.json
@@ -58,7 +58,6 @@
             },
             "original": {
               "doc_values": false,
-              "ignore_above": 1024,
               "index": false,
               "type": "keyword"
             },

diff --git a/generated/elasticsearch/component/log.json b/generated/elasticsearch/component/log.json
@@ -45,7 +45,6 @@
             },
             "original": {
               "doc_values": false,
-              "ignore_above": 1024,
               "index": false,
               "type": "keyword"
             },

diff --git a/generated/elasticsearch/component/threat.json b/generated/elasticsearch/component/threat.json
@@ -83,7 +83,6 @@
                     },
                     "original": {
                       "doc_values": false,
-                      "ignore_above": 1024,
                       "index": false,
                       "type": "keyword"
                     },

diff --git a/scripts/schema/cleaner.py b/scripts/schema/cleaner.py
@@ -153,6 +153,7 @@ def field_or_multi_field_datatype_defaults(field_details):
         field_details.pop('index', None)
     if 'index' in field_details and not field_details['index']:
         field_details.setdefault('doc_values', False)
+        field_details.pop('ignore_above', None)
 
 
 FIELD_MANDATORY_ATTRIBUTES = ['name', 'description', 'type', 'level']

diff --git a/scripts/tests/unit/test_schema_cleaner.py b/scripts/tests/unit/test_schema_cleaner.py
@@ -238,6 +238,18 @@ def test_field_defaults_dont_override(self):
         cleaner.field_defaults({'field_details': field_details})
         self.assertEqual(field_details['ignore_above'], 8000)
 
+    def test_field_defaults_index_false_doc_values_false(self):
+        field_details = {
+            'description': 'description',
+            'level': 'extended',
+            'name': 'my_non_indexed_field',
+            'type': 'keyword',
+            'index': False,
+            'doc_values': False
+        }
+        cleaner.field_defaults({'field_details': field_details})
+        self.assertNotIn("ignore_above", field_details)
+
     def test_multi_field_defaults_and_precalc(self):
         field_details = {
             'description': 'description',