[Fleet] Add data stream API tests for cases where event.ingested is not set

[kpollich]

In #135817, we fixed a bug caused by event.ingested being unset in some data stream documents for cloud environments. Some additional testing revealed that this is potentially because enterprise search documents are ingested in Cloud before the .fleet-final-pipeline ingest pipeline is installed, e.g.

Show code block

# GET logs-enterprise_search.audit-default/_search
{
  "took": 2,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 7,
      "relation": "eq"
    },
    "max_score": 1,
    "hits": [
      {
        "_index": ".ds-logs-enterprise_search.audit-default-2022.07.06-000001",
        "_id": "Rhj41IEBv5F_uAXu8Yfw",
        "_score": 1,
        "_source": {
          "@timestamp": "2022-07-06T19:24:48.000Z",
          "message": "Workplace Search Organization change (000000000000000000000001)",
          "event.id": "62c5e18042e4b31756a80682",
          "process.pid": 8,
          "service.version": "8.4.0",
          "event.type": [
            "change"
          ],
          "process.thread.id": 4504,
          "data_stream": {
            "type": "logs",
            "dataset": "enterprise_search.audit",
            "namespace": "default"
          },
          "event.kind": "event",
          "service.ephemeral_id": "62c5e17142e4b315c8a80677",
          "ecs": {
            "version": "1.7.0"
          },
          "input": {
            "type": "log"
          },
          "event.action": "audit",
          "event.outcome": "success",
          "log": {
            "offset": 6505,
            "file": {
              "path": "/app/logs/audit.log"
            }
          },
          "labels": {
            "index_date": "2022.07.06"
          },
          "enterprisesearch.entity": "FritoPie::Organization",
          "agent": {
            "ephemeral_id": "c8d092a4-9a33-4f25-a7e8-05a731f7feb8",
            "id": "fe72453b-b0ff-49f8-9d17-1644f7e5f257",
            "name": "e1139b0a8396",
            "type": "filebeat",
            "version": "8.1.1"
          },
          "service.type": "enterprise_search",
          "event": {
            "dataset": "enterprise-search-audit"
          },
          "event.category": [
            "process"
          ],
          "enterprisesearch.change": {
            "telemetry_last_sent_at": [
              null,
              "2022-07-06T19:24:48Z"
            ],
            "updated_at": [
              "2022-07-06T19:24:40Z",
              "2022-07-06T19:24:48Z"
            ]
          },
          "host": {
            "name": "e1139b0a8396"
          }
        }
      },
      {
        "_index": ".ds-logs-enterprise_search.audit-default-2022.07.06-000001",
        "_id": "dMf41IEB6XGlQZUq17sc",
        "_score": 1,
        "_source": {
          "@timestamp": "2022-07-06T19:24:33.000Z",
          "host": {
            "name": "e1139b0a8396"
          },
          "user.name": "system",
          "process.thread.id": 4004,
          "agent": {
            "id": "fe72453b-b0ff-49f8-9d17-1644f7e5f257",
            "name": "e1139b0a8396",
            "type": "filebeat",
            "version": "8.1.1",
            "ephemeral_id": "c8d092a4-9a33-4f25-a7e8-05a731f7feb8"
          },
          "event.action": "audit",
          "service.version": "8.4.0",
          "service.type": "enterprise_search",
          "event.outcome": "unknown",
          "labels": {
            "index_date": "2022.07.06"
          },
          "event": {
            "dataset": "enterprise-search-audit"
          },
          "input": {
            "type": "log"
          },
          "event.kind": "event",
          "event.type": [
            "change"
          ],
          "process.pid": 8,
          "ecs": {
            "version": "1.7.0"
          },
          "message": "[Starting] Creating indices for 46 models",
          "service.ephemeral_id": "62c5e17142e4b315c8a80677",
          "event.category": [
            "process"
          ],
          "data_stream": {
            "dataset": "enterprise_search.audit",
            "namespace": "default",
            "type": "logs"
          },
          "log": {
            "offset": 0,
            "file": {
              "path": "/app/logs/audit.log"
            }
          },
          "event.id": "62c5e17142e4b315c8a80676"
        }
      },
      {
        "_index": ".ds-logs-enterprise_search.audit-default-2022.07.06-000001",
        "_id": "dcf41IEB6XGlQZUq17sc",
        "_score": 1,
        "_source": {
          "@timestamp": "2022-07-06T19:24:39.000Z",
          "event.action": "audit",
          "service.ephemeral_id": "62c5e17142e4b315c8a80677",
          "log": {
            "file": {
              "path": "/app/logs/audit.log"
            },
            "offset": 668
          },
          "event.outcome": "success",
          "labels": {
            "index_date": "2022.07.06"
          },
          "agent": {
            "name": "e1139b0a8396",
            "type": "filebeat",
            "version": "8.1.1",
            "ephemeral_id": "c8d092a4-9a33-4f25-a7e8-05a731f7feb8",
            "id": "fe72453b-b0ff-49f8-9d17-1644f7e5f257"
          },
          "service.type": "enterprise_search",
          "event.type": [
            "change"
          ],
          "data_stream": {
            "dataset": "enterprise_search.audit",
            "namespace": "default",
            "type": "logs"
          },
          "event": {
            "dataset": "enterprise-search-audit"
          },
          "event.id": "62c5e17742e4b315c8a80678",
          "process.thread.id": 4004,
          "user.name": "system",
          "input": {
            "type": "log"
          },
          "host": {
            "name": "e1139b0a8396"
          },
          "service.version": "8.4.0",
          "event.category": [
            "process"
          ],
          "ecs": {
            "version": "1.7.0"
          },
          "message": "[Finished] Creating indices for 46 models",
          "process.pid": 8,
          "event.kind": "event"
        }
      },
      {
        "_index": ".ds-logs-enterprise_search.audit-default-2022.07.06-000001",
        "_id": "dsf41IEB6XGlQZUq17sc",
        "_score": 1,
        "_source": {
          "@timestamp": "2022-07-06T19:24:40.000Z",
          "event.action": "audit",
          "event": {
            "dataset": "enterprise-search-audit"
          },
          "process.pid": 8,
          "event.id": "62c5e17842e4b315c8a8067b",
          "event.outcome": "success",
          "enterprisesearch.entity": "LocoMoco::ApiToken",
          "labels": {
            "index_date": "2022.07.06"
          },
          "service.type": "enterprise_search",
          "host": {
            "name": "e1139b0a8396"
          },
          "event.kind": "event",
          "event.type": [
            "creation"
          ],
          "log": {
            "file": {
              "path": "/app/logs/audit.log"
            },
            "offset": 1336
          },
          "event.category": [
            "process"
          ],
          "data_stream": {
            "type": "logs",
            "dataset": "enterprise_search.audit",
            "namespace": "default"
          },
          "message": "App Search ApiToken creation (loco_moco_account_id:62c5e17842e4b315c8a8067a|name:search-key)",
          "input": {
            "type": "log"
          },
          "agent": {
            "ephemeral_id": "c8d092a4-9a33-4f25-a7e8-05a731f7feb8",
            "id": "fe72453b-b0ff-49f8-9d17-1644f7e5f257",
            "name": "e1139b0a8396",
            "type": "filebeat",
            "version": "8.1.1"
          },
          "ecs": {
            "version": "1.7.0"
          },
          "service.ephemeral_id": "62c5e17142e4b315c8a80677",
          "service.version": "8.4.0",
          "enterprisesearch.change": {
            "token_type": [
              null,
              "search"
            ],
            "read_access": [
              null,
              false
            ],
            "created_at": [
              null,
              "2022-07-06T19:24:40Z"
            ],
            "access_all_engines": [
              null,
              true
            ],
            "authentication_token": [
              null,
              {
                "hash": "9c04978fd17348eddd47d377d8b4375701bededa22bd133b2db037925c749d2a",
                "ciphertext": "NDN1c3VrVExydlVjRXpBSGMvd0JrTkRkTWRPb1FadENvYUVJeWxaNGNjYnE3c1JjTC94NDlabnNvRWxTTTB5dy0tN2dVamFZcDBEOHhWcWd6eDBZdXhBQT09--fb9d9051437f5997eb38a9aa51ceeaacb62346ed"
              }
            ],
            "write_access": [
              null,
              false
            ],
            "updated_at": [
              null,
              "2022-07-06T19:24:40Z"
            ],
            "loco_moco_account_id": [
              null,
              "62c5e17842e4b315c8a8067a"
            ],
            "id": [
              null,
              "loco_moco_account_id:62c5e17842e4b315c8a8067a|name:search-key"
            ],
            "name": [
              null,
              "search-key"
            ]
          },
          "process.thread.id": 4004
        }
      },
      {
        "_index": ".ds-logs-enterprise_search.audit-default-2022.07.06-000001",
        "_id": "d8f41IEB6XGlQZUq17sc",
        "_score": 1,
        "_source": {
          "@timestamp": "2022-07-06T19:24:40.000Z",
          "service.ephemeral_id": "62c5e17142e4b315c8a80677",
          "data_stream": {
            "type": "logs",
            "dataset": "enterprise_search.audit",
            "namespace": "default"
          },
          "event.kind": "event",
          "input": {
            "type": "log"
          },
          "process.thread.id": 4004,
          "enterprisesearch.change": {
            "write_access": [
              null,
              true
            ],
            "name": [
              null,
              "private-key"
            ],
            "access_all_engines": [
              null,
              true
            ],
            "authentication_token": [
              null,
              {
                "hash": "a2bec8a2dfc7a4a6d408362c0a5e17ef107117c904622092ee2f4142c9ee03ee",
                "ciphertext": "cGh4d1RhUGZaMVMxeGk5ZUVGcHpGWXo1M0I1NlMxVGJobjgxa1VyRkdDZ05wdTFVOE4xWU03Q294Sm10WkloQi0tclZUZEVRdktpanZNZ3JlblQzdU05Zz09--4d5f6f562ee7ee935ab84e856c1cc68b707bcd04"
              }
            ],
            "id": [
              null,
              "loco_moco_account_id:62c5e17842e4b315c8a8067a|name:private-key"
            ],
            "created_at": [
              null,
              "2022-07-06T19:24:40Z"
            ],
            "read_access": [
              null,
              true
            ],
            "token_type": [
              null,
              "private"
            ],
            "updated_at": [
              null,
              "2022-07-06T19:24:40Z"
            ],
            "loco_moco_account_id": [
              null,
              "62c5e17842e4b315c8a8067a"
            ]
          },
          "event.category": [
            "process"
          ],
          "event.outcome": "success",
          "labels": {
            "index_date": "2022.07.06"
          },
          "event.type": [
            "creation"
          ],
          "enterprisesearch.entity": "LocoMoco::ApiToken",
          "message": "App Search ApiToken creation (loco_moco_account_id:62c5e17842e4b315c8a8067a|name:private-key)",
          "event.action": "audit",
          "agent": {
            "id": "fe72453b-b0ff-49f8-9d17-1644f7e5f257",
            "name": "e1139b0a8396",
            "type": "filebeat",
            "version": "8.1.1",
            "ephemeral_id": "c8d092a4-9a33-4f25-a7e8-05a731f7feb8"
          },
          "log": {
            "offset": 2758,
            "file": {
              "path": "/app/logs/audit.log"
            }
          },
          "host": {
            "name": "e1139b0a8396"
          },
          "event.id": "62c5e17842e4b315c8a8067c",
          "event": {
            "dataset": "enterprise-search-audit"
          },
          "service.type": "enterprise_search",
          "service.version": "8.4.0",
          "ecs": {
            "version": "1.7.0"
          },
          "process.pid": 8
        }
      },
      {
        "_index": ".ds-logs-enterprise_search.audit-default-2022.07.06-000001",
        "_id": "eMf41IEB6XGlQZUq17sc",
        "_score": 1,
        "_source": {
          "@timestamp": "2022-07-06T19:24:41.000Z",
          "message": "Workplace Search Account creation (62c5e17942e4b315c8a8067e)",
          "agent": {
            "id": "fe72453b-b0ff-49f8-9d17-1644f7e5f257",
            "name": "e1139b0a8396",
            "type": "filebeat",
            "version": "8.1.1",
            "ephemeral_id": "c8d092a4-9a33-4f25-a7e8-05a731f7feb8"
          },
          "host": {
            "name": "e1139b0a8396"
          },
          "enterprisesearch.entity": "FritoPie::Account",
          "event.outcome": "success",
          "process.pid": 8,
          "event.type": [
            "creation"
          ],
          "data_stream": {
            "type": "logs",
            "dataset": "enterprise_search.audit",
            "namespace": "default"
          },
          "event.action": "audit",
          "labels": {
            "index_date": "2022.07.06"
          },
          "service.ephemeral_id": "62c5e17142e4b315c8a80677",
          "event.kind": "event",
          "log": {
            "offset": 4182,
            "file": {
              "path": "/app/logs/audit.log"
            }
          },
          "process.thread.id": 4004,
          "service.type": "enterprise_search",
          "event": {
            "dataset": "enterprise-search-audit"
          },
          "ecs": {
            "version": "1.7.0"
          },
          "event.category": [
            "process"
          ],
          "service.version": "8.4.0",
          "event.id": "62c5e17942e4b315c8a8067f",
          "input": {
            "type": "log"
          },
          "enterprisesearch.change": {
            "updated_at": [
              null,
              "2022-07-06T19:24:41Z"
            ],
            "created_at": [
              null,
              "2022-07-06T19:24:41Z"
            ],
            "user_oid": [
              null,
              "62c5e17942e4b315c8a8067d"
            ],
            "role_type": [
              null,
              "admin"
            ],
            "frito_pie_group_ids": [
              null,
              [
                "62c5e17842e4b315c8a80679"
              ]
            ],
            "memberships": [
              null,
              []
            ],
            "id": [
              null,
              "62c5e17942e4b315c8a8067e"
            ]
          }
        }
      },
      {
        "_index": ".ds-logs-enterprise_search.audit-default-2022.07.06-000001",
        "_id": "ecf41IEB6XGlQZUq17sc",
        "_score": 1,
        "_source": {
          "@timestamp": "2022-07-06T19:24:43.000Z",
          "service.type": "enterprise_search",
          "service.ephemeral_id": "62c5e17142e4b315c8a80677",
          "message": "Workplace Search Account change (62c5e17942e4b315c8a8067e)",
          "process.thread.id": 4004,
          "data_stream": {
            "dataset": "enterprise_search.audit",
            "namespace": "default",
            "type": "logs"
          },
          "agent": {
            "ephemeral_id": "c8d092a4-9a33-4f25-a7e8-05a731f7feb8",
            "id": "fe72453b-b0ff-49f8-9d17-1644f7e5f257",
            "name": "e1139b0a8396",
            "type": "filebeat",
            "version": "8.1.1"
          },
          "event.outcome": "success",
          "ecs": {
            "version": "1.7.0"
          },
          "enterprisesearch.change": {
            "api_key": [
              null,
              "NmlyZkplTGl5dUMxSUJzZExMTmRmMnptT3VHNFVKRWZheXIrY0JrUnNrV1pHREM3OXEzbWNMY1VJSFFwQnFYZG1iQWIwZ1V1TGgvQ1lyRGh4YTBJeDNIYVNDbDBtWk56TC9EK2RFY090Y21JMEdCWVJpUEVpcEFKYjRMYjl3NmpmRVpXOXYyUDNYNGQ2Mmt0TXZoOEdKWVJOalZ0aXdpWFAxYnQ3amlNazkvdHQxcmR3VVNNVExyd3FkUnh5QWJzNnNHR2NvSStIaHpTRU1CTDBHL2hRaUVaTEdqK3NERnVjRGs4S0RuK3dURm0rWTB5c3pBbE9WQnZWRzRZSWdkL3VRY1BxaVBHOWkvS0xhdWFrWGNhNmdYK3BXYzVBeHRSMU1jS2hLRVloalE9LS1SZWVLUy92NjBHczhTUVlneFptaE1BPT0=--e14094557e6ea916f607898509f2cea3c35d7cea"
            ],
            "updated_at": [
              "2022-07-06T19:24:41Z",
              "2022-07-06T19:24:43Z"
            ]
          },
          "event": {
            "dataset": "enterprise-search-audit"
          },
          "event.id": "62c5e17b42e4b315c8a80681",
          "event.action": "audit",
          "event.type": [
            "change"
          ],
          "service.version": "8.4.0",
          "event.category": [
            "process"
          ],
          "labels": {
            "index_date": "2022.07.06"
          },
          "process.pid": 8,
          "enterprisesearch.entity": "FritoPie::Account",
          "input": {
            "type": "log"
          },
          "host": {
            "name": "e1139b0a8396"
          },
          "log": {
            "offset": 5203,
            "file": {
              "path": "/app/logs/audit.log"
            }
          },
          "event.kind": "event"
        }
      }
    ]
  }
}

The missing event.ingested field caused issues when we fetched data streams in Fleet's API, so we introduced a guard clause here. However, we're lacking in test coverage for this case.

Adding tests for this is difficult, however, because we need to ingest data stream documents without Fleet's .fleet-final-pipeline adding the event.ingested field, or we need to manually delete that field. I spent a few hours trying to get that work in the data stream API integration tests without much luck.

It'd be great to get some test coverage for this case, or further investigate the issue causing enterprise search data to be ingested before Fleet's ingest pipeline can process the data.

[elasticmachine]

Pinging @elastic/fleet (Team:Fleet)