diff --git a/x-pack/plugins/fleet/server/collectors/agent_collectors.ts b/x-pack/plugins/fleet/server/collectors/agent_collectors.ts index 0f960df4545bab..7861e7ac606a45 100644 --- a/x-pack/plugins/fleet/server/collectors/agent_collectors.ts +++ b/x-pack/plugins/fleet/server/collectors/agent_collectors.ts @@ -7,7 +7,6 @@ import type { SavedObjectsClient, ElasticsearchClient } from '@kbn/core/server'; -import type { FleetConfigType } from '../../common/types'; import { AGENTS_INDEX } from '../../common'; import * as AgentService from '../services/agents'; import { appContextService } from '../services'; @@ -22,7 +21,6 @@ export interface AgentUsage { } export const getAgentUsage = async ( - config: FleetConfigType, soClient?: SavedObjectsClient, esClient?: ElasticsearchClient ): Promise => { @@ -56,16 +54,13 @@ export interface AgentData { error: number; degraded: number; }; - agent_checkin_status_last_1h: { - error: number; - degraded: number; - }; + agents_per_policy: number[]; } const DEFAULT_AGENT_DATA = { agent_versions: [], agent_checkin_status: { error: 0, degraded: 0 }, - agent_checkin_status_last_1h: { error: 0, degraded: 0 }, + agents_per_policy: [], }; export const getAgentData = async ( @@ -84,6 +79,17 @@ export const getAgentData = async ( const response = await esClient.search( { index: AGENTS_INDEX, + query: { + bool: { + filter: [ + { + term: { + active: 'true', + }, + }, + ], + }, + }, size: 0, aggs: { versions: { @@ -92,6 +98,9 @@ export const getAgentData = async ( last_checkin_status: { terms: { field: 'last_checkin_status' }, }, + policies: { + terms: { field: 'policy_id' }, + }, }, }, { signal: abortController.signal } @@ -101,44 +110,14 @@ export const getAgentData = async ( ); const statuses = transformLastCheckinStatusBuckets(response); - const responseLast1h = await esClient.search( - { - index: AGENTS_INDEX, - size: 0, - query: { - bool: { - filter: [ - { - bool: { - must: [ - { - range: { - last_checkin: { - gte: 'now-1h/h', - lt: 'now/h', - }, - }, - }, - ], - }, - }, - ], - }, - }, - aggs: { - last_checkin_status: { - terms: { field: 'last_checkin_status' }, - }, - }, - }, - { signal: abortController.signal } + const agentsPerPolicy = ((response?.aggregations?.policies as any).buckets ?? []).map( + (bucket: any) => bucket.doc_count ); - const statusesLast1h = transformLastCheckinStatusBuckets(responseLast1h); return { agent_versions: versions, agent_checkin_status: statuses, - agent_checkin_status_last_1h: statusesLast1h, + agents_per_policy: agentsPerPolicy, }; } catch (error) { if (error.statusCode === 404) { diff --git a/x-pack/plugins/fleet/server/collectors/register.ts b/x-pack/plugins/fleet/server/collectors/register.ts index 051bb902c5479d..2892de0685e2f9 100644 --- a/x-pack/plugins/fleet/server/collectors/register.ts +++ b/x-pack/plugins/fleet/server/collectors/register.ts @@ -38,7 +38,7 @@ export const fetchFleetUsage = async ( } const usage = { agents_enabled: getIsAgentsEnabled(config), - agents: await getAgentUsage(config, soClient, esClient), + agents: await getAgentUsage(soClient, esClient), fleet_server: await getFleetServerUsage(soClient, esClient), packages: await getPackageUsage(soClient), ...(await getAgentData(esClient, abortController)), @@ -53,7 +53,7 @@ const fetchUsage = async (core: CoreSetup, config: FleetConfigType) => { const [soClient, esClient] = await getInternalClients(core); const usage = { agents_enabled: getIsAgentsEnabled(config), - agents: await getAgentUsage(config, soClient, esClient), + agents: await getAgentUsage(soClient, esClient), fleet_server: await getFleetServerUsage(soClient, esClient), packages: await getPackageUsage(soClient), }; @@ -64,7 +64,7 @@ export const fetchAgentsUsage = async (core: CoreSetup, config: FleetConfigType) const [soClient, esClient] = await getInternalClients(core); const usage = { agents_enabled: getIsAgentsEnabled(config), - agents: await getAgentUsage(config, soClient, esClient), + agents: await getAgentUsage(soClient, esClient), fleet_server: await getFleetServerUsage(soClient, esClient), }; return usage; diff --git a/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts b/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts new file mode 100644 index 00000000000000..fff7291856af2a --- /dev/null +++ b/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts @@ -0,0 +1,256 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import path from 'path'; + +import * as kbnTestServer from '@kbn/core/test_helpers/kbn_server'; + +import { fetchFleetUsage } from '../collectors/register'; + +import { waitForFleetSetup } from './helpers'; + +const logFilePath = path.join(__dirname, 'logs.log'); + +describe('fleet usage telemetry', () => { + let core: any; + let esServer: kbnTestServer.TestElasticsearchUtils; + let kbnServer: kbnTestServer.TestKibanaUtils; + + const startServers = async () => { + const { startES } = kbnTestServer.createTestServers({ + adjustTimeout: (t) => jest.setTimeout(t), + settings: { + es: { + license: 'trial', + }, + kbn: {}, + }, + }); + + esServer = await startES(); + const startKibana = async () => { + const root = kbnTestServer.createRootWithCorePlugins( + { + xpack: { + fleet: { + agentPolicies: [ + { + name: 'Second preconfigured policy', + description: 'second policy', + is_default: false, + is_managed: true, + id: 'test-456789', + namespace: 'default', + monitoring_enabled: [], + package_policies: [], + }, + ], + }, + }, + logging: { + appenders: { + file: { + type: 'file', + fileName: logFilePath, + layout: { + type: 'json', + }, + }, + }, + loggers: [ + { + name: 'root', + appenders: ['file'], + }, + { + name: 'plugins.fleet', + level: 'info', + }, + ], + }, + }, + { oss: false } + ); + + await root.preboot(); + const coreSetup = await root.setup(); + const coreStart = await root.start(); + + return { + root, + coreSetup, + coreStart, + stop: async () => await root.shutdown(), + }; + }; + kbnServer = await startKibana(); + await waitForFleetSetup(kbnServer.root); + }; + + const stopServers = async () => { + if (kbnServer) { + await kbnServer.stop(); + } + + if (esServer) { + await esServer.stop(); + } + + await new Promise((res) => setTimeout(res, 10000)); + }; + + beforeAll(async () => { + await startServers(); + + const esClient = kbnServer.coreStart.elasticsearch.client.asInternalUser; + await esClient.bulk({ + index: '.fleet-agents', + body: [ + { + create: { + _id: 'agent1', + }, + }, + { + agent: { + version: '8.6.0', + }, + last_checkin_status: 'error', + last_checkin: '2022-11-21T12:26:24Z', + active: true, + policy_id: 'policy1', + }, + { + create: { + _id: 'agent2', + }, + }, + { + agent: { + version: '8.5.1', + }, + last_checkin_status: 'degraded', + last_checkin: '2022-11-21T12:27:24Z', + active: true, + policy_id: 'policy1', + }, + { + create: { + _id: 'inactive', + }, + }, + { + agent: { + version: '8.5.1', + }, + last_checkin_status: 'online', + last_checkin: '2021-11-21T12:27:24Z', + active: false, + policy_id: 'policy1', + }, + ], + refresh: 'wait_for', + }); + + await esClient.create({ + index: '.fleet-policies', + id: 'policy1', + body: { + data: { + id: 'fleet-server-policy', + outputs: { + default: { + type: 'elasticsearch', + }, + }, + }, + }, + refresh: 'wait_for', + }); + + const soClient = kbnServer.coreStart.savedObjects.createInternalRepository(); + await soClient.create('ingest-package-policies', { + name: 'fleet_server-1', + namespace: 'default', + package: { + name: 'fleet_server', + title: 'Fleet Server', + version: '1.2.0', + }, + enabled: true, + policy_id: 'fleet-server-policy', + inputs: [ + { + compiled_input: { + server: { + port: 8220, + host: '0.0.0.0', + 'limits.max_agents': 3000, + }, + 'server.runtime': 'gc_percent:20', + }, + }, + ], + }); + }); + + afterAll(async () => { + await stopServers(); + }); + + beforeEach(() => { + core = { getStartServices: jest.fn().mockResolvedValue([kbnServer.coreStart]) }; + }); + + it('should fetch usage telemetry', async () => { + const usage = await fetchFleetUsage(core, { agents: { enabled: true } }, new AbortController()); + + expect(usage).toEqual( + expect.objectContaining({ + agents_enabled: true, + agents: { + total_enrolled: 2, + healthy: 0, + unhealthy: 0, + offline: 2, + total_all_statuses: 3, + updating: 0, + }, + fleet_server: { + total_all_statuses: 0, + total_enrolled: 0, + healthy: 0, + unhealthy: 0, + offline: 0, + updating: 0, + num_host_urls: 0, + }, + packages: [ + expect.objectContaining({ + name: 'synthetics', + }), + ], + agent_versions: ['8.5.1', '8.6.0'], + agent_checkin_status: { error: 1, degraded: 1 }, + agents_per_policy: [2], + fleet_server_config: { + policies: [ + { + input_config: { + server: { + 'limits.max_agents': 3000, + }, + 'server.runtime': 'gc_percent:20', + }, + }, + ], + }, + agent_policies: { count: 3, output_types: ['elasticsearch'] }, + }) + ); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts b/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts index 959510e6f7faa0..9eeb867bd9b91a 100644 --- a/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts +++ b/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts @@ -113,6 +113,13 @@ export const fleetUsagesSchema: RootSchema = { _meta: { description: 'The agent versions enrolled in this deployment.' }, }, }, + agents_per_policy: { + type: 'array', + items: { + type: 'long', + _meta: { description: 'Agent counts enrolled per agent policy.' }, + }, + }, fleet_server_config: { properties: { policies: { @@ -158,20 +165,4 @@ export const fleetUsagesSchema: RootSchema = { }, }, }, - agent_checkin_status_last_1h: { - properties: { - error: { - type: 'long', - _meta: { - description: 'Count of agent last checkin status error', - }, - }, - degraded: { - type: 'long', - _meta: { - description: 'Count of agent last checkin status degraded', - }, - }, - }, - }, };