[Fleet] Only enable output secrets if all Fleet servers are compatible

x-pack/plugins/fleet/common/constants/secrets.ts

@@ -8,3 +8,4 @@
 export const SECRETS_ENDPOINT_PATH = '/_fleet/secret';
 
 export const SECRETS_MINIMUM_FLEET_SERVER_VERSION = '8.10.0';
+export const OUTPUT_SECRETS_MINIMUM_FLEET_SERVER_VERSION = '8.12.0';

x-pack/plugins/fleet/common/types/models/settings.ts

@@ -15,4 +15,5 @@ export interface Settings extends BaseSettings {
   id: string;
   preconfigured_fields?: Array<'fleet_server_hosts'>;
   secret_storage_requirements_met?: boolean;
+  output_secret_storage_requirements_met?: boolean;
 }

x-pack/plugins/fleet/server/constants/index.ts

@@ -81,6 +81,7 @@ export {
   // secrets
   SECRETS_ENDPOINT_PATH,
   SECRETS_MINIMUM_FLEET_SERVER_VERSION,
+  OUTPUT_SECRETS_MINIMUM_FLEET_SERVER_VERSION,
   // outputs
   OUTPUT_HEALTH_DATA_STREAM,
   type PrivilegeMapObject,

x-pack/plugins/fleet/server/saved_objects/index.ts

@@ -104,6 +104,7 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({
         has_seen_add_data_notice: { type: 'boolean', index: false },
         prerelease_integrations_enabled: { type: 'boolean' },
         secret_storage_requirements_met: { type: 'boolean' },
+        output_secret_storage_requirements_met: { type: 'boolean' },
       },
     },
     migrations: {

x-pack/plugins/fleet/server/services/output.ts

@@ -71,6 +71,7 @@ import {
   deleteSecrets,
   extractAndUpdateOutputSecrets,
   extractAndWriteOutputSecrets,
+  isOutputSecretStorageEnabled,
 } from './secrets';
 
 type Nullable<T> = { [P in keyof T]: T[P] | null };
@@ -122,12 +123,6 @@ function outputSavedObjectToOutput(so: SavedObject<OutputSOAttributes>): Output
   };
 }
 
-function isOutputSecretsEnabled() {
-  const { outputSecretsStorage } = appContextService.getExperimentalFeatures();
-
-  return !!outputSecretsStorage;
-}
-
 async function getAgentPoliciesPerOutput(
   soClient: SavedObjectsClientContract,
   outputId?: string,
@@ -588,7 +583,7 @@ class OutputService {
 
     const id = options?.id ? outputIdToUuid(options.id) : SavedObjectsUtils.generateId();
 
-    if (isOutputSecretsEnabled()) {
+    if (await isOutputSecretStorageEnabled(esClient, soClient)) {
       const { output: outputWithSecrets } = await extractAndWriteOutputSecrets({
         output,
         esClient,
@@ -802,7 +797,7 @@ class OutputService {
         );
       }
     }
-    if (isOutputSecretsEnabled()) {
+    if (await isOutputSecretStorageEnabled(esClient, soClient)) {
       const secretsRes = await extractAndUpdateOutputSecrets({
         oldOutput: originalOutput,
         outputUpdate: data,

x-pack/plugins/fleet/server/services/secrets.ts

@@ -47,7 +47,11 @@ import type {
 } from '../types';
 
 import { FleetError } from '../errors';
-import { SECRETS_ENDPOINT_PATH, SECRETS_MINIMUM_FLEET_SERVER_VERSION } from '../constants';
+import {
+  OUTPUT_SECRETS_MINIMUM_FLEET_SERVER_VERSION,
+  SECRETS_ENDPOINT_PATH,
+  SECRETS_MINIMUM_FLEET_SERVER_VERSION,
+} from '../constants';
 
 import { retryTransientEsErrors } from './epm/elasticsearch/retry';
 
@@ -642,6 +646,65 @@ export async function isSecretStorageEnabled(
   return false;
 }
 
+export async function isOutputSecretStorageEnabled(
+  esClient: ElasticsearchClient,
+  soClient: SavedObjectsClientContract
+): Promise<boolean> {
+  const logger = appContextService.getLogger();
+
+  // first check if the feature flag is enabled, if not output secrets are disabled
+  const { outputSecretsStorage: outputSecretsStorageEnabled } =
+    appContextService.getExperimentalFeatures();
+  if (!outputSecretsStorageEnabled) {
+    logger.debug('Output secrets storage is disabled by feature flag');
+    return false;
+  }
+
+  // if serverless then output secrets will always be supported
+  const isFleetServerStandalone =
+    appContextService.getConfig()?.internal?.fleetServerStandalone ?? false;
+
+  if (isFleetServerStandalone) {
+    logger.trace('Output secrets storage is enabled as fleet server is standalone');
+    return true;
+  }
+
+  // now check the flag in settings to see if the fleet server requirement has already been met
+  // once the requirement has been met, output secrets are always on
+  const settings = await settingsService.getSettingsOrUndefined(soClient);
+
+  if (settings && settings.output_secret_storage_requirements_met) {
+    logger.debug('Output secrets storage requirements already met, turned on in settings');
+    return true;
+  }
+
+  // otherwise check if we have the minimum fleet server version and enable secrets if so
+  if (
+    await allFleetServerVersionsAreAtLeast(
+      esClient,
+      soClient,
+      OUTPUT_SECRETS_MINIMUM_FLEET_SERVER_VERSION
+    )
+  ) {
+    logger.debug('Enabling output secrets storage as minimum fleet server version has been met');
+    try {
+      await settingsService.saveSettings(soClient, {
+        output_secret_storage_requirements_met: true,
+      });
+    } catch (err) {
+      // we can suppress this error as it will be retried on the next function call
+      logger.warn(`Failed to save settings after enabling output secrets storage: ${err.message}`);
+    }
+
+    return true;
+  }
+
+  logger.info(
+    'Output secrets storage is disabled as minimum fleet server version has not been met'
+  );
+  return false;
+}
+
 function _getPackageLevelSecretPaths(
   packagePolicy: NewPackagePolicy,
   packageInfo: PackageInfo

x-pack/test/fleet_api_integration/apis/outputs/crud.ts

@@ -60,6 +60,36 @@ export default function (providerContext: FtrProviderContext) {
     }
   };
 
+  const enableOutputSecrets = async () => {
+    try {
+      await kibanaServer.savedObjects.update({
+        type: GLOBAL_SETTINGS_SAVED_OBJECT_TYPE,
+        id: 'fleet-default-settings',
+        attributes: {
+          output_secret_storage_requirements_met: true,
+        },
+        overwrite: false,
+      });
+    } catch (e) {
+      throw e;
+    }
+  };
+
+  const disableOutputSecrets = async () => {
+    try {
+      await kibanaServer.savedObjects.update({
+        type: GLOBAL_SETTINGS_SAVED_OBJECT_TYPE,
+        id: 'fleet-default-settings',
+        attributes: {
+          output_secret_storage_requirements_met: false,
+        },
+        overwrite: false,
+      });
+    } catch (e) {
+      throw e;
+    }
+  };
+
   describe('fleet_outputs_crud', async function () {
     skipIfNoDockerRegistry(providerContext);
     before(async () => {
@@ -75,6 +105,7 @@ export default function (providerContext: FtrProviderContext) {
 
     before(async function () {
       await enableSecrets();
+      await enableOutputSecrets();
       // we must first force install the fleet_server package to override package verification error on policy create
       // https://github.com/elastic/kibana/issues/137450
       const getPkRes = await supertest
@@ -1305,6 +1336,46 @@ export default function (providerContext: FtrProviderContext) {
         // @ts-ignore _source unknown type
         expect(secret._source.value).to.equal('token');
       });
+
+      it('should not store secrets if fleet server does not meet minimum version', async function () {
+        await disableOutputSecrets();
+        await es.index({
+          index: '.fleet-agents',
+          refresh: true,
+          body: {
+            access_api_key_id: 'api-key-3',
+            active: true,
+            policy_id: fleetServerPolicyId,
+            type: 'PERMANENT',
+            local_metadata: {
+              host: { hostname: 'server_3' },
+              elastic: { agent: { version: '7.0.0' } },
+            },
+            user_provided_metadata: {},
+            enrolled_at: '2022-06-21T12:14:25Z',
+            last_checkin: '2022-06-27T12:28:29Z',
+            tags: ['tag1'],
+          },
+        });
+
+        const res = await supertest
+          .post(`/api/fleet/outputs`)
+          .set('kbn-xsrf', 'xxxx')
+          .send({
+            name: 'Logstash Output',
+            type: 'logstash',
+            hosts: ['test.fr:443'],
+            ssl: {
+              certificate: 'CERTIFICATE',
+              certificate_authorities: ['CA1', 'CA2'],
+            },
+            config_yaml: 'shipper: {}',
+            secrets: { ssl: { key: 'KEY' } },
+          })
+          .expect(200);
+
+        expect(Object.keys(res.body.item)).not.to.contain('secrets');
+      });
     });
 
     describe('DELETE /outputs/{outputId}', () => {
@@ -1378,50 +1449,6 @@ export default function (providerContext: FtrProviderContext) {
 
           expect(deleteResponse.id).to.eql(outputId);
         });
-
-        it('should delete secrets when deleting an output', async function () {
-          const res = await supertest
-            .post(`/api/fleet/outputs`)
-            .set('kbn-xsrf', 'xxxx')
-            .send({
-              name: 'Kafka Output With Secret',
-              type: 'kafka',
-              hosts: ['test.fr:2000'],
-              auth_type: 'ssl',
-              topics: [{ topic: 'topic1' }],
-              config_yaml: 'shipper: {}',
-              shipper: {
-                disk_queue_enabled: true,
-                disk_queue_path: 'path/to/disk/queue',
-                disk_queue_encryption_enabled: true,
-              },
-              ssl: {
-                certificate: 'CERTIFICATE',
-                certificate_authorities: ['CA1', 'CA2'],
-              },
-              secrets: {
-                ssl: {
-                  key: 'KEY',
-                },
-              },
-            })
-            .expect(200);
-
-          const outputWithSecretsId = res.body.item.id;
-          const secretId = res.body.item.secrets.ssl.key.id;
-
-          await supertest
-            .delete(`/api/fleet/outputs/${outputWithSecretsId}`)
-            .set('kbn-xsrf', 'xxxx')
-            .expect(200);
-
-          try {
-            await getSecretById(secretId);
-            expect().fail('Secret should have been deleted');
-          } catch (e) {
-            // not found
-          }
-        });
       });
 
       describe('Kafka output', () => {
@@ -1470,6 +1497,50 @@ export default function (providerContext: FtrProviderContext) {
 
           expect(deleteResponse.id).to.eql(outputId);
         });
+
+        it('should delete secrets when deleting an output', async function () {
+          const res = await supertest
+            .post(`/api/fleet/outputs`)
+            .set('kbn-xsrf', 'xxxx')
+            .send({
+              name: 'Kafka Output With Secret',
+              type: 'kafka',
+              hosts: ['test.fr:2000'],
+              auth_type: 'ssl',
+              topics: [{ topic: 'topic1' }],
+              config_yaml: 'shipper: {}',
+              shipper: {
+                disk_queue_enabled: true,
+                disk_queue_path: 'path/to/disk/queue',
+                disk_queue_encryption_enabled: true,
+              },
+              ssl: {
+                certificate: 'CERTIFICATE',
+                certificate_authorities: ['CA1', 'CA2'],
+              },
+              secrets: {
+                ssl: {
+                  key: 'KEY',
+                },
+              },
+            })
+            .expect(200);
+
+          const outputWithSecretsId = res.body.item.id;
+          const secretId = res.body.item.secrets.ssl.key.id;
+
+          await supertest
+            .delete(`/api/fleet/outputs/${outputWithSecretsId}`)
+            .set('kbn-xsrf', 'xxxx')
+            .expect(200);
+
+          try {
+            await getSecretById(secretId);
+            expect().fail('Secret should have been deleted');
+          } catch (e) {
+            // not found
+          }
+        });
       });
     });
   });