Make config-schema extensible for handling of unknown fields (#156214)

Related issue #155764. In this POC, I'm adding an `extendsDeep` function to the schema object. This feature allows you to create a copy of an existing schema definition and recursively modify options without mutating them. With `extendsDeep`, you can specify whether unknown attributes on objects should be allowed, forbidden or ignored. This new function is particularly useful for alerting scenarios where we need to drop unknown fields when reading from Elasticsearch without modifying the schema object. Since we don't control the schema definition in some areas, `extendsDeep` provides a convenient way to set the `unknowns` option to all objects recursively. By doing so, we can validate and drop unknown properties using the same defined schema, just with `unknowns: forbid` extension. Usage: ``` // Single, shared type definition const type = schema.object({ foo: schema.string() }); // Drop unknown fields (bar in this case) const savedObject = { foo: 'test', bar: 'test' }; const ignoreSchema = type.extendsDeep({ unknowns: 'ignore' }); ignoreSchema.validate(savedObject); // Prevent unknown fields (bar in this case) const soToUpdate = { foo: 'test', bar: 'test' }; const forbidSchema = type.extendsDeep({ unknowns: 'forbid' }); forbidSchema.validate(soToUpdate); ``` --------- Co-authored-by: Kibana Machine <[email protected]>
elastic · May 5, 2023 · 1cab306 · 1cab306
1 parent 75cebfe
commit 1cab306
Show file tree

Hide file tree

Showing 15 changed files with 347 additions and 14 deletions.
diff --git a/packages/kbn-config-schema/src/types/array_type.test.ts b/packages/kbn-config-schema/src/types/array_type.test.ts
@@ -188,3 +188,30 @@ describe('#maxSize', () => {
     ).toThrowErrorMatchingInlineSnapshot(`"array size is [2], but cannot be greater than [1]"`);
   });
 });
+
+describe('#extendsDeep', () => {
+  const type = schema.arrayOf(
+    schema.object({
+      foo: schema.string(),
+    })
+  );
+
+  test('objects with unknown attributes are kept when extending with unknowns=allow', () => {
+    const allowSchema = type.extendsDeep({ unknowns: 'allow' });
+    const result = allowSchema.validate([{ foo: 'test', bar: 'test' }]);
+    expect(result).toEqual([{ foo: 'test', bar: 'test' }]);
+  });
+
+  test('objects with unknown attributes are dropped when extending with unknowns=ignore', () => {
+    const ignoreSchema = type.extendsDeep({ unknowns: 'ignore' });
+    const result = ignoreSchema.validate([{ foo: 'test', bar: 'test' }]);
+    expect(result).toEqual([{ foo: 'test' }]);
+  });
+
+  test('objects with unknown attributes fail validation when extending with unknowns=forbid', () => {
+    const forbidSchema = type.extendsDeep({ unknowns: 'forbid' });
+    expect(() =>
+      forbidSchema.validate([{ foo: 'test', bar: 'test' }])
+    ).toThrowErrorMatchingInlineSnapshot(`"[0.bar]: definition for this key is missing"`);
+  });
+});
diff --git a/packages/kbn-config-schema/src/types/array_type.ts b/packages/kbn-config-schema/src/types/array_type.ts
@@ -8,14 +8,17 @@
 
 import typeDetect from 'type-detect';
 import { internals } from '../internals';
-import { Type, TypeOptions } from './type';
+import { Type, TypeOptions, ExtendsDeepOptions } from './type';
 
 export type ArrayOptions<T> = TypeOptions<T[]> & {
   minSize?: number;
   maxSize?: number;
 };
 
 export class ArrayType<T> extends Type<T[]> {
+  private readonly arrayType: Type<T>;
+  private readonly arrayOptions: ArrayOptions<T>;
+
   constructor(type: Type<T>, options: ArrayOptions<T> = {}) {
     let schema = internals.array().items(type.getSchema().optional()).sparse(false);
 
@@ -28,6 +31,12 @@ export class ArrayType<T> extends Type<T[]> {
     }
 
     super(schema, options);
+    this.arrayType = type;
+    this.arrayOptions = options;
+  }
+
+  public extendsDeep(options: ExtendsDeepOptions) {
+    return new ArrayType(this.arrayType.extendsDeep(options), this.arrayOptions);
   }
 
   protected handleError(type: string, { limit, reason, value }: Record<string, any>) {

diff --git a/packages/kbn-config-schema/src/types/conditional_type.test.ts b/packages/kbn-config-schema/src/types/conditional_type.test.ts
@@ -395,3 +395,84 @@ describe('#validate', () => {
     });
   });
 });
+
+describe('#extendsDeep', () => {
+  describe('#equalType', () => {
+    const type = schema.object({
+      foo: schema.string(),
+      test: schema.conditional(
+        schema.siblingRef('foo'),
+        'test',
+        schema.object({
+          bar: schema.string(),
+        }),
+        schema.string()
+      ),
+    });
+
+    test('objects with unknown attributes are kept when extending with unknowns=allow', () => {
+      const result = type
+        .extendsDeep({ unknowns: 'allow' })
+        .validate({ foo: 'test', test: { bar: 'test', baz: 'test' } });
+      expect(result).toEqual({
+        foo: 'test',
+        test: { bar: 'test', baz: 'test' },
+      });
+    });
+
+    test('objects with unknown attributes are dropped when extending with unknowns=ignore', () => {
+      const result = type
+        .extendsDeep({ unknowns: 'ignore' })
+        .validate({ foo: 'test', test: { bar: 'test', baz: 'test' } });
+      expect(result).toEqual({
+        foo: 'test',
+        test: { bar: 'test' },
+      });
+    });
+    test('objects with unknown attributes fail validation when extending with unknowns=forbid', () => {
+      expect(() =>
+        type
+          .extendsDeep({ unknowns: 'forbid' })
+          .validate({ foo: 'test', test: { bar: 'test', baz: 'test' } })
+      ).toThrowErrorMatchingInlineSnapshot(`"[test.baz]: definition for this key is missing"`);
+    });
+  });
+
+  describe('#notEqualType', () => {
+    const type = schema.object({
+      foo: schema.string(),
+      test: schema.conditional(
+        schema.siblingRef('foo'),
+        'test',
+        schema.string(),
+        schema.object({
+          bar: schema.string(),
+        })
+      ),
+    });
+
+    test('objects with unknown attributes are kept when extending with unknowns=allow', () => {
+      const allowSchema = type.extendsDeep({ unknowns: 'allow' });
+      const result = allowSchema.validate({ foo: 'not-test', test: { bar: 'test', baz: 'test' } });
+      expect(result).toEqual({
+        foo: 'not-test',
+        test: { bar: 'test', baz: 'test' },
+      });
+    });
+
+    test('objects with unknown attributes are dropped when extending with unknowns=ignore', () => {
+      const ignoreSchema = type.extendsDeep({ unknowns: 'ignore' });
+      const result = ignoreSchema.validate({ foo: 'not-test', test: { bar: 'test', baz: 'test' } });
+      expect(result).toEqual({
+        foo: 'not-test',
+        test: { bar: 'test' },
+      });
+    });
+    test('objects with unknown attributes fail validation when extending with unknowns=forbid', () => {
+      const forbidSchema = type.extendsDeep({ unknowns: 'forbid' });
+      expect(() =>
+        forbidSchema.validate({ foo: 'not-test', test: { bar: 'test', baz: 'test' } })
+      ).toThrowErrorMatchingInlineSnapshot(`"[test.baz]: definition for this key is missing"`);
+    });
+  });
+});
diff --git a/packages/kbn-config-schema/src/types/conditional_type.ts b/packages/kbn-config-schema/src/types/conditional_type.ts
@@ -9,11 +9,17 @@
 import typeDetect from 'type-detect';
 import { internals } from '../internals';
 import { Reference } from '../references';
-import { Type, TypeOptions } from './type';
+import { ExtendsDeepOptions, Type, TypeOptions } from './type';
 
 export type ConditionalTypeValue = string | number | boolean | object | null;
 
 export class ConditionalType<A extends ConditionalTypeValue, B, C> extends Type<B | C> {
+  private readonly leftOperand: Reference<A>;
+  private readonly rightOperand: Reference<A> | A | Type<unknown>;
+  private readonly equalType: Type<B>;
+  private readonly notEqualType: Type<C>;
+  private readonly options?: TypeOptions<B | C>;
+
   constructor(
     leftOperand: Reference<A>,
     rightOperand: Reference<A> | A | Type<unknown>,
@@ -31,6 +37,21 @@ export class ConditionalType<A extends ConditionalTypeValue, B, C> extends Type<
     });
 
     super(schema, options);
+    this.leftOperand = leftOperand;
+    this.rightOperand = rightOperand;
+    this.equalType = equalType;
+    this.notEqualType = notEqualType;
+    this.options = options;
+  }
+
+  public extendsDeep(options: ExtendsDeepOptions) {
+    return new ConditionalType(
+      this.leftOperand,
+      this.rightOperand,
+      this.equalType.extendsDeep(options),
+      this.notEqualType.extendsDeep(options),
+      this.options
+    );
   }
 
   protected handleError(type: string, { value }: Record<string, any>) {

diff --git a/packages/kbn-config-schema/src/types/map_of_type.test.ts b/packages/kbn-config-schema/src/types/map_of_type.test.ts
@@ -185,3 +185,28 @@ test('error preserves full path', () => {
     `"[grandParentKey.parentKey.ab]: expected value of type [number] but got [string]"`
   );
 });
+
+describe('#extendsDeep', () => {
+  describe('#keyType', () => {
+    const type = schema.mapOf(schema.string(), schema.object({ foo: schema.string() }));
+
+    test('objects with unknown attributes are kept when extending with unknowns=allow', () => {
+      const allowSchema = type.extendsDeep({ unknowns: 'allow' });
+      const result = allowSchema.validate({ key: { foo: 'test', bar: 'test' } });
+      expect(result.get('key')).toEqual({ foo: 'test', bar: 'test' });
+    });
+
+    test('objects with unknown attributes are dropped when extending with unknowns=ignore', () => {
+      const ignoreSchema = type.extendsDeep({ unknowns: 'ignore' });
+      const result = ignoreSchema.validate({ key: { foo: 'test', bar: 'test' } });
+      expect(result.get('key')).toEqual({ foo: 'test' });
+    });
+
+    test('objects with unknown attributes fail validation when extending with unknowns=forbid', () => {
+      const forbidSchema = type.extendsDeep({ unknowns: 'forbid' });
+      expect(() =>
+        forbidSchema.validate({ key: { foo: 'test', bar: 'test' } })
+      ).toThrowErrorMatchingInlineSnapshot(`"[key.bar]: definition for this key is missing"`);
+    });
+  });
+});
diff --git a/packages/kbn-config-schema/src/types/map_type.ts b/packages/kbn-config-schema/src/types/map_type.ts
@@ -9,11 +9,15 @@
 import typeDetect from 'type-detect';
 import { SchemaTypeError, SchemaTypesError } from '../errors';
 import { internals } from '../internals';
-import { Type, TypeOptions } from './type';
+import { Type, TypeOptions, ExtendsDeepOptions } from './type';
 
 export type MapOfOptions<K, V> = TypeOptions<Map<K, V>>;
 
 export class MapOfType<K, V> extends Type<Map<K, V>> {
+  private readonly keyType: Type<K>;
+  private readonly valueType: Type<V>;
+  private readonly mapOptions: MapOfOptions<K, V>;
+
   constructor(keyType: Type<K>, valueType: Type<V>, options: MapOfOptions<K, V> = {}) {
     const defaultValue = options.defaultValue;
     const schema = internals.map().entries(keyType.getSchema(), valueType.getSchema());
@@ -26,6 +30,17 @@ export class MapOfType<K, V> extends Type<Map<K, V>> {
       // default value instead.
       defaultValue: defaultValue instanceof Map ? () => defaultValue : defaultValue,
     });
+    this.keyType = keyType;
+    this.valueType = valueType;
+    this.mapOptions = options;
+  }
+
+  public extendsDeep(options: ExtendsDeepOptions) {
+    return new MapOfType(
+      this.keyType.extendsDeep(options),
+      this.valueType.extendsDeep(options),
+      this.mapOptions
+    );
   }
 
   protected handleError(

diff --git a/packages/kbn-config-schema/src/types/maybe_type.test.ts b/packages/kbn-config-schema/src/types/maybe_type.test.ts
@@ -95,3 +95,26 @@ describe('maybe + object', () => {
     expect(type.validate({})).toEqual({});
   });
 });
+
+describe('#extendsDeep', () => {
+  const type = schema.maybe(schema.object({ foo: schema.string() }));
+
+  test('objects with unknown attributes are kept when extending with unknowns=allow', () => {
+    const allowSchema = type.extendsDeep({ unknowns: 'allow' });
+    const result = allowSchema.validate({ foo: 'test', bar: 'test' });
+    expect(result).toEqual({ foo: 'test', bar: 'test' });
+  });
+
+  test('objects with unknown attributes are dropped when extending with unknowns=ignore', () => {
+    const ignoreSchema = type.extendsDeep({ unknowns: 'ignore' });
+    const result = ignoreSchema.validate({ foo: 'test', bar: 'test' });
+    expect(result).toEqual({ foo: 'test' });
+  });
+
+  test('objects with unknown attributes fail validation when extending with unknowns=forbid', () => {
+    const forbidSchema = type.extendsDeep({ unknowns: 'forbid' });
+    expect(() =>
+      forbidSchema.validate({ foo: 'test', bar: 'test' })
+    ).toThrowErrorMatchingInlineSnapshot(`"[bar]: definition for this key is missing"`);
+  });
+});
diff --git a/packages/kbn-config-schema/src/types/maybe_type.ts b/packages/kbn-config-schema/src/types/maybe_type.ts
@@ -6,15 +6,22 @@
  * Side Public License, v 1.
  */
 
-import { Type } from './type';
+import { Type, ExtendsDeepOptions } from './type';
 
 export class MaybeType<V> extends Type<V | undefined> {
+  private readonly maybeType: Type<V>;
+
   constructor(type: Type<V>) {
     super(
       type
         .getSchema()
         .optional()
         .default(() => undefined)
     );
+    this.maybeType = type;
+  }
+
+  public extendsDeep(options: ExtendsDeepOptions) {
+    return new MaybeType(this.maybeType.extendsDeep(options));
   }
 }
diff --git a/packages/kbn-config-schema/src/types/object_type.test.ts b/packages/kbn-config-schema/src/types/object_type.test.ts
@@ -563,3 +563,26 @@ test('returns schema structure', () => {
     { path: ['nested', 'uri'], type: 'string' },
   ]);
 });
+
+describe('#extendsDeep', () => {
+  const type = schema.object({ test: schema.object({ foo: schema.string() }) });
+
+  test('objects with unknown attributes are kept when extending with unknowns=allow', () => {
+    const allowSchema = type.extendsDeep({ unknowns: 'allow' });
+    const result = allowSchema.validate({ test: { foo: 'test', bar: 'test' } });
+    expect(result).toEqual({ test: { foo: 'test', bar: 'test' } });
+  });
+
+  test('objects with unknown attributes are dropped when extending with unknowns=ignore', () => {
+    const ignoreSchema = type.extendsDeep({ unknowns: 'ignore' });
+    const result = ignoreSchema.validate({ test: { foo: 'test', bar: 'test' } });
+    expect(result).toEqual({ test: { foo: 'test' } });
+  });
+
+  test('objects with unknown attributes fail validation when extending with unknowns=forbid', () => {
+    const forbidSchema = type.extendsDeep({ unknowns: 'forbid' });
+    expect(() =>
+      forbidSchema.validate({ test: { foo: 'test', bar: 'test' } })
+    ).toThrowErrorMatchingInlineSnapshot(`"[test.bar]: definition for this key is missing"`);
+  });
+});
diff --git a/packages/kbn-config-schema/src/types/object_type.ts b/packages/kbn-config-schema/src/types/object_type.ts
@@ -9,7 +9,7 @@
 import type { AnySchema } from 'joi';
 import typeDetect from 'type-detect';
 import { internals } from '../internals';
-import { Type, TypeOptions } from './type';
+import { Type, TypeOptions, ExtendsDeepOptions, OptionsForUnknowns } from './type';
 import { ValidationError } from '../errors';
 
 export type Props = Record<string, Type<any>>;
@@ -60,13 +60,7 @@ type ExtendedObjectTypeOptions<P extends Props, NP extends NullableProps> = Obje
 >;
 
 interface UnknownOptions {
-  /**
-   * Options for dealing with unknown keys:
-   * - allow: unknown keys will be permitted
-   * - ignore: unknown keys will not fail validation, but will be stripped out
-   * - forbid (default): unknown keys will fail validation
-   */
-  unknowns?: 'allow' | 'ignore' | 'forbid';
+  unknowns?: OptionsForUnknowns;
 }
 
 export type ObjectTypeOptions<P extends Props = any> = TypeOptions<ObjectResultType<P>> &
@@ -181,6 +175,25 @@ export class ObjectType<P extends Props = any> extends Type<ObjectResultType<P>>
     return new ObjectType(extendedProps, extendedOptions);
   }
 
+  public extendsDeep(options: ExtendsDeepOptions) {
+    const extendedProps = Object.entries(this.props).reduce((memo, [key, value]) => {
+      if (value !== null && value !== undefined) {
+        return {
+          ...memo,
+          [key]: value.extendsDeep(options),
+        };
+      }
+      return memo;
+    }, {} as P);
+
+    const extendedOptions: ObjectTypeOptions<P> = {
+      ...this.options,
+      ...(options.unknowns ? { unknowns: options.unknowns } : {}),
+    };
+
+    return new ObjectType(extendedProps, extendedOptions);
+  }
+
   protected handleError(type: string, { reason, value }: Record<string, any>) {
     switch (type) {
       case 'any.required':